fixes
This commit is contained in:
parent
45a61a032c
commit
655050bf46
|
|
@ -17,17 +17,19 @@ export async function GET(req: NextRequest) {
|
|||
// Hole alle E-Mail-Adressen aus den "to" Feldern für diese Domain
|
||||
const mailboxData = await db.select({ to: emails.to }).from(emails).where(eq(emails.domainId, domain.id));
|
||||
|
||||
// Extrahiere die Domain aus dem Bucket-Namen (z.B. "example-com-emails" -> "example.com")
|
||||
const domainName = bucket.replace('-emails', '').replace(/-/g, '.');
|
||||
// Extrahiere sowohl die konvertierte Domain (mit Punkt) als auch die Original-Bucket-Domain (mit Bindestrich)
|
||||
const domainNameWithDot = domain.domain; // z.B. "bayarea-cc.com"
|
||||
const domainNameWithDash = bucket.replace('-emails', ''); // z.B. "bayarea-cc-com"
|
||||
|
||||
const uniqueMailboxes = new Set<string>();
|
||||
|
||||
// Filtere nur E-Mail-Adressen, die zur aktuellen Domain gehören
|
||||
// Filtere E-Mail-Adressen, die zu einer der beiden Domain-Varianten gehören
|
||||
mailboxData.forEach(em => {
|
||||
em.to?.forEach(recipient => {
|
||||
const recipientLower = recipient.toLowerCase();
|
||||
// Prüfe, ob die E-Mail-Adresse zur Domain gehört
|
||||
if (recipientLower.endsWith(`@${domainName}`)) {
|
||||
// Prüfe beide Varianten: mit Punkt und mit Bindestrich
|
||||
if (recipientLower.endsWith(`@${domainNameWithDot}`) ||
|
||||
recipientLower.endsWith(`@${domainNameWithDash}`)) {
|
||||
uniqueMailboxes.add(recipientLower);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,37 +1,132 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { db } from '@/app/db/drizzle';
|
||||
import { domains, emails } from '@/app/db/schema';
|
||||
import { authenticate } from '@/app/lib/utils';
|
||||
import { eq } from 'drizzle-orm';
|
||||
'use client';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
if (!authenticate(req)) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useSearchParams } from 'next/navigation';
|
||||
import Link from 'next/link';
|
||||
|
||||
const { searchParams } = new URL(req.url);
|
||||
const bucket = searchParams.get('bucket');
|
||||
if (!bucket) return NextResponse.json({ error: 'Missing bucket' }, { status: 400 });
|
||||
|
||||
const [domain] = await db.select().from(domains).where(eq(domains.bucket, bucket));
|
||||
if (!domain) return NextResponse.json({ error: 'Domain not found' }, { status: 404 });
|
||||
|
||||
// Hole alle E-Mail-Adressen aus den "to" Feldern für diese Domain
|
||||
const mailboxData = await db.select({ to: emails.to }).from(emails).where(eq(emails.domainId, domain.id));
|
||||
|
||||
// Extrahiere die Domain aus dem Bucket-Namen (z.B. "example-com-emails" -> "example.com")
|
||||
const domainName = bucket.replace('-emails', '').replace(/-/g, '.');
|
||||
|
||||
const uniqueMailboxes = new Set<string>();
|
||||
|
||||
// Filtere nur E-Mail-Adressen, die zur aktuellen Domain gehören
|
||||
mailboxData.forEach(em => {
|
||||
em.to?.forEach(recipient => {
|
||||
const recipientLower = recipient.toLowerCase();
|
||||
// Prüfe, ob die E-Mail-Adresse zur Domain gehört
|
||||
if (recipientLower.endsWith(`@${domainName}`)) {
|
||||
uniqueMailboxes.add(recipientLower);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return NextResponse.json(Array.from(uniqueMailboxes).sort());
|
||||
interface Email {
|
||||
key: string;
|
||||
subject: string;
|
||||
date: string;
|
||||
processed: string;
|
||||
processedAt: string | null;
|
||||
processedBy: string | null;
|
||||
queuedTo: string | null;
|
||||
status: string | null;
|
||||
}
|
||||
|
||||
export default function Emails() {
|
||||
const searchParams = useSearchParams();
|
||||
const bucket = searchParams.get('bucket');
|
||||
const mailbox = searchParams.get('mailbox');
|
||||
const [emails, setEmails] = useState<Email[]>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
if (!bucket || !mailbox) {
|
||||
setError('Missing parameters');
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
const auth = localStorage.getItem('auth');
|
||||
if (!auth) {
|
||||
setError('Not authenticated');
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
fetch(`/api/emails?bucket=${bucket}&mailbox=${encodeURIComponent(mailbox)}`, {
|
||||
headers: { Authorization: `Basic ${auth}` }
|
||||
})
|
||||
.then(res => {
|
||||
if (!res.ok) throw new Error('Failed to fetch emails');
|
||||
return res.json();
|
||||
})
|
||||
.then(data => {
|
||||
const sorted = data.sort((a: Email, b: Email) => new Date(b.date).getTime() - new Date(a.date).getTime());
|
||||
setEmails(sorted);
|
||||
})
|
||||
.catch(err => setError(err.message))
|
||||
.finally(() => setLoading(false));
|
||||
}, [bucket, mailbox]);
|
||||
|
||||
if (loading) return <div className="min-h-screen flex items-center justify-center bg-gray-100">Loading...</div>;
|
||||
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
||||
|
||||
const formatDate = (dateStr: string | null) => {
|
||||
if (!dateStr) return 'N/A';
|
||||
const date = new Date(dateStr);
|
||||
return date.toLocaleString('en-US', { year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hour12: false });
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gradient-to-b from-blue-50 to-gray-100 p-8">
|
||||
<nav className="max-w-6xl mx-auto mb-6 bg-white p-4 rounded-lg shadow-sm">
|
||||
<ol className="flex flex-wrap space-x-2 text-sm text-gray-500">
|
||||
<li><Link href="/" className="hover:text-blue-600">Home</Link></li>
|
||||
<li className="mx-1">/</li>
|
||||
<li><Link href="/domains" className="hover:text-blue-600">Domains</Link></li>
|
||||
<li className="mx-1">/</li>
|
||||
<li><Link href={`/mailboxes?bucket=${bucket}`} className="hover:text-blue-600">Mailboxes</Link></li>
|
||||
<li className="mx-1">/</li>
|
||||
<li className="font-semibold text-gray-700">Emails</li>
|
||||
</ol>
|
||||
</nav>
|
||||
<h1 className="text-4xl font-bold mb-8 text-center text-gray-800">Emails for {mailbox} in {bucket}</h1>
|
||||
|
||||
<div className="overflow-x-auto max-w-6xl mx-auto bg-white rounded-lg shadow-md">
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead className="bg-blue-50">
|
||||
<tr>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Subject</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Date</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Processed</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Processed At</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Processed By</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Queued To</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Status</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="bg-white divide-y divide-gray-200">
|
||||
{emails.length === 0 ? (
|
||||
<tr>
|
||||
<td colSpan={8} className="px-4 py-8 text-center text-gray-500">No emails found</td>
|
||||
</tr>
|
||||
) : emails.map((e: Email) => (
|
||||
<tr key={e.key} className="hover:bg-blue-50 transition">
|
||||
<td className="px-4 py-4 text-sm text-gray-900 max-w-xs truncate">{e.subject}</td>
|
||||
<td className="px-4 py-4 whitespace-nowrap text-sm text-gray-500">{formatDate(e.date)}</td>
|
||||
<td className="px-4 py-4 whitespace-nowrap text-sm">
|
||||
<span className={`px-2 py-1 rounded-full text-xs font-medium ${e.processed === 'true' ? 'bg-green-100 text-green-800' : 'bg-yellow-100 text-yellow-800'}`}>
|
||||
{e.processed === 'true' ? 'Yes' : 'No'}
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-4 py-4 whitespace-nowrap text-sm text-gray-500">{formatDate(e.processedAt)}</td>
|
||||
<td className="px-4 py-4 whitespace-nowrap text-sm text-gray-500">{e.processedBy || '-'}</td>
|
||||
<td className="px-4 py-4 whitespace-nowrap text-sm text-gray-500">{e.queuedTo || '-'}</td>
|
||||
<td className="px-4 py-4 whitespace-nowrap text-sm">
|
||||
{e.status ? (
|
||||
<span className={`px-2 py-1 rounded-full text-xs font-medium ${
|
||||
e.status === 'delivered' ? 'bg-green-100 text-green-800' :
|
||||
e.status === 'failed' ? 'bg-red-100 text-red-800' :
|
||||
'bg-gray-100 text-gray-800'
|
||||
}`}>
|
||||
{e.status}
|
||||
</span>
|
||||
) : '-'}
|
||||
</td>
|
||||
<td className="px-4 py-4 whitespace-nowrap text-sm font-medium">
|
||||
<Link href={`/email?bucket=${bucket}&key=${e.key}&mailbox=${encodeURIComponent(mailbox || '')}`} className="text-blue-600 hover:text-blue-900">
|
||||
View
|
||||
</Link>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -13,6 +13,28 @@ const CONCURRENT_S3_OPERATIONS = 10;
|
|||
const BATCH_INSERT_SIZE = 100;
|
||||
const CONCURRENT_EMAIL_PARSING = 5;
|
||||
|
||||
// Globale Helper function für sichere Date-Konvertierung
|
||||
function parseDate(dateInput: string | Date | undefined | null): Date | null {
|
||||
if (!dateInput) return null;
|
||||
|
||||
try {
|
||||
const date = dateInput instanceof Date ? dateInput : new Date(dateInput);
|
||||
// Prüfe ob das Datum gültig ist
|
||||
if (isNaN(date.getTime())) {
|
||||
return null;
|
||||
}
|
||||
// Prüfe ob das Datum in einem vernünftigen Bereich liegt (1970-2100)
|
||||
const year = date.getFullYear();
|
||||
if (year < 1970 || year > 2100) {
|
||||
return null;
|
||||
}
|
||||
return date;
|
||||
} catch (error) {
|
||||
console.error('Error parsing date:', dateInput, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function syncAllDomains() {
|
||||
console.log('Starting optimized syncAllDomains...');
|
||||
const startTime = Date.now();
|
||||
|
|
@ -28,8 +50,17 @@ export async function syncAllDomains() {
|
|||
domainBuckets.map(bucketObj =>
|
||||
bucketLimit(async () => {
|
||||
const bucket = bucketObj.Name!;
|
||||
const domainName = bucket.replace('-emails', '').replace(/-/g, '.');
|
||||
console.log(`Processing bucket: ${bucket}`);
|
||||
|
||||
// Korrekte Domain-Konvertierung: Nur den letzten '-' vor '-emails' zu '.' machen
|
||||
// Beispiel: bayarea-cc-com-emails -> bayarea-cc-com -> bayarea-cc.com
|
||||
let domainName = bucket.replace('-emails', ''); // Entferne '-emails'
|
||||
const lastDashIndex = domainName.lastIndexOf('-'); // Finde letzten Bindestrich
|
||||
if (lastDashIndex !== -1) {
|
||||
// Ersetze nur den letzten Bindestrich durch einen Punkt
|
||||
domainName = domainName.substring(0, lastDashIndex) + '.' + domainName.substring(lastDashIndex + 1);
|
||||
}
|
||||
|
||||
console.log(`Processing bucket: ${bucket} -> Domain: ${domainName}`);
|
||||
|
||||
const [domain] = await db
|
||||
.insert(domains)
|
||||
|
|
@ -114,7 +145,7 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
const metadata = head.Metadata || {};
|
||||
|
||||
const processed = metadata[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!;
|
||||
const processedAt = metadata['processed_at'] ? new Date(metadata['processed_at']) : null;
|
||||
const processedAt = parseDate(metadata['processed_at']);
|
||||
const processedBy = metadata['processed_by'] || null;
|
||||
const queuedTo = metadata['queued_to'] || null;
|
||||
const status = metadata['status'] || null;
|
||||
|
|
@ -167,6 +198,25 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
const cc = extractAddresses(parsed.cc);
|
||||
const bcc = extractAddresses(parsed.bcc);
|
||||
|
||||
// Versuche verschiedene Datum-Quellen in Reihenfolge der Präferenz
|
||||
let emailDate: Date | null = null;
|
||||
|
||||
// 1. Versuche parsed.date
|
||||
if (parsed.date) {
|
||||
emailDate = parseDate(parsed.date);
|
||||
}
|
||||
|
||||
// 2. Falls parsed.date ungültig, versuche S3 LastModified
|
||||
if (!emailDate && headResponse.LastModified) {
|
||||
emailDate = parseDate(headResponse.LastModified);
|
||||
}
|
||||
|
||||
// 3. Falls beides ungültig, verwende aktuelles Datum als Fallback
|
||||
if (!emailDate) {
|
||||
console.warn(`No valid date found for ${key}, using current date`);
|
||||
emailDate = new Date();
|
||||
}
|
||||
|
||||
return {
|
||||
domainId,
|
||||
s3Key: key,
|
||||
|
|
@ -178,9 +228,8 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
html: parsed.html || parsed.textAsHtml,
|
||||
raw: raw.toString('utf-8'),
|
||||
processed: metadata[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!,
|
||||
date: parsed.date || headResponse.LastModified,
|
||||
// Neue Metadaten
|
||||
processedAt: metadata['processed_at'] ? new Date(metadata['processed_at']) : null,
|
||||
date: emailDate,
|
||||
processedAt: parseDate(metadata['processed_at']),
|
||||
processedBy: metadata['processed_by'] || null,
|
||||
queuedTo: metadata['queued_to'] || null,
|
||||
status: metadata['status'] || null,
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ export function getS3Client() {
|
|||
socketTimeout: 60000,
|
||||
httpsAgent: new https.Agent({
|
||||
keepAlive: true,
|
||||
maxSockets: 50 // Erhöhe parallele Verbindungen
|
||||
maxSockets: 50
|
||||
})
|
||||
})
|
||||
});
|
||||
|
|
@ -25,22 +25,66 @@ export function getS3Client() {
|
|||
|
||||
export function authenticate(req: NextRequest) {
|
||||
const auth = req.headers.get('Authorization');
|
||||
console.log('Received Auth Header:', auth); // Logge den Header
|
||||
console.log('Expected Password:', process.env.APP_PASSWORD); // Logge das Env-Passwort (Achtung: Sensibel, nur für Debug!)
|
||||
console.log('Received Auth Header:', auth);
|
||||
console.log('Expected Password:', process.env.APP_PASSWORD);
|
||||
if (!auth || !auth.startsWith('Basic ')) return false;
|
||||
const [user, pass] = Buffer.from(auth.slice(6), 'base64').toString().split(':');
|
||||
return user === 'admin' && pass === process.env.APP_PASSWORD;
|
||||
}
|
||||
|
||||
export async function getBody(stream: Readable): Promise<Buffer> {
|
||||
export async function getBody(stream: Readable, timeoutMs = 30000): Promise<Buffer> {
|
||||
console.log('Getting body from stream...');
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
stream.on('data', chunk => chunks.push(chunk));
|
||||
stream.on('error', reject);
|
||||
let totalSize = 0;
|
||||
let timeoutHandle: NodeJS.Timeout;
|
||||
|
||||
// Timeout-Handler
|
||||
const cleanup = () => {
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle);
|
||||
stream.removeAllListeners();
|
||||
};
|
||||
|
||||
timeoutHandle = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error(`Stream timeout after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
chunks.push(chunk);
|
||||
totalSize += chunk.length;
|
||||
|
||||
// Optional: Limit für maximale Größe (z.B. 50MB)
|
||||
if (totalSize > 50 * 1024 * 1024) {
|
||||
cleanup();
|
||||
reject(new Error('Stream size exceeded maximum limit of 50MB'));
|
||||
}
|
||||
});
|
||||
|
||||
stream.on('error', (err) => {
|
||||
cleanup();
|
||||
console.error('Stream error:', err);
|
||||
reject(err);
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
console.log('Body fetched, size:', Buffer.concat(chunks).length);
|
||||
resolve(Buffer.concat(chunks));
|
||||
cleanup();
|
||||
const buffer = Buffer.concat(chunks);
|
||||
console.log('Body fetched, size:', buffer.length);
|
||||
resolve(buffer);
|
||||
});
|
||||
|
||||
// Handle stream destroy/close
|
||||
stream.on('close', () => {
|
||||
cleanup();
|
||||
if (chunks.length > 0) {
|
||||
const buffer = Buffer.concat(chunks);
|
||||
console.log('Stream closed, partial data fetched, size:', buffer.length);
|
||||
resolve(buffer);
|
||||
} else {
|
||||
reject(new Error('Stream closed without data'));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
22
sync-test.ts
22
sync-test.ts
|
|
@ -9,15 +9,27 @@ dotenv.config({ path: '.env' });
|
|||
|
||||
console.log('DATABASE_URL:', process.env.DATABASE_URL);
|
||||
console.log('AWS_REGION:', process.env.AWS_REGION);
|
||||
console.log('AWS_ACCESS_KEY_ID:', process.env.AWS_ACCESS_KEY_ID ? 'Set' : 'Not set'); // Maskiere sensible Keys
|
||||
console.log('AWS_ACCESS_KEY_ID:', process.env.AWS_ACCESS_KEY_ID ? 'Set' : 'Not set');
|
||||
console.log('AWS_SECRET_ACCESS_KEY:', process.env.AWS_SECRET_ACCESS_KEY ? 'Set' : 'Not set');
|
||||
|
||||
// Timeout-Wrapper für den Sync
|
||||
async function runSyncWithTimeout(timeoutMs = 600000) { // 10 Minuten Standard-Timeout
|
||||
return Promise.race([
|
||||
syncAllDomains(),
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Sync timeout exceeded')), timeoutMs)
|
||||
)
|
||||
]);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
console.log('Starting sync...');
|
||||
await syncAllDomains();
|
||||
console.log('Sync done');
|
||||
console.log('Starting sync with timeout...');
|
||||
await runSyncWithTimeout();
|
||||
console.log('Sync completed successfully');
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Error:', error);
|
||||
console.error('Sync error:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
Loading…
Reference in New Issue