changes
This commit is contained in:
parent
eeac2c6904
commit
45a61a032c
|
|
@ -1,12 +1,8 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { db } from '@/app/db/drizzle';
|
||||
import { emails } from '@/app/db/schema';
|
||||
import { authenticate, getBody } from '@/app/lib/utils';
|
||||
import { authenticate } from '@/app/lib/utils';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { CopyObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { getS3Client } from '@/app/lib/utils';
|
||||
import nodemailer from 'nodemailer';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
if (!authenticate(req)) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
|
@ -26,67 +22,9 @@ export async function GET(req: NextRequest) {
|
|||
html: email.html,
|
||||
raw: email.raw,
|
||||
processed: email.processed ? 'true' : 'false',
|
||||
processedAt: email.processedAt?.toISOString() || null,
|
||||
processedBy: email.processedBy,
|
||||
queuedTo: email.queuedTo,
|
||||
status: email.status,
|
||||
});
|
||||
}
|
||||
|
||||
// PUT: Update processed in S3 and DB
|
||||
export async function PUT(req: NextRequest) {
|
||||
if (!authenticate(req)) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
||||
const { bucket, key, processed } = await req.json();
|
||||
if (!bucket || !key) return NextResponse.json({ error: 'Missing params' }, { status: 400 });
|
||||
|
||||
const s3 = getS3Client();
|
||||
const head = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
|
||||
const newMeta = { ...head.Metadata, [process.env.PROCESSED_META_KEY!]: processed };
|
||||
await s3.send(new CopyObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
CopySource: `${bucket}/${key}`,
|
||||
Metadata: newMeta,
|
||||
MetadataDirective: 'REPLACE'
|
||||
}));
|
||||
|
||||
await db.update(emails).set({ processed: processed === 'true' }).where(eq(emails.s3Key, key));
|
||||
|
||||
return NextResponse.json({ success: true });
|
||||
}
|
||||
|
||||
// POST: Resend, update in S3 and DB
|
||||
export async function POST(req: NextRequest) {
|
||||
if (!authenticate(req)) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
||||
const { bucket, key } = await req.json();
|
||||
if (!bucket || !key) return NextResponse.json({ error: 'Missing params' }, { status: 400 });
|
||||
|
||||
const s3 = getS3Client();
|
||||
const { Body } = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
|
||||
const raw = await getBody(Body as Readable);
|
||||
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: process.env.SMTP_HOST,
|
||||
port: Number(process.env.SMTP_PORT),
|
||||
secure: false,
|
||||
auth: { user: process.env.SMTP_USER, pass: process.env.SMTP_PASS },
|
||||
tls: { rejectUnauthorized: false }
|
||||
});
|
||||
|
||||
try {
|
||||
await transporter.sendMail({ raw });
|
||||
// Update S3 Metadata
|
||||
const head = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
|
||||
const newMeta = { ...head.Metadata, [process.env.PROCESSED_META_KEY!]: process.env.PROCESSED_META_VALUE! };
|
||||
await s3.send(new CopyObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
CopySource: `${bucket}/${key}`,
|
||||
Metadata: newMeta,
|
||||
MetadataDirective: 'REPLACE'
|
||||
}));
|
||||
// Update DB
|
||||
await db.update(emails).set({ processed: true }).where(eq(emails.s3Key, key));
|
||||
return NextResponse.json({ message: 'Resent successfully' });
|
||||
} catch (error) {
|
||||
return NextResponse.json({ error: (error as Error).message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
@ -20,7 +20,20 @@ export async function GET(req: NextRequest) {
|
|||
subject: emails.subject,
|
||||
date: emails.date,
|
||||
processed: emails.processed,
|
||||
processedAt: emails.processedAt,
|
||||
processedBy: emails.processedBy,
|
||||
queuedTo: emails.queuedTo,
|
||||
status: emails.status,
|
||||
}).from(emails).where(sql`${mailbox} = ANY(${emails.to}) AND ${emails.domainId} = ${domain.id}`);
|
||||
|
||||
return NextResponse.json(emailList.map(e => ({ key: e.key, subject: e.subject, date: e.date?.toISOString(), processed: e.processed ? 'true' : 'false' })));
|
||||
return NextResponse.json(emailList.map(e => ({
|
||||
key: e.key,
|
||||
subject: e.subject,
|
||||
date: e.date?.toISOString(),
|
||||
processed: e.processed ? 'true' : 'false',
|
||||
processedAt: e.processedAt?.toISOString() || null,
|
||||
processedBy: e.processedBy,
|
||||
queuedTo: e.queuedTo,
|
||||
status: e.status,
|
||||
})));
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
|
|||
import { db } from '@/app/db/drizzle';
|
||||
import { domains, emails } from '@/app/db/schema';
|
||||
import { authenticate } from '@/app/lib/utils';
|
||||
import { eq, sql } from 'drizzle-orm';
|
||||
import { eq } from 'drizzle-orm';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
if (!authenticate(req)) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
|
@ -14,9 +14,24 @@ export async function GET(req: NextRequest) {
|
|||
const [domain] = await db.select().from(domains).where(eq(domains.bucket, bucket));
|
||||
if (!domain) return NextResponse.json({ error: 'Domain not found' }, { status: 404 });
|
||||
|
||||
// Hole alle E-Mail-Adressen aus den "to" Feldern für diese Domain
|
||||
const mailboxData = await db.select({ to: emails.to }).from(emails).where(eq(emails.domainId, domain.id));
|
||||
const uniqueMailboxes = new Set<string>();
|
||||
mailboxData.forEach(em => em.to?.forEach(r => uniqueMailboxes.add(r.toLowerCase())));
|
||||
|
||||
return NextResponse.json(Array.from(uniqueMailboxes));
|
||||
// Extrahiere die Domain aus dem Bucket-Namen (z.B. "example-com-emails" -> "example.com")
|
||||
const domainName = bucket.replace('-emails', '').replace(/-/g, '.');
|
||||
|
||||
const uniqueMailboxes = new Set<string>();
|
||||
|
||||
// Filtere nur E-Mail-Adressen, die zur aktuellen Domain gehören
|
||||
mailboxData.forEach(em => {
|
||||
em.to?.forEach(recipient => {
|
||||
const recipientLower = recipient.toLowerCase();
|
||||
// Prüfe, ob die E-Mail-Adresse zur Domain gehört
|
||||
if (recipientLower.endsWith(`@${domainName}`)) {
|
||||
uniqueMailboxes.add(recipientLower);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return NextResponse.json(Array.from(uniqueMailboxes).sort());
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { db } from '@/app/db/drizzle';
|
||||
import { domains, emails } from '@/app/db/schema';
|
||||
import { authenticate } from '@/app/lib/utils';
|
||||
import { eq } from 'drizzle-orm';
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
if (!authenticate(req)) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
||||
const { bucket } = await req.json();
|
||||
if (!bucket) return NextResponse.json({ error: 'Missing bucket' }, { status: 400 });
|
||||
|
||||
const [domain] = await db.select().from(domains).where(eq(domains.bucket, bucket));
|
||||
if (!domain) return NextResponse.json({ error: 'Domain not found' }, { status: 404 });
|
||||
|
||||
const unprocessed = await db.select({ s3Key: emails.s3Key }).from(emails).where(eq(emails.processed, false));
|
||||
|
||||
let count = 0;
|
||||
for (const em of unprocessed) {
|
||||
// Call POST /api/email internally for resend (updates DB/S3)
|
||||
await fetch(`${req.headers.get('origin')}/api/email`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: req.headers.get('Authorization')! },
|
||||
body: JSON.stringify({ bucket, key: em.s3Key }),
|
||||
});
|
||||
count++;
|
||||
}
|
||||
|
||||
return NextResponse.json({ message: `Resent ${count} emails` });
|
||||
}
|
||||
|
|
@ -19,4 +19,9 @@ export const emails = pgTable('emails', {
|
|||
raw: text('raw'),
|
||||
processed: boolean('processed').default(false),
|
||||
date: timestamp('date'),
|
||||
// Neue Metadaten
|
||||
processedAt: timestamp('processed_at'),
|
||||
processedBy: text('processed_by'),
|
||||
queuedTo: text('queued_to'),
|
||||
status: text('status'),
|
||||
});
|
||||
|
|
@ -8,8 +8,19 @@ export default function EmailDetail() {
|
|||
const searchParams = useSearchParams();
|
||||
const bucket = searchParams.get('bucket');
|
||||
const key = searchParams.get('key');
|
||||
const mailbox = searchParams.get('mailbox'); // Für Breadcrumb
|
||||
const [email, setEmail] = useState({ subject: '', from: '', to: '', html: '', raw: '', processed: '' });
|
||||
const mailbox = searchParams.get('mailbox');
|
||||
const [email, setEmail] = useState({
|
||||
subject: '',
|
||||
from: '',
|
||||
to: '',
|
||||
html: '',
|
||||
raw: '',
|
||||
processed: '',
|
||||
processedAt: null as string | null,
|
||||
processedBy: null as string | null,
|
||||
queuedTo: null as string | null,
|
||||
status: null as string | null,
|
||||
});
|
||||
const [viewMode, setViewMode] = useState('html');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
|
@ -42,6 +53,20 @@ export default function EmailDetail() {
|
|||
if (loading) return <div className="min-h-screen flex items-center justify-center bg-gray-100">Loading...</div>;
|
||||
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
||||
|
||||
const formatDate = (dateStr: string | null) => {
|
||||
if (!dateStr) return 'N/A';
|
||||
const date = new Date(dateStr);
|
||||
return date.toLocaleString('en-US', {
|
||||
year: 'numeric',
|
||||
month: '2-digit',
|
||||
day: '2-digit',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
hour12: false
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gradient-to-b from-blue-50 to-gray-100 p-8">
|
||||
<nav className="max-w-4xl mx-auto mb-6 bg-white p-4 rounded-lg shadow-sm">
|
||||
|
|
@ -59,10 +84,38 @@ export default function EmailDetail() {
|
|||
</nav>
|
||||
<div className="max-w-4xl mx-auto bg-white rounded-lg shadow-md p-8">
|
||||
<h1 className="text-4xl font-bold mb-6 text-gray-800">{email.subject}</h1>
|
||||
<p className="text-gray-700 mb-2 text-lg"><strong>From:</strong> {email.from}</p>
|
||||
<p className="text-gray-700 mb-2 text-lg"><strong>To:</strong> {email.to}</p>
|
||||
<p className="text-gray-700 mb-2 text-lg"><strong>S3 Key:</strong> {key}</p>
|
||||
<p className="text-gray-700 mb-6 text-lg"><strong>Processed:</strong> {email.processed}</p>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4 mb-6 bg-gray-50 p-6 rounded-lg">
|
||||
<div>
|
||||
<p className="text-gray-700 mb-2"><strong>From:</strong> {email.from}</p>
|
||||
<p className="text-gray-700 mb-2"><strong>To:</strong> {email.to}</p>
|
||||
<p className="text-gray-700 mb-2"><strong>S3 Key:</strong> <span className="text-sm break-all">{key}</span></p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-gray-700 mb-2">
|
||||
<strong>Processed:</strong>
|
||||
<span className={`ml-2 px-2 py-1 rounded-full text-xs font-medium ${email.processed === 'true' ? 'bg-green-100 text-green-800' : 'bg-yellow-100 text-yellow-800'}`}>
|
||||
{email.processed === 'true' ? 'Yes' : 'No'}
|
||||
</span>
|
||||
</p>
|
||||
<p className="text-gray-700 mb-2"><strong>Processed At:</strong> {formatDate(email.processedAt)}</p>
|
||||
<p className="text-gray-700 mb-2"><strong>Processed By:</strong> {email.processedBy || 'N/A'}</p>
|
||||
<p className="text-gray-700 mb-2"><strong>Queued To:</strong> {email.queuedTo || 'N/A'}</p>
|
||||
<p className="text-gray-700 mb-2">
|
||||
<strong>Status:</strong>
|
||||
{email.status ? (
|
||||
<span className={`ml-2 px-2 py-1 rounded-full text-xs font-medium ${
|
||||
email.status === 'delivered' ? 'bg-green-100 text-green-800' :
|
||||
email.status === 'failed' ? 'bg-red-100 text-red-800' :
|
||||
'bg-gray-100 text-gray-800'
|
||||
}`}>
|
||||
{email.status}
|
||||
</span>
|
||||
) : 'N/A'}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex mb-6 space-x-2">
|
||||
<button
|
||||
onClick={() => setViewMode('html')}
|
||||
|
|
|
|||
|
|
@ -1,161 +1,37 @@
|
|||
'use client';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { db } from '@/app/db/drizzle';
|
||||
import { domains, emails } from '@/app/db/schema';
|
||||
import { authenticate } from '@/app/lib/utils';
|
||||
import { eq } from 'drizzle-orm';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useSearchParams } from 'next/navigation';
|
||||
import Link from 'next/link';
|
||||
export async function GET(req: NextRequest) {
|
||||
if (!authenticate(req)) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
||||
interface Email {
|
||||
key: string;
|
||||
subject: string;
|
||||
date: string;
|
||||
processed: string;
|
||||
}
|
||||
|
||||
export default function Emails() {
|
||||
const searchParams = useSearchParams();
|
||||
const { searchParams } = new URL(req.url);
|
||||
const bucket = searchParams.get('bucket');
|
||||
const mailbox = searchParams.get('mailbox');
|
||||
const [emails, setEmails] = useState<Email[]>([]);
|
||||
const [message, setMessage] = useState('');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
if (!bucket) return NextResponse.json({ error: 'Missing bucket' }, { status: 400 });
|
||||
|
||||
useEffect(() => {
|
||||
if (!bucket || !mailbox) {
|
||||
setError('Missing parameters');
|
||||
setLoading(false);
|
||||
return;
|
||||
const [domain] = await db.select().from(domains).where(eq(domains.bucket, bucket));
|
||||
if (!domain) return NextResponse.json({ error: 'Domain not found' }, { status: 404 });
|
||||
|
||||
// Hole alle E-Mail-Adressen aus den "to" Feldern für diese Domain
|
||||
const mailboxData = await db.select({ to: emails.to }).from(emails).where(eq(emails.domainId, domain.id));
|
||||
|
||||
// Extrahiere die Domain aus dem Bucket-Namen (z.B. "example-com-emails" -> "example.com")
|
||||
const domainName = bucket.replace('-emails', '').replace(/-/g, '.');
|
||||
|
||||
const uniqueMailboxes = new Set<string>();
|
||||
|
||||
// Filtere nur E-Mail-Adressen, die zur aktuellen Domain gehören
|
||||
mailboxData.forEach(em => {
|
||||
em.to?.forEach(recipient => {
|
||||
const recipientLower = recipient.toLowerCase();
|
||||
// Prüfe, ob die E-Mail-Adresse zur Domain gehört
|
||||
if (recipientLower.endsWith(`@${domainName}`)) {
|
||||
uniqueMailboxes.add(recipientLower);
|
||||
}
|
||||
const auth = localStorage.getItem('auth');
|
||||
if (!auth) {
|
||||
setError('Not authenticated');
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
fetch(`/api/emails?bucket=${bucket}&mailbox=${encodeURIComponent(mailbox)}`, {
|
||||
headers: { Authorization: `Basic ${auth}` }
|
||||
})
|
||||
.then(res => {
|
||||
if (!res.ok) throw new Error('Failed to fetch emails');
|
||||
return res.json();
|
||||
})
|
||||
.then(data => {
|
||||
// Sortiere nach date descending
|
||||
const sorted = data.sort((a: Email, b: Email) => new Date(b.date).getTime() - new Date(a.date).getTime());
|
||||
setEmails(sorted);
|
||||
})
|
||||
.catch(err => setError(err.message))
|
||||
.finally(() => setLoading(false));
|
||||
}, [bucket, mailbox]);
|
||||
|
||||
if (loading) return <div className="min-h-screen flex items-center justify-center bg-gray-100">Loading...</div>;
|
||||
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
||||
|
||||
const formatDate = (dateStr: string) => {
|
||||
const date = new Date(dateStr);
|
||||
return date.toLocaleString('en-US', { year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hour12: false });
|
||||
};
|
||||
|
||||
const handleResendAll = async () => {
|
||||
const auth = localStorage.getItem('auth');
|
||||
if (!auth) return setMessage('Not authenticated');
|
||||
|
||||
const response = await fetch('/api/resend-domain', {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Basic ${auth}`, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ bucket }),
|
||||
});
|
||||
const res = await response.json();
|
||||
setMessage(res.message || res.error);
|
||||
};
|
||||
|
||||
const handleUpdateProcessed = async (key: string, newValue: boolean) => {
|
||||
const auth = localStorage.getItem('auth');
|
||||
if (!auth) return;
|
||||
|
||||
await fetch('/api/email', {
|
||||
method: 'PUT',
|
||||
headers: { Authorization: `Basic ${auth}`, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ bucket, key, processed: newValue ? 'true' : 'false' }),
|
||||
});
|
||||
setEmails(emails.map(em => em.key === key ? { ...em, processed: newValue ? 'true' : 'false' } : em));
|
||||
};
|
||||
|
||||
const handleResend = async (key: string) => {
|
||||
const auth = localStorage.getItem('auth');
|
||||
if (!auth) return alert('Not authenticated');
|
||||
|
||||
const response = await fetch('/api/email', {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Basic ${auth}`, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ bucket, key }),
|
||||
});
|
||||
const res = await response.json();
|
||||
alert(res.message || res.error);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gradient-to-b from-blue-50 to-gray-100 p-8">
|
||||
<nav className="max-w-4xl mx-auto mb-6 bg-white p-4 rounded-lg shadow-sm">
|
||||
<ol className="flex flex-wrap space-x-2 text-sm text-gray-500">
|
||||
<li><Link href="/" className="hover:text-blue-600">Home</Link></li>
|
||||
<li className="mx-1">/</li>
|
||||
<li><Link href="/domains" className="hover:text-blue-600">Domains</Link></li>
|
||||
<li className="mx-1">/</li>
|
||||
<li><Link href={`/mailboxes?bucket=${bucket}`} className="hover:text-blue-600">Mailboxes</Link></li>
|
||||
<li className="mx-1">/</li>
|
||||
<li className="font-semibold text-gray-700">Emails</li>
|
||||
</ol>
|
||||
</nav>
|
||||
<h1 className="text-4xl font-bold mb-8 text-center text-gray-800">Emails for {mailbox} in {bucket}</h1>
|
||||
<div className="flex justify-center mb-6">
|
||||
<button
|
||||
onClick={handleResendAll}
|
||||
className="bg-green-500 text-white px-8 py-3 rounded-lg hover:bg-green-600 transition shadow-md"
|
||||
>
|
||||
Resend all unprocessed
|
||||
</button>
|
||||
</div>
|
||||
{message && <p className="text-center mb-6 text-blue-600 font-medium">{message}</p>}
|
||||
<div className="overflow-x-auto max-w-4xl mx-auto bg-white rounded-lg shadow-md">
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead className="bg-blue-50">
|
||||
<tr>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Subject</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Date</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">S3 Key</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Processed</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="bg-white divide-y divide-gray-200">
|
||||
{emails.map((e: Email) => (
|
||||
<tr key={e.key} className="hover:bg-blue-50 transition">
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">{e.subject}</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">{formatDate(e.date)}</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500 truncate max-w-xs">{e.key}</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={e.processed === 'true'}
|
||||
onChange={() => handleUpdateProcessed(e.key, e.processed !== 'true')}
|
||||
className="h-4 w-4 text-blue-600 focus:ring-blue-500 border-gray-300 rounded"
|
||||
/>
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm font-medium">
|
||||
<Link href={`/email?bucket=${bucket}&key=${e.key}&mailbox=${encodeURIComponent(mailbox || '')}`} className="text-blue-600 hover:text-blue-900 mr-4">
|
||||
View
|
||||
</Link>
|
||||
<button onClick={() => handleResend(e.key)} className="text-green-600 hover:text-green-900">
|
||||
Resend
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
return NextResponse.json(Array.from(uniqueMailboxes).sort());
|
||||
}
|
||||
145
app/lib/sync.ts
145
app/lib/sync.ts
|
|
@ -5,13 +5,13 @@ import { simpleParser } from 'mailparser';
|
|||
import { ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, HeadObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
import { eq, sql, inArray } from 'drizzle-orm';
|
||||
import { Readable } from 'stream';
|
||||
import pLimit from 'p-limit'; // Für parallele Verarbeitung mit Limit
|
||||
import pLimit from 'p-limit';
|
||||
import pRetry from 'p-retry';
|
||||
|
||||
// Konfigurierbare Konstanten
|
||||
const CONCURRENT_S3_OPERATIONS = 10; // Parallele S3 Operationen
|
||||
const BATCH_INSERT_SIZE = 100; // Batch-Größe für DB Inserts
|
||||
const CONCURRENT_EMAIL_PARSING = 5; // Parallele E-Mail Parser
|
||||
const CONCURRENT_S3_OPERATIONS = 10;
|
||||
const BATCH_INSERT_SIZE = 100;
|
||||
const CONCURRENT_EMAIL_PARSING = 5;
|
||||
|
||||
export async function syncAllDomains() {
|
||||
console.log('Starting optimized syncAllDomains...');
|
||||
|
|
@ -22,8 +22,7 @@ export async function syncAllDomains() {
|
|||
const domainBuckets = Buckets?.filter(b => b.Name?.endsWith('-emails')) || [];
|
||||
console.log(`Found ${domainBuckets.length} domain buckets`);
|
||||
|
||||
// Parallele Verarbeitung der Buckets
|
||||
const bucketLimit = pLimit(3); // Max 3 Buckets parallel
|
||||
const bucketLimit = pLimit(3);
|
||||
|
||||
await Promise.all(
|
||||
domainBuckets.map(bucketObj =>
|
||||
|
|
@ -32,7 +31,6 @@ export async function syncAllDomains() {
|
|||
const domainName = bucket.replace('-emails', '').replace(/-/g, '.');
|
||||
console.log(`Processing bucket: ${bucket}`);
|
||||
|
||||
// Upsert Domain
|
||||
const [domain] = await db
|
||||
.insert(domains)
|
||||
.values({ bucket, domain: domainName })
|
||||
|
|
@ -63,7 +61,7 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
const response = await pRetry(
|
||||
() => s3.send(new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
MaxKeys: 1000, // Maximum per Request
|
||||
MaxKeys: 1000,
|
||||
ContinuationToken: continuationToken
|
||||
})),
|
||||
{ retries: 3 }
|
||||
|
|
@ -94,37 +92,45 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
|
||||
// 3. Bestimme was zu tun ist
|
||||
const toInsert: string[] = [];
|
||||
const toCheckProcessed: string[] = [];
|
||||
const toUpdate: string[] = [];
|
||||
|
||||
for (const key of allS3Keys) {
|
||||
if (!existingKeysMap.has(key)) {
|
||||
toInsert.push(key);
|
||||
} else {
|
||||
toCheckProcessed.push(key);
|
||||
toUpdate.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`To insert: ${toInsert.length}, To check: ${toCheckProcessed.length}`);
|
||||
console.log(`To insert: ${toInsert.length}, To update: ${toUpdate.length}`);
|
||||
|
||||
// 4. Parallele Verarbeitung der Updates (Processed Status)
|
||||
if (toCheckProcessed.length > 0) {
|
||||
// 4. Parallele Verarbeitung der Updates (Metadaten)
|
||||
if (toUpdate.length > 0) {
|
||||
const updateLimit = pLimit(CONCURRENT_S3_OPERATIONS);
|
||||
const updatePromises = toCheckProcessed.map(key =>
|
||||
const updatePromises = toUpdate.map(key =>
|
||||
updateLimit(async () => {
|
||||
try {
|
||||
const head = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
|
||||
const processed = head.Metadata?.[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!;
|
||||
const currentProcessed = existingKeysMap.get(key);
|
||||
const metadata = head.Metadata || {};
|
||||
|
||||
const processed = metadata[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!;
|
||||
const processedAt = metadata['processed_at'] ? new Date(metadata['processed_at']) : null;
|
||||
const processedBy = metadata['processed_by'] || null;
|
||||
const queuedTo = metadata['queued_to'] || null;
|
||||
const status = metadata['status'] || null;
|
||||
|
||||
if (currentProcessed !== processed) {
|
||||
await db
|
||||
.update(emails)
|
||||
.set({ processed })
|
||||
.set({
|
||||
processed,
|
||||
processedAt,
|
||||
processedBy,
|
||||
queuedTo,
|
||||
status
|
||||
})
|
||||
.where(eq(emails.s3Key, key));
|
||||
console.log(`Updated processed status for ${key}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error checking ${key}:`, error);
|
||||
console.error(`Error updating ${key}:`, error);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
|
@ -136,17 +142,14 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
if (toInsert.length > 0) {
|
||||
console.log(`Processing ${toInsert.length} new emails...`);
|
||||
|
||||
// Verarbeite in Batches
|
||||
for (let i = 0; i < toInsert.length; i += BATCH_INSERT_SIZE) {
|
||||
const batch = toInsert.slice(i, i + BATCH_INSERT_SIZE);
|
||||
console.log(`Processing batch ${Math.floor(i/BATCH_INSERT_SIZE) + 1}/${Math.ceil(toInsert.length/BATCH_INSERT_SIZE)}`);
|
||||
|
||||
// Paralleles Fetching und Parsing
|
||||
const parseLimit = pLimit(CONCURRENT_EMAIL_PARSING);
|
||||
const emailDataPromises = batch.map(key =>
|
||||
parseLimit(async () => {
|
||||
try {
|
||||
// Hole Objekt und Metadata parallel
|
||||
const [getObjResponse, headResponse] = await Promise.all([
|
||||
pRetry(() => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })), { retries: 2 }),
|
||||
pRetry(() => s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })), { retries: 2 })
|
||||
|
|
@ -154,11 +157,12 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
|
||||
const raw = await getBody(getObjResponse.Body as Readable);
|
||||
const parsed = await simpleParser(raw, {
|
||||
skipHtmlToText: true, // Schneller, wenn Text nicht benötigt
|
||||
skipHtmlToText: true,
|
||||
skipTextContent: false,
|
||||
skipImageLinks: true
|
||||
});
|
||||
|
||||
const metadata = headResponse.Metadata || {};
|
||||
const to = extractAddresses(parsed.to);
|
||||
const cc = extractAddresses(parsed.cc);
|
||||
const bcc = extractAddresses(parsed.bcc);
|
||||
|
|
@ -173,8 +177,13 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
subject: parsed.subject,
|
||||
html: parsed.html || parsed.textAsHtml,
|
||||
raw: raw.toString('utf-8'),
|
||||
processed: headResponse.Metadata?.[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!,
|
||||
processed: metadata[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!,
|
||||
date: parsed.date || headResponse.LastModified,
|
||||
// Neue Metadaten
|
||||
processedAt: metadata['processed_at'] ? new Date(metadata['processed_at']) : null,
|
||||
processedBy: metadata['processed_by'] || null,
|
||||
queuedTo: metadata['queued_to'] || null,
|
||||
status: metadata['status'] || null,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error processing ${key}:`, error);
|
||||
|
|
@ -185,7 +194,6 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
|
||||
const emailData = (await Promise.all(emailDataPromises)).filter(Boolean);
|
||||
|
||||
// Batch Insert
|
||||
if (emailData.length > 0) {
|
||||
await db.insert(emails).values(emailData);
|
||||
console.log(`Inserted ${emailData.length} emails`);
|
||||
|
|
@ -197,7 +205,6 @@ async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3
|
|||
console.log(`Sync for ${bucket} completed in ${duration}s`);
|
||||
}
|
||||
|
||||
// Helper Funktion für Address-Extraktion
|
||||
function extractAddresses(addressObj: any): string[] {
|
||||
if (!addressObj) return [];
|
||||
if (Array.isArray(addressObj)) {
|
||||
|
|
@ -205,85 +212,3 @@ function extractAddresses(addressObj: any): string[] {
|
|||
}
|
||||
return addressObj.value?.map((v: any) => v.address?.toLowerCase() || '').filter(Boolean) || [];
|
||||
}
|
||||
|
||||
// Optimierte Version mit Stream-Processing für sehr große Buckets
|
||||
export async function syncEmailsForDomainStreaming(domainId: number, bucket: string, s3: S3Client) {
|
||||
console.log(`Starting streaming sync for bucket: ${bucket}`);
|
||||
|
||||
// Verwende S3 Select für große Datensätze (falls unterstützt)
|
||||
// Dies reduziert die übertragene Datenmenge erheblich
|
||||
|
||||
const existingKeys = new Set(
|
||||
(await db
|
||||
.select({ s3Key: emails.s3Key })
|
||||
.from(emails)
|
||||
.where(eq(emails.domainId, domainId))
|
||||
).map(e => e.s3Key)
|
||||
);
|
||||
|
||||
const processQueue: any[] = [];
|
||||
const QUEUE_SIZE = 50;
|
||||
|
||||
// Stream-basierte Verarbeitung
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
do {
|
||||
const response = await s3.send(new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
MaxKeys: 100,
|
||||
ContinuationToken: continuationToken
|
||||
}));
|
||||
|
||||
const newKeys = response.Contents?.filter(obj =>
|
||||
obj.Key && !existingKeys.has(obj.Key)
|
||||
) || [];
|
||||
|
||||
// Verarbeite parallel während wir weitere Keys holen
|
||||
if (newKeys.length > 0) {
|
||||
const batch = newKeys.slice(0, QUEUE_SIZE);
|
||||
// Prozessiere Batch async (ohne await)
|
||||
processBatchAsync(batch, bucket, domainId, s3);
|
||||
}
|
||||
|
||||
continuationToken = response.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
}
|
||||
|
||||
async function processBatchAsync(batch: any[], bucket: string, domainId: number, s3: S3Client) {
|
||||
// Async Batch Processing ohne den Hauptthread zu blockieren
|
||||
const emailData = await Promise.all(
|
||||
batch.map(async (obj) => {
|
||||
try {
|
||||
const [getObjResponse, headResponse] = await Promise.all([
|
||||
s3.send(new GetObjectCommand({ Bucket: bucket, Key: obj.Key })),
|
||||
s3.send(new HeadObjectCommand({ Bucket: bucket, Key: obj.Key }))
|
||||
]);
|
||||
|
||||
const raw = await getBody(getObjResponse.Body as Readable);
|
||||
const parsed = await simpleParser(raw);
|
||||
|
||||
return {
|
||||
domainId,
|
||||
s3Key: obj.Key,
|
||||
from: parsed.from?.value[0]?.address,
|
||||
to: extractAddresses(parsed.to),
|
||||
cc: extractAddresses(parsed.cc),
|
||||
bcc: extractAddresses(parsed.bcc),
|
||||
subject: parsed.subject,
|
||||
html: parsed.html || parsed.textAsHtml,
|
||||
raw: raw.toString('utf-8'),
|
||||
processed: headResponse.Metadata?.[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!,
|
||||
date: parsed.date || obj.LastModified,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error processing ${obj.Key}:`, error);
|
||||
return null;
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
const validData = emailData.filter(Boolean);
|
||||
if (validData.length > 0) {
|
||||
await db.insert(emails).values(validData);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue