changes
This commit is contained in:
parent
6d12e7e151
commit
2cac2af5ab
|
|
@ -32,16 +32,23 @@ export default function Domains() {
|
||||||
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen bg-gray-100 p-8">
|
<div className="min-h-screen bg-gradient-to-b from-blue-50 to-gray-100 p-8">
|
||||||
<h1 className="text-3xl font-bold mb-6 text-center">Domains</h1>
|
<nav className="max-w-4xl mx-auto mb-6 bg-white p-4 rounded-lg shadow-sm">
|
||||||
<ul className="max-w-md mx-auto bg-white rounded-lg shadow-md divide-y divide-gray-200">
|
<ol className="flex flex-wrap space-x-2 text-sm text-gray-500">
|
||||||
|
<li><Link href="/" className="hover:text-blue-600">Home</Link></li>
|
||||||
|
<li className="mx-1">/</li>
|
||||||
|
<li className="font-semibold text-gray-700">Domains</li>
|
||||||
|
</ol>
|
||||||
|
</nav>
|
||||||
|
<h1 className="text-4xl font-bold mb-8 text-center text-gray-800">Domains</h1>
|
||||||
|
<ul className="max-w-md mx-auto grid gap-4">
|
||||||
{domains.length === 0 ? (
|
{domains.length === 0 ? (
|
||||||
<li className="p-4 text-center text-gray-500">No domains found</li>
|
<li className="p-6 text-center text-gray-500 bg-white rounded-lg shadow-md">No domains found</li>
|
||||||
) : (
|
) : (
|
||||||
domains.map((d: any) => (
|
domains.map((d: any) => (
|
||||||
<li key={d.bucket} className="p-4 hover:bg-gray-50 transition">
|
<li key={d.bucket}>
|
||||||
<Link href={`/mailboxes?bucket=${d.bucket}`} className="text-blue-500 hover:underline">
|
<Link href={`/mailboxes?bucket=${d.bucket}`} className="block p-6 bg-white rounded-lg shadow-md hover:shadow-lg transition hover:bg-blue-50">
|
||||||
{d.domain}
|
<span className="text-lg font-medium text-blue-600">{d.domain}</span>
|
||||||
</Link>
|
</Link>
|
||||||
</li>
|
</li>
|
||||||
))
|
))
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,13 @@
|
||||||
|
|
||||||
import { useState, useEffect } from 'react';
|
import { useState, useEffect } from 'react';
|
||||||
import { useSearchParams } from 'next/navigation';
|
import { useSearchParams } from 'next/navigation';
|
||||||
import ReactMarkdown from 'react-markdown';
|
import Link from 'next/link';
|
||||||
|
|
||||||
export default function EmailDetail() {
|
export default function EmailDetail() {
|
||||||
const searchParams = useSearchParams();
|
const searchParams = useSearchParams();
|
||||||
const bucket = searchParams.get('bucket');
|
const bucket = searchParams.get('bucket');
|
||||||
const key = searchParams.get('key');
|
const key = searchParams.get('key');
|
||||||
|
const mailbox = searchParams.get('mailbox'); // Für Breadcrumb
|
||||||
const [email, setEmail] = useState({ subject: '', from: '', to: '', html: '', raw: '', processed: '' });
|
const [email, setEmail] = useState({ subject: '', from: '', to: '', html: '', raw: '', processed: '' });
|
||||||
const [viewMode, setViewMode] = useState('html');
|
const [viewMode, setViewMode] = useState('html');
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
@ -42,33 +43,45 @@ export default function EmailDetail() {
|
||||||
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen bg-gray-100 p-8">
|
<div className="min-h-screen bg-gradient-to-b from-blue-50 to-gray-100 p-8">
|
||||||
<div className="max-w-4xl mx-auto bg-white rounded-lg shadow-md p-6">
|
<nav className="max-w-4xl mx-auto mb-6 bg-white p-4 rounded-lg shadow-sm">
|
||||||
<h1 className="text-3xl font-bold mb-4">{email.subject}</h1>
|
<ol className="flex flex-wrap space-x-2 text-sm text-gray-500">
|
||||||
<p className="text-gray-700 mb-2"><strong>From:</strong> {email.from}</p>
|
<li><Link href="/" className="hover:text-blue-600">Home</Link></li>
|
||||||
<p className="text-gray-700 mb-2"><strong>To:</strong> {email.to}</p>
|
<li className="mx-1">/</li>
|
||||||
<p className="text-gray-700 mb-4"><strong>Processed:</strong> {email.processed}</p>
|
<li><Link href="/domains" className="hover:text-blue-600">Domains</Link></li>
|
||||||
<div className="flex mb-4">
|
<li className="mx-1">/</li>
|
||||||
|
<li><Link href={`/mailboxes?bucket=${bucket}`} className="hover:text-blue-600">Mailboxes</Link></li>
|
||||||
|
<li className="mx-1">/</li>
|
||||||
|
<li><Link href={`/emails?bucket=${bucket}&mailbox=${encodeURIComponent(mailbox || '')}`} className="hover:text-blue-600">Emails</Link></li>
|
||||||
|
<li className="mx-1">/</li>
|
||||||
|
<li className="font-semibold text-gray-700">Detail</li>
|
||||||
|
</ol>
|
||||||
|
</nav>
|
||||||
|
<div className="max-w-4xl mx-auto bg-white rounded-lg shadow-md p-8">
|
||||||
|
<h1 className="text-4xl font-bold mb-6 text-gray-800">{email.subject}</h1>
|
||||||
|
<p className="text-gray-700 mb-2 text-lg"><strong>From:</strong> {email.from}</p>
|
||||||
|
<p className="text-gray-700 mb-2 text-lg"><strong>To:</strong> {email.to}</p>
|
||||||
|
<p className="text-gray-700 mb-2 text-lg"><strong>S3 Key:</strong> {key}</p>
|
||||||
|
<p className="text-gray-700 mb-6 text-lg"><strong>Processed:</strong> {email.processed}</p>
|
||||||
|
<div className="flex mb-6 space-x-2">
|
||||||
<button
|
<button
|
||||||
onClick={() => setViewMode('html')}
|
onClick={() => setViewMode('html')}
|
||||||
className={`px-4 py-2 rounded-l ${viewMode === 'html' ? 'bg-blue-500 text-white' : 'bg-gray-200 text-gray-700'} hover:bg-blue-600 transition`}
|
className={`px-6 py-3 rounded-md font-medium ${viewMode === 'html' ? 'bg-blue-600 text-white' : 'bg-gray-200 text-gray-700'} hover:bg-blue-700 transition`}
|
||||||
>
|
>
|
||||||
HTML
|
HTML
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
onClick={() => setViewMode('raw')}
|
onClick={() => setViewMode('raw')}
|
||||||
className={`px-4 py-2 rounded-r ${viewMode === 'raw' ? 'bg-blue-500 text-white' : 'bg-gray-200 text-gray-700'} hover:bg-blue-600 transition`}
|
className={`px-6 py-3 rounded-md font-medium ${viewMode === 'raw' ? 'bg-blue-600 text-white' : 'bg-gray-200 text-gray-700'} hover:bg-blue-700 transition`}
|
||||||
>
|
>
|
||||||
RAW
|
RAW
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<div className="border border-gray-300 p-4 rounded bg-white overflow-auto max-h-96">
|
<div className="border border-gray-300 p-6 rounded-lg bg-white overflow-auto max-h-[600px] shadow-inner">
|
||||||
{viewMode === 'html' ? (
|
{viewMode === 'html' ? (
|
||||||
<div className="prose">
|
<div dangerouslySetInnerHTML={{ __html: email.html }} className="prose prose-lg max-w-none" />
|
||||||
<ReactMarkdown>{email.html}</ReactMarkdown>
|
|
||||||
</div>
|
|
||||||
) : (
|
) : (
|
||||||
<pre className="whitespace-pre-wrap text-sm">{email.raw}</pre>
|
<pre className="whitespace-pre-wrap text-sm font-mono bg-gray-50 p-4 rounded">{email.raw}</pre>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,11 @@ export default function Emails() {
|
||||||
if (!res.ok) throw new Error('Failed to fetch emails');
|
if (!res.ok) throw new Error('Failed to fetch emails');
|
||||||
return res.json();
|
return res.json();
|
||||||
})
|
})
|
||||||
.then(setEmails)
|
.then(data => {
|
||||||
|
// Sortiere nach date descending
|
||||||
|
const sorted = data.sort((a: Email, b: Email) => new Date(b.date).getTime() - new Date(a.date).getTime());
|
||||||
|
setEmails(sorted);
|
||||||
|
})
|
||||||
.catch(err => setError(err.message))
|
.catch(err => setError(err.message))
|
||||||
.finally(() => setLoading(false));
|
.finally(() => setLoading(false));
|
||||||
}, [bucket, mailbox]);
|
}, [bucket, mailbox]);
|
||||||
|
|
@ -48,6 +52,11 @@ export default function Emails() {
|
||||||
if (loading) return <div className="min-h-screen flex items-center justify-center bg-gray-100">Loading...</div>;
|
if (loading) return <div className="min-h-screen flex items-center justify-center bg-gray-100">Loading...</div>;
|
||||||
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
||||||
|
|
||||||
|
const formatDate = (dateStr: string) => {
|
||||||
|
const date = new Date(dateStr);
|
||||||
|
return date.toLocaleString('en-US', { year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hour12: false });
|
||||||
|
};
|
||||||
|
|
||||||
const handleResendAll = async () => {
|
const handleResendAll = async () => {
|
||||||
const auth = localStorage.getItem('auth');
|
const auth = localStorage.getItem('auth');
|
||||||
if (!auth) return setMessage('Not authenticated');
|
if (!auth) return setMessage('Not authenticated');
|
||||||
|
|
@ -87,32 +96,45 @@ export default function Emails() {
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen bg-gray-100 p-8">
|
<div className="min-h-screen bg-gradient-to-b from-blue-50 to-gray-100 p-8">
|
||||||
<h1 className="text-3xl font-bold mb-6 text-center">Emails for {mailbox} in {bucket}</h1>
|
<nav className="max-w-4xl mx-auto mb-6 bg-white p-4 rounded-lg shadow-sm">
|
||||||
|
<ol className="flex flex-wrap space-x-2 text-sm text-gray-500">
|
||||||
|
<li><Link href="/" className="hover:text-blue-600">Home</Link></li>
|
||||||
|
<li className="mx-1">/</li>
|
||||||
|
<li><Link href="/domains" className="hover:text-blue-600">Domains</Link></li>
|
||||||
|
<li className="mx-1">/</li>
|
||||||
|
<li><Link href={`/mailboxes?bucket=${bucket}`} className="hover:text-blue-600">Mailboxes</Link></li>
|
||||||
|
<li className="mx-1">/</li>
|
||||||
|
<li className="font-semibold text-gray-700">Emails</li>
|
||||||
|
</ol>
|
||||||
|
</nav>
|
||||||
|
<h1 className="text-4xl font-bold mb-8 text-center text-gray-800">Emails for {mailbox} in {bucket}</h1>
|
||||||
<div className="flex justify-center mb-6">
|
<div className="flex justify-center mb-6">
|
||||||
<button
|
<button
|
||||||
onClick={handleResendAll}
|
onClick={handleResendAll}
|
||||||
className="bg-green-500 text-white px-6 py-3 rounded hover:bg-green-600 transition"
|
className="bg-green-500 text-white px-8 py-3 rounded-lg hover:bg-green-600 transition shadow-md"
|
||||||
>
|
>
|
||||||
Resend all unprocessed
|
Resend all unprocessed
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
{message && <p className="text-center mb-4 text-blue-500">{message}</p>}
|
{message && <p className="text-center mb-6 text-blue-600 font-medium">{message}</p>}
|
||||||
<div className="overflow-x-auto max-w-4xl mx-auto bg-white rounded-lg shadow-md">
|
<div className="overflow-x-auto max-w-4xl mx-auto bg-white rounded-lg shadow-md">
|
||||||
<table className="min-w-full divide-y divide-gray-200">
|
<table className="min-w-full divide-y divide-gray-200">
|
||||||
<thead className="bg-gray-50">
|
<thead className="bg-blue-50">
|
||||||
<tr>
|
<tr>
|
||||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Subject</th>
|
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Subject</th>
|
||||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Date</th>
|
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Date</th>
|
||||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Processed</th>
|
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">S3 Key</th>
|
||||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Actions</th>
|
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Processed</th>
|
||||||
|
<th className="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">Actions</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody className="bg-white divide-y divide-gray-200">
|
<tbody className="bg-white divide-y divide-gray-200">
|
||||||
{emails.map((e: Email) => (
|
{emails.map((e: Email) => (
|
||||||
<tr key={e.key} className="hover:bg-gray-50 transition">
|
<tr key={e.key} className="hover:bg-blue-50 transition">
|
||||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">{e.subject}</td>
|
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">{e.subject}</td>
|
||||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">{e.date}</td>
|
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">{formatDate(e.date)}</td>
|
||||||
|
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500 truncate max-w-xs">{e.key}</td>
|
||||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
|
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
|
||||||
<input
|
<input
|
||||||
type="checkbox"
|
type="checkbox"
|
||||||
|
|
@ -122,7 +144,7 @@ export default function Emails() {
|
||||||
/>
|
/>
|
||||||
</td>
|
</td>
|
||||||
<td className="px-6 py-4 whitespace-nowrap text-sm font-medium">
|
<td className="px-6 py-4 whitespace-nowrap text-sm font-medium">
|
||||||
<Link href={`/email?bucket=${bucket}&key=${e.key}`} className="text-blue-600 hover:text-blue-900 mr-4">
|
<Link href={`/email?bucket=${bucket}&key=${e.key}&mailbox=${encodeURIComponent(mailbox || '')}`} className="text-blue-600 hover:text-blue-900 mr-4">
|
||||||
View
|
View
|
||||||
</Link>
|
</Link>
|
||||||
<button onClick={() => handleResend(e.key)} className="text-green-600 hover:text-green-900">
|
<button onClick={() => handleResend(e.key)} className="text-green-600 hover:text-green-900">
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,7 @@
|
||||||
@tailwind base;
|
@tailwind base;
|
||||||
@tailwind components;
|
@tailwind components;
|
||||||
@tailwind utilities;
|
@tailwind utilities;
|
||||||
|
|
||||||
|
li {
|
||||||
|
list-style-type: none;
|
||||||
|
}
|
||||||
349
app/lib/sync.ts
349
app/lib/sync.ts
|
|
@ -2,93 +2,288 @@ import { db } from '@/app/db/drizzle';
|
||||||
import { domains, emails } from '@/app/db/schema';
|
import { domains, emails } from '@/app/db/schema';
|
||||||
import { getS3Client, getBody } from '@/app/lib/utils';
|
import { getS3Client, getBody } from '@/app/lib/utils';
|
||||||
import { simpleParser } from 'mailparser';
|
import { simpleParser } from 'mailparser';
|
||||||
import { ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
import { ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, HeadObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||||
import { eq, sql } from 'drizzle-orm';
|
import { eq, sql, inArray } from 'drizzle-orm';
|
||||||
import { Readable } from 'stream';
|
import { Readable } from 'stream';
|
||||||
import pRetry from 'p-retry'; // Für Retry bei Timeouts
|
import pLimit from 'p-limit'; // Für parallele Verarbeitung mit Limit
|
||||||
|
import pRetry from 'p-retry';
|
||||||
|
|
||||||
|
// Konfigurierbare Konstanten
|
||||||
|
const CONCURRENT_S3_OPERATIONS = 10; // Parallele S3 Operationen
|
||||||
|
const BATCH_INSERT_SIZE = 100; // Batch-Größe für DB Inserts
|
||||||
|
const CONCURRENT_EMAIL_PARSING = 5; // Parallele E-Mail Parser
|
||||||
|
|
||||||
export async function syncAllDomains() {
|
export async function syncAllDomains() {
|
||||||
console.log('Starting syncAllDomains...');
|
console.log('Starting optimized syncAllDomains...');
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
const s3 = getS3Client();
|
const s3 = getS3Client();
|
||||||
const { Buckets } = await pRetry(() => s3.send(new ListBucketsCommand({})), { retries: 3 }); // Retry bei Fail
|
const { Buckets } = await pRetry(() => s3.send(new ListBucketsCommand({})), { retries: 3 });
|
||||||
const domainBuckets = Buckets?.filter(b => b.Name?.endsWith('-emails')) || [];
|
const domainBuckets = Buckets?.filter(b => b.Name?.endsWith('-emails')) || [];
|
||||||
console.log('Found domain buckets:', domainBuckets.map(b => b.Name).join(', ') || 'None');
|
console.log(`Found ${domainBuckets.length} domain buckets`);
|
||||||
|
|
||||||
for (const bucketObj of domainBuckets) {
|
// Parallele Verarbeitung der Buckets
|
||||||
const bucket = bucketObj.Name!;
|
const bucketLimit = pLimit(3); // Max 3 Buckets parallel
|
||||||
const domainName = bucket.replace('-emails', '').replace(/-/g, '.');
|
|
||||||
console.log(`Processing bucket: ${bucket} (domain: ${domainName})`);
|
await Promise.all(
|
||||||
|
domainBuckets.map(bucketObj =>
|
||||||
|
bucketLimit(async () => {
|
||||||
|
const bucket = bucketObj.Name!;
|
||||||
|
const domainName = bucket.replace('-emails', '').replace(/-/g, '.');
|
||||||
|
console.log(`Processing bucket: ${bucket}`);
|
||||||
|
|
||||||
// Upsert Domain
|
// Upsert Domain
|
||||||
const existingDomain = await db.select().from(domains).where(eq(domains.bucket, bucket)).limit(1);
|
const [domain] = await db
|
||||||
console.log(`Existing domain for ${bucket}: ${existingDomain.length > 0 ? 'Yes (ID: ' + existingDomain[0].id + ')' : 'No'}`);
|
.insert(domains)
|
||||||
let domainId;
|
.values({ bucket, domain: domainName })
|
||||||
if (existingDomain.length === 0) {
|
.onConflictDoUpdate({
|
||||||
const [newDomain] = await db.insert(domains).values({ bucket, domain: domainName }).returning({ id: domains.id });
|
target: domains.bucket,
|
||||||
domainId = newDomain.id;
|
set: { domain: domainName }
|
||||||
console.log(`Created new domain ID: ${domainId}`);
|
})
|
||||||
} else {
|
.returning({ id: domains.id });
|
||||||
domainId = existingDomain[0].id;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sync Emails
|
await syncEmailsForDomainOptimized(domain.id, bucket, s3);
|
||||||
await syncEmailsForDomain(domainId, bucket);
|
})
|
||||||
}
|
)
|
||||||
console.log('syncAllDomains completed.');
|
);
|
||||||
|
|
||||||
|
const duration = (Date.now() - startTime) / 1000;
|
||||||
|
console.log(`syncAllDomains completed in ${duration}s`);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function syncEmailsForDomain(domainId: number, bucket: string) {
|
async function syncEmailsForDomainOptimized(domainId: number, bucket: string, s3: S3Client) {
|
||||||
console.log(`Starting syncEmailsForDomain for bucket: ${bucket} (domainId: ${domainId})`);
|
console.log(`Starting optimized sync for bucket: ${bucket}`);
|
||||||
const s3 = getS3Client();
|
const startTime = Date.now();
|
||||||
const { Contents } = await pRetry(() => s3.send(new ListObjectsV2Command({ Bucket: bucket })), { retries: 3 });
|
|
||||||
console.log(`Found objects in bucket: ${Contents?.length || 0}`);
|
// 1. Hole alle S3 Keys auf einmal
|
||||||
|
let allS3Keys: string[] = [];
|
||||||
for (const obj of Contents || []) {
|
let continuationToken: string | undefined;
|
||||||
if (!obj.Key) continue;
|
|
||||||
console.log(`Processing object: ${obj.Key}`);
|
do {
|
||||||
|
const response = await pRetry(
|
||||||
// Check if exists
|
() => s3.send(new ListObjectsV2Command({
|
||||||
const existing = await db.select().from(emails).where(eq(emails.s3Key, obj.Key!)).limit(1);
|
Bucket: bucket,
|
||||||
console.log(`Existing email for ${obj.Key}: ${existing.length > 0 ? 'Yes' : 'No'}`);
|
MaxKeys: 1000, // Maximum per Request
|
||||||
if (existing.length > 0) {
|
ContinuationToken: continuationToken
|
||||||
// Update processed if changed
|
})),
|
||||||
const head = await pRetry(() => s3.send(new HeadObjectCommand({ Bucket: bucket, Key: obj.Key })), { retries: 3 });
|
{ retries: 3 }
|
||||||
const processed = head.Metadata?.[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!;
|
);
|
||||||
console.log(`Processed metadata for ${obj.Key}: ${processed} (DB: ${existing[0].processed})`);
|
|
||||||
if (existing[0].processed !== processed) {
|
allS3Keys.push(...(response.Contents?.map(obj => obj.Key!).filter(Boolean) || []));
|
||||||
await db.update(emails).set({ processed }).where(eq(emails.s3Key, obj.Key!));
|
continuationToken = response.NextContinuationToken;
|
||||||
console.log(`Updated processed for ${obj.Key}`);
|
} while (continuationToken);
|
||||||
}
|
|
||||||
await new Promise(resolve => setTimeout(resolve, 100)); // Kleine Delay gegen Throttling
|
console.log(`Found ${allS3Keys.length} objects in bucket`);
|
||||||
continue;
|
|
||||||
|
if (allS3Keys.length === 0) return;
|
||||||
|
|
||||||
|
// 2. Hole alle existierenden E-Mails aus der DB in einem Query
|
||||||
|
const existingEmails = await db
|
||||||
|
.select({
|
||||||
|
s3Key: emails.s3Key,
|
||||||
|
processed: emails.processed
|
||||||
|
})
|
||||||
|
.from(emails)
|
||||||
|
.where(inArray(emails.s3Key, allS3Keys));
|
||||||
|
|
||||||
|
const existingKeysMap = new Map(
|
||||||
|
existingEmails.map(e => [e.s3Key, e.processed])
|
||||||
|
);
|
||||||
|
|
||||||
|
console.log(`Found ${existingEmails.length} existing emails in DB`);
|
||||||
|
|
||||||
|
// 3. Bestimme was zu tun ist
|
||||||
|
const toInsert: string[] = [];
|
||||||
|
const toCheckProcessed: string[] = [];
|
||||||
|
|
||||||
|
for (const key of allS3Keys) {
|
||||||
|
if (!existingKeysMap.has(key)) {
|
||||||
|
toInsert.push(key);
|
||||||
|
} else {
|
||||||
|
toCheckProcessed.push(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
// New: Parse and insert
|
|
||||||
console.log(`Parsing new email: ${obj.Key}`);
|
|
||||||
const { Body } = await pRetry(() => s3.send(new GetObjectCommand({ Bucket: bucket, Key: obj.Key })), { retries: 3 });
|
|
||||||
const raw = await getBody(Body as Readable);
|
|
||||||
const parsed = await simpleParser(raw);
|
|
||||||
const head = await pRetry(() => s3.send(new HeadObjectCommand({ Bucket: bucket, Key: obj.Key })), { retries: 3 });
|
|
||||||
|
|
||||||
const to = parsed.to ? (Array.isArray(parsed.to) ? parsed.to.flatMap(t => t.value.map(v => v.address?.toLowerCase() || '')) : parsed.to.value.map(v => v.address?.toLowerCase() || '')) : [];
|
|
||||||
const cc = parsed.cc ? (Array.isArray(parsed.cc) ? parsed.cc.flatMap(c => c.value.map(v => v.address?.toLowerCase() || '')) : parsed.cc.value.map(v => v.address?.toLowerCase() || '')) : [];
|
|
||||||
const bcc = parsed.bcc ? (Array.isArray(parsed.bcc) ? parsed.bcc.flatMap(b => b.value.map(v => v.address?.toLowerCase() || '')) : parsed.bcc.value.map(v => v.address?.toLowerCase() || '')) : [];
|
|
||||||
|
|
||||||
await db.insert(emails).values({
|
|
||||||
domainId,
|
|
||||||
s3Key: obj.Key!,
|
|
||||||
from: parsed.from?.value[0].address,
|
|
||||||
to,
|
|
||||||
cc,
|
|
||||||
bcc,
|
|
||||||
subject: parsed.subject,
|
|
||||||
html: parsed.html || parsed.textAsHtml,
|
|
||||||
raw: raw.toString('utf-8'),
|
|
||||||
processed: head.Metadata?.[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!,
|
|
||||||
date: parsed.date || obj.LastModified,
|
|
||||||
});
|
|
||||||
console.log(`Inserted new email: ${obj.Key}`);
|
|
||||||
await new Promise(resolve => setTimeout(resolve, 100)); // Delay gegen Throttling
|
|
||||||
}
|
}
|
||||||
console.log(`syncEmailsForDomain completed for bucket: ${bucket}`);
|
|
||||||
|
console.log(`To insert: ${toInsert.length}, To check: ${toCheckProcessed.length}`);
|
||||||
|
|
||||||
|
// 4. Parallele Verarbeitung der Updates (Processed Status)
|
||||||
|
if (toCheckProcessed.length > 0) {
|
||||||
|
const updateLimit = pLimit(CONCURRENT_S3_OPERATIONS);
|
||||||
|
const updatePromises = toCheckProcessed.map(key =>
|
||||||
|
updateLimit(async () => {
|
||||||
|
try {
|
||||||
|
const head = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
|
||||||
|
const processed = head.Metadata?.[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!;
|
||||||
|
const currentProcessed = existingKeysMap.get(key);
|
||||||
|
|
||||||
|
if (currentProcessed !== processed) {
|
||||||
|
await db
|
||||||
|
.update(emails)
|
||||||
|
.set({ processed })
|
||||||
|
.where(eq(emails.s3Key, key));
|
||||||
|
console.log(`Updated processed status for ${key}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error checking ${key}:`, error);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
await Promise.all(updatePromises);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Batch-Insert für neue E-Mails
|
||||||
|
if (toInsert.length > 0) {
|
||||||
|
console.log(`Processing ${toInsert.length} new emails...`);
|
||||||
|
|
||||||
|
// Verarbeite in Batches
|
||||||
|
for (let i = 0; i < toInsert.length; i += BATCH_INSERT_SIZE) {
|
||||||
|
const batch = toInsert.slice(i, i + BATCH_INSERT_SIZE);
|
||||||
|
console.log(`Processing batch ${Math.floor(i/BATCH_INSERT_SIZE) + 1}/${Math.ceil(toInsert.length/BATCH_INSERT_SIZE)}`);
|
||||||
|
|
||||||
|
// Paralleles Fetching und Parsing
|
||||||
|
const parseLimit = pLimit(CONCURRENT_EMAIL_PARSING);
|
||||||
|
const emailDataPromises = batch.map(key =>
|
||||||
|
parseLimit(async () => {
|
||||||
|
try {
|
||||||
|
// Hole Objekt und Metadata parallel
|
||||||
|
const [getObjResponse, headResponse] = await Promise.all([
|
||||||
|
pRetry(() => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })), { retries: 2 }),
|
||||||
|
pRetry(() => s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })), { retries: 2 })
|
||||||
|
]);
|
||||||
|
|
||||||
|
const raw = await getBody(getObjResponse.Body as Readable);
|
||||||
|
const parsed = await simpleParser(raw, {
|
||||||
|
skipHtmlToText: true, // Schneller, wenn Text nicht benötigt
|
||||||
|
skipTextContent: false,
|
||||||
|
skipImageLinks: true
|
||||||
|
});
|
||||||
|
|
||||||
|
const to = extractAddresses(parsed.to);
|
||||||
|
const cc = extractAddresses(parsed.cc);
|
||||||
|
const bcc = extractAddresses(parsed.bcc);
|
||||||
|
|
||||||
|
return {
|
||||||
|
domainId,
|
||||||
|
s3Key: key,
|
||||||
|
from: parsed.from?.value[0]?.address,
|
||||||
|
to,
|
||||||
|
cc,
|
||||||
|
bcc,
|
||||||
|
subject: parsed.subject,
|
||||||
|
html: parsed.html || parsed.textAsHtml,
|
||||||
|
raw: raw.toString('utf-8'),
|
||||||
|
processed: headResponse.Metadata?.[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!,
|
||||||
|
date: parsed.date || headResponse.LastModified,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error processing ${key}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
const emailData = (await Promise.all(emailDataPromises)).filter(Boolean);
|
||||||
|
|
||||||
|
// Batch Insert
|
||||||
|
if (emailData.length > 0) {
|
||||||
|
await db.insert(emails).values(emailData);
|
||||||
|
console.log(`Inserted ${emailData.length} emails`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const duration = (Date.now() - startTime) / 1000;
|
||||||
|
console.log(`Sync for ${bucket} completed in ${duration}s`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper Funktion für Address-Extraktion
|
||||||
|
function extractAddresses(addressObj: any): string[] {
|
||||||
|
if (!addressObj) return [];
|
||||||
|
if (Array.isArray(addressObj)) {
|
||||||
|
return addressObj.flatMap(t => t.value.map((v: any) => v.address?.toLowerCase() || '')).filter(Boolean);
|
||||||
|
}
|
||||||
|
return addressObj.value?.map((v: any) => v.address?.toLowerCase() || '').filter(Boolean) || [];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Optimierte Version mit Stream-Processing für sehr große Buckets
|
||||||
|
export async function syncEmailsForDomainStreaming(domainId: number, bucket: string, s3: S3Client) {
|
||||||
|
console.log(`Starting streaming sync for bucket: ${bucket}`);
|
||||||
|
|
||||||
|
// Verwende S3 Select für große Datensätze (falls unterstützt)
|
||||||
|
// Dies reduziert die übertragene Datenmenge erheblich
|
||||||
|
|
||||||
|
const existingKeys = new Set(
|
||||||
|
(await db
|
||||||
|
.select({ s3Key: emails.s3Key })
|
||||||
|
.from(emails)
|
||||||
|
.where(eq(emails.domainId, domainId))
|
||||||
|
).map(e => e.s3Key)
|
||||||
|
);
|
||||||
|
|
||||||
|
const processQueue: any[] = [];
|
||||||
|
const QUEUE_SIZE = 50;
|
||||||
|
|
||||||
|
// Stream-basierte Verarbeitung
|
||||||
|
let continuationToken: string | undefined;
|
||||||
|
|
||||||
|
do {
|
||||||
|
const response = await s3.send(new ListObjectsV2Command({
|
||||||
|
Bucket: bucket,
|
||||||
|
MaxKeys: 100,
|
||||||
|
ContinuationToken: continuationToken
|
||||||
|
}));
|
||||||
|
|
||||||
|
const newKeys = response.Contents?.filter(obj =>
|
||||||
|
obj.Key && !existingKeys.has(obj.Key)
|
||||||
|
) || [];
|
||||||
|
|
||||||
|
// Verarbeite parallel während wir weitere Keys holen
|
||||||
|
if (newKeys.length > 0) {
|
||||||
|
const batch = newKeys.slice(0, QUEUE_SIZE);
|
||||||
|
// Prozessiere Batch async (ohne await)
|
||||||
|
processBatchAsync(batch, bucket, domainId, s3);
|
||||||
|
}
|
||||||
|
|
||||||
|
continuationToken = response.NextContinuationToken;
|
||||||
|
} while (continuationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processBatchAsync(batch: any[], bucket: string, domainId: number, s3: S3Client) {
|
||||||
|
// Async Batch Processing ohne den Hauptthread zu blockieren
|
||||||
|
const emailData = await Promise.all(
|
||||||
|
batch.map(async (obj) => {
|
||||||
|
try {
|
||||||
|
const [getObjResponse, headResponse] = await Promise.all([
|
||||||
|
s3.send(new GetObjectCommand({ Bucket: bucket, Key: obj.Key })),
|
||||||
|
s3.send(new HeadObjectCommand({ Bucket: bucket, Key: obj.Key }))
|
||||||
|
]);
|
||||||
|
|
||||||
|
const raw = await getBody(getObjResponse.Body as Readable);
|
||||||
|
const parsed = await simpleParser(raw);
|
||||||
|
|
||||||
|
return {
|
||||||
|
domainId,
|
||||||
|
s3Key: obj.Key,
|
||||||
|
from: parsed.from?.value[0]?.address,
|
||||||
|
to: extractAddresses(parsed.to),
|
||||||
|
cc: extractAddresses(parsed.cc),
|
||||||
|
bcc: extractAddresses(parsed.bcc),
|
||||||
|
subject: parsed.subject,
|
||||||
|
html: parsed.html || parsed.textAsHtml,
|
||||||
|
raw: raw.toString('utf-8'),
|
||||||
|
processed: headResponse.Metadata?.[process.env.PROCESSED_META_KEY!] === process.env.PROCESSED_META_VALUE!,
|
||||||
|
date: parsed.date || obj.LastModified,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error processing ${obj.Key}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
const validData = emailData.filter(Boolean);
|
||||||
|
if (validData.length > 0) {
|
||||||
|
await db.insert(emails).values(validData);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1,17 +1,25 @@
|
||||||
import { S3Client } from '@aws-sdk/client-s3';
|
import { S3Client } from '@aws-sdk/client-s3';
|
||||||
|
import { NodeHttpHandler } from '@smithy/node-http-handler';
|
||||||
import { NextRequest } from 'next/server';
|
import { NextRequest } from 'next/server';
|
||||||
import { Readable } from 'stream';
|
import { Readable } from 'stream';
|
||||||
|
import https from 'https';
|
||||||
|
|
||||||
export function getS3Client() {
|
export function getS3Client() {
|
||||||
console.log('Creating S3Client...');
|
|
||||||
console.log('AWS_REGION:', process.env.AWS_REGION);
|
|
||||||
console.log('AWS_ACCESS_KEY_ID:', process.env.AWS_ACCESS_KEY_ID ? 'Set' : 'Not set'); // Maskiere sensible Infos
|
|
||||||
console.log('AWS_SECRET_ACCESS_KEY:', process.env.AWS_SECRET_ACCESS_KEY ? 'Set' : 'Not set');
|
|
||||||
|
|
||||||
return new S3Client({
|
return new S3Client({
|
||||||
region: process.env.AWS_REGION,
|
region: process.env.AWS_REGION,
|
||||||
credentials: { accessKeyId: process.env.AWS_ACCESS_KEY_ID!, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY! },
|
credentials: {
|
||||||
httpOptions: { connectTimeout: 60000, timeout: 60000 }, // 60s, verhindert Timeouts
|
accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
|
||||||
|
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
|
||||||
|
},
|
||||||
|
maxAttempts: 3,
|
||||||
|
requestHandler: new NodeHttpHandler({
|
||||||
|
connectionTimeout: 10000,
|
||||||
|
socketTimeout: 60000,
|
||||||
|
httpsAgent: new https.Agent({
|
||||||
|
keepAlive: true,
|
||||||
|
maxSockets: 50 // Erhöhe parallele Verbindungen
|
||||||
|
})
|
||||||
|
})
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -40,16 +40,25 @@ export default function Mailboxes() {
|
||||||
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
if (error) return <div className="min-h-screen flex items-center justify-center bg-gray-100 text-red-500">{error}</div>;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen bg-gray-100 p-8">
|
<div className="min-h-screen bg-gradient-to-b from-blue-50 to-gray-100 p-8">
|
||||||
<h1 className="text-3xl font-bold mb-6 text-center">Mailboxes for {bucket}</h1>
|
<nav className="max-w-4xl mx-auto mb-6 bg-white p-4 rounded-lg shadow-sm">
|
||||||
<ul className="max-w-md mx-auto bg-white rounded-lg shadow-md divide-y divide-gray-200">
|
<ol className="flex flex-wrap space-x-2 text-sm text-gray-500">
|
||||||
|
<li><Link href="/" className="hover:text-blue-600">Home</Link></li>
|
||||||
|
<li className="mx-1">/</li>
|
||||||
|
<li><Link href="/domains" className="hover:text-blue-600">Domains</Link></li>
|
||||||
|
<li className="mx-1">/</li>
|
||||||
|
<li className="font-semibold text-gray-700">Mailboxes</li>
|
||||||
|
</ol>
|
||||||
|
</nav>
|
||||||
|
<h1 className="text-4xl font-bold mb-8 text-center text-gray-800">Mailboxes for {bucket}</h1>
|
||||||
|
<ul className="max-w-md mx-auto grid gap-4">
|
||||||
{mailboxes.length === 0 ? (
|
{mailboxes.length === 0 ? (
|
||||||
<li className="p-4 text-center text-gray-500">No mailboxes found</li>
|
<li className="p-6 text-center text-gray-500 bg-white rounded-lg shadow-md">No mailboxes found</li>
|
||||||
) : (
|
) : (
|
||||||
mailboxes.map((m: string) => (
|
mailboxes.map((m: string) => (
|
||||||
<li key={m} className="p-4 hover:bg-gray-50 transition">
|
<li key={m}>
|
||||||
<Link href={`/emails?bucket=${bucket}&mailbox=${encodeURIComponent(m)}`} className="text-blue-500 hover:underline">
|
<Link href={`/emails?bucket=${bucket}&mailbox=${encodeURIComponent(m)}`} className="block p-6 bg-white rounded-lg shadow-md hover:shadow-lg transition hover:bg-blue-50">
|
||||||
{m}
|
<span className="text-lg font-medium text-blue-600">{m}</span>
|
||||||
</Link>
|
</Link>
|
||||||
</li>
|
</li>
|
||||||
))
|
))
|
||||||
|
|
|
||||||
16
app/page.tsx
16
app/page.tsx
|
|
@ -28,19 +28,19 @@ export default function Home() {
|
||||||
|
|
||||||
if (!loggedIn) {
|
if (!loggedIn) {
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen flex items-center justify-center bg-gray-100">
|
<div className="min-h-screen flex items-center justify-center bg-gradient-to-b from-blue-50 to-gray-100">
|
||||||
<div className="bg-white p-8 rounded-lg shadow-md w-96">
|
<div className="bg-white p-10 rounded-xl shadow-xl w-96">
|
||||||
<h1 className="text-2xl font-bold mb-6 text-center">Login</h1>
|
<h1 className="text-3xl font-bold mb-8 text-center text-gray-800">Login to Mail S3 Admin</h1>
|
||||||
<input
|
<input
|
||||||
type="password"
|
type="password"
|
||||||
value={password}
|
value={password}
|
||||||
onChange={e => setPassword(e.target.value)}
|
onChange={e => setPassword(e.target.value)}
|
||||||
className="border border-gray-300 p-3 mb-4 w-full rounded focus:outline-none focus:ring-2 focus:ring-blue-500"
|
className="border border-gray-300 p-4 mb-6 w-full rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500"
|
||||||
placeholder="Enter password"
|
placeholder="Enter password"
|
||||||
/>
|
/>
|
||||||
<button
|
<button
|
||||||
onClick={handleLogin}
|
onClick={handleLogin}
|
||||||
className="bg-blue-500 text-white p-3 rounded w-full hover:bg-blue-600 transition"
|
className="bg-blue-600 text-white p-4 rounded-lg w-full hover:bg-blue-700 transition font-medium"
|
||||||
>
|
>
|
||||||
Login
|
Login
|
||||||
</button>
|
</button>
|
||||||
|
|
@ -50,10 +50,10 @@ export default function Home() {
|
||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen bg-gray-100 p-8">
|
<div className="min-h-screen bg-gradient-to-b from-blue-50 to-gray-100 p-8">
|
||||||
<h1 className="text-3xl font-bold mb-6 text-center">Mail S3 Admin</h1>
|
<h1 className="text-4xl font-bold mb-8 text-center text-gray-800">Mail S3 Admin</h1>
|
||||||
<div className="flex justify-center">
|
<div className="flex justify-center">
|
||||||
<Link href="/domains" className="bg-blue-500 text-white px-6 py-3 rounded hover:bg-blue-600 transition">
|
<Link href="/domains" className="bg-blue-600 text-white px-8 py-4 rounded-lg hover:bg-blue-700 transition shadow-md font-medium">
|
||||||
Go to Domains
|
Go to Domains
|
||||||
</Link>
|
</Link>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.888.0",
|
"@aws-sdk/client-s3": "^3.888.0",
|
||||||
|
"@smithy/node-http-handler": "^4.2.1",
|
||||||
"dotenv": "^17.2.2",
|
"dotenv": "^17.2.2",
|
||||||
"drizzle-orm": "^0.44.5",
|
"drizzle-orm": "^0.44.5",
|
||||||
"email-addresses": "^5.0.0",
|
"email-addresses": "^5.0.0",
|
||||||
|
|
@ -19,6 +20,7 @@
|
||||||
"next": "15.5.3",
|
"next": "15.5.3",
|
||||||
"node-cron": "^4.2.1",
|
"node-cron": "^4.2.1",
|
||||||
"nodemailer": "^7.0.6",
|
"nodemailer": "^7.0.6",
|
||||||
|
"p-limit": "^7.1.1",
|
||||||
"p-retry": "^7.0.0",
|
"p-retry": "^7.0.0",
|
||||||
"pg": "^8.16.3",
|
"pg": "^8.16.3",
|
||||||
"postgres": "^3.4.7",
|
"postgres": "^3.4.7",
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue