diff --git a/TESTS b/TESTS deleted file mode 100644 index ff9d340..0000000 --- a/TESTS +++ /dev/null @@ -1,13 +0,0 @@ -# Via AWS SES CLI -aws ses send-email \ - --from "sender@example.com" \ - --destination "ToAddresses=test@andreasknuth.de" \ - --message "Subject={Data='Test',Charset=utf-8},Body={Text={Data='Test message',Charset=utf-8}}" \ - --region us-east-2 - -# Mail an mehrere Domains -aws ses send-email \ - --from "sender@example.com" \ - --destination "ToAddresses=test@andreasknuth.de,test@bizmatch.net" \ - --message "Subject={Data='Multi-Domain Test',Charset=utf-8},Body={Text={Data='Testing multiple domains',Charset=utf-8}}" \ - --region us-east-2 \ No newline at end of file diff --git a/Dockerfile b/deprecated_worker/Dockerfile similarity index 100% rename from Dockerfile rename to deprecated_worker/Dockerfile diff --git a/docker-compose.yml b/deprecated_worker/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to deprecated_worker/docker-compose.yml diff --git a/manage-worker.sh b/deprecated_worker/manage-worker.sh similarity index 100% rename from manage-worker.sh rename to deprecated_worker/manage-worker.sh diff --git a/update-all-workers.sh b/deprecated_worker/update-all-workers.sh similarity index 100% rename from update-all-workers.sh rename to deprecated_worker/update-all-workers.sh diff --git a/worker.py b/deprecated_worker/worker.py similarity index 100% rename from worker.py rename to deprecated_worker/worker.py diff --git a/worker.py.old b/deprecated_worker/worker.py.old similarity index 100% rename from worker.py.old rename to deprecated_worker/worker.py.old diff --git a/extract_email_headers.py b/extract_email_headers.py deleted file mode 100644 index 13c5475..0000000 --- a/extract_email_headers.py +++ /dev/null @@ -1,24 +0,0 @@ -import sys -import email -from email.utils import getaddresses - -if len(sys.argv) < 2: - print("Usage: python3 extract_email_headers.py ") - sys.exit(1) - -file_path = sys.argv[1] -with open(file_path, 'rb') as f: - msg = email.message_from_bytes(f.read()) - -# From: Nur die reine Email-Adresse -from_header = msg.get('From', '') -from_name, from_addr = email.utils.parseaddr(from_header) - -# Recipients: Alle To und Cc, nur reine Emails, kommagetrennt -to_addrs = getaddresses(msg.get_all('To', [])) -cc_addrs = getaddresses(msg.get_all('Cc', [])) -all_addrs = to_addrs + cc_addrs -recipients = ','.join([addr for name, addr in all_addrs if addr]) if all_addrs else '' - -print(f'FROM:{from_addr}') -print(f'RECIPIENTS:{recipients}') \ No newline at end of file diff --git a/lambda_function_outbound.py b/lambda/lambda_function_bounce.py similarity index 100% rename from lambda_function_outbound.py rename to lambda/lambda_function_bounce.py diff --git a/basic_setup/ses_sns_shim_global.py b/lambda/ses_sns_shim_global.py similarity index 100% rename from basic_setup/ses_sns_shim_global.py rename to lambda/ses_sns_shim_global.py diff --git a/lambda_function.py b/lambda_function.py deleted file mode 100644 index 3eb945f..0000000 --- a/lambda_function.py +++ /dev/null @@ -1,123 +0,0 @@ -import json -import os -import boto3 -import uuid -import logging -from datetime import datetime -from botocore.exceptions import ClientError -import time -import random - -# Logging konfigurieren -logger = logging.getLogger() -logger.setLevel(logging.INFO) - -sqs = boto3.client('sqs') - -# Retry-Konfiguration -MAX_RETRIES = 3 -BASE_BACKOFF = 1 # Sekunden - -def exponential_backoff(attempt): - """Exponential Backoff mit Jitter""" - return BASE_BACKOFF * (2 ** attempt) + random.uniform(0, 1) - -def get_queue_url(domain): - """ - Generiert Queue-Namen aus Domain und holt URL. - Konvention: domain.tld -> domain-tld-queue - """ - queue_name = domain.replace('.', '-') + '-queue' - try: - response = sqs.get_queue_url(QueueName=queue_name) - return response['QueueUrl'] - except ClientError as e: - if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue': - logger.error(f"Queue nicht gefunden für Domain: {domain}") - raise ValueError(f"Keine Queue für Domain {domain}") - else: - raise - -def lambda_handler(event, context): - """ - Nimmt SES Event entgegen, extrahiert Domain dynamisch, - verpackt Metadaten als 'Fake SNS' und sendet an die domain-spezifische SQS. - Mit integrierter Retry-Logik für SQS-Send. - """ - try: - records = event.get('Records', []) - logger.info(f"Received event with {len(records)} records.") - - for record in records: - ses_data = record.get('ses', {}) - if not ses_data: - logger.warning(f"Invalid SES event: Missing 'ses' in record: {record}") - continue - - mail = ses_data.get('mail', {}) - receipt = ses_data.get('receipt', {}) - - # Domain extrahieren (aus erstem Recipient) - recipients = receipt.get('recipients', []) or mail.get('destination', []) - if not recipients: - logger.warning("No recipients in event - skipping") - continue - - first_recipient = recipients[0] - domain = first_recipient.split('@')[-1].lower() - if not domain: - logger.error("Could not extract domain from recipient") - continue - - # Wichtige Metadaten loggen - msg_id = mail.get('messageId', 'unknown') - source = mail.get('source', 'unknown') - logger.info(f"Processing Message-ID: {msg_id} for domain: {domain}") - logger.info(f" From: {source}") - logger.info(f" To: {recipients}") - - # SES JSON als String serialisieren - ses_json_string = json.dumps(ses_data) - - # Payload Größe loggen und checken (Safeguard) - payload_size = len(ses_json_string.encode('utf-8')) - logger.info(f" Metadata Payload Size: {payload_size} bytes") - if payload_size > 200000: # Arbitrary Limit < SQS 256KB - raise ValueError("Payload too large for SQS") - - # Fake SNS Payload - fake_sns_payload = { - "Type": "Notification", - "MessageId": str(uuid.uuid4()), - "TopicArn": "arn:aws:sns:ses-shim:global-topic", - "Subject": "Amazon SES Email Receipt Notification", - "Message": ses_json_string, - "Timestamp": datetime.utcnow().isoformat() + "Z" - } - - # Queue URL dynamisch holen - queue_url = get_queue_url(domain) - - # SQS Send mit Retries - attempt = 0 - while attempt < MAX_RETRIES: - try: - sqs.send_message( - QueueUrl=queue_url, - MessageBody=json.dumps(fake_sns_payload) - ) - logger.info(f"✅ Successfully forwarded {msg_id} to SQS: {queue_url}") - break - except ClientError as e: - attempt += 1 - error_code = e.response['Error']['Code'] - logger.warning(f"Retry {attempt}/{MAX_RETRIES} for SQS send: {error_code} - {str(e)}") - if attempt == MAX_RETRIES: - raise - time.sleep(exponential_backoff(attempt)) - - return {'status': 'ok'} - - except Exception as e: - logger.error(f"❌ Critical Error in Lambda Shim: {str(e)}", exc_info=True) - raise e \ No newline at end of file diff --git a/check-dlq.py b/queue_helper/check-dlq.py similarity index 100% rename from check-dlq.py rename to queue_helper/check-dlq.py diff --git a/get-queue-urls.sh b/queue_helper/get-queue-urls.sh similarity index 100% rename from get-queue-urls.sh rename to queue_helper/get-queue-urls.sh diff --git a/monitor-queues.py b/queue_helper/monitor-queues.py similarity index 100% rename from monitor-queues.py rename to queue_helper/monitor-queues.py diff --git a/requeue-dlq.py b/queue_helper/requeue-dlq.py similarity index 100% rename from requeue-dlq.py rename to queue_helper/requeue-dlq.py diff --git a/requeue-email.sh b/queue_helper/requeue-email.sh similarity index 100% rename from requeue-email.sh rename to queue_helper/requeue-email.sh diff --git a/test_extract_v2.py b/test_extract_v2.py deleted file mode 100755 index 4cfb49f..0000000 --- a/test_extract_v2.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script für Message-ID Extraktion - VERBESSERTE VERSION -Kann lokal ausgeführt werden ohne AWS-Verbindung -""" - -import re -from email.parser import BytesParser -from email.policy import SMTP as SMTPPolicy - -def log(message: str, level: str = 'INFO'): - """Dummy log für Tests""" - print(f"[{level}] {message}") - -def extract_original_message_id(parsed): - """ - Extrahiert Original SES Message-ID aus Email - SES Format: 010f[hex32]-[hex8]-[hex4]-[hex4]-[hex4]-[hex12]-000000 - """ - import re - - # SES Message-ID Pattern (endet immer mit -000000) - ses_pattern = re.compile(r'010f[0-9a-f]{12}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-000000') - - # Die Message-ID der aktuellen Email (Bounce selbst) - diese wollen wir NICHT - current_msg_id = (parsed.get('Message-ID') or '').strip() - current_match = ses_pattern.search(current_msg_id) - current_id = current_match.group(0) if current_match else None - - log(f"Current Message-ID: {current_id}", 'DEBUG') - - # 1. Versuche Standard-Header (In-Reply-To, References) - for header in ['In-Reply-To', 'References']: - value = (parsed.get(header) or '').strip() - if value: - match = ses_pattern.search(value) - if match: - found_id = match.group(0) - # Nur nehmen wenn es NICHT die aktuelle Bounce-ID ist - if found_id != current_id: - log(f" Found Message-ID in {header}: {found_id}") - return found_id - - # 2. Durchsuche den kompletten Email-Body (inkl. ALLE Attachments/Parts) - try: - body_text = '' - - # Hole den kompletten Body als String - if parsed.is_multipart(): - for part in parsed.walk(): - content_type = part.get_content_type() - - # SPEZIALFALL: message/rfc822 (eingebettete Messages) - if content_type == 'message/rfc822': - log(f" Processing embedded message/rfc822", 'DEBUG') - try: - # get_payload() gibt eine Liste mit einem EmailMessage-Objekt zurück! - payload = part.get_payload() - if isinstance(payload, list) and len(payload) > 0: - embedded_msg = payload[0] - # Hole Message-ID aus dem eingebetteten Message - embedded_id = (embedded_msg.get('Message-ID') or '').strip() - match = ses_pattern.search(embedded_id) - if match: - found_id = match.group(0) - log(f" Found ID in embedded msg: {found_id}", 'DEBUG') - # Nur nehmen wenn es NICHT die aktuelle Bounce-ID ist - if found_id != current_id: - log(f" ✓ Found Message-ID in embedded message: {found_id}") - return found_id - # Fallback: Konvertiere eingebettete Message zu String - body_text += embedded_msg.as_string() - except Exception as e: - log(f" Warning: Could not process embedded message: {e}", 'WARNING') - - # Durchsuche ALLE anderen Parts (außer Binärdaten wie images) - elif content_type.startswith('text/') or content_type.startswith('application/'): - try: - payload = part.get_payload(decode=True) - if payload: - # Versuche als UTF-8, fallback auf Latin-1 - try: - body_text += payload.decode('utf-8', errors='ignore') - except: - try: - body_text += payload.decode('latin-1', errors='ignore') - except: - # Letzter Versuch: als ASCII mit ignore - body_text += str(payload, errors='ignore') - except: - # Falls decode fehlschlägt, String-Payload holen - payload = part.get_payload() - if isinstance(payload, str): - body_text += payload - else: - # Nicht-Multipart Message - payload = parsed.get_payload(decode=True) - if payload: - try: - body_text = payload.decode('utf-8', errors='ignore') - except: - body_text = payload.decode('latin-1', errors='ignore') - - # Suche alle SES Message-IDs im Body - matches = ses_pattern.findall(body_text) - if matches: - log(f" Found {len(matches)} total IDs in body: {matches}", 'DEBUG') - # Filtere die aktuelle Bounce-ID raus - candidates = [m for m in matches if m != current_id] - - if candidates: - # Nehme die ERSTE der verbleibenden (meist die Original-ID) - log(f" Found {len(matches)} SES Message-ID(s) in body, using first (not bounce): {candidates[0]}") - return candidates[0] - else: - log(f" Found {len(matches)} SES Message-ID(s) but all match the bounce ID") - - except Exception as e: - log(f" Warning: Could not search body for Message-ID: {e}", 'WARNING') - - return None - - -def test_with_file(filepath: str): - """Test mit einer echten Email-Datei""" - print(f"\n{'='*70}") - print(f"Testing: {filepath}") - print('='*70) - - with open(filepath, 'rb') as f: - raw_bytes = f.read() - - parsed = BytesParser(policy=SMTPPolicy).parsebytes(raw_bytes) - - print(f"\nEmail Headers:") - print(f" From: {parsed.get('From')}") - print(f" To: {parsed.get('To')}") - print(f" Subject: {parsed.get('Subject')}") - print(f" Message-ID: {parsed.get('Message-ID')}") - print(f" In-Reply-To: {parsed.get('In-Reply-To')}") - print(f" References: {parsed.get('References')}") - - print(f"\n--- EXTRACTION ---") - result = extract_original_message_id(parsed) - - print(f"\n{'='*70}") - print(f"RESULT: {result}") - print('='*70) - - return result - - -if __name__ == '__main__': - import sys - - if len(sys.argv) > 1: - # Email-Datei als Argument - result = test_with_file(sys.argv[1]) - - # Exit code: 0 = success (ID found), 1 = failure (no ID) - sys.exit(0 if result else 1) - else: - print("Usage: python3 test_extract_v2.py ") - sys.exit(1) \ No newline at end of file