202 lines
8.1 KiB
Python
202 lines
8.1 KiB
Python
# Optimiertes Lambda mit Verbesserungen
|
|
import json
|
|
import os
|
|
import urllib.request
|
|
import urllib.error
|
|
import urllib.parse
|
|
import logging
|
|
import ssl
|
|
import boto3
|
|
import base64
|
|
import hashlib
|
|
|
|
# Konfiguration
|
|
API_BASE_URL = os.environ['API_BASE_URL']
|
|
API_TOKEN = os.environ['API_TOKEN']
|
|
DEBUG_MODE = os.environ.get('DEBUG_MODE', 'false').lower() == 'true'
|
|
MAX_EMAIL_SIZE = int(os.environ.get('MAX_EMAIL_SIZE', '10485760')) # 10MB Standard
|
|
|
|
# Bucket zu Domain Mapping
|
|
BUCKET_DOMAIN_MAP = {
|
|
'bizmatch-ses-mails': 'bizmatch.net',
|
|
'haiky-app-emails-339712845857': 'haiky.app',
|
|
'andreasknuth-de-emails': 'andreasknuth.de',
|
|
'wetter-playadelingles-de-emails': 'wetter-playadelingles.de',
|
|
}
|
|
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.INFO)
|
|
|
|
def lambda_handler(event, context):
|
|
"""Optimierter Lambda-Handler mit verbesserter Fehlerbehandlung"""
|
|
|
|
# Eindeutige Request-ID für Tracking
|
|
request_id = context.aws_request_id
|
|
logger.info(f"[{request_id}] S3-Ereignis empfangen")
|
|
|
|
try:
|
|
# S3-Event-Details extrahieren
|
|
s3_event = event['Records'][0]['s3']
|
|
bucket_name = s3_event['bucket']['name']
|
|
object_key = urllib.parse.unquote_plus(s3_event['object']['key'])
|
|
|
|
logger.info(f"[{request_id}] Verarbeite: {bucket_name}/{object_key}")
|
|
|
|
# Domain ermitteln
|
|
domain = BUCKET_DOMAIN_MAP.get(bucket_name)
|
|
if not domain:
|
|
logger.warning(f"[{request_id}] Keine Domain-Zuordnung für Bucket {bucket_name}")
|
|
return {'statusCode': 400, 'body': json.dumps('Unbekannter Bucket')}
|
|
|
|
# Duplikat-Check mit S3-ETag
|
|
s3_client = boto3.client('s3')
|
|
head_response = s3_client.head_object(Bucket=bucket_name, Key=object_key)
|
|
etag = head_response['ETag'].strip('"')
|
|
content_length = head_response['ContentLength']
|
|
|
|
# E-Mail-Größe prüfen
|
|
if content_length > MAX_EMAIL_SIZE:
|
|
logger.warning(f"[{request_id}] E-Mail zu groß: {content_length} Bytes (max: {MAX_EMAIL_SIZE})")
|
|
# Große E-Mails markieren statt löschen
|
|
mark_large_email(s3_client, bucket_name, object_key, content_length)
|
|
return {'statusCode': 413, 'body': json.dumps('E-Mail zu groß')}
|
|
|
|
# E-Mail laden
|
|
try:
|
|
response = s3_client.get_object(Bucket=bucket_name, Key=object_key)
|
|
email_content = response['Body'].read()
|
|
logger.info(f"[{request_id}] E-Mail geladen: {len(email_content)} Bytes")
|
|
|
|
except Exception as e:
|
|
logger.error(f"[{request_id}] Fehler beim Laden der E-Mail: {str(e)}")
|
|
return {'statusCode': 500, 'body': json.dumps(f'S3-Fehler: {str(e)}')}
|
|
|
|
# E-Mail-Content komprimieren für Übertragung (optional)
|
|
import gzip
|
|
compressed_content = gzip.compress(email_content)
|
|
email_base64 = base64.b64encode(compressed_content).decode('utf-8')
|
|
|
|
# Payload erstellen
|
|
payload = {
|
|
'bucket': bucket_name,
|
|
'key': object_key,
|
|
'domain': domain,
|
|
'email_content': email_base64,
|
|
'compressed': True, # Flag für API
|
|
'etag': etag,
|
|
'request_id': request_id,
|
|
'original_size': len(email_content),
|
|
'compressed_size': len(compressed_content)
|
|
}
|
|
|
|
# API aufrufen mit Retry-Logik
|
|
success = call_api_with_retry(payload, domain, request_id)
|
|
|
|
if success:
|
|
# Bei Erfolg: E-Mail aus S3 löschen
|
|
try:
|
|
s3_client.delete_object(Bucket=bucket_name, Key=object_key)
|
|
logger.info(f"[{request_id}] E-Mail erfolgreich verarbeitet und gelöscht")
|
|
return {
|
|
'statusCode': 200,
|
|
'body': json.dumps({
|
|
'message': 'E-Mail erfolgreich verarbeitet',
|
|
'request_id': request_id
|
|
})
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"[{request_id}] Konnte E-Mail nicht löschen: {str(e)}")
|
|
# Trotzdem Erfolg melden, da E-Mail verarbeitet wurde
|
|
return {'statusCode': 200, 'body': json.dumps('Verarbeitet, aber nicht gelöscht')}
|
|
else:
|
|
# Bei Fehler: E-Mail in S3 belassen für späteren Retry
|
|
return {'statusCode': 500, 'body': json.dumps('API-Fehler')}
|
|
|
|
except Exception as e:
|
|
logger.error(f"[{request_id}] Unerwarteter Fehler: {str(e)}", exc_info=True)
|
|
return {'statusCode': 500, 'body': json.dumps(f'Lambda-Fehler: {str(e)}')}
|
|
|
|
def call_api_with_retry(payload, domain, request_id, max_retries=3):
|
|
"""API-Aufruf mit Retry-Logik"""
|
|
|
|
sync_url = f"{API_BASE_URL}/process/{domain}"
|
|
payload_json = json.dumps(payload).encode('utf-8')
|
|
|
|
for attempt in range(max_retries):
|
|
logger.info(f"[{request_id}] API-Aufruf Versuch {attempt + 1}/{max_retries}")
|
|
|
|
try:
|
|
# Request erstellen
|
|
req = urllib.request.Request(sync_url, data=payload_json, method="POST")
|
|
req.add_header('Authorization', f'Bearer {API_TOKEN}')
|
|
req.add_header('Content-Type', 'application/json')
|
|
req.add_header('User-Agent', f'AWS-Lambda-EmailProcessor/2.0-{request_id}')
|
|
req.add_header('X-Request-ID', request_id)
|
|
|
|
# SSL-Kontext
|
|
ssl_context = ssl._create_unverified_context() if DEBUG_MODE else None
|
|
|
|
# API-Request mit Timeout
|
|
timeout = 25 # Lambda hat 30s Timeout, also etwas weniger
|
|
|
|
if DEBUG_MODE and ssl_context:
|
|
with urllib.request.urlopen(req, context=ssl_context, timeout=timeout) as response:
|
|
response_body = response.read().decode('utf-8')
|
|
response_code = response.getcode()
|
|
else:
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
response_body = response.read().decode('utf-8')
|
|
response_code = response.getcode()
|
|
|
|
# Erfolgreiche Antwort
|
|
if response_code == 200:
|
|
logger.info(f"[{request_id}] API-Erfolg nach Versuch {attempt + 1}")
|
|
return True
|
|
else:
|
|
logger.warning(f"[{request_id}] API-Antwort {response_code}: {response_body}")
|
|
|
|
except urllib.error.HTTPError as e:
|
|
error_body = ""
|
|
try:
|
|
error_body = e.read().decode('utf-8')
|
|
except:
|
|
pass
|
|
|
|
logger.error(f"[{request_id}] HTTP-Fehler {e.code} (Versuch {attempt + 1}): {e.reason}")
|
|
logger.error(f"[{request_id}] Fehlerdetails: {error_body}")
|
|
|
|
# Bei 4xx-Fehlern nicht retry (Client-Fehler)
|
|
if 400 <= e.code < 500:
|
|
logger.error(f"[{request_id}] Client-Fehler - kein Retry")
|
|
return False
|
|
|
|
except Exception as e:
|
|
logger.error(f"[{request_id}] Netzwerk-Fehler (Versuch {attempt + 1}): {str(e)}")
|
|
|
|
# Warten vor nächstem Versuch (exponential backoff)
|
|
if attempt < max_retries - 1:
|
|
wait_time = 2 ** attempt # 1s, 2s, 4s
|
|
logger.info(f"[{request_id}] Warte {wait_time}s vor nächstem Versuch")
|
|
import time
|
|
time.sleep(wait_time)
|
|
|
|
logger.error(f"[{request_id}] API-Aufruf nach {max_retries} Versuchen fehlgeschlagen")
|
|
return False
|
|
|
|
def mark_large_email(s3_client, bucket, key, size):
|
|
"""Markiert große E-Mails mit Metadaten statt sie zu löschen"""
|
|
try:
|
|
s3_client.copy_object(
|
|
Bucket=bucket,
|
|
Key=key,
|
|
CopySource={'Bucket': bucket, 'Key': key},
|
|
Metadata={
|
|
'status': 'too_large',
|
|
'size': str(size),
|
|
'marked_at': str(int(time.time()))
|
|
},
|
|
MetadataDirective='REPLACE'
|
|
)
|
|
logger.info(f"E-Mail als zu groß markiert: {key} ({size} Bytes)")
|
|
except Exception as e:
|
|
logger.error(f"Konnte E-Mail nicht markieren: {str(e)}") |