update

basic_setup/awsses.sh

@@ -1,218 +1,187 @@
 #!/bin/bash
-# awsses.sh - Konfiguriert Amazon SES mit SNS->SQS Fanout Architektur & Outbound Tracking
+# awsses_lambda_global.sh - SES Setup mit S3 + Global Lambda Shim -> SQS
-#
+# Dieses Skript ist idempotent: Es kann sicher mehrfach ausgeführt werden.
-# Ablauf:
+# Globale Lambda für alle Domains.
-# 1. SES Domain Identity erstellen/verifizieren
-# 2. Domain mit Configuration Set verknüpfen (für Outbound Tracking)
-# 3. SNS Topic erstellen
-# 4. SNS Topic Policy setzen (damit SES hineinschreiben darf)
-# 5. SQS Queue verbinden (Subscription)
-# 6. SQS Queue Policy setzen (damit SNS hineinschreiben darf)
-# 7. SES Receipt Rule erstellen (S3 Action + SNS Action)
 
 set -e
 
-# Überprüfen, ob jq installiert ist
+# --- CHECKS ---
-if ! command -v jq &> /dev/null; then
+if ! command -v jq &> /dev/null; then echo "Fehler: 'jq' fehlt."; exit 1; fi
-    echo "Fehler: 'jq' ist nicht installiert. Bitte installieren (sudo apt-get install jq)."
+if [ -z "$DOMAIN_NAME" ]; then echo "Fehler: DOMAIN_NAME ist nicht gesetzt."; exit 1; fi
+
+# Prüfen ob Python Code da ist
+PYTHON_FILE="ses_sns_shim_global.py"
+if [ ! -f "$PYTHON_FILE" ]; then
+    echo "Fehler: $PYTHON_FILE nicht gefunden!"
     exit 1
 fi
 
-# Überprüfen, ob die Domain-Variable gesetzt ist
+# --- VARIABLEN ---
-if [ -z "$DOMAIN_NAME" ]; then
-  echo "Fehler: DOMAIN_NAME ist nicht gesetzt."
-  echo "Bitte setzen Sie die Variable mit: export DOMAIN_NAME='IhreDomain.de'"
-  exit 1
-fi
-
-# Überprüfen, ob S3_BUCKET_NAME gesetzt ist
-if [ -z "$S3_BUCKET_NAME" ]; then
-  echo "Warnung: S3_BUCKET_NAME ist nicht gesetzt."
-  S3_BUCKET_NAME=$(echo "$DOMAIN_NAME" | tr '.' '-' | awk '{print $0 "-emails"}')
-  echo "Generierter Bucket-Name: $S3_BUCKET_NAME"
-fi
-
-# Konfiguration
 AWS_REGION=${AWS_REGION:-"us-east-2"}
 EMAIL_PREFIX=${EMAIL_PREFIX:-""}
-CONFIGURATION_SET_NAME="relay-outbound" # Name deines globalen Config Sets
+CONFIGURATION_SET_NAME="relay-outbound"
 
-# Naming Conventions
+# Bucket Name generieren falls leer
+if [ -z "$S3_BUCKET_NAME" ]; then
+  S3_BUCKET_NAME=$(echo "$DOMAIN_NAME" | tr '.' '-' | awk '{print $0 "-emails"}')
+fi
+
+# Namen (Global Lambda!)
 RULE_SET_NAME="bizmatch-ruleset"
 RULE_NAME="store-${DOMAIN_NAME//./-}-to-s3"
-TOPIC_NAME="${DOMAIN_NAME//./-}-topic"
 QUEUE_NAME="${DOMAIN_NAME//./-}-queue"
+LAMBDA_NAME="ses-shim-global"
+LAMBDA_ROLE_NAME="SesShimGlobalRole"
 
-echo "========================================================"
+echo "=========================================================="
-echo "   SES Setup (Full Architecture) für $DOMAIN_NAME"
+echo " SES Setup (S3 -> Global Lambda Shim -> SQS) für $DOMAIN_NAME"
-echo "========================================================"
+echo "=========================================================="
-echo "Region: $AWS_REGION"
-echo "S3 Bucket: $S3_BUCKET_NAME"
-echo "Config Set: $CONFIGURATION_SET_NAME"
-echo "--------------------------------------------------------"
-
-# ------------------------
-# 1. SES Domain Identität
-# ------------------------
-echo "[1/7] Prüfe SES Domain Identität..."
 
+# ---------------------------------------------------------
+# 1. SES Identity & Config Set
+# ---------------------------------------------------------
+echo "[1/6] SES Identity Setup..."
 if ! aws sesv2 get-email-identity --email-identity ${DOMAIN_NAME} --region ${AWS_REGION} >/dev/null 2>&1; then
-  echo "-> Erstelle Identity..."
   aws sesv2 create-email-identity --email-identity ${DOMAIN_NAME} --region ${AWS_REGION} >/dev/null
-else
-  echo "-> Identity existiert bereits."
 fi
-
+# Update Attributes (Idempotent)
-# Config Updates (Idempotent)
-echo "-> Konfiguriere DKIM & Mail-From..."
 aws sesv2 put-email-identity-dkim-attributes --email-identity ${DOMAIN_NAME} --signing-enabled --region ${AWS_REGION}
 aws sesv2 put-email-identity-mail-from-attributes --email-identity ${DOMAIN_NAME} --mail-from-domain "mail.${DOMAIN_NAME}" --behavior-on-mx-failure USE_DEFAULT_VALUE --region ${AWS_REGION}
+aws sesv2 put-email-identity-configuration-set-attributes --email-identity ${DOMAIN_NAME} --configuration-set-name "$CONFIGURATION_SET_NAME" --region ${AWS_REGION}
 
-# ------------------------
+# ---------------------------------------------------------
-# 2. Configuration Set Verknüpfung (NEU!)
+# 2. SQS Queue holen (nur zur Validierung, Lambda holt dynamisch)
-# ------------------------
+# ---------------------------------------------------------
-echo "[2/7] Verknüpfe Domain mit Outbound Configuration Set..."
+echo "[2/6] Queue URL ermitteln (zur Validierung)..."
-# Dies sorgt dafür, dass ausgehende Mails getrackt werden (für OOO/Bounces)
-aws sesv2 put-email-identity-configuration-set-attributes \
-  --email-identity ${DOMAIN_NAME} \
-  --configuration-set-name "$CONFIGURATION_SET_NAME" \
-  --region ${AWS_REGION}
-
-# ------------------------
-# 3. SNS Topic erstellen
-# ------------------------
-echo "[3/7] Erstelle/Prüfe SNS Topic..."
-TOPIC_ARN=$(aws sns create-topic --name "$TOPIC_NAME" --region "$AWS_REGION" --output text --query 'TopicArn')
-echo "-> Topic ARN: $TOPIC_ARN"
-
-# ------------------------
-# 4. SNS Policy (SES -> SNS)
-# ------------------------
-echo "[4/7] Setze SNS Policy (SES darf publishen)..."
-ACCOUNT_ID=$(echo "$TOPIC_ARN" | cut -d: -f5)
-
-SNS_POLICY=$(jq -n \
-  --arg topic_arn "$TOPIC_ARN" \
-  --arg account_id "$ACCOUNT_ID" \
-  '{
-    Version: "2008-10-17",
-    Id: "__default_policy_ID",
-    Statement: [
-      {
-        Sid: "Allow-SES-Publish",
-        Effect: "Allow",
-        Principal: { Service: "ses.amazonaws.com" },
-        Action: "sns:Publish",
-        Resource: $topic_arn,
-        Condition: {
-          StringEquals: { "AWS:SourceAccount": $account_id }
-        }
-      }
-    ]
-  }' | jq -c .)
-
-aws sns set-topic-attributes --topic-arn "$TOPIC_ARN" --attribute-name Policy --attribute-value "$SNS_POLICY" --region "$AWS_REGION"
-
-# ------------------------
-# 5. SQS Queue Verbindung
-# ------------------------
-echo "[5/7] Verbinde SQS Queue..."
-
-# Queue URL & ARN holen (Queue muss existieren -> create-queue.sh vorher ausführen!)
 QUEUE_URL=$(aws sqs get-queue-url --queue-name "$QUEUE_NAME" --region "$AWS_REGION" --output text --query 'QueueUrl' 2>/dev/null)
-
+if [ -z "$QUEUE_URL" ]; then echo "FEHLER: Queue $QUEUE_NAME nicht gefunden! ./create-queue.sh zuerst ausführen."; exit 1; fi
-if [ -z "$QUEUE_URL" ]; then
-    echo "FEHLER: Queue $QUEUE_NAME nicht gefunden!"
-    echo "Bitte führen Sie erst ./create-queue.sh aus."
-    exit 1
-fi
-
 QUEUE_ARN=$(aws sqs get-queue-attributes --queue-url "$QUEUE_URL" --attribute-names QueueArn --region "$AWS_REGION" --output text --query 'Attributes.QueueArn')
 
-# Subscription erstellen (Idempotent)
+# ---------------------------------------------------------
-aws sns subscribe --topic-arn "$TOPIC_ARN" --protocol sqs --notification-endpoint "$QUEUE_ARN" --region "$AWS_REGION" > /dev/null
+# 3. IAM Role für Global Lambda erstellen
-
+# ---------------------------------------------------------
-# ------------------------
+echo "[3/6] IAM Role für Lambda prüfen/erstellen..."
-# 6. SQS Policy (SNS -> SQS)
+TRUST_POLICY='{"Version": "2012-10-17","Statement": [{"Effect": "Allow","Principal": {"Service": "lambda.amazonaws.com"},"Action": "sts:AssumeRole"}]}'
-# ------------------------
+if ! aws iam get-role --role-name "$LAMBDA_ROLE_NAME" >/dev/null 2>&1; then
-echo "[6/7] Setze SQS Policy (SNS darf schreiben)..."
+  aws iam create-role --role-name "$LAMBDA_ROLE_NAME" --assume-role-policy-document "$TRUST_POLICY" >/dev/null
-
+  echo " -> Rolle erstellt."
-SQS_POLICY=$(jq -n \
+else
-  --arg queue_arn "$QUEUE_ARN" \
+  echo " -> Rolle existiert bereits."
-  --arg topic_arn "$TOPIC_ARN" \
-  '{
-    Version: "2012-10-17",
-    Id: "SNS-to-SQS",
-    Statement: [
-      {
-        Sid: "Allow-SNS-SendMessage",
-        Effect: "Allow",
-        Principal: { Service: "sns.amazonaws.com" },
-        Action: "sqs:SendMessage",
-        Resource: $queue_arn,
-        Condition: {
-          ArnEquals: { "aws:SourceArn": $topic_arn }
-        }
-      }
-    ]
-  }' | jq -c .)
-
-# Policy setzen (mit Single-Quote Schutz für AWS CLI)
-aws sqs set-queue-attributes --queue-url "$QUEUE_URL" --attributes Policy="'$SQS_POLICY'" --region "$AWS_REGION"
-
-# ------------------------
-# 7. SES Rule Set & Rule
-# ------------------------
-echo "[7/7] Konfiguriere SES Receipt Rule..."
-
-# Rule Set prüfen
-RULESET_EXISTS=$(aws ses list-receipt-rule-sets --region ${AWS_REGION} | jq -r '.RuleSets[] | select(.Name == "bizmatch-ruleset") | .Name')
-if [ "$RULESET_EXISTS" != "bizmatch-ruleset" ]; then
-  echo "-> Erstelle Rule Set 'bizmatch-ruleset'..."
-  aws ses create-receipt-rule-set --rule-set-name "bizmatch-ruleset" --region ${AWS_REGION}
 fi
 
-# Rule prüfen/erstellen
+# Permissions Policy (Lambda darf Logs schreiben und in ALLE Queues mit *-queue senden)
-if ! aws ses describe-receipt-rule --rule-set-name "$RULE_SET_NAME" --rule-name "${RULE_NAME}" --region ${AWS_REGION} >/dev/null 2>&1; then
+LAMBDA_POLICY=$(jq -n '{
-  
+    Version: "2012-10-17",
-  echo "-> Erstelle Receipt Rule '${RULE_NAME}'..."
+    Statement: [
-  
+        {
-  # Rule mit S3 Action UND SNS Action
+            Effect: "Allow",
-  # HINWEIS: Hier fügen wir initial die Hauptdomain als Recipient hinzu.
+            Action: ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"],
-  # Denke daran, später ./manage_mail_user.sh sync ... auszuführen!
+            Resource: "arn:aws:logs:*:*:*"
-  aws ses create-receipt-rule --rule-set-name "$RULE_SET_NAME" --rule '{
+        },
-    "Name": "'"${RULE_NAME}"'",
+        {
-    "Enabled": true,
+            Effect: "Allow",
-    "ScanEnabled": true,
+            Action: "sqs:SendMessage",
-    "Actions": [
+            Resource: "arn:aws:sqs:*:*:*-queue"
+        },
+        {
+            Effect: "Allow",
+            Action: "sqs:GetQueueUrl",
+            Resource: "*"
+        }
+    ]
+}' | jq -c .)
+aws iam put-role-policy --role-name "$LAMBDA_ROLE_NAME" --policy-name "SesShimGlobalPermissions" --policy-document "$LAMBDA_POLICY"
+echo " -> Permissions aktualisiert."
+
+# Kurze Pause für IAM Propagation, falls Rolle neu war
+sleep 5
+
+# ---------------------------------------------------------
+# 4. Lambda Funktion erstellen/updaten (Global!)
+# ---------------------------------------------------------
+echo "[4/6] Global Lambda Shim deployen..."
+# Zip erstellen
+cp "$PYTHON_FILE" lambda_function.py
+zip -q lambda.zip lambda_function.py
+# Keine Env-Vars nötig, da dynamisch
+ROLE_ARN=$(aws iam get-role --role-name "$LAMBDA_ROLE_NAME" --query 'Role.Arn' --output text)
+if ! aws lambda get-function --function-name "$LAMBDA_NAME" --region "$AWS_REGION" >/dev/null 2>&1; then
+  echo " -> Erstelle neue Lambda-Funktion..."
+  aws lambda create-function --function-name "$LAMBDA_NAME" \
+    --runtime python3.11 --handler lambda_function.lambda_handler \
+    --role "$ROLE_ARN" --zip-file fileb://lambda.zip \
+    --region "$AWS_REGION" >/dev/null
+else
+  echo " -> Aktualisiere existierende Lambda-Funktion..."
+  aws lambda update-function-code --function-name "$LAMBDA_NAME" --zip-file fileb://lambda.zip --region "$AWS_REGION" >/dev/null
+
+  # Warte kurz
+  sleep 2
+
+  aws lambda update-function-configuration --function-name "$LAMBDA_NAME" --region "$AWS_REGION" >/dev/null
+fi
+# Aufräumen
+rm lambda.zip lambda_function.py
+
+# ---------------------------------------------------------
+# 5. Permission: SES darf Lambda aufrufen (Global, einmalig)
+# ---------------------------------------------------------
+echo "[5/6] SES Permission für Lambda..."
+aws lambda add-permission --function-name "$LAMBDA_NAME" \
+    --statement-id "AllowSESInvoke-Global" \
+    --action "lambda:InvokeFunction" \
+    --principal "ses.amazonaws.com" \
+    --region "$AWS_REGION" 2>/dev/null || true
+
+# ---------------------------------------------------------
+# 6. SES Rule (S3 + Global Lambda)
+# ---------------------------------------------------------
+echo "[6/6] SES Receipt Rule (S3 + Lambda) konfigurieren..."
+LAMBDA_ARN=$(aws lambda get-function --function-name "$LAMBDA_NAME" --region "$AWS_REGION" --query 'Configuration.FunctionArn' --output text)
+# Rule Set prüfen
+if ! aws ses list-receipt-rule-sets --region ${AWS_REGION} | grep -q "bizmatch-ruleset"; then
+  aws ses create-receipt-rule-set --rule-set-name "bizmatch-ruleset" --region ${AWS_REGION}
+fi
+# Regel-Definition
+RULE_JSON=$(jq -n \
+  --arg bucket "$S3_BUCKET_NAME" \
+  --arg prefix "$EMAIL_PREFIX" \
+  --arg larn "$LAMBDA_ARN" \
+  --arg rule "$RULE_NAME" \
+  --arg domain "$DOMAIN_NAME" \
+  --arg subdomain "mail.$DOMAIN_NAME" \
+  '{
+    Name: $rule,
+    Enabled: true,
+    ScanEnabled: true,
+    TlsPolicy: "Require",
+    Recipients: [$domain, $subdomain],
+    Actions: [
       {
-        "S3Action": {
+        S3Action: {
-          "BucketName": "'"${S3_BUCKET_NAME}"'",
+          BucketName: $bucket,
-          "ObjectKeyPrefix": "'"${EMAIL_PREFIX}"'"
+          ObjectKeyPrefix: $prefix
         }
       },
       {
-        "SNSAction": {
+        LambdaAction: {
-          "TopicArn": "'"${TOPIC_ARN}"'",
+          FunctionArn: $larn,
-          "Encoding": "UTF-8"
+          InvocationType: "Event"
         }
       }
-    ],
+    ]
-    "TlsPolicy": "Require",
+  }')
-    "Recipients": ["'"${DOMAIN_NAME}"'"]
+# Check ob Regel existiert -> Update, sonst Create
-  }' --region ${AWS_REGION}
+if aws ses describe-receipt-rule --rule-set-name "bizmatch-ruleset" --rule-name "$RULE_NAME" --region "$AWS_REGION" >/dev/null 2>&1; then
-  
+    echo " -> Aktualisiere existierende Regel..."
+    aws ses update-receipt-rule --rule-set-name "bizmatch-ruleset" --rule "$RULE_JSON" --region "$AWS_REGION"
 else
-  echo "-> Receipt Rule '${RULE_NAME}' existiert bereits (Überspringe Erstellung)."
+    echo " -> Erstelle neue Regel..."
+    aws ses create-receipt-rule --rule-set-name "bizmatch-ruleset" --rule "$RULE_JSON" --region "$AWS_REGION"
 fi
-
+# Aktivieren
-# Rule Set aktivieren
+aws ses set-active-receipt-rule-set --rule-set-name "bizmatch-ruleset" --region ${AWS_REGION}
-ACTIVE_RULESET=$(aws ses describe-active-receipt-rule-set --region ${AWS_REGION} | jq -r '.Metadata.Name')
-if [ "$ACTIVE_RULESET" != "bizmatch-ruleset" ]; then
-  echo "-> Aktiviere Rule Set..."
-  aws ses set-active-receipt-rule-set --rule-set-name "bizmatch-ruleset" --region ${AWS_REGION}
-fi
-
 echo "========================================================"
-echo "✅ Setup erfolgreich abgeschlossen für $DOMAIN_NAME"
+echo "✅ Setup erfolgreich. Globale Lambda ($LAMBDA_NAME) für alle Domains."
+echo " S3 -> Lambda -> Domain-spezifische SQS"
 echo "========================================================"

basic_setup/ses_sns_shim_global.py

@@ -0,0 +1,123 @@
+import json
+import os
+import boto3
+import uuid
+import logging
+from datetime import datetime
+from botocore.exceptions import ClientError
+import time
+import random
+
+# Logging konfigurieren
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+sqs = boto3.client('sqs')
+
+# Retry-Konfiguration
+MAX_RETRIES = 3
+BASE_BACKOFF = 1  # Sekunden
+
+def exponential_backoff(attempt):
+    """Exponential Backoff mit Jitter"""
+    return BASE_BACKOFF * (2 ** attempt) + random.uniform(0, 1)
+
+def get_queue_url(domain):
+    """
+    Generiert Queue-Namen aus Domain und holt URL.
+    Konvention: domain.tld -> domain-tld-queue
+    """
+    queue_name = domain.replace('.', '-') + '-queue'
+    try:
+        response = sqs.get_queue_url(QueueName=queue_name)
+        return response['QueueUrl']
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue':
+            logger.error(f"Queue nicht gefunden für Domain: {domain}")
+            raise ValueError(f"Keine Queue für Domain {domain}")
+        else:
+            raise
+
+def lambda_handler(event, context):
+    """
+    Nimmt SES Event entgegen, extrahiert Domain dynamisch,
+    verpackt Metadaten als 'Fake SNS' und sendet an die domain-spezifische SQS.
+    Mit integrierter Retry-Logik für SQS-Send.
+    """
+    try:
+        records = event.get('Records', [])
+        logger.info(f"Received event with {len(records)} records.")
+
+        for record in records:
+            ses_data = record.get('ses', {})
+            if not ses_data:
+                logger.warning(f"Invalid SES event: Missing 'ses' in record: {record}")
+                continue
+
+            mail = ses_data.get('mail', {})
+            receipt = ses_data.get('receipt', {})
+
+            # Domain extrahieren (aus erstem Recipient)
+            recipients = receipt.get('recipients', []) or mail.get('destination', [])
+            if not recipients:
+                logger.warning("No recipients in event - skipping")
+                continue
+
+            first_recipient = recipients[0]
+            domain = first_recipient.split('@')[-1].lower()
+            if not domain:
+                logger.error("Could not extract domain from recipient")
+                continue
+
+            # Wichtige Metadaten loggen
+            msg_id = mail.get('messageId', 'unknown')
+            source = mail.get('source', 'unknown')
+            logger.info(f"Processing Message-ID: {msg_id} for domain: {domain}")
+            logger.info(f" From: {source}")
+            logger.info(f" To: {recipients}")
+
+            # SES JSON als String serialisieren
+            ses_json_string = json.dumps(ses_data)
+
+            # Payload Größe loggen und checken (Safeguard)
+            payload_size = len(ses_json_string.encode('utf-8'))
+            logger.info(f" Metadata Payload Size: {payload_size} bytes")
+            if payload_size > 200000:  # Arbitrary Limit < SQS 256KB
+                raise ValueError("Payload too large for SQS")
+
+            # Fake SNS Payload
+            fake_sns_payload = {
+                "Type": "Notification",
+                "MessageId": str(uuid.uuid4()),
+                "TopicArn": "arn:aws:sns:ses-shim:global-topic",
+                "Subject": "Amazon SES Email Receipt Notification",
+                "Message": ses_json_string,
+                "Timestamp": datetime.utcnow().isoformat() + "Z"
+            }
+
+            # Queue URL dynamisch holen
+            queue_url = get_queue_url(domain)
+
+            # SQS Send mit Retries
+            attempt = 0
+            while attempt < MAX_RETRIES:
+                try:
+                    sqs.send_message(
+                        QueueUrl=queue_url,
+                        MessageBody=json.dumps(fake_sns_payload)
+                    )
+                    logger.info(f"✅ Successfully forwarded {msg_id} to SQS: {queue_url}")
+                    break
+                except ClientError as e:
+                    attempt += 1
+                    error_code = e.response['Error']['Code']
+                    logger.warning(f"Retry {attempt}/{MAX_RETRIES} for SQS send: {error_code} - {str(e)}")
+                    if attempt == MAX_RETRIES:
+                        raise
+                    time.sleep(exponential_backoff(attempt))
+
+        return {'status': 'ok'}
+
+    except Exception as e:
+        logger.error(f"❌ Critical Error in Lambda Shim: {str(e)}", exc_info=True)
+        raise e