diff --git a/basic_setup/awsses.sh b/basic_setup/awsses.sh index 1aea4f5..13b00dc 100755 --- a/basic_setup/awsses.sh +++ b/basic_setup/awsses.sh @@ -1,218 +1,187 @@ #!/bin/bash -# awsses.sh - Konfiguriert Amazon SES mit SNS->SQS Fanout Architektur & Outbound Tracking -# -# Ablauf: -# 1. SES Domain Identity erstellen/verifizieren -# 2. Domain mit Configuration Set verknüpfen (für Outbound Tracking) -# 3. SNS Topic erstellen -# 4. SNS Topic Policy setzen (damit SES hineinschreiben darf) -# 5. SQS Queue verbinden (Subscription) -# 6. SQS Queue Policy setzen (damit SNS hineinschreiben darf) -# 7. SES Receipt Rule erstellen (S3 Action + SNS Action) +# awsses_lambda_global.sh - SES Setup mit S3 + Global Lambda Shim -> SQS +# Dieses Skript ist idempotent: Es kann sicher mehrfach ausgeführt werden. +# Globale Lambda für alle Domains. set -e -# Überprüfen, ob jq installiert ist -if ! command -v jq &> /dev/null; then - echo "Fehler: 'jq' ist nicht installiert. Bitte installieren (sudo apt-get install jq)." +# --- CHECKS --- +if ! command -v jq &> /dev/null; then echo "Fehler: 'jq' fehlt."; exit 1; fi +if [ -z "$DOMAIN_NAME" ]; then echo "Fehler: DOMAIN_NAME ist nicht gesetzt."; exit 1; fi + +# Prüfen ob Python Code da ist +PYTHON_FILE="ses_sns_shim_global.py" +if [ ! -f "$PYTHON_FILE" ]; then + echo "Fehler: $PYTHON_FILE nicht gefunden!" exit 1 fi -# Überprüfen, ob die Domain-Variable gesetzt ist -if [ -z "$DOMAIN_NAME" ]; then - echo "Fehler: DOMAIN_NAME ist nicht gesetzt." - echo "Bitte setzen Sie die Variable mit: export DOMAIN_NAME='IhreDomain.de'" - exit 1 -fi - -# Überprüfen, ob S3_BUCKET_NAME gesetzt ist -if [ -z "$S3_BUCKET_NAME" ]; then - echo "Warnung: S3_BUCKET_NAME ist nicht gesetzt." - S3_BUCKET_NAME=$(echo "$DOMAIN_NAME" | tr '.' '-' | awk '{print $0 "-emails"}') - echo "Generierter Bucket-Name: $S3_BUCKET_NAME" -fi - -# Konfiguration +# --- VARIABLEN --- AWS_REGION=${AWS_REGION:-"us-east-2"} EMAIL_PREFIX=${EMAIL_PREFIX:-""} -CONFIGURATION_SET_NAME="relay-outbound" # Name deines globalen Config Sets +CONFIGURATION_SET_NAME="relay-outbound" -# Naming Conventions +# Bucket Name generieren falls leer +if [ -z "$S3_BUCKET_NAME" ]; then + S3_BUCKET_NAME=$(echo "$DOMAIN_NAME" | tr '.' '-' | awk '{print $0 "-emails"}') +fi + +# Namen (Global Lambda!) RULE_SET_NAME="bizmatch-ruleset" RULE_NAME="store-${DOMAIN_NAME//./-}-to-s3" -TOPIC_NAME="${DOMAIN_NAME//./-}-topic" QUEUE_NAME="${DOMAIN_NAME//./-}-queue" +LAMBDA_NAME="ses-shim-global" +LAMBDA_ROLE_NAME="SesShimGlobalRole" -echo "========================================================" -echo " SES Setup (Full Architecture) für $DOMAIN_NAME" -echo "========================================================" -echo "Region: $AWS_REGION" -echo "S3 Bucket: $S3_BUCKET_NAME" -echo "Config Set: $CONFIGURATION_SET_NAME" -echo "--------------------------------------------------------" - -# ------------------------ -# 1. SES Domain Identität -# ------------------------ -echo "[1/7] Prüfe SES Domain Identität..." +echo "==========================================================" +echo " SES Setup (S3 -> Global Lambda Shim -> SQS) für $DOMAIN_NAME" +echo "==========================================================" +# --------------------------------------------------------- +# 1. SES Identity & Config Set +# --------------------------------------------------------- +echo "[1/6] SES Identity Setup..." if ! aws sesv2 get-email-identity --email-identity ${DOMAIN_NAME} --region ${AWS_REGION} >/dev/null 2>&1; then - echo "-> Erstelle Identity..." aws sesv2 create-email-identity --email-identity ${DOMAIN_NAME} --region ${AWS_REGION} >/dev/null -else - echo "-> Identity existiert bereits." fi - -# Config Updates (Idempotent) -echo "-> Konfiguriere DKIM & Mail-From..." +# Update Attributes (Idempotent) aws sesv2 put-email-identity-dkim-attributes --email-identity ${DOMAIN_NAME} --signing-enabled --region ${AWS_REGION} aws sesv2 put-email-identity-mail-from-attributes --email-identity ${DOMAIN_NAME} --mail-from-domain "mail.${DOMAIN_NAME}" --behavior-on-mx-failure USE_DEFAULT_VALUE --region ${AWS_REGION} +aws sesv2 put-email-identity-configuration-set-attributes --email-identity ${DOMAIN_NAME} --configuration-set-name "$CONFIGURATION_SET_NAME" --region ${AWS_REGION} -# ------------------------ -# 2. Configuration Set Verknüpfung (NEU!) -# ------------------------ -echo "[2/7] Verknüpfe Domain mit Outbound Configuration Set..." -# Dies sorgt dafür, dass ausgehende Mails getrackt werden (für OOO/Bounces) -aws sesv2 put-email-identity-configuration-set-attributes \ - --email-identity ${DOMAIN_NAME} \ - --configuration-set-name "$CONFIGURATION_SET_NAME" \ - --region ${AWS_REGION} - -# ------------------------ -# 3. SNS Topic erstellen -# ------------------------ -echo "[3/7] Erstelle/Prüfe SNS Topic..." -TOPIC_ARN=$(aws sns create-topic --name "$TOPIC_NAME" --region "$AWS_REGION" --output text --query 'TopicArn') -echo "-> Topic ARN: $TOPIC_ARN" - -# ------------------------ -# 4. SNS Policy (SES -> SNS) -# ------------------------ -echo "[4/7] Setze SNS Policy (SES darf publishen)..." -ACCOUNT_ID=$(echo "$TOPIC_ARN" | cut -d: -f5) - -SNS_POLICY=$(jq -n \ - --arg topic_arn "$TOPIC_ARN" \ - --arg account_id "$ACCOUNT_ID" \ - '{ - Version: "2008-10-17", - Id: "__default_policy_ID", - Statement: [ - { - Sid: "Allow-SES-Publish", - Effect: "Allow", - Principal: { Service: "ses.amazonaws.com" }, - Action: "sns:Publish", - Resource: $topic_arn, - Condition: { - StringEquals: { "AWS:SourceAccount": $account_id } - } - } - ] - }' | jq -c .) - -aws sns set-topic-attributes --topic-arn "$TOPIC_ARN" --attribute-name Policy --attribute-value "$SNS_POLICY" --region "$AWS_REGION" - -# ------------------------ -# 5. SQS Queue Verbindung -# ------------------------ -echo "[5/7] Verbinde SQS Queue..." - -# Queue URL & ARN holen (Queue muss existieren -> create-queue.sh vorher ausführen!) +# --------------------------------------------------------- +# 2. SQS Queue holen (nur zur Validierung, Lambda holt dynamisch) +# --------------------------------------------------------- +echo "[2/6] Queue URL ermitteln (zur Validierung)..." QUEUE_URL=$(aws sqs get-queue-url --queue-name "$QUEUE_NAME" --region "$AWS_REGION" --output text --query 'QueueUrl' 2>/dev/null) - -if [ -z "$QUEUE_URL" ]; then - echo "FEHLER: Queue $QUEUE_NAME nicht gefunden!" - echo "Bitte führen Sie erst ./create-queue.sh aus." - exit 1 -fi - +if [ -z "$QUEUE_URL" ]; then echo "FEHLER: Queue $QUEUE_NAME nicht gefunden! ./create-queue.sh zuerst ausführen."; exit 1; fi QUEUE_ARN=$(aws sqs get-queue-attributes --queue-url "$QUEUE_URL" --attribute-names QueueArn --region "$AWS_REGION" --output text --query 'Attributes.QueueArn') -# Subscription erstellen (Idempotent) -aws sns subscribe --topic-arn "$TOPIC_ARN" --protocol sqs --notification-endpoint "$QUEUE_ARN" --region "$AWS_REGION" > /dev/null - -# ------------------------ -# 6. SQS Policy (SNS -> SQS) -# ------------------------ -echo "[6/7] Setze SQS Policy (SNS darf schreiben)..." - -SQS_POLICY=$(jq -n \ - --arg queue_arn "$QUEUE_ARN" \ - --arg topic_arn "$TOPIC_ARN" \ - '{ - Version: "2012-10-17", - Id: "SNS-to-SQS", - Statement: [ - { - Sid: "Allow-SNS-SendMessage", - Effect: "Allow", - Principal: { Service: "sns.amazonaws.com" }, - Action: "sqs:SendMessage", - Resource: $queue_arn, - Condition: { - ArnEquals: { "aws:SourceArn": $topic_arn } - } - } - ] - }' | jq -c .) - -# Policy setzen (mit Single-Quote Schutz für AWS CLI) -aws sqs set-queue-attributes --queue-url "$QUEUE_URL" --attributes Policy="'$SQS_POLICY'" --region "$AWS_REGION" - -# ------------------------ -# 7. SES Rule Set & Rule -# ------------------------ -echo "[7/7] Konfiguriere SES Receipt Rule..." - -# Rule Set prüfen -RULESET_EXISTS=$(aws ses list-receipt-rule-sets --region ${AWS_REGION} | jq -r '.RuleSets[] | select(.Name == "bizmatch-ruleset") | .Name') -if [ "$RULESET_EXISTS" != "bizmatch-ruleset" ]; then - echo "-> Erstelle Rule Set 'bizmatch-ruleset'..." - aws ses create-receipt-rule-set --rule-set-name "bizmatch-ruleset" --region ${AWS_REGION} +# --------------------------------------------------------- +# 3. IAM Role für Global Lambda erstellen +# --------------------------------------------------------- +echo "[3/6] IAM Role für Lambda prüfen/erstellen..." +TRUST_POLICY='{"Version": "2012-10-17","Statement": [{"Effect": "Allow","Principal": {"Service": "lambda.amazonaws.com"},"Action": "sts:AssumeRole"}]}' +if ! aws iam get-role --role-name "$LAMBDA_ROLE_NAME" >/dev/null 2>&1; then + aws iam create-role --role-name "$LAMBDA_ROLE_NAME" --assume-role-policy-document "$TRUST_POLICY" >/dev/null + echo " -> Rolle erstellt." +else + echo " -> Rolle existiert bereits." fi -# Rule prüfen/erstellen -if ! aws ses describe-receipt-rule --rule-set-name "$RULE_SET_NAME" --rule-name "${RULE_NAME}" --region ${AWS_REGION} >/dev/null 2>&1; then - - echo "-> Erstelle Receipt Rule '${RULE_NAME}'..." - - # Rule mit S3 Action UND SNS Action - # HINWEIS: Hier fügen wir initial die Hauptdomain als Recipient hinzu. - # Denke daran, später ./manage_mail_user.sh sync ... auszuführen! - aws ses create-receipt-rule --rule-set-name "$RULE_SET_NAME" --rule '{ - "Name": "'"${RULE_NAME}"'", - "Enabled": true, - "ScanEnabled": true, - "Actions": [ +# Permissions Policy (Lambda darf Logs schreiben und in ALLE Queues mit *-queue senden) +LAMBDA_POLICY=$(jq -n '{ + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Action: ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"], + Resource: "arn:aws:logs:*:*:*" + }, + { + Effect: "Allow", + Action: "sqs:SendMessage", + Resource: "arn:aws:sqs:*:*:*-queue" + }, + { + Effect: "Allow", + Action: "sqs:GetQueueUrl", + Resource: "*" + } + ] +}' | jq -c .) +aws iam put-role-policy --role-name "$LAMBDA_ROLE_NAME" --policy-name "SesShimGlobalPermissions" --policy-document "$LAMBDA_POLICY" +echo " -> Permissions aktualisiert." + +# Kurze Pause für IAM Propagation, falls Rolle neu war +sleep 5 + +# --------------------------------------------------------- +# 4. Lambda Funktion erstellen/updaten (Global!) +# --------------------------------------------------------- +echo "[4/6] Global Lambda Shim deployen..." +# Zip erstellen +cp "$PYTHON_FILE" lambda_function.py +zip -q lambda.zip lambda_function.py +# Keine Env-Vars nötig, da dynamisch +ROLE_ARN=$(aws iam get-role --role-name "$LAMBDA_ROLE_NAME" --query 'Role.Arn' --output text) +if ! aws lambda get-function --function-name "$LAMBDA_NAME" --region "$AWS_REGION" >/dev/null 2>&1; then + echo " -> Erstelle neue Lambda-Funktion..." + aws lambda create-function --function-name "$LAMBDA_NAME" \ + --runtime python3.11 --handler lambda_function.lambda_handler \ + --role "$ROLE_ARN" --zip-file fileb://lambda.zip \ + --region "$AWS_REGION" >/dev/null +else + echo " -> Aktualisiere existierende Lambda-Funktion..." + aws lambda update-function-code --function-name "$LAMBDA_NAME" --zip-file fileb://lambda.zip --region "$AWS_REGION" >/dev/null + + # Warte kurz + sleep 2 + + aws lambda update-function-configuration --function-name "$LAMBDA_NAME" --region "$AWS_REGION" >/dev/null +fi +# Aufräumen +rm lambda.zip lambda_function.py + +# --------------------------------------------------------- +# 5. Permission: SES darf Lambda aufrufen (Global, einmalig) +# --------------------------------------------------------- +echo "[5/6] SES Permission für Lambda..." +aws lambda add-permission --function-name "$LAMBDA_NAME" \ + --statement-id "AllowSESInvoke-Global" \ + --action "lambda:InvokeFunction" \ + --principal "ses.amazonaws.com" \ + --region "$AWS_REGION" 2>/dev/null || true + +# --------------------------------------------------------- +# 6. SES Rule (S3 + Global Lambda) +# --------------------------------------------------------- +echo "[6/6] SES Receipt Rule (S3 + Lambda) konfigurieren..." +LAMBDA_ARN=$(aws lambda get-function --function-name "$LAMBDA_NAME" --region "$AWS_REGION" --query 'Configuration.FunctionArn' --output text) +# Rule Set prüfen +if ! aws ses list-receipt-rule-sets --region ${AWS_REGION} | grep -q "bizmatch-ruleset"; then + aws ses create-receipt-rule-set --rule-set-name "bizmatch-ruleset" --region ${AWS_REGION} +fi +# Regel-Definition +RULE_JSON=$(jq -n \ + --arg bucket "$S3_BUCKET_NAME" \ + --arg prefix "$EMAIL_PREFIX" \ + --arg larn "$LAMBDA_ARN" \ + --arg rule "$RULE_NAME" \ + --arg domain "$DOMAIN_NAME" \ + --arg subdomain "mail.$DOMAIN_NAME" \ + '{ + Name: $rule, + Enabled: true, + ScanEnabled: true, + TlsPolicy: "Require", + Recipients: [$domain, $subdomain], + Actions: [ { - "S3Action": { - "BucketName": "'"${S3_BUCKET_NAME}"'", - "ObjectKeyPrefix": "'"${EMAIL_PREFIX}"'" + S3Action: { + BucketName: $bucket, + ObjectKeyPrefix: $prefix } }, { - "SNSAction": { - "TopicArn": "'"${TOPIC_ARN}"'", - "Encoding": "UTF-8" + LambdaAction: { + FunctionArn: $larn, + InvocationType: "Event" } } - ], - "TlsPolicy": "Require", - "Recipients": ["'"${DOMAIN_NAME}"'"] - }' --region ${AWS_REGION} - + ] + }') +# Check ob Regel existiert -> Update, sonst Create +if aws ses describe-receipt-rule --rule-set-name "bizmatch-ruleset" --rule-name "$RULE_NAME" --region "$AWS_REGION" >/dev/null 2>&1; then + echo " -> Aktualisiere existierende Regel..." + aws ses update-receipt-rule --rule-set-name "bizmatch-ruleset" --rule "$RULE_JSON" --region "$AWS_REGION" else - echo "-> Receipt Rule '${RULE_NAME}' existiert bereits (Überspringe Erstellung)." + echo " -> Erstelle neue Regel..." + aws ses create-receipt-rule --rule-set-name "bizmatch-ruleset" --rule "$RULE_JSON" --region "$AWS_REGION" fi - -# Rule Set aktivieren -ACTIVE_RULESET=$(aws ses describe-active-receipt-rule-set --region ${AWS_REGION} | jq -r '.Metadata.Name') -if [ "$ACTIVE_RULESET" != "bizmatch-ruleset" ]; then - echo "-> Aktiviere Rule Set..." - aws ses set-active-receipt-rule-set --rule-set-name "bizmatch-ruleset" --region ${AWS_REGION} -fi - +# Aktivieren +aws ses set-active-receipt-rule-set --rule-set-name "bizmatch-ruleset" --region ${AWS_REGION} echo "========================================================" -echo "✅ Setup erfolgreich abgeschlossen für $DOMAIN_NAME" +echo "✅ Setup erfolgreich. Globale Lambda ($LAMBDA_NAME) für alle Domains." +echo " S3 -> Lambda -> Domain-spezifische SQS" echo "========================================================" \ No newline at end of file diff --git a/basic_setup/ses_sns_shim_global.py b/basic_setup/ses_sns_shim_global.py new file mode 100644 index 0000000..0eb0c40 --- /dev/null +++ b/basic_setup/ses_sns_shim_global.py @@ -0,0 +1,123 @@ +import json +import os +import boto3 +import uuid +import logging +from datetime import datetime +from botocore.exceptions import ClientError +import time +import random + +# Logging konfigurieren +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +sqs = boto3.client('sqs') + +# Retry-Konfiguration +MAX_RETRIES = 3 +BASE_BACKOFF = 1 # Sekunden + +def exponential_backoff(attempt): + """Exponential Backoff mit Jitter""" + return BASE_BACKOFF * (2 ** attempt) + random.uniform(0, 1) + +def get_queue_url(domain): + """ + Generiert Queue-Namen aus Domain und holt URL. + Konvention: domain.tld -> domain-tld-queue + """ + queue_name = domain.replace('.', '-') + '-queue' + try: + response = sqs.get_queue_url(QueueName=queue_name) + return response['QueueUrl'] + except ClientError as e: + if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue': + logger.error(f"Queue nicht gefunden für Domain: {domain}") + raise ValueError(f"Keine Queue für Domain {domain}") + else: + raise + +def lambda_handler(event, context): + """ + Nimmt SES Event entgegen, extrahiert Domain dynamisch, + verpackt Metadaten als 'Fake SNS' und sendet an die domain-spezifische SQS. + Mit integrierter Retry-Logik für SQS-Send. + """ + try: + records = event.get('Records', []) + logger.info(f"Received event with {len(records)} records.") + + for record in records: + ses_data = record.get('ses', {}) + if not ses_data: + logger.warning(f"Invalid SES event: Missing 'ses' in record: {record}") + continue + + mail = ses_data.get('mail', {}) + receipt = ses_data.get('receipt', {}) + + # Domain extrahieren (aus erstem Recipient) + recipients = receipt.get('recipients', []) or mail.get('destination', []) + if not recipients: + logger.warning("No recipients in event - skipping") + continue + + first_recipient = recipients[0] + domain = first_recipient.split('@')[-1].lower() + if not domain: + logger.error("Could not extract domain from recipient") + continue + + # Wichtige Metadaten loggen + msg_id = mail.get('messageId', 'unknown') + source = mail.get('source', 'unknown') + logger.info(f"Processing Message-ID: {msg_id} for domain: {domain}") + logger.info(f" From: {source}") + logger.info(f" To: {recipients}") + + # SES JSON als String serialisieren + ses_json_string = json.dumps(ses_data) + + # Payload Größe loggen und checken (Safeguard) + payload_size = len(ses_json_string.encode('utf-8')) + logger.info(f" Metadata Payload Size: {payload_size} bytes") + if payload_size > 200000: # Arbitrary Limit < SQS 256KB + raise ValueError("Payload too large for SQS") + + # Fake SNS Payload + fake_sns_payload = { + "Type": "Notification", + "MessageId": str(uuid.uuid4()), + "TopicArn": "arn:aws:sns:ses-shim:global-topic", + "Subject": "Amazon SES Email Receipt Notification", + "Message": ses_json_string, + "Timestamp": datetime.utcnow().isoformat() + "Z" + } + + # Queue URL dynamisch holen + queue_url = get_queue_url(domain) + + # SQS Send mit Retries + attempt = 0 + while attempt < MAX_RETRIES: + try: + sqs.send_message( + QueueUrl=queue_url, + MessageBody=json.dumps(fake_sns_payload) + ) + logger.info(f"✅ Successfully forwarded {msg_id} to SQS: {queue_url}") + break + except ClientError as e: + attempt += 1 + error_code = e.response['Error']['Code'] + logger.warning(f"Retry {attempt}/{MAX_RETRIES} for SQS send: {error_code} - {str(e)}") + if attempt == MAX_RETRIES: + raise + time.sleep(exponential_backoff(attempt)) + + return {'status': 'ok'} + + except Exception as e: + logger.error(f"❌ Critical Error in Lambda Shim: {str(e)}", exc_info=True) + raise e \ No newline at end of file