From f7fe2852005442870e95c84ed9f19d764c167ed9 Mon Sep 17 00:00:00 2001 From: Andreas Knuth Date: Fri, 16 Jan 2026 22:16:09 -0600 Subject: [PATCH] cleanup --- DMS/content_filter.py | 201 +++++++++++++++--- DMS/docker-data/dms/config/postfix-main.cf | 13 -- .../dms/config/postfix/header_checks | 11 - .../dms/config/postfix/main.cf.append | 12 +- .../dms/config/postfix/sasl_passwd | 1 - .../dms/config/postfix/smtp_header_checks | 22 -- DMS/docker-data/dms/config/user-patches.sh | 86 +++++++- 7 files changed, 251 insertions(+), 95 deletions(-) delete mode 100644 DMS/docker-data/dms/config/postfix-main.cf delete mode 100644 DMS/docker-data/dms/config/postfix/header_checks delete mode 100644 DMS/docker-data/dms/config/postfix/sasl_passwd delete mode 100644 DMS/docker-data/dms/config/postfix/smtp_header_checks diff --git a/DMS/content_filter.py b/DMS/content_filter.py index 03f36a7..cade95f 100644 --- a/DMS/content_filter.py +++ b/DMS/content_filter.py @@ -2,6 +2,8 @@ """ Postfix Content Filter for Internal Email Processing Handles forwarding and auto-reply for local deliveries + +Version: 2.0 (Optimized) """ import os @@ -25,27 +27,62 @@ logging.basicConfig( ) # AWS Configuration -AWS_REGION = 'us-east-2' -DYNAMODB_RULES_TABLE = 'email-rules' +AWS_REGION = os.environ.get('AWS_REGION', 'us-east-2') +DYNAMODB_RULES_TABLE = os.environ.get('DYNAMODB_RULES_TABLE', 'email-rules') # SMTP Configuration -REINJECT_HOST = 'localhost' -REINJECT_PORT = 10026 +REINJECT_HOST = os.environ.get('REINJECT_HOST', 'localhost') +REINJECT_PORT = int(os.environ.get('REINJECT_PORT', '10026')) -# Cache for DynamoDB rules (TTL 5 minutes) +# Cache Configuration +CACHE_TTL_MINUTES = int(os.environ.get('CACHE_TTL_MINUTES', '5')) +CACHE_TTL = timedelta(minutes=CACHE_TTL_MINUTES) + +# Auto-reply throttling (prevent sending multiple auto-replies to same sender) +# Key: (recipient, sender), Value: last_sent_timestamp +AUTOREPLY_SENT = {} +AUTOREPLY_THROTTLE = timedelta(hours=24) # Only one auto-reply per sender per day + +# Cache for DynamoDB rules RULES_CACHE = {} -CACHE_TTL = timedelta(minutes=5) # Initialize boto3 (lazy import to catch errors) +DYNAMODB_AVAILABLE = False try: import boto3 dynamodb = boto3.resource('dynamodb', region_name=AWS_REGION) rules_table = dynamodb.Table(DYNAMODB_RULES_TABLE) + # Test connection + rules_table.table_status DYNAMODB_AVAILABLE = True - logging.info("DynamoDB connection initialized") + logging.info("✓ DynamoDB connection initialized") except Exception as e: - DYNAMODB_AVAILABLE = False - logging.error(f"DynamoDB initialization failed: {e}") + logging.error(f"✗ DynamoDB initialization failed: {e}") + logging.warning("Auto-reply and forwarding will be DISABLED") + +def extract_domain(email_addr): + """ + Extract domain from email address, handling various formats + + Examples: + "user@example.com" -> "example.com" + "Name " -> "example.com" + "invalid" -> "" + """ + if not email_addr or '@' not in email_addr: + return '' + + # parseaddr handles "Name " format + _, addr = parseaddr(email_addr) + + if '@' not in addr: + return '' + + try: + domain = addr.split('@')[1].lower() + return domain + except (IndexError, AttributeError): + return '' def get_email_rules(email_address): """Fetch forwarding and auto-reply rules from DynamoDB with caching""" @@ -54,23 +91,40 @@ def get_email_rules(email_address): now = datetime.now() cache_key = email_address.lower() - if cache_key in RULES_CACHE and now - RULES_CACHE[cache_key]['time'] < CACHE_TTL: - logging.debug(f"Cache hit for {email_address}") - return RULES_CACHE[cache_key]['rules'] + # Check cache + if cache_key in RULES_CACHE: + cached_entry = RULES_CACHE[cache_key] + if now - cached_entry['time'] < CACHE_TTL: + logging.debug(f"Cache hit for {email_address}") + return cached_entry['rules'] + + # Fetch from DynamoDB try: response = rules_table.get_item(Key={'email_address': email_address}) item = response.get('Item', {}) + if item: - logging.info(f"Rules found for {email_address}: forwards={len(item.get('forwards', []))}, ooo={item.get('ooo_active', False)}") + forwards_count = len(item.get('forwards', [])) + ooo_active = item.get('ooo_active', False) + logging.info(f"Rules for {email_address}: forwards={forwards_count}, ooo={ooo_active}") + else: + logging.debug(f"No rules found for {email_address}") + + # Update cache RULES_CACHE[cache_key] = {'rules': item, 'time': now} return item + except Exception as e: logging.error(f"DynamoDB error for {email_address}: {e}") return {} -def should_send_autoreply(original_msg, sender_addr): - """Check if we should send auto-reply to this sender""" +def should_send_autoreply(original_msg, sender_addr, recipient_addr): + """ + Check if we should send auto-reply to this sender + + Returns: (bool, str) - (should_send, reason_if_not) + """ sender_lower = sender_addr.lower() # Don't reply to automated senders @@ -80,24 +134,43 @@ def should_send_autoreply(original_msg, sender_addr): 'noreply', 'no-reply', 'donotreply', + 'do-not-reply', 'bounce', - 'amazonses.com' + 'amazonses.com', + 'notification', ] for pattern in blocked_patterns: if pattern in sender_lower: - logging.info(f"Skipping auto-reply to automated sender: {sender_addr}") - return False + return (False, f"automated sender pattern: {pattern}") # Check for auto-submitted header to prevent loops (RFC 3834) - if original_msg.get('Auto-Submitted', '').startswith('auto-'): - logging.info(f"Skipping auto-reply due to Auto-Submitted header: {sender_addr}") - return False + auto_submitted = original_msg.get('Auto-Submitted', '') + if auto_submitted and auto_submitted.lower().startswith('auto-'): + return (False, f"Auto-Submitted header: {auto_submitted}") - return True + # Check precedence header (mailing lists, bulk mail) + precedence = original_msg.get('Precedence', '').lower() + if precedence in ['bulk', 'list', 'junk']: + return (False, f"Precedence: {precedence}") + + # Check List-* headers (mailing lists) + if original_msg.get('List-Id') or original_msg.get('List-Unsubscribe'): + return (False, "mailing list headers detected") + + # Throttle: Only send one auto-reply per sender per 24 hours + throttle_key = (recipient_addr.lower(), sender_lower) + if throttle_key in AUTOREPLY_SENT: + last_sent = AUTOREPLY_SENT[throttle_key] + if datetime.now() - last_sent < AUTOREPLY_THROTTLE: + time_left = AUTOREPLY_THROTTLE - (datetime.now() - last_sent) + hours_left = int(time_left.total_seconds() / 3600) + return (False, f"throttled (sent {hours_left}h ago)") + + return (True, "") def send_autoreply(original_msg, recipient_rules, recipient_addr): - """Send auto-reply if enabled""" + """Send auto-reply if enabled and appropriate""" if not recipient_rules.get('ooo_active'): return @@ -109,7 +182,14 @@ def send_autoreply(original_msg, recipient_rules, recipient_addr): # Extract email from "Name " format sender_name, sender_addr = parseaddr(sender) - if not should_send_autoreply(original_msg, sender_addr): + if not sender_addr or '@' not in sender_addr: + logging.warning(f"Invalid sender address: {sender}, skipping auto-reply") + return + + # Check if we should send auto-reply + should_send, reason = should_send_autoreply(original_msg, sender_addr, recipient_addr) + if not should_send: + logging.info(f"Skipping auto-reply to {sender_addr}: {reason}") return subject = original_msg.get('Subject', 'No Subject') @@ -136,6 +216,7 @@ def send_autoreply(original_msg, recipient_rules, recipient_addr): reply['Message-ID'] = make_msgid() reply['Auto-Submitted'] = 'auto-replied' # RFC 3834 reply['Precedence'] = 'bulk' + reply['X-Auto-Response-Suppress'] = 'All' # Microsoft Exchange if message_id: reply['In-Reply-To'] = message_id @@ -145,11 +226,16 @@ def send_autoreply(original_msg, recipient_rules, recipient_addr): try: with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp: smtp.send_message(reply) + + # Update throttle timestamp + throttle_key = (recipient_addr.lower(), sender_addr.lower()) + AUTOREPLY_SENT[throttle_key] = datetime.now() + logging.info(f"✓ Sent auto-reply from {recipient_addr} to {sender_addr}") except Exception as e: logging.error(f"✗ Auto-reply failed: {e}") -def send_forwards(original_msg_bytes, recipient_rules, recipient_addr): +def send_forwards(original_msg_bytes, recipient_rules, recipient_addr, sender_addr): """Forward email to configured addresses""" forwards = recipient_rules.get('forwards', []) if not forwards: @@ -157,17 +243,27 @@ def send_forwards(original_msg_bytes, recipient_rules, recipient_addr): for forward_addr in forwards: try: + # Validate forward address + if '@' not in forward_addr: + logging.warning(f"Invalid forward address: {forward_addr}, skipping") + continue + # Parse message again for clean forwarding msg = message_from_binary_file(BytesIO(original_msg_bytes)) # Add forwarding headers msg['X-Forwarded-For'] = recipient_addr msg['X-Original-To'] = recipient_addr + msg['X-Forwarded-By'] = 'content_filter.py' + + # Preserve original sender in envelope + # (This way replies go to original sender, not to recipient) + envelope_sender = sender_addr if sender_addr else recipient_addr # Send via local SMTP with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp: smtp.sendmail( - from_addr=recipient_addr, + from_addr=envelope_sender, to_addrs=[forward_addr], msg=msg.as_bytes() ) @@ -175,6 +271,20 @@ def send_forwards(original_msg_bytes, recipient_rules, recipient_addr): except Exception as e: logging.error(f"✗ Forward to {forward_addr} failed: {e}") +def is_internal_mail(sender, recipient): + """ + Check if this is internal mail (same domain) + + This is a safety check in addition to transport_maps filtering + """ + sender_domain = extract_domain(sender) + recipient_domain = extract_domain(recipient) + + if not sender_domain or not recipient_domain: + return False + + return sender_domain == recipient_domain + def main(): """Main content filter logic""" if len(sys.argv) < 3: @@ -184,53 +294,74 @@ def main(): sender = sys.argv[1] recipients = sys.argv[2:] - logging.info(f"Processing email from {sender} to {', '.join(recipient)}") + logging.info(f"Processing email from {sender} to {', '.join(recipients)}") # Read email from stdin try: msg_bytes = sys.stdin.buffer.read() + if not msg_bytes: + logging.error("No email data received on stdin") + sys.exit(75) # EX_TEMPFAIL + msg = message_from_binary_file(BytesIO(msg_bytes)) except Exception as e: logging.error(f"Failed to read email: {e}") sys.exit(75) # EX_TEMPFAIL # Process each recipient + processed_count = 0 for recipient in recipients: try: - # Check if mail is internal (same domain) - skip if external - _, recipient_domain = parseaddr(recipient)[1].rsplit('@', 1) if '@' in recipient else ('', '') - _, sender_domain = parseaddr(sender)[1].rsplit('@', 1) if '@' in sender else ('', '') - if recipient_domain != sender_domain: - logging.info(f"Skipping external mail to {recipient} (sender domain: {sender_domain})") + # Safety check: Only process internal mail + # (transport_maps should already filter, but defense-in-depth) + if not is_internal_mail(sender, recipient): + logging.debug(f"Skipping external mail: {sender} -> {recipient}") continue + # Fetch rules from DynamoDB rules = get_email_rules(recipient) if rules: + processed_count += 1 + # Send auto-reply if configured - send_autoreply(msg, rules, recipient) + if rules.get('ooo_active'): + send_autoreply(msg, rules, recipient) # Send forwards if configured - send_forwards(msg_bytes, rules, recipient) + if rules.get('forwards'): + send_forwards(msg_bytes, rules, recipient, sender) else: logging.debug(f"No rules for {recipient}") except Exception as e: logging.error(f"Error processing rules for {recipient}: {e}") + import traceback + logging.error(traceback.format_exc()) + + if processed_count > 0: + logging.info(f"Processed rules for {processed_count}/{len(recipients)} recipients") # Re-inject original email for normal delivery try: with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp: smtp.sendmail(sender, recipients, msg_bytes) - logging.info(f"✓ Delivered to {', '.join(recipient)}") + logging.info(f"✓ Delivered to {', '.join(recipients)}") sys.exit(0) except Exception as e: logging.error(f"✗ Delivery failed: {e}") + import traceback + logging.error(traceback.format_exc()) sys.exit(75) # EX_TEMPFAIL - Postfix will retry if __name__ == '__main__': try: main() + except KeyboardInterrupt: + logging.info("Interrupted by user") + sys.exit(1) except Exception as e: logging.error(f"Fatal error: {e}") + import traceback + logging.error(traceback.format_exc()) sys.exit(75) \ No newline at end of file diff --git a/DMS/docker-data/dms/config/postfix-main.cf b/DMS/docker-data/dms/config/postfix-main.cf deleted file mode 100644 index 1a4859d..0000000 --- a/DMS/docker-data/dms/config/postfix-main.cf +++ /dev/null @@ -1,13 +0,0 @@ -# persistente Overrides -smtp_host_lookup = dns -smtp_tls_security_level = encrypt -smtp_tls_note_starttls_offer = yes - -# smtp_sasl_auth_enable = yes -# smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd -# smtp_sasl_security_options = noanonymous - -# transport_maps = hash:/etc/postfix/transport - -header_checks = pcre:/etc/postfix/header_checks -smtp_tls_loglevel = 1 \ No newline at end of file diff --git a/DMS/docker-data/dms/config/postfix/header_checks b/DMS/docker-data/dms/config/postfix/header_checks deleted file mode 100644 index 8059f18..0000000 --- a/DMS/docker-data/dms/config/postfix/header_checks +++ /dev/null @@ -1,11 +0,0 @@ -# X-SES-CONFIGURATION-SET für ausgehende Mails -/^Subject:/ PREPEND X-SES-CONFIGURATION-SET: relay-outbound - -# === DEBUG SECTION - Logging für Weitergeleitete Mails === -/^From:/ WARN Debugging: Original From Header -/^To:/ WARN Debugging: To Header -/^Return-Path:/ WARN Debugging: Return-Path -/^X-Forwarded/ WARN Debugging: Forwarding detected - -# Entferne doppelte Delivered-To Headers bei Weiterleitungen -/^Delivered-To:/ IGNORE \ No newline at end of file diff --git a/DMS/docker-data/dms/config/postfix/main.cf.append b/DMS/docker-data/dms/config/postfix/main.cf.append index bb5f67b..8d2f074 100644 --- a/DMS/docker-data/dms/config/postfix/main.cf.append +++ b/DMS/docker-data/dms/config/postfix/main.cf.append @@ -1,3 +1,11 @@ # Content Filter Configuration -# Use transport_maps for selective filtering (only locals) -transport_maps = regexp:/etc/postfix/local_transport_maps \ No newline at end of file +# Routes local/internal mail through content filter for forwarding and auto-reply + +# Use transport_maps for selective filtering +# Only internal deliveries go through content filter +# Transport map is auto-generated from postfix-accounts.cf by user-patches.sh +transport_maps = regexp:/etc/postfix/local_transport_maps + +# Optional: If you want ALL local deliveries to go through filter (not recommended) +# Uncomment this line and comment out transport_maps above: +# content_filter = smtp:[localhost]:10025 diff --git a/DMS/docker-data/dms/config/postfix/sasl_passwd b/DMS/docker-data/dms/config/postfix/sasl_passwd deleted file mode 100644 index 68af6b4..0000000 --- a/DMS/docker-data/dms/config/postfix/sasl_passwd +++ /dev/null @@ -1 +0,0 @@ -[email-smtp.us-east-2.amazonaws.com]:587 AKIAU6G......../ARbpotim1m........... \ No newline at end of file diff --git a/DMS/docker-data/dms/config/postfix/smtp_header_checks b/DMS/docker-data/dms/config/postfix/smtp_header_checks deleted file mode 100644 index 7fe17e2..0000000 --- a/DMS/docker-data/dms/config/postfix/smtp_header_checks +++ /dev/null @@ -1,22 +0,0 @@ -# 1. EIGENE DOMAINS SCHÜTZEN (Whitelist) -# Wenn der Absender @bayarea-cc.com oder @email-srvr.com ist, tue NICHTS (DUNNO). -# Das Postfix bricht die Prüfung hier ab, die Mail bleibt original. -/.*@bayarea-cc\.com/ DUNNO -/.*@email-srvr\.com/ DUNNO -/.*@andreasknuth\.de/ DUNNO -# 2. FREMDE DOMAINS UMSCHREIBEN (Rewriting) -# Nur wenn wir hier ankommen (also keine eigene Domain), schreiben wir um. -# Ersetzt den Absender durch eine generische Adresse deiner Domain. - -# Fall A: Mit Name -> "Name (original@email)" -/^From:(.*)\s+<(.*)>/ REPLACE From: "$1 ($2)" - -# Fall B: Ohne Name -> "original@email" -/^From:\s*([^<>\s]+)$/ REPLACE From: "$1" - -# 3. AUFRÄUMEN -# Return-Path im Header entfernen (verwirrt manche Clients, da SRS den Envelope regelt) -/^Return-Path:/ IGNORE - -# Entferne Sieve-spezifische Headers bei Weiterleitungen -/^\s*Delivered-To:/ IGNORE \ No newline at end of file diff --git a/DMS/docker-data/dms/config/user-patches.sh b/DMS/docker-data/dms/config/user-patches.sh index f9360e0..d3aec6d 100644 --- a/DMS/docker-data/dms/config/user-patches.sh +++ b/DMS/docker-data/dms/config/user-patches.sh @@ -1,4 +1,6 @@ #!/bin/bash +# user-patches.sh - Optimized version with dynamic transport_maps generation + set -euo pipefail CFG_ROOT="/tmp/docker-mailserver" @@ -8,8 +10,15 @@ DST_DIR="/etc/postfix" echo "[user-patches.sh] Starting Postfix customizations..." # Existing patches (header_checks, etc.) -install -D -m 0644 "$SRC_DIR/header_checks" "$DST_DIR/header_checks" -install -D -m 0644 "$SRC_DIR/smtp_header_checks" "$DST_DIR/maps/sender_header_filter.pcre" +if [ -f "$SRC_DIR/header_checks" ]; then + install -D -m 0644 "$SRC_DIR/header_checks" "$DST_DIR/header_checks" + echo "[user-patches.sh] ✓ header_checks installed" +fi + +if [ -f "$SRC_DIR/smtp_header_checks" ]; then + install -D -m 0644 "$SRC_DIR/smtp_header_checks" "$DST_DIR/maps/sender_header_filter.pcre" + echo "[user-patches.sh] ✓ smtp_header_checks installed" +fi # NEW: Append content filter configuration to main.cf if [ -f "$SRC_DIR/main.cf.append" ]; then @@ -29,24 +38,79 @@ else echo "[user-patches.sh] ⚠ master.cf.append not found, skipping" fi -# NEW: Create and postmap local_transport_maps for selective filtering -echo "[user-patches.sh] Creating local_transport_maps..." -install -D -m 0644 /dev/null "$DST_DIR/local_transport_maps" -cat > "$DST_DIR/local_transport_maps" << 'EOF' -# Filter only local/internal deliveries (adjust to your domains) -/^.*@example\.com$/ smtp:[localhost]:10025 # Replace with your domains, e.g. /^.*@andreasknuth\.de$/ -/^.*@another-domain\.com$/ smtp:[localhost]:10025 +# NEW: Generate local_transport_maps dynamically from postfix-accounts.cf +echo "[user-patches.sh] Generating local_transport_maps..." + +TRANSPORT_MAP="$DST_DIR/local_transport_maps" +ACCOUNTS_FILE="$CFG_ROOT/postfix-accounts.cf" + +# Create empty transport map +> "$TRANSPORT_MAP" + +if [ -f "$ACCOUNTS_FILE" ]; then + # Extract unique domains from postfix-accounts.cf + # Format of postfix-accounts.cf: user@domain.com|{PLAIN}password + + echo "# Auto-generated transport map for content filter" >> "$TRANSPORT_MAP" + echo "# Generated at: $(date)" >> "$TRANSPORT_MAP" + echo "" >> "$TRANSPORT_MAP" + + # Extract domains and create regex patterns + awk -F'@|\\|' '{print $2}' "$ACCOUNTS_FILE" | \ + sort -u | \ + while read -r domain; do + if [ -n "$domain" ]; then + # Escape dots for regex + escaped_domain=$(echo "$domain" | sed 's/\./\\./g') + echo "/^.*@${escaped_domain}\$/ smtp:[localhost]:10025" >> "$TRANSPORT_MAP" + echo "[user-patches.sh] - Added filter for: $domain" + fi + done + + # Compile the map + if [ -s "$TRANSPORT_MAP" ]; then + postmap "$TRANSPORT_MAP" + echo "[user-patches.sh] ✓ local_transport_maps created with $(grep -c '^/' "$TRANSPORT_MAP" || echo 0) domains" + else + echo "[user-patches.sh] ⚠ No domains found in $ACCOUNTS_FILE" + fi +else + echo "[user-patches.sh] ⚠ $ACCOUNTS_FILE not found, creating minimal transport_maps" + + # Fallback: Create minimal config + cat > "$TRANSPORT_MAP" << 'EOF' +# Minimal transport map - edit manually or populate postfix-accounts.cf +# Format: /^.*@domain\.com$/ smtp:[localhost]:10025 + +# Example (replace with your domains): +# /^.*@example\.com$/ smtp:[localhost]:10025 +# /^.*@another\.com$/ smtp:[localhost]:10025 EOF -postmap "$DST_DIR/local_transport_maps" -echo "[user-patches.sh] ✓ local_transport_maps created and mapped" + postmap "$TRANSPORT_MAP" +fi # Verify content filter script exists and is executable if [ -x "/usr/local/bin/content_filter.py" ]; then echo "[user-patches.sh] ✓ Content filter script found" + + # Test Python dependencies + if python3 -c "import boto3" 2>/dev/null; then + echo "[user-patches.sh] ✓ boto3 installed" + else + echo "[user-patches.sh] ⚠ WARNING: boto3 not installed!" + fi else echo "[user-patches.sh] ⚠ WARNING: content_filter.py not found or not executable!" fi +# Create log file if it doesn't exist +if [ ! -f "/var/log/mail/content_filter.log" ]; then + touch /var/log/mail/content_filter.log + chown mail:mail /var/log/mail/content_filter.log + chmod 644 /var/log/mail/content_filter.log + echo "[user-patches.sh] ✓ Created content_filter.log" +fi + echo "[user-patches.sh] Postfix customizations complete" # Postfix neu laden (nachdem docker-mailserver seine eigene Konfig geladen hat)