From d1677a656c765101870453744b6bca74d551d3c8 Mon Sep 17 00:00:00 2001 From: Andreas Knuth Date: Sat, 24 Jan 2026 14:51:52 -0600 Subject: [PATCH] sieve generation --- DMS/.gitignore | 3 + DMS/content_filter.py | 367 ------------------ DMS/docker-compose.yml | 1 + .../dms/config/postfix/local_transport_maps | 1 - .../dms/config/postfix/main.cf.append | 11 - .../dms/config/postfix/master.cf.append | 32 -- DMS/docker-data/dms/config/user-patches.sh | 117 ------ DMS/sync_dynamodb_to_sieve.py | 67 ++++ 8 files changed, 71 insertions(+), 528 deletions(-) create mode 100644 DMS/.gitignore delete mode 100644 DMS/content_filter.py delete mode 100644 DMS/docker-data/dms/config/postfix/local_transport_maps delete mode 100644 DMS/docker-data/dms/config/postfix/main.cf.append delete mode 100644 DMS/docker-data/dms/config/postfix/master.cf.append delete mode 100644 DMS/docker-data/dms/config/user-patches.sh create mode 100644 DMS/sync_dynamodb_to_sieve.py diff --git a/DMS/.gitignore b/DMS/.gitignore new file mode 100644 index 0000000..ba88a06 --- /dev/null +++ b/DMS/.gitignore @@ -0,0 +1,3 @@ +mail-data +mail-logs +mail-state \ No newline at end of file diff --git a/DMS/content_filter.py b/DMS/content_filter.py deleted file mode 100644 index cade95f..0000000 --- a/DMS/content_filter.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env python3 -""" -Postfix Content Filter for Internal Email Processing -Handles forwarding and auto-reply for local deliveries - -Version: 2.0 (Optimized) -""" - -import os -import sys -import smtplib -import logging -from email import message_from_binary_file -from email.mime.text import MIMEText -from email.utils import parseaddr, formatdate, make_msgid -from datetime import datetime, timedelta -from io import BytesIO - -# Setup logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler('/var/log/mail/content_filter.log'), - logging.StreamHandler(sys.stderr) - ] -) - -# AWS Configuration -AWS_REGION = os.environ.get('AWS_REGION', 'us-east-2') -DYNAMODB_RULES_TABLE = os.environ.get('DYNAMODB_RULES_TABLE', 'email-rules') - -# SMTP Configuration -REINJECT_HOST = os.environ.get('REINJECT_HOST', 'localhost') -REINJECT_PORT = int(os.environ.get('REINJECT_PORT', '10026')) - -# Cache Configuration -CACHE_TTL_MINUTES = int(os.environ.get('CACHE_TTL_MINUTES', '5')) -CACHE_TTL = timedelta(minutes=CACHE_TTL_MINUTES) - -# Auto-reply throttling (prevent sending multiple auto-replies to same sender) -# Key: (recipient, sender), Value: last_sent_timestamp -AUTOREPLY_SENT = {} -AUTOREPLY_THROTTLE = timedelta(hours=24) # Only one auto-reply per sender per day - -# Cache for DynamoDB rules -RULES_CACHE = {} - -# Initialize boto3 (lazy import to catch errors) -DYNAMODB_AVAILABLE = False -try: - import boto3 - dynamodb = boto3.resource('dynamodb', region_name=AWS_REGION) - rules_table = dynamodb.Table(DYNAMODB_RULES_TABLE) - # Test connection - rules_table.table_status - DYNAMODB_AVAILABLE = True - logging.info("✓ DynamoDB connection initialized") -except Exception as e: - logging.error(f"✗ DynamoDB initialization failed: {e}") - logging.warning("Auto-reply and forwarding will be DISABLED") - -def extract_domain(email_addr): - """ - Extract domain from email address, handling various formats - - Examples: - "user@example.com" -> "example.com" - "Name " -> "example.com" - "invalid" -> "" - """ - if not email_addr or '@' not in email_addr: - return '' - - # parseaddr handles "Name " format - _, addr = parseaddr(email_addr) - - if '@' not in addr: - return '' - - try: - domain = addr.split('@')[1].lower() - return domain - except (IndexError, AttributeError): - return '' - -def get_email_rules(email_address): - """Fetch forwarding and auto-reply rules from DynamoDB with caching""" - if not DYNAMODB_AVAILABLE: - return {} - - now = datetime.now() - cache_key = email_address.lower() - - # Check cache - if cache_key in RULES_CACHE: - cached_entry = RULES_CACHE[cache_key] - if now - cached_entry['time'] < CACHE_TTL: - logging.debug(f"Cache hit for {email_address}") - return cached_entry['rules'] - - # Fetch from DynamoDB - try: - response = rules_table.get_item(Key={'email_address': email_address}) - item = response.get('Item', {}) - - if item: - forwards_count = len(item.get('forwards', [])) - ooo_active = item.get('ooo_active', False) - logging.info(f"Rules for {email_address}: forwards={forwards_count}, ooo={ooo_active}") - else: - logging.debug(f"No rules found for {email_address}") - - # Update cache - RULES_CACHE[cache_key] = {'rules': item, 'time': now} - return item - - except Exception as e: - logging.error(f"DynamoDB error for {email_address}: {e}") - return {} - -def should_send_autoreply(original_msg, sender_addr, recipient_addr): - """ - Check if we should send auto-reply to this sender - - Returns: (bool, str) - (should_send, reason_if_not) - """ - sender_lower = sender_addr.lower() - - # Don't reply to automated senders - blocked_patterns = [ - 'mailer-daemon', - 'postmaster', - 'noreply', - 'no-reply', - 'donotreply', - 'do-not-reply', - 'bounce', - 'amazonses.com', - 'notification', - ] - - for pattern in blocked_patterns: - if pattern in sender_lower: - return (False, f"automated sender pattern: {pattern}") - - # Check for auto-submitted header to prevent loops (RFC 3834) - auto_submitted = original_msg.get('Auto-Submitted', '') - if auto_submitted and auto_submitted.lower().startswith('auto-'): - return (False, f"Auto-Submitted header: {auto_submitted}") - - # Check precedence header (mailing lists, bulk mail) - precedence = original_msg.get('Precedence', '').lower() - if precedence in ['bulk', 'list', 'junk']: - return (False, f"Precedence: {precedence}") - - # Check List-* headers (mailing lists) - if original_msg.get('List-Id') or original_msg.get('List-Unsubscribe'): - return (False, "mailing list headers detected") - - # Throttle: Only send one auto-reply per sender per 24 hours - throttle_key = (recipient_addr.lower(), sender_lower) - if throttle_key in AUTOREPLY_SENT: - last_sent = AUTOREPLY_SENT[throttle_key] - if datetime.now() - last_sent < AUTOREPLY_THROTTLE: - time_left = AUTOREPLY_THROTTLE - (datetime.now() - last_sent) - hours_left = int(time_left.total_seconds() / 3600) - return (False, f"throttled (sent {hours_left}h ago)") - - return (True, "") - -def send_autoreply(original_msg, recipient_rules, recipient_addr): - """Send auto-reply if enabled and appropriate""" - if not recipient_rules.get('ooo_active'): - return - - sender = original_msg.get('From') - if not sender: - logging.warning("No sender address, skipping auto-reply") - return - - # Extract email from "Name " format - sender_name, sender_addr = parseaddr(sender) - - if not sender_addr or '@' not in sender_addr: - logging.warning(f"Invalid sender address: {sender}, skipping auto-reply") - return - - # Check if we should send auto-reply - should_send, reason = should_send_autoreply(original_msg, sender_addr, recipient_addr) - if not should_send: - logging.info(f"Skipping auto-reply to {sender_addr}: {reason}") - return - - subject = original_msg.get('Subject', 'No Subject') - message_id = original_msg.get('Message-ID') - - # Get auto-reply message - ooo_message = recipient_rules.get('ooo_message', 'I am currently unavailable.') - content_type = recipient_rules.get('ooo_content_type', 'text') - - # Create auto-reply - if content_type == 'html': - from email.mime.multipart import MIMEMultipart - from email.mime.text import MIMEText as MIMETextPart - reply = MIMEMultipart('alternative') - reply.attach(MIMETextPart(ooo_message, 'plain')) - reply.attach(MIMETextPart(ooo_message, 'html')) - else: - reply = MIMEText(ooo_message, 'plain', 'utf-8') - - reply['From'] = recipient_addr - reply['To'] = sender_addr - reply['Subject'] = f"Automatic Reply: {subject}" - reply['Date'] = formatdate(localtime=True) - reply['Message-ID'] = make_msgid() - reply['Auto-Submitted'] = 'auto-replied' # RFC 3834 - reply['Precedence'] = 'bulk' - reply['X-Auto-Response-Suppress'] = 'All' # Microsoft Exchange - - if message_id: - reply['In-Reply-To'] = message_id - reply['References'] = message_id - - # Send via local SMTP - try: - with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp: - smtp.send_message(reply) - - # Update throttle timestamp - throttle_key = (recipient_addr.lower(), sender_addr.lower()) - AUTOREPLY_SENT[throttle_key] = datetime.now() - - logging.info(f"✓ Sent auto-reply from {recipient_addr} to {sender_addr}") - except Exception as e: - logging.error(f"✗ Auto-reply failed: {e}") - -def send_forwards(original_msg_bytes, recipient_rules, recipient_addr, sender_addr): - """Forward email to configured addresses""" - forwards = recipient_rules.get('forwards', []) - if not forwards: - return - - for forward_addr in forwards: - try: - # Validate forward address - if '@' not in forward_addr: - logging.warning(f"Invalid forward address: {forward_addr}, skipping") - continue - - # Parse message again for clean forwarding - msg = message_from_binary_file(BytesIO(original_msg_bytes)) - - # Add forwarding headers - msg['X-Forwarded-For'] = recipient_addr - msg['X-Original-To'] = recipient_addr - msg['X-Forwarded-By'] = 'content_filter.py' - - # Preserve original sender in envelope - # (This way replies go to original sender, not to recipient) - envelope_sender = sender_addr if sender_addr else recipient_addr - - # Send via local SMTP - with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp: - smtp.sendmail( - from_addr=envelope_sender, - to_addrs=[forward_addr], - msg=msg.as_bytes() - ) - logging.info(f"✓ Forwarded from {recipient_addr} to {forward_addr}") - except Exception as e: - logging.error(f"✗ Forward to {forward_addr} failed: {e}") - -def is_internal_mail(sender, recipient): - """ - Check if this is internal mail (same domain) - - This is a safety check in addition to transport_maps filtering - """ - sender_domain = extract_domain(sender) - recipient_domain = extract_domain(recipient) - - if not sender_domain or not recipient_domain: - return False - - return sender_domain == recipient_domain - -def main(): - """Main content filter logic""" - if len(sys.argv) < 3: - logging.error("Usage: content_filter.py [recipient2] ...") - sys.exit(1) - - sender = sys.argv[1] - recipients = sys.argv[2:] - - logging.info(f"Processing email from {sender} to {', '.join(recipients)}") - - # Read email from stdin - try: - msg_bytes = sys.stdin.buffer.read() - if not msg_bytes: - logging.error("No email data received on stdin") - sys.exit(75) # EX_TEMPFAIL - - msg = message_from_binary_file(BytesIO(msg_bytes)) - except Exception as e: - logging.error(f"Failed to read email: {e}") - sys.exit(75) # EX_TEMPFAIL - - # Process each recipient - processed_count = 0 - for recipient in recipients: - try: - # Safety check: Only process internal mail - # (transport_maps should already filter, but defense-in-depth) - if not is_internal_mail(sender, recipient): - logging.debug(f"Skipping external mail: {sender} -> {recipient}") - continue - - # Fetch rules from DynamoDB - rules = get_email_rules(recipient) - - if rules: - processed_count += 1 - - # Send auto-reply if configured - if rules.get('ooo_active'): - send_autoreply(msg, rules, recipient) - - # Send forwards if configured - if rules.get('forwards'): - send_forwards(msg_bytes, rules, recipient, sender) - else: - logging.debug(f"No rules for {recipient}") - - except Exception as e: - logging.error(f"Error processing rules for {recipient}: {e}") - import traceback - logging.error(traceback.format_exc()) - - if processed_count > 0: - logging.info(f"Processed rules for {processed_count}/{len(recipients)} recipients") - - # Re-inject original email for normal delivery - try: - with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp: - smtp.sendmail(sender, recipients, msg_bytes) - logging.info(f"✓ Delivered to {', '.join(recipients)}") - sys.exit(0) - except Exception as e: - logging.error(f"✗ Delivery failed: {e}") - import traceback - logging.error(traceback.format_exc()) - sys.exit(75) # EX_TEMPFAIL - Postfix will retry - -if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - logging.info("Interrupted by user") - sys.exit(1) - except Exception as e: - logging.error(f"Fatal error: {e}") - import traceback - logging.error(traceback.format_exc()) - sys.exit(75) \ No newline at end of file diff --git a/DMS/docker-compose.yml b/DMS/docker-compose.yml index acdb8c3..ef231ed 100644 --- a/DMS/docker-compose.yml +++ b/DMS/docker-compose.yml @@ -24,6 +24,7 @@ services: - ./docker-data/dms/config/:/tmp/docker-mailserver/ - ./docker-data/dms/config/dovecot/conf.d/95-sieve-redirect.conf:/etc/dovecot/conf.d/95-sieve-redirect.conf:ro - /etc/localtime:/etc/localtime:ro + - ./sync_dynamodb_to_sieve.py:/scripts/sync.py:ro environment: # Wichtig: Rspamd und andere Services deaktivieren für ersten Test - ENABLE_RSPAMD=0 diff --git a/DMS/docker-data/dms/config/postfix/local_transport_maps b/DMS/docker-data/dms/config/postfix/local_transport_maps deleted file mode 100644 index 6539214..0000000 --- a/DMS/docker-data/dms/config/postfix/local_transport_maps +++ /dev/null @@ -1 +0,0 @@ -# Filter only local/internal deliveries (adjust to your domains) diff --git a/DMS/docker-data/dms/config/postfix/main.cf.append b/DMS/docker-data/dms/config/postfix/main.cf.append deleted file mode 100644 index 8d2f074..0000000 --- a/DMS/docker-data/dms/config/postfix/main.cf.append +++ /dev/null @@ -1,11 +0,0 @@ -# Content Filter Configuration -# Routes local/internal mail through content filter for forwarding and auto-reply - -# Use transport_maps for selective filtering -# Only internal deliveries go through content filter -# Transport map is auto-generated from postfix-accounts.cf by user-patches.sh -transport_maps = regexp:/etc/postfix/local_transport_maps - -# Optional: If you want ALL local deliveries to go through filter (not recommended) -# Uncomment this line and comment out transport_maps above: -# content_filter = smtp:[localhost]:10025 diff --git a/DMS/docker-data/dms/config/postfix/master.cf.append b/DMS/docker-data/dms/config/postfix/master.cf.append deleted file mode 100644 index 1b382dc..0000000 --- a/DMS/docker-data/dms/config/postfix/master.cf.append +++ /dev/null @@ -1,32 +0,0 @@ -# -# Content Filter Setup -# Two additional SMTP services for content filtering -# -# Port 10025: Content filter input -# Receives mail from main Postfix, passes to content_filter.py -localhost:10025 inet n - n - - smtpd - -o content_filter= - -o local_recipient_maps= - -o relay_recipient_maps= - -o smtpd_restriction_classes= - -o smtpd_client_restrictions= - -o smtpd_helo_restrictions= - -o smtpd_sender_restrictions= - -o smtpd_recipient_restrictions=permit_mynetworks,reject - -o mynetworks=127.0.0.0/8 - -o smtpd_authorized_xforward_hosts=127.0.0.0/8 - -o receive_override_options=no_unknown_recipient_checks -# Port 10026: Content filter output (re-injection) -# Receives processed mail from content_filter.py for final delivery -localhost:10026 inet n - n - - smtpd - -o content_filter= - -o local_recipient_maps= - -o relay_recipient_maps= - -o smtpd_restriction_classes= - -o smtpd_client_restrictions= - -o smtpd_helo_restrictions= - -o smtpd_sender_restrictions= - -o smtpd_recipient_restrictions=permit_mynetworks,reject - -o mynetworks=127.0.0.0/8,[::1]/128 - -o smtpd_authorized_xforward_hosts=127.0.0.0/8,[::1]/128 - -o receive_override_options=no_header_body_checks,no_unknown_recipient_checks \ No newline at end of file diff --git a/DMS/docker-data/dms/config/user-patches.sh b/DMS/docker-data/dms/config/user-patches.sh deleted file mode 100644 index d3aec6d..0000000 --- a/DMS/docker-data/dms/config/user-patches.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -# user-patches.sh - Optimized version with dynamic transport_maps generation - -set -euo pipefail - -CFG_ROOT="/tmp/docker-mailserver" -SRC_DIR="$CFG_ROOT/postfix" -DST_DIR="/etc/postfix" - -echo "[user-patches.sh] Starting Postfix customizations..." - -# Existing patches (header_checks, etc.) -if [ -f "$SRC_DIR/header_checks" ]; then - install -D -m 0644 "$SRC_DIR/header_checks" "$DST_DIR/header_checks" - echo "[user-patches.sh] ✓ header_checks installed" -fi - -if [ -f "$SRC_DIR/smtp_header_checks" ]; then - install -D -m 0644 "$SRC_DIR/smtp_header_checks" "$DST_DIR/maps/sender_header_filter.pcre" - echo "[user-patches.sh] ✓ smtp_header_checks installed" -fi - -# NEW: Append content filter configuration to main.cf -if [ -f "$SRC_DIR/main.cf.append" ]; then - echo "[user-patches.sh] Appending content filter config to main.cf..." - cat "$SRC_DIR/main.cf.append" >> "$DST_DIR/main.cf" - echo "[user-patches.sh] ✓ main.cf updated" -else - echo "[user-patches.sh] ⚠ main.cf.append not found, skipping" -fi - -# NEW: Append content filter services to master.cf -if [ -f "$SRC_DIR/master.cf.append" ]; then - echo "[user-patches.sh] Appending content filter services to master.cf..." - cat "$SRC_DIR/master.cf.append" >> "$DST_DIR/master.cf" - echo "[user-patches.sh] ✓ master.cf updated" -else - echo "[user-patches.sh] ⚠ master.cf.append not found, skipping" -fi - -# NEW: Generate local_transport_maps dynamically from postfix-accounts.cf -echo "[user-patches.sh] Generating local_transport_maps..." - -TRANSPORT_MAP="$DST_DIR/local_transport_maps" -ACCOUNTS_FILE="$CFG_ROOT/postfix-accounts.cf" - -# Create empty transport map -> "$TRANSPORT_MAP" - -if [ -f "$ACCOUNTS_FILE" ]; then - # Extract unique domains from postfix-accounts.cf - # Format of postfix-accounts.cf: user@domain.com|{PLAIN}password - - echo "# Auto-generated transport map for content filter" >> "$TRANSPORT_MAP" - echo "# Generated at: $(date)" >> "$TRANSPORT_MAP" - echo "" >> "$TRANSPORT_MAP" - - # Extract domains and create regex patterns - awk -F'@|\\|' '{print $2}' "$ACCOUNTS_FILE" | \ - sort -u | \ - while read -r domain; do - if [ -n "$domain" ]; then - # Escape dots for regex - escaped_domain=$(echo "$domain" | sed 's/\./\\./g') - echo "/^.*@${escaped_domain}\$/ smtp:[localhost]:10025" >> "$TRANSPORT_MAP" - echo "[user-patches.sh] - Added filter for: $domain" - fi - done - - # Compile the map - if [ -s "$TRANSPORT_MAP" ]; then - postmap "$TRANSPORT_MAP" - echo "[user-patches.sh] ✓ local_transport_maps created with $(grep -c '^/' "$TRANSPORT_MAP" || echo 0) domains" - else - echo "[user-patches.sh] ⚠ No domains found in $ACCOUNTS_FILE" - fi -else - echo "[user-patches.sh] ⚠ $ACCOUNTS_FILE not found, creating minimal transport_maps" - - # Fallback: Create minimal config - cat > "$TRANSPORT_MAP" << 'EOF' -# Minimal transport map - edit manually or populate postfix-accounts.cf -# Format: /^.*@domain\.com$/ smtp:[localhost]:10025 - -# Example (replace with your domains): -# /^.*@example\.com$/ smtp:[localhost]:10025 -# /^.*@another\.com$/ smtp:[localhost]:10025 -EOF - postmap "$TRANSPORT_MAP" -fi - -# Verify content filter script exists and is executable -if [ -x "/usr/local/bin/content_filter.py" ]; then - echo "[user-patches.sh] ✓ Content filter script found" - - # Test Python dependencies - if python3 -c "import boto3" 2>/dev/null; then - echo "[user-patches.sh] ✓ boto3 installed" - else - echo "[user-patches.sh] ⚠ WARNING: boto3 not installed!" - fi -else - echo "[user-patches.sh] ⚠ WARNING: content_filter.py not found or not executable!" -fi - -# Create log file if it doesn't exist -if [ ! -f "/var/log/mail/content_filter.log" ]; then - touch /var/log/mail/content_filter.log - chown mail:mail /var/log/mail/content_filter.log - chmod 644 /var/log/mail/content_filter.log - echo "[user-patches.sh] ✓ Created content_filter.log" -fi - -echo "[user-patches.sh] Postfix customizations complete" - -# Postfix neu laden (nachdem docker-mailserver seine eigene Konfig geladen hat) -postfix reload || true \ No newline at end of file diff --git a/DMS/sync_dynamodb_to_sieve.py b/DMS/sync_dynamodb_to_sieve.py new file mode 100644 index 0000000..b9e152d --- /dev/null +++ b/DMS/sync_dynamodb_to_sieve.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +""" +sync_dynamodb_to_sieve.py - Sync DynamoDB rules to Dovecot Sieve +""" +import boto3 +import os +from pathlib import Path + +# Config +REGION = 'us-east-2' +TABLE = 'email-rules' +VMAIL_BASE = '/var/mail' + +dynamodb = boto3.resource('dynamodb', region_name=REGION) +table = dynamodb.Table(TABLE) + +def generate_sieve(email, rules): + """Generate Sieve script from DynamoDB rules""" + script = [ + 'require ["copy","vacation","variables"];', + '', + '# Skip if already processed by worker', + 'if header :contains "X-SES-Worker-Processed" "" {', + ' keep;', + ' stop;', + '}', + '' + ] + + # Forwards + forwards = rules.get('forwards', []) + if forwards: + script.append('# rule:[forward]') + for fwd in forwards: + script.append(f'redirect :copy "{fwd}";') + + # OOO + if rules.get('ooo_active'): + msg = rules.get('ooo_message', 'I am away') + script.append('# rule:[reply]') + script.append(f'vacation :days 1 :from "{email}" "{msg}";') + + return '\n'.join(script) + +def sync(): + """Sync all rules from DynamoDB to Sieve""" + response = table.scan() + + for item in response.get('Items', []): + email = item['email_address'] + domain = email.split('@')[1] + user = email.split('@')[0] + + # Path: /var/mail/domain.de/user/.dovecot.sieve + sieve_path = Path(VMAIL_BASE) / domain / user / '.dovecot.sieve' + sieve_path.parent.mkdir(parents=True, exist_ok=True) + + # Generate & write + script = generate_sieve(email, item) + sieve_path.write_text(script) + + # Compile + os.system(f'sievec {sieve_path}') + print(f'✓ {email}') + +if __name__ == '__main__': + sync() \ No newline at end of file