sieve generation

This commit is contained in:
Andreas Knuth 2026-01-24 14:51:52 -06:00
parent 182598c402
commit d1677a656c
8 changed files with 71 additions and 528 deletions

3
DMS/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
mail-data
mail-logs
mail-state

View File

@ -1,367 +0,0 @@
#!/usr/bin/env python3
"""
Postfix Content Filter for Internal Email Processing
Handles forwarding and auto-reply for local deliveries
Version: 2.0 (Optimized)
"""
import os
import sys
import smtplib
import logging
from email import message_from_binary_file
from email.mime.text import MIMEText
from email.utils import parseaddr, formatdate, make_msgid
from datetime import datetime, timedelta
from io import BytesIO
# Setup logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('/var/log/mail/content_filter.log'),
logging.StreamHandler(sys.stderr)
]
)
# AWS Configuration
AWS_REGION = os.environ.get('AWS_REGION', 'us-east-2')
DYNAMODB_RULES_TABLE = os.environ.get('DYNAMODB_RULES_TABLE', 'email-rules')
# SMTP Configuration
REINJECT_HOST = os.environ.get('REINJECT_HOST', 'localhost')
REINJECT_PORT = int(os.environ.get('REINJECT_PORT', '10026'))
# Cache Configuration
CACHE_TTL_MINUTES = int(os.environ.get('CACHE_TTL_MINUTES', '5'))
CACHE_TTL = timedelta(minutes=CACHE_TTL_MINUTES)
# Auto-reply throttling (prevent sending multiple auto-replies to same sender)
# Key: (recipient, sender), Value: last_sent_timestamp
AUTOREPLY_SENT = {}
AUTOREPLY_THROTTLE = timedelta(hours=24) # Only one auto-reply per sender per day
# Cache for DynamoDB rules
RULES_CACHE = {}
# Initialize boto3 (lazy import to catch errors)
DYNAMODB_AVAILABLE = False
try:
import boto3
dynamodb = boto3.resource('dynamodb', region_name=AWS_REGION)
rules_table = dynamodb.Table(DYNAMODB_RULES_TABLE)
# Test connection
rules_table.table_status
DYNAMODB_AVAILABLE = True
logging.info("✓ DynamoDB connection initialized")
except Exception as e:
logging.error(f"✗ DynamoDB initialization failed: {e}")
logging.warning("Auto-reply and forwarding will be DISABLED")
def extract_domain(email_addr):
"""
Extract domain from email address, handling various formats
Examples:
"user@example.com" -> "example.com"
"Name <user@example.com>" -> "example.com"
"invalid" -> ""
"""
if not email_addr or '@' not in email_addr:
return ''
# parseaddr handles "Name <email@domain.com>" format
_, addr = parseaddr(email_addr)
if '@' not in addr:
return ''
try:
domain = addr.split('@')[1].lower()
return domain
except (IndexError, AttributeError):
return ''
def get_email_rules(email_address):
"""Fetch forwarding and auto-reply rules from DynamoDB with caching"""
if not DYNAMODB_AVAILABLE:
return {}
now = datetime.now()
cache_key = email_address.lower()
# Check cache
if cache_key in RULES_CACHE:
cached_entry = RULES_CACHE[cache_key]
if now - cached_entry['time'] < CACHE_TTL:
logging.debug(f"Cache hit for {email_address}")
return cached_entry['rules']
# Fetch from DynamoDB
try:
response = rules_table.get_item(Key={'email_address': email_address})
item = response.get('Item', {})
if item:
forwards_count = len(item.get('forwards', []))
ooo_active = item.get('ooo_active', False)
logging.info(f"Rules for {email_address}: forwards={forwards_count}, ooo={ooo_active}")
else:
logging.debug(f"No rules found for {email_address}")
# Update cache
RULES_CACHE[cache_key] = {'rules': item, 'time': now}
return item
except Exception as e:
logging.error(f"DynamoDB error for {email_address}: {e}")
return {}
def should_send_autoreply(original_msg, sender_addr, recipient_addr):
"""
Check if we should send auto-reply to this sender
Returns: (bool, str) - (should_send, reason_if_not)
"""
sender_lower = sender_addr.lower()
# Don't reply to automated senders
blocked_patterns = [
'mailer-daemon',
'postmaster',
'noreply',
'no-reply',
'donotreply',
'do-not-reply',
'bounce',
'amazonses.com',
'notification',
]
for pattern in blocked_patterns:
if pattern in sender_lower:
return (False, f"automated sender pattern: {pattern}")
# Check for auto-submitted header to prevent loops (RFC 3834)
auto_submitted = original_msg.get('Auto-Submitted', '')
if auto_submitted and auto_submitted.lower().startswith('auto-'):
return (False, f"Auto-Submitted header: {auto_submitted}")
# Check precedence header (mailing lists, bulk mail)
precedence = original_msg.get('Precedence', '').lower()
if precedence in ['bulk', 'list', 'junk']:
return (False, f"Precedence: {precedence}")
# Check List-* headers (mailing lists)
if original_msg.get('List-Id') or original_msg.get('List-Unsubscribe'):
return (False, "mailing list headers detected")
# Throttle: Only send one auto-reply per sender per 24 hours
throttle_key = (recipient_addr.lower(), sender_lower)
if throttle_key in AUTOREPLY_SENT:
last_sent = AUTOREPLY_SENT[throttle_key]
if datetime.now() - last_sent < AUTOREPLY_THROTTLE:
time_left = AUTOREPLY_THROTTLE - (datetime.now() - last_sent)
hours_left = int(time_left.total_seconds() / 3600)
return (False, f"throttled (sent {hours_left}h ago)")
return (True, "")
def send_autoreply(original_msg, recipient_rules, recipient_addr):
"""Send auto-reply if enabled and appropriate"""
if not recipient_rules.get('ooo_active'):
return
sender = original_msg.get('From')
if not sender:
logging.warning("No sender address, skipping auto-reply")
return
# Extract email from "Name <email>" format
sender_name, sender_addr = parseaddr(sender)
if not sender_addr or '@' not in sender_addr:
logging.warning(f"Invalid sender address: {sender}, skipping auto-reply")
return
# Check if we should send auto-reply
should_send, reason = should_send_autoreply(original_msg, sender_addr, recipient_addr)
if not should_send:
logging.info(f"Skipping auto-reply to {sender_addr}: {reason}")
return
subject = original_msg.get('Subject', 'No Subject')
message_id = original_msg.get('Message-ID')
# Get auto-reply message
ooo_message = recipient_rules.get('ooo_message', 'I am currently unavailable.')
content_type = recipient_rules.get('ooo_content_type', 'text')
# Create auto-reply
if content_type == 'html':
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText as MIMETextPart
reply = MIMEMultipart('alternative')
reply.attach(MIMETextPart(ooo_message, 'plain'))
reply.attach(MIMETextPart(ooo_message, 'html'))
else:
reply = MIMEText(ooo_message, 'plain', 'utf-8')
reply['From'] = recipient_addr
reply['To'] = sender_addr
reply['Subject'] = f"Automatic Reply: {subject}"
reply['Date'] = formatdate(localtime=True)
reply['Message-ID'] = make_msgid()
reply['Auto-Submitted'] = 'auto-replied' # RFC 3834
reply['Precedence'] = 'bulk'
reply['X-Auto-Response-Suppress'] = 'All' # Microsoft Exchange
if message_id:
reply['In-Reply-To'] = message_id
reply['References'] = message_id
# Send via local SMTP
try:
with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp:
smtp.send_message(reply)
# Update throttle timestamp
throttle_key = (recipient_addr.lower(), sender_addr.lower())
AUTOREPLY_SENT[throttle_key] = datetime.now()
logging.info(f"✓ Sent auto-reply from {recipient_addr} to {sender_addr}")
except Exception as e:
logging.error(f"✗ Auto-reply failed: {e}")
def send_forwards(original_msg_bytes, recipient_rules, recipient_addr, sender_addr):
"""Forward email to configured addresses"""
forwards = recipient_rules.get('forwards', [])
if not forwards:
return
for forward_addr in forwards:
try:
# Validate forward address
if '@' not in forward_addr:
logging.warning(f"Invalid forward address: {forward_addr}, skipping")
continue
# Parse message again for clean forwarding
msg = message_from_binary_file(BytesIO(original_msg_bytes))
# Add forwarding headers
msg['X-Forwarded-For'] = recipient_addr
msg['X-Original-To'] = recipient_addr
msg['X-Forwarded-By'] = 'content_filter.py'
# Preserve original sender in envelope
# (This way replies go to original sender, not to recipient)
envelope_sender = sender_addr if sender_addr else recipient_addr
# Send via local SMTP
with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp:
smtp.sendmail(
from_addr=envelope_sender,
to_addrs=[forward_addr],
msg=msg.as_bytes()
)
logging.info(f"✓ Forwarded from {recipient_addr} to {forward_addr}")
except Exception as e:
logging.error(f"✗ Forward to {forward_addr} failed: {e}")
def is_internal_mail(sender, recipient):
"""
Check if this is internal mail (same domain)
This is a safety check in addition to transport_maps filtering
"""
sender_domain = extract_domain(sender)
recipient_domain = extract_domain(recipient)
if not sender_domain or not recipient_domain:
return False
return sender_domain == recipient_domain
def main():
"""Main content filter logic"""
if len(sys.argv) < 3:
logging.error("Usage: content_filter.py <sender> <recipient1> [recipient2] ...")
sys.exit(1)
sender = sys.argv[1]
recipients = sys.argv[2:]
logging.info(f"Processing email from {sender} to {', '.join(recipients)}")
# Read email from stdin
try:
msg_bytes = sys.stdin.buffer.read()
if not msg_bytes:
logging.error("No email data received on stdin")
sys.exit(75) # EX_TEMPFAIL
msg = message_from_binary_file(BytesIO(msg_bytes))
except Exception as e:
logging.error(f"Failed to read email: {e}")
sys.exit(75) # EX_TEMPFAIL
# Process each recipient
processed_count = 0
for recipient in recipients:
try:
# Safety check: Only process internal mail
# (transport_maps should already filter, but defense-in-depth)
if not is_internal_mail(sender, recipient):
logging.debug(f"Skipping external mail: {sender} -> {recipient}")
continue
# Fetch rules from DynamoDB
rules = get_email_rules(recipient)
if rules:
processed_count += 1
# Send auto-reply if configured
if rules.get('ooo_active'):
send_autoreply(msg, rules, recipient)
# Send forwards if configured
if rules.get('forwards'):
send_forwards(msg_bytes, rules, recipient, sender)
else:
logging.debug(f"No rules for {recipient}")
except Exception as e:
logging.error(f"Error processing rules for {recipient}: {e}")
import traceback
logging.error(traceback.format_exc())
if processed_count > 0:
logging.info(f"Processed rules for {processed_count}/{len(recipients)} recipients")
# Re-inject original email for normal delivery
try:
with smtplib.SMTP(REINJECT_HOST, REINJECT_PORT, timeout=30) as smtp:
smtp.sendmail(sender, recipients, msg_bytes)
logging.info(f"✓ Delivered to {', '.join(recipients)}")
sys.exit(0)
except Exception as e:
logging.error(f"✗ Delivery failed: {e}")
import traceback
logging.error(traceback.format_exc())
sys.exit(75) # EX_TEMPFAIL - Postfix will retry
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
logging.info("Interrupted by user")
sys.exit(1)
except Exception as e:
logging.error(f"Fatal error: {e}")
import traceback
logging.error(traceback.format_exc())
sys.exit(75)

View File

@ -24,6 +24,7 @@ services:
- ./docker-data/dms/config/:/tmp/docker-mailserver/
- ./docker-data/dms/config/dovecot/conf.d/95-sieve-redirect.conf:/etc/dovecot/conf.d/95-sieve-redirect.conf:ro
- /etc/localtime:/etc/localtime:ro
- ./sync_dynamodb_to_sieve.py:/scripts/sync.py:ro
environment:
# Wichtig: Rspamd und andere Services deaktivieren für ersten Test
- ENABLE_RSPAMD=0

View File

@ -1 +0,0 @@
# Filter only local/internal deliveries (adjust to your domains)

View File

@ -1,11 +0,0 @@
# Content Filter Configuration
# Routes local/internal mail through content filter for forwarding and auto-reply
# Use transport_maps for selective filtering
# Only internal deliveries go through content filter
# Transport map is auto-generated from postfix-accounts.cf by user-patches.sh
transport_maps = regexp:/etc/postfix/local_transport_maps
# Optional: If you want ALL local deliveries to go through filter (not recommended)
# Uncomment this line and comment out transport_maps above:
# content_filter = smtp:[localhost]:10025

View File

@ -1,32 +0,0 @@
#
# Content Filter Setup
# Two additional SMTP services for content filtering
#
# Port 10025: Content filter input
# Receives mail from main Postfix, passes to content_filter.py
localhost:10025 inet n - n - - smtpd
-o content_filter=
-o local_recipient_maps=
-o relay_recipient_maps=
-o smtpd_restriction_classes=
-o smtpd_client_restrictions=
-o smtpd_helo_restrictions=
-o smtpd_sender_restrictions=
-o smtpd_recipient_restrictions=permit_mynetworks,reject
-o mynetworks=127.0.0.0/8
-o smtpd_authorized_xforward_hosts=127.0.0.0/8
-o receive_override_options=no_unknown_recipient_checks
# Port 10026: Content filter output (re-injection)
# Receives processed mail from content_filter.py for final delivery
localhost:10026 inet n - n - - smtpd
-o content_filter=
-o local_recipient_maps=
-o relay_recipient_maps=
-o smtpd_restriction_classes=
-o smtpd_client_restrictions=
-o smtpd_helo_restrictions=
-o smtpd_sender_restrictions=
-o smtpd_recipient_restrictions=permit_mynetworks,reject
-o mynetworks=127.0.0.0/8,[::1]/128
-o smtpd_authorized_xforward_hosts=127.0.0.0/8,[::1]/128
-o receive_override_options=no_header_body_checks,no_unknown_recipient_checks

View File

@ -1,117 +0,0 @@
#!/bin/bash
# user-patches.sh - Optimized version with dynamic transport_maps generation
set -euo pipefail
CFG_ROOT="/tmp/docker-mailserver"
SRC_DIR="$CFG_ROOT/postfix"
DST_DIR="/etc/postfix"
echo "[user-patches.sh] Starting Postfix customizations..."
# Existing patches (header_checks, etc.)
if [ -f "$SRC_DIR/header_checks" ]; then
install -D -m 0644 "$SRC_DIR/header_checks" "$DST_DIR/header_checks"
echo "[user-patches.sh] ✓ header_checks installed"
fi
if [ -f "$SRC_DIR/smtp_header_checks" ]; then
install -D -m 0644 "$SRC_DIR/smtp_header_checks" "$DST_DIR/maps/sender_header_filter.pcre"
echo "[user-patches.sh] ✓ smtp_header_checks installed"
fi
# NEW: Append content filter configuration to main.cf
if [ -f "$SRC_DIR/main.cf.append" ]; then
echo "[user-patches.sh] Appending content filter config to main.cf..."
cat "$SRC_DIR/main.cf.append" >> "$DST_DIR/main.cf"
echo "[user-patches.sh] ✓ main.cf updated"
else
echo "[user-patches.sh] ⚠ main.cf.append not found, skipping"
fi
# NEW: Append content filter services to master.cf
if [ -f "$SRC_DIR/master.cf.append" ]; then
echo "[user-patches.sh] Appending content filter services to master.cf..."
cat "$SRC_DIR/master.cf.append" >> "$DST_DIR/master.cf"
echo "[user-patches.sh] ✓ master.cf updated"
else
echo "[user-patches.sh] ⚠ master.cf.append not found, skipping"
fi
# NEW: Generate local_transport_maps dynamically from postfix-accounts.cf
echo "[user-patches.sh] Generating local_transport_maps..."
TRANSPORT_MAP="$DST_DIR/local_transport_maps"
ACCOUNTS_FILE="$CFG_ROOT/postfix-accounts.cf"
# Create empty transport map
> "$TRANSPORT_MAP"
if [ -f "$ACCOUNTS_FILE" ]; then
# Extract unique domains from postfix-accounts.cf
# Format of postfix-accounts.cf: user@domain.com|{PLAIN}password
echo "# Auto-generated transport map for content filter" >> "$TRANSPORT_MAP"
echo "# Generated at: $(date)" >> "$TRANSPORT_MAP"
echo "" >> "$TRANSPORT_MAP"
# Extract domains and create regex patterns
awk -F'@|\\|' '{print $2}' "$ACCOUNTS_FILE" | \
sort -u | \
while read -r domain; do
if [ -n "$domain" ]; then
# Escape dots for regex
escaped_domain=$(echo "$domain" | sed 's/\./\\./g')
echo "/^.*@${escaped_domain}\$/ smtp:[localhost]:10025" >> "$TRANSPORT_MAP"
echo "[user-patches.sh] - Added filter for: $domain"
fi
done
# Compile the map
if [ -s "$TRANSPORT_MAP" ]; then
postmap "$TRANSPORT_MAP"
echo "[user-patches.sh] ✓ local_transport_maps created with $(grep -c '^/' "$TRANSPORT_MAP" || echo 0) domains"
else
echo "[user-patches.sh] ⚠ No domains found in $ACCOUNTS_FILE"
fi
else
echo "[user-patches.sh] ⚠ $ACCOUNTS_FILE not found, creating minimal transport_maps"
# Fallback: Create minimal config
cat > "$TRANSPORT_MAP" << 'EOF'
# Minimal transport map - edit manually or populate postfix-accounts.cf
# Format: /^.*@domain\.com$/ smtp:[localhost]:10025
# Example (replace with your domains):
# /^.*@example\.com$/ smtp:[localhost]:10025
# /^.*@another\.com$/ smtp:[localhost]:10025
EOF
postmap "$TRANSPORT_MAP"
fi
# Verify content filter script exists and is executable
if [ -x "/usr/local/bin/content_filter.py" ]; then
echo "[user-patches.sh] ✓ Content filter script found"
# Test Python dependencies
if python3 -c "import boto3" 2>/dev/null; then
echo "[user-patches.sh] ✓ boto3 installed"
else
echo "[user-patches.sh] ⚠ WARNING: boto3 not installed!"
fi
else
echo "[user-patches.sh] ⚠ WARNING: content_filter.py not found or not executable!"
fi
# Create log file if it doesn't exist
if [ ! -f "/var/log/mail/content_filter.log" ]; then
touch /var/log/mail/content_filter.log
chown mail:mail /var/log/mail/content_filter.log
chmod 644 /var/log/mail/content_filter.log
echo "[user-patches.sh] ✓ Created content_filter.log"
fi
echo "[user-patches.sh] Postfix customizations complete"
# Postfix neu laden (nachdem docker-mailserver seine eigene Konfig geladen hat)
postfix reload || true

View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""
sync_dynamodb_to_sieve.py - Sync DynamoDB rules to Dovecot Sieve
"""
import boto3
import os
from pathlib import Path
# Config
REGION = 'us-east-2'
TABLE = 'email-rules'
VMAIL_BASE = '/var/mail'
dynamodb = boto3.resource('dynamodb', region_name=REGION)
table = dynamodb.Table(TABLE)
def generate_sieve(email, rules):
"""Generate Sieve script from DynamoDB rules"""
script = [
'require ["copy","vacation","variables"];',
'',
'# Skip if already processed by worker',
'if header :contains "X-SES-Worker-Processed" "" {',
' keep;',
' stop;',
'}',
''
]
# Forwards
forwards = rules.get('forwards', [])
if forwards:
script.append('# rule:[forward]')
for fwd in forwards:
script.append(f'redirect :copy "{fwd}";')
# OOO
if rules.get('ooo_active'):
msg = rules.get('ooo_message', 'I am away')
script.append('# rule:[reply]')
script.append(f'vacation :days 1 :from "{email}" "{msg}";')
return '\n'.join(script)
def sync():
"""Sync all rules from DynamoDB to Sieve"""
response = table.scan()
for item in response.get('Items', []):
email = item['email_address']
domain = email.split('@')[1]
user = email.split('@')[0]
# Path: /var/mail/domain.de/user/.dovecot.sieve
sieve_path = Path(VMAIL_BASE) / domain / user / '.dovecot.sieve'
sieve_path.parent.mkdir(parents=True, exist_ok=True)
# Generate & write
script = generate_sieve(email, item)
sieve_path.write_text(script)
# Compile
os.system(f'sievec {sieve_path}')
print(f'{email}')
if __name__ == '__main__':
sync()