#!/usr/bin/env python3
"""
Gmail IMAP email exporter v3 — optimized.
Single IMAP query for all emails since Dec 20, 2025, then filter by domains in Python.
Each email becomes a PDF with body + attachments merged.
Organized by domain and also in a flat 'all' folder.
"""
import os
import sys
import re
import time
import logging
from pathlib import Path
from datetime import datetime, date
from email.utils import parsedate_to_datetime
from imap_tools import MailBox, AND
from weasyprint import HTML
from fpdf import FPDF
import fitz # PyMuPDF
# === CONFIG ===
IMAP_SERVER = "imap.gmail.com"
# Извлекаем учётку из .muttrc (сначала в проекте, потом в home)
MUTT_CONF = Path(__file__).parent / ".muttrc"
if not MUTT_CONF.exists():
MUTT_CONF = Path.home() / ".muttrc"
# Domains to search
DOMAINS = [
"gosuslugi.ru", "dom.gosuslugi.ru", "gov.ru", "duma.gov.ru",
"mvd.gov.ru", "mvd.ru", "rkn.gov.ru", "nalog.gov.ru", "nalog.ru",
"genproc.gov.ru", "pfr.gov.ru", "fss.ru", "rospotrebnadzor.ru",
"russianpost.ru", "mailop.ru", "73.mailop.ru", "uk-service.ru",
"ulgss.ru", "ulgov.ru", "ulgkh.ru", "dgi.ru", "mos.ru", "pfrf.ru",
"fsin.gov.ru", "skrf.ru", "epp.genproc.gov.ru", "ugpr.ru",
"fas.gov.ru", "minstroyrf.gov.ru", "gkh.ru", "sudrf.ru",
]
# From this date
SINCE_DATE = date(2025, 12, 20)
# Gmail folder to search
GMAIL_FOLDER = "[Gmail]/Вся почта"
# Output
BASE_OUTPUT = Path(__file__).parent / "exported_emails"
ALL_FOLDER = BASE_OUTPUT / "all"
# Progress tracking
PROGRESS_FILE = Path(__file__).parent / ".export_progress"
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler(Path(__file__).parent / "export.log", encoding="utf-8"),
logging.StreamHandler(sys.stdout),
],
)
log = logging.getLogger(__name__)
def load_progress():
"""Load set of already processed UIDs."""
if PROGRESS_FILE.exists():
return set(PROGRESS_FILE.read_text().strip().splitlines())
return set()
def save_progress(uid, processed_set):
"""Save progress after each email."""
processed_set.add(str(uid))
PROGRESS_FILE.write_text("\n".join(processed_set))
def sanitize_filename(name, max_len=80):
"""Make filename safe."""
name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', name)
name = re.sub(r'\s+', ' ', name).strip()
return name[:max_len]
def parse_mutt_config():
"""Extract IMAP user and password from .muttrc."""
config = {}
with open(MUTT_CONF) as f:
for line in f:
line = line.strip()
if line.startswith('#') or not line:
continue
match = re.match(r'set\s+(\w+)\s*=\s*"([^"]*)"', line)
if not match:
match = re.match(r'set\s+(\w+)\s*=\s*(\S+)', line)
if match:
key, val = match.group(1), match.group(2)
config[key] = val
return config
def html_to_pdf_stream(html_content):
"""Convert HTML to PDF bytes using weasyprint, fallback to fpdf."""
try:
html_doc = HTML(string=html_content)
return html_doc.write_pdf()
except Exception as e:
log.warning(f"weasyprint failed, using fpdf fallback: {e}")
return html_to_pdf_fpdf(html_content)
def html_to_pdf_fpdf(html_content):
"""Fallback: strip HTML and use FPDF."""
from bs4 import BeautifulSoup
soup = BeautifulSoup(html_content, 'lxml')
text = soup.get_text(separator='\n')
pdf = FPDF()
pdf.add_page()
pdf.add_font('DejaVu', '', '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', uni=True)
pdf.add_font('DejaVu', 'B', '/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', uni=True)
pdf.set_font('DejaVu', '', 10)
lines = [line for line in text.split('\n') if line.strip()]
for line in lines:
pdf.multi_cell(0, 6, line)
pdf.ln(1)
return pdf.output()
def convert_attachment_to_pdf(attachment_data, filename, content_type):
"""Convert an attachment to PDF."""
ext = Path(filename).suffix.lower()
try:
if ext in ('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg'):
pdf = FPDF()
pdf.add_page()
img_w, img_h = get_image_dims(attachment_data)
page_w = pdf.w - 20
scale = min(page_w / img_w, (pdf.h - 20) / img_h, 1.0)
w, h = img_w * scale, img_h * scale
import tempfile
with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp:
tmp.write(attachment_data)
tmp_path = tmp.name
try:
pdf.image(tmp_path, x=10, y=10, w=w, h=h)
finally:
os.unlink(tmp_path)
return pdf.output()
elif ext in ('.doc', '.docx'):
import tempfile, subprocess
with tempfile.TemporaryDirectory() as tmpdir:
src = Path(tmpdir) / filename
src.write_bytes(attachment_data)
subprocess.run(
['libreoffice', '--headless', '--convert-to', 'pdf',
'--outdir', tmpdir, str(src)],
capture_output=True, timeout=120
)
pdf_path = src.with_suffix('.pdf')
if pdf_path.exists():
return pdf_path.read_bytes()
elif ext == '.pdf':
return attachment_data
elif ext in ('.xls', '.xlsx'):
import tempfile, subprocess
with tempfile.TemporaryDirectory() as tmpdir:
src = Path(tmpdir) / filename
src.write_bytes(attachment_data)
subprocess.run(
['libreoffice', '--headless', '--convert-to', 'pdf',
'--outdir', tmpdir, str(src)],
capture_output=True, timeout=120
)
pdf_path = src.with_suffix('.pdf')
if pdf_path.exists():
return pdf_path.read_bytes()
elif ext in ('.txt', '.csv', '.log', '.xml', '.html', '.htm'):
try:
text = attachment_data.decode('utf-8', errors='replace')
except:
text = attachment_data.decode('latin-1', errors='replace')
return html_to_pdf_fpdf(text)
else:
info = f"Attachment: {filename}\nType: {content_type}\nSize: {len(attachment_data)} bytes\n\n[Binary content - not converted]"
return html_to_pdf_fpdf(info)
except Exception as e:
log.warning(f"Failed to convert {filename}: {e}")
info = f"Attachment: {filename}\nType: {content_type}\nSize: {len(attachment_data)} bytes\n\nConversion failed: {e}"
return html_to_pdf_fpdf(info)
def get_image_dims(data):
"""Get image dimensions."""
import io
from PIL import Image
img = Image.open(io.BytesIO(data))
w_px, h_px = img.size
w_mm = w_px * 25.4 / 96
h_mm = h_px * 25.4 / 96
return w_mm, h_mm
def merge_pdfs(pdf_bytes_list):
"""Merge multiple PDFs into one."""
if not pdf_bytes_list:
return None
if len(pdf_bytes_list) == 1:
return pdf_bytes_list[0]
merger = fitz.open()
for pdf_bytes in pdf_bytes_list:
try:
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
merger.insert_pdf(doc)
except Exception as e:
log.warning(f"Failed to merge a PDF section: {e}")
output = merger.tobytes()
merger.close()
return output
def build_email_html(msg):
"""Build nice HTML representation of email with headers."""
subject = msg.subject or "(Без темы)"
from_addr = msg.from_ or "unknown"
to_addr = ", ".join(msg.to) if msg.to else "unknown"
cc_addr = ", ".join(msg.cc) if msg.cc else ""
date_str = msg.date_str or str(msg.date) if msg.date else "unknown"
body_html = msg.html
if not body_html and msg.text:
text = msg.text
text_escaped = text.replace('&', '&').replace('<', '<').replace('>', '>')
body_html = text_escaped.replace('\n', '
\n')
elif not body_html:
body_html = "(Нет тела письма)
"
attachments_info = ""
if msg.attachments:
att_items = []
for att in msg.attachments:
att_items.append(f'{att.filename} ({att.content_type}, {att.size} bytes)')
attachments_info = 'Вложения:
' + ''.join(att_items) + '
'
html = f"""
{body_html}
{f'{attachments_info}
' if attachments_info else ''}
"""
return html
def matches_domain(addr, domains):
"""Check if address matches any domain. Returns matched domain or None."""
if not addr:
return None
addr_lower = addr.lower()
for domain in domains:
if domain.lower() in addr_lower:
return domain
return None
def main():
log.info("=" * 60)
log.info("Gmail Email Exporter v3 — starting")
log.info("=" * 60)
# Parse config
config = parse_mutt_config()
imap_user = config.get('imap_user', '')
imap_pass = config.get('imap_pass', '')
if not imap_user or imap_user == 'your.email@gmail.com':
log.error("ERROR: Не настроены учётные данные в .muttrc!")
log.error("Укажите реальный imap_user и imap_pass (App Password)")
sys.exit(1)
log.info(f"IMAP User: {imap_user}")
log.info(f"Domains: {len(DOMAINS)} domains")
log.info(f"Since: {SINCE_DATE.strftime('%Y-%m-%d')}")
# Create output dirs
BASE_OUTPUT.mkdir(parents=True, exist_ok=True)
ALL_FOLDER.mkdir(parents=True, exist_ok=True)
# Load progress
processed = load_progress()
log.info(f"Previously processed: {len(processed)} emails")
# Connect
log.info("Connecting to Gmail IMAP...")
try:
mailbox = MailBox(IMAP_SERVER).login(imap_user, imap_pass, initial_folder=GMAIL_FOLDER)
log.info("Connected successfully!")
except Exception as e:
log.error(f"Failed to connect: {e}")
sys.exit(1)
# Phase 1: Fetch headers only to find matching emails
log.info("Phase 1: Scanning emails (headers only) to find matches...")
scan_start = time.time()
matching_uids = []
total_scanned = 0
last_log_time = time.time()
# Fetch headers only first — much faster
for msg in mailbox.fetch(
AND(date_gte=SINCE_DATE),
bulk=True,
headers_only=True,
mark_seen=False,
):
total_scanned += 1
# Log every 10 seconds
now = time.time()
if now - last_log_time > 10:
elapsed = now - scan_start
rate = total_scanned / elapsed if elapsed > 0 else 0
log.info(f" Scanned: {total_scanned} ({rate:.0f}/sec), Matching so far: {len(matching_uids)}")
last_log_time = now
# Check if matches any domain
from_match = matches_domain(msg.from_, DOMAINS)
to_match = None
if not from_match:
for to_addr in (msg.to or []):
to_match = matches_domain(to_addr, DOMAINS)
if to_match:
break
if from_match or to_match:
# Also verify date since IMAP date filtering can be imprecise
msg_date = msg.date
if msg_date and hasattr(msg_date, 'date'):
msg_date = msg_date.date()
elif isinstance(msg_date, datetime):
pass # ok
if not msg_date or msg_date >= SINCE_DATE:
matching_uids.append(msg.uid)
scan_elapsed = time.time() - scan_start
log.info(f"Phase 1 complete: scanned {total_scanned} emails in {scan_elapsed/60:.1f}min")
log.info(f"Found {len(matching_uids)} matching emails")
if not matching_uids:
log.info("No matching emails found. Exiting.")
mailbox.logout()
return
# Phase 2: Download full emails and convert to PDF
log.info("\n" + "=" * 60)
log.info("Phase 2: Downloading full emails and converting to PDF...")
log.info("=" * 60)
start_time = time.time()
total_processed = 0
total_errors = 0
def reconnect_mailbox():
"""Reconnect to IMAP with retry."""
for attempt in range(5):
try:
log.info(f"Reconnecting... attempt {attempt+1}")
mb = MailBox(IMAP_SERVER).login(imap_user, imap_pass, initial_folder=GMAIL_FOLDER)
log.info("Reconnected!")
return mb
except Exception as e:
log.warning(f"Reconnect failed: {e}")
time.sleep(10 * (attempt + 1))
log.error("Failed to reconnect after 5 attempts")
sys.exit(1)
for idx, uid in enumerate(matching_uids, 1):
if str(uid) in processed:
log.info(f"[{idx}/{len(matching_uids)}] SKIP (already processed) UID={uid}")
continue
elapsed = time.time() - start_time
if idx > 1:
eta = (elapsed / (idx - 1)) * (len(matching_uids) - idx + 1)
else:
eta = 0
log.info(f"[{idx}/{len(matching_uids)}] Downloading UID={uid} | ETA: {eta/60:.1f}m")
try:
# Fetch full message
msgs = list(mailbox.fetch(
AND(uid=uid),
headers_only=False,
mark_seen=False,
))
if not msgs:
log.warning(f" Could not fetch UID={uid}")
total_errors += 1
save_progress(uid, processed)
continue
msg = msgs[0]
# Build email info
subject = msg.subject or "(Без темы)"
from_addr = msg.from_ or "unknown"
date_str = msg.date_str or "unknown"
# Determine matching domain
match_domain = matches_domain(from_addr, DOMAINS)
if not match_domain:
for to_addr in (msg.to or []):
match_domain = matches_domain(to_addr, DOMAINS)
if match_domain:
break
if not match_domain:
match_domain = "other"
safe_subject = sanitize_filename(subject)
safe_from = sanitize_filename(from_addr.split('<')[-1].rstrip('>').split('@')[0], 30)
date_part = msg.date.strftime('%Y-%m-%d') if msg.date else 'unknown'
filename = f"{date_part}_{safe_from}_{safe_subject}.pdf"
if len(filename) > 150:
filename = filename[:150] + ".pdf"
# Build HTML for email body
html_content = build_email_html(msg)
pdf_parts = []
# Convert body to PDF
body_pdf = html_to_pdf_stream(html_content)
if body_pdf:
pdf_parts.append(body_pdf)
# Convert attachments to PDF
if msg.attachments:
log.info(f" Has {len(msg.attachments)} attachment(s)")
for att in msg.attachments:
try:
att_pdf = convert_attachment_to_pdf(
att.payload, att.filename, att.content_type
)
if att_pdf:
pdf_parts.append(att_pdf)
except Exception as e:
log.warning(f" Failed to convert attachment {att.filename}: {e}")
# Merge all PDFs
if pdf_parts:
merged_pdf = merge_pdfs(pdf_parts)
if merged_pdf:
# Save to domain folder
domain_folder = BASE_OUTPUT / sanitize_filename(match_domain, 50)
domain_folder.mkdir(parents=True, exist_ok=True)
filepath_domain = domain_folder / filename
filepath_domain.write_bytes(merged_pdf)
# Save to all folder
filepath_all = ALL_FOLDER / f"{match_domain}_{filename}"
filepath_all.write_bytes(merged_pdf)
log.info(f" SAVED: {filename}")
total_processed += 1
else:
log.warning(f" Failed to create merged PDF")
total_errors += 1
else:
log.warning(f" No content to convert")
total_errors += 1
# Save progress
save_progress(uid, processed)
except Exception as e:
# Check if it's a socket error — try to reconnect
if 'socket error' in str(e) or 'EOF' in str(e) or 'abort' in str(e):
log.warning(f"Connection lost, trying to reconnect...")
nonlocal mailbox
mailbox = reconnect_mailbox()
# Don't mark as processed, retry next time
total_errors += 1
continue
else:
log.error(f" ERROR processing UID={uid}: {e}", exc_info=True)
total_errors += 1
save_progress(uid, processed)
mailbox.logout()
# Summary
total_time = time.time() - start_time
log.info("\n" + "=" * 60)
log.info("EXPORT COMPLETE!")
log.info(f"Total scanned: {total_scanned}")
log.info(f"Matching: {len(matching_uids)}")
log.info(f"Processed this run: {total_processed}")
log.info(f"Errors: {total_errors}")
log.info(f"Total time: {total_time/60:.1f} minutes")
log.info(f"Output: {BASE_OUTPUT}")
domain_count = len([d for d in BASE_OUTPUT.iterdir() if d.is_dir() and d != ALL_FOLDER])
log.info(f" Domain folders: {domain_count}")
log.info(f" All folder: {len(list(ALL_FOLDER.glob('*.pdf')))} files")
log.info("=" * 60)
if __name__ == '__main__':
main()