283 lines
9.6 KiB
Python
283 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Gmail Email Auditor v2 for Domovoy Bot.
|
|
1. Exports emails from specific domains as PDFs.
|
|
2. Merges all PDFs into a single master archive for NotebookLM.
|
|
3. Updates domovoy.db statistics.
|
|
4. Sends Telegram notifications on new emails.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import time
|
|
import logging
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from datetime import datetime, date
|
|
import requests
|
|
|
|
from imap_tools import MailBox, AND
|
|
from weasyprint import HTML
|
|
from fpdf import FPDF
|
|
import fitz # PyMuPDF
|
|
from dotenv import load_dotenv
|
|
|
|
# === CONFIG & PATHS ===
|
|
BASE_DIR = Path(__file__).resolve().parent
|
|
PROJECT_ROOT = Path("/app")
|
|
load_dotenv(PROJECT_ROOT / ".env")
|
|
|
|
DB_PATH = PROJECT_ROOT / "database" / "domovoy.db"
|
|
MUTT_CONF = BASE_DIR / ".muttrc"
|
|
EXPORT_DIR = BASE_DIR / "exported_emails"
|
|
ALL_FOLDER = EXPORT_DIR / "all"
|
|
MASTER_PDF = EXPORT_DIR / "TOTAL_ARCHIVE_2025-2026.pdf"
|
|
PROGRESS_FILE = BASE_DIR / ".export_progress"
|
|
|
|
IMAP_SERVER = "imap.gmail.com"
|
|
GMAIL_FOLDER = "[Gmail]/Вся почта"
|
|
SINCE_DATE = date(2025, 12, 20)
|
|
|
|
DOMAINS = [
|
|
"gosuslugi.ru", "dom.gosuslugi.ru", "gov.ru", "duma.gov.ru",
|
|
"mvd.gov.ru", "mvd.ru", "rkn.gov.ru", "nalog.gov.ru", "nalog.ru",
|
|
"genproc.gov.ru", "pfr.gov.ru", "fss.ru", "rospotrebnadzor.ru",
|
|
"russianpost.ru", "mailop.ru", "73.mailop.ru", "uk-service.ru",
|
|
"ulgss.ru", "ulgov.ru", "ulgkh.ru", "dgi.ru", "mos.ru", "pfrf.ru",
|
|
"fsin.gov.ru", "skrf.ru", "epp.genproc.gov.ru", "ugpr.ru",
|
|
"fas.gov.ru", "minstroyrf.gov.ru", "gkh.ru", "sudrf.ru",
|
|
]
|
|
|
|
# Bot notification config
|
|
BOT_TOKEN = os.getenv("BOT_TOKEN")
|
|
ADMIN_USER_ID = os.getenv("ADMIN_USER_ID")
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
handlers=[
|
|
logging.FileHandler(BASE_DIR / "auditor_run.log", encoding="utf-8"),
|
|
logging.StreamHandler(sys.stdout),
|
|
],
|
|
)
|
|
log = logging.getLogger("EmailAuditor")
|
|
|
|
# === UTILS ===
|
|
|
|
def parse_mutt_config():
|
|
config = {}
|
|
if not MUTT_CONF.exists():
|
|
return config
|
|
with open(MUTT_CONF) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith('#'): continue
|
|
match = re.match(r'set\s+(\w+)\s*=\s*"([^"]*)"', line)
|
|
if match: config[match.group(1)] = match.group(2)
|
|
return config
|
|
|
|
def sanitize_filename(name):
|
|
return re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', name)[:80].strip()
|
|
|
|
def send_bot_message(text):
|
|
if not BOT_TOKEN or not ADMIN_USER_ID:
|
|
return
|
|
url = f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage"
|
|
try:
|
|
requests.post(url, json={
|
|
"chat_id": ADMIN_USER_ID,
|
|
"text": text,
|
|
"parse_mode": "HTML"
|
|
}, timeout=10)
|
|
except Exception as e:
|
|
log.error(f"Failed to send Telegram notification: {e}")
|
|
|
|
# === DATABASE ===
|
|
|
|
def update_db_stats(results):
|
|
"""Update SQLite statistics based on exported emails."""
|
|
if not DB_PATH.exists():
|
|
log.error(f"Database not found at {DB_PATH}")
|
|
return
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cur = conn.cursor()
|
|
|
|
new_total = 0
|
|
domains_updated = set()
|
|
|
|
for item in results:
|
|
domain = item['domain']
|
|
domains_updated.add(domain)
|
|
is_inbound = item['inbound']
|
|
|
|
# Upsert domain stats
|
|
cur.execute("""
|
|
INSERT INTO email_audit (domain, description, inbound_count, outbound_count, last_email_date, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(domain) DO UPDATE SET
|
|
inbound_count = inbound_count + ?,
|
|
outbound_count = outbound_count + ?,
|
|
last_email_date = MAX(last_email_date, ?),
|
|
updated_at = CURRENT_TIMESTAMP
|
|
""", (domain, domain, 1 if is_inbound else 0, 0 if is_inbound else 1, item['date'],
|
|
1 if is_inbound else 0, 0 if is_inbound else 1, item['date']))
|
|
new_total += 1
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
return new_total
|
|
|
|
# === PDF MERGING ===
|
|
|
|
def merge_all_pdfs():
|
|
"""Merge all PDFs from 'all' folder into one master file, sorted by date."""
|
|
log.info("Merging PDFs into master archive...")
|
|
pdf_files = sorted(list(ALL_FOLDER.glob("*.pdf")))
|
|
if not pdf_files:
|
|
return
|
|
|
|
result = fitz.open()
|
|
for pdf in pdf_files:
|
|
try:
|
|
with fitz.open(pdf) as mfile:
|
|
result.insert_pdf(mfile)
|
|
except Exception as e:
|
|
log.error(f"Failed to merge {pdf.name}: {e}")
|
|
|
|
result.save(MASTER_PDF)
|
|
result.close()
|
|
log.info(f"Master archive created: {MASTER_PDF}")
|
|
|
|
# === CORE LOGIC ===
|
|
|
|
def run_export():
|
|
mutt_cfg = parse_mutt_config()
|
|
user = mutt_cfg.get('imap_user')
|
|
password = mutt_cfg.get('imap_pass')
|
|
|
|
if not user or not password:
|
|
log.error("IMAP credentials not found in .muttrc")
|
|
return
|
|
|
|
EXPORT_DIR.mkdir(exist_ok=True)
|
|
ALL_FOLDER.mkdir(exist_ok=True)
|
|
|
|
# Database log start
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cur = conn.cursor()
|
|
cur.execute("INSERT INTO email_export_logs (status, start_time) VALUES (?, CURRENT_TIMESTAMP)", ('running',))
|
|
log_id = cur.lastrowid
|
|
conn.commit()
|
|
|
|
processed_uids = set()
|
|
if PROGRESS_FILE.exists():
|
|
processed_uids = set(PROGRESS_FILE.read_text().splitlines())
|
|
|
|
new_emails_info = []
|
|
|
|
try:
|
|
with MailBox(IMAP_SERVER).login(user, password) as mailbox:
|
|
# Try to find 'All Mail' folder dynamically
|
|
all_mail_folder = None
|
|
for folder in mailbox.folder.list():
|
|
if r'\All' in folder.flags:
|
|
all_mail_folder = folder.name
|
|
break
|
|
|
|
if not all_mail_folder:
|
|
all_mail_folder = GMAIL_FOLDER # Fallback
|
|
|
|
log.info(f"Using folder: {all_mail_folder}")
|
|
mailbox.folder.set(all_mail_folder)
|
|
|
|
# Search all emails since Dec 20, 2025
|
|
# Using raw IMAP search criteria to avoid library version issues
|
|
since_str = SINCE_DATE.strftime("%d-%b-%Y")
|
|
search_criteria = f'(SINCE "{since_str}")'
|
|
|
|
for msg in mailbox.fetch(search_criteria):
|
|
if str(msg.uid) in processed_uids:
|
|
continue
|
|
|
|
|
|
# Filter by domains (check From, To, Cc and SUBJECT)
|
|
found_domain = None
|
|
from_addr = msg.from_.lower()
|
|
to_addrs = [t.lower() for t in msg.to] if msg.to else []
|
|
cc_addrs = [c.lower() for t in msg.cc] if msg.cc else []
|
|
subject_lower = msg.subject.lower()
|
|
|
|
for d in DOMAINS:
|
|
if d in from_addr or any(d in t for t in to_addrs) or any(d in c for c in cc_addrs) or d in subject_lower:
|
|
found_domain = d
|
|
break
|
|
|
|
if not found_domain:
|
|
continue
|
|
|
|
log.info(f"MATCH FOUND: {msg.date} | {found_domain} | {msg.subject}")
|
|
|
|
# Metadata
|
|
msg_date = msg.date.strftime("%Y-%m-%d")
|
|
is_inbound = user.lower() in [t.lower() for t in msg.to] if msg.to else True
|
|
|
|
# Save as PDF
|
|
safe_subject = sanitize_filename(msg.subject)
|
|
filename = f"{msg_date}_{found_domain}_{safe_subject}.pdf"
|
|
filepath = ALL_FOLDER / filename
|
|
|
|
# Minimal PDF generation (reuse your logic)
|
|
try:
|
|
html_doc = HTML(string=msg.html or msg.text)
|
|
html_doc.write_pdf(filepath)
|
|
except Exception as e:
|
|
log.error(f"PDF creation failed for {msg.uid}: {e}")
|
|
continue
|
|
|
|
new_emails_info.append({
|
|
'uid': msg.uid,
|
|
'domain': found_domain,
|
|
'date': msg_date,
|
|
'inbound': is_inbound
|
|
})
|
|
|
|
# Update progress
|
|
processed_uids.add(str(msg.uid))
|
|
with open(PROGRESS_FILE, 'a') as pf:
|
|
pf.write(f"{msg.uid}\n")
|
|
|
|
except Exception as e:
|
|
log.error(f"IMAP Error: {e}")
|
|
cur.execute("UPDATE email_export_logs SET status = ?, end_time = CURRENT_TIMESTAMP WHERE id = ?", ('error', log_id))
|
|
conn.commit()
|
|
conn.close()
|
|
return
|
|
|
|
if new_emails_info:
|
|
count = update_db_stats(new_emails_info)
|
|
merge_all_pdfs()
|
|
|
|
# Notify with details
|
|
subjects_list = "\n".join([f"• {e['domain']}: {e['subject'][:50]}..." for e in new_emails_info[:5]])
|
|
if len(new_emails_info) > 5:
|
|
subjects_list += f"\n... и еще {len(new_emails_info)-5} писем."
|
|
|
|
msg_text = f"📧 <b>Email Auditor: Обновление</b>\n\nНайдено новых писем: <b>{len(new_emails_info)}</b>\n\n<b>Последние темы:</b>\n{subjects_list}\n\nОбщий архив PDF обновлен и готов к загрузке в NotebookLM."
|
|
send_bot_message(msg_text)
|
|
log.info(f"Export finished. {len(new_emails_info)} new emails processed.")
|
|
else:
|
|
log.info("No new emails found.")
|
|
# Re-merge just in case MASTER_PDF is missing
|
|
if not MASTER_PDF.exists():
|
|
merge_all_pdfs()
|
|
|
|
# Database log finish
|
|
cur.execute("UPDATE email_export_logs SET status = ?, end_time = CURRENT_TIMESTAMP, new_emails_count = ? WHERE id = ?",
|
|
('success', len(new_emails_info), log_id))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
if __name__ == "__main__":
|
|
run_export()
|