diff --git a/export_emails.py b/export_emails.py index 962d402..7d9be2b 100644 --- a/export_emails.py +++ b/export_emails.py @@ -37,6 +37,9 @@ DOMAINS = [ "ulgss.ru", "ulgov.ru", "ulgkh.ru", "dgi.ru", "mos.ru", "pfrf.ru", "fsin.gov.ru", "skrf.ru", "epp.genproc.gov.ru", "ugpr.ru", "fas.gov.ru", "minstroyrf.gov.ru", "gkh.ru", "sudrf.ru", + "msud.ru", "arbitr.ru", "fssp.gov.ru", "fssprus.ru", "mirsud", + "msud73.ru", "mirsud73.ru", "court.ru", "sud.ru", "vkks.ru", + "uloblsud.ru", "sledcom.ru" ] # From this date @@ -331,60 +334,74 @@ def main(): # Connect log.info("Connecting to Gmail IMAP...") try: - mailbox = MailBox(IMAP_SERVER).login(imap_user, imap_pass, initial_folder=GMAIL_FOLDER) + mailbox = MailBox(IMAP_SERVER).login(imap_user, imap_pass) log.info("Connected successfully!") except Exception as e: log.error(f"Failed to connect: {e}") sys.exit(1) - # Phase 1: Fetch headers only to find matching emails - log.info("Phase 1: Scanning emails (headers only) to find matches...") - scan_start = time.time() + # Auto-detect folder names + all_mail_folder = "[Gmail]/Вся почта" + spam_folder = "[Gmail]/Спам" + try: + for folder in mailbox.folder.list(): + if r'\All' in folder.flags: + all_mail_folder = folder.name + elif r'\Junk' in folder.flags: + spam_folder = folder.name + except Exception as e: + log.warning(f"Failed to auto-detect folder names, using defaults: {e}") + folders_to_scan = [all_mail_folder, spam_folder] matching_uids = [] total_scanned = 0 + + # Phase 1: Fetch headers from all target folders + log.info("Phase 1: Scanning emails (headers only) to find matches...") + scan_start = time.time() last_log_time = time.time() - # Fetch headers only first — much faster - for msg in mailbox.fetch( - AND(date_gte=SINCE_DATE), - bulk=True, - headers_only=True, - mark_seen=False, - ): - total_scanned += 1 + for folder_name in folders_to_scan: + log.info(f"Scanning folder: {folder_name}") + try: + mailbox.folder.set(folder_name) + except Exception as e: + log.error(f"Failed to set folder {folder_name}: {e}") + continue - # Log every 10 seconds - now = time.time() - if now - last_log_time > 10: - elapsed = now - scan_start - rate = total_scanned / elapsed if elapsed > 0 else 0 - log.info(f" Scanned: {total_scanned} ({rate:.0f}/sec), Matching so far: {len(matching_uids)}") - last_log_time = now + for msg in mailbox.fetch( + AND(date_gte=SINCE_DATE), + bulk=True, + headers_only=True, + mark_seen=False, + ): + total_scanned += 1 + now = time.time() + if now - last_log_time > 10: + elapsed = now - scan_start + rate = total_scanned / elapsed if elapsed > 0 else 0 + log.info(f" Scanned in {folder_name}: {total_scanned} ({rate:.0f}/sec), Matching so far: {len(matching_uids)}") + last_log_time = now - # Check if matches any domain - from_match = matches_domain(msg.from_, DOMAINS) - to_match = None - if not from_match: - for to_addr in (msg.to or []): - to_match = matches_domain(to_addr, DOMAINS) - if to_match: - break + # Check if matches any domain + from_match = matches_domain(msg.from_, DOMAINS) + to_match = None + if not from_match: + for to_addr in (msg.to or []): + to_match = matches_domain(to_addr, DOMAINS) + if to_match: + break - if from_match or to_match: - # Also verify date since IMAP date filtering can be imprecise - msg_date = msg.date - if msg_date and hasattr(msg_date, 'date'): - msg_date = msg_date.date() - elif isinstance(msg_date, datetime): - pass # ok - - if not msg_date or msg_date >= SINCE_DATE: - matching_uids.append(msg.uid) + if from_match or to_match: + msg_date = msg.date + if msg_date and hasattr(msg_date, 'date'): + msg_date = msg_date.date() + if not msg_date or msg_date >= SINCE_DATE: + matching_uids.append((msg.uid, folder_name)) scan_elapsed = time.time() - scan_start log.info(f"Phase 1 complete: scanned {total_scanned} emails in {scan_elapsed/60:.1f}min") - log.info(f"Found {len(matching_uids)} matching emails") + log.info(f"Found {len(matching_uids)} matching emails across folders") if not matching_uids: log.info("No matching emails found. Exiting.") @@ -400,13 +417,16 @@ def main(): total_processed = 0 total_errors = 0 - def reconnect_mailbox(): - """Reconnect to IMAP with retry.""" + current_folder = "" + + def reconnect_mailbox(target_folder): + """Reconnect to IMAP with retry and set folder.""" for attempt in range(5): try: log.info(f"Reconnecting... attempt {attempt+1}") - mb = MailBox(IMAP_SERVER).login(imap_user, imap_pass, initial_folder=GMAIL_FOLDER) - log.info("Reconnected!") + mb = MailBox(IMAP_SERVER).login(imap_user, imap_pass) + mb.folder.set(target_folder) + log.info(f"Reconnected and folder set to {target_folder}!") return mb except Exception as e: log.warning(f"Reconnect failed: {e}") @@ -414,30 +434,35 @@ def main(): log.error("Failed to reconnect after 5 attempts") sys.exit(1) - for idx, uid in enumerate(matching_uids, 1): - if str(uid) in processed: - log.info(f"[{idx}/{len(matching_uids)}] SKIP (already processed) UID={uid}") + for idx, (uid, f_name) in enumerate(matching_uids, 1): + if current_folder != f_name: + try: + mailbox.folder.set(f_name) + current_folder = f_name + except Exception as e: + log.error(f"Failed to switch to folder {f_name}: {e}") + continue + + progress_key = f"{f_name}:{uid}" if f_name == spam_folder else str(uid) + if progress_key in processed or str(uid) in processed: + log.info(f"[{idx}/{len(matching_uids)}] SKIP (already processed) UID={uid} in {f_name}") continue elapsed = time.time() - start_time - if idx > 1: - eta = (elapsed / (idx - 1)) * (len(matching_uids) - idx + 1) - else: - eta = 0 + eta = (elapsed / (idx - 1)) * (len(matching_uids) - idx + 1) if idx > 1 else 0 - log.info(f"[{idx}/{len(matching_uids)}] Downloading UID={uid} | ETA: {eta/60:.1f}m") + log.info(f"[{idx}/{len(matching_uids)}] Downloading UID={uid} from {f_name} | ETA: {eta/60:.1f}m") try: - # Fetch full message msgs = list(mailbox.fetch( AND(uid=uid), headers_only=False, mark_seen=False, )) if not msgs: - log.warning(f" Could not fetch UID={uid}") + log.warning(f" Could not fetch UID={uid} in {f_name}") total_errors += 1 - save_progress(uid, processed) + save_progress(progress_key, processed) continue msg = msgs[0] @@ -514,20 +539,20 @@ def main(): total_errors += 1 # Save progress - save_progress(uid, processed) + save_progress(progress_key, processed) except Exception as e: # Check if it's a socket error — try to reconnect if 'socket error' in str(e) or 'EOF' in str(e) or 'abort' in str(e): log.warning(f"Connection lost, trying to reconnect...") - mailbox = reconnect_mailbox() - # Don't mark as processed, retry next time + mailbox = reconnect_mailbox(f_name) + current_folder = f_name total_errors += 1 continue else: - log.error(f" ERROR processing UID={uid}: {e}", exc_info=True) + log.error(f" ERROR processing UID={uid} in {f_name}: {e}", exc_info=True) total_errors += 1 - save_progress(uid, processed) + save_progress(progress_key, processed) mailbox.logout() @@ -566,9 +591,8 @@ def main(): "text": "📬 ВАЖНОЕ ОБНОВЛЕНИЕ ПОЧТЫ!\n" + summary_text + "\n🚀 Пора обновить источники в NotebookLM!", "parse_mode": "HTML" } - # Use proxy if needed - proxies = {"https": "socks5h://127.0.0.1:10808"} - requests.post(url, json=params, proxies=proxies, timeout=20) + # Send directly (NanoPi routes to VPN transparently) + requests.post(url, json=params, timeout=20) log.info("✅ Telegram notification sent.") except Exception as e: log.error(f"❌ Failed to send Telegram notification: {e}") diff --git a/master_ai_sync.py b/master_ai_sync.py new file mode 100755 index 0000000..ebc252d --- /dev/null +++ b/master_ai_sync.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +import subprocess +import os +import sys +import logging +import requests +from pathlib import Path +from datetime import datetime + +# === CONFIG === +BASE_DIR = Path("/home/matrixhasyou/mutt") +PYTHON_BIN = "/usr/bin/python3" +GOOGLE_PYTHON_BIN = "/home/matrixhasyou/swarm-services/domovoy-bot/venv-google/bin/python" +EXPORT_SCRIPT = BASE_DIR / "export_emails.py" +CONSOLIDATE_SCRIPT = BASE_DIR / "consolidate_to_mega_pdf.py" +DRIVE_SYNC_SCRIPT = Path("/home/matrixhasyou/domovoy_drive_sync.py") +MASTER_PDF = BASE_DIR / "ALL_EMAILS_CONSOLIDATED.pdf" + +BOT_TOKEN = "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M" +CHAT_ID = "197957361" + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +log = logging.getLogger("MasterSync") + +def send_tg(text): + url = f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage" + try: + requests.post(url, json={"chat_id": CHAT_ID, "text": text, "parse_mode": "HTML"}, + timeout=20) + except Exception as e: + log.error(f"TG notification failed: {e}") + +def run_step(name, cmd): + log.info(f"--- Step: {name} ---") + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + log.error(f"Step {name} FAILED!") + log.error(result.stderr) + return False, result.stderr + log.info(f"Step {name} completed.") + return True, result.stdout + +def main(): + start_time = datetime.now() + log.info(f"Starting Master Sync at {start_time}") + + # 1. Export Emails + ok, out_export = run_step("Export Emails", [PYTHON_BIN, str(EXPORT_SCRIPT)]) + if not ok: + send_tg(f"❌ Mutt Master Sync: FAILED\nStep: Export Emails\nError: {out_export[:200]}") + return + + # 2. Consolidate PDF + ok, out_consolidate = run_step("Consolidate PDF", [PYTHON_BIN, str(CONSOLIDATE_SCRIPT)]) + if not ok: + send_tg(f"❌ Mutt Master Sync: FAILED\nStep: Consolidate PDF\nError: {out_consolidate[:200]}") + return + + # 3. Sync to G-Drive + ok, out_drive = run_step("G-Drive Sync", [GOOGLE_PYTHON_BIN, str(DRIVE_SYNC_SCRIPT), str(MASTER_PDF)]) + if not ok: + send_tg(f"❌ Mutt Master Sync: FAILED\nStep: G-Drive Sync\nError: {out_drive[:200]}") + return + + end_time = datetime.now() + + # Parsing Stats + new_emails = 0 + total_emails = 0 + for line in out_export.split('\n'): + if "Processed this run:" in line: + try: new_emails = int(line.split(':')[-1].strip()) + except: pass + + for line in out_consolidate.split('\n'): + if "Found" in line and "PDFs" in line: + try: total_emails = int(line.split('Found')[-1].split('PDFs')[0].strip()) + except: pass + + pdf_size_mb = os.path.getsize(MASTER_PDF) / (1024 * 1024) + + summary = f""" +🚀 AI ПОЧТОВЫЙ ТУРБО-СИНХРОН: OK 🚀 + +📥 Новых писем: {new_emails} +📚 Всего в архиве: {total_emails} +🐘 Размер PDF: {pdf_size_mb:.2f} МБ +☁️ G-Drive: Обновлено ✅ + +📅 {end_time.strftime("%Y-%m-%d %H:%M")} +Робот закончил работу. NotebookLM готов к анализу. +""" + send_tg(summary) + + log.info("Master Sync completed successfully.") + + # Save last sync time for web UI + sync_time_file = Path("/home/matrixhasyou/mutt/last_ai_sync.txt") + with open(sync_time_file, "w") as f: + f.write(end_time.strftime("%Y-%m-%d %H:%M:%S")) + +if __name__ == "__main__": + main()