diff --git a/scraper_gis/sudrf_pdf_compiler.py b/scraper_gis/sudrf_pdf_compiler.py new file mode 100644 index 0000000..0177cfe --- /dev/null +++ b/scraper_gis/sudrf_pdf_compiler.py @@ -0,0 +1,145 @@ +import os +import sys +import logging +import json +from pathlib import Path +from pypdf import PdfWriter, PdfReader +from reportlab.pdfgen import canvas +from reportlab.lib.pagesizes import A4 +from reportlab.lib import colors +from PIL import Image +import subprocess + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [%(levelname)s] %(message)s', + handlers=[ + logging.FileHandler("/home/matrixhasyou/swarm-services/domovoy-bot/scraper_gis/logs/sudrf_compiler.log", encoding="utf-8"), + logging.StreamHandler(sys.stdout) + ] +) +log = logging.getLogger("SUD-Compiler") + +# Add libs path +sys.path.append("/home/matrixhasyou/swarm-services/libs") +try: + from swarmlib.gdrive import GDriveManager +except ImportError as e: + log.error(f"Cannot import swarmlib: {e}") + GDriveManager = None + +EXPORT_DIR = Path("/home/matrixhasyou/swarm-services/domovoy-bot/scraper_gis/exports/sudrf") +OUTPUT_DIR = Path("/home/matrixhasyou/swarm-services/domovoy-bot/exports/compiled_sudrf") +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + +BOT_TOKEN = "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M" +CHAT_ID = "197957361" + +def send_tg(msg): + subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage", + "-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"]) + +def tag_image_to_pdf(image_path, output_pdf_path, label): + try: + c = canvas.Canvas(output_pdf_path, pagesize=A4) + width, height = A4 + c.setFillColor(colors.black) + c.rect(0, height - 50, width, 50, fill=1) + c.setFillColor(colors.white) + c.setFont("Helvetica-Bold", 10) + c.drawString(20, height - 30, f"SUD: {label}".upper()) + + img = Image.open(image_path) + img_width, img_height = img.size + aspect = img_height / float(img_width) + + display_width = width - 40 + display_height = display_width * aspect + if display_height > (height - 70): + display_height = height - 70 + display_width = display_height / aspect + + c.drawImage(image_path, 20, height - 60 - display_height, width=display_width, height=display_height) + c.showPage() + c.save() + return True + except Exception as e: + log.error(f"Failed to tag image {image_path}: {e}") + return False + +def compile_appeals(): + send_tg("🧬 **ГАС Правосудие: Запуск PDF-компилятора**\nОбрабатываю выгруженные материалы...") + + if not EXPORT_DIR.exists(): + log.error(f"Export dir not found: {EXPORT_DIR}") + send_tg("❌ **Компиляция прервана**: папка с выгрузками не найдена.") + return + + gdrive = None + if GDriveManager: + try: + gdrive = GDriveManager() + except Exception as e: + log.error(f"Failed to initialize GDriveManager: {e}") + + folders = [d for d in EXPORT_DIR.iterdir() if d.is_dir() and d.name.startswith("appeal_")] + processed = 0 + + for folder in folders: + appeal_id = folder.name.replace("appeal_", "") + log.info(f"Compiling appeal: {appeal_id}") + + writer = PdfWriter() + + # 1. Convert detail screenshot (evidence.png) to PDF first + evidence_img = folder / "evidence.png" + evidence_pdf = folder / "evidence.pdf" + if evidence_img.exists(): + if tag_image_to_pdf(str(evidence_img), str(evidence_pdf), f"APPEAL {appeal_id}"): + writer.append(str(evidence_pdf)) + + # 2. Append all other PDF files inside the folder + files = sorted(list(folder.iterdir())) + for f in files: + if f.suffix.lower() == ".pdf" and f.name != "evidence.pdf": + try: + writer.append(str(f)) + log.info(f"Appended PDF: {f.name}") + except Exception as e: + log.error(f"Failed to append PDF {f.name}: {e}") + elif f.suffix.lower() in [".png", ".jpg", ".jpeg"] and f.name != "evidence.png": + # Convert image attachments to PDF and append + img_pdf = folder / f"{f.stem}_attachment.pdf" + if tag_image_to_pdf(str(f), str(img_pdf), f"ATTACHMENT {f.name}"): + try: + writer.append(str(img_pdf)) + log.info(f"Appended converted image: {f.name}") + except Exception as e: + log.error(f"Failed to append converted image {f.name}: {e}") + + output_pdf = OUTPUT_DIR / f"SUD_APPEAL_{appeal_id}.pdf" + try: + with open(output_pdf, "wb") as out_f: + writer.write(out_f) + processed += 1 + log.info(f"Successfully compiled: {output_pdf.name}") + + # Upload to GDrive + drive_status = "не настроен" + if gdrive: + try: + drive_status = gdrive.upload_or_update(str(output_pdf), output_pdf.name) + except Exception as e: + drive_status = f"ошибка GDrive: {e}" + + send_tg(f"📄 **Обращение {appeal_id} собрано в PDF**\n☁️ GDrive: `{drive_status}`") + + except Exception as e: + log.error(f"Failed to write compiled PDF for {appeal_id}: {e}") + send_tg(f"❌ **Ошибка сборки обращения {appeal_id}**: {e}") + + send_tg(f"🏁 **Компиляция судебных дел завершена!**\nСобрано обращений: {processed}") + +if __name__ == "__main__": + compile_appeals() diff --git a/scraper_gis/sudrf_reaper.py b/scraper_gis/sudrf_reaper.py new file mode 100644 index 0000000..c83877e --- /dev/null +++ b/scraper_gis/sudrf_reaper.py @@ -0,0 +1,230 @@ +import asyncio +import os +import sys +import logging +import random +from datetime import datetime +from pathlib import Path +from playwright.async_api import async_playwright + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [%(levelname)s] %(message)s', + handlers=[ + logging.FileHandler("/app/logs/sudrf.log", encoding="utf-8"), + logging.StreamHandler(sys.stdout) + ] +) +logger = logging.getLogger("SUD-Reaper") + +USER_DATA_DIR = "/app/browser_context_sudrf" +EXPORT_DIR = "/app/exports/sudrf" +os.makedirs(EXPORT_DIR, exist_ok=True) +os.makedirs("/app/logs", exist_ok=True) + +BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M") +CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "197957361") + +def send_tg(msg): + import subprocess + subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage", + "-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"]) + +async def human_delay(min_sec=3, max_sec=6): + await asyncio.sleep(random.uniform(min_sec, max_sec)) + +async def run_mission(): + logger.info("Starting Playwright in VNC display...") + send_tg("🕵️‍♂️ **ГАС Правосудие: Запуск браузера**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html") + + async with async_playwright() as p: + # Launch persistent browser context + context = await p.chromium.launch_persistent_context( + user_data_dir=USER_DATA_DIR, + headless=False, + args=["--no-sandbox", "--disable-setuid-sandbox", "--start-maximized"] + ) + + page = context.pages[0] if context.pages else await context.new_page() + await page.set_viewport_size({"width": 1280, "height": 1024}) + + # Navigate to portal + logger.info("Navigating to sudrf...") + await page.goto("https://ej.sudrf.ru/") + await human_delay(4, 7) + + # Phase 0: Wait for login + logger.info("Waiting for user authorization...") + logged_in = False + for i in range(120): # 10 minutes total + content = "" + try: + content = await page.content() + except Exception as e: + logger.error(f"Error reading page content: {e}") + + # Check for common logout or profile indicators + if any(indicator in content for indicator in ["Выйти", "Выход", "Личный кабинет", "История обращений"]): + logged_in = True + break + + if i % 6 == 0: + logger.info(f"Still waiting... i={i}") + await asyncio.sleep(5) + + if not logged_in: + logger.error("Login timeout exceeded.") + send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.") + await context.close() + return + + logger.info("Login detected. Navigating to appeals history...") + send_tg("✅ **Авторизация успешна!** Начинаю сбор обращений...") + + await page.goto("https://ej.sudrf.ru/#/appeals") + await asyncio.sleep(10) # wait for page to render appeals list + + # Take list page screenshot + list_screenshot = os.path.join(EXPORT_DIR, "appeals_list.png") + await page.screenshot(path=list_screenshot, full_page=True) + + # Dump HTML for structural analysis + list_html = os.path.join(EXPORT_DIR, "appeals_list.html") + with open(list_html, "w", encoding="utf-8") as f: + f.write(await page.content()) + + logger.info(f"Appeals list screenshot saved to {list_screenshot}") + + # Extract appeal links + links = [] + all_links = await page.query_selector_all("a") + for link in all_links: + href = await link.get_attribute("href") + text = await link.inner_text() + if href and ("appeal" in href or "card" in href or "request" in href): + links.append({"href": href, "text": text.strip()}) + + logger.info(f"Found {len(links)} matching links on page: {links}") + + # Process each appeal + # We can extract rows from the table to get exact dates, numbers and link objects + # For sudrf, appeals table often has tr tags with status and numbers + rows = await page.query_selector_all("tr, .appeals-item") + logger.info(f"Found {len(rows)} rows/items in appeals list.") + + # Dump structured report of links to a json file + import json + with open(os.path.join(EXPORT_DIR, "found_links.json"), "w", encoding="utf-8") as f: + json.dump({"links": links, "rows_count": len(rows)}, f, indent=4, ensure_ascii=False) + + # Try to parse the table rows if they contain appeal details + parsed_appeals = [] + for index, row in enumerate(rows[:20]): # parse top 20 appeals first for safety + cells = await row.query_selector_all("td") + if not cells: continue + + row_data = [] + for cell in cells: + row_data.append((await cell.inner_text()).strip()) + + # Look for link inside the row + row_link = await row.query_selector("a") + href = await row_link.get_attribute("href") if row_link else None + + parsed_appeals.append({ + "row_index": index, + "data": row_data, + "href": href + }) + + with open(os.path.join(EXPORT_DIR, "parsed_table.json"), "w", encoding="utf-8") as f: + json.dump(parsed_appeals, f, indent=4, ensure_ascii=False) + + # Loop through found details pages and download documents + count = 0 + processed_hrefs = set() + + # Collect actual detail pages + detail_hrefs = [] + for appeal in parsed_appeals: + href = appeal.get("href") + if href and href not in processed_hrefs: + detail_hrefs.append(href) + processed_hrefs.add(href) + + # Also check all links for hrefs like #/appeals/view/ or similar + for link in links: + href = link["href"] + if href and href not in processed_hrefs: + detail_hrefs.append(href) + processed_hrefs.add(href) + + logger.info(f"Unique detail pages to process: {detail_hrefs}") + + for href in detail_hrefs[:10]: # Safe limit to top 10 appeals in recon mode + try: + full_url = href + if href.startswith("#"): + full_url = f"https://ej.sudrf.ru/{href}" + elif href.startswith("/"): + full_url = f"https://ej.sudrf.ru{href}" + + logger.info(f"Navigating to detail page: {full_url}") + await page.goto(full_url) + await asyncio.sleep(8) # Wait for page load + + safe_id = "".join([c for c in href if c.isalnum() or c in ("-", "_")]).strip() + appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}") + os.makedirs(appeal_dir, exist_ok=True) + + # Take detail screenshot + detail_screenshot = os.path.join(appeal_dir, "evidence.png") + await page.screenshot(path=detail_screenshot, full_page=True) + + # Dump detail HTML + detail_html = os.path.join(appeal_dir, "page.html") + with open(detail_html, "w", encoding="utf-8") as f: + f.write(await page.content()) + + # Look for download links + download_selectors = [ + "a[href*='download']", + "a[href*='file']", + "button:has-text('Скачать')", + "a:has-text('Скачать')" + ] + + files_downloaded = [] + for selector in download_selectors: + elements = await page.locator(selector).all() + for el in elements: + if not await el.is_visible(): continue + try: + async with page.expect_download(timeout=15000) as di: + await el.click(force=True) + download = await di.value + fname = download.suggested_filename + dest = os.path.join(appeal_dir, fname) + await download.save_as(dest) + files_downloaded.append(fname) + logger.info(f"Downloaded: {fname}") + except Exception as e: + logger.warning(f"Failed download click: {e}") + + count += 1 + send_tg(f"📥 **Обращение {count} собрано**\nПапка: `appeal_{safe_id}`\nФайлов скачано: {len(files_downloaded)}") + + except Exception as e: + logger.error(f"Error processing appeal details {href}: {e}") + + send_tg(f"🏁 **Миссия ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.") + + # Trigger PDF compilation script + import subprocess + subprocess.Popen(["python3", "/app/sudrf_pdf_compiler.py"], start_new_session=True) + + await context.close() + +if __name__ == "__main__": + asyncio.run(run_mission()) diff --git a/web/app.py b/web/app.py index ff74640..28152bf 100644 --- a/web/app.py +++ b/web/app.py @@ -884,6 +884,28 @@ async def api_gis_status(username: str = Depends(get_current_admin)): except Exception as e: return JSONResponse(status_code=500, content={"message": str(e)}) +@app.post("/api/infra/run_sudrf_parser") +async def api_run_sudrf_parser(username: str = Depends(get_current_admin)): + try: + subprocess.run(["docker", "exec", "lkm37-gis-scraper", "pkill", "-f", "sudrf_reaper.py"]) + subprocess.Popen(["docker", "exec", "-d", "lkm37-gis-scraper", "python3", "/app/sudrf_reaper.py"]) + return JSONResponse(content={"message": "Сессия ГАС Правосудие успешно запущена. Перейдите по ссылке VNC и авторизуйтесь."}) + except Exception as e: + return JSONResponse(status_code=500, content={"message": str(e)}) + +@app.get("/api/infra/sudrf_status") +async def api_sudrf_status(username: str = Depends(get_current_admin)): + try: + proc = subprocess.run(["docker", "exec", "lkm37-gis-scraper", "ps", "aux"], capture_output=True, text=True) + is_alive = "sudrf_reaper.py" in proc.stdout + log_proc = subprocess.run(["docker", "exec", "lkm37-gis-scraper", "tail", "-n", "30", "/app/logs/sudrf.log"], capture_output=True, text=True) + return JSONResponse(content={ + "alive": is_alive, + "log": log_proc.stdout or "Log is empty or file not found." + }) + except Exception as e: + return JSONResponse(status_code=500, content={"message": str(e)}) + if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/web/templates/infra.html b/web/templates/infra.html index 5fc4402..8785f59 100644 --- a/web/templates/infra.html +++ b/web/templates/infra.html @@ -78,6 +78,30 @@ +
+
+
ГАС ПРАВОСУДИЕ (СУДЕБНЫЙ АРХИВ)
+
+
+ +
СТАТУС: НЕИЗВЕСТНО
+
+ +
+ ПОСЛЕДНИЕ ЛОГИ: + +
+

+                
Выполняет: Открытие сессии -> Ввод 2FA пользователем -> Парсинг ej.sudrf.ru -> Сборка PDF -> Google Drive
+
+
+
ДЕЙСТВИЯ
@@ -155,9 +179,62 @@ async function refreshGisStatus() { } } +document.getElementById('runSudrfParserBtn').addEventListener('click', async function() { + const btn = this; + if (!confirm('Запустить сборку судебного архива ГАС Правосудие? Потребуется войти через Госуслуги во VNC.')) return; + + btn.disabled = true; + btn.innerHTML = ' СЕССИЯ СУДОВ...'; + + try { + const response = await fetch('/api/infra/run_sudrf_parser', { method: 'POST' }); + const result = await response.json(); + alert(result.message); + } catch (e) { + alert('Ошибка: ' + e); + } finally { + btn.disabled = false; + btn.innerHTML = ' ЗАПУСТИТЬ ЖАТВУ СУДОВ'; + refreshSudrfStatus(); + } +}); + +async function refreshSudrfStatus() { + const badge = document.getElementById('sudrfStatusBadge'); + const logs = document.getElementById('sudrfLogs'); + + try { + const response = await fetch('/api/infra/sudrf_status'); + const data = await response.json(); + + if (data.alive) { + badge.className = 'badge bg-success p-2'; + badge.innerText = 'СТАТУС: РАБОТАЕТ'; + } else { + badge.className = 'badge bg-danger p-2'; + badge.innerText = 'СТАТУС: ОСТАНОВЛЕН'; + } + + logs.innerText = data.log; + logs.scrollTop = logs.scrollHeight; + } catch (e) { + badge.className = 'badge bg-warning p-2'; + badge.innerText = 'ОШИБКА API'; + } +} + document.getElementById('refreshGisStatusBtn').addEventListener('click', refreshGisStatus); -window.addEventListener('load', refreshGisStatus); +document.getElementById('refreshSudrfStatusBtn').addEventListener('click', refreshSudrfStatus); + +window.addEventListener('load', () => { + refreshGisStatus(); + refreshSudrfStatus(); +}); + // Авто-обновление раз в 30 секунд -setInterval(refreshGisStatus, 30000); +setInterval(() => { + refreshGisStatus(); + refreshSudrfStatus(); +}, 30000); {% endblock %}