feat: Implement semi-automatic GAS Pravosudie sudrf scraper and web UI integration

scraper_gis/sudrf_pdf_compiler.py

@@ -0,0 +1,145 @@
+import os
+import sys
+import logging
+import json
+from pathlib import Path
+from pypdf import PdfWriter, PdfReader
+from reportlab.pdfgen import canvas
+from reportlab.lib.pagesizes import A4
+from reportlab.lib import colors
+from PIL import Image
+import subprocess
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s [%(levelname)s] %(message)s',
+    handlers=[
+        logging.FileHandler("/home/matrixhasyou/swarm-services/domovoy-bot/scraper_gis/logs/sudrf_compiler.log", encoding="utf-8"),
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+log = logging.getLogger("SUD-Compiler")
+
+# Add libs path
+sys.path.append("/home/matrixhasyou/swarm-services/libs")
+try:
+    from swarmlib.gdrive import GDriveManager
+except ImportError as e:
+    log.error(f"Cannot import swarmlib: {e}")
+    GDriveManager = None
+
+EXPORT_DIR = Path("/home/matrixhasyou/swarm-services/domovoy-bot/scraper_gis/exports/sudrf")
+OUTPUT_DIR = Path("/home/matrixhasyou/swarm-services/domovoy-bot/exports/compiled_sudrf")
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+BOT_TOKEN = "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M"
+CHAT_ID = "197957361"
+
+def send_tg(msg):
+    subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage", 
+                    "-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"])
+
+def tag_image_to_pdf(image_path, output_pdf_path, label):
+    try:
+        c = canvas.Canvas(output_pdf_path, pagesize=A4)
+        width, height = A4
+        c.setFillColor(colors.black)
+        c.rect(0, height - 50, width, 50, fill=1)
+        c.setFillColor(colors.white)
+        c.setFont("Helvetica-Bold", 10)
+        c.drawString(20, height - 30, f"SUD: {label}".upper())
+        
+        img = Image.open(image_path)
+        img_width, img_height = img.size
+        aspect = img_height / float(img_width)
+        
+        display_width = width - 40
+        display_height = display_width * aspect
+        if display_height > (height - 70):
+            display_height = height - 70
+            display_width = display_height / aspect
+            
+        c.drawImage(image_path, 20, height - 60 - display_height, width=display_width, height=display_height)
+        c.showPage()
+        c.save()
+        return True
+    except Exception as e:
+        log.error(f"Failed to tag image {image_path}: {e}")
+        return False
+
+def compile_appeals():
+    send_tg("🧬 **ГАС Правосудие: Запуск PDF-компилятора**\nОбрабатываю выгруженные материалы...")
+    
+    if not EXPORT_DIR.exists():
+        log.error(f"Export dir not found: {EXPORT_DIR}")
+        send_tg("❌ **Компиляция прервана**: папка с выгрузками не найдена.")
+        return
+        
+    gdrive = None
+    if GDriveManager:
+        try:
+            gdrive = GDriveManager()
+        except Exception as e:
+            log.error(f"Failed to initialize GDriveManager: {e}")
+            
+    folders = [d for d in EXPORT_DIR.iterdir() if d.is_dir() and d.name.startswith("appeal_")]
+    processed = 0
+    
+    for folder in folders:
+        appeal_id = folder.name.replace("appeal_", "")
+        log.info(f"Compiling appeal: {appeal_id}")
+        
+        writer = PdfWriter()
+        
+        # 1. Convert detail screenshot (evidence.png) to PDF first
+        evidence_img = folder / "evidence.png"
+        evidence_pdf = folder / "evidence.pdf"
+        if evidence_img.exists():
+            if tag_image_to_pdf(str(evidence_img), str(evidence_pdf), f"APPEAL {appeal_id}"):
+                writer.append(str(evidence_pdf))
+                
+        # 2. Append all other PDF files inside the folder
+        files = sorted(list(folder.iterdir()))
+        for f in files:
+            if f.suffix.lower() == ".pdf" and f.name != "evidence.pdf":
+                try:
+                    writer.append(str(f))
+                    log.info(f"Appended PDF: {f.name}")
+                except Exception as e:
+                    log.error(f"Failed to append PDF {f.name}: {e}")
+            elif f.suffix.lower() in [".png", ".jpg", ".jpeg"] and f.name != "evidence.png":
+                # Convert image attachments to PDF and append
+                img_pdf = folder / f"{f.stem}_attachment.pdf"
+                if tag_image_to_pdf(str(f), str(img_pdf), f"ATTACHMENT {f.name}"):
+                    try:
+                        writer.append(str(img_pdf))
+                        log.info(f"Appended converted image: {f.name}")
+                    except Exception as e:
+                        log.error(f"Failed to append converted image {f.name}: {e}")
+                        
+        output_pdf = OUTPUT_DIR / f"SUD_APPEAL_{appeal_id}.pdf"
+        try:
+            with open(output_pdf, "wb") as out_f:
+                writer.write(out_f)
+            processed += 1
+            log.info(f"Successfully compiled: {output_pdf.name}")
+            
+            # Upload to GDrive
+            drive_status = "не настроен"
+            if gdrive:
+                try:
+                    drive_status = gdrive.upload_or_update(str(output_pdf), output_pdf.name)
+                except Exception as e:
+                    drive_status = f"ошибка GDrive: {e}"
+                    
+            send_tg(f"📄 **Обращение {appeal_id} собрано в PDF**\n☁️ GDrive: `{drive_status}`")
+            
+        except Exception as e:
+            log.error(f"Failed to write compiled PDF for {appeal_id}: {e}")
+            send_tg(f"❌ **Ошибка сборки обращения {appeal_id}**: {e}")
+            
+    send_tg(f"🏁 **Компиляция судебных дел завершена!**\nСобрано обращений: {processed}")
+
+if __name__ == "__main__":
+    compile_appeals()

scraper_gis/sudrf_reaper.py

@@ -0,0 +1,230 @@
+import asyncio
+import os
+import sys
+import logging
+import random
+from datetime import datetime
+from pathlib import Path
+from playwright.async_api import async_playwright
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s [%(levelname)s] %(message)s',
+    handlers=[
+        logging.FileHandler("/app/logs/sudrf.log", encoding="utf-8"),
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+logger = logging.getLogger("SUD-Reaper")
+
+USER_DATA_DIR = "/app/browser_context_sudrf"
+EXPORT_DIR = "/app/exports/sudrf"
+os.makedirs(EXPORT_DIR, exist_ok=True)
+os.makedirs("/app/logs", exist_ok=True)
+
+BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M")
+CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "197957361")
+
+def send_tg(msg):
+    import subprocess
+    subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage", 
+                    "-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"])
+
+async def human_delay(min_sec=3, max_sec=6):
+    await asyncio.sleep(random.uniform(min_sec, max_sec))
+
+async def run_mission():
+    logger.info("Starting Playwright in VNC display...")
+    send_tg("🕵️‍♂️ **ГАС Правосудие: Запуск браузера**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html")
+    
+    async with async_playwright() as p:
+        # Launch persistent browser context
+        context = await p.chromium.launch_persistent_context(
+            user_data_dir=USER_DATA_DIR,
+            headless=False,
+            args=["--no-sandbox", "--disable-setuid-sandbox", "--start-maximized"]
+        )
+        
+        page = context.pages[0] if context.pages else await context.new_page()
+        await page.set_viewport_size({"width": 1280, "height": 1024})
+        
+        # Navigate to portal
+        logger.info("Navigating to sudrf...")
+        await page.goto("https://ej.sudrf.ru/")
+        await human_delay(4, 7)
+        
+        # Phase 0: Wait for login
+        logger.info("Waiting for user authorization...")
+        logged_in = False
+        for i in range(120): # 10 minutes total
+            content = ""
+            try:
+                content = await page.content()
+            except Exception as e:
+                logger.error(f"Error reading page content: {e}")
+            
+            # Check for common logout or profile indicators
+            if any(indicator in content for indicator in ["Выйти", "Выход", "Личный кабинет", "История обращений"]):
+                logged_in = True
+                break
+            
+            if i % 6 == 0:
+                logger.info(f"Still waiting... i={i}")
+            await asyncio.sleep(5)
+            
+        if not logged_in:
+            logger.error("Login timeout exceeded.")
+            send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.")
+            await context.close()
+            return
+            
+        logger.info("Login detected. Navigating to appeals history...")
+        send_tg("✅ **Авторизация успешна!** Начинаю сбор обращений...")
+        
+        await page.goto("https://ej.sudrf.ru/#/appeals")
+        await asyncio.sleep(10) # wait for page to render appeals list
+        
+        # Take list page screenshot
+        list_screenshot = os.path.join(EXPORT_DIR, "appeals_list.png")
+        await page.screenshot(path=list_screenshot, full_page=True)
+        
+        # Dump HTML for structural analysis
+        list_html = os.path.join(EXPORT_DIR, "appeals_list.html")
+        with open(list_html, "w", encoding="utf-8") as f:
+            f.write(await page.content())
+            
+        logger.info(f"Appeals list screenshot saved to {list_screenshot}")
+        
+        # Extract appeal links
+        links = []
+        all_links = await page.query_selector_all("a")
+        for link in all_links:
+            href = await link.get_attribute("href")
+            text = await link.inner_text()
+            if href and ("appeal" in href or "card" in href or "request" in href):
+                links.append({"href": href, "text": text.strip()})
+        
+        logger.info(f"Found {len(links)} matching links on page: {links}")
+        
+        # Process each appeal
+        # We can extract rows from the table to get exact dates, numbers and link objects
+        # For sudrf, appeals table often has tr tags with status and numbers
+        rows = await page.query_selector_all("tr, .appeals-item")
+        logger.info(f"Found {len(rows)} rows/items in appeals list.")
+        
+        # Dump structured report of links to a json file
+        import json
+        with open(os.path.join(EXPORT_DIR, "found_links.json"), "w", encoding="utf-8") as f:
+            json.dump({"links": links, "rows_count": len(rows)}, f, indent=4, ensure_ascii=False)
+            
+        # Try to parse the table rows if they contain appeal details
+        parsed_appeals = []
+        for index, row in enumerate(rows[:20]): # parse top 20 appeals first for safety
+            cells = await row.query_selector_all("td")
+            if not cells: continue
+            
+            row_data = []
+            for cell in cells:
+                row_data.append((await cell.inner_text()).strip())
+            
+            # Look for link inside the row
+            row_link = await row.query_selector("a")
+            href = await row_link.get_attribute("href") if row_link else None
+            
+            parsed_appeals.append({
+                "row_index": index,
+                "data": row_data,
+                "href": href
+            })
+            
+        with open(os.path.join(EXPORT_DIR, "parsed_table.json"), "w", encoding="utf-8") as f:
+            json.dump(parsed_appeals, f, indent=4, ensure_ascii=False)
+
+        # Loop through found details pages and download documents
+        count = 0
+        processed_hrefs = set()
+        
+        # Collect actual detail pages
+        detail_hrefs = []
+        for appeal in parsed_appeals:
+            href = appeal.get("href")
+            if href and href not in processed_hrefs:
+                detail_hrefs.append(href)
+                processed_hrefs.add(href)
+                
+        # Also check all links for hrefs like #/appeals/view/ or similar
+        for link in links:
+            href = link["href"]
+            if href and href not in processed_hrefs:
+                detail_hrefs.append(href)
+                processed_hrefs.add(href)
+                
+        logger.info(f"Unique detail pages to process: {detail_hrefs}")
+        
+        for href in detail_hrefs[:10]: # Safe limit to top 10 appeals in recon mode
+            try:
+                full_url = href
+                if href.startswith("#"):
+                    full_url = f"https://ej.sudrf.ru/{href}"
+                elif href.startswith("/"):
+                    full_url = f"https://ej.sudrf.ru{href}"
+                    
+                logger.info(f"Navigating to detail page: {full_url}")
+                await page.goto(full_url)
+                await asyncio.sleep(8) # Wait for page load
+                
+                safe_id = "".join([c for c in href if c.isalnum() or c in ("-", "_")]).strip()
+                appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}")
+                os.makedirs(appeal_dir, exist_ok=True)
+                
+                # Take detail screenshot
+                detail_screenshot = os.path.join(appeal_dir, "evidence.png")
+                await page.screenshot(path=detail_screenshot, full_page=True)
+                
+                # Dump detail HTML
+                detail_html = os.path.join(appeal_dir, "page.html")
+                with open(detail_html, "w", encoding="utf-8") as f:
+                    f.write(await page.content())
+                    
+                # Look for download links
+                download_selectors = [
+                    "a[href*='download']", 
+                    "a[href*='file']", 
+                    "button:has-text('Скачать')", 
+                    "a:has-text('Скачать')"
+                ]
+                
+                files_downloaded = []
+                for selector in download_selectors:
+                    elements = await page.locator(selector).all()
+                    for el in elements:
+                        if not await el.is_visible(): continue
+                        try:
+                            async with page.expect_download(timeout=15000) as di:
+                                await el.click(force=True)
+                            download = await di.value
+                            fname = download.suggested_filename
+                            dest = os.path.join(appeal_dir, fname)
+                            await download.save_as(dest)
+                            files_downloaded.append(fname)
+                            logger.info(f"Downloaded: {fname}")
+                        except Exception as e:
+                            logger.warning(f"Failed download click: {e}")
+                            
+                count += 1
+                send_tg(f"📥 **Обращение {count} собрано**\nПапка: `appeal_{safe_id}`\nФайлов скачано: {len(files_downloaded)}")
+                
+            except Exception as e:
+                logger.error(f"Error processing appeal details {href}: {e}")
+                
+        send_tg(f"🏁 **Миссия ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.")
+        
+        # Trigger PDF compilation script
+        import subprocess
+        subprocess.Popen(["python3", "/app/sudrf_pdf_compiler.py"], start_new_session=True)
+        
+        await context.close()
+
+if __name__ == "__main__":
+    asyncio.run(run_mission())

web/app.py

@@ -884,6 +884,28 @@ async def api_gis_status(username: str = Depends(get_current_admin)):
     except Exception as e:
         return JSONResponse(status_code=500, content={"message": str(e)})
 
+@app.post("/api/infra/run_sudrf_parser")
+async def api_run_sudrf_parser(username: str = Depends(get_current_admin)):
+    try:
+        subprocess.run(["docker", "exec", "lkm37-gis-scraper", "pkill", "-f", "sudrf_reaper.py"])
+        subprocess.Popen(["docker", "exec", "-d", "lkm37-gis-scraper", "python3", "/app/sudrf_reaper.py"])
+        return JSONResponse(content={"message": "Сессия ГАС Правосудие успешно запущена. Перейдите по ссылке VNC и авторизуйтесь."})
+    except Exception as e:
+        return JSONResponse(status_code=500, content={"message": str(e)})
+
+@app.get("/api/infra/sudrf_status")
+async def api_sudrf_status(username: str = Depends(get_current_admin)):
+    try:
+        proc = subprocess.run(["docker", "exec", "lkm37-gis-scraper", "ps", "aux"], capture_output=True, text=True)
+        is_alive = "sudrf_reaper.py" in proc.stdout
+        log_proc = subprocess.run(["docker", "exec", "lkm37-gis-scraper", "tail", "-n", "30", "/app/logs/sudrf.log"], capture_output=True, text=True)
+        return JSONResponse(content={
+            "alive": is_alive,
+            "log": log_proc.stdout or "Log is empty or file not found."
+        })
+    except Exception as e:
+        return JSONResponse(status_code=500, content={"message": str(e)})
+
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=8000)

web/templates/infra.html

@@ -78,6 +78,30 @@
             </div>
         </div>
     </div>
+    <div class="col-md-6">
+        <div class="card bg-dark border-danger">
+            <div class="card-header text-danger">ГАС ПРАВОСУДИЕ (СУДЕБНЫЙ АРХИВ)</div>
+            <div class="card-body">
+                <div class="d-flex justify-content-between align-items-center mb-3">
+                    <button id="runSudrfParserBtn" class="btn btn-outline-danger flex-grow-1 me-2">
+                        <i class="bi bi-play-circle"></i> ЗАПУСТИТЬ ЖАТВУ СУДОВ
+                    </button>
+                    <div id="sudrfStatusBadge" class="badge bg-secondary p-2">СТАТУС: НЕИЗВЕСТНО</div>
+                </div>
+                <div class="d-flex justify-content-between align-items-center mb-3">
+                    <a href="http://192.168.10.116:6080/vnc.html" target="_blank" class="btn btn-outline-info w-100">
+                        <i class="bi bi-window"></i> ОТКРЫТЬ VNC БРАУЗЕР (ДЛЯ ЕСИА)
+                    </a>
+                </div>
+                <div class="mb-2 d-flex justify-content-between">
+                    <span class="small text-muted">ПОСЛЕДНИЕ ЛОГИ:</span>
+                    <button id="refreshSudrfStatusBtn" class="btn btn-sm btn-link p-0 text-danger">ОБНОВИТЬ СТАТУС</button>
+                </div>
+                <pre id="sudrfLogs" class="bg-black text-danger p-2 small border border-secondary" style="height: 200px; overflow-y: scroll; font-size: 0.7rem;"></pre>
+                <div class="small opacity-50 mt-2">Выполняет: Открытие сессии -> Ввод 2FA пользователем -> Парсинг ej.sudrf.ru -> Сборка PDF -> Google Drive</div>
+            </div>
+        </div>
+    </div>
     <div class="col-md-6">
         <div class="card bg-dark border-warning">
             <div class="card-header text-warning">ДЕЙСТВИЯ</div>
@@ -155,9 +179,62 @@ async function refreshGisStatus() {
     }
 }
 
+document.getElementById('runSudrfParserBtn').addEventListener('click', async function() {
+    const btn = this;
+    if (!confirm('Запустить сборку судебного архива ГАС Правосудие? Потребуется войти через Госуслуги во VNC.')) return;
+    
+    btn.disabled = true;
+    btn.innerHTML = '<span class="spinner-border spinner-border-sm"></span> СЕССИЯ СУДОВ...';
+    
+    try {
+        const response = await fetch('/api/infra/run_sudrf_parser', { method: 'POST' });
+        const result = await response.json();
+        alert(result.message);
+    } catch (e) {
+        alert('Ошибка: ' + e);
+    } finally {
+        btn.disabled = false;
+        btn.innerHTML = '<i class="bi bi-play-circle"></i> ЗАПУСТИТЬ ЖАТВУ СУДОВ';
+        refreshSudrfStatus();
+    }
+});
+
+async function refreshSudrfStatus() {
+    const badge = document.getElementById('sudrfStatusBadge');
+    const logs = document.getElementById('sudrfLogs');
+    
+    try {
+        const response = await fetch('/api/infra/sudrf_status');
+        const data = await response.json();
+        
+        if (data.alive) {
+            badge.className = 'badge bg-success p-2';
+            badge.innerText = 'СТАТУС: РАБОТАЕТ';
+        } else {
+            badge.className = 'badge bg-danger p-2';
+            badge.innerText = 'СТАТУС: ОСТАНОВЛЕН';
+        }
+        
+        logs.innerText = data.log;
+        logs.scrollTop = logs.scrollHeight;
+    } catch (e) {
+        badge.className = 'badge bg-warning p-2';
+        badge.innerText = 'ОШИБКА API';
+    }
+}
+
 document.getElementById('refreshGisStatusBtn').addEventListener('click', refreshGisStatus);
-window.addEventListener('load', refreshGisStatus);
+document.getElementById('refreshSudrfStatusBtn').addEventListener('click', refreshSudrfStatus);
+
+window.addEventListener('load', () => {
+    refreshGisStatus();
+    refreshSudrfStatus();
+});
+
 // Авто-обновление раз в 30 секунд
-setInterval(refreshGisStatus, 30000);
+setInterval(() => {
+    refreshGisStatus();
+    refreshSudrfStatus();
+}, 30000);
 </script>
 {% endblock %}