domovoy_bot/scraper_gis/sudrf_reaper.py

import asyncio
import os
import sys
import logging
import random
from datetime import datetime
from pathlib import Path
from playwright.async_api import async_playwright

# Setup logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    handlers=[
        logging.FileHandler("/app/logs/sudrf.log", encoding="utf-8"),
        logging.StreamHandler(sys.stdout)
    ]
)
logger = logging.getLogger("SUD-Reaper")

USER_DATA_DIR = "/app/browser_context_sudrf"
EXPORT_DIR = "/app/exports/sudrf"
os.makedirs(EXPORT_DIR, exist_ok=True)
os.makedirs("/app/logs", exist_ok=True)

BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M")
CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "197957361")

def send_tg(msg):
    import subprocess
    subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage", 
                    "-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"])

async def human_delay(min_sec=3, max_sec=6):
    await asyncio.sleep(random.uniform(min_sec, max_sec))

async def run_mission():
    logger.info("Starting Playwright in VNC display...")
    send_tg("🕵️‍♂️ **ГАС Правосудие: Запуск браузера**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html")
    
    async with async_playwright() as p:
        # Launch persistent browser context
        context = await p.chromium.launch_persistent_context(
            user_data_dir=USER_DATA_DIR,
            headless=False,
            args=["--no-sandbox", "--disable-setuid-sandbox", "--start-maximized"]
        )
        
        page = context.pages[0] if context.pages else await context.new_page()
        await page.set_viewport_size({"width": 1280, "height": 1024})
        
        # Navigate to portal
        logger.info("Navigating to sudrf...")
        await page.goto("https://ej.sudrf.ru/")
        await human_delay(4, 7)
        
        # Phase 0: Wait for login
        logger.info("Waiting for user authorization...")
        logged_in = False
        for i in range(120): # 10 minutes total
            content = ""
            try:
                content = await page.content()
            except Exception as e:
                logger.error(f"Error reading page content: {e}")
            
            # Check for common logout or profile indicators
            if any(indicator in content for indicator in ["Выйти", "Выход", "Личный кабинет", "История обращений"]):
                logged_in = True
                break
            
            if i % 6 == 0:
                logger.info(f"Still waiting... i={i}")
            await asyncio.sleep(5)
            
        if not logged_in:
            logger.error("Login timeout exceeded.")
            send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.")
            await context.close()
            return
            
        logger.info("Login detected. Navigating to appeals history...")
        send_tg("✅ **Авторизация успешна!** Начинаю сбор обращений...")
        
        await page.goto("https://ej.sudrf.ru/appeals/")
        await asyncio.sleep(10) # wait for page to render appeals list
        
        # Take list page screenshot
        list_screenshot = os.path.join(EXPORT_DIR, "appeals_list.png")
        await page.screenshot(path=list_screenshot, full_page=True)
        
        # Dump HTML for structural analysis
        list_html = os.path.join(EXPORT_DIR, "appeals_list.html")
        with open(list_html, "w", encoding="utf-8") as f:
            f.write(await page.content())
            
        logger.info(f"Appeals list screenshot saved to {list_screenshot}")
        
        # Extract appeal links
        links = []
        all_links = await page.query_selector_all("a")
        for link in all_links:
            href = await link.get_attribute("href")
            text = await link.inner_text()
            if href and ("appeal" in href or "card" in href or "request" in href):
                # Исключаем общие страницы
                if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
                    links.append({"href": href, "text": text.strip()})
        
        logger.info(f"Found {len(links)} matching links on page: {links}")
        
        # Process each appeal
        # We can extract rows from the table to get exact dates, numbers and link objects
        # For sudrf, appeals table often has tr tags with status and numbers
        rows = await page.query_selector_all("tr, .appeals-item")
        logger.info(f"Found {len(rows)} rows/items in appeals list.")
        
        # Dump structured report of links to a json file
        import json
        with open(os.path.join(EXPORT_DIR, "found_links.json"), "w", encoding="utf-8") as f:
            json.dump({"links": links, "rows_count": len(rows)}, f, indent=4, ensure_ascii=False)
            
        # Try to parse the table rows if they contain appeal details
        parsed_appeals = []
        for index, row in enumerate(rows[:20]): # parse top 20 appeals first for safety
            cells = await row.query_selector_all("td")
            if not cells: continue
            
            row_data = []
            for cell in cells:
                row_data.append((await cell.inner_text()).strip())
            
            # Look for link inside the row
            row_link = await row.query_selector("a")
            href = await row_link.get_attribute("href") if row_link else None
            
            parsed_appeals.append({
                "row_index": index,
                "data": row_data,
                "href": href
            })
            
        with open(os.path.join(EXPORT_DIR, "parsed_table.json"), "w", encoding="utf-8") as f:
            json.dump(parsed_appeals, f, indent=4, ensure_ascii=False)

        # Loop through found details pages and download documents
        count = 0
        processed_hrefs = set()
        
        # Collect actual detail pages
        detail_hrefs = []
        for appeal in parsed_appeals:
            href = appeal.get("href")
            if href and href not in processed_hrefs:
                if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
                    detail_hrefs.append(href)
                    processed_hrefs.add(href)
                
        # Also check all links for hrefs like #/appeals/view/ or similar
        for link in links:
            href = link["href"]
            if href and href not in processed_hrefs:
                if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
                    detail_hrefs.append(href)
                    processed_hrefs.add(href)
                
        logger.info(f"Unique detail pages to process: {detail_hrefs}")
        
        for href in detail_hrefs[:10]: # Safe limit to top 10 appeals in recon mode
            try:
                full_url = href
                if href.startswith("#"):
                    full_url = f"https://ej.sudrf.ru/{href}"
                elif href.startswith("/"):
                    full_url = f"https://ej.sudrf.ru{href}"
                    
                logger.info(f"Navigating to detail page: {full_url}")
                await page.goto(full_url)
                await asyncio.sleep(8) # Wait for page load
                
                safe_id = "".join([c for c in href if c.isalnum() or c in ("-", "_")]).strip()
                appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}")
                os.makedirs(appeal_dir, exist_ok=True)
                
                # Take detail screenshot
                detail_screenshot = os.path.join(appeal_dir, "evidence.png")
                await page.screenshot(path=detail_screenshot, full_page=True)
                
                # Dump detail HTML
                detail_html = os.path.join(appeal_dir, "page.html")
                with open(detail_html, "w", encoding="utf-8") as f:
                    f.write(await page.content())
                    
                # Look for download links
                download_selectors = [
                    "a[href*='download']", 
                    "a[href*='file']", 
                    "button:has-text('Скачать')", 
                    "a:has-text('Скачать')"
                ]
                
                files_downloaded = []
                for selector in download_selectors:
                    elements = await page.locator(selector).all()
                    for el in elements:
                        if not await el.is_visible(): continue
                        try:
                            async with page.expect_download(timeout=15000) as di:
                                await el.click(force=True)
                            download = await di.value
                            fname = download.suggested_filename
                            dest = os.path.join(appeal_dir, fname)
                            await download.save_as(dest)
                            files_downloaded.append(fname)
                            logger.info(f"Downloaded: {fname}")
                        except Exception as e:
                            logger.warning(f"Failed download click: {e}")
                            
                count += 1
                send_tg(f"📥 **Обращение {count} собрано**\nПапка: `appeal_{safe_id}`\nФайлов скачано: {len(files_downloaded)}")
                
            except Exception as e:
                logger.error(f"Error processing appeal details {href}: {e}")
                
        send_tg(f"🏁 **Миссия ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.")
        
        # Trigger PDF compilation script
        import subprocess
        subprocess.Popen(["python3", "/app/sudrf_pdf_compiler.py"], start_new_session=True)
        
        await context.close()

if __name__ == "__main__":
    asyncio.run(run_mission())