domovoy_bot/scraper_gis/sudrf_reaper.py

import asyncio
import os
import sys
import logging
import random
from datetime import datetime
from pathlib import Path
from playwright.async_api import async_playwright

# Setup logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    handlers=[
        logging.FileHandler("/app/logs/sudrf.log", encoding="utf-8"),
        logging.StreamHandler(sys.stdout)
    ]
)
logger = logging.getLogger("SUD-Reaper")

USER_DATA_DIR = "/app/browser_context_sudrf"
EXPORT_DIR = "/app/exports/sudrf"
os.makedirs(EXPORT_DIR, exist_ok=True)
os.makedirs("/app/logs", exist_ok=True)

BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M")
CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "197957361")

def send_tg(msg):
    import subprocess
    subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage", 
                    "-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"])

async def human_delay(min_sec=3, max_sec=6):
    await asyncio.sleep(random.uniform(min_sec, max_sec))

async def run_mission():
    logger.info("Starting Playwright in VNC display...")
    send_tg("🕵️‍♂️ **ГАС Правосудие: Запуск робота-разведчика**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html")
    
    async with async_playwright() as p:
        # Launch persistent browser context
        context = await p.chromium.launch_persistent_context(
            user_data_dir=USER_DATA_DIR,
            headless=False,
            args=["--no-sandbox", "--disable-setuid-sandbox", "--start-maximized"]
        )
        
        page = context.pages[0] if context.pages else await context.new_page()
        await page.set_viewport_size({"width": 1280, "height": 1024})
        
        # Navigate to portal
        logger.info("Navigating to sudrf...")
        await page.goto("https://ej.sudrf.ru/")
        await asyncio.sleep(8)
        
        # Check login screen
        url = page.url
        try:
            visible_text = await page.locator("body").inner_text()
        except Exception:
            visible_text = ""
            
        logger.info(f"Current page URL: {url}")
        
        is_logged_in = "sudrf.ru" in url and any(ind in visible_text for ind in ["Выйти", "Выход", "Личный кабинет", "Тырин"]) and "Авторизация" not in visible_text
        
        if not is_logged_in:
            logger.info("Login/Agreement page detected. Auto-accepting terms...")
            try:
                checkbox = await page.query_selector("input[type='checkbox']")
                if checkbox:
                    await checkbox.check()
                    logger.info("Checked user agreement checkbox.")
                else:
                    await page.click("text=Я ознакомился(ась)", timeout=5000)
                    logger.info("Clicked user agreement label.")
                await asyncio.sleep(2)
                
                buttons = await page.query_selector_all("button, .btn, a.btn")
                for b in buttons:
                    text = (await b.inner_text()).strip()
                    if "Войти" in text:
                        logger.info(f"Clicking Войти button: '{text}'")
                        await b.click()
                        break
            except Exception as e:
                logger.info(f"Note (agreement handling): {e}")
                
            logger.info("Waiting up to 300s for user authorization...")
            logged_in_False = True
            for i in range(60):
                await asyncio.sleep(5)
                current_url = page.url
                try:
                    current_text = await page.locator("body").inner_text()
                except Exception:
                    current_text = ""
                    
                if "sudrf.ru" in current_url and any(ind in current_text for ind in ["Выйти", "Выход", "Личный кабинет", "Тырин"]):
                    logger.info(f"Login detected! URL: {current_url}")
                    logged_in_False = False
                    break
                if i % 6 == 0:
                    logger.info(f"Still waiting... {i*5}s")
            if logged_in_False:
                logger.error("Login timeout exceeded.")
                send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.")
                await context.close()
                return
        else:
            logger.info("Already logged in.")
            
        # Navigate to appeals history page
        logger.info("Navigating to appeals history...")
        send_tg("✅ **Авторизация подтверждена!** Загружаю историю обращений...")
        await page.goto("https://ej.sudrf.ru/appeals/")
        await asyncio.sleep(8)
        
        # Click search button
        logger.info("Locating Search button...")
        buttons = await page.query_selector_all("button, input[type='submit'], input[type='button'], .btn, a.btn")
        search_clicked = False
        for b in buttons:
            try:
                text = (await b.inner_text()).strip()
                if "Найти" in text:
                    logger.info(f"Clicking Search button: '{text}'")
                    await b.click(force=True)
                    search_clicked = True
                    break
            except Exception as e:
                logger.error(f"Error inspecting search button: {e}")
                
        if not search_clicked:
            logger.error("Could not click Search button.")
            send_tg("❌ **Ошибка**: Не удалось нажать кнопку Поиск для загрузки списка обращений.")
            await page.screenshot(path=os.path.join(EXPORT_DIR, "search_failed.png"), full_page=True)
            await context.close()
            return
            
        logger.info("Waiting for search results...")
        await asyncio.sleep(8)
        
        # Check if table loaded
        tbodies = await page.query_selector_all("table.table-history tbody")
        logger.info(f"Found {len(tbodies)} tbodies in history table.")
        
        if not tbodies:
            logger.warning("No tbodies found in history table.")
            send_tg("⚠️ **Предупреждение**: История обращений пуста или не загрузилась.")
            await page.screenshot(path=os.path.join(EXPORT_DIR, "empty_table.png"), full_page=True)
            await context.close()
            return
            
        # Process each appeal / tbody
        count = 0
        for index, tbody in enumerate(tbodies[:15]): # Process top 15 appeals
            main_tr = await tbody.query_selector("tr:not(.tr-expanded)")
            if not main_tr:
                continue
                
            cells = await main_tr.query_selector_all("td")
            if len(cells) < 5:
                continue
                
            # Get registration number details
            num_text = (await cells[1].inner_text()).strip()
            # Clean registration number to get safe directory ID
            safe_id = "".join([c for c in num_text if c.isalnum() or c in ("-", "_")]).strip()
            if not safe_id:
                safe_id = f"row_{index}"
                
            appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}")
            os.makedirs(appeal_dir, exist_ok=True)
            
            # Find and click "История рассмотрения" link in 5th cell
            history_link = await cells[4].query_selector("a:has-text('История рассмотрения')")
            if not history_link:
                logger.info(f"No history link for row {index}, safe_id={safe_id}")
                continue
                
            logger.info(f"Processing appeal {safe_id} (row {index})...")
            
            try:
                # Expand
                await history_link.click(force=True)
                await asyncio.sleep(4)
                
                # Take screenshot of expanded tbody segment
                evidence_img = os.path.join(appeal_dir, "evidence.png")
                await tbody.screenshot(path=evidence_img)
                
                # Save tbody HTML
                detail_html = os.path.join(appeal_dir, "page.html")
                with open(detail_html, "w", encoding="utf-8") as f:
                    f.write(await tbody.inner_html())
                    
                # Download attachments
                file_links = await tbody.query_selector_all("a[href*='getFile']")
                files_downloaded = []
                
                for file_link in file_links:
                    file_text = (await file_link.inner_text()).strip()
                    logger.info(f"Downloading file: '{file_text}'")
                    
                    try:
                        async with page.expect_download(timeout=15000) as download_info:
                            await file_link.click()
                        download = await download_info.value
                        dest = os.path.join(appeal_dir, download.suggested_filename)
                        await download.save_as(dest)
                        files_downloaded.append(download.suggested_filename)
                        logger.info(f"Saved: {download.suggested_filename}")
                    except Exception as e:
                        logger.warning(f"Error downloading file {file_text}: {e}")
                        
                count += 1
                send_tg(f"📥 **Обращение {count} скачано: № {safe_id}**\nФайлов: {len(files_downloaded)}")
                
                # Collapse
                await history_link.click(force=True)
                await asyncio.sleep(2)
                
            except Exception as e:
                logger.error(f"Error parsing appeal {safe_id}: {e}")
                
        send_tg(f"🏁 **Жатва ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.")
        
        # Trigger compiler
        import subprocess
        subprocess.Popen(["python3", "/app/sudrf_pdf_compiler.py"], start_new_session=True)
        
        await context.close()

if __name__ == "__main__":
    asyncio.run(run_mission())