domovoy_bot/scraper_gis/ultimate_reaper.py

import asyncio
import os
import logging
import random
from datetime import datetime
from playwright.async_api import async_playwright

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[logging.FileHandler("logs/reaper.log"), logging.StreamHandler()])
logger = logging.getLogger("Ultimate-Reaper")

USER_DATA_DIR = "/app/browser_context"
OBSIDIAN_BASE = "/app/obsidian/Gemini/Projects/GIS_ZHKH"
NAS_DOWNLOADS = "/app/downloads/nas/gis_archive"

async def send_tg_report(message):
    token = os.getenv("TELEGRAM_BOT_TOKEN")
    chat_id = os.getenv("TELEGRAM_CHAT_ID")
    if not token or not chat_id: return
    import aiohttp
    url = f"https://api.telegram.org/bot{token}/sendMessage"
    try:
        async with aiohttp.ClientSession() as session:
            await session.post(url, json={"chat_id": chat_id, "text": message, "parse_mode": "HTML"})
    except: pass

async def human_delay(min_sec=5, max_sec=22):
    # Имитируем медлительного человека
    delay = random.uniform(min_sec, max_sec)
    logger.info(f"Human delay: {delay:.2f}s")
    await asyncio.sleep(delay)

async def take_screenshot(page, title, subfolder):
    safe_name = "".join([c for c in title if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filename = f"{safe_name}_{timestamp}.png"
    path = f"{OBSIDIAN_BASE}/{subfolder}/{filename}"
    os.makedirs(os.path.dirname(path), exist_ok=True)
    await page.screenshot(path=path, full_page=True)
    return filename

async def get_all_frames(page):
    return [page] + page.frames

async def download_files_from_frames(page, save_folder):
    files_saved = []
    dest_dir = f"{NAS_DOWNLOADS}/{save_folder}"
    os.makedirs(dest_dir, exist_ok=True)
    
    selectors = [
        "a[title*='Скачать']", "button:has-text('Скачать')", "a:has-text('Скачать все')",
        "a[href*='.pdf']", "a[href*='.zip']", "a[href*='.doc']", "a[href*='.rar']"
    ]
    
    for f in await get_all_frames(page):
        for selector in selectors:
            try:
                elements = await f.locator(selector).all()
                for el in elements:
                    if not await el.is_visible(): continue
                    try:
                        async with page.expect_download(timeout=15000) as di:
                            await el.click()
                        download = await di.value
                        fname = download.suggested_filename
                        dest = f"{dest_dir}/{fname}"
                        await download.save_as(dest)
                        files_saved.append(fname)
                    except: continue
            except: continue
    return files_saved

async def wait_for_user_login(page):
    await send_tg_report("🔍 <b>Инициализация.</b>\nЯ открыл страницу ГИС ЖКХ и жду сессию...")
    await page.goto("https://dom.gosuslugi.ru/")
    await human_delay()
    
    for _ in range(60): # Ждем долго (около 20 минут в сумме с задержками)
        content = ""
        for f in await get_all_frames(page):
            try: content += await f.content()
            except: pass
            
        if "Выйти" in content or "Александрович" in content:
            await send_tg_report("✅ <b>Успех:</b> Я вижу Личный Кабинет! Начинаю работу.")
            return True
            
        if "Личный кабинет" in content:
            for f in await get_all_frames(page):
                try:
                    btn = f.locator("text='Личный кабинет'").first
                    if await btn.is_visible():
                        await send_tg_report("🚪 Вижу кнопку 'Личный кабинет'. Нажимаю сам...")
                        await btn.scroll_into_view_if_needed()
                        await btn.click()
                        await human_delay()
                        break
                except: pass
            
        await asyncio.sleep(10)
    return False

async def map_dashboard(page):
    await send_tg_report("🗺️ <b>Начинаю картографирование Личного Кабинета.</b>")
    await human_delay()
    scr = await take_screenshot(page, "Dashboard", "reports")
    
    buttons = set()
    for f in await get_all_frames(page):
        try:
            els = await f.locator("a, button, [role='button'], .menu-item").all()
            for el in els:
                if await el.is_visible():
                    txt = (await el.inner_text()).strip()
                    if txt and len(txt) > 3:
                        buttons.add(txt)
        except: pass
        
    path = f"{OBSIDIAN_BASE}/reports/Dashboard_Map.md"
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w", encoding="utf-8") as f:
        f.write("# Карта Личного Кабинета\n\n")
        f.write(f"![[Gemini/Projects/GIS_ZHKH/reports/{scr}]]\n\n")
        f.write("## Найденные элементы:\n")
        for b in sorted(buttons):
            f.write(f"- `{b}`\n")
            
    await send_tg_report("✅ <b>Карта составлена</b> и сохранена в Obsidian (reports/Dashboard_Map.md).")
    await human_delay()

async def process_notifications(page):
    await send_tg_report("📦 <b>Раздел: Информационные сообщения.</b> Захожу...")
    await page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/hcsi/nnl?tab=INCOMING")
    await human_delay()
    
    scr = await take_screenshot(page, "Уведомления", "notifications")
    files = await download_files_from_frames(page, "notifications")
    
    path = f"{OBSIDIAN_BASE}/reports/Notifications.md"
    with open(path, "w", encoding="utf-8") as f:
        f.write("# Информационные сообщения\n\n")
        f.write(f"![[Gemini/Projects/GIS_ZHKH/notifications/{scr}]]\n\n")
        f.write("## Файлы:\n")
        for file in files: f.write(f"- `{file}`\n")
        
    await send_tg_report(f"✅ <b>Уведомления:</b> Скриншот сделан, скачано {len(files)} файлов.")
    await human_delay()

async def process_appeals(page):
    await send_tg_report("✉️ <b>Раздел: Обращения и ответы.</b> Захожу...")
    await page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/appeals")
    await human_delay()
    
    # Ждем загрузки списка
    found_appeals = False
    for _ in range(30):
        for f in await get_all_frames(page):
            try:
                content = await f.content()
                if "Всего записей:" in content or "Направить обращение" in content:
                    found_appeals = True
                    break
            except: pass
        if found_appeals: break
        await asyncio.sleep(1)

    await human_delay(3, 5)
    scr = await take_screenshot(page, "Обращения_Список", "appeals")
    
    page_num = 1
    processed = set()
    
    while True:
        await send_tg_report(f"📄 Обращения: Обрабатываю страницу {page_num}...")
        
        while True:
            targets = []
            for f in await get_all_frames(page):
                try:
                    links = await f.locator("a").all()
                    for l in links:
                        if await l.is_visible():
                            txt = (await l.inner_text()).strip()
                            clean_txt = re.sub(r'\s+', ' ', txt.lower())
                            # Ищем ссылки-номера обращений или темы
                            if len(txt) > 10 and txt not in processed and ("№" in txt or "ответ" in clean_txt or "обращение" in clean_txt):
                                # Исключаем пагинацию и МЕНЮ (в службу поддержки)
                                if "следующая" not in clean_txt and "предыдущая" not in clean_txt and "в службу поддержки" not in clean_txt and "направить" not in clean_txt:
                                    targets.append((l, txt))
                except: pass
                if targets: break
                
            if not targets: break
            
            el, txt = targets[0]
            safe_name = "".join([c for c in txt if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
            processed.add(txt)
            
            await send_tg_report(f"⏳ Открываю обращение: <code>{txt[:40]}...</code>")
            try:
                await el.click()
                await human_delay() # Имитация человека
                
                inner_scr = await take_screenshot(page, safe_name, "appeals")
                files = await download_files_from_frames(page, f"appeals/{safe_name}")
                
                path = f"{OBSIDIAN_BASE}/reports/Appeal_{safe_name}.md"
                with open(path, "w", encoding="utf-8") as f_out:
                    f_out.write(f"# Обращение: {txt}\n\n![[Gemini/Projects/GIS_ZHKH/appeals/{inner_scr}]]\n\n## Файлы:\n")
                    if files:
                        for fn in files: f_out.write(f"- `{fn}`\n")
                    else:
                        f_out.write("Файлов нет.\n")
                
                await send_tg_report(f"✅ Готово. Файлов: {len(files)}")
                await page.go_back()
                await human_delay()
            except Exception as e:
                logger.error(f"Error on appeal {txt}: {e}")
                await page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/appeals")
                await human_delay()

        # Пагинация
        next_found = False
        for f in await get_all_frames(page):
            try:
                btn = f.locator("a:has-text('следующая')").first
                if await btn.is_visible():
                    await btn.click()
                    page_num += 1
                    next_found = True
                    await human_delay()
                    break
            except: pass
        if not next_found: break

    await send_tg_report("🏆 <b>Раздел Обращений выкачан полностью!</b>")

async def process_contracts(page):
    await send_tg_report("📑 <b>Раздел: Информация о договорах по дому.</b> Захожу...")
    # Идем по правильному URL Личного Кабинета, а не публичного портала
    await page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/house-info")
    await human_delay()
    
    # Клик по иконке раздела
    clicked = False
    for _ in range(30):
        for f in await get_all_frames(page):
            try:
                # Ищем по частичному тексту, чтобы обойти неразрывные пробелы (&nbsp;)
                btns = await f.locator("text=/договорах по дому/i").all()
                for btn in btns:
                    if await btn.is_visible():
                        await btn.scroll_into_view_if_needed()
                        await btn.click()
                        await human_delay()
                        clicked = True
                        break
            except: pass
            if clicked: break
        if clicked: break
        await asyncio.sleep(1)
        
    if not clicked:
        await send_tg_report("⚠️ Не смог найти раздел 'Информация о договорах по дому'.")
        return

    # Клик Найти
    found_search = False
    for _ in range(15):
        for f in await get_all_frames(page):
            try:
                btns = await f.locator("button:has-text('Найти')").all()
                for btn in btns:
                    if await btn.is_visible():
                        await btn.scroll_into_view_if_needed()
                        await btn.click()
                        await human_delay()
                        found_search = True
                        break
            except: pass
            if found_search: break
        if found_search: break
        await asyncio.sleep(1)

    processed = set()
    
    while True:
        targets = []
        for f in await get_all_frames(page):
            try:
                elements = await f.locator("a").all()
                for el in elements:
                    if await el.is_visible():
                        txt = (await el.inner_text()).strip()
                        if txt.startswith("Договор") and "Закон" not in txt and txt not in processed and len(txt) > 15:
                            targets.append((el, txt))
            except: pass
            if targets: break
            
        if not targets: break
        
        el, txt = targets[0]
        safe_name = "".join([c for c in txt if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
        processed.add(txt)
        
        await send_tg_report(f"⏳ Открываю договор: <code>{txt[:40]}...</code>")
        try:
            await el.click()
            await human_delay()
            
            inner_scr = await take_screenshot(page, safe_name, "contracts")
            files = await download_files_from_frames(page, f"contracts/{safe_name}")
            
            path = f"{OBSIDIAN_BASE}/reports/Contract_{safe_name}.md"
            with open(path, "w", encoding="utf-8") as f_out:
                f_out.write(f"# {txt}\n\n![[Gemini/Projects/GIS_ZHKH/contracts/{inner_scr}]]\n\n## Файлы:\n")
                if files:
                    for fn in files: f_out.write(f"- `{fn}`\n")
                else:
                    f_out.write("Файлов нет.\n")
            
            await send_tg_report(f"✅ Готово. Файлов: {len(files)}")
            await page.go_back()
            await human_delay()
            
        except Exception as e:
            logger.error(f"Error on {txt}: {e}")
            await page.go_back()
            await human_delay()

    await send_tg_report("🏆 <b>Раздел Договоров выкачан полностью!</b>")

async def ultimate_mission():
    async with async_playwright() as p:
        context = await p.chromium.launch_persistent_context(
            user_data_dir=USER_DATA_DIR, headless=False, args=["--no-sandbox", "--disable-setuid-sandbox", "--start-maximized"]
        )
        page = context.pages[0] if context.pages else await context.new_page()
        await page.set_viewport_size({"width": 1600, "height": 1024})

        if await wait_for_user_login(page):
            await map_dashboard(page)
            await process_notifications(page)
            await process_appeals(page)
            await process_contracts(page)
            await send_tg_report("🎉 <b>ЖНЕЦ-УЛЬТИМАТУМ ЗАВЕРШИЛ РАБОТУ!</b>")
        else:
            await send_tg_report("❌ Таймаут ожидания логина. Отбой миссии.")

        while True: await asyncio.sleep(3600)

if __name__ == "__main__":
    asyncio.run(ultimate_mission())