domovoy_bot/scraper_gis/master_scraper.py

import asyncio
import os
import logging
import random
from datetime import datetime
from playwright.async_api import async_playwright

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[logging.FileHandler("logs/contracts.log"), logging.StreamHandler()])
logger = logging.getLogger("GIS-Ultimate")

USER_DATA_DIR = "/app/browser_context"
OBSIDIAN_BASE = "/app/obsidian/Gemini/Projects/GIS_ZHKH"
NAS_DOWNLOADS = "/app/downloads/nas/gis_archive"

BLACKLIST = [
    "Справка по системе", "Регламенты и инструкции", "Часто задаваемые вопросы", 
    "Карта сайта", "Версия для слабовидящих", "Обращение в службу поддержки", 
    "Личный кабинет", "Главная страница", "Помещения (жилые дома)", 
    "Информационные сообщения", "Обращения и ответы на них", "Подключенные ЛС", 
    "Оплата ЖКУ", "Аналитика и отчеты", "Форум", "Свернуть поиск", "Найти", 
    "Очистить", "предыдущая", "следующая", "Выводить по"
]

async def send_tg_report(message):
    token = os.getenv("TELEGRAM_BOT_TOKEN")
    chat_id = os.getenv("TELEGRAM_CHAT_ID")
    if not token or not chat_id: return
    import aiohttp
    url = f"https://api.telegram.org/bot{token}/sendMessage"
    try:
        async with aiohttp.ClientSession() as session:
            await session.post(url, json={"chat_id": chat_id, "text": message, "parse_mode": "HTML"})
    except: pass

async def human_delay(min_sec=4, max_sec=8):
    await asyncio.sleep(random.uniform(min_sec, max_sec))

async def take_screenshot(page, title, subfolder):
    safe_name = "".join([c for c in title if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filename = f"{safe_name}_{timestamp}.png"
    path = f"{OBSIDIAN_BASE}/{subfolder}/{filename}"
    os.makedirs(os.path.dirname(path), exist_ok=True)
    await page.screenshot(path=path, full_page=True)
    return filename

async def download_all_files(page, save_folder):
    files_saved = []
    dest_dir = f"{NAS_DOWNLOADS}/{save_folder}"
    os.makedirs(dest_dir, exist_ok=True)
    
    selectors = ["a[title*='Скачать']", "button:has-text('Скачать')", "a:has-text('Скачать все')", "a[href*='.pdf']", "a[href*='.zip']", "a[href*='.doc']", "a[href*='.rar']"]
    
    for f in [page] + page.frames:
        for selector in selectors:
            try:
                elements = await f.locator(selector).all()
                for el in elements:
                    if not await el.is_visible(): continue
                    try:
                        async with page.expect_download(timeout=15000) as di:
                            await el.click()
                        download = await di.value
                        fname = download.suggested_filename
                        dest = f"{dest_dir}/{fname}"
                        await download.save_as(dest)
                        files_saved.append(fname)
                    except: continue
            except: continue
    return files_saved

async def phase_0_login(page):
    await send_tg_report("🔍 <b>Инициализация. Проверка сессии.</b>")
    await page.goto("https://dom.gosuslugi.ru/")
    await human_delay()
    
    for _ in range(30):
        content = ""
        for f in [page] + page.frames:
            try: content += await f.content()
            except: pass
            
        if "Выйти" in content or "Александрович" in content:
            await send_tg_report("✅ Сессия активна.")
            return True
            
        await asyncio.sleep(5)
    return False

async def process_generic_list(page, url, section_name, folder_name, link_prefix=""):
    await send_tg_report(f"📂 <b>Захожу в раздел: {section_name}</b>")
    await page.goto(url)
    await human_delay(5, 8)
    
    processed = set()
    page_num = 1
    
    while True:
        await send_tg_report(f"📄 Сканирую страницу {page_num}...")
        await take_screenshot(page, f"Список_стр_{page_num}", f"{folder_name}/lists")
        
        while True:
            targets = []
            for f in [page] + page.frames:
                try:
                    elements = await f.locator("a").all()
                    for el in elements:
                        if await el.is_visible():
                            txt = (await el.inner_text()).strip()
                            if len(txt) > 10 and txt not in processed:
                                is_blacklisted = any(b in txt for b in BLACKLIST)
                                if not is_blacklisted:
                                    if not link_prefix or txt.startswith(link_prefix):
                                        targets.append((el, txt))
                except: pass
                if targets: break
                
            if not targets: break # Все ссылки на этой странице обработаны
            
            el, txt = targets[0]
            safe_name = "".join([c for c in txt if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
            processed.add(txt)
            
            await send_tg_report(f"⏳ Открываю: <code>{txt[:40]}...</code>")
            try:
                await el.click()
                await human_delay(4, 7)
                
                inner_scr = await take_screenshot(page, safe_name, f"{folder_name}/details")
                files = await download_all_files(page, f"{folder_name}/{safe_name}")
                
                path = f"{OBSIDIAN_BASE}/{folder_name}/{safe_name}.md"
                os.makedirs(os.path.dirname(path), exist_ok=True)
                with open(path, "w", encoding="utf-8") as f_out:
                    f_out.write(f"# {txt}\n\n![[Gemini/Projects/GIS_ZHKH/{folder_name}/details/{inner_scr}]]\n\n## Файлы\n")
                    if files:
                        for fn in files: f_out.write(f"- `{fn}`\n")
                    else:
                        f_out.write("Файлов нет.\n")
                
                await send_tg_report(f"✅ Готово. Файлов: {len(files)}")
                await page.go_back()
                await human_delay(4, 7)
            except Exception as e:
                logger.error(f"Error on {txt}: {e}")
                await page.goto(url)
                await human_delay(5, 8)
                
        # Пагинация
        next_found = False
        for f in [page] + page.frames:
            try:
                btn = f.locator("a:has-text('следующая')").first
                if await btn.is_visible():
                    await btn.click()
                    page_num += 1
                    next_found = True
                    await human_delay(5, 8)
                    break
            except: pass
        
        if not next_found: break
        
    await send_tg_report(f"🏆 <b>Раздел {section_name} полностью выкачан!</b>")

async def phase_3_contracts(page):
    await send_tg_report("📂 <b>Захожу в раздел: Договора</b>")
    await page.goto("https://dom.gosuslugi.ru/#!/house-info")
    await human_delay(4, 6)
    
    # Клик по иконке раздела
    for f in [page] + page.frames:
        try:
            btn = f.locator("text='Информация о договорах по дому'").first
            if await btn.is_visible():
                await btn.click()
                await human_delay(4, 6)
                break
        except: pass
        
    # Клик Найти
    for f in [page] + page.frames:
        try:
            btn = f.locator("button:has-text('Найти')").first
            if await btn.is_visible():
                await btn.click()
                await human_delay(6, 10)
                break
        except: pass

    processed = set()
    page_num = 1
    
    while True:
        await send_tg_report(f"📄 Договора: Страница {page_num}...")
        await take_screenshot(page, f"Договора_Стр_{page_num}", "contracts/lists")
        
        while True:
            targets = []
            for f in [page] + page.frames:
                try:
                    elements = await f.locator("a").all()
                    for el in elements:
                        if await el.is_visible():
                            txt = (await el.inner_text()).strip()
                            if txt.startswith("Договор") and "Закон" not in txt and txt not in processed and len(txt) > 15:
                                targets.append((el, txt))
                except: pass
                if targets: break
                
            if not targets: break
            
            el, txt = targets[0]
            safe_name = "".join([c for c in txt if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
            processed.add(txt)
            
            await send_tg_report(f"⏳ Открываю договор: <code>{txt[:40]}...</code>")
            try:
                await el.click()
                await human_delay(5, 8)
                
                inner_scr = await take_screenshot(page, safe_name, "contracts/details")
                files = await download_all_files(page, f"contracts/{safe_name}")
                
                path = f"{OBSIDIAN_BASE}/contracts/{safe_name}.md"
                os.makedirs(os.path.dirname(path), exist_ok=True)
                with open(path, "w", encoding="utf-8") as f_out:
                    f_out.write(f"# {txt}\n\n![[Gemini/Projects/GIS_ZHKH/contracts/details/{inner_scr}]]\n\n## Файлы\n")
                    if files:
                        for fn in files: f_out.write(f"- `{fn}`\n")
                    else:
                        f_out.write("Файлов нет.\n")
                
                await send_tg_report(f"✅ Готово. Файлов: {len(files)}")
                
                # Клик по хлебным крошкам для возврата
                back_clicked = False
                for f in [page] + page.frames:
                    try:
                        bbtn = f.locator("text='Информация о договорах по дому'").first
                        if await bbtn.is_visible(timeout=1000):
                            await bbtn.click()
                            back_clicked = True
                            break
                    except: pass
                if not back_clicked:
                    await page.go_back()
                await human_delay(5, 8)
                
            except Exception as e:
                logger.error(f"Error on {txt}: {e}")
                await page.go_back()
                await human_delay(5, 8)
                
        # Пагинация
        next_found = False
        for f in [page] + page.frames:
            try:
                btn = f.locator("a:has-text('следующая')").first
                if await btn.is_visible():
                    await btn.click()
                    page_num += 1
                    next_found = True
                    await human_delay(5, 8)
                    break
            except: pass
        if not next_found: break

    await send_tg_report("🏆 <b>Раздел Договора полностью выкачан!</b>")

async def run_ultimate_mission():
    async with async_playwright() as p:
        context = await p.chromium.launch_persistent_context(
            user_data_dir=USER_DATA_DIR, headless=False, args=["--no-sandbox", "--disable-setuid-sandbox", "--start-maximized"]
        )
        page = context.pages[0] if context.pages else await context.new_page()
        await page.set_viewport_size({"width": 1600, "height": 1024})

        await send_tg_report("🚀 <b>ФИНАЛЬНЫЙ АГЕНТ ЗАПУЩЕН.</b> Жду сессию.")

        if await phase_0_login(page):
            # 1. Госпочта (Уведомления)
            await process_generic_list(page, "https://my.dom.gosuslugi.ru/citizen-cabinet/#!/hcsi/nnl?tab=INCOMING", "Госпочта (Уведомления)", "notifications")
            
            # 2. Обращения
            await process_generic_list(page, "https://my.dom.gosuslugi.ru/citizen-cabinet/#!/appeals", "Обращения", "appeals")
            
            # 3. Договоры
            await phase_3_contracts(page)
            
            await send_tg_report("🎉 <b>ВСЕ ЗАДАЧИ ВЫПОЛНЕНЫ! СВОБОДА!</b>")
        
        while True: await asyncio.sleep(3600)

if __name__ == "__main__":
    asyncio.run(run_ultimate_mission())