296 lines
13 KiB
Python
296 lines
13 KiB
Python
import asyncio
|
||
import os
|
||
import logging
|
||
import random
|
||
from datetime import datetime
|
||
from playwright.async_api import async_playwright
|
||
|
||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[logging.FileHandler("logs/contracts.log"), logging.StreamHandler()])
|
||
logger = logging.getLogger("GIS-Ultimate")
|
||
|
||
USER_DATA_DIR = "/app/browser_context"
|
||
OBSIDIAN_BASE = "/app/obsidian/Gemini/Projects/GIS_ZHKH"
|
||
NAS_DOWNLOADS = "/app/downloads/nas/gis_archive"
|
||
|
||
BLACKLIST = [
|
||
"Справка по системе", "Регламенты и инструкции", "Часто задаваемые вопросы",
|
||
"Карта сайта", "Версия для слабовидящих", "Обращение в службу поддержки",
|
||
"Личный кабинет", "Главная страница", "Помещения (жилые дома)",
|
||
"Информационные сообщения", "Обращения и ответы на них", "Подключенные ЛС",
|
||
"Оплата ЖКУ", "Аналитика и отчеты", "Форум", "Свернуть поиск", "Найти",
|
||
"Очистить", "предыдущая", "следующая", "Выводить по"
|
||
]
|
||
|
||
async def send_tg_report(message):
|
||
token = os.getenv("TELEGRAM_BOT_TOKEN")
|
||
chat_id = os.getenv("TELEGRAM_CHAT_ID")
|
||
if not token or not chat_id: return
|
||
import aiohttp
|
||
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
||
try:
|
||
async with aiohttp.ClientSession() as session:
|
||
await session.post(url, json={"chat_id": chat_id, "text": message, "parse_mode": "HTML"})
|
||
except: pass
|
||
|
||
async def human_delay(min_sec=4, max_sec=8):
|
||
await asyncio.sleep(random.uniform(min_sec, max_sec))
|
||
|
||
async def take_screenshot(page, title, subfolder):
|
||
safe_name = "".join([c for c in title if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
|
||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
filename = f"{safe_name}_{timestamp}.png"
|
||
path = f"{OBSIDIAN_BASE}/{subfolder}/{filename}"
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
await page.screenshot(path=path, full_page=True)
|
||
return filename
|
||
|
||
async def download_all_files(page, save_folder):
|
||
files_saved = []
|
||
dest_dir = f"{NAS_DOWNLOADS}/{save_folder}"
|
||
os.makedirs(dest_dir, exist_ok=True)
|
||
|
||
selectors = ["a[title*='Скачать']", "button:has-text('Скачать')", "a:has-text('Скачать все')", "a[href*='.pdf']", "a[href*='.zip']", "a[href*='.doc']", "a[href*='.rar']"]
|
||
|
||
for f in [page] + page.frames:
|
||
for selector in selectors:
|
||
try:
|
||
elements = await f.locator(selector).all()
|
||
for el in elements:
|
||
if not await el.is_visible(): continue
|
||
try:
|
||
async with page.expect_download(timeout=15000) as di:
|
||
await el.click()
|
||
download = await di.value
|
||
fname = download.suggested_filename
|
||
dest = f"{dest_dir}/{fname}"
|
||
await download.save_as(dest)
|
||
files_saved.append(fname)
|
||
except: continue
|
||
except: continue
|
||
return files_saved
|
||
|
||
async def phase_0_login(page):
|
||
await send_tg_report("🔍 <b>Инициализация. Проверка сессии.</b>")
|
||
await page.goto("https://dom.gosuslugi.ru/")
|
||
await human_delay()
|
||
|
||
for _ in range(30):
|
||
content = ""
|
||
for f in [page] + page.frames:
|
||
try: content += await f.content()
|
||
except: pass
|
||
|
||
if "Выйти" in content or "Александрович" in content:
|
||
await send_tg_report("✅ Сессия активна.")
|
||
return True
|
||
|
||
await asyncio.sleep(5)
|
||
return False
|
||
|
||
async def process_generic_list(page, url, section_name, folder_name, link_prefix=""):
|
||
await send_tg_report(f"📂 <b>Захожу в раздел: {section_name}</b>")
|
||
await page.goto(url)
|
||
await human_delay(5, 8)
|
||
|
||
processed = set()
|
||
page_num = 1
|
||
|
||
while True:
|
||
await send_tg_report(f"📄 Сканирую страницу {page_num}...")
|
||
await take_screenshot(page, f"Список_стр_{page_num}", f"{folder_name}/lists")
|
||
|
||
while True:
|
||
targets = []
|
||
for f in [page] + page.frames:
|
||
try:
|
||
elements = await f.locator("a").all()
|
||
for el in elements:
|
||
if await el.is_visible():
|
||
txt = (await el.inner_text()).strip()
|
||
if len(txt) > 10 and txt not in processed:
|
||
is_blacklisted = any(b in txt for b in BLACKLIST)
|
||
if not is_blacklisted:
|
||
if not link_prefix or txt.startswith(link_prefix):
|
||
targets.append((el, txt))
|
||
except: pass
|
||
if targets: break
|
||
|
||
if not targets: break # Все ссылки на этой странице обработаны
|
||
|
||
el, txt = targets[0]
|
||
safe_name = "".join([c for c in txt if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
|
||
processed.add(txt)
|
||
|
||
await send_tg_report(f"⏳ Открываю: <code>{txt[:40]}...</code>")
|
||
try:
|
||
await el.click()
|
||
await human_delay(4, 7)
|
||
|
||
inner_scr = await take_screenshot(page, safe_name, f"{folder_name}/details")
|
||
files = await download_all_files(page, f"{folder_name}/{safe_name}")
|
||
|
||
path = f"{OBSIDIAN_BASE}/{folder_name}/{safe_name}.md"
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
with open(path, "w", encoding="utf-8") as f_out:
|
||
f_out.write(f"# {txt}\n\n![[Gemini/Projects/GIS_ZHKH/{folder_name}/details/{inner_scr}]]\n\n## Файлы\n")
|
||
if files:
|
||
for fn in files: f_out.write(f"- `{fn}`\n")
|
||
else:
|
||
f_out.write("Файлов нет.\n")
|
||
|
||
await send_tg_report(f"✅ Готово. Файлов: {len(files)}")
|
||
await page.go_back()
|
||
await human_delay(4, 7)
|
||
except Exception as e:
|
||
logger.error(f"Error on {txt}: {e}")
|
||
await page.goto(url)
|
||
await human_delay(5, 8)
|
||
|
||
# Пагинация
|
||
next_found = False
|
||
for f in [page] + page.frames:
|
||
try:
|
||
btn = f.locator("a:has-text('следующая')").first
|
||
if await btn.is_visible():
|
||
await btn.click()
|
||
page_num += 1
|
||
next_found = True
|
||
await human_delay(5, 8)
|
||
break
|
||
except: pass
|
||
|
||
if not next_found: break
|
||
|
||
await send_tg_report(f"🏆 <b>Раздел {section_name} полностью выкачан!</b>")
|
||
|
||
async def phase_3_contracts(page):
|
||
await send_tg_report("📂 <b>Захожу в раздел: Договора</b>")
|
||
await page.goto("https://dom.gosuslugi.ru/#!/house-info")
|
||
await human_delay(4, 6)
|
||
|
||
# Клик по иконке раздела
|
||
for f in [page] + page.frames:
|
||
try:
|
||
btn = f.locator("text='Информация о договорах по дому'").first
|
||
if await btn.is_visible():
|
||
await btn.click()
|
||
await human_delay(4, 6)
|
||
break
|
||
except: pass
|
||
|
||
# Клик Найти
|
||
for f in [page] + page.frames:
|
||
try:
|
||
btn = f.locator("button:has-text('Найти')").first
|
||
if await btn.is_visible():
|
||
await btn.click()
|
||
await human_delay(6, 10)
|
||
break
|
||
except: pass
|
||
|
||
processed = set()
|
||
page_num = 1
|
||
|
||
while True:
|
||
await send_tg_report(f"📄 Договора: Страница {page_num}...")
|
||
await take_screenshot(page, f"Договора_Стр_{page_num}", "contracts/lists")
|
||
|
||
while True:
|
||
targets = []
|
||
for f in [page] + page.frames:
|
||
try:
|
||
elements = await f.locator("a").all()
|
||
for el in elements:
|
||
if await el.is_visible():
|
||
txt = (await el.inner_text()).strip()
|
||
if txt.startswith("Договор") and "Закон" not in txt and txt not in processed and len(txt) > 15:
|
||
targets.append((el, txt))
|
||
except: pass
|
||
if targets: break
|
||
|
||
if not targets: break
|
||
|
||
el, txt = targets[0]
|
||
safe_name = "".join([c for c in txt if c.isalnum() or c in (' ', '_', '-')]).strip().replace(" ", "_")
|
||
processed.add(txt)
|
||
|
||
await send_tg_report(f"⏳ Открываю договор: <code>{txt[:40]}...</code>")
|
||
try:
|
||
await el.click()
|
||
await human_delay(5, 8)
|
||
|
||
inner_scr = await take_screenshot(page, safe_name, "contracts/details")
|
||
files = await download_all_files(page, f"contracts/{safe_name}")
|
||
|
||
path = f"{OBSIDIAN_BASE}/contracts/{safe_name}.md"
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
with open(path, "w", encoding="utf-8") as f_out:
|
||
f_out.write(f"# {txt}\n\n![[Gemini/Projects/GIS_ZHKH/contracts/details/{inner_scr}]]\n\n## Файлы\n")
|
||
if files:
|
||
for fn in files: f_out.write(f"- `{fn}`\n")
|
||
else:
|
||
f_out.write("Файлов нет.\n")
|
||
|
||
await send_tg_report(f"✅ Готово. Файлов: {len(files)}")
|
||
|
||
# Клик по хлебным крошкам для возврата
|
||
back_clicked = False
|
||
for f in [page] + page.frames:
|
||
try:
|
||
bbtn = f.locator("text='Информация о договорах по дому'").first
|
||
if await bbtn.is_visible(timeout=1000):
|
||
await bbtn.click()
|
||
back_clicked = True
|
||
break
|
||
except: pass
|
||
if not back_clicked:
|
||
await page.go_back()
|
||
await human_delay(5, 8)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error on {txt}: {e}")
|
||
await page.go_back()
|
||
await human_delay(5, 8)
|
||
|
||
# Пагинация
|
||
next_found = False
|
||
for f in [page] + page.frames:
|
||
try:
|
||
btn = f.locator("a:has-text('следующая')").first
|
||
if await btn.is_visible():
|
||
await btn.click()
|
||
page_num += 1
|
||
next_found = True
|
||
await human_delay(5, 8)
|
||
break
|
||
except: pass
|
||
if not next_found: break
|
||
|
||
await send_tg_report("🏆 <b>Раздел Договора полностью выкачан!</b>")
|
||
|
||
async def run_ultimate_mission():
|
||
async with async_playwright() as p:
|
||
context = await p.chromium.launch_persistent_context(
|
||
user_data_dir=USER_DATA_DIR, headless=False, args=["--no-sandbox", "--disable-setuid-sandbox", "--start-maximized"]
|
||
)
|
||
page = context.pages[0] if context.pages else await context.new_page()
|
||
await page.set_viewport_size({"width": 1600, "height": 1024})
|
||
|
||
await send_tg_report("🚀 <b>ФИНАЛЬНЫЙ АГЕНТ ЗАПУЩЕН.</b> Жду сессию.")
|
||
|
||
if await phase_0_login(page):
|
||
# 1. Госпочта (Уведомления)
|
||
await process_generic_list(page, "https://my.dom.gosuslugi.ru/citizen-cabinet/#!/hcsi/nnl?tab=INCOMING", "Госпочта (Уведомления)", "notifications")
|
||
|
||
# 2. Обращения
|
||
await process_generic_list(page, "https://my.dom.gosuslugi.ru/citizen-cabinet/#!/appeals", "Обращения", "appeals")
|
||
|
||
# 3. Договоры
|
||
await phase_3_contracts(page)
|
||
|
||
await send_tg_report("🎉 <b>ВСЕ ЗАДАЧИ ВЫПОЛНЕНЫ! СВОБОДА!</b>")
|
||
|
||
while True: await asyncio.sleep(3600)
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(run_ultimate_mission())
|