98 lines
4.7 KiB
Python
98 lines
4.7 KiB
Python
import os, subprocess, time, re
|
|
from pathlib import Path
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
STORAGE_ROOT = Path("/app/services/gis_harvester/contracts")
|
|
NAS_IP = "192.168.10.105"
|
|
NAS_PATH = "/volume1/web/legal/gis/contracts"
|
|
BOT_TOKEN = "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M"
|
|
CHAT_ID = "197957361"
|
|
|
|
def send_tg(msg):
|
|
subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage",
|
|
"-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"])
|
|
|
|
def sync_to_nas(local_path, remote_folder):
|
|
target = f"{NAS_PATH}/{remote_folder}"
|
|
subprocess.run(["ssh", f"matrixhasyou@{NAS_IP}", f"mkdir -p '{target}'"])
|
|
for f in os.listdir(local_path):
|
|
src = local_path / f
|
|
dst = f"{target}/{f}"
|
|
with open(src, "rb") as fd:
|
|
subprocess.run(["ssh", f"matrixhasyou@{NAS_IP}", f"cat > '{dst}'"], stdin=fd)
|
|
|
|
def run_harvest():
|
|
with sync_playwright() as p:
|
|
try:
|
|
browser = p.chromium.connect_over_cdp("http://127.0.0.1:9222")
|
|
page = browser.contexts[0].pages[0]
|
|
|
|
# Переход в раздел договоров
|
|
page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/agreements/citizen/4b1aae97-01e6-4083-bf0c-029d01bbda2a?apartmentNumber=152")
|
|
page.wait_for_load_state("networkidle")
|
|
time.sleep(10)
|
|
|
|
# Собираем задачи
|
|
page.locator("a:has-text('Договор')").first.wait_for(timeout=30000)
|
|
links = page.locator("a:has-text('Договор')").all()
|
|
tasks = []
|
|
for i, l in enumerate(links):
|
|
text = l.inner_text().strip()
|
|
tasks.append({'name': text[:100].replace('/', '_').replace(' ', '_'), 'text': text, 'index': i})
|
|
|
|
send_tg(f"🛡️ **ЖНЕЦ ДОГОВОРОВ v4.0 (Anti-404)**\nЦелей: {len(tasks)}")
|
|
|
|
for t in tasks:
|
|
try:
|
|
print(f"[*] Обработка: {t['name']}")
|
|
# Кликаем и ПРОВЕРЯЕМ на 404
|
|
retry_count = 0
|
|
success_load = False
|
|
while retry_count < 3 and not success_load:
|
|
page.locator(f"a:has-text('{t['text']}')").first.click()
|
|
page.wait_for_load_state("networkidle")
|
|
time.sleep(10)
|
|
|
|
if "404" in page.content() or "не найдена" in page.content():
|
|
print(f"[!] Детект 404 на {t['name']}, рефреш...")
|
|
page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/agreements/citizen/4b1aae97-01e6-4083-bf0c-029d01bbda2a?apartmentNumber=152")
|
|
time.sleep(10)
|
|
retry_count += 1
|
|
else:
|
|
success_load = True
|
|
|
|
if not success_load:
|
|
send_tg(f"❌ Пропуск {t['name']} (Вечный 404)")
|
|
continue
|
|
|
|
case_dir = STORAGE_ROOT / t['name']
|
|
case_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Ждем контент перед скрином
|
|
page.locator("h1, .agreement-view-header, :has-text('Информация')").first.wait_for(timeout=20000)
|
|
page.screenshot(path=str(case_dir / "evidence.png"), full_page=True)
|
|
|
|
# Качаем
|
|
status = "⚠️ Скрин OK"
|
|
btn = page.locator("button:has-text('Скачать все'), a:has-text('Скачать все')").first
|
|
if btn.is_visible():
|
|
with page.expect_download(timeout=120000) as d_info:
|
|
btn.click()
|
|
d_info.value.save_as(str(case_dir / "archive.zip"))
|
|
status = "✅ ZIP + Скрин"
|
|
|
|
sync_to_nas(case_dir, t['name'])
|
|
send_tg(f"📜 **{t['name'][:40]}**\nСтатус: {status} [{t['index']+1}/{len(tasks)}]")
|
|
|
|
page.go_back()
|
|
time.sleep(10)
|
|
except Exception as e:
|
|
send_tg(f"⚠️ Ошибка на {t['name'][:30]}: {str(e)[:50]}")
|
|
page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/agreements/citizen/4b1aae97-01e6-4083-bf0c-029d01bbda2a?apartmentNumber=152")
|
|
time.sleep(10)
|
|
|
|
send_tg("🏁 Жатва завершена.")
|
|
except Exception as e:
|
|
send_tg(f"🚨 КАТАСТРОФА v4.0: {str(e)}")
|
|
|
|
run_harvest()
|