76 lines
3.3 KiB
Python
76 lines
3.3 KiB
Python
import json, os, subprocess, time, re
|
||
from pathlib import Path
|
||
from playwright.sync_api import sync_playwright
|
||
|
||
QUEUE_FILE = "/app/services/gis_harvester/data/full_queue.json"
|
||
STORAGE_ROOT = Path("/app/services/gis_harvester/archive")
|
||
NAS_IP = "192.168.10.105"
|
||
NAS_LEGAL_RAW = "/volume1/web/legal/gis/raw"
|
||
BOT_TOKEN = "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M"
|
||
CHAT_ID = "197957361"
|
||
|
||
def send_tg(msg):
|
||
subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage",
|
||
"-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"])
|
||
|
||
def sync_to_nas_ssh(local_path, remote_folder):
|
||
target = f"{NAS_LEGAL_RAW}/{remote_folder}"
|
||
subprocess.run(["ssh", f"matrixhasyou@{NAS_IP}", f"mkdir -p '{target}'"])
|
||
for f in os.listdir(local_path):
|
||
src = local_path / f
|
||
if os.path.isfile(src):
|
||
dst = f"{target}/{f}"
|
||
with open(src, "rb") as fd:
|
||
subprocess.run(["ssh", f"matrixhasyou@{NAS_IP}", f"cat > '{dst}'"], stdin=fd)
|
||
|
||
def run_harvest():
|
||
with open(QUEUE_FILE, "r") as f: queue = json.load(f)
|
||
total = len(queue)
|
||
send_tg(f"🔥 **ЖНЕЦ v5.1 (LEGAL ARCHIVE MODE)**\nЦель: /volume1/web/legal/gis/raw")
|
||
|
||
with sync_playwright() as p:
|
||
try:
|
||
browser = p.chromium.connect_over_cdp("http://127.0.0.1:9222")
|
||
page = browser.contexts[0].pages[0]
|
||
|
||
for index, item in enumerate(queue):
|
||
num = item['number'].replace('/', '_').replace(' ', '_')[:50]
|
||
local_dir = STORAGE_ROOT / num
|
||
local_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
page.goto("https://my.dom.gosuslugi.ru" + item['href'])
|
||
page.wait_for_load_state("networkidle")
|
||
time.sleep(15)
|
||
|
||
if "404" in page.content() or "не найдена" in page.content():
|
||
page.reload()
|
||
time.sleep(15)
|
||
|
||
page.screenshot(path=str(local_dir / "evidence.png"), full_page=True)
|
||
|
||
status = "📸 Скрин OK"
|
||
btn = page.locator("button:has-text('Скачать все'), a:has-text('Скачать все')").first
|
||
if btn.is_visible():
|
||
with page.expect_download(timeout=120000) as d_info:
|
||
btn.click()
|
||
d_info.value.save_as(str(local_dir / "attachments.zip"))
|
||
subprocess.run(["unzip", "-o", str(local_dir / "attachments.zip"), "-d", str(local_dir)])
|
||
status = "✅ ZIP + Скрин"
|
||
|
||
# СИНХРОНИЗАЦИЯ ЧЕРЕЗ SSH (НЕ В YOUTUBE!)
|
||
sync_to_nas_ssh(local_dir, num)
|
||
|
||
bar = "█" * ((index + 1) * 10 // total) + "░" * (10 - ((index + 1) * 10 // total))
|
||
send_tg(f"🌾 **ЖНЕЦ 5.1**\n🎯 {num}\n{status}\n[{bar}] {index+1}/{total}")
|
||
|
||
except Exception as e:
|
||
send_tg(f"❌ Ошибка на {num}: {str(e)[:50]}")
|
||
page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/Pafo/Home")
|
||
time.sleep(10)
|
||
|
||
browser.close()
|
||
except Exception as e:
|
||
send_tg(f"🚨 КАТАСТРОФА v5.1: {str(e)}")
|
||
|
||
run_harvest()
|