118 lines
5.9 KiB
Python
118 lines
5.9 KiB
Python
import os
|
||
import subprocess
|
||
import time
|
||
import re
|
||
from pathlib import Path
|
||
from playwright.sync_api import sync_playwright
|
||
|
||
# --- CONFIG ---
|
||
STORAGE_ROOT = Path("/app/services/gis_harvester/contracts")
|
||
NAS_IP = "192.168.10.105"
|
||
NAS_PATH = "/volume1/web/legal/gis/contracts"
|
||
BOT_TOKEN = "8725618164:AAH1tGalq-pw1l0t4P5c0sdLkCVJdh7IE7M"
|
||
CHAT_ID = "197957361"
|
||
|
||
def send_tg_status(msg):
|
||
subprocess.run(["curl", "-s", "-X", "POST", f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage",
|
||
"-d", f"chat_id={CHAT_ID}&text={msg}&parse_mode=markdown"])
|
||
|
||
def sync_to_nas(local_path, remote_folder):
|
||
target = f"{NAS_PATH}/{remote_folder}"
|
||
subprocess.run(["ssh", f"matrixhasyou@{NAS_IP}", f"mkdir -p '{target}'"])
|
||
for f in os.listdir(local_path):
|
||
src = local_path / f
|
||
dst = f"{target}/{f}"
|
||
with open(src, "rb") as fd:
|
||
subprocess.run(["ssh", f"matrixhasyou@{NAS_IP}", f"cat > '{dst}'"], stdin=fd)
|
||
|
||
def run_harvest():
|
||
with sync_playwright() as p:
|
||
try:
|
||
browser = p.chromium.connect_over_cdp("http://127.0.0.1:9222")
|
||
page = browser.contexts[0].pages[0]
|
||
|
||
print("[*] Ожидание загрузки списка договоров...")
|
||
# Ждем появления хотя бы одного элемента с текстом 'Договор'
|
||
page.locator("a:has-text('Договор')").first.wait_for(timeout=30000)
|
||
|
||
# Находим все ссылки
|
||
links_locators = page.locator("a:has-text('Договор')").all()
|
||
total = len(links_locators)
|
||
|
||
# Предварительно соберем тексты и номера, чтобы не зависеть от DOM
|
||
contract_tasks = []
|
||
for i in range(total):
|
||
text = links_locators[i].inner_text().strip()
|
||
# Извлекаем номер для папки
|
||
safe_name = text[:100].replace('/', '_').replace(' ', '_')
|
||
contract_tasks.append({'index': i, 'name': safe_name, 'original_text': text})
|
||
|
||
send_tg_status(f"📜 **ЖНЕЦ ДОГОВОРОВ v3.0**\nОбнаружено: {total} договоров.")
|
||
|
||
for task in contract_tasks:
|
||
try:
|
||
print(f"[*] Обработка: {task['name']}")
|
||
|
||
# Находим элемент по тексту (надежнее при обновлении SPA)
|
||
target_link = page.locator(f"a:has-text('{task['original_text']}')").first
|
||
target_link.click()
|
||
page.wait_for_load_state("networkidle")
|
||
time.sleep(15)
|
||
|
||
case_dir = STORAGE_ROOT / task['name']
|
||
case_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# 1. СКРИНШОТ
|
||
page.screenshot(path=str(case_dir / "contract_evidence.png"), full_page=True)
|
||
|
||
# 2. СКАЧИВАНИЕ
|
||
status = "⚠️ Только скрин"
|
||
# ГИС ЖКХ: В договорах файлы могут быть в разных вкладках.
|
||
# Пытаемся найти кнопку 'Скачать все'
|
||
download_btn = page.locator("button:has-text('Скачать все'), a:has-text('Скачать все')").first
|
||
|
||
if download_btn.is_visible():
|
||
with page.expect_download(timeout=120000) as download_info:
|
||
download_btn.click()
|
||
download = download_info.value
|
||
zip_path = case_dir / "contract_archive.zip"
|
||
download.save_as(str(zip_path))
|
||
status = "✅ ZIP + Скрин"
|
||
else:
|
||
# Попробуем найти одиночные файлы если нет 'Скачать все'
|
||
file_links = page.locator("a[title*='Скачать']").all()
|
||
if len(file_links) > 0:
|
||
for idx, fl in enumerate(file_links[:5]): # Ограничим первыми 5
|
||
with page.expect_download() as d_info:
|
||
fl.click()
|
||
d = d_info.value
|
||
d.save_as(str(case_dir / f"file_{idx}_{d.suggested_filename}"))
|
||
status = f"✅ {len(file_links)} файлов + Скрин"
|
||
|
||
# СИНХРОНИЗАЦИЯ
|
||
sync_to_nas(case_dir, task['name'])
|
||
|
||
# ОТЧЕТ
|
||
bar = "█" * ((task['index']+1)*10//total) + "░" * (10 - ((task['index']+1)*10//total))
|
||
send_tg_status(f"📜 **{task['name'][:50]}...**\nСтатус: {status}\n\n[{bar}] {task['index']+1}/{total}")
|
||
|
||
# НАЗАД
|
||
page.go_back()
|
||
time.sleep(10)
|
||
# Ждем возврата списка
|
||
page.locator("a:has-text('Договор')").first.wait_for()
|
||
|
||
except Exception as inner_e:
|
||
print(f"Ошибка на договоре {task['name']}: {inner_e}")
|
||
send_tg_status(f"⚠️ Пропуск {task['name'][:30]}: {str(inner_e)[:50]}")
|
||
page.goto("https://my.dom.gosuslugi.ru/citizen-cabinet/#!/agreements/citizen/4b1aae97-01e6-4083-bf0c-029d01bbda2a?apartmentNumber=152")
|
||
time.sleep(10)
|
||
|
||
send_tg_status("🏁 Жатва договоров ЗАВЕРШЕНА.")
|
||
browser.close()
|
||
except Exception as e:
|
||
send_tg_status(f"🚨 КАТАСТРОФА ЖНЕЦА v3.0: {str(e)}")
|
||
|
||
if __name__ == "__main__":
|
||
os.makedirs(STORAGE_ROOT, exist_ok=True)
|
||
run_harvest()
|