feat: Implement robust search and inline accordion details scraper for sudrf appeals
This commit is contained in:
Admin
parent
c1ad8b3d4c
commit
7101e617ab
1 changed files with 152 additions and 149 deletions
|
|
@ -36,7 +36,7 @@ async def human_delay(min_sec=3, max_sec=6):
|
||||||
|
|
||||||
async def run_mission():
|
async def run_mission():
|
||||||
logger.info("Starting Playwright in VNC display...")
|
logger.info("Starting Playwright in VNC display...")
|
||||||
send_tg("🕵️♂️ **ГАС Правосудие: Запуск браузера**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html")
|
send_tg("🕵️♂️ **ГАС Правосудие: Запуск робота-разведчика**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html")
|
||||||
|
|
||||||
async with async_playwright() as p:
|
async with async_playwright() as p:
|
||||||
# Launch persistent browser context
|
# Launch persistent browser context
|
||||||
|
|
@ -52,179 +52,182 @@ async def run_mission():
|
||||||
# Navigate to portal
|
# Navigate to portal
|
||||||
logger.info("Navigating to sudrf...")
|
logger.info("Navigating to sudrf...")
|
||||||
await page.goto("https://ej.sudrf.ru/")
|
await page.goto("https://ej.sudrf.ru/")
|
||||||
await human_delay(4, 7)
|
await asyncio.sleep(8)
|
||||||
|
|
||||||
# Phase 0: Wait for login
|
# Check login screen
|
||||||
logger.info("Waiting for user authorization...")
|
url = page.url
|
||||||
logged_in = False
|
try:
|
||||||
for i in range(120): # 10 minutes total
|
visible_text = await page.locator("body").inner_text()
|
||||||
content = ""
|
except Exception:
|
||||||
|
visible_text = ""
|
||||||
|
|
||||||
|
logger.info(f"Current page URL: {url}")
|
||||||
|
|
||||||
|
is_logged_in = "sudrf.ru" in url and any(ind in visible_text for ind in ["Выйти", "Выход", "Личный кабинет", "Тырин"]) and "Авторизация" not in visible_text
|
||||||
|
|
||||||
|
if not is_logged_in:
|
||||||
|
logger.info("Login/Agreement page detected. Auto-accepting terms...")
|
||||||
try:
|
try:
|
||||||
content = await page.content()
|
checkbox = await page.query_selector("input[type='checkbox']")
|
||||||
|
if checkbox:
|
||||||
|
await checkbox.check()
|
||||||
|
logger.info("Checked user agreement checkbox.")
|
||||||
|
else:
|
||||||
|
await page.click("text=Я ознакомился(ась)", timeout=5000)
|
||||||
|
logger.info("Clicked user agreement label.")
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
|
||||||
|
buttons = await page.query_selector_all("button, .btn, a.btn")
|
||||||
|
for b in buttons:
|
||||||
|
text = (await b.inner_text()).strip()
|
||||||
|
if "Войти" in text:
|
||||||
|
logger.info(f"Clicking Войти button: '{text}'")
|
||||||
|
await b.click()
|
||||||
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error reading page content: {e}")
|
logger.info(f"Note (agreement handling): {e}")
|
||||||
|
|
||||||
# Check for common logout or profile indicators
|
logger.info("Waiting up to 300s for user authorization...")
|
||||||
if any(indicator in content for indicator in ["Выйти", "Выход", "Личный кабинет", "История обращений"]):
|
logged_in_False = True
|
||||||
logged_in = True
|
for i in range(60):
|
||||||
break
|
await asyncio.sleep(5)
|
||||||
|
current_url = page.url
|
||||||
|
try:
|
||||||
|
current_text = await page.locator("body").inner_text()
|
||||||
|
except Exception:
|
||||||
|
current_text = ""
|
||||||
|
|
||||||
if i % 6 == 0:
|
if "sudrf.ru" in current_url and any(ind in current_text for ind in ["Выйти", "Выход", "Личный кабинет", "Тырин"]):
|
||||||
logger.info(f"Still waiting... i={i}")
|
logger.info(f"Login detected! URL: {current_url}")
|
||||||
await asyncio.sleep(5)
|
logged_in_False = False
|
||||||
|
break
|
||||||
|
if i % 6 == 0:
|
||||||
|
logger.info(f"Still waiting... {i*5}s")
|
||||||
|
if logged_in_False:
|
||||||
|
logger.error("Login timeout exceeded.")
|
||||||
|
send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.")
|
||||||
|
await context.close()
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
logger.info("Already logged in.")
|
||||||
|
|
||||||
if not logged_in:
|
# Navigate to appeals history page
|
||||||
logger.error("Login timeout exceeded.")
|
logger.info("Navigating to appeals history...")
|
||||||
send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.")
|
send_tg("✅ **Авторизация подтверждена!** Загружаю историю обращений...")
|
||||||
|
await page.goto("https://ej.sudrf.ru/appeals/")
|
||||||
|
await asyncio.sleep(8)
|
||||||
|
|
||||||
|
# Click search button
|
||||||
|
logger.info("Locating Search button...")
|
||||||
|
buttons = await page.query_selector_all("button, input[type='submit'], input[type='button'], .btn, a.btn")
|
||||||
|
search_clicked = False
|
||||||
|
for b in buttons:
|
||||||
|
try:
|
||||||
|
text = (await b.inner_text()).strip()
|
||||||
|
if "Найти" in text:
|
||||||
|
logger.info(f"Clicking Search button: '{text}'")
|
||||||
|
await b.click(force=True)
|
||||||
|
search_clicked = True
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error inspecting search button: {e}")
|
||||||
|
|
||||||
|
if not search_clicked:
|
||||||
|
logger.error("Could not click Search button.")
|
||||||
|
send_tg("❌ **Ошибка**: Не удалось нажать кнопку Поиск для загрузки списка обращений.")
|
||||||
|
await page.screenshot(path=os.path.join(EXPORT_DIR, "search_failed.png"), full_page=True)
|
||||||
await context.close()
|
await context.close()
|
||||||
return
|
return
|
||||||
|
|
||||||
logger.info("Login detected. Navigating to appeals history...")
|
logger.info("Waiting for search results...")
|
||||||
send_tg("✅ **Авторизация успешна!** Начинаю сбор обращений...")
|
await asyncio.sleep(8)
|
||||||
|
|
||||||
await page.goto("https://ej.sudrf.ru/appeals/")
|
# Check if table loaded
|
||||||
await asyncio.sleep(10) # wait for page to render appeals list
|
tbodies = await page.query_selector_all("table.table-history tbody")
|
||||||
|
logger.info(f"Found {len(tbodies)} tbodies in history table.")
|
||||||
|
|
||||||
# Take list page screenshot
|
if not tbodies:
|
||||||
list_screenshot = os.path.join(EXPORT_DIR, "appeals_list.png")
|
logger.warning("No tbodies found in history table.")
|
||||||
await page.screenshot(path=list_screenshot, full_page=True)
|
send_tg("⚠️ **Предупреждение**: История обращений пуста или не загрузилась.")
|
||||||
|
await page.screenshot(path=os.path.join(EXPORT_DIR, "empty_table.png"), full_page=True)
|
||||||
|
await context.close()
|
||||||
|
return
|
||||||
|
|
||||||
# Dump HTML for structural analysis
|
# Process each appeal / tbody
|
||||||
list_html = os.path.join(EXPORT_DIR, "appeals_list.html")
|
|
||||||
with open(list_html, "w", encoding="utf-8") as f:
|
|
||||||
f.write(await page.content())
|
|
||||||
|
|
||||||
logger.info(f"Appeals list screenshot saved to {list_screenshot}")
|
|
||||||
|
|
||||||
# Extract appeal links
|
|
||||||
links = []
|
|
||||||
all_links = await page.query_selector_all("a")
|
|
||||||
for link in all_links:
|
|
||||||
href = await link.get_attribute("href")
|
|
||||||
text = await link.inner_text()
|
|
||||||
if href and ("appeal" in href or "card" in href or "request" in href):
|
|
||||||
# Исключаем общие страницы
|
|
||||||
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
|
|
||||||
links.append({"href": href, "text": text.strip()})
|
|
||||||
|
|
||||||
logger.info(f"Found {len(links)} matching links on page: {links}")
|
|
||||||
|
|
||||||
# Process each appeal
|
|
||||||
# We can extract rows from the table to get exact dates, numbers and link objects
|
|
||||||
# For sudrf, appeals table often has tr tags with status and numbers
|
|
||||||
rows = await page.query_selector_all("tr, .appeals-item")
|
|
||||||
logger.info(f"Found {len(rows)} rows/items in appeals list.")
|
|
||||||
|
|
||||||
# Dump structured report of links to a json file
|
|
||||||
import json
|
|
||||||
with open(os.path.join(EXPORT_DIR, "found_links.json"), "w", encoding="utf-8") as f:
|
|
||||||
json.dump({"links": links, "rows_count": len(rows)}, f, indent=4, ensure_ascii=False)
|
|
||||||
|
|
||||||
# Try to parse the table rows if they contain appeal details
|
|
||||||
parsed_appeals = []
|
|
||||||
for index, row in enumerate(rows[:20]): # parse top 20 appeals first for safety
|
|
||||||
cells = await row.query_selector_all("td")
|
|
||||||
if not cells: continue
|
|
||||||
|
|
||||||
row_data = []
|
|
||||||
for cell in cells:
|
|
||||||
row_data.append((await cell.inner_text()).strip())
|
|
||||||
|
|
||||||
# Look for link inside the row
|
|
||||||
row_link = await row.query_selector("a")
|
|
||||||
href = await row_link.get_attribute("href") if row_link else None
|
|
||||||
|
|
||||||
parsed_appeals.append({
|
|
||||||
"row_index": index,
|
|
||||||
"data": row_data,
|
|
||||||
"href": href
|
|
||||||
})
|
|
||||||
|
|
||||||
with open(os.path.join(EXPORT_DIR, "parsed_table.json"), "w", encoding="utf-8") as f:
|
|
||||||
json.dump(parsed_appeals, f, indent=4, ensure_ascii=False)
|
|
||||||
|
|
||||||
# Loop through found details pages and download documents
|
|
||||||
count = 0
|
count = 0
|
||||||
processed_hrefs = set()
|
for index, tbody in enumerate(tbodies[:15]): # Process top 15 appeals
|
||||||
|
main_tr = await tbody.query_selector("tr:not(.tr-expanded)")
|
||||||
|
if not main_tr:
|
||||||
|
continue
|
||||||
|
|
||||||
# Collect actual detail pages
|
cells = await main_tr.query_selector_all("td")
|
||||||
detail_hrefs = []
|
if len(cells) < 5:
|
||||||
for appeal in parsed_appeals:
|
continue
|
||||||
href = appeal.get("href")
|
|
||||||
if href and href not in processed_hrefs:
|
|
||||||
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
|
|
||||||
detail_hrefs.append(href)
|
|
||||||
processed_hrefs.add(href)
|
|
||||||
|
|
||||||
# Also check all links for hrefs like #/appeals/view/ or similar
|
# Get registration number details
|
||||||
for link in links:
|
num_text = (await cells[1].inner_text()).strip()
|
||||||
href = link["href"]
|
# Clean registration number to get safe directory ID
|
||||||
if href and href not in processed_hrefs:
|
safe_id = "".join([c for c in num_text if c.isalnum() or c in ("-", "_")]).strip()
|
||||||
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
|
if not safe_id:
|
||||||
detail_hrefs.append(href)
|
safe_id = f"row_{index}"
|
||||||
processed_hrefs.add(href)
|
|
||||||
|
|
||||||
logger.info(f"Unique detail pages to process: {detail_hrefs}")
|
appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}")
|
||||||
|
os.makedirs(appeal_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# Find and click "История рассмотрения" link in 5th cell
|
||||||
|
history_link = await cells[4].query_selector("a:has-text('История рассмотрения')")
|
||||||
|
if not history_link:
|
||||||
|
logger.info(f"No history link for row {index}, safe_id={safe_id}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.info(f"Processing appeal {safe_id} (row {index})...")
|
||||||
|
|
||||||
for href in detail_hrefs[:10]: # Safe limit to top 10 appeals in recon mode
|
|
||||||
try:
|
try:
|
||||||
full_url = href
|
# Expand
|
||||||
if href.startswith("#"):
|
await history_link.click(force=True)
|
||||||
full_url = f"https://ej.sudrf.ru/{href}"
|
await asyncio.sleep(4)
|
||||||
elif href.startswith("/"):
|
|
||||||
full_url = f"https://ej.sudrf.ru{href}"
|
|
||||||
|
|
||||||
logger.info(f"Navigating to detail page: {full_url}")
|
# Take screenshot of expanded tbody segment
|
||||||
await page.goto(full_url)
|
evidence_img = os.path.join(appeal_dir, "evidence.png")
|
||||||
await asyncio.sleep(8) # Wait for page load
|
await tbody.screenshot(path=evidence_img)
|
||||||
|
|
||||||
safe_id = "".join([c for c in href if c.isalnum() or c in ("-", "_")]).strip()
|
# Save tbody HTML
|
||||||
appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}")
|
|
||||||
os.makedirs(appeal_dir, exist_ok=True)
|
|
||||||
|
|
||||||
# Take detail screenshot
|
|
||||||
detail_screenshot = os.path.join(appeal_dir, "evidence.png")
|
|
||||||
await page.screenshot(path=detail_screenshot, full_page=True)
|
|
||||||
|
|
||||||
# Dump detail HTML
|
|
||||||
detail_html = os.path.join(appeal_dir, "page.html")
|
detail_html = os.path.join(appeal_dir, "page.html")
|
||||||
with open(detail_html, "w", encoding="utf-8") as f:
|
with open(detail_html, "w", encoding="utf-8") as f:
|
||||||
f.write(await page.content())
|
f.write(await tbody.inner_html())
|
||||||
|
|
||||||
# Look for download links
|
|
||||||
download_selectors = [
|
|
||||||
"a[href*='download']",
|
|
||||||
"a[href*='file']",
|
|
||||||
"button:has-text('Скачать')",
|
|
||||||
"a:has-text('Скачать')"
|
|
||||||
]
|
|
||||||
|
|
||||||
|
# Download attachments
|
||||||
|
file_links = await tbody.query_selector_all("a[href*='getFile']")
|
||||||
files_downloaded = []
|
files_downloaded = []
|
||||||
for selector in download_selectors:
|
|
||||||
elements = await page.locator(selector).all()
|
for file_link in file_links:
|
||||||
for el in elements:
|
file_text = (await file_link.inner_text()).strip()
|
||||||
if not await el.is_visible(): continue
|
logger.info(f"Downloading file: '{file_text}'")
|
||||||
try:
|
|
||||||
async with page.expect_download(timeout=15000) as di:
|
try:
|
||||||
await el.click(force=True)
|
async with page.expect_download(timeout=15000) as download_info:
|
||||||
download = await di.value
|
await file_link.click()
|
||||||
fname = download.suggested_filename
|
download = await download_info.value
|
||||||
dest = os.path.join(appeal_dir, fname)
|
dest = os.path.join(appeal_dir, download.suggested_filename)
|
||||||
await download.save_as(dest)
|
await download.save_as(dest)
|
||||||
files_downloaded.append(fname)
|
files_downloaded.append(download.suggested_filename)
|
||||||
logger.info(f"Downloaded: {fname}")
|
logger.info(f"Saved: {download.suggested_filename}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed download click: {e}")
|
logger.warning(f"Error downloading file {file_text}: {e}")
|
||||||
|
|
||||||
count += 1
|
count += 1
|
||||||
send_tg(f"📥 **Обращение {count} собрано**\nПапка: `appeal_{safe_id}`\nФайлов скачано: {len(files_downloaded)}")
|
send_tg(f"📥 **Обращение {count} скачано: № {safe_id}**\nФайлов: {len(files_downloaded)}")
|
||||||
|
|
||||||
|
# Collapse
|
||||||
|
await history_link.click(force=True)
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error processing appeal details {href}: {e}")
|
logger.error(f"Error parsing appeal {safe_id}: {e}")
|
||||||
|
|
||||||
send_tg(f"🏁 **Миссия ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.")
|
send_tg(f"🏁 **Жатва ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.")
|
||||||
|
|
||||||
# Trigger PDF compilation script
|
# Trigger compiler
|
||||||
import subprocess
|
import subprocess
|
||||||
subprocess.Popen(["python3", "/app/sudrf_pdf_compiler.py"], start_new_session=True)
|
subprocess.Popen(["python3", "/app/sudrf_pdf_compiler.py"], start_new_session=True)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue