feat: Implement robust search and inline accordion details scraper for sudrf appeals
This commit is contained in:
Admin
parent
c1ad8b3d4c
commit
7101e617ab
1 changed files with 152 additions and 149 deletions
|
|
@ -36,7 +36,7 @@ async def human_delay(min_sec=3, max_sec=6):
|
|||
|
||||
async def run_mission():
|
||||
logger.info("Starting Playwright in VNC display...")
|
||||
send_tg("🕵️♂️ **ГАС Правосудие: Запуск браузера**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html")
|
||||
send_tg("🕵️♂️ **ГАС Правосудие: Запуск робота-разведчика**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html")
|
||||
|
||||
async with async_playwright() as p:
|
||||
# Launch persistent browser context
|
||||
|
|
@ -52,179 +52,182 @@ async def run_mission():
|
|||
# Navigate to portal
|
||||
logger.info("Navigating to sudrf...")
|
||||
await page.goto("https://ej.sudrf.ru/")
|
||||
await human_delay(4, 7)
|
||||
await asyncio.sleep(8)
|
||||
|
||||
# Phase 0: Wait for login
|
||||
logger.info("Waiting for user authorization...")
|
||||
logged_in = False
|
||||
for i in range(120): # 10 minutes total
|
||||
content = ""
|
||||
# Check login screen
|
||||
url = page.url
|
||||
try:
|
||||
content = await page.content()
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading page content: {e}")
|
||||
visible_text = await page.locator("body").inner_text()
|
||||
except Exception:
|
||||
visible_text = ""
|
||||
|
||||
# Check for common logout or profile indicators
|
||||
if any(indicator in content for indicator in ["Выйти", "Выход", "Личный кабинет", "История обращений"]):
|
||||
logged_in = True
|
||||
logger.info(f"Current page URL: {url}")
|
||||
|
||||
is_logged_in = "sudrf.ru" in url and any(ind in visible_text for ind in ["Выйти", "Выход", "Личный кабинет", "Тырин"]) and "Авторизация" not in visible_text
|
||||
|
||||
if not is_logged_in:
|
||||
logger.info("Login/Agreement page detected. Auto-accepting terms...")
|
||||
try:
|
||||
checkbox = await page.query_selector("input[type='checkbox']")
|
||||
if checkbox:
|
||||
await checkbox.check()
|
||||
logger.info("Checked user agreement checkbox.")
|
||||
else:
|
||||
await page.click("text=Я ознакомился(ась)", timeout=5000)
|
||||
logger.info("Clicked user agreement label.")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
buttons = await page.query_selector_all("button, .btn, a.btn")
|
||||
for b in buttons:
|
||||
text = (await b.inner_text()).strip()
|
||||
if "Войти" in text:
|
||||
logger.info(f"Clicking Войти button: '{text}'")
|
||||
await b.click()
|
||||
break
|
||||
except Exception as e:
|
||||
logger.info(f"Note (agreement handling): {e}")
|
||||
|
||||
if i % 6 == 0:
|
||||
logger.info(f"Still waiting... i={i}")
|
||||
logger.info("Waiting up to 300s for user authorization...")
|
||||
logged_in_False = True
|
||||
for i in range(60):
|
||||
await asyncio.sleep(5)
|
||||
current_url = page.url
|
||||
try:
|
||||
current_text = await page.locator("body").inner_text()
|
||||
except Exception:
|
||||
current_text = ""
|
||||
|
||||
if not logged_in:
|
||||
if "sudrf.ru" in current_url and any(ind in current_text for ind in ["Выйти", "Выход", "Личный кабинет", "Тырин"]):
|
||||
logger.info(f"Login detected! URL: {current_url}")
|
||||
logged_in_False = False
|
||||
break
|
||||
if i % 6 == 0:
|
||||
logger.info(f"Still waiting... {i*5}s")
|
||||
if logged_in_False:
|
||||
logger.error("Login timeout exceeded.")
|
||||
send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.")
|
||||
await context.close()
|
||||
return
|
||||
else:
|
||||
logger.info("Already logged in.")
|
||||
|
||||
logger.info("Login detected. Navigating to appeals history...")
|
||||
send_tg("✅ **Авторизация успешна!** Начинаю сбор обращений...")
|
||||
|
||||
# Navigate to appeals history page
|
||||
logger.info("Navigating to appeals history...")
|
||||
send_tg("✅ **Авторизация подтверждена!** Загружаю историю обращений...")
|
||||
await page.goto("https://ej.sudrf.ru/appeals/")
|
||||
await asyncio.sleep(10) # wait for page to render appeals list
|
||||
await asyncio.sleep(8)
|
||||
|
||||
# Take list page screenshot
|
||||
list_screenshot = os.path.join(EXPORT_DIR, "appeals_list.png")
|
||||
await page.screenshot(path=list_screenshot, full_page=True)
|
||||
|
||||
# Dump HTML for structural analysis
|
||||
list_html = os.path.join(EXPORT_DIR, "appeals_list.html")
|
||||
with open(list_html, "w", encoding="utf-8") as f:
|
||||
f.write(await page.content())
|
||||
|
||||
logger.info(f"Appeals list screenshot saved to {list_screenshot}")
|
||||
|
||||
# Extract appeal links
|
||||
links = []
|
||||
all_links = await page.query_selector_all("a")
|
||||
for link in all_links:
|
||||
href = await link.get_attribute("href")
|
||||
text = await link.inner_text()
|
||||
if href and ("appeal" in href or "card" in href or "request" in href):
|
||||
# Исключаем общие страницы
|
||||
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
|
||||
links.append({"href": href, "text": text.strip()})
|
||||
|
||||
logger.info(f"Found {len(links)} matching links on page: {links}")
|
||||
|
||||
# Process each appeal
|
||||
# We can extract rows from the table to get exact dates, numbers and link objects
|
||||
# For sudrf, appeals table often has tr tags with status and numbers
|
||||
rows = await page.query_selector_all("tr, .appeals-item")
|
||||
logger.info(f"Found {len(rows)} rows/items in appeals list.")
|
||||
|
||||
# Dump structured report of links to a json file
|
||||
import json
|
||||
with open(os.path.join(EXPORT_DIR, "found_links.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({"links": links, "rows_count": len(rows)}, f, indent=4, ensure_ascii=False)
|
||||
|
||||
# Try to parse the table rows if they contain appeal details
|
||||
parsed_appeals = []
|
||||
for index, row in enumerate(rows[:20]): # parse top 20 appeals first for safety
|
||||
cells = await row.query_selector_all("td")
|
||||
if not cells: continue
|
||||
|
||||
row_data = []
|
||||
for cell in cells:
|
||||
row_data.append((await cell.inner_text()).strip())
|
||||
|
||||
# Look for link inside the row
|
||||
row_link = await row.query_selector("a")
|
||||
href = await row_link.get_attribute("href") if row_link else None
|
||||
|
||||
parsed_appeals.append({
|
||||
"row_index": index,
|
||||
"data": row_data,
|
||||
"href": href
|
||||
})
|
||||
|
||||
with open(os.path.join(EXPORT_DIR, "parsed_table.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(parsed_appeals, f, indent=4, ensure_ascii=False)
|
||||
|
||||
# Loop through found details pages and download documents
|
||||
count = 0
|
||||
processed_hrefs = set()
|
||||
|
||||
# Collect actual detail pages
|
||||
detail_hrefs = []
|
||||
for appeal in parsed_appeals:
|
||||
href = appeal.get("href")
|
||||
if href and href not in processed_hrefs:
|
||||
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
|
||||
detail_hrefs.append(href)
|
||||
processed_hrefs.add(href)
|
||||
|
||||
# Also check all links for hrefs like #/appeals/view/ or similar
|
||||
for link in links:
|
||||
href = link["href"]
|
||||
if href and href not in processed_hrefs:
|
||||
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
|
||||
detail_hrefs.append(href)
|
||||
processed_hrefs.add(href)
|
||||
|
||||
logger.info(f"Unique detail pages to process: {detail_hrefs}")
|
||||
|
||||
for href in detail_hrefs[:10]: # Safe limit to top 10 appeals in recon mode
|
||||
# Click search button
|
||||
logger.info("Locating Search button...")
|
||||
buttons = await page.query_selector_all("button, input[type='submit'], input[type='button'], .btn, a.btn")
|
||||
search_clicked = False
|
||||
for b in buttons:
|
||||
try:
|
||||
full_url = href
|
||||
if href.startswith("#"):
|
||||
full_url = f"https://ej.sudrf.ru/{href}"
|
||||
elif href.startswith("/"):
|
||||
full_url = f"https://ej.sudrf.ru{href}"
|
||||
text = (await b.inner_text()).strip()
|
||||
if "Найти" in text:
|
||||
logger.info(f"Clicking Search button: '{text}'")
|
||||
await b.click(force=True)
|
||||
search_clicked = True
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error inspecting search button: {e}")
|
||||
|
||||
logger.info(f"Navigating to detail page: {full_url}")
|
||||
await page.goto(full_url)
|
||||
await asyncio.sleep(8) # Wait for page load
|
||||
if not search_clicked:
|
||||
logger.error("Could not click Search button.")
|
||||
send_tg("❌ **Ошибка**: Не удалось нажать кнопку Поиск для загрузки списка обращений.")
|
||||
await page.screenshot(path=os.path.join(EXPORT_DIR, "search_failed.png"), full_page=True)
|
||||
await context.close()
|
||||
return
|
||||
|
||||
logger.info("Waiting for search results...")
|
||||
await asyncio.sleep(8)
|
||||
|
||||
# Check if table loaded
|
||||
tbodies = await page.query_selector_all("table.table-history tbody")
|
||||
logger.info(f"Found {len(tbodies)} tbodies in history table.")
|
||||
|
||||
if not tbodies:
|
||||
logger.warning("No tbodies found in history table.")
|
||||
send_tg("⚠️ **Предупреждение**: История обращений пуста или не загрузилась.")
|
||||
await page.screenshot(path=os.path.join(EXPORT_DIR, "empty_table.png"), full_page=True)
|
||||
await context.close()
|
||||
return
|
||||
|
||||
# Process each appeal / tbody
|
||||
count = 0
|
||||
for index, tbody in enumerate(tbodies[:15]): # Process top 15 appeals
|
||||
main_tr = await tbody.query_selector("tr:not(.tr-expanded)")
|
||||
if not main_tr:
|
||||
continue
|
||||
|
||||
cells = await main_tr.query_selector_all("td")
|
||||
if len(cells) < 5:
|
||||
continue
|
||||
|
||||
# Get registration number details
|
||||
num_text = (await cells[1].inner_text()).strip()
|
||||
# Clean registration number to get safe directory ID
|
||||
safe_id = "".join([c for c in num_text if c.isalnum() or c in ("-", "_")]).strip()
|
||||
if not safe_id:
|
||||
safe_id = f"row_{index}"
|
||||
|
||||
safe_id = "".join([c for c in href if c.isalnum() or c in ("-", "_")]).strip()
|
||||
appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}")
|
||||
os.makedirs(appeal_dir, exist_ok=True)
|
||||
|
||||
# Take detail screenshot
|
||||
detail_screenshot = os.path.join(appeal_dir, "evidence.png")
|
||||
await page.screenshot(path=detail_screenshot, full_page=True)
|
||||
# Find and click "История рассмотрения" link in 5th cell
|
||||
history_link = await cells[4].query_selector("a:has-text('История рассмотрения')")
|
||||
if not history_link:
|
||||
logger.info(f"No history link for row {index}, safe_id={safe_id}")
|
||||
continue
|
||||
|
||||
# Dump detail HTML
|
||||
logger.info(f"Processing appeal {safe_id} (row {index})...")
|
||||
|
||||
try:
|
||||
# Expand
|
||||
await history_link.click(force=True)
|
||||
await asyncio.sleep(4)
|
||||
|
||||
# Take screenshot of expanded tbody segment
|
||||
evidence_img = os.path.join(appeal_dir, "evidence.png")
|
||||
await tbody.screenshot(path=evidence_img)
|
||||
|
||||
# Save tbody HTML
|
||||
detail_html = os.path.join(appeal_dir, "page.html")
|
||||
with open(detail_html, "w", encoding="utf-8") as f:
|
||||
f.write(await page.content())
|
||||
|
||||
# Look for download links
|
||||
download_selectors = [
|
||||
"a[href*='download']",
|
||||
"a[href*='file']",
|
||||
"button:has-text('Скачать')",
|
||||
"a:has-text('Скачать')"
|
||||
]
|
||||
f.write(await tbody.inner_html())
|
||||
|
||||
# Download attachments
|
||||
file_links = await tbody.query_selector_all("a[href*='getFile']")
|
||||
files_downloaded = []
|
||||
for selector in download_selectors:
|
||||
elements = await page.locator(selector).all()
|
||||
for el in elements:
|
||||
if not await el.is_visible(): continue
|
||||
|
||||
for file_link in file_links:
|
||||
file_text = (await file_link.inner_text()).strip()
|
||||
logger.info(f"Downloading file: '{file_text}'")
|
||||
|
||||
try:
|
||||
async with page.expect_download(timeout=15000) as di:
|
||||
await el.click(force=True)
|
||||
download = await di.value
|
||||
fname = download.suggested_filename
|
||||
dest = os.path.join(appeal_dir, fname)
|
||||
async with page.expect_download(timeout=15000) as download_info:
|
||||
await file_link.click()
|
||||
download = await download_info.value
|
||||
dest = os.path.join(appeal_dir, download.suggested_filename)
|
||||
await download.save_as(dest)
|
||||
files_downloaded.append(fname)
|
||||
logger.info(f"Downloaded: {fname}")
|
||||
files_downloaded.append(download.suggested_filename)
|
||||
logger.info(f"Saved: {download.suggested_filename}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed download click: {e}")
|
||||
logger.warning(f"Error downloading file {file_text}: {e}")
|
||||
|
||||
count += 1
|
||||
send_tg(f"📥 **Обращение {count} собрано**\nПапка: `appeal_{safe_id}`\nФайлов скачано: {len(files_downloaded)}")
|
||||
send_tg(f"📥 **Обращение {count} скачано: № {safe_id}**\nФайлов: {len(files_downloaded)}")
|
||||
|
||||
# Collapse
|
||||
await history_link.click(force=True)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing appeal details {href}: {e}")
|
||||
logger.error(f"Error parsing appeal {safe_id}: {e}")
|
||||
|
||||
send_tg(f"🏁 **Миссия ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.")
|
||||
send_tg(f"🏁 **Жатва ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.")
|
||||
|
||||
# Trigger PDF compilation script
|
||||
# Trigger compiler
|
||||
import subprocess
|
||||
subprocess.Popen(["python3", "/app/sudrf_pdf_compiler.py"], start_new_session=True)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue