From 7101e617aba901d23077c9c61c50187d52a80d57 Mon Sep 17 00:00:00 2001 From: Admin Date: Fri, 3 Jul 2026 14:52:14 +0400 Subject: [PATCH] feat: Implement robust search and inline accordion details scraper for sudrf appeals --- scraper_gis/sudrf_reaper.py | 301 ++++++++++++++++++------------------ 1 file changed, 152 insertions(+), 149 deletions(-) diff --git a/scraper_gis/sudrf_reaper.py b/scraper_gis/sudrf_reaper.py index 54d8f14..56143f9 100644 --- a/scraper_gis/sudrf_reaper.py +++ b/scraper_gis/sudrf_reaper.py @@ -36,7 +36,7 @@ async def human_delay(min_sec=3, max_sec=6): async def run_mission(): logger.info("Starting Playwright in VNC display...") - send_tg("🕵️‍♂️ **ГАС Правосудие: Запуск браузера**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html") + send_tg("🕵️‍♂️ **ГАС Правосудие: Запуск робота-разведчика**\nОжидаю авторизацию через Госуслуги во VNC: http://192.168.10.116:6080/vnc.html") async with async_playwright() as p: # Launch persistent browser context @@ -52,179 +52,182 @@ async def run_mission(): # Navigate to portal logger.info("Navigating to sudrf...") await page.goto("https://ej.sudrf.ru/") - await human_delay(4, 7) + await asyncio.sleep(8) - # Phase 0: Wait for login - logger.info("Waiting for user authorization...") - logged_in = False - for i in range(120): # 10 minutes total - content = "" + # Check login screen + url = page.url + try: + visible_text = await page.locator("body").inner_text() + except Exception: + visible_text = "" + + logger.info(f"Current page URL: {url}") + + is_logged_in = "sudrf.ru" in url and any(ind in visible_text for ind in ["Выйти", "Выход", "Личный кабинет", "Тырин"]) and "Авторизация" not in visible_text + + if not is_logged_in: + logger.info("Login/Agreement page detected. Auto-accepting terms...") try: - content = await page.content() + checkbox = await page.query_selector("input[type='checkbox']") + if checkbox: + await checkbox.check() + logger.info("Checked user agreement checkbox.") + else: + await page.click("text=Я ознакомился(ась)", timeout=5000) + logger.info("Clicked user agreement label.") + await asyncio.sleep(2) + + buttons = await page.query_selector_all("button, .btn, a.btn") + for b in buttons: + text = (await b.inner_text()).strip() + if "Войти" in text: + logger.info(f"Clicking Войти button: '{text}'") + await b.click() + break except Exception as e: - logger.error(f"Error reading page content: {e}") + logger.info(f"Note (agreement handling): {e}") + + logger.info("Waiting up to 300s for user authorization...") + logged_in_False = True + for i in range(60): + await asyncio.sleep(5) + current_url = page.url + try: + current_text = await page.locator("body").inner_text() + except Exception: + current_text = "" + + if "sudrf.ru" in current_url and any(ind in current_text for ind in ["Выйти", "Выход", "Личный кабинет", "Тырин"]): + logger.info(f"Login detected! URL: {current_url}") + logged_in_False = False + break + if i % 6 == 0: + logger.info(f"Still waiting... {i*5}s") + if logged_in_False: + logger.error("Login timeout exceeded.") + send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.") + await context.close() + return + else: + logger.info("Already logged in.") - # Check for common logout or profile indicators - if any(indicator in content for indicator in ["Выйти", "Выход", "Личный кабинет", "История обращений"]): - logged_in = True - break - - if i % 6 == 0: - logger.info(f"Still waiting... i={i}") - await asyncio.sleep(5) - - if not logged_in: - logger.error("Login timeout exceeded.") - send_tg("❌ **ГАС Правосудие: Превышено время ожидания входа**.") + # Navigate to appeals history page + logger.info("Navigating to appeals history...") + send_tg("✅ **Авторизация подтверждена!** Загружаю историю обращений...") + await page.goto("https://ej.sudrf.ru/appeals/") + await asyncio.sleep(8) + + # Click search button + logger.info("Locating Search button...") + buttons = await page.query_selector_all("button, input[type='submit'], input[type='button'], .btn, a.btn") + search_clicked = False + for b in buttons: + try: + text = (await b.inner_text()).strip() + if "Найти" in text: + logger.info(f"Clicking Search button: '{text}'") + await b.click(force=True) + search_clicked = True + break + except Exception as e: + logger.error(f"Error inspecting search button: {e}") + + if not search_clicked: + logger.error("Could not click Search button.") + send_tg("❌ **Ошибка**: Не удалось нажать кнопку Поиск для загрузки списка обращений.") + await page.screenshot(path=os.path.join(EXPORT_DIR, "search_failed.png"), full_page=True) await context.close() return - logger.info("Login detected. Navigating to appeals history...") - send_tg("✅ **Авторизация успешна!** Начинаю сбор обращений...") + logger.info("Waiting for search results...") + await asyncio.sleep(8) - await page.goto("https://ej.sudrf.ru/appeals/") - await asyncio.sleep(10) # wait for page to render appeals list + # Check if table loaded + tbodies = await page.query_selector_all("table.table-history tbody") + logger.info(f"Found {len(tbodies)} tbodies in history table.") - # Take list page screenshot - list_screenshot = os.path.join(EXPORT_DIR, "appeals_list.png") - await page.screenshot(path=list_screenshot, full_page=True) - - # Dump HTML for structural analysis - list_html = os.path.join(EXPORT_DIR, "appeals_list.html") - with open(list_html, "w", encoding="utf-8") as f: - f.write(await page.content()) + if not tbodies: + logger.warning("No tbodies found in history table.") + send_tg("⚠️ **Предупреждение**: История обращений пуста или не загрузилась.") + await page.screenshot(path=os.path.join(EXPORT_DIR, "empty_table.png"), full_page=True) + await context.close() + return - logger.info(f"Appeals list screenshot saved to {list_screenshot}") - - # Extract appeal links - links = [] - all_links = await page.query_selector_all("a") - for link in all_links: - href = await link.get_attribute("href") - text = await link.inner_text() - if href and ("appeal" in href or "card" in href or "request" in href): - # Исключаем общие страницы - if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"): - links.append({"href": href, "text": text.strip()}) - - logger.info(f"Found {len(links)} matching links on page: {links}") - - # Process each appeal - # We can extract rows from the table to get exact dates, numbers and link objects - # For sudrf, appeals table often has tr tags with status and numbers - rows = await page.query_selector_all("tr, .appeals-item") - logger.info(f"Found {len(rows)} rows/items in appeals list.") - - # Dump structured report of links to a json file - import json - with open(os.path.join(EXPORT_DIR, "found_links.json"), "w", encoding="utf-8") as f: - json.dump({"links": links, "rows_count": len(rows)}, f, indent=4, ensure_ascii=False) - - # Try to parse the table rows if they contain appeal details - parsed_appeals = [] - for index, row in enumerate(rows[:20]): # parse top 20 appeals first for safety - cells = await row.query_selector_all("td") - if not cells: continue - - row_data = [] - for cell in cells: - row_data.append((await cell.inner_text()).strip()) - - # Look for link inside the row - row_link = await row.query_selector("a") - href = await row_link.get_attribute("href") if row_link else None - - parsed_appeals.append({ - "row_index": index, - "data": row_data, - "href": href - }) - - with open(os.path.join(EXPORT_DIR, "parsed_table.json"), "w", encoding="utf-8") as f: - json.dump(parsed_appeals, f, indent=4, ensure_ascii=False) - - # Loop through found details pages and download documents + # Process each appeal / tbody count = 0 - processed_hrefs = set() - - # Collect actual detail pages - detail_hrefs = [] - for appeal in parsed_appeals: - href = appeal.get("href") - if href and href not in processed_hrefs: - if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"): - detail_hrefs.append(href) - processed_hrefs.add(href) + for index, tbody in enumerate(tbodies[:15]): # Process top 15 appeals + main_tr = await tbody.query_selector("tr:not(.tr-expanded)") + if not main_tr: + continue - # Also check all links for hrefs like #/appeals/view/ or similar - for link in links: - href = link["href"] - if href and href not in processed_hrefs: - if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"): - detail_hrefs.append(href) - processed_hrefs.add(href) + cells = await main_tr.query_selector_all("td") + if len(cells) < 5: + continue - logger.info(f"Unique detail pages to process: {detail_hrefs}") - - for href in detail_hrefs[:10]: # Safe limit to top 10 appeals in recon mode + # Get registration number details + num_text = (await cells[1].inner_text()).strip() + # Clean registration number to get safe directory ID + safe_id = "".join([c for c in num_text if c.isalnum() or c in ("-", "_")]).strip() + if not safe_id: + safe_id = f"row_{index}" + + appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}") + os.makedirs(appeal_dir, exist_ok=True) + + # Find and click "История рассмотрения" link in 5th cell + history_link = await cells[4].query_selector("a:has-text('История рассмотрения')") + if not history_link: + logger.info(f"No history link for row {index}, safe_id={safe_id}") + continue + + logger.info(f"Processing appeal {safe_id} (row {index})...") + try: - full_url = href - if href.startswith("#"): - full_url = f"https://ej.sudrf.ru/{href}" - elif href.startswith("/"): - full_url = f"https://ej.sudrf.ru{href}" - - logger.info(f"Navigating to detail page: {full_url}") - await page.goto(full_url) - await asyncio.sleep(8) # Wait for page load + # Expand + await history_link.click(force=True) + await asyncio.sleep(4) - safe_id = "".join([c for c in href if c.isalnum() or c in ("-", "_")]).strip() - appeal_dir = os.path.join(EXPORT_DIR, f"appeal_{safe_id}") - os.makedirs(appeal_dir, exist_ok=True) + # Take screenshot of expanded tbody segment + evidence_img = os.path.join(appeal_dir, "evidence.png") + await tbody.screenshot(path=evidence_img) - # Take detail screenshot - detail_screenshot = os.path.join(appeal_dir, "evidence.png") - await page.screenshot(path=detail_screenshot, full_page=True) - - # Dump detail HTML + # Save tbody HTML detail_html = os.path.join(appeal_dir, "page.html") with open(detail_html, "w", encoding="utf-8") as f: - f.write(await page.content()) + f.write(await tbody.inner_html()) - # Look for download links - download_selectors = [ - "a[href*='download']", - "a[href*='file']", - "button:has-text('Скачать')", - "a:has-text('Скачать')" - ] - + # Download attachments + file_links = await tbody.query_selector_all("a[href*='getFile']") files_downloaded = [] - for selector in download_selectors: - elements = await page.locator(selector).all() - for el in elements: - if not await el.is_visible(): continue - try: - async with page.expect_download(timeout=15000) as di: - await el.click(force=True) - download = await di.value - fname = download.suggested_filename - dest = os.path.join(appeal_dir, fname) - await download.save_as(dest) - files_downloaded.append(fname) - logger.info(f"Downloaded: {fname}") - except Exception as e: - logger.warning(f"Failed download click: {e}") - + + for file_link in file_links: + file_text = (await file_link.inner_text()).strip() + logger.info(f"Downloading file: '{file_text}'") + + try: + async with page.expect_download(timeout=15000) as download_info: + await file_link.click() + download = await download_info.value + dest = os.path.join(appeal_dir, download.suggested_filename) + await download.save_as(dest) + files_downloaded.append(download.suggested_filename) + logger.info(f"Saved: {download.suggested_filename}") + except Exception as e: + logger.warning(f"Error downloading file {file_text}: {e}") + count += 1 - send_tg(f"📥 **Обращение {count} собрано**\nПапка: `appeal_{safe_id}`\nФайлов скачано: {len(files_downloaded)}") + send_tg(f"📥 **Обращение {count} скачано: № {safe_id}**\nФайлов: {len(files_downloaded)}") + + # Collapse + await history_link.click(force=True) + await asyncio.sleep(2) except Exception as e: - logger.error(f"Error processing appeal details {href}: {e}") + logger.error(f"Error parsing appeal {safe_id}: {e}") - send_tg(f"🏁 **Миссия ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.") + send_tg(f"🏁 **Жатва ГАС Правосудие завершена!** Собрано обращений: {count}. Запускаю сборку в PDF.") - # Trigger PDF compilation script + # Trigger compiler import subprocess subprocess.Popen(["python3", "/app/sudrf_pdf_compiler.py"], start_new_session=True)