From c1ad8b3d4ce46903ada57cc29b5e01a2785c9af9 Mon Sep 17 00:00:00 2001 From: Admin Date: Fri, 3 Jul 2026 11:33:32 +0400 Subject: [PATCH] fix: Update sudrf routing to /appeals/ and exclude general navigation URLs from details crawler --- scraper_gis/sudrf_reaper.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scraper_gis/sudrf_reaper.py b/scraper_gis/sudrf_reaper.py index c83877e..54d8f14 100644 --- a/scraper_gis/sudrf_reaper.py +++ b/scraper_gis/sudrf_reaper.py @@ -82,7 +82,7 @@ async def run_mission(): logger.info("Login detected. Navigating to appeals history...") send_tg("✅ **Авторизация успешна!** Начинаю сбор обращений...") - await page.goto("https://ej.sudrf.ru/#/appeals") + await page.goto("https://ej.sudrf.ru/appeals/") await asyncio.sleep(10) # wait for page to render appeals list # Take list page screenshot @@ -103,7 +103,9 @@ async def run_mission(): href = await link.get_attribute("href") text = await link.inner_text() if href and ("appeal" in href or "card" in href or "request" in href): - links.append({"href": href, "text": text.strip()}) + # Исключаем общие страницы + if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"): + links.append({"href": href, "text": text.strip()}) logger.info(f"Found {len(links)} matching links on page: {links}") @@ -150,15 +152,17 @@ async def run_mission(): for appeal in parsed_appeals: href = appeal.get("href") if href and href not in processed_hrefs: - detail_hrefs.append(href) - processed_hrefs.add(href) + if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"): + detail_hrefs.append(href) + processed_hrefs.add(href) # Also check all links for hrefs like #/appeals/view/ or similar for link in links: href = link["href"] if href and href not in processed_hrefs: - detail_hrefs.append(href) - processed_hrefs.add(href) + if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"): + detail_hrefs.append(href) + processed_hrefs.add(href) logger.info(f"Unique detail pages to process: {detail_hrefs}")