matrixhasyou/domovoy_bot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Update sudrf routing to /appeals/ and exclude general navigation URLs from details crawler

Browse source
This commit is contained in:
Admin 2026-07-03 11:33:32 +04:00
parent 378ec2ebca
commit c1ad8b3d4c
1 changed files with 10 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
scraper_gis/sudrf_reaper.py
View file

@ -82,7 +82,7 @@ async def run_mission():
logger.info("Login detected. Navigating to appeals history...")
send_tg("✅ **Авторизация успешна!** Начинаю сбор обращений...")
await page.goto("https://ej.sudrf.ru/#/appeals")
await page.goto("https://ej.sudrf.ru/appeals/")
await asyncio.sleep(10) # wait for page to render appeals list
# Take list page screenshot
@ -103,7 +103,9 @@ async def run_mission():
href = await link.get_attribute("href")
text = await link.inner_text()
if href and ("appeal" in href or "card" in href or "request" in href):
links.append({"href": href, "text": text.strip()})
# Исключаем общие страницы
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
links.append({"href": href, "text": text.strip()})
logger.info(f"Found {len(links)} matching links on page: {links}")
@ -150,15 +152,17 @@ async def run_mission():
for appeal in parsed_appeals:
href = appeal.get("href")
if href and href not in processed_hrefs:
detail_hrefs.append(href)
processed_hrefs.add(href)
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
detail_hrefs.append(href)
processed_hrefs.add(href)
# Also check all links for hrefs like #/appeals/view/ or similar
for link in links:
href = link["href"]
if href and href not in processed_hrefs:
detail_hrefs.append(href)
processed_hrefs.add(href)
if href.strip() not in ("/appeal/", "/appeals/", "/appeal", "/appeals", "#/appeal/", "#/appeals/"):
detail_hrefs.append(href)
processed_hrefs.add(href)
logger.info(f"Unique detail pages to process: {detail_hrefs}")