domovoy_bot/scraper_gis/harvest_final.py

58 lines
2.3 KiB
Python

from playwright.sync_api import sync_playwright
import time, os
def run():
with sync_playwright() as p:
try:
browser = p.chromium.connect_over_cdp('http://127.0.0.1:9222')
page = browser.contexts[0].pages[0]
# Собираем видимые ссылки
links = page.query_selector_all('a[href*="appeals/view/"]')
print(f'Visible links: {len(links)}')
if not links:
print('No links found. Scrolling...')
page.evaluate('window.scrollTo(0, 500)')
time.sleep(2)
links = page.query_selector_all('a[href*="appeals/view/"]')
for i in range(min(3, len(links))):
# Находим ссылку заново после навигации назад
links = page.query_selector_all('a[href*="appeals/view/"]')
target = links[i]
num = target.inner_text().strip()
print(f'Step {i+1}: Entering {num}')
target.click()
page.wait_for_load_state('networkidle')
time.sleep(10)
# Скриншот внутри
page.screenshot(path=f'/app/downloads/screen_{num}.png', full_page=True)
# Download All
btn = page.get_by_text('Скачать все').first
if btn.is_visible():
print(f'Downloading all files for {num}...')
with page.expect_download(timeout=60000) as download_info:
btn.click()
download = download_info.value
download.save_as(f'/app/downloads/archive_{num}.zip')
print(f'SUCCESS: Saved archive_{num}.zip')
else:
print(f'Download button NOT found for {num}')
# Go back
print('Going back...')
page.go_back()
page.wait_for_load_state('networkidle')
time.sleep(5)
browser.close()
except Exception as e:
print(f'ERROR: {e}')
if __name__ == '__main__':
if not os.path.exists('/app/downloads'): os.makedirs('/app/downloads')
run()