28 lines
1.5 KiB
Python
28 lines
1.5 KiB
Python
"""python scrape_interactive.py URL --output records.jsonl [--channel chrome]"""
|
|
import argparse
|
|
from playwright.sync_api import sync_playwright
|
|
from common import extract,save
|
|
|
|
def scrape(url,channel=None):
|
|
with sync_playwright() as p:
|
|
browser=p.chromium.launch(headless=True,**({'channel':channel} if channel else {}))
|
|
try:
|
|
page=browser.new_page()
|
|
response=page.goto(url)
|
|
if not response or not response.ok: raise RuntimeError('Публикация недоступна')
|
|
page.locator('.lab-records').wait_for(timeout=30000)
|
|
while True:
|
|
more=page.get_by_role('button',name='Загрузить ещё',exact=True)
|
|
if not more.is_visible(): break
|
|
old=page.locator('.record[data-record-id]').count()
|
|
more.click()
|
|
page.wait_for_function('(n)=>document.querySelectorAll(".record[data-record-id]").length > n',arg=old,timeout=10000)
|
|
records=extract(page.content())
|
|
if len({r['id'] for r in records})!=len(records): raise RuntimeError('Обнаружены дубликаты записей')
|
|
return records
|
|
finally: browser.close()
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('url');p.add_argument('--output',default='records.jsonl');p.add_argument('--channel');a=p.parse_args()
|
|
records=scrape(a.url,a.channel);save(records,a.output);print(f'Сохранено записей: {len(records)}')
|