MVP сайта

2026-10-01 00:28:16 +10:00
commit 25f4255404
55 changed files with 3325 additions and 0 deletions
@@ -0,0 +1,27 @@
+"""python scrape_interactive.py URL --output records.jsonl [--channel chrome]"""
+import argparse
+from playwright.sync_api import sync_playwright
+from common import extract,save
+
+def scrape(url,channel=None):
+    with sync_playwright() as p:
+        browser=p.chromium.launch(headless=True,**({'channel':channel} if channel else {}))
+        try:
+            page=browser.new_page()
+            response=page.goto(url)
+            if not response or not response.ok: raise RuntimeError('Публикация недоступна')
+            page.locator('.lab-records').wait_for(timeout=30000)
+            while True:
+                more=page.get_by_role('button',name='Загрузить ещё',exact=True)
+                if not more.is_visible(): break
+                old=page.locator('.record[data-record-id]').count()
+                more.click()
+                page.wait_for_function('(n)=>document.querySelectorAll(".record[data-record-id]").length > n',arg=old,timeout=10000)
+            records=extract(page.content())
+            if len({r['id'] for r in records})!=len(records): raise RuntimeError('Обнаружены дубликаты записей')
+            return records
+        finally: browser.close()
+
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('url');p.add_argument('--output',default='records.jsonl');p.add_argument('--channel');a=p.parse_args()
+    records=scrape(a.url,a.channel);save(records,a.output);print(f'Сохранено записей: {len(records)}')