MVP сайта

examples/parsers/scrape_html.py

@@ -0,0 +1,26 @@
+"""python scrape_html.py URL --output records.jsonl"""
+import argparse
+from urllib.parse import urljoin,urlsplit
+import requests
+from bs4 import BeautifulSoup
+from common import extract,save
+
+def scrape(url):
+    records=[]; pages=set(); ids=set(); origin=urlsplit(url).netloc
+    with requests.Session() as session:
+        while url:
+            if url in pages: raise RuntimeError('Зацикленная пагинация')
+            pages.add(url)
+            response=session.get(url,timeout=30); response.raise_for_status()
+            batch=extract(response.text)
+            for record in batch:
+                if record['id'] in ids: raise RuntimeError('Повтор записи между страницами')
+                ids.add(record['id']); records.append(record)
+            next_link=BeautifulSoup(response.text,'html.parser').select_one('a[rel=next]')
+            url=urljoin(url,next_link['href']) if next_link else ''
+            if url and urlsplit(url).netloc!=origin: raise RuntimeError('Ссылка ведёт на другой сервер')
+    return records
+
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('url');p.add_argument('--output',default='records.jsonl');a=p.parse_args()
+    records=scrape(a.url);save(records,a.output);print(f'Сохранено записей: {len(records)}')