27 lines
1.3 KiB
Python
27 lines
1.3 KiB
Python
"""python scrape_html.py URL --output records.jsonl"""
|
|
import argparse
|
|
from urllib.parse import urljoin,urlsplit
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from common import extract,save
|
|
|
|
def scrape(url):
|
|
records=[]; pages=set(); ids=set(); origin=urlsplit(url).netloc
|
|
with requests.Session() as session:
|
|
while url:
|
|
if url in pages: raise RuntimeError('Зацикленная пагинация')
|
|
pages.add(url)
|
|
response=session.get(url,timeout=30); response.raise_for_status()
|
|
batch=extract(response.text)
|
|
for record in batch:
|
|
if record['id'] in ids: raise RuntimeError('Повтор записи между страницами')
|
|
ids.add(record['id']); records.append(record)
|
|
next_link=BeautifulSoup(response.text,'html.parser').select_one('a[rel=next]')
|
|
url=urljoin(url,next_link['href']) if next_link else ''
|
|
if url and urlsplit(url).netloc!=origin: raise RuntimeError('Ссылка ведёт на другой сервер')
|
|
return records
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('url');p.add_argument('--output',default='records.jsonl');a=p.parse_args()
|
|
records=scrape(a.url);save(records,a.output);print(f'Сохранено записей: {len(records)}')
|