"""python scrape_html.py URL --output records.jsonl"""
import argparse
from urllib.parse import urljoin,urlsplit
import requests
from bs4 import BeautifulSoup
from common import extract,save

def scrape(url):
    records=[]; pages=set(); ids=set(); origin=urlsplit(url).netloc
    with requests.Session() as session:
        while url:
            if url in pages: raise RuntimeError('Зацикленная пагинация')
            pages.add(url)
            response=session.get(url,timeout=30); response.raise_for_status()
            batch=extract(response.text)
            for record in batch:
                if record['id'] in ids: raise RuntimeError('Повтор записи между страницами')
                ids.add(record['id']); records.append(record)
            next_link=BeautifulSoup(response.text,'html.parser').select_one('a[rel=next]')
            url=urljoin(url,next_link['href']) if next_link else ''
            if url and urlsplit(url).netloc!=origin: raise RuntimeError('Ссылка ведёт на другой сервер')
    return records

if __name__=='__main__':
    p=argparse.ArgumentParser();p.add_argument('url');p.add_argument('--output',default='records.jsonl');a=p.parse_args()
    records=scrape(a.url);save(records,a.output);print(f'Сохранено записей: {len(records)}')