обновлен формат генерации датасета

assembly_dataset/__init__.py

@@ -0,0 +1 @@
+"""Reproducible, logically validated assembly planning datasets."""

assembly_dataset/__main__.py

@@ -0,0 +1,51 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+import sys
+
+from .config import load_config
+from .pipeline import generate, export, validate
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Датасет высокоуровневых планов сборки: full / step / branch')
+    parser.add_argument('--config', type=Path, default=Path(__file__).resolve().parents[1] / 'config.yaml')
+    parser.add_argument('command', choices=['generate', 'export', 'validate'], nargs='?', default='generate')
+    parser.add_argument('--count', type=int, help='Точное суммарное число SFT-записей; 0 допустим')
+    parser.add_argument('--output-dir', type=Path, help='Папка корпуса (относительно текущего каталога)')
+    parser.add_argument('--dry-run', action='store_true', help='Показать настройки без записи и запросов')
+    parser.add_argument('--no-progress', action='store_true', help='Отключить индикаторы прогресса')
+    args = parser.parse_args()
+    try:
+        config = load_config(args.config)
+        data = config.model_dump()
+        if args.count is not None:
+            data['target_examples'] = args.count
+        if args.output_dir is not None:
+            data['output_dir'] = args.output_dir.resolve()
+        config = type(config).model_validate(data)
+        if args.dry_run:
+            print(config.model_dump_json(indent=2))
+            return 0
+        if args.command == 'generate':
+            if config.teacher.enabled:
+                from dotenv import load_dotenv
+                load_dotenv(Path(__file__).resolve().parents[1] / '.env')
+            asyncio.run(generate(config, show_progress=not args.no_progress))
+            result = export(config, show_progress=not args.no_progress)
+            result = {k: result[k] for k in ('examples', 'scenarios', 'counts', 'teacher_failures')}
+        elif args.command == 'export':
+            result = export(config, show_progress=not args.no_progress)
+            result = {k: result[k] for k in ('examples', 'scenarios', 'counts', 'teacher_failures')}
+        else:
+            result = validate(config)
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+        return 0
+    except (ValueError, OSError) as exc:
+        print(f'Ошибка: {exc}', file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

assembly_dataset/config.py

@@ -0,0 +1,76 @@
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import Field, model_validator
+
+from llm.config import ConfigModel, LLMConfig, GenerationConfig, CodexConfig
+
+from .prompts import Prompts, load_prompts
+
+DOMAINS = (
+    'manipulation', 'mechanical', 'fastening', 'press_fit', 'electrical',
+    'electronics', 'adhesive', 'welding', 'collaborative', 'inspection', 'operator_assistance',
+)
+
+
+class TeacherConfig(ConfigModel):
+    enabled: bool = False
+    fraction: float = Field(default=0.15, ge=0, le=1)
+    llm: LLMConfig | None = None
+    generation: GenerationConfig = Field(default_factory=GenerationConfig)
+    concurrency: int = Field(default=4, ge=1, le=128)
+    timeout_seconds: float = Field(default=180, gt=0)
+    retries: int = Field(default=2, ge=0, le=10)
+
+    @model_validator(mode='after')
+    def require_model(self):
+        if self.enabled and self.llm is None:
+            raise ValueError('teacher.llm обязателен при teacher.enabled: true')
+        return self
+
+
+class Config(ConfigModel):
+    # Count means exported SFT rows, not episodes or API calls.
+    target_examples: int = Field(default=500_000, ge=0, strict=True)
+    seed: int = 20261021
+    output_dir: Path = Path('dataset/assembly')
+    modes: list[Literal['full', 'step', 'branch']] = Field(default_factory=lambda: ['full', 'step', 'branch'], min_length=1)
+    step_sampling: Literal['one', 'all'] = 'one'
+    shard_size: int = Field(default=1000, ge=1)
+    split: tuple[float, float, float] = (0.9, 0.05, 0.05)
+    domains: list[str] = Field(default_factory=lambda: list(DOMAINS), min_length=1)
+    min_components: int = Field(default=2, ge=1, le=12)
+    max_components: int = Field(default=6, ge=1, le=12)
+    blocked_fraction: float = Field(default=0.1, ge=0, le=1)
+    prompts_file: Path = Path("prompts.yaml")
+    prompts: Prompts = Field(default_factory=load_prompts)
+    teacher: TeacherConfig = Field(default_factory=TeacherConfig)
+
+    @model_validator(mode='after')
+    def check(self):
+        if len(set(self.modes)) != len(self.modes):
+            raise ValueError('modes не должны повторяться')
+        if len(set(self.domains)) != len(self.domains) or set(self.domains) - set(DOMAINS):
+            raise ValueError(f'Допустимые domains: {DOMAINS}')
+        if set(self.domains) - set(self.prompts.task_instructions):
+            raise ValueError("В task_instructions отсутствуют инструкции выбранных domains")
+        if self.min_components > self.max_components:
+            raise ValueError('min_components > max_components')
+        if any(v < 0 or v > 1 for v in self.split) or abs(sum(self.split) - 1) > 1e-9:
+            raise ValueError('split должен содержать три доли с суммой 1')
+        return self
+
+
+def load_config(path: Path) -> Config:
+    path = path.resolve()
+    data = yaml.safe_load(path.read_text(encoding='utf-8')) or {}
+    prompts_path = (path.parent / data.get('prompts_file', 'prompts.yaml')).resolve()
+    data['prompts'] = load_prompts(prompts_path)
+    data['prompts_file'] = prompts_path
+    config = Config.model_validate(data)
+    config.output_dir = (path.parent / config.output_dir).resolve()
+    llm = config.teacher.llm
+    if isinstance(llm, CodexConfig) and llm.credential_path:
+        llm.credential_path = (path.parent / llm.credential_path).resolve()
+    return config

assembly_dataset/csv_export.py

@@ -0,0 +1,30 @@
+"""Readable Russian plans with metadata kept after the primary columns."""
+import json
+
+COLUMNS = [
+    'Задание', 'Сцена', 'План', 'Статус', 'Формат', 'Источник текста',
+    'ID записи', 'ID сценария', 'Выборка', 'Вход (задание и сцена)',
+    'Системная инструкция', 'Провайдер', 'Модель', 'Уровень проверки',
+]
+STATUSES = {'ready': 'План готов', 'blocked': 'Задача невыполнима', 'continue': 'Следующее действие'}
+MODES = {'full': 'Полный план', 'step': 'Следующее действие', 'branch': 'План с условиями'}
+
+
+def csv_record(record):
+    messages = {m['role']: m['content'] for m in record['messages']}
+    user = messages['user']
+    try:
+        user = json.dumps(json.loads(user), ensure_ascii=False, indent=2)
+    except json.JSONDecodeError:
+        pass
+    return {
+        'Задание': record['task'], 'Сцена': record['scene_summary'],
+        'План': messages['assistant'], 'Статус': STATUSES[record['status']],
+        'Формат': MODES[record['mode']],
+        'Источник текста': 'LLM' if record['text_source'] == 'teacher' else 'Шаблон русского описания',
+        'ID записи': record['record_id'], 'ID сценария': record['scenario_id'],
+        'Выборка': record['split'], 'Вход (задание и сцена)': user,
+        'Системная инструкция': messages['system'],
+        'Провайдер': record['source']['provider'], 'Модель': record['source']['model'] or '',
+        'Уровень проверки': record['validation_level'],
+    }

assembly_dataset/language.py

@@ -0,0 +1,114 @@
+"""Russian action descriptions and readable scenes for the planning model."""
+import re
+
+LOCATIONS = {'tray': 'лоток', 'buffer': 'буферная зона', 'conveyor': 'конвейер'}
+SIZES = {'small': 'маленький', 'medium': 'средний', 'large': 'большой'}
+
+
+def component_for(action, scene):
+    return next(c for c in scene['components'] if c['id'] == action['object'])
+
+
+def object_name(component, scene, prompts):
+    return component.get('name') or f"{prompts.object_names.get(scene['domain'], 'деталь')} №{scene['components'].index(component) + 1}"
+
+
+def target_name(component, scene):
+    return component.get('target_name') or f"сборочная позиция №{scene['components'].index(component) + 1}"
+
+
+def action_text(action, scene, prompts):
+    component = component_for(action, scene)
+    return prompts.action_templates[action['skill']].format(
+        object=object_name(component, scene, prompts), target=target_name(component, scene))
+
+
+def normalized_action_text(text):
+    words = re.findall(r'[\w]+', text.casefold())
+    while words and words[0] in {'сначала', 'затем', 'далее'}:
+        words.pop(0)
+    synonyms = {'перенести': 'переместить', 'взять': 'захватить',
+                'проконтролировать': 'проверить', 'возвратиться': 'вернуться',
+                'местоположение': 'положение'}
+    return [synonyms.get(word, word) for word in words]
+
+
+def validate_texts(episode, texts, prompts):
+    if len(texts) != len(episode['plan']['actions']):
+        raise ValueError('Число русских описаний не совпадает с числом действий')
+    for action, text in zip(episode['plan']['actions'], texts):
+        if not text.strip() or not re.search('[А-Яа-яЁё]', text):
+            raise ValueError('Каждое действие должно быть описано на русском языке')
+        if re.search(r'\b(?:part_|slot_)\d+|\b(?:robot|camera)\.', text):
+            raise ValueError('В тексте плана используй названия объектов, а не код')
+        component = component_for(action, episode['scene'])
+        if action['skill'] != 'return_home' and component.get('name') and component['name'].casefold() not in text.casefold():
+            raise ValueError(f"В описании действия укажи название объекта: {component['name']}")
+        expected = action_text(action, episode['scene'], prompts)
+        if normalized_action_text(text) != normalized_action_text(expected):
+            raise ValueError(f'Описание не соответствует проверяемому действию. Используй: {expected}')
+
+
+def step_texts(episode, prompts):
+    if 'step_texts' in episode:
+        validate_texts(episode, episode['step_texts'], prompts)
+        return episode['step_texts']
+    return [action_text(action, episode['scene'], prompts) for action in episode['plan']['actions']]
+
+
+def terminal_text(status):
+    return ('Задание выполнено.' if status == 'ready' else
+            'Задание невозможно завершить в текущих условиях. Сообщить оператору о недоступных ресурсах или зависимостях и запросить уточнение.')
+
+
+def answer_text(episode, mode, index, prompts):
+    steps = step_texts(episode, prompts)
+    status = episode['plan']['status']
+    if mode == 'step':
+        return steps[index] if steps else terminal_text(status)
+    if not steps:
+        return terminal_text(status)
+    if mode == 'full':
+        lines = [f'{i}. {text}' for i, text in enumerate(steps, 1)]
+        if status == 'blocked':
+            lines.append(terminal_text(status))
+        return '\n'.join(lines)
+    lines = []
+    for i, text in enumerate(steps, 1):
+        success = f'перейти к пункту {i + 1}' if i < len(steps) else terminal_text(status)
+        lines.append(f'{i}. {text}\n   Если действие выполнено успешно — {success}.\n'
+                     '   Если произошла ошибка — остановиться, уточнить состояние сцены и запросить новый план.')
+    return '\n'.join(lines)
+
+
+def readable_scene(scene, state, prompts):
+    state = set(state)
+    names = {c['id']: object_name(c, scene, prompts) for c in scene['components']}
+    objects = []
+    for c in scene['components']:
+        prefix = f"at:{c['id']}:"
+        location = next((s[len(prefix):] for s in state if s.startswith(prefix)), None)
+        location = target_name(c, scene) if location == c['target'] else LOCATIONS.get(location, location)
+        if f"holding:{c['id']}" in state:
+            location = 'в захвате робота'
+        stage = next((int(s.rsplit(':', 1)[1]) for s in state if s.startswith(f"{c['id']}:stage:")), 0)
+        operations = [op['action'] for op in scene['operations'] if op['action']['object'] == c['id']]
+        objects.append({'название': names[c['id']], 'размер': SIZES.get(c['type'], c['type']),
+                        'положение': location or 'не определено', 'назначение': target_name(c, scene),
+                        'доступен': f"available:{c['id']}" in state,
+                        'после_завершения': [names[d] for d in c['dependencies']],
+                        'выполненные_действия': [action_text(a, scene, prompts) for a in operations[:stage]]})
+    return {'объекты': objects, 'захват_свободен': 'gripper_empty' in state,
+            'оператор_готов': 'operator_ready' in state,
+            'цель': 'Выполнить задание для всех перечисленных объектов и освободить захват.',
+            'ограничение': 'Соблюдать зависимости объектов и не начинать следующий этап до завершения предыдущего.'}
+
+
+def scene_summary(scene):
+    lines = []
+    for obj in scene['объекты']:
+        lines.append(f"{obj['название']}: {obj['положение']}; назначение — {obj['назначение']}; "
+                     f"{'доступен' if obj['доступен'] else 'недоступен'}.")
+        if obj['после_завершения']:
+            lines.append('  Начать после: ' + ', '.join(obj['после_завершения']) + '.')
+    return '\n'.join(lines)

assembly_dataset/pipeline.py

@@ -0,0 +1,247 @@
+"""Bounded generation, transactional checkpoints, streaming sharded export."""
+import asyncio
+import csv
+from collections import Counter
+from contextlib import contextmanager
+import fcntl
+import hashlib
+import json
+from itertools import zip_longest
+from pathlib import Path
+import shutil
+import sqlite3
+import tempfile
+
+from .config import Config
+from .language import validate_texts
+from .csv_export import COLUMNS, csv_record
+from .progress import ProgressDisplay
+from .world import canonical, digest, make_episode, training_rows, validate_plan, validate_branch
+
+VERSION = 3
+STORAGE_VERSION = 2
+
+
+@contextmanager
+def storage(config):
+    root = config.output_dir
+    root.mkdir(parents=True, exist_ok=True)
+    with (root / '.lock').open('a') as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise ValueError('Другой процесс уже работает с output_dir') from None
+        db = sqlite3.connect(root / 'episodes.sqlite3')
+        try:
+            db.execute('PRAGMA journal_mode=WAL')
+            db.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
+            db.execute('CREATE TABLE IF NOT EXISTS episodes (seq INTEGER PRIMARY KEY, signature TEXT UNIQUE NOT NULL, payload TEXT NOT NULL, rows INTEGER NOT NULL)')
+            settings = config.model_dump(mode='json', exclude={'target_examples', 'output_dir', 'shard_size', 'prompts_file'})
+            # Structured prompts only affect export, so existing v2 episodes remain reusable.
+            settings['prompts'].pop('structured_system', None)
+            fingerprint = digest({'version': STORAGE_VERSION, 'settings': settings})
+            saved = db.execute("SELECT value FROM meta WHERE key='fingerprint'").fetchone()
+            if saved and saved[0] != fingerprint:
+                raise ValueError('Изменён config: для других настроек используйте новый output_dir')
+            if not saved:
+                with db:
+                    db.execute('INSERT INTO meta VALUES (?, ?)', ('fingerprint', fingerprint))
+                    db.execute('INSERT INTO meta VALUES (?, ?)', ('settings', canonical(settings)))
+                    db.execute('INSERT INTO meta VALUES (?, ?)', ('next_index', '0'))
+            yield db
+        finally:
+            db.close()
+
+
+async def generate(config: Config, agent=None, *, show_progress=False):
+    with storage(config) as db:
+        count = db.execute('SELECT COALESCE(SUM(rows), 0) FROM episodes').fetchone()[0]
+        index = int(db.execute("SELECT value FROM meta WHERE key='next_index'").fetchone()[0])
+        with ProgressDisplay('Генерация', config.target_examples, initial=count, enabled=show_progress) as progress:
+            if count >= config.target_examples:
+                return count
+            if config.teacher.enabled and config.teacher.fraction > 0 and agent is None:
+                from .teacher import create_agent
+                progress.update(status='подключение LLM')
+                agent = await create_agent(config.teacher, config.prompts)
+            batch_size = config.teacher.concurrency if agent is not None else 32
+            duplicates = 0
+            while count < config.target_examples:
+                # Never construct 500k coroutines or keep the entire dataset in RAM.
+                size = min(batch_size, max(1, config.target_examples - count))
+                episodes = [make_episode(i, config) for i in range(index, index + size)]
+                if agent is not None:
+                    from .teacher import enrich
+                    progress.update(status='ожидание LLM')
+                    episodes = [item[0] for item in await asyncio.gather(*(
+                        enrich(e, config.teacher, agent, config.prompts) for e in episodes))]
+                with db:
+                    for episode in episodes:
+                        rows = sum(1 for _ in training_rows(episode, config))
+                        cursor = db.execute('INSERT OR IGNORE INTO episodes(signature,payload,rows) VALUES (?,?,?)',
+                                            (episode['signature'], canonical(episode), rows))
+                        if cursor.rowcount:
+                            count += rows
+                            duplicates = 0
+                        else:
+                            duplicates += 1
+                        index = episode['candidate_index'] + 1
+                        if count >= config.target_examples:
+                            break
+                    db.execute("UPDATE meta SET value=? WHERE key='next_index'", (str(index),))
+                if duplicates > 10000:
+                    raise ValueError('Пространство сценариев исчерпано: расширьте диапазон компонентов или domains')
+                progress.update(count, status='')
+            return count
+
+
+class Shards:
+    def __init__(self, root, size):
+        self.root, self.size = root, size
+        self.counts = Counter()
+        self.handles = {}
+        self.csv_handles = {}
+        self.csv_writers = {}
+        self.files = []
+
+    def write(self, row, structured_row):
+        key = row['split'], row['mode']
+        count = self.counts[key]
+        for variant, record in (('text', row), ('structured', structured_row)):
+            handle_key = (*key, variant)
+            if count % self.size == 0:
+                if handle_key in self.handles:
+                    self.handles[handle_key].close()
+                    self.csv_handles[handle_key].close()
+                suffix = '.jsonl' if variant == 'text' else '.structured.jsonl'
+                path = self.root / key[0] / key[1] / f'{count // self.size:06d}{suffix}'
+                path.parent.mkdir(parents=True, exist_ok=True)
+                self.handles[handle_key] = path.open('w', encoding='utf-8')
+                self.files.append(path)
+                csv_path = path.with_suffix('.csv')
+                self.csv_handles[handle_key] = csv_path.open('w', encoding='utf-8-sig', newline='')
+                self.csv_writers[handle_key] = csv.DictWriter(self.csv_handles[handle_key], fieldnames=COLUMNS, delimiter=';')
+                self.csv_writers[handle_key].writeheader()
+                self.files.append(csv_path)
+            self.handles[handle_key].write(canonical(record) + '\n')
+            self.csv_writers[handle_key].writerow(csv_record(record))
+        self.counts[key] += 1
+
+    def close(self):
+        for file in (*self.handles.values(), *self.csv_handles.values()):
+            file.close()
+
+
+def file_digest(path):
+    hasher = hashlib.sha256()
+    with path.open('rb') as file:
+        for block in iter(lambda: file.read(1024 * 1024), b''):
+            hasher.update(block)
+    return hasher.hexdigest()
+
+
+def export(config, *, show_progress=False):
+    with storage(config) as db, ProgressDisplay('Экспорт', config.target_examples, enabled=show_progress) as progress:
+        capacity = db.execute('SELECT COALESCE(SUM(rows), 0) FROM episodes').fetchone()[0]
+        if capacity < config.target_examples:
+            raise ValueError('Недостаточно сценариев: сначала выполните generate')
+        destination = config.output_dir / 'sft'
+        backup = config.output_dir / '.sft-previous'
+        # Recover an interrupted directory swap before making another export.
+        if backup.exists() and not destination.exists():
+            backup.rename(destination)
+        if destination.exists() and not (destination / 'manifest.json').exists():
+            raise ValueError('sft не является управляемым экспортом; выберите другой output_dir')
+        stage = Path(tempfile.mkdtemp(prefix='.sft-', dir=config.output_dir))
+        writer = Shards(stage, config.shard_size)
+        total, scenarios, failures = 0, 0, 0
+        domains, sources = Counter(), Counter()
+        try:
+            for (payload,) in db.execute('SELECT payload FROM episodes ORDER BY seq'):
+                if total >= config.target_examples:
+                    break
+                episode = json.loads(payload)
+                scenarios += 1
+                domains[episode['scene']['domain']] += 1
+                sources[episode['source']['provider']] += 1
+                failures += int('teacher_error' in episode)
+                for record, structured_record in zip(training_rows(episode, config),
+                                                     training_rows(episode, config, output_format='structured'), strict=True):
+                    if total >= config.target_examples:
+                        break
+                    writer.write(record, structured_record)
+                    total += 1
+                progress.update(total)
+            progress.update(status="контрольные суммы")
+            writer.close()
+            manifest = {'schema_version': VERSION, 'examples': total, 'scenarios': scenarios,
+                        'examples_per_variant': {'text': total, 'structured': total},
+                        'counts': {f'{split}/{mode}': n for (split, mode), n in sorted(writer.counts.items())},
+                        'domains': dict(domains), 'sources': dict(sources), 'teacher_failures': failures,
+                        'validation_level': 'logical_only',
+                        'config': config.model_dump(mode='json'),
+                        'files': {str(p.relative_to(stage)): file_digest(p) for p in writer.files}}
+            (stage / 'manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding='utf-8')
+            if backup.exists():
+                shutil.rmtree(backup)
+            if destination.exists():
+                destination.rename(backup)
+            stage.rename(destination)
+            if backup.exists():
+                shutil.rmtree(backup)
+            progress.update(total, status='')
+            return manifest
+        finally:
+            writer.close()
+            if stage.exists():
+                shutil.rmtree(stage)
+
+
+def validate(config):
+    with storage(config) as db:
+        episodes, paths = 0, 0
+        for signature, payload, stored_rows in db.execute('SELECT signature,payload,rows FROM episodes'):
+            episode = json.loads(payload)
+            if signature != digest(episode['scene']) or episode['signature'] != signature or episode['scenario_id'] != signature:
+                raise ValueError('Неверная сигнатура сценария')
+            if 'step_texts' in episode:
+                validate_texts(episode, episode['step_texts'], config.prompts)
+            trace = validate_plan(episode, episode['plan'])
+            if trace != episode['trace']:
+                raise ValueError('Trace не соответствует плану')
+            paths += validate_branch(episode, episode['branch'])
+            if sum(1 for _ in training_rows(episode, config)) != stored_rows:
+                raise ValueError('Неверное число записей эпизода')
+            episodes += 1
+        root = config.output_dir / 'sft'
+        manifest_path = root / 'manifest.json'
+        if not manifest_path.exists():
+            raise ValueError('Нет экспорта SFT: выполните generate или export')
+        manifest = json.loads(manifest_path.read_text())
+        total = 0
+        expected_files = set(manifest['files'])
+        actual_files = {str(p.relative_to(root)) for pattern in ('*/*/*.jsonl', '*/*/*.csv') for p in root.glob(pattern)}
+        if expected_files != actual_files:
+            raise ValueError('Набор шардов не соответствует manifest')
+        for relative, checksum in manifest['files'].items():
+            path = root / relative
+            if path.resolve().is_relative_to(root.resolve()) is False or file_digest(path) != checksum:
+                raise ValueError(f'Повреждён шард {relative}')
+        for relative in manifest['files']:
+            path = root / relative
+            if path.suffix == '.jsonl' and '.structured.' not in path.name:
+                counterpart = path.with_suffix('.structured.jsonl')
+                if str(counterpart.relative_to(root)) not in expected_files:
+                    raise ValueError('Отсутствует структурированная пара JSONL')
+                with path.open(encoding='utf-8') as file, counterpart.open(encoding='utf-8') as other:
+                    for text_line, structured_line in zip_longest(file, other):
+                        if text_line is None or structured_line is None:
+                            raise ValueError('Разное число записей в паре JSONL')
+                        text_row, structured_row = json.loads(text_line), json.loads(structured_line)
+                        if (text_row['record_id'] != structured_row['record_id'] or
+                                text_row['messages'][1] != structured_row['messages'][1]):
+                            raise ValueError('Записи в паре JSONL не совпадают')
+                        total += 1
+        if total != manifest['examples']:
+            raise ValueError('Неверное число экспортированных записей')
+        return {'episodes_checked': episodes, 'branch_paths_checked': paths, 'examples_checked': total}

assembly_dataset/progress.py

@@ -0,0 +1,45 @@
+"""Terminal progress on stderr; dataset statistics remain machine-readable on stdout."""
+from rich.console import Console
+from rich.progress import (
+    BarColumn, MofNCompleteColumn, Progress, ProgressColumn,
+    TaskProgressColumn, TextColumn, TimeElapsedColumn, TimeRemainingColumn,
+)
+from rich.text import Text
+
+
+class RateColumn(ProgressColumn):
+    def render(self, task):
+        speed = task.finished_speed or task.speed
+        return Text(f'{speed:.1f} зап/с' if speed is not None else '— зап/с')
+
+
+class ProgressDisplay:
+    def __init__(self, label, total, *, initial=0, enabled=True):
+        self.label = label
+        self.total = total
+        self.progress = Progress(
+            TextColumn('{task.description}'), BarColumn(bar_width=None),
+            TaskProgressColumn(), MofNCompleteColumn(), RateColumn(),
+            TextColumn('прошло'), TimeElapsedColumn(),
+            TextColumn('осталось'), TimeRemainingColumn(),
+            console=Console(stderr=True), refresh_per_second=4,
+            redirect_stdout=False, redirect_stderr=False, disable=not enabled,
+        )
+        self.task = self.progress.add_task(label, total=total, completed=min(initial, total))
+
+    def __enter__(self):
+        self.progress.start()
+        return self
+
+    def update(self, completed=None, *, status=None):
+        kwargs = {}
+        if completed is not None:
+            kwargs['completed'] = min(completed, self.total)
+        if status is not None:
+            kwargs['description'] = f'{self.label}: {status}' if status else self.label
+        self.progress.update(self.task, **kwargs)
+
+    def __exit__(self, exc_type, exc, traceback):
+        if exc_type is not None:
+            self.update(status='прервано' if issubclass(exc_type, (KeyboardInterrupt, SystemExit)) else 'ошибка')
+        self.progress.stop()

assembly_dataset/prompts.py

@@ -0,0 +1,51 @@
+"""Editable prompts and validated payload templates."""
+from pathlib import Path
+from string import Template, Formatter
+
+import yaml
+from pydantic import model_validator
+from llm.config import ConfigModel
+
+DEFAULT_PROMPTS = Path(__file__).resolve().parents[1] / 'prompts.yaml'
+
+
+class Prompts(ConfigModel):
+    teacher_system: str
+    teacher_user: str
+    teacher_retry: str
+    training_system: dict[str, str]
+    structured_system: dict[str, str]
+    training_user: str
+    task_instructions: dict[str, str]
+    object_names: dict[str, str]
+    action_templates: dict[str, str]
+
+    @model_validator(mode='after')
+    def validate_texts(self):
+        texts = [self.teacher_system, *self.training_system.values(), *self.structured_system.values(), *self.task_instructions.values()]
+        if any(not text.strip() for text in texts):
+            raise ValueError('Промпты не должны быть пустыми')
+        if set(self.training_system) != {'full', 'step', 'branch'}:
+            raise ValueError('training_system должен содержать full, step, branch')
+        if set(self.structured_system) != {'full', 'step', 'branch'}:
+            raise ValueError('structured_system должен содержать full, step, branch')
+        for value in (self.teacher_user, self.teacher_retry, self.training_user):
+            template = Template(value)
+            if not template.is_valid() or set(template.get_identifiers()) != {'payload'}:
+                raise ValueError('Шаблон должен содержать ${payload}; другие переменные не поддерживаются')
+        for value in self.action_templates.values():
+            if not value.strip():
+                raise ValueError('Пустой шаблон действия')
+            fields = {field for _, field, _, _ in Formatter().parse(value) if field is not None}
+            if fields - {'object', 'target'}:
+                raise ValueError('В action_templates допустимы только {object} и {target}')
+            value.format(object='объект', target='место')
+        return self
+
+
+def load_prompts(path: Path = DEFAULT_PROMPTS) -> Prompts:
+    return Prompts.model_validate(yaml.safe_load(path.read_text(encoding='utf-8')))
+
+
+def render(template: str, payload: str) -> str:
+    return Template(template).substitute(payload=payload)

assembly_dataset/structured.py

@@ -0,0 +1,37 @@
+"""Public skill plans derived from the same validated actions as Russian text."""
+from .language import component_for, object_name, target_name
+
+PLACEMENT_SKILLS = {
+    'position', 'place', 'align', 'insert', 'insert_fastener', 'tighten',
+    'press_fit', 'route_wire', 'connect', 'seat_component', 'bond', 'clamp',
+    'weld', 'move_to_storage',
+}
+OPERATOR_SKILLS = {'move_to_operator', 'hold', 'handover'}
+
+
+def public_action(action, scene, prompts):
+    component = component_for(action, scene)
+    skill = action['skill']
+    result = {'skill': skill, 'object': object_name(component, scene, prompts)}
+    if skill in PLACEMENT_SKILLS:
+        result['target'] = target_name(component, scene)
+    elif skill in OPERATOR_SKILLS:
+        result['target'] = 'оператор'
+    elif skill == 'return_home':
+        result.update(object='робот', target='исходное положение')
+    return result
+
+
+def structured_answer(episode, mode, index, prompts):
+    actions = episode['plan']['actions']
+    status = episode['plan']['status']
+    if mode == 'step' and actions:
+        actions = actions[index:index + 1]
+        status = 'ready'
+    plan = [public_action(action, episode['scene'], prompts) for action in actions]
+    if mode == 'branch':
+        for i, action in enumerate(plan):
+            action.update(id=f'step_{i + 1}',
+                          on_success=f'step_{i + 2}' if i + 1 < len(plan) else status,
+                          on_failure='replan_required')
+    return {'status': status, 'plan': plan}

assembly_dataset/teacher.py

@@ -0,0 +1,71 @@
+"""Optional teacher plans through the project's existing model factory."""
+import asyncio
+import re
+from typing import Literal
+
+from pydantic import Field, field_validator
+from llm.config import ConfigModel
+from .prompts import render
+from .language import validate_texts, action_text
+from .world import canonical, validate_plan, branch_plan, validate_branch
+
+
+class Action(ConfigModel):
+    skill: str
+    object: str
+    target: str
+    text: str = Field(description='Одно действие на русском языке с названием объекта из сцены; без кода')
+
+    @field_validator('text')
+    @classmethod
+    def russian_description(cls, value):
+        value = value.strip()
+        if not re.search('[А-Яа-яЁё]', value) or re.search(r'\b(?:part_|slot_)\d+|\b(?:robot|camera)\.', value):
+            raise ValueError('Ожидается русское описание действия, без технических идентификаторов')
+        return value
+
+
+class Plan(ConfigModel):
+    status: Literal['ready', 'blocked']
+    actions: list[Action] = Field(default_factory=list)
+
+
+async def create_agent(config, prompts):
+    from pydantic_ai import Agent
+    from llm.factory import LLMModelProvider
+    model = await LLMModelProvider.create_from_config(config.llm)
+    return Agent(model=model, output_type=Plan, retries=config.retries,
+                 system_prompt=prompts.teacher_system)
+
+
+async def enrich(episode, config, agent, prompts):
+    if int(episode['signature'][16:32], 16) / 2**64 >= config.fraction:
+        return episode, None
+    teacher_scene = {**episode['scene'], 'operations': [
+        {**operation, 'reference_text': action_text(operation['action'], episode['scene'], prompts)}
+        for operation in episode['scene']['operations']]}
+    prompt = render(prompts.teacher_user, canonical({'instruction': episode['instruction'], 'scene': teacher_scene}))
+    error = None
+    for attempt in range(config.retries + 1):
+        try:
+            async with asyncio.timeout(config.timeout_seconds):
+                result = await agent.run(prompt, model_settings=config.generation.model_dump(exclude_none=True))
+            raw = result.output.model_dump()
+            texts = [action['text'] for action in raw['actions']]
+            plan = {'status': raw['status'], 'actions': [{k: v for k, v in a.items() if k != 'text'} for a in raw['actions']]}
+            validate_texts({**episode, 'plan': plan}, texts, prompts)
+            trace = validate_plan(episode, plan)
+            branch = branch_plan(plan)
+            validate_branch(episode, branch)
+            episode.update(plan=plan, trace=trace, branch=branch, step_texts=texts, text_source='teacher',
+                           source={'provider': config.llm.provider, 'model': config.llm.model})
+            return episode, None
+        except Exception as exc:
+            # API exceptions may contain request headers. Persist the class only.
+            error = {'scenario_id': episode['scenario_id'], 'error_type': type(exc).__name__, 'attempts': attempt + 1}
+            if isinstance(exc, ValueError):
+                prompt = render(prompts.teacher_retry, canonical({'instruction': episode['instruction'], 'scene': teacher_scene,
+                                    'validation_error': str(exc)[:500]}))
+    # A failed teacher is never silently labelled as a successful teacher plan.
+    episode['teacher_error'] = error
+    return episode, error

assembly_dataset/world.py

@@ -0,0 +1,298 @@
+"""Finite symbolic assembly worlds; model output cannot supply its own effects.
+
+These recipes describe logical prerequisites only, not physical feasibility.
+Every grounded operation in a scene is supplied by this trusted generator.
+"""
+import hashlib
+import json
+import random
+from copy import deepcopy
+
+from .config import Config
+from .prompts import render
+from .structured import structured_answer
+from .language import answer_text, readable_scene, scene_summary, step_texts
+
+# Skills before release must retain the component; 'installed' is a logical
+# stage, not an assertion that real geometry or process parameters are safe.
+RECIPES = {
+    'manipulation': ('Перемести детали в сборочные позиции', ['pick', 'position', 'place', 'release', 'inspect'], None),
+    'mechanical': ('Собери механический узел', ['pick', 'align', 'insert', 'release', 'inspect'], None),
+    'fastening': ('Установи и затяни крепёж', ['pick', 'align', 'insert_fastener', 'tighten', 'release', 'inspect'], 'driver'),
+    'press_fit': ('Собери прессовые соединения', ['pick', 'align', 'press_fit', 'release', 'inspect'], 'press'),
+    'electrical': ('Выполни электромонтаж', ['pick', 'route_wire', 'connect', 'release', 'test_continuity', 'inspect'], 'tester'),
+    'electronics': ('Установи электронные компоненты', ['pick', 'align', 'seat_component', 'release', 'test_connection', 'inspect'], 'tester'),
+    'adhesive': ('Собери клеевые соединения', ['clean_surface', 'apply_adhesive', 'pick', 'align', 'bond', 'release', 'wait_cure', 'inspect'], 'dispenser'),
+    'welding': ('Соедини детали сваркой', ['pick', 'align', 'clamp', 'release', 'weld', 'inspect'], 'welder'),
+    'collaborative': ('Передай детали оператору для сборки', ['wait_operator', 'pick', 'handover', 'release', 'confirm_assembly', 'inspect'], None),
+    'operator_assistance': ('Удерживай коробку возле оператора', ['detect', 'approach', 'pick', 'move_to_operator', 'hold', 'wait_assembly', 'move_to_storage', 'release', 'return_home'], None),
+    'inspection': ('Проверь детали в контрольной позиции', ['pick', 'position', 'place', 'release', 'measure', 'inspect'], 'gauge'),
+}
+# Explicit perception and approach precede grasping in the human-level plan.
+for _domain, (_title, _recipe, _tool) in list(RECIPES.items()):
+    if _domain != 'operator_assistance':
+        _pick = _recipe.index('pick')
+        RECIPES[_domain] = (_title, _recipe[:_pick] + ['detect', 'approach'] + _recipe[_pick:], _tool)
+
+
+
+def canonical(value):
+    return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
+
+
+def digest(value):
+    return hashlib.sha256(canonical(value).encode()).hexdigest()
+
+
+def action_key(action):
+    if not isinstance(action, dict) or set(action) != {'skill', 'object', 'target'}:
+        raise ValueError('Действие должно иметь только skill, object, target')
+    return canonical(action)
+
+
+def enabled(state, operation):
+    return set(operation['requires']) <= state and not (set(operation['forbids']) & state)
+
+
+def apply(state, operation):
+    if not enabled(state, operation):
+        raise ValueError(f"Нарушены предусловия: {operation['action']}")
+    return (state - set(operation['delete'])) | set(operation['add'])
+
+
+def solve(scene):
+    state = set(scene['initial_state'])
+    actions, trace = [], [sorted(state)]
+    remaining = list(scene['operations'])
+    while remaining:
+        operation = next((op for op in remaining if enabled(state, op)), None)
+        if operation is None:
+            break
+        state = apply(state, operation)
+        actions.append(operation['action'])
+        trace.append(sorted(state))
+        remaining.remove(operation)
+    status = 'ready' if set(scene['goal']) <= state else 'blocked'
+    return {'status': status, 'actions': actions}, trace
+
+
+def validate_plan(episode, plan):
+    if not isinstance(plan, dict) or set(plan) != {'status', 'actions'}:
+        raise ValueError('Ожидаются status и actions')
+    if plan['status'] not in ('ready', 'blocked') or not isinstance(plan['actions'], list):
+        raise ValueError('Неверный статус/список действий')
+    operations = {action_key(op['action']): op for op in episode['scene']['operations']}
+    state = set(episode['scene']['initial_state'])
+    trace = [sorted(state)]
+    for action in plan['actions']:
+        operation = operations.get(action_key(action))
+        if operation is None:
+            raise ValueError(f'Неизвестное действие или объект: {action}')
+        state = apply(state, operation)
+        trace.append(sorted(state))
+    achieved = set(episode['scene']['goal']) <= state
+    if plan['status'] == 'ready' and not achieved:
+        raise ValueError('План не достигает цели')
+    if plan['status'] == 'blocked':
+        # This domain has an acyclic recipe dependency graph. The deterministic
+        # solver is complete for its grounded operations (not arbitrary PDDL).
+        if achieved or solve(episode['scene'])[0]['status'] != 'blocked':
+            raise ValueError('Необоснованная блокировка выполнимой задачи')
+    return trace
+
+
+def branch_plan(plan):
+    nodes = []
+    for index, action in enumerate(plan['actions']):
+        nodes.append({'id': f'n{index}', 'action': action,
+                      'observation': 'action_succeeded',
+                      'on_success': f'n{index + 1}' if index + 1 < len(plan['actions']) else 'done',
+                      'on_failure': 'replan'})
+    nodes.extend([
+        {'id': 'done', 'terminal': plan['status']},
+        {'id': 'replan', 'terminal': 'replan_required'},
+    ])
+    return {'entry': 'n0' if plan['actions'] else 'done', 'nodes': nodes}
+
+
+def validate_branch(episode, branch):
+    if set(branch) != {'entry', 'nodes'} or not isinstance(branch['nodes'], list):
+        raise ValueError('Неверная схема графа')
+    nodes = {node['id']: node for node in branch['nodes']}
+    if len(nodes) != len(branch['nodes']):
+        raise ValueError('Повторяющиеся узлы')
+    operations = {action_key(op['action']): op for op in episode['scene']['operations']}
+    pending = [(branch['entry'], set(episode['scene']['initial_state']), frozenset(), False)]
+    paths = 0
+    while pending:
+        node_id, state, visited, failed = pending.pop()
+        if node_id not in nodes or node_id in visited:
+            raise ValueError('Отсутствующий узел или цикл')
+        node = nodes[node_id]
+        if failed and node.get('terminal') != 'replan_required':
+            raise ValueError('После сбоя необходима новая сцена и перепланирование')
+        if 'terminal' in node:
+            if set(node) != {'id', 'terminal'}:
+                raise ValueError('Неверный терминальный узел')
+            status = node['terminal']
+            if status == 'ready':
+                if not set(episode['scene']['goal']) <= state:
+                    raise ValueError('Ветка не достигает цели')
+            elif status == 'blocked':
+                if solve(episode['scene'])[0]['status'] != 'blocked':
+                    raise ValueError('Необоснованная блокировка')
+            elif status == 'replan_required':
+                if not failed:
+                    raise ValueError('Перепланирование без наблюдаемого сбоя')
+            else:
+                raise ValueError('Неизвестный терминальный статус')
+            paths += 1
+            continue
+        if set(node) != {'id', 'action', 'observation', 'on_success', 'on_failure'} or node['observation'] != 'action_succeeded':
+            raise ValueError('Неполный узел или неизвестное наблюдение')
+        operation = operations.get(action_key(node['action']))
+        if operation is None:
+            raise ValueError('Неизвестная операция')
+        success = apply(state, operation)
+        visited = visited | {node_id}
+        # The defined failure outcome has no success effects. Stop and obtain a
+        # fresh scene; never blindly retry a physical action after a failure.
+        pending.append((node['on_failure'], state, visited, True))
+        pending.append((node['on_success'], success, visited, failed))
+        if len(pending) + paths > 10000:
+            raise ValueError('Слишком много путей в графе')
+    return paths
+
+
+def make_episode(index: int, config: Config):
+    rng = random.Random(f'{config.seed}:{index}')
+    domain = config.domains[index % len(config.domains)]
+    title, recipe, tool = RECIPES[domain]
+    missing = set(recipe) - set(config.prompts.action_templates)
+    if missing:
+        raise ValueError(f'В action_templates отсутствуют навыки: {sorted(missing)}')
+    count = rng.randint(config.min_components, config.max_components)
+    state = {'gripper_empty', 'fixture_ready', 'operator_ready'}
+    if domain == 'operator_assistance':
+        state.add('robot_home')
+    if tool:
+        state.add(f'tool_available:{tool}')
+    components, operations = [], []
+    blocked = rng.random() < config.blocked_fraction
+    unavailable = rng.randrange(count) if blocked else -1
+    for i in range(count):
+        obj, target = f'part_{i}', f'slot_{i}'
+        dependencies = [f'part_{j}' for j in range(i) if rng.random() < 0.35]
+        # Positions, part classes and progress have operational meaning, and are
+        # included in the signature. No free-form names enter deduplication.
+        location = rng.choice(['tray', 'buffer', 'conveyor'])
+        components.append({'id': obj, 'target': target, 'type': rng.choice(['small', 'medium', 'large']),
+                           'source_location': location, 'dependencies': dependencies,
+                           'name': f"{config.prompts.object_names.get(domain, 'деталь')} №{i + 1}",
+                           'target_name': f"{'зона хранения' if domain == 'operator_assistance' else 'сборочная позиция'} №{i + 1}"})
+        state.add(f'at:{obj}:{location}')
+        if i != unavailable:
+            state.add(f'available:{obj}')
+        prefix = f'{obj}:stage:'
+        # Some parts are already installed; final inspection is still required.
+        start = len(recipe) - 1 if domain != 'operator_assistance' and i != unavailable and not dependencies and rng.random() < 0.2 else 0
+        state.add(f'{prefix}{start}')
+        for stage, skill in enumerate(recipe):
+            requires = [f'{prefix}{stage}', 'fixture_ready', f'available:{obj}']
+            adds, deletes = [f'{prefix}{stage + 1}'], [f'{prefix}{stage}']
+            if skill == 'pick':
+                requires += ['gripper_empty', f'at:{obj}:{location}']
+                requires += [f'complete:{dependency}' for dependency in dependencies]
+                adds += [f'holding:{obj}']
+                deletes += ['gripper_empty', f'at:{obj}:{location}']
+            elif skill == 'release':
+                requires += [f'holding:{obj}']
+                adds += ['gripper_empty', f'at:{obj}:{target}']
+                deletes += [f'holding:{obj}']
+            elif 'pick' in recipe[:stage] and 'release' not in recipe[:stage]:
+                requires += [f'holding:{obj}']
+            if skill in ('tighten', 'press_fit', 'apply_adhesive', 'weld', 'measure', 'test_connection', 'test_continuity') and tool:
+                requires += [f'tool_available:{tool}']
+            if skill == 'approach':
+                requires += ['gripper_empty']
+                if domain == 'operator_assistance':
+                    deletes += ['robot_home']
+            if skill == 'wait_assembly':
+                adds += [f'operator_finished:{obj}']
+            if skill == 'move_to_storage':
+                requires += [f'operator_finished:{obj}']
+            if skill == 'return_home':
+                requires += ['gripper_empty']
+                adds += ['robot_home']
+            if skill in ('wait_operator', 'handover', 'confirm_assembly', 'move_to_operator', 'hold', 'wait_assembly'):
+                requires += ['operator_ready']
+            if stage == len(recipe) - 1:
+                adds += [f'complete:{obj}']
+            operations.append({'action': {'skill': skill, 'object': obj, 'target': target},
+                               'requires': requires, 'forbids': [], 'add': adds, 'delete': deletes,
+                               'failure_effects': []})
+        if start:
+            state.discard(f'at:{obj}:{location}')
+            state.add(f'at:{obj}:{target}')
+    scene = {'domain': domain, 'components': components,
+             'initial_state': sorted(state), 'goal': ['gripper_empty'] + [f'complete:part_{i}' for i in range(count)],
+             'operations': operations}
+    if domain == 'operator_assistance':
+        scene['goal'].append('robot_home')
+    signature = digest(scene)
+    plan, trace = solve(scene)
+    episode = {'scenario_id': signature, 'signature': signature, 'candidate_index': index,
+               'instruction': config.prompts.task_instructions[domain],
+               'scene': scene, 'plan': plan, 'trace': trace, 'branch': branch_plan(plan),
+               'validation_level': 'logical_only', 'source': {'provider': 'symbolic', 'model': None}}
+    validate_plan(episode, plan)
+    validate_branch(episode, episode['branch'])
+    return episode
+
+
+def split_for(signature, config):
+    value = int(signature[:16], 16) / 2**64
+    return 'train' if value < config.split[0] else 'validation' if value < sum(config.split[:2]) else 'test'
+
+
+def training_rows(episode, config, *, output_format='text'):
+    """All views share the same split; only pre-action state enters step input."""
+    if output_format not in ('text', 'structured'):
+        raise ValueError('Неизвестный формат ответа')
+    common = {'instruction': episode['instruction'],
+              'scene': readable_scene(episode['scene'], episode['scene']['initial_state'], config.prompts)}
+    for mode in config.modes:
+        if mode == 'step':
+            actions = episode['plan']['actions']
+            indices = range(len(actions)) if config.step_sampling == 'all' else [int(episode['signature'][:8], 16) % max(1, len(actions))]
+            if not actions:
+                indices = [0]
+            for step in indices:
+                prompt = deepcopy(common)
+                prompt['scene'] = readable_scene(episode['scene'], episode['trace'][step], config.prompts)
+                prompt['history'] = step_texts(episode, config.prompts)[:step]
+                answer = {'status': 'continue', 'action': actions[step]} if actions else {'status': episode['plan']['status'], 'action': None}
+                yield row(episode, mode, step, prompt, answer, config, output_format)
+        else:
+            prompt = deepcopy(common)
+            if mode == 'branch':
+                prompt['observations'] = ['действие выполнено успешно', 'при выполнении произошла ошибка']
+            yield row(episode, mode, 0, prompt, episode['plan'] if mode == 'full' else episode['branch'], config, output_format)
+
+
+def row(episode, mode, index, prompt, answer, config, output_format):
+    prompt['mode'] = mode
+    structured = output_format == 'structured'
+    public_answer = structured_answer(episode, mode, index, config.prompts) if structured else None
+    return {'record_id': f"{episode['scenario_id']}:{mode}:{index}",
+            'scenario_id': episode['scenario_id'], 'split': split_for(episode['signature'], config),
+            'mode': mode, 'validation_level': 'logical_only', 'source': episode['source'],
+            'task': episode['instruction'], 'scene_summary': scene_summary(prompt['scene']),
+            'status': public_answer['status'] if structured else (('continue' if episode['plan']['actions'] else episode['plan']['status']) if mode == 'step' else episode['plan']['status']),
+            'text_source': episode.get('text_source', 'template'),
+            'output_format': output_format,
+            'messages': [
+                {'role': 'system', 'content': (config.prompts.structured_system if structured else config.prompts.training_system)[mode]},
+                {'role': 'user', 'content': render(config.prompts.training_user, canonical(prompt))},
+                {'role': 'assistant', 'content': canonical(public_answer) if structured else answer_text(episode, mode, index, config.prompts)},
+            ]}