обновлен формат генерации датасета

llm/config.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 from typing import Annotated, Literal
 
-import yaml
 from pydantic import BaseModel, ConfigDict, Field
 
 
@@ -31,36 +30,8 @@ LLMConfig = Annotated[
 
 # Generation
 class GenerationConfig(ConfigModel):
-    temperature: float = Field(default=.7,
+    temperature: float | None = Field(default=None,
                                ge=.0,
                                le=2.0)
     max_tokens: int = Field(default=4096,
                             gt=0)
-    
-
-# Runtime
-class RuntimeConfig(ConfigModel):
-    concurrency: int = Field(ge=1, default=1)
-
-# Prompts
-class PromptsConfig(ConfigModel):
-    instructions: str
-    system: str
-    user: str
-
-# Application
-class AppConfig(ConfigModel):
-    llm: LLMConfig
-    generation: GenerationConfig = GenerationConfig()
-    runtime: RuntimeConfig = RuntimeConfig()
-    prompts: PromptsConfig
-
-
-# Loader
-def load_config(path: str | Path = "config.yaml") -> AppConfig:
-    path = Path(path)
-
-    with path.open("r", encoding="utf-8") as file:
-        data = yaml.safe_load(file)
-
-    return AppConfig.model_validate(data)

llm/factory.py

@@ -1,6 +1,5 @@
 import os
 import webbrowser
-import pandas as pd
 from pathlib import Path
 
 from pydantic_ai.models.openai import OpenAIChatModel
@@ -13,7 +12,6 @@ from pydantic_ai.providers.openai_codex import (
 
 from .config import CodexConfig, LLMConfig, OpenAiConfig
 from .credentials import FileCodexCredentionalSource
-from .schemas import BaseDatasetRecord, ChatRecord, ChatMessage
 
 
 class LLMModelProvider:
@@ -22,7 +20,7 @@ class LLMModelProvider:
     def create_openai_compatible(model_name: str,
                                  api_key: str,
                                  base_url: str
-                                ) -> OpenAICodexModel:
+                                ) -> OpenAIChatModel:
 
         provider = OpenAIProvider(base_url=base_url,
                                 api_key=api_key)
@@ -84,68 +82,3 @@ class LLMModelProvider:
             )
 
         raise ValueError(f"Unsupported LLM config: {type(config)}")
-
-class DatasetWriter:
-    def __init__(self,
-                 dataset_record: BaseDatasetRecord,
-                 base_path: str | Path):
-
-
-        self._dataset_record = dataset_record    
-        self._path = Path(base_path)
-        self._path.mkdir(parents=True, exist_ok=True)
-
-    def __check_file_exists(self, filename :str)-> bool:
-        file_path = self._path / filename
-        return file_path.exists()
-
-    def to_csv(self,
-               filename: str) -> None:
-
-        exists=self.__check_file_exists(filename)
-        
-        df = pd.json_normalize(
-            self._dataset_record.model_dump()
-        )
-
-        df.to_csv(self._path / filename,
-                  mode="a" if exists else "w",
-                  header=not exists,
-                  index=False,
-                  encoding="utf-8")
-
-    def to_chat_record(self) -> ChatRecord:
-        response = self._dataset_record.response
-        prompts = self._dataset_record.prompts
-
-        assistant_content = (
-            response.assistant_content
-            or response.model_response
-        )
-
-        return ChatRecord(
-            messages=[
-                ChatMessage(
-                    role="system",
-                    content=prompts.system_prompt,
-                ),
-                ChatMessage(
-                    role="user",
-                    content=prompts.user_prompt,
-                ),
-                ChatMessage(
-                    role="assistant",
-                    content=assistant_content,
-                ),
-            ]
-        )
-
-    def to_jsonl(self,
-                 filename: str) -> None:
-        
-        chat_record = self.to_chat_record()
-        path = self._path/filename
-
-        with path.open("a", encoding='utf-8') as file:
-            file.write(chat_record.model_dump_json())
-            file.write("\n")

llm/schemas.py

@@ -1,49 +0,0 @@
-from typing import Literal, Optional
-
-from pydantic import BaseModel, Field
-
-
-class RobotPlan(BaseModel):
-    steps: list[str] = Field(
-        description="Последовательность высокоуровневых действий робота",
-        min_length=1
-    )
-
-
-class ChatMessage(BaseModel):
-    role: Literal["system",
-                  "user",
-                  "assistant"]
-
-    content: str
-
-
-class ChatRecord(BaseModel):
-    messages: list[ChatMessage]
-
-class Prompts(BaseModel):
-    system_prompt: str
-    instructions: str
-    user_prompt: str
-
-
-class ModelResponse(BaseModel):
-    model_response: str
-    assistant_content: Optional[str] = None
-
-class ModelInfo(BaseModel):
-    provider: str
-    model: str
-
-class GenerationSettings(BaseModel):
-    temperature: float
-    max_tokens: int
-
-class BaseDatasetRecord(BaseModel):
-    prompts: Prompts
-    response: ModelResponse
-    model_info: ModelInfo
-    generation: GenerationSettings
-
-class RobotDatasetRecord(BaseDatasetRecord):
-    task: str