synth-ai 0.2.9.dev7__py3-none-any.whl → 0.2.9.dev8__py3-none-any.whl

synth_ai/learning/sft/config.py

@@ -0,0 +1,270 @@
+"""Utilities for validating and constructing SFT job payloads."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field, fields
+from typing import Any
+
+from synth_ai.api.models.supported import (
+    UnsupportedModelError,
+    normalize_model_identifier,
+)
+
+_STEP_KEYS = ("n_epochs", "total_steps", "train_steps", "steps")
+
+
+def _ensure_positive_int(value: Any, *, key: str) -> int:
+    if isinstance(value, bool):
+        raise ValueError(f"hyperparameters.{key} must be an integer greater than zero")
+    try:
+        ivalue = int(value)
+    except (TypeError, ValueError) as exc:  # pragma: no cover - defensive
+        raise ValueError(f"hyperparameters.{key} must be an integer greater than zero") from exc
+    if ivalue <= 0:
+        raise ValueError(f"hyperparameters.{key} must be an integer greater than zero")
+    return ivalue
+
+
+def _ensure_non_negative_float(value: Any, *, key: str) -> float:
+    if isinstance(value, bool):
+        raise ValueError(f"hyperparameters.{key} must be a float greater than or equal to zero")
+    try:
+        fvalue = float(value)
+    except (TypeError, ValueError) as exc:  # pragma: no cover - defensive
+        raise ValueError(
+            f"hyperparameters.{key} must be a float greater than or equal to zero"
+        ) from exc
+    if fvalue < 0:
+        raise ValueError(f"hyperparameters.{key} must be a float greater than or equal to zero")
+    return fvalue
+
+
+def _ensure_positive_float(value: Any, *, key: str) -> float:
+    fvalue = _ensure_non_negative_float(value, key=key)
+    if fvalue == 0.0:
+        raise ValueError(f"hyperparameters.{key} must be greater than zero")
+    return fvalue
+
+
+@dataclass(slots=True)
+class SFTTrainingHyperparameters:
+    """Typed representation of SFT training hyperparameters."""
+
+    n_epochs: int | None = None
+    total_steps: int | None = None
+    train_steps: int | None = None
+    steps: int | None = None
+    batch_size: int | None = None
+    global_batch: int | None = None
+    per_device_batch: int | None = None
+    gradient_accumulation_steps: int | None = None
+    sequence_length: int | None = None
+    learning_rate: float | None = None
+    warmup_ratio: float | None = None
+    train_kind: str | None = None
+    extras: dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_mapping(cls, data: Mapping[str, Any] | None) -> SFTTrainingHyperparameters:
+        if data is None:
+            raise ValueError("hyperparameters must not be empty")
+        normalized: dict[str, Any] = dict(data)
+        if not normalized:
+            raise ValueError("hyperparameters must not be empty")
+
+        kwargs: dict[str, Any] = {}
+
+        def pop_int(name: str) -> int | None:
+            if name not in normalized:
+                return None
+            value = _ensure_positive_int(normalized.pop(name), key=name)
+            return value
+
+        def pop_optional_int(name: str) -> int | None:
+            if name not in normalized:
+                return None
+            value = _ensure_positive_int(normalized.pop(name), key=name)
+            return value
+
+        def pop_positive_float(name: str) -> float | None:
+            if name not in normalized:
+                return None
+            return _ensure_positive_float(normalized.pop(name), key=name)
+
+        def pop_non_negative_float(name: str) -> float | None:
+            if name not in normalized:
+                return None
+            value = _ensure_non_negative_float(normalized.pop(name), key=name)
+            return value
+
+        # Step-derived keys
+        step_values = {
+            "n_epochs": pop_int("n_epochs"),
+            "total_steps": pop_int("total_steps"),
+            "train_steps": pop_int("train_steps"),
+            "steps": pop_int("steps"),
+        }
+        if not any(step_values.values()):
+            keys = ", ".join(_STEP_KEYS)
+            raise ValueError(f"hyperparameters must include at least one of: {keys}")
+        kwargs.update(step_values)
+
+        kwargs["batch_size"] = pop_optional_int("batch_size")
+        kwargs["global_batch"] = pop_optional_int("global_batch")
+        kwargs["per_device_batch"] = pop_optional_int("per_device_batch")
+        kwargs["gradient_accumulation_steps"] = pop_optional_int("gradient_accumulation_steps")
+        kwargs["sequence_length"] = pop_optional_int("sequence_length")
+        kwargs["learning_rate"] = pop_positive_float("learning_rate")
+        kwargs["warmup_ratio"] = pop_non_negative_float("warmup_ratio")
+
+        if "warmup_ratio" in kwargs and kwargs["warmup_ratio"] is not None:
+            ratio = kwargs["warmup_ratio"]
+            if ratio > 1:
+                raise ValueError("hyperparameters.warmup_ratio must be between 0 and 1 inclusive")
+
+        if "train_kind" in normalized:
+            value = normalized.pop("train_kind")
+            if not isinstance(value, str):
+                raise ValueError("hyperparameters.train_kind must be a string")
+            kwargs["train_kind"] = value
+
+        extras = normalized
+
+        return cls(extras=extras, **kwargs)
+
+    def to_dict(self) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for field_info in fields(self):
+            if field_info.name == "extras":
+                continue
+            value = getattr(self, field_info.name)
+            if value is not None:
+                result[field_info.name] = value
+        result.update(self.extras)
+        return result
+
+
+def _coerce_mapping(value: Mapping[str, Any] | None, *, name: str) -> dict[str, Any]:
+    if value is None:
+        return {}
+    if not isinstance(value, Mapping):
+        raise ValueError(f"{name} must be a mapping")
+    return dict(value)
+
+
+@dataclass(slots=True)
+class SFTJobConfig:
+    """Structured representation of an SFT training job request."""
+
+    model: str
+    hyperparameters: Mapping[str, Any] | SFTTrainingHyperparameters
+    training_file: str | None = None
+    metadata: Mapping[str, Any] | None = None
+    training_type: str | None = "sft_offline"
+    validation_file: str | None = None
+    suffix: str | None = None
+    integrations: Mapping[str, Any] | None = None
+
+    def to_payload(
+        self,
+        *,
+        training_file_field: str = "training_file_id",
+        require_training_file: bool = True,
+        include_training_file_when_none: bool = False,
+        allow_finetuned_prefixes: bool = False,
+    ) -> dict[str, Any]:
+        model = normalize_model_identifier(
+            self.model, allow_finetuned_prefixes=allow_finetuned_prefixes
+        )
+        if isinstance(self.hyperparameters, SFTTrainingHyperparameters):
+            hyper_config = self.hyperparameters
+        else:
+            hyper_config = SFTTrainingHyperparameters.from_mapping(
+                _coerce_mapping(self.hyperparameters, name="hyperparameters")
+            )
+        hyperparameters = hyper_config.to_dict()
+
+        payload: dict[str, Any] = {
+            "model": model,
+            "hyperparameters": hyperparameters,
+        }
+
+        training_type = (self.training_type or "").strip() if self.training_type else ""
+        if training_type:
+            payload["training_type"] = training_type
+
+        metadata = _coerce_mapping(self.metadata, name="metadata")
+        if metadata:
+            payload["metadata"] = metadata
+
+        integrations = _coerce_mapping(self.integrations, name="integrations")
+        if integrations:
+            payload["integrations"] = integrations
+
+        suffix = (self.suffix or "").strip()
+        if suffix:
+            payload["suffix"] = suffix
+
+        validation_file = (self.validation_file or "").strip()
+        if validation_file:
+            payload["validation_file"] = validation_file
+
+        if training_file_field:
+            training_file = (self.training_file or "").strip() if self.training_file else ""
+            if training_file:
+                payload[training_file_field] = training_file
+            elif require_training_file:
+                raise ValueError("training file identifier is required for SFT jobs")
+            elif include_training_file_when_none:
+                payload[training_file_field] = None
+
+        return payload
+
+
+def prepare_sft_job_payload(
+    *,
+    model: str,
+    hyperparameters: Mapping[str, Any] | SFTTrainingHyperparameters | None,
+    training_file: str | None = None,
+    metadata: Mapping[str, Any] | None = None,
+    training_type: str | None = "sft_offline",
+    validation_file: str | None = None,
+    suffix: str | None = None,
+    integrations: Mapping[str, Any] | None = None,
+    training_file_field: str = "training_file_id",
+    require_training_file: bool = True,
+    include_training_file_when_none: bool = False,
+    allow_finetuned_prefixes: bool = False,
+) -> dict[str, Any]:
+    """Validate inputs and return an SFT job payload suitable for API calls."""
+
+    if isinstance(hyperparameters, SFTTrainingHyperparameters):
+        hyper_config = hyperparameters
+    else:
+        hyper_config = SFTTrainingHyperparameters.from_mapping(hyperparameters or {})
+
+    config = SFTJobConfig(
+        model=model,
+        training_file=training_file,
+        hyperparameters=hyper_config,
+        metadata=metadata,
+        training_type=training_type,
+        validation_file=validation_file,
+        suffix=suffix,
+        integrations=integrations,
+    )
+    return config.to_payload(
+        training_file_field=training_file_field,
+        require_training_file=require_training_file,
+        include_training_file_when_none=include_training_file_when_none,
+        allow_finetuned_prefixes=allow_finetuned_prefixes,
+    )
+
+
+__all__ = [
+    "SFTTrainingHyperparameters",
+    "SFTJobConfig",
+    "prepare_sft_job_payload",
+    "UnsupportedModelError",
+]

synth_ai/learning/sft/data.py

@@ -0,0 +1,295 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Iterator, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+SFTMessageContent = str | dict[str, Any] | list[Any] | None
+
+
+class SFTDataError(ValueError):
+    """Raised when a JSONL record cannot be coerced into an SFTExample."""
+
+
+@dataclass(slots=True)
+class SFTToolDefinition:
+    name: str
+    description: str | None
+    parameters: dict[str, Any] | None
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class SFTToolCall:
+    name: str
+    arguments: Any
+    call_id: str | None = None
+    type: str | None = None
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class SFTMessage:
+    role: str
+    content: SFTMessageContent
+    tool_calls: list[SFTToolCall] = field(default_factory=list)
+    tool_call_id: str | None = None
+    name: str | None = None
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class SFTExample:
+    messages: list[SFTMessage]
+    tools: list[SFTToolDefinition] = field(default_factory=list)
+    tool_choice: Any | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+def _parse_tool_arguments(value: Any) -> Any:
+    if isinstance(value, str):
+        try:
+            return json.loads(value)
+        except json.JSONDecodeError:
+            return value
+    return value
+
+
+def _coerce_tool_definition(raw: Any, *, index: int) -> SFTToolDefinition:
+    if not isinstance(raw, dict):
+        raise SFTDataError(f"tool {index} is not an object")
+    name = raw.get("name")
+    if not isinstance(name, str) or not name.strip():
+        raise SFTDataError(f"tool {index} missing name")
+    description = raw.get("description")
+    if description is not None and not isinstance(description, str):
+        raise SFTDataError(f"tool {index} description must be a string if present")
+    parameters = raw.get("parameters")
+    if parameters is not None and not isinstance(parameters, dict):
+        raise SFTDataError(f"tool {index} parameters must be an object if present")
+    return SFTToolDefinition(
+        name=name, description=description, parameters=parameters, raw=dict(raw)
+    )
+
+
+def _coerce_tool_call(raw: Any, *, index: int) -> SFTToolCall:
+    if not isinstance(raw, dict):
+        raise SFTDataError(f"tool_call {index} is not an object")
+
+    call_id = raw.get("id")
+    call_type = raw.get("type")
+
+    fn_payload: dict[str, Any] | None = None
+    name: str | None = None
+    arguments: Any = None
+
+    if isinstance(raw.get("function"), dict):
+        fn_payload = raw["function"]
+        name = fn_payload.get("name") if isinstance(fn_payload.get("name"), str) else None
+        arguments = fn_payload.get("arguments")
+    if name is None:
+        maybe_name = raw.get("name")
+        if isinstance(maybe_name, str):
+            name = maybe_name
+            arguments = raw.get("arguments")
+
+    if not isinstance(name, str) or not name.strip():
+        raise SFTDataError(f"tool_call {index} missing function name")
+
+    parsed_arguments = _parse_tool_arguments(arguments)
+
+    normalized_id = None
+    if call_id is not None:
+        normalized_id = str(call_id)
+    normalized_type = None
+    if call_type is not None:
+        normalized_type = str(call_type)
+
+    return SFTToolCall(
+        name=name,
+        arguments=parsed_arguments,
+        call_id=normalized_id,
+        type=normalized_type,
+        raw=dict(raw),
+    )
+
+
+def _coerce_message(raw: Any, *, index: int) -> SFTMessage:
+    if not isinstance(raw, dict):
+        raise SFTDataError(f"message {index} is not an object")
+    role = raw.get("role")
+    if not isinstance(role, str) or not role.strip():
+        raise SFTDataError(f"message {index} has invalid role")
+
+    content = raw.get("content")
+    if content is not None and not isinstance(content, str | list | dict):
+        raise SFTDataError(f"message {index} has unsupported content type {type(content).__name__}")
+
+    raw_tool_calls = raw.get("tool_calls")
+    tool_calls: list[SFTToolCall] = []
+    if raw_tool_calls is not None:
+        if not isinstance(raw_tool_calls, list | tuple):
+            raise SFTDataError(f"message {index} tool_calls must be a list")
+        for call_index, call in enumerate(raw_tool_calls):
+            tool_calls.append(_coerce_tool_call(call, index=call_index))
+
+    tool_call_id = raw.get("tool_call_id")
+    if tool_call_id is not None and not isinstance(tool_call_id, str):
+        tool_call_id = str(tool_call_id)
+
+    name = raw.get("name")
+    if name is not None and not isinstance(name, str):
+        raise SFTDataError(f"message {index} name must be a string if present")
+
+    extra = {
+        key: value
+        for key, value in raw.items()
+        if key not in {"role", "content", "tool_calls", "tool_call_id", "name"}
+    }
+
+    return SFTMessage(
+        role=role,
+        content=content,
+        tool_calls=tool_calls,
+        tool_call_id=tool_call_id,
+        name=name,
+        extra=extra,
+    )
+
+
+def coerce_example(raw: Any, *, min_messages: int = 1) -> SFTExample:
+    if not isinstance(raw, dict):
+        raise SFTDataError("record is not an object")
+
+    messages_raw = raw.get("messages")
+    if not isinstance(messages_raw, Sequence):
+        raise SFTDataError("missing messages[] list")
+    if len(messages_raw) < min_messages:
+        raise SFTDataError(f"missing messages[] with at least {min_messages} turns")
+
+    messages = [_coerce_message(msg, index=i) for i, msg in enumerate(messages_raw)]
+
+    tools: list[SFTToolDefinition] = []
+    if "tools" in raw and raw["tools"] is not None:
+        tools_raw = raw["tools"]
+        if not isinstance(tools_raw, Sequence):
+            raise SFTDataError("tools must be provided as a list when present")
+        for tool_index, tool in enumerate(tools_raw):
+            tools.append(_coerce_tool_definition(tool, index=tool_index))
+
+    tool_choice = raw.get("tool_choice")
+
+    metadata_field = raw.get("metadata")
+    metadata: dict[str, Any] = {}
+    if metadata_field is not None:
+        if not isinstance(metadata_field, dict):
+            raise SFTDataError("metadata must be an object if present")
+        metadata = dict(metadata_field)
+
+    extra = {
+        key: value
+        for key, value in raw.items()
+        if key not in {"messages", "tools", "tool_choice", "metadata"}
+    }
+
+    return SFTExample(
+        messages=messages,
+        tools=tools,
+        tool_choice=tool_choice,
+        metadata=metadata,
+        extra=extra,
+    )
+
+
+def parse_jsonl_line(line: str, *, min_messages: int = 1) -> SFTExample:
+    record = json.loads(line)
+    return coerce_example(record, min_messages=min_messages)
+
+
+def iter_sft_examples(
+    source: Iterable[str], *, min_messages: int = 1, skip_empty: bool = True
+) -> Iterator[SFTExample]:
+    for line in source:
+        if skip_empty and not line.strip():
+            continue
+        yield parse_jsonl_line(line, min_messages=min_messages)
+
+
+def collect_sft_jsonl_errors(
+    path: Path,
+    *,
+    min_messages: int = 1,
+    max_lines: int | None = None,
+    max_errors: int | None = None,
+) -> list[str]:
+    errors: list[str] = []
+    lines_checked = 0
+
+    with path.open("r", encoding="utf-8") as fh:
+        for lineno, raw_line in enumerate(fh, start=1):
+            if max_lines is not None and lines_checked >= max_lines:
+                break
+            stripped = raw_line.strip()
+            if not stripped:
+                continue
+            lines_checked += 1
+            try:
+                parse_jsonl_line(stripped, min_messages=min_messages)
+            except json.JSONDecodeError as exc:
+                errors.append(f"Line {lineno}: invalid JSON ({exc.msg})")
+            except SFTDataError as exc:
+                errors.append(f"Line {lineno}: {exc}")
+            if max_errors is not None and len(errors) >= max_errors:
+                break
+    if lines_checked == 0 and (max_errors is None or len(errors) < max_errors):
+        errors.append("File contains no SFT examples")
+    return errors
+
+
+def validate_jsonl_or_raise(
+    path: Path,
+    *,
+    min_messages: int = 1,
+    max_lines: int | None = None,
+    max_errors: int | None = None,
+    error_factory: type[Exception] = ValueError,
+) -> None:
+    if not path.exists():
+        raise FileNotFoundError(str(path))
+
+    issues = collect_sft_jsonl_errors(
+        path,
+        min_messages=min_messages,
+        max_lines=max_lines,
+        max_errors=max_errors,
+    )
+    if issues:
+        truncated = max_errors is not None and len(issues) >= max_errors
+        suffix = "" if not truncated else f" (showing first {max_errors} issues)"
+        details = "\n - ".join(issues)
+        raise error_factory(f"{path}: Dataset validation failed{suffix}:\n - {details}")
+
+
+def load_jsonl(path: Path, *, min_messages: int = 1) -> list[SFTExample]:
+    if not path.exists():
+        raise FileNotFoundError(str(path))
+    with path.open("r", encoding="utf-8") as fh:
+        return list(iter_sft_examples(fh, min_messages=min_messages))
+
+
+__all__ = [
+    "SFTDataError",
+    "SFTExample",
+    "SFTMessage",
+    "SFTToolCall",
+    "SFTToolDefinition",
+    "collect_sft_jsonl_errors",
+    "coerce_example",
+    "iter_sft_examples",
+    "load_jsonl",
+    "parse_jsonl_line",
+    "validate_jsonl_or_raise",
+]

synth_ai/learning/sse.py

@@ -2,7 +2,8 @@ from __future__ import annotations
 
 import json
 import time
-from typing import Any, Callable, Optional
+from collections.abc import Callable
+from contextlib import suppress
 
 import aiohttp
 
@@ -18,7 +19,7 @@ async def stream_events(
     job_id: str,
     *,
     seconds: int = 60,
-    on_event: Optional[Callable[[dict], None]] = None,
+    on_event: Callable[[dict], None] | None = None,
 ) -> None:
     if seconds <= 0:
         return
@@ -29,28 +30,28 @@ async def stream_events(
     ]
     for url in candidates:
         try:
-            async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session:
-                async with session.get(url, headers=headers) as resp:
-                    if resp.status != 200:
+            async with (
+                aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session,
+                session.get(url, headers=headers) as resp,
+            ):
+                if resp.status != 200:
+                    continue
+                start_t = time.time()
+                async for raw in resp.content:
+                    line = raw.decode(errors="ignore").strip()
+                    if not line or line.startswith(":"):
                         continue
-                    start_t = time.time()
-                    async for raw in resp.content:
-                        line = raw.decode(errors="ignore").strip()
-                        if not line or line.startswith(":"):
-                            continue
-                        if not line.startswith("data:"):
-                            continue
-                        data = line[5:].strip()
-                        try:
-                            obj = json.loads(data)
-                        except Exception:
-                            continue
-                        if on_event:
-                            try:
-                                on_event(obj)
-                            except Exception:
-                                pass
-                        if (time.time() - start_t) >= seconds:
-                            return
+                    if not line.startswith("data:"):
+                        continue
+                    data = line[5:].strip()
+                    try:
+                        obj = json.loads(data)
+                    except Exception:
+                        continue
+                    if on_event:
+                        with suppress(Exception):
+                            on_event(obj)
+                    if (time.time() - start_t) >= seconds:
+                        return
         except Exception:
             continue