synth-ai 0.2.14__py3-none-any.whl → 0.2.16__py3-none-any.whl

synth_ai/task/rubrics.py

@@ -0,0 +1,219 @@
+"""Rubric schema, loading, and scoring helpers for Task Apps."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Criterion(BaseModel):
+    id: str
+    description: str
+    weight: float = 1.0
+    required: bool = False
+
+    @field_validator("weight")
+    @classmethod
+    def _validate_weight(cls, value: float) -> float:
+        if value <= 0:
+            raise ValueError("criterion weight must be positive")
+        return value
+
+
+class Rubric(BaseModel):
+    version: str
+    goal_text: str | None = None
+    criteria: list[Criterion] = Field(default_factory=list)
+    aggregation: str = "weighted_sum"
+
+    @field_validator("aggregation")
+    @classmethod
+    def _validate_aggregation(cls, value: str) -> str:
+        allowed = {"sum", "weighted_sum", "custom", "inherit"}
+        if value not in allowed:
+            raise ValueError(f"aggregation must be one of {sorted(allowed)}")
+        return value
+
+    @field_validator("criteria")
+    @classmethod
+    def _validate_criteria(cls, criteria: list[Criterion]) -> list[Criterion]:
+        seen = set()
+        for criterion in criteria:
+            if criterion.id in seen:
+                raise ValueError(f"duplicate criterion id: {criterion.id}")
+            seen.add(criterion.id)
+        return criteria
+
+
+def _load_text(source: str) -> tuple[str, str | None]:
+    path = Path(source)
+    if path.exists():
+        return path.read_text(encoding="utf-8"), path.suffix.lower()
+    return source, None
+
+
+def _parse_structured(text: str, suffix: str | None) -> dict[str, Any]:
+    text = text.strip()
+    if not text:
+        raise ValueError("Rubric source is empty")
+    if suffix in (".yaml", ".yml"):
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load YAML rubrics") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric YAML must produce a mapping") from None
+        return data
+    if text.startswith("{"):
+        return json.loads(text)
+    if text.startswith("http://") or text.startswith("https://"):
+        import requests  # type: ignore
+
+        response = requests.get(text, timeout=15)
+        response.raise_for_status()
+        return _parse_structured(response.text, suffix)
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load rubric text") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric text must decode to a mapping") from None
+        return data
+
+
+def load_rubric(source: str | dict[str, Any] | Rubric | None) -> Rubric | None:
+    if source is None:
+        return None
+    if isinstance(source, Rubric):
+        return source
+    if isinstance(source, dict):
+        return Rubric.model_validate(source)
+    text, suffix = _load_text(str(source))
+    data = _parse_structured(text, suffix)
+    return Rubric.model_validate(data)
+
+
+def _merge_weights(base: Criterion, override: Criterion) -> float:
+    if override.weight != 1.0 and base.weight != 1.0:
+        return base.weight * override.weight
+    if override.weight != 1.0:
+        return override.weight
+    return base.weight
+
+
+def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None:
+    if override is None and base is None:
+        return None
+    if base is None:
+        return override
+    if override is None:
+        return base
+
+    base_map = {criterion.id: criterion for criterion in base.criteria}
+    merged: list[Criterion] = []
+
+    for ov in override.criteria:
+        if ov.id in base_map:
+            existing = base_map.pop(ov.id)
+            merged.append(
+                Criterion(
+                    id=ov.id,
+                    description=ov.description or existing.description,
+                    weight=_merge_weights(existing, ov),
+                    required=ov.required if ov.required is not None else existing.required,
+                )
+            )
+        else:
+            merged.append(ov)
+
+    merged.extend(base_map.values())
+
+    aggregation = override.aggregation
+    if aggregation == "inherit":
+        aggregation = base.aggregation
+
+    return Rubric(
+        version=override.version or base.version,
+        goal_text=override.goal_text or base.goal_text,
+        criteria=merged,
+        aggregation=aggregation,
+    )
+
+
+def _as_float(value: Any) -> float | None:
+    try:
+        return float(value)
+    except Exception:
+        return None
+
+
+def _score(
+    criteria: Iterable[Criterion], values: dict[str, float], aggregation: str
+) -> dict[str, Any]:
+    if aggregation == "inherit":
+        aggregation = "weighted_sum"
+    per_criterion: dict[str, dict[str, Any]] = {}
+    total = 0.0
+    total_weight = 0.0
+    for criterion in criteria:
+        score = values.get(criterion.id, 0.0)
+        per_criterion[criterion.id] = {
+            "score": score,
+            "weight": criterion.weight,
+            "required": criterion.required,
+        }
+        if aggregation == "sum":
+            total += score
+        elif aggregation == "weighted_sum":
+            total += score * criterion.weight
+            total_weight += criterion.weight
+    if aggregation == "weighted_sum" and total_weight > 0:
+        total = total / total_weight
+    if aggregation == "custom":
+        total = None  # type: ignore[assignment]
+    return {
+        "aggregation": aggregation,
+        "score": total,
+        "per_criterion": per_criterion,
+    }
+
+
+def score_events_against_rubric(
+    events: list[dict[str, Any]], rubric: Rubric | None
+) -> dict[str, Any]:
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    for event in events or []:
+        if not isinstance(event, dict):
+            continue
+        cid = event.get("criterion_id") or event.get("id") or event.get("criterion")
+        score = _as_float(event.get("score"))
+        if cid and score is not None:
+            values[str(cid)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)
+
+
+def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None) -> dict[str, Any]:
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    if isinstance(outcome, dict):
+        candidates = (
+            outcome.get("criteria") if isinstance(outcome.get("criteria"), dict) else outcome
+        )
+        if isinstance(candidates, dict):
+            for key, value in candidates.items():
+                score = _as_float(value)
+                if score is not None:
+                    values[str(key)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)

synth_ai/task/trace_correlation_helpers.py

@@ -7,8 +7,9 @@ This module provides utilities for task apps to:
 See monorepo/trace_creation_and_judgement.txt "Fatal Guards" section for requirements.
 """
 
+import importlib
 import logging
-from typing import Any
+from typing import Any, cast
 from urllib.parse import parse_qs, urlparse
 
 logger = logging.getLogger(__name__)
@@ -63,13 +64,25 @@ def extract_trace_correlation_id(
                 return stripped
     
     # Determine if we're in EVAL mode (trace_correlation_id not required for eval)
+    rollout_mode_cls: Any | None = None
     try:
-        from synth_ai.task.contracts import RolloutMode
-        is_eval_mode = (mode == "eval" or mode == RolloutMode.EVAL or 
-                        (hasattr(mode, 'value') and mode.value == "eval"))
-    except ImportError:
-        # If RolloutMode not available, fall back to string comparison
-        is_eval_mode = (mode == "eval")
+        contracts_module = importlib.import_module("synth_ai.task.contracts")
+        rollout_mode_cls = getattr(contracts_module, "RolloutMode", None)
+    except Exception:
+        rollout_mode_cls = None
+
+    is_eval_mode = False
+    if rollout_mode_cls is not None:
+        try:
+            is_eval_mode = (
+                mode == "eval"
+                or mode == rollout_mode_cls.EVAL
+                or getattr(mode, "value", None) == "eval"
+            )
+        except Exception:
+            is_eval_mode = mode == "eval"
+    else:
+        is_eval_mode = mode == "eval" or getattr(mode, "value", None) == "eval"
     
     # Fallback: try to extract from inference_url query params
     if not inference_url or not isinstance(inference_url, str):
@@ -87,10 +100,12 @@ def extract_trace_correlation_id(
     
     try:
         parsed = urlparse(inference_url)
-        query_params = parse_qs(parsed.query or "")
+        query_params = cast(dict[str, list[str]], parse_qs(parsed.query or ""))
         # Try multiple possible query param names
         for param_name in ["cid", "trace_correlation_id", "trace"]:
-            values = query_params.get(param_name, [])
+            values = query_params.get(param_name)
+            if not values:
+                continue
             for value in values:
                 if isinstance(value, str) and value.strip():
                     correlation_id = value.strip()
@@ -311,5 +326,3 @@ def verify_trace_correlation_id_in_response(
         expected_correlation_id
     )
     return True
-
-

synth_ai/task/tracing_utils.py

@@ -4,9 +4,12 @@ from __future__ import annotations
 
 import os
 from collections.abc import Callable
+from datetime import datetime
 from pathlib import Path
 from typing import Any
 
+from synth_ai.tracing_v3.constants import TRACE_DB_DIR, canonical_trace_db_name
+
 
 def tracing_env_enabled(default: bool = False) -> bool:
     """Return True when tracing is enabled for task apps via environment variable."""
@@ -40,9 +43,17 @@ def resolve_tracing_db_url() -> str | None:
             path.parent.mkdir(parents=True, exist_ok=True)
             return f"sqlite+aiosqlite:///{path}"
 
-    fallback_path = Path("traces/v3/synth_ai.db").expanduser()
-    fallback_path.parent.mkdir(parents=True, exist_ok=True)
-    return f"sqlite+aiosqlite:///{fallback_path}"
+    existing = os.getenv("TASKAPP_TRACE_DB_PATH")
+    if existing:
+        path = Path(existing).expanduser()
+    else:
+        base_dir = TRACE_DB_DIR.expanduser()
+        base_dir.mkdir(parents=True, exist_ok=True)
+        path = base_dir / canonical_trace_db_name(timestamp=datetime.now())
+        os.environ["TASKAPP_TRACE_DB_PATH"] = str(path)
+        os.environ.setdefault("SQLD_DB_PATH", str(path))
+    path.parent.mkdir(parents=True, exist_ok=True)
+    return f"sqlite+aiosqlite:///{path}"
 
 
 def build_tracer_factory(

synth_ai/task/validators.py

@@ -3,12 +3,11 @@
 from __future__ import annotations
 
 import re
-from typing import Any
+from typing import Any, cast
 from urllib.parse import urlparse, urlunparse
 
 import click
 import httpx
-
 from synth_ai.task.contracts import TaskAppEndpoints  # type: ignore[attr-defined]
 
 
@@ -152,7 +151,7 @@ def normalize_inference_url(url: str | None, *, default: str = "https://api.open
         new_path = f"{path}/v1/chat/completions" if path else "/v1/chat/completions"
     
     # Reconstruct URL with new path and original query/fragment
-    return urlunparse(parsed._replace(path=new_path))
+    return cast(str, urlunparse(parsed._replace(path=new_path)))
 
 
 def validate_task_app_url(url: str | None) -> str:

synth_ai/tracing_v3/abstractions.py

@@ -37,7 +37,7 @@ Concepts:
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass, field
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import Any
 
 from .lm_call_record_abstractions import LLMCallRecord
@@ -249,7 +249,7 @@ class SessionTimeStep:
 
     step_id: str = ""
     step_index: int = 0
-    timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
     turn_number: int | None = None
     events: list[BaseEvent] = field(default_factory=list)
     markov_blanket_messages: list[SessionEventMarkovBlanketMessage] = field(default_factory=list)
@@ -283,7 +283,7 @@ class SessionTrace:
     """
 
     session_id: str = ""
-    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    created_at: datetime = field(default_factory=lambda: datetime.now(UTC))
     session_time_steps: list[SessionTimeStep] = field(default_factory=list)
     event_history: list[BaseEvent] = field(default_factory=list)
     markov_blanket_message_history: list[SessionEventMarkovBlanketMessage] = field(

synth_ai/tracing_v3/config.py

@@ -3,27 +3,29 @@
 import os
 from dataclasses import dataclass
 
+from synth_ai.tracing_v3.constants import canonical_trace_db_path
+
+DEFAULT_DB_FILE = str(canonical_trace_db_path())
+
+
+def _default_sqlite_url() -> str:
+    base_path = os.path.abspath(os.getenv("SQLD_DB_PATH", DEFAULT_DB_FILE))
+    candidate = os.path.join(base_path, "dbs", "default", "data")
+    if os.path.isdir(base_path) and os.path.exists(candidate):
+        return f"sqlite+aiosqlite:///{candidate}"
+    return f"sqlite+aiosqlite:///{base_path}"
+
 
 @dataclass
 class TursoConfig:
     """Configuration for Turso/sqld connection."""
 
     # Default values matching serve.sh
-    DEFAULT_DB_FILE = "traces/v3/synth_ai.db"
+    DEFAULT_DB_FILE = DEFAULT_DB_FILE
     DEFAULT_HTTP_PORT = 8080
 
-    # Local embedded database for async SQLAlchemy
-    # Resolve to the actual SQLite file used by sqld if the base path is a directory
-    def _resolve_sqlite_db_url() -> str:  # type: ignore[no-redef]
-        base_path = os.path.abspath(os.getenv("SQLD_DB_PATH", "traces/v3/synth_ai.db"))
-        # If sqld is managing this DB, the real SQLite file lives under dbs/default/data
-        candidate = os.path.join(base_path, "dbs", "default", "data")
-        if os.path.isdir(base_path) and os.path.exists(candidate):
-            return f"sqlite+aiosqlite:///{candidate}"
-        return f"sqlite+aiosqlite:///{base_path}"
-
     # Use env override if provided; otherwise resolve based on SQLD layout
-    db_url: str = os.getenv("TURSO_LOCAL_DB_URL", _resolve_sqlite_db_url())
+    db_url: str = os.getenv("TURSO_LOCAL_DB_URL", _default_sqlite_url())
 
     # Remote database sync configuration
     sync_url: str = os.getenv("TURSO_DATABASE_URL", "")
@@ -48,7 +50,7 @@ class TursoConfig:
 
     # Daemon settings (for local sqld) - match serve.sh defaults
     sqld_binary: str = os.getenv("SQLD_BINARY", "sqld")
-    sqld_db_path: str = os.getenv("SQLD_DB_PATH", "traces/v3/synth_ai.db")
+    sqld_db_path: str = os.getenv("SQLD_DB_PATH", DEFAULT_DB_FILE)
     sqld_http_port: int = int(os.getenv("SQLD_HTTP_PORT", "8080"))
     sqld_idle_shutdown: int = int(os.getenv("SQLD_IDLE_SHUTDOWN", "0"))  # 0 = no idle shutdown
 

synth_ai/tracing_v3/constants.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+TRACE_DB_DIR = Path("traces")
+TRACE_DB_BASENAME = "task_app_traces"
+
+
+def canonical_trace_db_name(*, timestamp: datetime | None = None) -> str:
+    """Return the canonical trace database filename (with optional timestamp suffix)."""
+
+    if timestamp is None:
+        return f"{TRACE_DB_BASENAME}.db"
+    return f"{TRACE_DB_BASENAME}_{timestamp.strftime('%Y-%m-%d_%H-%M-%S')}.db"
+
+
+def canonical_trace_db_path(*, timestamp: datetime | None = None) -> Path:
+    """Return the canonical trace database path within the default trace directory."""
+
+    return TRACE_DB_DIR / canonical_trace_db_name(timestamp=timestamp)

synth_ai/tracing_v3/db_config.py

@@ -7,6 +7,8 @@ import os
 import shutil
 from typing import TYPE_CHECKING, Optional
 
+from synth_ai.tracing_v3.constants import canonical_trace_db_path
+
 if TYPE_CHECKING:
     from .turso.daemon import SqldDaemon
 
@@ -17,7 +19,7 @@ class DatabaseConfig:
     """Centralized database configuration management."""
 
     # Default values from serve.sh
-    DEFAULT_DB_FILE = "traces/v3/synth_ai.db"
+    DEFAULT_DB_FILE = str(canonical_trace_db_path())
     DEFAULT_HTTP_PORT = 8080
 
     def __init__(

synth_ai/tracing_v3/decorators.py

@@ -29,6 +29,7 @@ import contextvars
 import functools
 import time
 from collections.abc import Awaitable, Callable, Mapping
+from contextvars import Token
 from typing import Any, TypeVar, cast, overload
 
 from .abstractions import LMCAISEvent, TimeRecord
@@ -367,11 +368,11 @@ class SessionContext:
         ```
     """
 
-    def __init__(self, session_id: str, tracer=None):
+    def __init__(self, session_id: str, tracer: Any | None = None):
         self.session_id = session_id
         self.tracer = tracer
-        self._token = None
-        self._tracer_token = None
+        self._token: Token[str | None] | None = None
+        self._tracer_token: Token[Any] | None = None
 
     def __enter__(self):
         # Store tokens to restore previous context on exit
@@ -382,8 +383,9 @@ class SessionContext:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         # Restore previous context - this is crucial for proper isolation
-        _session_id_ctx.reset(self._token)
-        if self._tracer_token:
+        if self._token is not None:
+            _session_id_ctx.reset(self._token)
+        if self._tracer_token is not None:
             _session_tracer_ctx.reset(self._tracer_token)
 
     async def __aenter__(self):
@@ -393,6 +395,7 @@ class SessionContext:
         return self
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
-        _session_id_ctx.reset(self._token)
-        if self._tracer_token:
+        if self._token is not None:
+            _session_id_ctx.reset(self._token)
+        if self._tracer_token is not None:
             _session_tracer_ctx.reset(self._tracer_token)

synth_ai/tracing_v3/llm_call_record_helpers.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import uuid
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import Any, TypedDict, cast
 
 from .lm_call_record_abstractions import (
@@ -180,8 +180,8 @@ def create_llm_call_record_from_response(
         api_type=api_type,
         provider=provider,
         model_name=model_name,
-        started_at=started_at or datetime.now(timezone.utc),
-        completed_at=completed_at or datetime.now(timezone.utc),
+        started_at=started_at or datetime.now(UTC),
+        completed_at=completed_at or datetime.now(UTC),
         latency_ms=latency_ms,
         request_params=params,
         input_messages=input_messages,
@@ -376,8 +376,8 @@ def create_llm_call_record_from_streaming(
         api_type="responses",  # Streaming typically from Responses API
         provider=provider,
         model_name=model_name,
-        started_at=started_at or datetime.now(timezone.utc),
-        completed_at=completed_at or datetime.now(timezone.utc),
+        started_at=started_at or datetime.now(UTC),
+        completed_at=completed_at or datetime.now(UTC),
         latency_ms=latency_ms,
         request_params=params,
         input_messages=input_messages,

synth_ai/tracing_v3/session_tracer.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import asyncio
 import json
 from contextlib import asynccontextmanager
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import Any
 
 from .abstractions import (
@@ -106,7 +106,7 @@ class SessionTracer:
 
             self._current_trace = SessionTrace(
                 session_id=session_id,
-                created_at=datetime.now(timezone.utc),
+                created_at=datetime.now(UTC),
                 session_time_steps=[],
                 event_history=[],
                 markov_blanket_message_history=[],
@@ -152,7 +152,7 @@ class SessionTracer:
         step = SessionTimeStep(
             step_id=step_id,
             step_index=len(self._current_trace.session_time_steps),
-            timestamp=datetime.now(timezone.utc),
+            timestamp=datetime.now(UTC),
             turn_number=turn_number,
             step_metadata=metadata or {},
         )
@@ -197,7 +197,7 @@ class SessionTracer:
             step = self._current_step
 
         if step and step.completed_at is None:
-            step.completed_at = datetime.now(timezone.utc)
+            step.completed_at = datetime.now(UTC)
 
             # Trigger hooks
             await self.hooks.trigger(
@@ -294,7 +294,7 @@ class SessionTracer:
             content=normalised_content,
             message_type=message_type,
             time_record=TimeRecord(
-                event_time=event_time or datetime.now(timezone.utc).timestamp(), message_time=message_time
+                event_time=event_time or datetime.now(UTC).timestamp(), message_time=message_time
             ),
             metadata=metadata or {},
         )
@@ -368,7 +368,7 @@ class SessionTracer:
             # End any open timesteps
             for step in self._current_trace.session_time_steps:
                 if step.completed_at is None:
-                    step.completed_at = datetime.now(timezone.utc)
+                    step.completed_at = datetime.now(UTC)
 
             # Trigger pre-save hooks
             await self.hooks.trigger("before_save", session=self._current_trace)
@@ -384,7 +384,7 @@ class SessionTracer:
             if should_save and self.db:
                 _logger.info(f"[TRACE_DEBUG] Calling insert_session_trace with {len(self._current_trace.markov_blanket_message_history)} messages")
                 await self.db.insert_session_trace(self._current_trace)
-                _logger.info(f"[TRACE_DEBUG] insert_session_trace completed")
+                _logger.info("[TRACE_DEBUG] insert_session_trace completed")
 
                 # Trigger post-save hooks
                 await self.hooks.trigger("after_save", session=self._current_trace)