synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/cli/local/experiment_queue/trace_storage.py

@@ -0,0 +1,65 @@
+"""Helpers for persisting trials and trace metadata gleaned from job outputs."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+from sqlalchemy.orm import Session
+
+from .models import Experiment, ExperimentJob, Trial, TrialStatus
+from .results import ResultSummary
+
+
+def persist_trials_from_summary(
+    session: Session,
+    job: ExperimentJob,
+    summary: ResultSummary,
+) -> list[Trial]:
+    """Create Trial rows from learning curve checkpoints."""
+    trials: list[Trial] = []
+    if not summary.learning_curve_points:
+        return trials
+
+    now = datetime.now(UTC)
+    for idx, point in enumerate(summary.learning_curve_points, start=1):
+        metadata = {
+            "rollout_count": point.rollout_count,
+            "checkpoint_pct": point.checkpoint_pct,
+            **point.metadata,
+        }
+        rewards = {}
+        if point.performance is not None:
+            rewards["performance"] = point.performance
+
+        trial = Trial(
+            trial_id=str(uuid4()),
+            experiment_id=job.experiment_id,
+            job_id=job.job_id,
+            trial_number=idx,
+            system_name=metadata.get("system_name") or f"{job.job_type}_trial_{idx}",
+            reward=point.performance,
+            aggregate_score=point.performance,
+            rewards=rewards or None,
+            metadata=metadata,
+            status=TrialStatus.COMPLETED,
+            completed_at=now,
+        )
+        session.add(trial)
+        trials.append(trial)
+
+    return trials
+
+
+def update_experiment_metadata(
+    experiment: Experiment,
+    summary: ResultSummary,
+) -> None:
+    """Attach aggregate metrics from results to the experiment metadata."""
+    experiment.metadata_json.setdefault("aggregate", {})
+    aggregate = experiment.metadata_json["aggregate"]
+    aggregate["best_score"] = summary.best_score
+    aggregate["baseline_score"] = summary.baseline_score
+    aggregate["total_rollouts"] = summary.total_rollouts
+    aggregate["total_time"] = summary.total_time
+    aggregate["artifacts"] = summary.artifacts

synth_ai/cli/local/experiment_queue/validation.py

@@ -0,0 +1,157 @@
+"""Validation utilities and decorators for type safety and data integrity."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Callable, TypeVar
+
+T = TypeVar("T")
+
+
+def validate_not_none(value: Any, name: str) -> None:
+    """Assert that a value is not None.
+    
+    Args:
+        value: Value to check
+        name: Name of the value (for error messages)
+        
+    Raises:
+        AssertionError: If value is None
+    """
+    assert value is not None, f"{name} cannot be None"
+
+
+def validate_non_empty_string(value: str | None, name: str) -> str:
+    """Validate that a string is not None and not empty.
+    
+    Args:
+        value: String to validate
+        name: Name of the value (for error messages)
+        
+    Returns:
+        The validated string
+        
+    Raises:
+        AssertionError: If value is None or empty
+    """
+    assert value is not None, f"{name} cannot be None"
+    assert isinstance(value, str), f"{name} must be str, got {type(value).__name__}"
+    assert value.strip(), f"{name} cannot be empty"
+    return value
+
+
+def validate_positive_int(value: int | None, name: str, allow_none: bool = True) -> int | None:
+    """Validate that an integer is positive (if not None).
+    
+    Args:
+        value: Integer to validate
+        name: Name of the value (for error messages)
+        allow_none: Whether None is allowed
+        
+    Returns:
+        The validated integer or None
+        
+    Raises:
+        AssertionError: If value is invalid
+    """
+    if value is None:
+        assert allow_none, f"{name} cannot be None"
+        return None
+    assert isinstance(value, int), f"{name} must be int, got {type(value).__name__}: {value}"
+    assert value >= 0, f"{name} must be >= 0, got {value}"
+    return value
+
+
+def validate_range(
+    value: int | float | None,
+    name: str,
+    min_val: float | None = None,
+    max_val: float | None = None,
+    allow_none: bool = True,
+) -> int | float | None:
+    """Validate that a numeric value is within a range.
+    
+    Args:
+        value: Numeric value to validate
+        name: Name of the value (for error messages)
+        min_val: Minimum allowed value (inclusive)
+        max_val: Maximum allowed value (inclusive)
+        allow_none: Whether None is allowed
+        
+    Returns:
+        The validated value or None
+        
+    Raises:
+        AssertionError: If value is out of range
+    """
+    if value is None:
+        assert allow_none, f"{name} cannot be None"
+        return None
+    assert isinstance(value, int | float), (
+        f"{name} must be int | float, got {type(value).__name__}: {value}"
+    )
+    if min_val is not None:
+        assert value >= min_val, f"{name} must be >= {min_val}, got {value}"
+    if max_val is not None:
+        assert value <= max_val, f"{name} must be <= {max_val}, got {value}"
+    return value
+
+
+def validate_dict(value: dict[str, Any] | None, name: str, allow_none: bool = True) -> dict[str, Any]:
+    """Validate that a value is a dict (or None if allowed).
+    
+    Args:
+        value: Value to validate
+        name: Name of the value (for error messages)
+        allow_none: Whether None is allowed (returns empty dict if None)
+        
+    Returns:
+        The validated dict (or empty dict if None and allow_none=True)
+        
+    Raises:
+        AssertionError: If value is not a dict
+    """
+    if value is None:
+        assert allow_none, f"{name} cannot be None"
+        return {}
+    assert isinstance(value, dict), (
+        f"{name} must be dict, got {type(value).__name__}: {value}"
+    )
+    return value
+
+
+def validate_path(value: str | Path, name: str, must_exist: bool = False) -> Path:
+    """Validate that a path is valid and optionally exists.
+    
+    Args:
+        value: Path to validate
+        name: Name of the value (for error messages)
+        must_exist: Whether the path must exist
+        
+    Returns:
+        The validated Path object
+        
+    Raises:
+        AssertionError: If path is invalid or doesn't exist (if required)
+    """
+    from pathlib import Path
+    
+    assert value is not None, f"{name} cannot be None"
+    path = Path(value) if isinstance(value, str) else value
+    assert isinstance(path, Path), f"{name} must be str or Path, got {type(value).__name__}"
+    if must_exist:
+        assert path.exists(), f"{name} path does not exist: {path}"
+    return path
+
+
+def validated(func: Callable[..., T]) -> Callable[..., T]:
+    """Decorator to add validation to function arguments.
+    
+    This is a placeholder for future validation decorator implementation.
+    Currently just passes through the function unchanged.
+    """
+    return func
+
+
+
+

synth_ai/cli/local/session/__init__.py

@@ -0,0 +1,92 @@
+"""Agent session module initialization."""
+
+from __future__ import annotations
+
+from synth_ai.cli.local.session.client import AgentSessionClient
+from synth_ai.cli.local.session.constants import (
+    LIMIT_TYPE_HARD,
+    LIMIT_TYPE_ROLLING_WINDOW,
+    LIMIT_TYPE_SOFT,
+    METRIC_TYPE_API_CALLS,
+    METRIC_TYPE_COST_USD,
+    METRIC_TYPE_GPU_HOURS,
+    METRIC_TYPE_TOKENS,
+    SESSION_STATUS_ACTIVE,
+    SESSION_STATUS_ENDED,
+    SESSION_STATUS_EXPIRED,
+    SESSION_STATUS_LIMIT_EXCEEDED,
+    VALID_LIMIT_TYPES,
+    VALID_METRIC_TYPES,
+    VALID_SESSION_STATUSES,
+)
+from synth_ai.cli.local.session.exceptions import (
+    InvalidLimitError,
+    LimitExceededError,
+    SessionNotActiveError,
+    SessionNotFoundError,
+    SessionUsageError,
+)
+from synth_ai.cli.local.session.manager import AgentSessionManager
+from synth_ai.cli.local.session.models import (
+    AgentSession,
+    AgentSessionLimit,
+    AgentSessionUsage,
+    LimitCheckResult,
+    SessionUsageRecord,
+)
+from synth_ai.cli.local.session.query import AgentSessionQuery
+
+# Import submodules for re-export
+from . import (
+    client,
+    constants,
+    exceptions,
+    manager,
+    models,
+    query,
+)
+
+__all__ = [
+    # Submodules
+    "client",
+    "constants",
+    "exceptions",
+    "manager",
+    "models",
+    "query",
+    # Exceptions
+    "SessionUsageError",
+    "LimitExceededError",
+    "SessionNotFoundError",
+    "SessionNotActiveError",
+    "InvalidLimitError",
+    # Models
+    "AgentSession",
+    "AgentSessionLimit",
+    "AgentSessionUsage",
+    "LimitCheckResult",
+    "SessionUsageRecord",
+    # Client
+    "AgentSessionClient",
+    # Manager
+    "AgentSessionManager",
+    # Query
+    "AgentSessionQuery",
+    # Constants
+    "SESSION_STATUS_ACTIVE",
+    "SESSION_STATUS_ENDED",
+    "SESSION_STATUS_LIMIT_EXCEEDED",
+    "SESSION_STATUS_EXPIRED",
+    "LIMIT_TYPE_HARD",
+    "LIMIT_TYPE_SOFT",
+    "LIMIT_TYPE_ROLLING_WINDOW",
+    "METRIC_TYPE_TOKENS",
+    "METRIC_TYPE_COST_USD",
+    "METRIC_TYPE_GPU_HOURS",
+    "METRIC_TYPE_API_CALLS",
+    "VALID_SESSION_STATUSES",
+    "VALID_LIMIT_TYPES",
+    "VALID_METRIC_TYPES",
+]
+
+

synth_ai/cli/local/session/client.py

@@ -0,0 +1,383 @@
+"""AgentSession API client for SDK."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from decimal import Decimal
+from typing import Any, List, Optional
+from uuid import UUID
+
+from synth_ai.core.http import AsyncHttpClient, HTTPError
+
+from .exceptions import (
+    InvalidLimitError,
+    SessionNotActiveError,
+    SessionNotFoundError,
+)
+from .models import (
+    AgentSession,
+    AgentSessionLimit,
+    AgentSessionUsage,
+    LimitCheckResult,
+    SessionUsageRecord,
+)
+
+
+class AgentSessionClient:
+    """Client for managing agent sessions via API."""
+
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str,
+        timeout: float = 30.0,
+        http: Optional[AsyncHttpClient] = None,
+    ):
+        self._base_url = base_url.rstrip("/")
+        self._api_key = api_key
+        self._timeout = timeout
+        self._http = http or AsyncHttpClient(base_url, api_key, timeout)
+
+    def _parse_session(self, data: dict[str, Any]) -> AgentSession:
+        """Parse session from API response."""
+        usage_data = data.get("current_usage", {})
+        usage = AgentSessionUsage(
+            tokens=Decimal(str(usage_data.get("tokens", 0))),
+            cost_usd=Decimal(str(usage_data.get("cost_usd", 0))),
+            gpu_hours=Decimal(str(usage_data.get("gpu_hours", 0))),
+            api_calls=int(usage_data.get("api_calls", 0)),
+        )
+
+        limits = []
+        for limit_data in data.get("limits", []):
+            limits.append(
+                AgentSessionLimit(
+                    limit_id=UUID(limit_data["limit_id"]) if limit_data.get("limit_id") else None,
+                    limit_type=limit_data["limit_type"],
+                    metric_type=limit_data["metric_type"],
+                    limit_value=Decimal(str(limit_data["limit_value"])),
+                    current_usage=Decimal(str(limit_data.get("current_usage", 0))),
+                    warning_threshold=Decimal(str(limit_data["warning_threshold"]))
+                    if limit_data.get("warning_threshold")
+                    else None,
+                    window_seconds=limit_data.get("window_seconds"),
+                    exceeded_at=datetime.fromisoformat(limit_data["exceeded_at"].replace("Z", "+00:00"))
+                    if limit_data.get("exceeded_at")
+                    else None,
+                    exceeded_reason=limit_data.get("exceeded_reason"),
+                    created_at=datetime.fromisoformat(limit_data["created_at"].replace("Z", "+00:00"))
+                    if limit_data.get("created_at")
+                    else None,
+                    expires_at=datetime.fromisoformat(limit_data["expires_at"].replace("Z", "+00:00"))
+                    if limit_data.get("expires_at")
+                    else None,
+                )
+            )
+
+        return AgentSession(
+            session_id=data["session_id"],
+            org_id=UUID(data["org_id"]),
+            user_id=UUID(data["user_id"]) if data.get("user_id") else None,
+            api_key_id=UUID(data["api_key_id"]) if data.get("api_key_id") else None,
+            created_at=datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")),
+            expires_at=datetime.fromisoformat(data["expires_at"].replace("Z", "+00:00"))
+            if data.get("expires_at")
+            else None,
+            ended_at=datetime.fromisoformat(data["ended_at"].replace("Z", "+00:00"))
+            if data.get("ended_at")
+            else None,
+            status=data["status"],
+            limit_exceeded_reason=data.get("limit_exceeded_reason"),
+            usage=usage,
+            limits=limits,
+            tracing_session_id=data.get("tracing_session_id"),
+            session_type=data.get("session_type", "api_call"),
+            metadata=data.get("metadata", {}),
+        )
+
+    async def create(
+        self,
+        org_id: Optional[UUID] = None,
+        limits: Optional[List[dict[str, Any]]] = None,
+        tracing_session_id: Optional[str] = None,
+        session_type: Optional[str] = None,
+        expires_at: Optional[datetime] = None,
+        metadata: Optional[dict[str, Any]] = None,
+        session_id: Optional[str] = None,
+        make_active: bool = True,  # If True, ends existing active session and makes this one active
+    ) -> AgentSession:
+        """Create a new agent session.
+        
+        If org_id is not provided, it will be fetched from /api/v1/me endpoint.
+        """
+        # Get org_id from /me endpoint if not provided
+        if org_id is None:
+            try:
+                async with self._http:
+                    me_data = await self._http.get("/api/v1/me")
+                    org_id = UUID(me_data["org_id"])
+            except Exception as e:
+                raise ValueError(
+                    f"Failed to get org_id from API endpoint '/api/v1/me': {e}. "
+                    f"Please provide org_id explicitly when creating a session, "
+                    f"or ensure your API key is valid and has access to the /api/v1/me endpoint."
+                ) from e
+        
+        payload: dict[str, Any] = {}
+        if org_id:
+            payload["org_id"] = str(org_id)
+        if limits:
+            payload["limits"] = limits
+        if tracing_session_id:
+            payload["tracing_session_id"] = tracing_session_id
+        if session_type:
+            payload["session_type"] = session_type
+        if expires_at:
+            payload["expires_at"] = expires_at.isoformat()
+        if metadata:
+            payload["metadata"] = metadata
+        if session_id:
+            payload["session_id"] = session_id
+        if make_active is not None:
+            payload["make_active"] = make_active
+
+        try:
+            async with self._http:
+                data = await self._http.post_json("/api/v1/sessions", json=payload)
+                return self._parse_session(data)
+        except HTTPError as e:
+            if e.status == 400:
+                raise InvalidLimitError(str(e)) from e
+            raise
+
+    async def get(self, session_id: str) -> AgentSession:
+        """Get session by ID."""
+        try:
+            async with self._http:
+                data = await self._http.get(f"/api/v1/sessions/{session_id}")
+                return self._parse_session(data)
+        except HTTPError as e:
+            if e.status == 404:
+                raise SessionNotFoundError(session_id) from e
+            raise
+
+    async def end(self, session_id: str) -> AgentSession:
+        """End a session."""
+        try:
+            async with self._http:
+                data = await self._http.post_json(f"/api/v1/sessions/{session_id}/end", json={})
+                return self._parse_session(data)
+        except HTTPError as e:
+            if e.status == 404:
+                raise SessionNotFoundError(session_id) from e
+            if e.status == 400:
+                raise SessionNotActiveError(session_id, "unknown") from e
+            raise
+
+    async def check_limit(
+        self,
+        session_id: str,
+        metric_type: str,
+        requested_value: Decimal,
+    ) -> LimitCheckResult:
+        """Check if usage would exceed limits."""
+        # Note: This endpoint doesn't exist yet, but could be added
+        # For now, we'll get the session and check limits client-side
+        session = await self.get(session_id)
+        if session.status != "active":
+            return LimitCheckResult(
+                allowed=False,
+                limit_value=Decimal("0"),
+                current_usage=Decimal("0"),
+                remaining=Decimal("0"),
+                reason=f"Session not active (status: {session.status})",
+            )
+
+        # Find relevant limits
+        for limit in session.limits:
+            if limit.metric_type == metric_type:
+                projected = limit.current_usage + requested_value
+                if projected > limit.limit_value:
+                    return LimitCheckResult(
+                        allowed=False,
+                        limit_value=limit.limit_value,
+                        current_usage=limit.current_usage,
+                        remaining=limit.remaining,
+                        reason=f"Limit exceeded: {projected} > {limit.limit_value}",
+                    )
+
+        return LimitCheckResult(
+            allowed=True,
+            limit_value=Decimal("0"),
+            current_usage=Decimal("0"),
+            remaining=Decimal("999999999"),
+            reason=None,
+        )
+
+    async def increase_limit(
+        self,
+        session_id: str,
+        metric_type: str,
+        increase_by: Decimal,
+    ) -> AgentSessionLimit:
+        """Increase a limit by amount."""
+        try:
+            async with self._http:
+                data = await self._http.post_json(
+                    f"/api/v1/sessions/{session_id}/limits/increase",
+                    json={"metric_type": metric_type, "increase_by": float(increase_by)},
+                )
+                return AgentSessionLimit(
+                    limit_id=UUID(data["limit_id"]) if data.get("limit_id") else None,
+                    limit_type=data["limit_type"],
+                    metric_type=data["metric_type"],
+                    limit_value=Decimal(str(data["limit_value"])),
+                    current_usage=Decimal(str(data.get("current_usage", 0))),
+                    warning_threshold=Decimal(str(data["warning_threshold"]))
+                    if data.get("warning_threshold")
+                    else None,
+                    window_seconds=data.get("window_seconds"),
+                    exceeded_at=datetime.fromisoformat(data["exceeded_at"].replace("Z", "+00:00"))
+                    if data.get("exceeded_at")
+                    else None,
+                    exceeded_reason=data.get("exceeded_reason"),
+                    created_at=datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
+                    if data.get("created_at")
+                    else None,
+                    expires_at=datetime.fromisoformat(data["expires_at"].replace("Z", "+00:00"))
+                    if data.get("expires_at")
+                    else None,
+                )
+        except HTTPError as e:
+            if e.status == 404:
+                raise SessionNotFoundError(session_id) from e
+            if e.status == 400:
+                raise SessionNotActiveError(session_id, "unknown") from e
+            raise
+
+    async def add_limit(
+        self,
+        session_id: str,
+        limit: dict[str, Any],
+    ) -> AgentSessionLimit:
+        """Add a new limit to session."""
+        try:
+            async with self._http:
+                data = await self._http.post_json(
+                    f"/api/v1/sessions/{session_id}/limits",
+                    json=limit,
+                )
+                return AgentSessionLimit(
+                    limit_id=UUID(data["limit_id"]) if data.get("limit_id") else None,
+                    limit_type=data["limit_type"],
+                    metric_type=data["metric_type"],
+                    limit_value=Decimal(str(data["limit_value"])),
+                    current_usage=Decimal(str(data.get("current_usage", 0))),
+                    warning_threshold=Decimal(str(data["warning_threshold"]))
+                    if data.get("warning_threshold")
+                    else None,
+                    window_seconds=data.get("window_seconds"),
+                    exceeded_at=datetime.fromisoformat(data["exceeded_at"].replace("Z", "+00:00"))
+                    if data.get("exceeded_at")
+                    else None,
+                    exceeded_reason=data.get("exceeded_reason"),
+                    created_at=datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
+                    if data.get("created_at")
+                    else None,
+                    expires_at=datetime.fromisoformat(data["expires_at"].replace("Z", "+00:00"))
+                    if data.get("expires_at")
+                    else None,
+                )
+        except HTTPError as e:
+            if e.status == 404:
+                raise SessionNotFoundError(session_id) from e
+            if e.status == 400:
+                raise InvalidLimitError(str(e)) from e
+            raise
+
+    async def list_limits(self, session_id: str) -> List[AgentSessionLimit]:
+        """Get all limits for a session."""
+        try:
+            async with self._http:
+                data = await self._http.get(f"/api/v1/sessions/{session_id}/limits")
+                return [
+                    AgentSessionLimit(
+                        limit_id=UUID(limit_data["limit_id"]) if limit_data.get("limit_id") else None,
+                        limit_type=limit_data["limit_type"],
+                        metric_type=limit_data["metric_type"],
+                        limit_value=Decimal(str(limit_data["limit_value"])),
+                        current_usage=Decimal(str(limit_data.get("current_usage", 0))),
+                        warning_threshold=Decimal(str(limit_data["warning_threshold"]))
+                        if limit_data.get("warning_threshold")
+                        else None,
+                        window_seconds=limit_data.get("window_seconds"),
+                        exceeded_at=datetime.fromisoformat(limit_data["exceeded_at"].replace("Z", "+00:00"))
+                        if limit_data.get("exceeded_at")
+                        else None,
+                        exceeded_reason=limit_data.get("exceeded_reason"),
+                        created_at=datetime.fromisoformat(limit_data["created_at"].replace("Z", "+00:00"))
+                        if limit_data.get("created_at")
+                        else None,
+                        expires_at=datetime.fromisoformat(limit_data["expires_at"].replace("Z", "+00:00"))
+                        if limit_data.get("expires_at")
+                        else None,
+                    )
+                    for limit_data in data
+                ]
+        except HTTPError as e:
+            if e.status == 404:
+                raise SessionNotFoundError(session_id) from e
+            raise
+
+    async def list(
+        self,
+        status: Optional[str] = None,
+        limit: int = 100,
+        offset: int = 0,
+    ) -> List[AgentSession]:
+        """List sessions for the authenticated organization."""
+        params: dict[str, Any] = {"limit": limit, "offset": offset}
+        if status:
+            params["status"] = status
+
+        try:
+            async with self._http:
+                data = await self._http.get("/api/v1/sessions", params=params)
+                return [self._parse_session(session_data) for session_data in data]
+        except HTTPError:
+            raise
+
+    async def get_usage(
+        self,
+        session_id: str,
+        metric_type: Optional[str] = None,
+        limit: int = 100,
+        offset: int = 0,
+    ) -> List[SessionUsageRecord]:
+        """Get usage records for a session."""
+        params: dict[str, Any] = {"limit": limit, "offset": offset}
+        if metric_type:
+            params["metric_type"] = metric_type
+
+        try:
+            async with self._http:
+                data = await self._http.get(f"/api/v1/sessions/{session_id}/usage", params=params)
+                return [
+                    SessionUsageRecord(
+                        id=UUID(record_data["id"]),
+                        session_id=record_data["session_id"],
+                        metric_type=record_data["metric_type"],
+                        metric_value=Decimal(str(record_data["metric_value"])),
+                        reference_type=record_data.get("reference_type"),
+                        reference_id=record_data.get("reference_id"),
+                        org_id=UUID(record_data["org_id"]),
+                        user_id=UUID(record_data["user_id"]) if record_data.get("user_id") else None,
+                        metadata=record_data.get("metadata", {}),
+                        created_at=datetime.fromisoformat(record_data["created_at"].replace("Z", "+00:00")),
+                    )
+                    for record_data in data
+                ]
+        except HTTPError as e:
+            if e.status == 404:
+                raise SessionNotFoundError(session_id) from e
+            raise