synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/environments/service/external_registry.py

@@ -1,56 +0,0 @@
-"""
-External environment registry support.
-
-This module provides functionality to register environments from external packages.
-"""
-
-import importlib
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class ExternalRegistryConfig:
-    """Configuration for external environment registries."""
-
-    def __init__(self, external_environments: list[dict[str, str]] = None):
-        self.external_environments = external_environments or []
-
-
-def load_external_environments(config: ExternalRegistryConfig):
-    """
-    Load and register environments from external packages.
-
-    Args:
-        config: Configuration specifying external environment sources
-    """
-    for env_config in config.external_environments:
-        module_name = env_config.get("module")
-        function_name = env_config.get("function", "integrate_with_environments_service")
-
-        if not module_name:
-            logger.warning("External environment config missing 'module' field")
-            continue
-
-        try:
-            # Import the module
-            module = importlib.import_module(module_name)
-
-            # Get the registration function
-            if hasattr(module, function_name):
-                register_func = getattr(module, function_name)
-                register_func()
-                logger.info(f"Successfully loaded environments from {module_name}")
-            else:
-                logger.warning(f"Module {module_name} does not have function {function_name}")
-
-        except ImportError as e:
-            logger.error(f"Failed to import module {module_name}: {e}")
-        except Exception as e:
-            logger.error(f"Error loading environments from {module_name}: {e}")
-
-
-__all__ = [
-    "ExternalRegistryConfig",
-    "load_external_environments",
-]

synth_ai/environments/service/registry.py

@@ -1,9 +0,0 @@
-# This file re-exports the actual registry functions from synth_ai.environments.environment.registry
-# to be used by the service layer, maintaining a clean separation if needed.
-from synth_ai.environments.environment.registry import (
-    get_environment_cls,
-    list_supported_env_types,
-    register_environment,
-)
-
-__all__ = ["register_environment", "get_environment_cls", "list_supported_env_types"]

synth_ai/environments/stateful/__init__.py

@@ -1 +0,0 @@
-"""Stateful environment components."""

synth_ai/environments/stateful/core.py

@@ -1,163 +0,0 @@
-from abc import abstractmethod
-
-from synth_ai.environments.environment.shared_engine import Engine, InternalObservation
-from synth_ai.environments.environment.tools import EnvToolCall
-
-
-class StatefulEnvironment(Engine):
-    """
-    Abstract base class for stateful environments in the Synth AI framework.
-
-    This class defines the interface for environments that maintain state between
-    interactions and support agent-environment interactions through tool calls.
-    StatefulEnvironments are designed to work with AI agents that can observe
-    the environment, take actions through tool calls, and receive feedback.
-
-    The environment follows a standard lifecycle:
-    1. Initialize - Set up initial state and return first observation
-    2. Step - Process agent tool calls and return new observations
-    3. Checkpoint - Save current state for potential restoration
-    4. Terminate - Clean up and finalize the environment
-
-    All methods are async to support non-blocking operations and integration
-    with modern async/await patterns in AI applications.
-
-    Example:
-        >>> class MyGameEnv(StatefulEnvironment):
-        ...     async def initialize(self):
-        ...         # Set up game state
-        ...         return initial_observation
-        ...
-        ...     async def step(self, tool_calls):
-        ...         # Process player actions
-        ...         return new_observation
-        >>>
-        >>> env = MyGameEnv(task)
-        >>> obs = await env.initialize()
-        >>> result = await env.step([tool_call])
-    """
-
-    @abstractmethod
-    async def initialize(self) -> InternalObservation:
-        """
-        Initialize the environment and return the initial observation.
-
-        This method sets up the environment's initial state, loads any
-        necessary resources, and prepares the environment for agent interaction.
-        It should be called once before any step() calls.
-
-        Returns:
-            InternalObservation: The initial state observation that the agent
-                will use to understand the environment and plan its first action.
-
-        Raises:
-            EnvironmentError: If initialization fails due to invalid configuration
-                or resource unavailability.
-
-        Example:
-            >>> env = MyEnvironment(task)
-            >>> initial_obs = await env.initialize()
-            >>> print(initial_obs.observation)  # Agent-visible state
-        """
-        pass
-
-    @abstractmethod
-    async def terminate(self) -> InternalObservation:
-        """
-        Terminate the environment and return the final observation.
-
-        This method performs cleanup operations, saves any persistent state,
-        and prepares the final observation that summarizes the environment's
-        end state. It should be called when the episode or session is complete.
-
-        Returns:
-            InternalObservation: The final state observation, typically including
-                summary information, final scores, or completion status.
-
-        Example:
-            >>> final_obs = await env.terminate()
-            >>> print(final_obs.observation.get('final_score'))
-        """
-        pass
-
-    # main external api
-    @abstractmethod
-    def validate_tool_calls(self, tool_calls: EnvToolCall):
-        """
-        Validate that tool calls are properly formatted and executable.
-
-        This method checks tool calls before execution to ensure they:
-        - Reference valid tools available in this environment
-        - Provide required arguments with correct types
-        - Follow any environment-specific constraints or rules
-
-        Args:
-            tool_calls: The tool call(s) to validate. Can be a single call
-                or a list of calls depending on the environment's capabilities.
-
-        Raises:
-            ValidationError: If tool calls are invalid, malformed, or not
-                supported by this environment.
-            TypeError: If tool_calls is not the expected type.
-
-        Example:
-            >>> tool_call = EnvToolCall(tool="move", args={"direction": "north"})
-            >>> env.validate_tool_calls(tool_call)  # Raises if invalid
-        """
-        pass
-
-    @abstractmethod
-    async def step(self, tool_calls: list[EnvToolCall]) -> InternalObservation:
-        """
-        Execute tool calls and return the resulting observation.
-
-        This is the main interaction method where agents submit actions
-        (as tool calls) and receive feedback from the environment. The method:
-        1. Validates the tool calls (may call validate_tool_calls)
-        2. Executes the actions in the environment
-        3. Updates the environment state
-        4. Returns the new observation for the agent
-
-        Args:
-            tool_calls: List of tool calls representing the agent's actions.
-                Each tool call specifies a tool name and arguments.
-
-        Returns:
-            InternalObservation: The new state observation after executing
-                the tool calls, including any changes, rewards, or feedback.
-
-        Raises:
-            ValidationError: If tool calls are invalid or cannot be executed.
-            EnvironmentError: If execution fails due to environment state issues.
-
-        Example:
-            >>> tool_calls = [
-            ...     EnvToolCall(tool="move", args={"direction": "north"}),
-            ...     EnvToolCall(tool="pick_up", args={"item": "key"})
-            ... ]
-            >>> obs = await env.step(tool_calls)
-            >>> print(obs.observation.get('player_location'))
-        """
-        pass
-
-    @abstractmethod
-    async def checkpoint(self) -> InternalObservation:
-        """
-        Create a checkpoint of the current environment state.
-
-        This method saves the current state of the environment for potential
-        restoration later. It's useful for:
-        - Implementing save/load functionality
-        - Creating branching scenarios for exploration
-        - Debugging and development
-        - Rollback mechanisms for error recovery
-
-        Returns:
-            InternalObservation: Current state observation, potentially including
-                checkpoint metadata or state identifiers.
-
-        Example:
-            >>> checkpoint_obs = await env.checkpoint()
-            >>> checkpoint_id = checkpoint_obs.metadata.get('checkpoint_id')
-        """
-        pass

synth_ai/environments/stateful/engine.py

@@ -1,21 +0,0 @@
-from typing import TypeVar
-
-from synth_ai.environments.environment.shared_engine import Engine
-
-SnapshotType = TypeVar("SnapshotType", bound="StatefulEngineSnapshot")
-
-
-class StatefulEngineSnapshot:
-    pass
-
-
-class StatefulEngine(Engine):
-    async def serialize(self):
-        pass
-
-    @classmethod
-    async def deserialize(cls, engine_snapshot: StatefulEngineSnapshot):
-        pass
-
-    async def _step_engine(self):
-        pass

synth_ai/environments/stateful/state.py

@@ -1,7 +0,0 @@
-from synth_ai.environments.v0_observability.log import EnvironmentStepRecord
-
-
-class State:
-    async def get_environment_step_record(self) -> EnvironmentStepRecord:
-        """Return the latest environment step record."""
-        raise NotImplementedError

synth_ai/environments/tasks/api.py

@@ -1,19 +0,0 @@
-from synth_ai.environments.tasks.core import (
-    Impetus,
-    Intent,
-    SplitInfo,
-    Task,
-    TaskInstance,
-    TaskInstanceMetadata,
-    TaskInstanceSet,
-)
-
-__all__ = [
-    "Task",
-    "TaskInstance",
-    "TaskInstanceSet",
-    "TaskInstanceMetadata",
-    "SplitInfo",
-    "Impetus",
-    "Intent",
-]

synth_ai/environments/tasks/core.py

@@ -1,81 +0,0 @@
-from abc import abstractmethod
-from collections.abc import Callable
-from dataclasses import dataclass, field
-from typing import Any, Optional
-from uuid import UUID
-
-from synth_ai.environments.stateful.engine import StatefulEngineSnapshot
-from synth_ai.environments.v0_observability.history import SynthGlobalTrajectory
-
-
-@dataclass
-class Task:
-    global_premises: str
-    global_constraints: str
-    global_objectives: str
-
-    shared_env_params: dict | None
-
-
-@dataclass
-class TaskInstanceMetadata:
-    pass
-
-
-@dataclass
-class Intent:
-    rubric: dict[str, Any]
-    gold_trajectories: SynthGlobalTrajectory | None
-    gold_state_diff: dict
-    deterministic_eval_functions: list[Callable] = field(default_factory=list)
-
-
-@dataclass
-class Impetus:
-    instructions: str
-
-    # ?
-
-
-@dataclass
-class TaskInstance:
-    id: UUID
-    impetus: Impetus
-    intent: Intent
-    metadata: TaskInstanceMetadata
-    is_reproducible: bool
-    initial_engine_snapshot: Optional["StatefulEngineSnapshot"]
-
-    @abstractmethod
-    async def serialize(self) -> dict:
-        pass
-
-    @abstractmethod
-    async def deserialize(self) -> "TaskInstance":
-        pass
-
-
-@dataclass
-class TaskInstanceMetadataFilter:
-    @abstractmethod
-    def __call__(
-        self, instance: TaskInstance
-    ) -> bool:  # Use Any temporarily for broader compatibility
-        # Using Any avoids strict dependency on AgentStatefulTaskInstance here
-        # Subclasses like MetadataFilter in helpers.py can specify the type.
-        """Return True if the instance passes the filter."""
-
-
-@dataclass
-class SplitInfo:
-    val_instance_ids: set[str]
-    test_instance_ids: set[str]
-    _is_split_defined: bool
-
-
-@dataclass
-class TaskInstanceSet:
-    name: str
-    description: str
-    instances: list[TaskInstance]
-    split_info: SplitInfo

synth_ai/environments/tasks/filters.py

@@ -1,40 +0,0 @@
-from collections.abc import Collection
-from dataclasses import dataclass
-from typing import Any
-
-from synth_ai.environments.tasks.core import TaskInstance, TaskInstanceMetadataFilter
-
-
-@dataclass
-class ValueFilter(TaskInstanceMetadataFilter):
-    key: str
-    values: Collection[Any]
-
-    def __call__(self, instance: TaskInstance) -> bool:
-        instance_value = getattr(instance.metadata, self.key, None)
-        if instance_value is None:
-            return False
-        return instance_value in self.values
-
-
-@dataclass
-class RangeFilter(TaskInstanceMetadataFilter):
-    key: str
-    min_val: float | None = None
-    max_val: float | None = None
-
-    def __call__(self, instance: TaskInstance) -> bool:
-        instance_value = getattr(instance.metadata, self.key, None)
-        if instance_value is None:
-            # If the attribute doesn't exist on the metadata, it can't be in range.
-            return False
-
-        if not isinstance(instance_value, int | float):
-            # If the attribute is not a number, it can't be in a numerical range.
-            # Or, we could raise an error, depending on desired strictness.
-            return False
-
-        return not (
-            (self.min_val is not None and instance_value < self.min_val)
-            or (self.max_val is not None and instance_value > self.max_val)
-        )

synth_ai/environments/tasks/utils.py

@@ -1,90 +0,0 @@
-"""
-Utility functions and generic filters for taskset creation.
-"""
-
-from collections.abc import Collection
-from typing import Any
-from uuid import UUID, uuid4
-
-from synth_ai.environments.tasks.core import (
-    SplitInfo,
-    TaskInstance,
-    TaskInstanceMetadataFilter,
-    TaskInstanceSet,
-)
-
-
-def parse_or_new_uuid(raw_id: str | None) -> UUID:
-    """
-    Parse a raw ID string into a UUID, or generate a new one if invalid or missing.
-    """
-    try:
-        return UUID(raw_id)  # type: ignore[arg-type]
-    except Exception:
-        return uuid4()
-
-
-class ValueFilter(TaskInstanceMetadataFilter):
-    """
-    Filter TaskInstances by exact match of a metadata attribute.
-    """
-
-    def __init__(self, key: str, values: Collection[Any]):
-        self.key = key
-        self.values = set(values)
-
-    def __call__(self, instance: TaskInstance) -> bool:
-        return getattr(instance.metadata, self.key, None) in self.values
-
-
-class RangeFilter(TaskInstanceMetadataFilter):
-    """
-    Filter TaskInstances where a numeric metadata attribute falls within [min_value, max_value].
-    """
-
-    def __init__(
-        self,
-        key: str,
-        min_value: float | None = None,
-        max_value: float | None = None,
-    ):
-        self.key = key
-        self.min_value = min_value
-        self.max_value = max_value
-
-    def __call__(self, instance: TaskInstance) -> bool:
-        value = getattr(instance.metadata, self.key, None)
-        return not (
-            (self.min_value is not None and (value is None or value < self.min_value))
-            or (self.max_value is not None and (value is None or value > self.max_value))
-        )
-
-
-def make_taskset(
-    name: str,
-    description: str,
-    instances: list[TaskInstance],
-    val_filter: TaskInstanceMetadataFilter | None = None,
-    test_filter: TaskInstanceMetadataFilter | None = None,
-) -> TaskInstanceSet:
-    """
-    Assemble a TaskInstanceSet by applying optional validation and test filters.
-    """
-    val_ids: set[Any] = set()
-    test_ids: set[Any] = set()
-    if val_filter:
-        val_ids = {inst.id for inst in instances if val_filter(inst)}
-    if test_filter:
-        test_ids = {inst.id for inst in instances if test_filter(inst)}
-    is_defined = val_filter is not None or test_filter is not None
-    split_info = SplitInfo(
-        val_instance_ids=val_ids,
-        test_instance_ids=test_ids,
-        _is_split_defined=is_defined,
-    )
-    return TaskInstanceSet(
-        name=name,
-        description=description,
-        instances=instances,
-        split_info=split_info,
-    )

synth_ai/environments/v0_observability/history.py

@@ -1,3 +0,0 @@
-# Env and Graph
-class SynthGlobalTrajectory:
-    pass

synth_ai/environments/v0_observability/log.py

@@ -1,2 +0,0 @@
-class EnvironmentStepRecord:
-    pass

synth_ai/evals/__init__.py

@@ -1,15 +0,0 @@
-from .client import JudgeClient, JudgeOptions, JudgeScoreResponse
-from .types import Judgement, RewardJudgement, RewardMetadata, Track, TrackAggregate
-
-__all__ = [
-	"JudgeClient",
-	"JudgeOptions",
-	"JudgeScoreResponse",
-	"Judgement",
-	"RewardJudgement",
-	"RewardMetadata",
-	"Track",
-	"TrackAggregate",
-]
-
-

synth_ai/evals/base.py

@@ -1,13 +0,0 @@
-class Judgement:
-    def __init__(
-        self, criteria: str, score: float, reasoning: str = "", evidence: list[str] = None
-    ):
-        self.criteria = criteria
-        self.score = score
-        self.reasoning = reasoning
-        self.evidence = evidence or []
-
-
-class BaseEval:
-    async def run(self, data: any) -> list[Judgement]:
-        pass

synth_ai/evals/client.py

@@ -1,82 +0,0 @@
-"""Experimental Judge API client.
-
-This surface is experimental and subject to change without notice.
-Set environment variable `SYNTH_SILENCE_EXPERIMENTAL=1` to silence warnings.
-"""
-
-from __future__ import annotations
-
-import os
-import warnings
-from typing import Any, Literal, TypedDict
-
-from synth_ai.http import AsyncHttpClient, HTTPError
-from synth_ai.tracing_v3.serialization import normalize_for_json
-
-Provider = Literal["groq", "gemini"]
-
-
-class JudgeOptions(TypedDict, total=False):
-    event: bool
-    outcome: bool
-    rubric_id: str
-    rubric_overrides: dict[str, Any]
-    provider: Provider
-    model: str
-    max_concurrency: int
-
-
-class JudgeScoreResponse(TypedDict, total=False):
-    status: str
-    event_rewards: list[dict[str, Any]]
-    outcome_reward: dict[str, Any]
-    details: dict[str, Any]
-
-
-class JudgeClient:
-    def __init__(self, base_url: str, api_key: str, *, timeout: float = 60.0) -> None:
-        _silence = (os.getenv("SYNTH_SILENCE_EXPERIMENTAL") or "").strip().lower()
-        if _silence not in {"1", "true", "t", "yes", "y", "on"}:
-            warnings.warn(
-                "Experimental API: synth_ai.evals.JudgeClient is experimental and may change without notice.",
-                UserWarning,
-                stacklevel=2,
-            )
-        self._base = base_url.rstrip("/")
-        self._key = api_key
-        self._timeout = timeout
-
-    async def score(
-        self,
-        *,
-        trace: dict[str, Any] | Any,
-        policy_name: str,
-        task_app_id: str,
-        options: JudgeOptions,
-        task_app_base_url: str | None = None,
-    ) -> JudgeScoreResponse:
-        body = {
-            "policy_name": policy_name,
-            "task_app": {"id": task_app_id, **({"base_url": task_app_base_url} if task_app_base_url else {})},
-            "trace": normalize_for_json(trace),
-            "options": options or {},
-        }
-        try:
-            async with AsyncHttpClient(self._base, self._key, timeout=self._timeout) as http:
-                js = await http.post_json("/api/judge/v1/score", json=body)
-                if not isinstance(js, dict):
-                    raise ValueError("invalid_judge_response_shape")
-                return js  # type: ignore[return-value]
-        except HTTPError as err:  # map to friendlier exceptions
-            status = int(getattr(err, "status", 0) or 0)
-            if status in (400, 422):
-                raise ValueError(f"judge_validation_error: {err.detail}") from err
-            if status in (401, 403):
-                raise PermissionError(f"judge_auth_error: {err.detail}") from err
-            if status == 404:
-                raise FileNotFoundError(f"judge_route_not_found: {err.detail}") from err
-            if status == 429:
-                raise Exception("judge_rate_limited") from err  # replace with RetryLater in future
-            if status >= 500:
-                raise Exception("judge_transient_error") from err  # replace with TransientError in future
-            raise

synth_ai/evals/types.py

@@ -1,42 +0,0 @@
-from __future__ import annotations
-
-from typing import Literal, TypedDict
-
-Track = Literal["process", "reasoning", "progress", "outcome"]
-
-
-class Judgement(TypedDict, total=False):
-	key: str
-	title: str
-	description: str
-	score: float
-	reason: str
-	confidence: float
-	scale: Literal["binary", "bounded", "count", "custom"]
-	source: dict
-
-
-class RewardJudgement(TypedDict, total=False):
-	judgement: Judgement
-	scope: Literal["step", "event", "outcome"]
-	turn: int | None
-	episode_id: str | None
-	reward_value: float | None
-	links: dict
-
-
-class TrackAggregate(TypedDict, total=False):
-	mean: float
-	median: float
-	std: float
-	n: int
-
-
-class RewardMetadata(TypedDict, total=False):
-	per_window: list[RewardJudgement]
-	aggregates: dict[Track, TrackAggregate]
-	overall: dict[str, float]  # {"final_outcome_score": float}
-	rubric: dict               # {"ids": {...}, "hash": "..."}
-	model_info: dict           # {"model": "...", ...}
-
-