synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/sdk/judging/client.py

@@ -1,191 +0,0 @@
-"""Experimental Judge API client.
-
-This surface is experimental and subject to change without notice.
-Set environment variable `SYNTH_SILENCE_EXPERIMENTAL=1` to silence warnings.
-"""
-
-from __future__ import annotations
-
-import os
-import warnings
-from typing import Any, Literal, TypedDict
-
-from synth_ai.core.http import AsyncHttpClient, HTTPError
-from synth_ai.core.tracing_v3.serialization import normalize_for_json
-from synth_ai.sdk.graphs import VerifierClient as GraphVerifierClient
-
-Provider = Literal["groq", "gemini"]
-
-
-class JudgeOptions(TypedDict, total=False):
-    event: bool
-    outcome: bool
-    rubric_id: str
-    rubric_overrides: dict[str, Any]
-    provider: Provider
-    model: str
-    max_concurrency: int
-    verifier_type: str
-
-
-class JudgeScoreResponse(TypedDict, total=False):
-    status: str
-    event_rewards: list[dict[str, Any]]
-    outcome_reward: dict[str, Any]
-    details: dict[str, Any]
-
-
-class JudgeClient:
-    """Legacy client for LLM-based evaluation of task app traces.
-    
-    This client provides programmatic access to Synth AI's judge API, which uses
-    LLMs to evaluate task execution traces and generate rewards. The judge can
-    evaluate both event-level (step-by-step) and outcome-level (episode-level) rewards.
-    
-    .. warning::
-        This API is experimental and subject to change without notice.
-        Set `SYNTH_SILENCE_EXPERIMENTAL=1` to silence warnings.
-    
-    Example:
-        >>> from synth_ai.sdk.judging import JudgeClient, JudgeOptions
-        >>> 
-        >>> client = JudgeClient(
-        ...     base_url="https://api.usesynth.ai",
-        ...     api_key=os.environ["SYNTH_API_KEY"],
-        ... )
-        >>> 
-        >>> # Score a trace with outcome reward
-        >>> result = await client.score(
-        ...     trace=my_trace_dict,
-        ...     policy_name="my_policy",
-        ...     task_app_id="heartdisease",
-        ...     options=JudgeOptions(
-        ...         outcome=True,
-        ...         rubric_id="accuracy",
-        ...         provider="groq",
-        ...         model="llama-3.1-8b-instant",
-        ...     ),
-        ... )
-        >>> 
-        >>> print(f"Outcome reward: {result['outcome_reward']}")
-    """
-    
-    def __init__(self, base_url: str, api_key: str, *, timeout: float = 60.0) -> None:
-        """Initialize the judge client.
-        
-        Args:
-            base_url: Base URL for the Synth AI API
-            api_key: API key for authentication
-            timeout: Request timeout in seconds (default: 60.0)
-        """
-        _silence = (os.getenv("SYNTH_SILENCE_EXPERIMENTAL") or "").strip().lower()
-        if _silence not in {"1", "true", "t", "yes", "y", "on"}:
-            warnings.warn(
-                "Legacy API: synth_ai.sdk.judging.JudgeClient is legacy. "
-                "Use synth_ai.sdk.graphs.VerifierClient or GraphCompletionsClient instead.",
-                UserWarning,
-                stacklevel=2,
-            )
-        self._base = base_url.rstrip("/")
-        self._key = api_key
-        self._timeout = timeout
-
-    async def score(
-        self,
-        *,
-        trace: dict[str, Any] | Any,
-        policy_name: str,
-        task_app_id: str,
-        options: JudgeOptions,
-        rubric: dict[str, Any] | None = None,
-        verifier_type: str | None = None,
-        task_app_base_url: str | None = None,
-    ) -> JudgeScoreResponse:
-        """Score a task execution trace using LLM-based evaluation.
-        
-        This method sends a trace to the judge API, which evaluates it according
-        to the provided rubric and returns event-level and/or outcome-level rewards.
-        
-        Args:
-            trace: Task execution trace (SessionTrace dict or compatible object)
-            policy_name: Name of the policy that generated this trace
-            task_app_id: Identifier for the task app (e.g., "heartdisease")
-            options: Judge configuration options:
-                - event: Whether to generate event-level rewards (default: False)
-                - outcome: Whether to generate outcome-level reward (default: False)
-                - rubric_id: Rubric identifier to use for evaluation
-                - rubric_overrides: Optional rubric modifications
-                - provider: LLM provider ("groq" or "gemini")
-                - model: Model identifier (e.g., "llama-3.1-8b-instant")
-                - max_concurrency: Max concurrent judge calls (default: 1)
-            rubric: Optional explicit rubric criteria (event/outcome lists)
-            verifier_type: Optional zero-shot verifier graph ID (e.g., "zero_shot_verifier_single")
-            task_app_base_url: Optional base URL for task app (for rubric fetching)
-            
-        Returns:
-            JudgeScoreResponse with:
-                - status: "ok" or error status
-                - event_rewards: List of event-level reward dicts (if event=True)
-                - outcome_reward: Outcome-level reward dict (if outcome=True)
-                - details: Additional evaluation details
-                
-        Raises:
-            ValueError: If validation fails or rubric is invalid
-            PermissionError: If authentication fails
-            FileNotFoundError: If task app or rubric not found
-            Exception: For rate limiting or transient errors
-        """
-        trace_payload = normalize_for_json(trace)
-        task_app_payload = {"id": task_app_id}
-        if task_app_base_url:
-            task_app_payload["base_url"] = task_app_base_url
-
-        selected_verifier = verifier_type or (options or {}).get("verifier_type")
-        if selected_verifier:
-            graph_input = {
-                "policy_name": policy_name,
-                "task_app": task_app_payload,
-                "session_trace": trace_payload,
-                "trace": trace_payload,
-                "options": options or {},
-            }
-            if rubric is not None:
-                graph_input["rubric"] = normalize_for_json(rubric)
-            body = {"job_id": selected_verifier, "input": graph_input}
-        else:
-            body = {
-                "policy_name": policy_name,
-                "task_app": task_app_payload,
-                "trace": trace_payload,
-                "options": options or {},
-            }
-            if rubric is not None:
-                body["rubric"] = normalize_for_json(rubric)
-        try:
-            async with AsyncHttpClient(self._base, self._key, timeout=self._timeout) as http:
-                if selected_verifier:
-                    js = await http.post_json("/api/graphs/completions", json=body)
-                    if isinstance(js, dict) and "output" in js:
-                        js = js["output"]
-                else:
-                    js = await http.post_json("/api/judge/v1/score", json=body)
-                if not isinstance(js, dict):
-                    raise ValueError("invalid_judge_response_shape")
-                return js  # type: ignore[return-value]
-        except HTTPError as err:  # map to friendlier exceptions
-            status = int(getattr(err, "status", 0) or 0)
-            if status in (400, 422):
-                raise ValueError(f"judge_validation_error: {err.detail}") from err
-            if status in (401, 403):
-                raise PermissionError(f"judge_auth_error: {err.detail}") from err
-            if status == 404:
-                raise FileNotFoundError(f"judge_route_not_found: {err.detail}") from err
-            if status == 429:
-                raise Exception("judge_rate_limited") from err  # replace with RetryLater in future
-            if status >= 500:
-                raise Exception("judge_transient_error") from err  # replace with TransientError in future
-            raise
-
-
-class VerifierClient(GraphVerifierClient):
-    """Deprecated alias for graph-based VerifierClient."""

synth_ai/sdk/judging/types.py

@@ -1,42 +0,0 @@
-from __future__ import annotations
-
-from typing import Literal, TypedDict
-
-Track = Literal["process", "reasoning", "progress", "outcome"]
-
-
-class Judgement(TypedDict, total=False):
-	key: str
-	title: str
-	description: str
-	score: float
-	reason: str
-	confidence: float
-	scale: Literal["binary", "bounded", "count", "custom"]
-	source: dict
-
-
-class RewardJudgement(TypedDict, total=False):
-	judgement: Judgement
-	scope: Literal["step", "event", "outcome"]
-	turn: int | None
-	episode_id: str | None
-	reward_value: float | None
-	links: dict
-
-
-class TrackAggregate(TypedDict, total=False):
-	mean: float
-	median: float
-	std: float
-	n: int
-
-
-class RewardMetadata(TypedDict, total=False):
-	per_window: list[RewardJudgement]
-	aggregates: dict[Track, TrackAggregate]
-	overall: dict[str, float]  # {"final_outcome_score": float}
-	rubric: dict               # {"ids": {...}, "hash": "..."}
-	model_info: dict           # {"model": "...", ...}
-
-

synth_ai/sdk/research_agent/__init__.py

@@ -1,34 +0,0 @@
-from synth_ai.sdk.research_agent.container_builder import (
-    ContainerBackend,
-    DockerBackend,
-    ModalBackend,
-    get_backend,
-)
-from synth_ai.sdk.research_agent.container_spec import ContainerSpec
-from synth_ai.sdk.research_agent.defaults import (
-    DEFAULT_BACKEND,
-    DEFAULT_BASE_IMAGE,
-    DEFAULT_INSTRUCTIONS,
-    DEFAULT_PACKAGES,
-    DEFAULT_PYTHON_VERSION,
-    DEFAULT_REASONING_EFFORT,
-    DEFAULT_RESULT_PATTERNS,
-)
-from synth_ai.sdk.research_agent.results_collector import ResultsCollector
-
-__all__ = [
-    "ContainerBackend",
-    "ContainerSpec",
-    "DockerBackend",
-    "ModalBackend",
-    "ResultsCollector",
-    "get_backend",
-    "DEFAULT_BACKEND",
-    "DEFAULT_BASE_IMAGE",
-    "DEFAULT_INSTRUCTIONS",
-    "DEFAULT_PACKAGES",
-    "DEFAULT_PYTHON_VERSION",
-    "DEFAULT_REASONING_EFFORT",
-    "DEFAULT_RESULT_PATTERNS",
-]
-

synth_ai/sdk/research_agent/container_builder.py

@@ -1,328 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import base64
-import contextlib
-import fnmatch
-import io
-import tarfile
-import tempfile
-import time
-import uuid
-from abc import ABC, abstractmethod
-from pathlib import Path
-from typing import Dict, Iterable, Tuple
-
-from synth_ai.sdk.research_agent.container_spec import ContainerSpec
-from synth_ai.sdk.research_agent.defaults import DEFAULT_BACKEND
-
-
-class ContainerBackend(ABC):
-    """Abstract base for container execution backends."""
-
-    @abstractmethod
-    async def provision(self, spec: ContainerSpec) -> str:
-        """Provision a new container and return its id/handle."""
-
-    @abstractmethod
-    async def execute(
-        self,
-        container_id: str,
-        command: str,
-        *,
-        env: Dict[str, str] | None = None,
-        workdir: Path | None = None,
-    ) -> Dict[str, str | int]:
-        """Execute a command in the container."""
-
-    @abstractmethod
-    async def collect_artifacts(self, container_id: str, patterns: Iterable[str]) -> Dict[str, bytes]:
-        """Pull artifacts that match any of the glob patterns."""
-
-    @abstractmethod
-    async def destroy(self, container_id: str) -> None:
-        """Tear down container resources."""
-
-
-class DockerBackend(ContainerBackend):
-    """Docker implementation using docker-py."""
-
-    def __init__(self, *, client=None):
-        self._client = client
-        self._containers: Dict[str, Tuple[object, ContainerSpec]] = {}
-
-    def _ensure_client(self):
-        if self._client is None:
-            try:
-                import docker  # type: ignore
-            except ImportError as exc:
-                raise RuntimeError("docker SDK is not installed. Add docker>=7.0.0 to dependencies.") from exc
-            self._client = docker.from_env()
-        return self._client
-
-    async def provision(self, spec: ContainerSpec) -> str:
-        spec.validate()
-        client = self._ensure_client()
-        context_bytes = spec.build_context()
-        image_tag = f"research-agent:{int(time.time())}"
-
-        def _build():
-            return client.images.build(
-                fileobj=io.BytesIO(context_bytes),
-                custom_context=True,
-                rm=True,
-                nocache=True,
-                tag=image_tag,
-                buildargs=spec.build_args,
-            )
-
-        loop = asyncio.get_running_loop()
-        image, _ = await loop.run_in_executor(None, _build)
-
-        container = client.containers.create(
-            image=image.id,
-            command="sleep infinity",
-            environment={**spec.env_vars, **spec.secrets},
-            tty=True,
-            detach=True,
-            working_dir=str(spec.workdir),
-        )
-        container.start()
-        self._containers[container.id] = (container, spec)
-        return container.id
-
-    async def execute(
-        self,
-        container_id: str,
-        command: str,
-        *,
-        env: Dict[str, str] | None = None,
-        workdir: Path | None = None,
-    ) -> Dict[str, str | int]:
-        container, spec = self._containers[container_id]
-        if env is None:
-            env = {}
-        exec_env = {**spec.env_vars, **spec.secrets, **env}
-        workdir_str = str(workdir or spec.workdir)
-
-        def _run():
-            result = container.exec_run(  # type: ignore[attr-defined]
-                cmd=["bash", "-lc", command],
-                environment=exec_env,
-                workdir=workdir_str,
-                demux=True,
-            )
-            stdout, stderr = result.output
-            return {
-                "exit_code": result.exit_code,
-                "stdout": (stdout or b"").decode(),
-                "stderr": (stderr or b"").decode(),
-            }
-
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, _run)
-
-    async def collect_artifacts(self, container_id: str, patterns: Iterable[str]) -> Dict[str, bytes]:
-        container, spec = self._containers[container_id]
-        loop = asyncio.get_running_loop()
-
-        def _pull():
-            try:
-                stream, _ = container.get_archive(str(spec.artifacts_dir))  # type: ignore[attr-defined]
-            except Exception:
-                return {}
-            tar_bytes = b"".join(stream)
-            collected: Dict[str, bytes] = {}
-            with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:*") as tar:
-                for member in tar.getmembers():
-                    if not member.isfile():
-                        continue
-                    relative_name = str(Path(member.name).name)
-                    if not any(fnmatch.fnmatch(relative_name, pat) for pat in patterns):
-                        continue
-                    file_obj = tar.extractfile(member)
-                    if file_obj:
-                        collected[relative_name] = file_obj.read()
-            return collected
-
-        return await loop.run_in_executor(None, _pull)
-
-    async def destroy(self, container_id: str) -> None:
-        container, _ = self._containers.pop(container_id, (None, None))
-        if container is None:
-            return
-
-        def _stop():
-            with contextlib.suppress(Exception):
-                container.kill()  # type: ignore[attr-defined]
-            with contextlib.suppress(Exception):
-                container.remove(force=True)  # type: ignore[attr-defined]
-
-        loop = asyncio.get_running_loop()
-        await loop.run_in_executor(None, _stop)
-
-
-class ModalBackend(ContainerBackend):
-    """Modal implementation using modal SDK. Returns artifacts inline from execute()."""
-
-    def __init__(self):
-        self._runs: Dict[str, Dict[str, object]] = {}
-
-    def _write_build_context(self, spec: ContainerSpec) -> tempfile.TemporaryDirectory:
-        """Materialize Dockerfile + overlay files to a temp dir for Modal build."""
-        temp_dir = tempfile.TemporaryDirectory()
-        ctx = Path(temp_dir.name)
-        (ctx / "Dockerfile").write_text(spec.to_dockerfile())
-
-        overlay_root = ctx / "overlay_files"
-        for rel_path, content in spec.rendered_overlay_files().items():
-            target = overlay_root / rel_path
-            target.parent.mkdir(parents=True, exist_ok=True)
-            target.write_bytes(content)
-
-        for rel_path, content in spec.files.items():
-            if not str(rel_path).startswith("/"):
-                continue
-            data = content.encode() if isinstance(content, str) else content
-            target = ctx / str(rel_path).lstrip("/")
-            target.parent.mkdir(parents=True, exist_ok=True)
-            target.write_bytes(data)
-
-        return temp_dir
-
-    async def provision(self, spec: ContainerSpec) -> str:
-        spec.validate()
-        try:
-            import modal  # type: ignore
-        except ImportError as exc:  # pragma: no cover - runtime import guard
-            raise RuntimeError("modal SDK is not installed. Add modal>=1.1.1 to dependencies.") from exc
-
-        ctx_dir = self._write_build_context(spec)
-        loop = asyncio.get_running_loop()
-
-        def _build_image():
-            return modal.Image.from_dockerfile(
-                path=ctx_dir.name,
-                build_args=spec.build_args,
-                force_build=True,
-            )
-
-        image = await loop.run_in_executor(None, _build_image)
-
-        # Combine env_vars and secrets into a Modal Secret
-        # Modal function decorator doesn't accept 'env' parameter directly
-        # Environment variables must be passed via secrets
-        combined_env: dict[str, str | None] = {**spec.env_vars, **spec.secrets}
-        secret_obj = None
-        if combined_env:
-            secret_obj = modal.Secret.from_dict(combined_env)
-
-        app = modal.App(f"oneshot-research-{int(time.time())}")
-
-        workdir_str = str(spec.workdir)
-        
-        @app.function(
-            image=image,
-            timeout=60 * 60,
-            secrets=[secret_obj] if secret_obj else [],
-        )
-        def run_task(command: str, patterns: list[str], artifacts_dir: str = "/app/artifacts") -> Dict:
-            """Execute the agent and pull artifacts matching patterns."""
-            import glob
-            import os
-            import subprocess
-
-            result = subprocess.run(
-                ["bash", "-lc", command],
-                capture_output=True,
-                text=True,
-                cwd=workdir_str,
-            )
-
-            artifacts: Dict[str, str] = {}
-            for pat in patterns:
-                for path in glob.glob(os.path.join(artifacts_dir, pat)):
-                    if not os.path.isfile(path):
-                        continue
-                    name = os.path.basename(path)
-                    with open(path, "rb") as f:
-                        artifacts[name] = base64.b64encode(f.read()).decode()
-
-            return {
-                "exit_code": result.returncode,
-                "stdout": result.stdout,
-                "stderr": result.stderr,
-                "artifacts": artifacts,
-            }
-
-        container_id = str(uuid.uuid4())
-        self._runs[container_id] = {
-            "app": app,
-            "function": run_task,
-            "result": None,
-            "ctx_dir": ctx_dir,
-            "patterns": tuple(spec.result_matchers()),
-        }
-        return container_id
-
-    async def execute(
-        self,
-        container_id: str,
-        command: str,
-        *,
-        env: Dict[str, str] | None = None,
-        workdir: Path | None = None,
-    ) -> Dict[str, str | int]:
-        run_info = self._runs.get(container_id)
-        if not run_info:
-            raise ValueError(f"Unknown container_id: {container_id}")
-        app = run_info["app"]
-        run_fn = run_info["function"]
-        patterns = list(run_info["patterns"])
-
-        loop = asyncio.get_running_loop()
-
-        def _call():
-            with app.run():
-                return run_fn.call(command, patterns)
-
-        result = await loop.run_in_executor(None, _call)
-        run_info["result"] = result
-        return {
-            "exit_code": result.get("exit_code", -1),
-            "stdout": result.get("stdout", ""),
-            "stderr": result.get("stderr", ""),
-        }
-
-    async def collect_artifacts(self, container_id: str, patterns: Iterable[str]) -> Dict[str, bytes]:
-        run_info = self._runs.get(container_id)
-        if not run_info:
-            return {}
-        result = run_info.get("result") or {}
-        artifacts: Dict[str, bytes] = {}
-        encoded = result.get("artifacts") or {}  # type: ignore[misc]
-        for name, b64 in encoded.items():
-            try:
-                artifacts[name] = base64.b64decode(b64)
-            except Exception:
-                continue
-        return artifacts
-
-    async def destroy(self, container_id: str) -> None:
-        info = self._runs.pop(container_id, None)
-        if not info:
-            return
-        ctx_dir = info.get("ctx_dir")
-        if ctx_dir and hasattr(ctx_dir, "cleanup"):
-            with contextlib.suppress(Exception):
-                ctx_dir.cleanup()  # type: ignore[call-arg]
-
-
-def get_backend(name: str = DEFAULT_BACKEND) -> ContainerBackend:
-    """Resolve backend by name."""
-    normalized = (name or DEFAULT_BACKEND).lower()
-    if normalized == "docker":
-        return DockerBackend()
-    if normalized == "modal":
-        return ModalBackend()
-    raise ValueError(f"Unsupported container backend: {name}")