synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/sdk/inference/client.py

@@ -0,0 +1,128 @@
+"""Inference client for model inference via Synth AI.
+
+This module provides a client for making inference requests through Synth AI's
+inference proxy, which routes requests to appropriate model providers (OpenAI,
+Groq, etc.) based on the model identifier.
+
+Example:
+    >>> from synth_ai.sdk.inference import InferenceClient
+    >>> 
+    >>> client = InferenceClient(
+    ...     base_url="https://api.usesynth.ai",
+    ...     api_key=os.environ["SYNTH_API_KEY"],
+    ... )
+    >>> 
+    >>> response = await client.create_chat_completion(
+    ...     model="gpt-4o-mini",
+    ...     messages=[
+    ...         {"role": "user", "content": "Hello!"}
+    ...     ],
+    ...     temperature=0.7,
+    ...     max_tokens=100,
+    ... )
+    >>> 
+    >>> print(response["choices"][0]["message"]["content"])
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from synth_ai.core._utils.http import AsyncHttpClient
+from synth_ai.sdk.api.models.supported import (
+    UnsupportedModelError,
+    normalize_model_identifier,
+)
+
+
+class InferenceClient:
+    """Client for making inference requests through Synth AI's inference proxy.
+    
+    This client provides a unified interface for calling LLMs through Synth AI's
+    backend, which handles routing to appropriate providers (OpenAI, Groq, etc.)
+    based on the model identifier.
+    
+    Example:
+        >>> from synth_ai.sdk.inference import InferenceClient
+        >>> 
+        >>> client = InferenceClient(
+        ...     base_url="https://api.usesynth.ai",
+        ...     api_key=os.environ["SYNTH_API_KEY"],
+        ... )
+        >>> 
+        >>> # Simple completion
+        >>> response = await client.create_chat_completion(
+        ...     model="gpt-4o-mini",
+        ...     messages=[{"role": "user", "content": "Hello!"}],
+        ... )
+        >>> 
+        >>> # With options
+        >>> response = await client.create_chat_completion(
+        ...     model="Qwen/Qwen3-4B",
+        ...     messages=[{"role": "user", "content": "Explain RL"}],
+        ...     temperature=0.7,
+        ...     max_tokens=500,
+        ...     thinking_budget=512,
+        ... )
+    """
+    
+    def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0) -> None:
+        """Initialize the inference client.
+        
+        Args:
+            base_url: Base URL for the Synth AI API
+            api_key: API key for authentication
+            timeout: Request timeout in seconds (default: 30.0)
+        """
+        self._base_url = base_url.rstrip("/")
+        self._api_key = api_key
+        self._timeout = timeout
+
+    async def create_chat_completion(
+        self, *, model: str, messages: list[dict], **kwargs: Any
+    ) -> dict[str, Any]:
+        """Create a chat completion request.
+        
+        This method sends a chat completion request to the Synth AI inference proxy,
+        which routes it to the appropriate provider based on the model identifier.
+        
+        Args:
+            model: Model identifier (e.g., "gpt-4o-mini", "Qwen/Qwen3-4B")
+            messages: List of message dicts with "role" and "content" keys
+            **kwargs: Additional OpenAI-compatible parameters:
+                - temperature: Sampling temperature (0.0-2.0)
+                - max_tokens: Maximum tokens to generate
+                - thinking_budget: Budget for thinking tokens (default: 256)
+                - top_p: Nucleus sampling parameter
+                - frequency_penalty: Frequency penalty (-2.0 to 2.0)
+                - presence_penalty: Presence penalty (-2.0 to 2.0)
+                - stop: Stop sequences
+                - tools: Function calling tools
+                - tool_choice: Tool choice strategy
+                - stream: Whether to stream responses
+                - ... (other OpenAI API parameters)
+                
+        Returns:
+            Completion response dict with:
+                - id: Request ID
+                - choices: List of completion choices
+                - usage: Token usage statistics
+                - ... (other OpenAI-compatible fields)
+                
+        Raises:
+            ValueError: If model is not supported or request is invalid
+            HTTPError: If the API request fails
+        """
+        try:
+            normalized_model = normalize_model_identifier(model)
+        except UnsupportedModelError as exc:
+            raise ValueError(str(exc)) from exc
+
+        body: dict[str, Any] = {"model": normalized_model, "messages": messages}
+        body.update(kwargs)
+        # Backend now expects an explicit thinking_budget; provide a sensible default if omitted
+        if "thinking_budget" not in body:
+            body["thinking_budget"] = 256
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            # Route through backend inference proxy to Modal
+            return await http.post_json("/api/inference/v1/chat/completions", json=body)

synth_ai/sdk/jobs/__init__.py

@@ -0,0 +1,16 @@
+"""Jobs API client for Synth AI."""
+from synth_ai.sdk.jobs.client import (
+    FilesApi,
+    JobsClient,
+    ModelsApi,
+    RlJobsApi,
+    SftJobsApi,
+)
+
+__all__ = [
+    "JobsClient",
+    "FilesApi",
+    "SftJobsApi",
+    "RlJobsApi",
+    "ModelsApi",
+]

synth_ai/sdk/jobs/client.py

@@ -0,0 +1,371 @@
+"""Jobs API client for managing training jobs, files, and models.
+
+This module provides a unified interface for interacting with Synth AI's job APIs,
+including file uploads, SFT jobs, RL jobs, and model management.
+
+Example:
+    >>> from synth_ai.sdk.jobs import JobsClient
+    >>> 
+    >>> async with JobsClient(
+    ...     base_url="https://api.usesynth.ai",
+    ...     api_key=os.environ["SYNTH_API_KEY"],
+    ... ) as client:
+    ...     # Upload training data
+    ...     file_result = await client.files.upload(
+    ...         filename="train.jsonl",
+    ...         content=b"...",
+    ...         purpose="training",
+    ...     )
+    ...     
+    ...     # Create SFT job
+    ...     job = await client.sft.create(
+    ...         training_file=file_result["id"],
+    ...         model="Qwen/Qwen3-4B",
+    ...     )
+    ...     
+    ...     # Check job status
+    ...     status = await client.sft.retrieve(job["id"])
+"""
+
+from __future__ import annotations
+
+import importlib
+from collections.abc import Callable
+from typing import Any, cast
+
+try:
+    _supported_module = cast(
+        Any, importlib.import_module("synth_ai.sdk.api.models.supported")
+    )
+    normalize_model_identifier = cast(
+        Callable[[str], str], _supported_module.normalize_model_identifier
+    )
+except Exception as exc:  # pragma: no cover - critical dependency
+    raise RuntimeError("Unable to load supported model utilities") from exc
+
+try:
+    _http_module = cast(Any, importlib.import_module("synth_ai.core.http"))
+    AsyncHttpClient = _http_module.AsyncHttpClient
+except Exception as exc:  # pragma: no cover - critical dependency
+    raise RuntimeError("Unable to load HTTP client") from exc
+
+try:
+    _sft_config_module = cast(
+        Any, importlib.import_module("synth_ai.sdk.learning.sft.config")
+    )
+    prepare_sft_job_payload = cast(
+        Callable[..., dict[str, Any]], _sft_config_module.prepare_sft_job_payload
+    )
+except Exception as exc:  # pragma: no cover - critical dependency
+    raise RuntimeError("Unable to load SFT configuration helpers") from exc
+
+
+class FilesApi:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def upload(
+        self,
+        *,
+        filename: str,
+        content: bytes,
+        purpose: str,
+        content_type: str | None = None,
+        idempotency_key: str | None = None,
+    ) -> dict[str, Any]:
+        data = {"purpose": purpose}
+        files = {"file": (filename, content, content_type)}
+        headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
+        return await self._http.post_multipart(
+            "/api/files", data=data, files=files, headers=headers
+        )
+
+    async def list(
+        self, *, purpose: str | None = None, after: str | None = None, limit: int = 20
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {}
+        if purpose is not None:
+            params["purpose"] = purpose
+        if after is not None:
+            params["after"] = after
+        params["limit"] = limit
+        return await self._http.get("/api/files", params=params)
+
+    async def retrieve(self, file_id: str) -> dict[str, Any]:
+        return await self._http.get(f"/api/files/{file_id}")
+
+    async def delete(self, file_id: str) -> Any:
+        return await self._http.delete(f"/api/files/{file_id}")
+
+    async def list_jobs(
+        self, file_id: str, *, after: str | None = None, limit: int = 20
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit}
+        if after is not None:
+            params["after"] = after
+        return await self._http.get(f"/api/files/{file_id}/jobs", params=params)
+
+
+class SftJobsApi:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def create(
+        self,
+        *,
+        training_file: str,
+        model: str,
+        validation_file: str | None = None,
+        hyperparameters: dict[str, Any] | None = None,
+        suffix: str | None = None,
+        integrations: dict[str, Any] | None = None,
+        metadata: dict[str, Any] | None = None,
+        idempotency_key: str | None = None,
+    ) -> dict[str, Any]:
+        payload = prepare_sft_job_payload(
+            model=model,
+            training_file=training_file,
+            hyperparameters=hyperparameters,
+            metadata=metadata,
+            training_type=None,
+            validation_file=validation_file,
+            suffix=suffix,
+            integrations=integrations,
+            training_file_field="training_file",
+            require_training_file=True,
+        )
+        headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
+        return await self._http.post_json("/api/sft/jobs", json=payload, headers=headers)
+
+    async def list(
+        self,
+        *,
+        status: str | None = None,
+        model: str | None = None,
+        file_id: str | None = None,
+        created_after: int | None = None,
+        created_before: int | None = None,
+        after: str | None = None,
+        limit: int = 20,
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit}
+        if status is not None:
+            params["status"] = status
+        if model is not None:
+            params["model"] = model
+        if file_id is not None:
+            params["file_id"] = file_id
+        if created_after is not None:
+            params["created_after"] = created_after
+        if created_before is not None:
+            params["created_before"] = created_before
+        if after is not None:
+            params["after"] = after
+        return await self._http.get("/api/sft/jobs", params=params)
+
+    async def retrieve(self, job_id: str) -> dict[str, Any]:
+        return await self._http.get(f"/api/sft/jobs/{job_id}")
+
+    async def cancel(self, job_id: str) -> dict[str, Any]:
+        return await self._http.post_json(f"/api/sft/jobs/{job_id}/cancel", json={})
+
+    async def list_events(
+        self, job_id: str, *, since_seq: int = 0, limit: int = 200
+    ) -> dict[str, Any]:
+        params = {"since_seq": since_seq, "limit": limit}
+        return await self._http.get(f"/api/sft/jobs/{job_id}/events", params=params)
+
+    async def checkpoints(
+        self, job_id: str, *, after: str | None = None, limit: int = 10
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit}
+        if after is not None:
+            params["after"] = after
+        return await self._http.get(f"/api/sft/jobs/{job_id}/checkpoints", params=params)
+
+
+class RlJobsApi:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def create(
+        self,
+        *,
+        model: str,
+        endpoint_base_url: str,
+        trainer_id: str,
+        trainer: dict[str, Any] | None = None,
+        job_config_id: str | None = None,
+        config: dict[str, Any] | None = None,
+        metadata: dict[str, Any] | None = None,
+        idempotency_key: str | None = None,
+    ) -> dict[str, Any]:
+        payload: dict[str, Any] = {
+            "model": normalize_model_identifier(model),
+            "endpoint_base_url": endpoint_base_url,
+            "trainer_id": trainer_id,
+        }
+        if trainer is not None:
+            payload["trainer"] = trainer
+        if job_config_id is not None:
+            payload["job_config_id"] = job_config_id
+        if config is not None:
+            payload["config"] = config
+        if metadata is not None:
+            payload["metadata"] = metadata
+        headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
+        return await self._http.post_json("/api/rl/jobs", json=payload, headers=headers)
+
+    async def list(
+        self,
+        *,
+        status: str | None = None,
+        model: str | None = None,
+        created_after: int | None = None,
+        created_before: int | None = None,
+        after: str | None = None,
+        limit: int = 20,
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit}
+        if status is not None:
+            params["status"] = status
+        if model is not None:
+            params["model"] = model
+        if created_after is not None:
+            params["created_after"] = created_after
+        if created_before is not None:
+            params["created_before"] = created_before
+        if after is not None:
+            params["after"] = after
+        return await self._http.get("/api/rl/jobs", params=params)
+
+    async def retrieve(self, job_id: str) -> dict[str, Any]:
+        return await self._http.get(f"/api/rl/jobs/{job_id}")
+
+    async def cancel(self, job_id: str) -> dict[str, Any]:
+        return await self._http.post_json(f"/api/rl/jobs/{job_id}/cancel", json={})
+
+    async def list_events(
+        self, job_id: str, *, since_seq: int = 0, limit: int = 200
+    ) -> dict[str, Any]:
+        params = {"since_seq": since_seq, "limit": limit}
+        return await self._http.get(f"/api/rl/jobs/{job_id}/events", params=params)
+
+    async def metrics(
+        self, job_id: str, *, after_step: int = -1, limit: int = 200
+    ) -> dict[str, Any]:
+        params = {"after_step": after_step, "limit": limit}
+        return await self._http.get(f"/api/rl/jobs/{job_id}/metrics", params=params)
+
+
+class ModelsApi:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def list(
+        self,
+        *,
+        source: str | None = None,
+        base_model: str | None = None,
+        status: str | None = None,
+        after: str | None = None,
+        limit: int = 20,
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit}
+        if source is not None:
+            params["source"] = source
+        if base_model is not None:
+            params["base_model"] = base_model
+        if status is not None:
+            params["status"] = status
+        if after is not None:
+            params["after"] = after
+        return await self._http.get("/api/models", params=params)
+
+    async def retrieve(self, model_id: str) -> dict[str, Any]:
+        return await self._http.get(f"/api/models/{model_id}")
+
+    async def delete(self, model_id: str) -> Any:
+        return await self._http.delete(f"/api/models/{model_id}")
+
+    async def list_jobs(
+        self, model_id: str, *, after: str | None = None, limit: int = 20
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit}
+        if after is not None:
+            params["after"] = after
+        return await self._http.get(f"/api/models/{model_id}/jobs", params=params)
+
+
+class JobsClient:
+    """High-level client for interacting with Synth AI job APIs.
+    
+    This client provides a unified interface for managing training jobs, files, and models
+    across different job types (SFT, RL, prompt learning). It aggregates multiple API
+    endpoints into a single, easy-to-use interface.
+    
+    The client is an async context manager, so it should be used with `async with`:
+    
+    Example:
+        >>> from synth_ai.sdk.jobs import JobsClient
+        >>> 
+        >>> async with JobsClient(
+        ...     base_url="https://api.usesynth.ai",
+        ...     api_key=os.environ["SYNTH_API_KEY"],
+        ... ) as client:
+        ...     # Upload a file
+        ...     file_result = await client.files.upload(
+        ...         filename="data.jsonl",
+        ...         content=b"...",
+        ...         purpose="training",
+        ...     )
+        ...     
+        ...     # Create an SFT job
+        ...     job_result = await client.sft.create(
+        ...         training_file=file_result["id"],
+        ...         model="Qwen/Qwen3-4B",
+        ...     )
+        ...     
+        ...     # Check RL job status
+        ...     rl_status = await client.rl.retrieve("rl_job_123")
+        ...     
+        ...     # List models
+        ...     models = await client.models.list()
+    
+    Attributes:
+        files: FilesApi - File upload and management
+        sft: SftJobsApi - Supervised fine-tuning jobs
+        rl: RlJobsApi - Reinforcement learning jobs
+        models: ModelsApi - Model management
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str,
+        timeout: float = 30.0,
+        http: AsyncHttpClient | None = None,
+    ) -> None:
+        """Initialize the Jobs API client.
+        
+        Args:
+            base_url: Base URL for the Synth AI API (e.g., "https://api.usesynth.ai")
+            api_key: API key for authentication
+            timeout: Request timeout in seconds (default: 30.0)
+            http: Optional pre-configured HTTP client (for testing/customization)
+        """
+        self._base_url = base_url
+        self._api_key = api_key
+        self._timeout = timeout
+        self._http = http or AsyncHttpClient(base_url, api_key, timeout=timeout)
+        self.files = FilesApi(self._http)
+        self.sft = SftJobsApi(self._http)
+        self.rl = RlJobsApi(self._http)
+        self.models = ModelsApi(self._http)
+
+    async def __aenter__(self) -> JobsClient:
+        await self._http.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:  # noqa: ANN001
+        await self._http.__aexit__(exc_type, exc, tb)

synth_ai/sdk/judging/__init__.py

@@ -0,0 +1,15 @@
+from .client import JudgeClient, JudgeOptions, JudgeScoreResponse, VerifierClient
+from .types import Judgement, RewardJudgement, RewardMetadata, Track, TrackAggregate
+
+__all__ = [
+	"JudgeClient",
+	"VerifierClient",
+	"JudgeOptions",
+	"JudgeScoreResponse",
+	"Judgement",
+	"RewardJudgement",
+	"RewardMetadata",
+	"Track",
+	"TrackAggregate",
+]
+

synth_ai/sdk/judging/base.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class Judgement:
+    def __init__(
+        self,
+        criteria: str,
+        score: float,
+        reasoning: str = "",
+        evidence: list[str] | None = None,
+    ) -> None:
+        self.criteria = criteria
+        self.score = score
+        self.reasoning = reasoning
+        self.evidence = evidence or []
+
+
+class BaseEval(ABC):
+    @abstractmethod
+    async def run(self, data: Any) -> list[Judgement]:
+        """Execute the evaluation and return a list of judgements."""