synthiaresearch 0.0.1__tar.gz

synthiaresearch-0.0.1/.gitignore

@@ -0,0 +1,11 @@
+node_modules/
+.env*
+!.env.example
+.vercel/
+__pycache__/
+.venv/
+.next/
+dist/
+.DS_Store
+.vercel
+*.tsbuildinfo

synthiaresearch-0.0.1/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: synthiaresearch
+Version: 0.0.1
+Summary: Python SDK for the Synthia API
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27

synthiaresearch-0.0.1/README.md

@@ -0,0 +1,33 @@
+# synthia
+
+Python SDK for the Synthia API.
+
+```python
+from synthia import Synthia
+
+client = Synthia(api_key="sk_...")
+
+# 1. Probe your agent to infer an ideal user model.
+#    The agent runs locally; any `(probe: str) -> str` callable works.
+user_model = client.user_models.create_from_probe(my_agent, max_turns=10)
+
+# 2. Generate synthetic data from it.
+job = client.datasets.generate(user_model, count=1000)
+dataset = job.wait()
+
+# 3. Play the scenarios against your agent, then quality-check the rollouts:
+#    a server-side parallel job analyzes each rollout's agentic states and
+#    judges whether the agent passed each scenario.
+results = client.rollouts.run(my_rollout_agent, dataset)
+evaluations = client.rollouts.quality_check(results).wait().rollouts()
+```
+
+## Layout
+
+- `src/synthia/client.py` — the whole SDK: the `Synthia` client, resource classes (seeds, user models, datasets), and the local probe loop with tool-call tracing.
+- `src/synthia/__init__.py` — public exports.
+- `pyproject.toml` — the `synthia` package definition (httpx as the sole runtime dependency).
+- `uv.lock` — uv lockfile.
+
+For an end-to-end demo against a real tool-using agent, see
+`examples/turo-support-agent/` at the repo root.

synthiaresearch-0.0.1/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "synthiaresearch"
+version = "0.0.1"
+description = "Python SDK for the Synthia API"
+requires-python = ">=3.10"
+dependencies = [
+    "httpx>=0.27",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/synthia"]

synthiaresearch-0.0.1/src/synthia/__init__.py

@@ -0,0 +1,17 @@
+from .client import (
+    AgentReply,
+    Dataset,
+    GenerationJob,
+    PrepareResult,
+    QualityCheck,
+    RolloutResult,
+    Synthia,
+    ToolCall,
+    ToolSandbox,
+    UserModel,
+    ValidationRun,
+)
+
+__all__ = ["Synthia", "UserModel", "GenerationJob", "Dataset", "AgentReply",
+           "ToolCall", "ValidationRun", "QualityCheck", "RolloutResult",
+           "ToolSandbox", "PrepareResult"]

synthiaresearch-0.0.1/src/synthia/client.py

@@ -0,0 +1,637 @@
+import hashlib
+import json
+import os
+import sys
+import time
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import asdict, dataclass, field
+from importlib import metadata
+from pathlib import Path
+from typing import Callable, Union
+
+import httpx
+
+DEFAULT_BASE_URL = "https://synthia-research--synthia-api-web.modal.run"
+
+# Launcher argv[0] values that don't identify a script (notebooks, REPLs).
+_INTERACTIVE_STEMS = {"", "-c", "-m", "ipykernel_launcher", "pydevconsole"}
+
+
+def _default_session_name() -> str:
+    """Stable session name derived from the entry-point script.
+
+    Last two path components only ("project/script"): readable, survives
+    moving the project, and never leaks the full directory layout. Falls
+    back to the working directory's name for notebooks and REPLs.
+    """
+    argv0 = sys.argv[0] if sys.argv else ""
+    if not argv0 or Path(argv0).stem in _INTERACTIVE_STEMS:
+        return Path.cwd().resolve().name
+    p = Path(argv0).resolve()
+    return f"{p.parent.name}/{p.stem}"
+
+
+def _sdk_version() -> str | None:
+    try:
+        return metadata.version("synthia")
+    except metadata.PackageNotFoundError:
+        return None
+
+
+@dataclass
+class ToolCall:
+    """One traced tool invocation made by the agent while answering a probe."""
+    name: str
+    input: dict
+    output: dict | None = None
+    is_error: bool = False
+
+
+@dataclass
+class AgentReply:
+    """An agent's reply plus the tool calls it made producing it."""
+    reply: str
+    tool_calls: list[ToolCall] = field(default_factory=list)
+
+
+# An agent takes a probe question and returns its reply — a plain string, or
+# an AgentReply carrying traced tool calls.
+Agent = Callable[[str], Union[str, AgentReply]]
+
+
+class ToolSandbox:
+    """Local replica of the server's deterministic tool sandbox.
+
+    Must stay hash-identical to synthia_api.pipeline.rollout.ToolSandbox:
+    outputs are a pure function of (tool name, input, state version, seed),
+    which is how the server reproduces the same tool behavior from the
+    events the SDK reports. Tool inputs must be JSON-native values that
+    round-trip exactly (e.g. tuples arrive as lists, 1 and 1.0 differ).
+    """
+
+    def __init__(self, seed: int, fail_tools: set[str] | None = None,
+                 state: dict | None = None):
+        self.seed = seed
+        self.fail_tools = fail_tools or set()
+        self.state = dict(state or {})
+        self.events: list[dict] = []
+
+    @classmethod
+    def from_config(cls, config: dict) -> "ToolSandbox":
+        return cls(seed=config["seed"], fail_tools=set(config["fail_tools"]),
+                   state=config["state"])
+
+    def call(self, name: str, tool_input: dict) -> dict:
+        is_error = name in self.fail_tools
+        if is_error:
+            output = {"error": f"{name} is unavailable"}
+        else:
+            digest = hashlib.sha256(
+                json.dumps([name, tool_input, self.state.get("version", 0),
+                            self.seed], sort_keys=True).encode()
+            ).hexdigest()[:8]
+            output = {"ok": True, "result_id": digest}
+            self.state["version"] = self.state.get("version", 0) + 1
+            self.state[f"last_{name}"] = digest
+        self.events.append({"name": name, "input": tool_input,
+                            "output": output, "is_error": is_error})
+        return output
+
+
+# A rollout agent takes the conversation transcript ([{role, content}]) and a
+# ToolSandbox for its tool calls, and returns its reply text. It is invoked
+# from worker threads when rollouts run concurrently, so it must be
+# thread-safe.
+RolloutAgent = Callable[[list, ToolSandbox], str]
+
+
+class _EventStream:
+    """Cursor-based poller that prints a run's server-side telemetry."""
+
+    def __init__(self, http: httpx.Client, path: str, enabled: bool):
+        self._http = http
+        self._path = path
+        self._after = 0
+        self._enabled = enabled
+
+    def pump(self) -> None:
+        """Fetch and print events newer than the cursor."""
+        if not self._enabled:
+            return
+        r = self._http.get(self._path, params={"after": self._after})
+        r.raise_for_status()
+        for event in r.json()["data"]:
+            self._after = event["seq"]
+            detail = " ".join(f"{k}={v}" for k, v in event["data"].items())
+            print(f"    ~ [{event['stage']}] {event['message']}"
+                  + (f"  ({detail})" if detail else ""))
+
+
+@dataclass
+class UserModel:
+    id: str
+    probe_session_id: str
+    persona: str
+    traits: list[str]
+    representation_id: str | None
+
+
+@dataclass
+class ValidationRun:
+    """A validation of a dataset: a per-scenario judge gate plus
+    collective fidelity/diversity reports and an advisory verdict."""
+    id: str
+    dataset_id: str
+    status: str
+    verdict: str | None
+    reference: dict | None
+    validity: dict | None
+    fidelity: dict | None
+    diversity: dict | None
+    error: str | None
+    _http: httpx.Client
+
+    def wait(self, poll_interval: float = 2.0, timeout: float = 1800.0,
+             verbose: bool = False) -> "ValidationRun":
+        """Poll until validation finishes; verbose prints server telemetry."""
+        events = _EventStream(
+            self._http, f"/v1/validation-runs/{self.id}/events", verbose)
+        deadline = time.monotonic() + timeout
+        while self.status == "running":
+            if time.monotonic() > deadline:
+                raise TimeoutError(
+                    f"validation run {self.id} still running after {timeout}s")
+            time.sleep(poll_interval)
+            r = self._http.get(f"/v1/validation-runs/{self.id}")
+            r.raise_for_status()
+            data = r.json()
+            for key in ("status", "verdict", "reference", "validity",
+                        "fidelity", "diversity", "error"):
+                setattr(self, key, data[key])
+            events.pump()
+        events.pump()  # catch events written after the final status poll
+        if self.status != "succeeded":
+            raise RuntimeError(f"validation run {self.id} failed: {self.error}")
+        return self
+
+    def scenarios(self) -> list[dict]:
+        """Per-scenario judge verdicts: scenario_id, passed, judge."""
+        r = self._http.get(f"/v1/validation-runs/{self.id}/scenarios")
+        r.raise_for_status()
+        return r.json()["data"]
+
+
+@dataclass
+class Dataset:
+    id: str
+    generation_id: str
+    user_model_id: str
+    row_count: int
+    _http: httpx.Client
+
+    def download(self) -> list[dict]:
+        r = self._http.get(f"/v1/datasets/{self.id}/rows")
+        r.raise_for_status()
+        return r.json()["data"]
+
+    def validate(self) -> ValidationRun:
+        """Start an async validation run over this dataset."""
+        r = self._http.post(f"/v1/datasets/{self.id}/validations")
+        r.raise_for_status()
+        return ValidationRun(**r.json(), _http=self._http)
+
+    def rollout(self, agent: "RolloutAgent", **kwargs) -> "list[RolloutResult]":
+        """Play this dataset's scenarios against a local agent."""
+        return Rollouts(self._http).run(agent, dataset=self, **kwargs)
+
+
+@dataclass
+class GenerationJob:
+    id: str
+    status: str
+    user_model_id: str
+    count: int
+    dataset_id: str | None
+    error: str | None
+    _http: httpx.Client
+
+    def wait(self, poll_interval: float = 2.0, timeout: float = 1800.0,
+             verbose: bool = False) -> Dataset:
+        """Poll until the job finishes; verbose prints server telemetry live."""
+        events = _EventStream(
+            self._http, f"/v1/generations/{self.id}/events", verbose)
+        deadline = time.monotonic() + timeout
+        while self.status == "running":
+            if time.monotonic() > deadline:
+                raise TimeoutError(f"generation {self.id} still running after {timeout}s")
+            time.sleep(poll_interval)
+            r = self._http.get(f"/v1/generations/{self.id}")
+            r.raise_for_status()
+            data = r.json()
+            self.status = data["status"]
+            self.dataset_id = data["dataset_id"]
+            self.error = data["error"]
+            events.pump()
+        events.pump()  # catch events written after the final status poll
+        if self.status != "succeeded":
+            raise RuntimeError(f"generation {self.id} failed: {self.error}")
+        r = self._http.get(f"/v1/datasets/{self.dataset_id}")
+        r.raise_for_status()
+        return Dataset(**r.json(), _http=self._http)
+
+
+class Seeds:
+    def __init__(self, http: httpx.Client):
+        self._http = http
+
+    def ingest(self, *, kind: str, source: str, content: dict,
+               version: str = "1", metadata: dict | None = None) -> dict:
+        """Upload seed material (documents, tool schemas, policies, traces...)."""
+        r = self._http.post("/v1/seeds", json={
+            "kind": kind, "source": source, "content": content,
+            "version": version, "metadata": metadata or {},
+        })
+        r.raise_for_status()
+        return r.json()
+
+
+class UserModels:
+    def __init__(self, http: httpx.Client):
+        self._http = http
+
+    def create_from_probe(self, agent: Agent, max_turns: int = 10,
+                          verbose: bool = False) -> UserModel:
+        """Probe `agent` until the server converges on a user model.
+
+        The agent runs locally; only probe questions, replies, and traced
+        tool calls travel over the wire. `verbose` prints the server's
+        telemetry (probe decisions, ingestion, inference) after each turn.
+        """
+        r = self._http.post("/v1/probe-sessions", json={"max_turns": max_turns})
+        r.raise_for_status()
+        session = r.json()
+        events = _EventStream(
+            self._http, f"/v1/probe-sessions/{session['id']}/events", verbose)
+        while session["status"] == "active":
+            raw = agent(session["next_probe"])
+            reply = raw if isinstance(raw, str) else raw.reply
+            tool_calls = [] if isinstance(raw, str) else [
+                asdict(tc) for tc in raw.tool_calls
+            ]
+            r = self._http.post(
+                f"/v1/probe-sessions/{session['id']}/responses",
+                json={"reply": reply, "tool_calls": tool_calls},
+            )
+            r.raise_for_status()
+            session = r.json()
+            events.pump()
+        return self.get(session["user_model_id"])
+
+    def get(self, model_id: str) -> UserModel:
+        r = self._http.get(f"/v1/user-models/{model_id}")
+        r.raise_for_status()
+        return UserModel(**r.json())
+
+    def list(self, session: str | None = None) -> list[UserModel]:
+        params = {"sdk_session": session} if session else {}
+        r = self._http.get("/v1/user-models", params=params)
+        r.raise_for_status()
+        return [UserModel(**m) for m in r.json()["data"]]
+
+
+class Datasets:
+    def __init__(self, http: httpx.Client):
+        self._http = http
+
+    def get(self, dataset_id: str) -> Dataset:
+        r = self._http.get(f"/v1/datasets/{dataset_id}")
+        r.raise_for_status()
+        return Dataset(**r.json(), _http=self._http)
+
+    def list(self, session: str | None = None) -> list[Dataset]:
+        """Datasets newest first; `session` filters to one SDK session."""
+        params = {"sdk_session": session} if session else {}
+        r = self._http.get("/v1/datasets", params=params)
+        r.raise_for_status()
+        return [Dataset(**d, _http=self._http) for d in r.json()["data"]]
+
+    def generate(self, user_model: UserModel | str, count: int = 20, *,
+                 quality_check_id: str | None = None) -> GenerationJob:
+        """Start a generation job. quality_check_id names a completed quality
+        check whose results calibrate the batch's difficulty and coverage."""
+        model_id = user_model.id if isinstance(user_model, UserModel) else user_model
+        body = {"user_model_id": model_id, "count": count}
+        if quality_check_id:
+            body["quality_check_id"] = quality_check_id
+        r = self._http.post("/v1/generations", json=body)
+        r.raise_for_status()
+        return GenerationJob(**r.json(), _http=self._http)
+
+
+@dataclass
+class PrepareResult:
+    """Outcome of Synthia.prepare(): the dataset to roll out, the user model
+    behind it, and how the decision was made."""
+    dataset: Dataset
+    user_model: UserModel
+    action: str                     # "generated" | "reused"
+    reason: str                     # human-readable decision trail
+    success_rate: float | None      # latest completed check's rate, if any
+    quality_check_id: str | None    # check that calibrated generation, if any
+
+
+@dataclass
+class QualityCheck:
+    """An async evaluation of finished rollouts: per rollout, the server
+    analyzes the agent's state trajectory and judges pass/fail. The
+    per-rollout results are the product; there is no aggregate verdict."""
+    id: str
+    status: str
+    rollout_ids: list[str]
+    error: str | None
+    _http: httpx.Client
+
+    def wait(self, poll_interval: float = 2.0, timeout: float = 1800.0,
+             verbose: bool = False) -> "QualityCheck":
+        """Poll until the check finishes; verbose prints server telemetry."""
+        events = _EventStream(
+            self._http, f"/v1/quality-checks/{self.id}/events", verbose)
+        deadline = time.monotonic() + timeout
+        while self.status == "running":
+            if time.monotonic() > deadline:
+                raise TimeoutError(
+                    f"quality check {self.id} still running after {timeout}s")
+            time.sleep(poll_interval)
+            r = self._http.get(f"/v1/quality-checks/{self.id}")
+            r.raise_for_status()
+            data = r.json()
+            self.status = data["status"]
+            self.error = data["error"]
+            events.pump()
+        events.pump()  # catch events written after the final status poll
+        if self.status != "succeeded":
+            raise RuntimeError(f"quality check {self.id} failed: {self.error}")
+        return self
+
+    def rollouts(self) -> list[dict]:
+        """Per-rollout results: rollout_id, passed, states (the agentic-state
+        trajectory), and judge (dimensions + issues)."""
+        r = self._http.get(f"/v1/quality-checks/{self.id}/rollouts")
+        r.raise_for_status()
+        return r.json()["data"]
+
+
+@dataclass
+class RolloutResult:
+    """One finished rollout: the conversation a scenario produced, plus every
+    tool call the agent made along the way (each tagged with its turn_idx)."""
+    rollout_id: str
+    scenario_id: str
+    status: str
+    turns: int
+    transcript: list[dict]
+    tool_events: list[dict]
+
+
+class Rollouts:
+    def __init__(self, http: httpx.Client, session_id: str | None = None):
+        self._http = http
+        self._session_id = session_id
+
+    def get(self, rollout_id: str) -> dict:
+        """A stored rollout's full captured state: status, seed, transcript,
+        tool events, and sandbox."""
+        r = self._http.get(f"/v1/rollouts/{rollout_id}")
+        r.raise_for_status()
+        return r.json()
+
+    def run(self, agent: RolloutAgent, dataset: Union[Dataset, str, None] = None,
+            *, max_turns: int = 12, concurrency: int = 4) -> list[RolloutResult]:
+        """Play a dataset's scenarios against `agent` (most recent dataset
+        when none is given).
+
+        The agent runs locally: each turn it gets the transcript so far and
+        a deterministic ToolSandbox for its tool calls; only its reply and
+        tool events travel over the wire. Scenarios run on `concurrency`
+        worker threads (turns within one conversation are sequential), so
+        the agent must be thread-safe — pass concurrency=1 to opt out.
+        """
+        if dataset is None:
+            # Session-scoped default: this script's latest dataset, so two
+            # concurrent scripts never pick up each other's data.
+            data = []
+            if self._session_id:
+                r = self._http.get("/v1/datasets",
+                                   params={"sdk_session": self._session_id})
+                r.raise_for_status()
+                data = r.json()["data"]
+            if not data:
+                r = self._http.get("/v1/datasets")
+                r.raise_for_status()
+                data = r.json()["data"]
+                if data and self._session_id:
+                    print(f"note: no dataset in this session yet; "
+                          f"using latest dataset {data[0]['id']}")
+            if not data:
+                raise RuntimeError("no datasets exist yet; generate one first")
+            dataset_id = data[0]["id"]
+        else:
+            dataset_id = dataset.id if isinstance(dataset, Dataset) else dataset
+        r = self._http.get(f"/v1/datasets/{dataset_id}/rows")
+        r.raise_for_status()
+        rows = r.json()["data"]
+        with ThreadPoolExecutor(max_workers=concurrency) as pool:
+            return list(pool.map(
+                lambda row: self.run_scenario(
+                    agent, row["scenario_id"], max_turns=max_turns),
+                rows))
+
+    def quality_check(
+            self, rollouts: "list[Union[RolloutResult, str]]") -> QualityCheck:
+        """Start an async quality check over finished rollouts: the server
+        analyzes each rollout's agentic states in parallel and judges
+        whether the agent passed each scenario."""
+        rollout_ids = [r.rollout_id if isinstance(r, RolloutResult) else r
+                       for r in rollouts]
+        r = self._http.post("/v1/quality-checks",
+                            json={"rollout_ids": rollout_ids})
+        r.raise_for_status()
+        return QualityCheck(**r.json(), _http=self._http)
+
+    def run_scenario(self, agent: RolloutAgent, scenario_id: str, *,
+                     max_turns: int = 12,
+                     random_seed: int | None = None) -> RolloutResult:
+        """Run one rollout session; one HTTP round-trip per agent turn."""
+        r = self._http.post("/v1/rollouts", json={
+            "scenario_id": scenario_id, "random_seed": random_seed,
+            "max_turns": max_turns,
+        })
+        r.raise_for_status()
+        session = r.json()
+        while session["status"] == "running":
+            sandbox = ToolSandbox.from_config(session["sandbox"])
+            reply = agent(session["transcript"], sandbox)
+            r = self._http.post(
+                f"/v1/rollouts/{session['id']}/turns",
+                json={"reply": reply, "tool_calls": sandbox.events},
+            )
+            r.raise_for_status()
+            session = r.json()
+        return RolloutResult(
+            rollout_id=session["id"], scenario_id=scenario_id,
+            status=session["status"], turns=session["turn"],
+            transcript=session["transcript"],
+            tool_events=session["tool_events"])
+
+
+class Synthia:
+    """Client entry point.
+
+    Session identity: every client belongs to a named session — the stable,
+    account-scoped identity of one script, persisted across executions
+    (same name resumes the same session; re-runs reuse its datasets instead
+    of re-probing/re-generating). Resolution order: `session` arg >
+    SYNTHIA_SESSION env var > derived "project/script" name from the entry
+    point. `session=False` opts out into a fresh ephemeral session.
+
+    Degradation: an old server without /v1/sdk-sessions -> no session
+    (today's behavior); keyless against a keyed server -> anonymous session;
+    an invalid api_key fails immediately with the server's message.
+    """
+
+    def __init__(self, api_key: str | None = None, base_url: str | None = None,
+                 session: Union[str, bool, None] = None):
+        api_key = api_key or os.environ.get("SYNTHIA_API_KEY")
+        base_url = base_url or os.environ.get("SYNTHIA_BASE_URL", DEFAULT_BASE_URL)
+        headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        # Generous timeout: probe convergence runs model inference server-side.
+        self._http = httpx.Client(base_url=base_url, headers=headers, timeout=300.0)
+        if session is False:
+            self.session_name = (f"{_default_session_name()}"
+                                 f"/eph-{uuid.uuid4().hex[:8]}")
+        elif isinstance(session, str) and session:
+            self.session_name = session
+        else:
+            self.session_name = (os.environ.get("SYNTHIA_SESSION")
+                                 or _default_session_name())
+        self.session_id: str | None = None
+        self.invocation_id: str | None = None
+        self._start_session()
+        self.seeds = Seeds(self._http)
+        self.user_models = UserModels(self._http)
+        self.datasets = Datasets(self._http)
+        self.rollouts = Rollouts(self._http, session_id=self.session_id)
+
+    def _start_session(self) -> None:
+        """One handshake per process: get-or-create the named session and
+        mint this invocation; all later requests carry both ids as headers."""
+        try:
+            r = self._http.post("/v1/sdk-sessions", json={
+                "name": self.session_name, "sdk_version": _sdk_version(),
+            })
+        except httpx.HTTPError:
+            return  # transient network trouble: run without session tracking
+        if r.status_code == 404:
+            return  # server predates sessions: degrade to old behavior
+        if r.status_code == 401:
+            raise RuntimeError(r.json().get("detail", "invalid API key"))
+        r.raise_for_status()
+        data = r.json()
+        self.session_id = data["sdk_session_id"]
+        self.invocation_id = data["sdk_invocation_id"]
+        self._http.headers["X-Synthia-Session"] = self.session_id
+        self._http.headers["X-Synthia-Invocation"] = self.invocation_id
+
+    def prepare(self, agent: Agent, *, count: int = 20, max_turns: int = 10,
+                min_success_rate: float = 0.6, max_success_rate: float = 0.9,
+                verbose: bool = False) -> PrepareResult:
+        """Probe + generate only when needed; otherwise reuse the latest dataset.
+
+        The main entry point for the probe and generation steps. `count` is
+        exact: the returned dataset has exactly that many rows, so reuse
+        requires the latest dataset to match it in addition to the quality
+        gate. Probing and generation run only when no dataset exists yet,
+        when the row count differs (generation-only — the session's probed
+        user model is reused), or when the latest completed quality check's
+        pass rate falls outside [min_success_rate, max_success_rate].
+        Out-of-band regeneration passes that quality check to the server,
+        which feeds its real results (pass rate, per-scenario outcomes,
+        judge issues) into scenario generation so the new batch recalibrates
+        difficulty and coverage.
+
+        All lookups are scoped to this client's session: re-running the
+        same script reuses its own dataset, and drift signals from other
+        scripts/sessions never trigger regeneration here.
+        """
+        existing = (self.datasets.list(session=self.session_id)
+                    if self.session_id else self.datasets.list())  # newest first
+        if not existing:
+            return self._probe_and_generate(
+                agent, count=count, max_turns=max_turns, verbose=verbose,
+                reason=("no datasets in this session yet" if self.session_id
+                        else "no datasets exist yet"),
+                success_rate=None, quality_check_id=None)
+
+        r = self._http.get(
+            "/v1/quality-checks/latest",
+            params={"sdk_session": self.session_id} if self.session_id else {})
+        r.raise_for_status()
+        latest = r.json()
+        rate = (latest["passed"] / latest["total"]
+                if latest["id"] is not None and latest["total"] > 0 else None)
+
+        if rate is not None and not (min_success_rate <= rate <= max_success_rate):
+            direction = ("below" if rate < min_success_rate else "above")
+            bound = (min_success_rate if rate < min_success_rate
+                     else max_success_rate)
+            return self._probe_and_generate(
+                agent, count=count, max_turns=max_turns, verbose=verbose,
+                reason=f"success rate {rate:.0%} {direction} {bound:.0%}; "
+                       f"regenerating calibrated on {latest['id']}",
+                success_rate=rate, quality_check_id=latest["id"])
+
+        # Quality is in band (or unjudged): reuse only on an exact size
+        # match; otherwise regenerate at the requested count — without
+        # re-probing, since nothing suggests the agent changed.
+        if existing[0].row_count != count:
+            return self._probe_and_generate(
+                agent, count=count, max_turns=max_turns, verbose=verbose,
+                reason=f"latest dataset has {existing[0].row_count} rows; "
+                       f"requested {count}",
+                success_rate=rate, quality_check_id=None)
+
+        return PrepareResult(
+            dataset=existing[0],
+            user_model=self.user_models.get(existing[0].user_model_id),
+            action="reused",
+            reason=(f"success rate {rate:.0%} within "
+                    f"{min_success_rate:.0%}-{max_success_rate:.0%} band"
+                    if rate is not None else
+                    "no completed quality check to judge by; "
+                    "reusing latest dataset"),
+            success_rate=rate, quality_check_id=None)
+
+    def _probe_and_generate(self, agent: Agent, *, count: int, max_turns: int,
+                            verbose: bool, reason: str,
+                            success_rate: float | None,
+                            quality_check_id: str | None) -> PrepareResult:
+        # Without a drift signal the agent hasn't been shown to change, so a
+        # user model this session already probed is still good — skip the
+        # probe. Drift-triggered regeneration re-probes deliberately.
+        user_model = None
+        if self.session_id and not quality_check_id:
+            session_models = self.user_models.list(session=self.session_id)
+            if session_models:
+                user_model = session_models[-1]  # newest (list is oldest-first)
+                reason += "; reusing session user model (no drift signal)"
+        if user_model is None:
+            user_model = self.user_models.create_from_probe(
+                agent, max_turns=max_turns, verbose=verbose)
+        job = self.datasets.generate(user_model, count=count,
+                                     quality_check_id=quality_check_id)
+        dataset = job.wait(verbose=verbose)
+        return PrepareResult(dataset=dataset, user_model=user_model,
+                             action="generated", reason=reason,
+                             success_rate=success_rate,
+                             quality_check_id=quality_check_id)

synthiaresearch-0.0.1/uv.lock

@@ -0,0 +1,104 @@
+version = 1
+revision = 3
+requires-python = ">=3.10"
+
+[[package]]
+name = "anyio"
+version = "4.14.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
+    { name = "idna" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" },
+]
+
+[[package]]
+name = "certifi"
+version = "2026.6.17"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" },
+]
+
+[[package]]
+name = "exceptiongroup"
+version = "1.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" },
+]
+
+[[package]]
+name = "h11"
+version = "0.16.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
+]
+
+[[package]]
+name = "httpcore"
+version = "1.0.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },
+]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "certifi" },
+    { name = "httpcore" },
+    { name = "idna" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
+]
+
+[[package]]
+name = "idna"
+version = "3.18"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" },
+]
+
+[[package]]
+name = "synthia"
+version = "0.0.1"
+source = { editable = "." }
+dependencies = [
+    { name = "httpx" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "httpx", specifier = ">=0.27" }]
+
+[[package]]
+name = "typing-extensions"
+version = "4.16.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" },
+]