runapprentice 0.4.0__tar.gz → 0.4.1__tar.gz

{runapprentice-0.4.0 → runapprentice-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runapprentice
-Version: 0.4.0
+Version: 0.4.1
 Summary: Cut LLM cost without losing quality: capture traffic, build golden datasets, and optimize prompts.
 Project-URL: Homepage, https://runapprentice.com
 Project-URL: Documentation, https://docs.runapprentice.com

{runapprentice-0.4.0 → runapprentice-0.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runapprentice"
-version = "0.4.0"
+version = "0.4.1"
 description = "Cut LLM cost without losing quality: capture traffic, build golden datasets, and optimize prompts."
 readme = "README.md"
 requires-python = ">=3.11"

{runapprentice-0.4.0 → runapprentice-0.4.1}/src/runapprentice/client.py

@@ -16,6 +16,7 @@ import os
 import time
 import uuid
 from pathlib import Path
+from urllib.parse import quote
 from typing import Any, Literal
 
 import httpx
@@ -164,18 +165,18 @@ class Apprentice:
         return Job(self, JobInfo.model_validate(data))
 
     def jobs(self, task: str) -> list[JobSummary]:
-        data = self._request("GET", f"/v1/tasks/{task}/jobs")
+        data = self._request("GET", f"/v1/tasks/{quote(task, safe='')}/jobs")
         return [JobSummary.model_validate(item) for item in data]
 
     def cancel_job(self, job_id: str) -> dict[str, Any]:
         return dict(self._request("POST", f"/v1/jobs/{job_id}/cancel"))
 
     def local_runs(self, task: str) -> list[LocalRun]:
-        data = self._request("GET", f"/v1/tasks/{task}/local-runs")
+        data = self._request("GET", f"/v1/tasks/{quote(task, safe='')}/local-runs")
         return [LocalRun.model_validate(item) for item in data]
 
     def train(self, task: str) -> Job:
-        data = self._request("POST", f"/v1/tasks/{task}/train", json={})
+        data = self._request("POST", f"/v1/tasks/{quote(task, safe='')}/train", json={})
         return Job(self, JobInfo.model_validate(data))
 
     def optimize_local(
@@ -441,13 +442,13 @@ class _DatasetsAPI:
         payload_rows = [_dataset_row_payload(r, input_col, output_col) for r in rows]
         data = self._client._request(
             "POST",
-            f"/v1/datasets/{task}/rows",
+            f"/v1/datasets/{quote(task, safe='')}/rows",
             json={"rows": payload_rows, "prompt": prompt},
         )
         return DatasetStatus.model_validate(data)
 
     def status(self, task: str) -> DatasetStatus:
-        data = self._client._request("GET", f"/v1/datasets/{task}/status")
+        data = self._client._request("GET", f"/v1/datasets/{quote(task, safe='')}/status")
         return DatasetStatus.model_validate(data)
 
     def rows(self, task: str, tier: str, limit: int = 200) -> list[DatasetRow]:
@@ -456,7 +457,7 @@ class _DatasetsAPI:
         while True:
             data = self._client._request(
                 "GET",
-                f"/v1/datasets/{task}/rows?tier={tier}&limit={limit}&offset={offset}",
+                f"/v1/datasets/{quote(task, safe='')}/rows?tier={tier}&limit={limit}&offset={offset}",
             )
             batch = [DatasetRow.model_validate(item) for item in data.get("rows", [])]
             rows.extend(batch)
@@ -481,21 +482,21 @@ class _PromptsAPI:
 
     def register(self, task: str, template: Any) -> dict[str, Any]:
         artifact = template if isinstance(template, dict) else _serialize_langchain_prompt(template)
-        return dict(self._client._request("POST", f"/v1/templates/{task}", json=artifact))
+        return dict(self._client._request("POST", f"/v1/templates/{quote(task, safe='')}", json=artifact))
 
     def to_langchain(self, task: str | None = None, artifact: dict[str, Any] | None = None) -> Any:
         if artifact is None:
             if task is None:
                 raise ApprenticeError("provide task= or artifact=")
-            artifact = dict(self._client._request("GET", f"/v1/templates/{task}"))
+            artifact = dict(self._client._request("GET", f"/v1/templates/{quote(task, safe='')}"))
         return _artifact_to_langchain(artifact)
 
     def get(self, task: str, version: int | None = None) -> PromptVersion:
-        path = f"/v1/prompts/{task}" + (f"?version={version}" if version is not None else "")
+        path = f"/v1/prompts/{quote(task, safe='')}" + (f"?version={version}" if version is not None else "")
         return PromptVersion.model_validate(self._client._request("GET", path))
 
     def history(self, task: str) -> list[PromptVersion]:
-        data = self._client._request("GET", f"/v1/prompts/{task}/history")
+        data = self._client._request("GET", f"/v1/prompts/{quote(task, safe='')}/history")
         return [PromptVersion.model_validate(p) for p in data]
 
 

{runapprentice-0.4.0 → runapprentice-0.4.1}/src/runapprentice/models.py

@@ -56,6 +56,9 @@ class HoldoutEvidenceRow(BaseModel):
     baseline_score: float | None = None
     optimized_output: str | None = None
     optimized_score: float | None = None
+    # Set when the row's 0 came from an evaluation error, not a wrong answer.
+    baseline_error: str | None = None
+    optimized_error: str | None = None
 
 
 class OptimizationReport(BaseModel):

{runapprentice-0.4.0 → runapprentice-0.4.1}/src/runapprentice/openai.py

@@ -49,8 +49,10 @@ def wrap(openai_client: Any, client: Apprentice, task: str, redact: RedactFn | N
     helper and posts on its own rather than calling `create`.
 
     Returns the same client (mutated in place) for convenient one-line use.
-    Idempotent: wrapping an already-wrapped client is a no-op. Supports both sync
-    (`OpenAI`) and async (`AsyncOpenAI`) clients.
+    Re-wrapping the same client for the same task is a no-op; re-wrapping it for a
+    different task or a different Apprentice client rebinds it, so reusing one
+    OpenAI client across tasks sends each task's traffic to its own dataset.
+    Supports both sync (`OpenAI`) and async (`AsyncOpenAI`) clients.
     """
     chat = getattr(openai_client, "chat", None)
     resources = (
@@ -87,10 +89,13 @@ def _wrap_method(
     input_tokens_field: str,
     output_tokens_field: str,
 ) -> None:
-    flag = f"_apprentice_wrapped_{method_name}"
-    original = getattr(resource, method_name)
-    if getattr(original, "_apprentice_wrapped", False) or getattr(resource, flag, False):
+    current = getattr(resource, method_name)
+    binding = (client, task)
+    if getattr(current, "_apprentice_binding", None) == binding:
         return
+    # Already wrapped for a different task or client: rebind by wrapping the real
+    # method again, never a wrapper. Otherwise the first task keeps every trace.
+    original = getattr(current, "_apprentice_original", current)
 
     def _capture(response: Any, kwargs: dict[str, Any], args: tuple[Any, ...], t0: float) -> None:
         try:
@@ -135,9 +140,9 @@ def _wrap_method(
                 _capture(response, kwargs, args, t0)
             return response
 
-    wrapped._apprentice_wrapped = True  # type: ignore[attr-defined]
+    wrapped._apprentice_original = original  # type: ignore[attr-defined]
+    wrapped._apprentice_binding = binding  # type: ignore[attr-defined]
     setattr(resource, method_name, wrapped)
-    setattr(resource, flag, True)
 
 
 def _messages_to_text(messages: Any) -> str:

{runapprentice-0.4.0 → runapprentice-0.4.1}/tests/test_client.py

@@ -283,6 +283,9 @@ def test_optimization_report_round_trips_every_api_field() -> None:
                 "baseline_score": 0.0,
                 "optimized_output": '{"total": 10}',
                 "optimized_score": 1.0,
+                # Distinguishes a 0 from a wrong answer and a 0 from a crashed row.
+                "baseline_error": None,
+                "optimized_error": "TimeoutError: judge call timed out",
             }
         ],
         "improved_count": 1,
@@ -774,3 +777,21 @@ def test_messages_needs_no_keywords_when_partials_fill_everything() -> None:
 def test_messages_without_an_artifact_still_demands_inputs() -> None:
     with pytest.raises(ValueError, match="keyword arguments"):
         _prompt_version(artifact=None).messages()
+
+
+def test_task_names_with_url_delimiters_are_quoted() -> None:
+    """The API accepts '/' and '?' in task names (main.py validates length only),
+    so an unquoted name reshapes the request path and the task becomes
+    unreachable from the SDK. httpx decodes `.path`, so assert on the raw target."""
+    seen: list[bytes] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(request.url.raw_path)
+        return httpx.Response(200, json={"task": "a/b?c", "gold": 0, "silver": 0, "raw": 0})
+
+    client = Apprentice(api_key="test", base_url="http://mock", capture_async=False)
+    client._http = httpx.Client(base_url="http://mock", transport=httpx.MockTransport(handler))
+
+    client.datasets.status("a/b?c")
+
+    assert seen[0] == b"/v1/datasets/a%2Fb%3Fc/status"

{runapprentice-0.4.0 → runapprentice-0.4.1}/tests/test_openai.py

@@ -106,11 +106,15 @@ def test_wrap_captures_and_returns_response() -> None:
 
 
 def test_wrap_is_idempotent() -> None:
+    """Same Apprentice client and same task: the wrapper is left alone. A
+    different client or task rebinds instead, so traces follow what was asked
+    for (see test_rewrapping_for_a_new_task_reroutes_traces)."""
     oa = _FakeOpenAI()
-    wrap(oa, _client(), task="t")
+    client = _client()
+    wrap(oa, client, task="t")
     chat_once = oa.chat.completions.create
     responses_once = oa.responses.create
-    wrap(oa, _client(), task="t")
+    wrap(oa, client, task="t")
     assert oa.chat.completions.create is chat_once
     assert oa.responses.create is responses_once
 
@@ -252,3 +256,33 @@ def test_resource_without_parse_still_wraps(monkeypatch: Any) -> None:
     oa.responses.create(model="gpt-5-mini", input="x")
 
     assert len(captured) == 1
+
+
+def test_rewrapping_for_a_new_task_reroutes_traces(monkeypatch: Any) -> None:
+    """Reusing one OpenAI client for a second task must not keep filing traces
+    under the first. Shipped wrong in 0.4.0: the flag made re-wrap a no-op, so
+    task B's traffic silently landed in task A's dataset."""
+    client = _client()
+    captured: list[Any] = []
+    monkeypatch.setattr(client, "post_trace_failopen", lambda record: captured.append(record))
+    oa = _FakeOpenAI()
+
+    wrap(oa, client, task="task-a")
+    oa.responses.create(model="gpt-5-mini", input="first")
+    wrap(oa, client, task="task-b")
+    oa.responses.create(model="gpt-5-mini", input="second")
+
+    assert [r.task for r in captured] == ["task-a", "task-b"]
+
+
+def test_rewrapping_for_the_same_task_still_captures_once(monkeypatch: Any) -> None:
+    client = _client()
+    captured: list[Any] = []
+    monkeypatch.setattr(client, "post_trace_failopen", lambda record: captured.append(record))
+    oa = _FakeOpenAI()
+
+    wrap(oa, client, task="t")
+    wrap(oa, client, task="t")
+    oa.responses.create(model="gpt-5-mini", input="once")
+
+    assert len(captured) == 1