runapprentice 0.1.0__py3-none-any.whl

runapprentice/__init__.py

@@ -0,0 +1,26 @@
+"""Apprentice SDK: capture LLM traffic, build golden datasets, and optimize prompts."""
+
+from runapprentice.client import Apprentice, ApprenticeError, Job, enable_debug_logging
+from runapprentice.models import (
+    DatasetRow,
+    DatasetStatus,
+    JobInfo,
+    OptimizationReport,
+    PromptVersion,
+    TraceRecord,
+)
+
+__all__ = [
+    "Apprentice",
+    "ApprenticeError",
+    "enable_debug_logging",
+    "Job",
+    "DatasetRow",
+    "DatasetStatus",
+    "JobInfo",
+    "OptimizationReport",
+    "PromptVersion",
+    "TraceRecord",
+]
+
+__version__ = "0.1.0"

runapprentice/_capture.py

@@ -0,0 +1,117 @@
+"""Background capture dispatcher.
+
+Capture runs inside the customer's production request path, so it must be both
+NON-BLOCKING (never add latency to their call) and RESILIENT (a transient network
+blip must not silently drop a trace on the first try). This dispatcher gives both:
+
+- `submit()` enqueues and returns immediately (the customer's thread never waits on
+  Apprentice's network).
+- A single daemon worker drains the queue and delivers each trace with bounded
+  retry + exponential backoff.
+- The queue is bounded; if Apprentice's backend is down long enough to fill it, the
+  oldest pending traces are dropped (fail-open — the customer app is never affected).
+- `flush()` (also run at interpreter exit) drains in-flight traces so short scripts
+  don't lose their last captures.
+"""
+
+from __future__ import annotations
+
+import atexit
+import logging
+import queue
+import random
+import threading
+import time
+from typing import Callable
+
+from runapprentice.models import TraceRecord
+
+logger = logging.getLogger("runapprentice")
+
+# send(record) -> True if delivered, False if permanently rejected (do not retry);
+# raises on a transient failure (network/5xx) that should be retried.
+SendFn = Callable[[TraceRecord], bool]
+_SHUTDOWN = object()
+
+
+class CaptureDispatcher:
+    def __init__(
+        self,
+        send: SendFn,
+        on_exhausted: Callable[[str], None],
+        max_queue: int = 1000,
+        retries: int = 2,
+        backoff_base: float = 0.5,
+    ) -> None:
+        self._send = send
+        self._on_exhausted = on_exhausted
+        self._retries = retries
+        self._backoff_base = backoff_base
+        self._queue: queue.Queue[object] = queue.Queue(maxsize=max_queue)
+        self._dropped = 0
+        self._worker = threading.Thread(
+            target=self._run, name="apprentice-capture", daemon=True
+        )
+        self._worker.start()
+        atexit.register(self.flush)
+
+    def submit(self, record: TraceRecord) -> None:
+        """Enqueue a trace without blocking. Drops the oldest if the queue is full."""
+        try:
+            self._queue.put_nowait(record)
+        except queue.Full:
+            # Backend is unreachable and the buffer is full — drop the oldest pending
+            # trace to make room for the newest, so we keep the freshest signal.
+            try:
+                self._queue.get_nowait()
+                self._queue.task_done()
+            except queue.Empty:
+                pass
+            self._dropped += 1
+            if self._dropped == 1 or self._dropped % 100 == 0:
+                logger.warning(
+                    "runapprentice: capture buffer full, dropped %d trace(s) "
+                    "(backend unreachable; app unaffected)",
+                    self._dropped,
+                )
+            try:
+                self._queue.put_nowait(record)
+            except queue.Full:
+                pass
+
+    def _run(self) -> None:
+        while True:
+            item = self._queue.get()
+            try:
+                if item is _SHUTDOWN:
+                    return
+                assert isinstance(item, TraceRecord)
+                self._deliver(item)
+            finally:
+                self._queue.task_done()
+
+    def _deliver(self, record: TraceRecord) -> None:
+        last_error: Exception | None = None
+        for attempt in range(self._retries + 1):
+            try:
+                self._send(record)  # True=delivered, False=permanent reject
+                return
+            except Exception as exc:  # noqa: BLE001 — transient; retry then give up
+                last_error = exc
+                if attempt < self._retries:
+                    time.sleep(self._backoff_base * (2**attempt) + random.uniform(0, 0.1))
+        if last_error is not None:
+            self._on_exhausted(f"capture failed after {self._retries + 1} attempts: {last_error}")
+
+    def flush(self, timeout: float = 2.0) -> None:
+        """Wait up to `timeout` seconds for in-flight traces to finish delivering.
+
+        Counts items as done only after the worker calls `task_done()` (i.e. after
+        delivery + any retries complete), not merely when they leave the queue.
+        """
+        deadline = time.monotonic() + timeout
+        while self._queue.unfinished_tasks > 0 and time.monotonic() < deadline:
+            time.sleep(0.02)
+
+    def close(self) -> None:
+        self._queue.put(_SHUTDOWN)

runapprentice/client.py

@@ -0,0 +1,391 @@
+"""Apprentice client.
+
+Two transport disciplines coexist here, by design:
+- Control-plane calls (upload, optimize, reports) RAISE on failure — the caller is a
+  developer at a terminal who needs to know.
+- Capture calls (`post_trace_failopen`) NEVER raise — they run inside the customer's
+  production request path. A dead Apprentice backend must never break a customer app.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import logging
+import os
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from runapprentice._capture import CaptureDispatcher
+from runapprentice.models import (
+    DatasetStatus,
+    JobInfo,
+    OptimizationReport,
+    PromptVersion,
+    TraceRecord,
+)
+
+logger = logging.getLogger("runapprentice")
+
+_DEFAULT_BASE_URL = "https://api.runapprentice.com"
+_DEFAULT_TIMEOUT = 10.0
+_CAPTURE_TIMEOUT = 3.0
+
+
+def enable_debug_logging(level: int = logging.DEBUG) -> None:
+    """One-liner for development: `runapprentice.enable_debug_logging()`.
+
+    Shows every API call, capture attempt, and failure reason on stderr.
+    Also enabled by setting APPRENTICE_DEBUG=1 in the environment.
+    """
+    handler = logging.StreamHandler()
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s [apprentice] %(levelname)s %(name)s: %(message)s")
+    )
+    root = logging.getLogger("runapprentice")
+    root.addHandler(handler)
+    root.setLevel(level)
+
+
+if os.environ.get("APPRENTICE_DEBUG"):  # pragma: no cover — env-dependent convenience
+    enable_debug_logging()
+
+
+class ApprenticeError(Exception):
+    """Raised for control-plane API failures."""
+
+
+class Apprentice:
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        timeout: float = _DEFAULT_TIMEOUT,
+        capture_async: bool = True,
+    ) -> None:
+        # base_url precedence: constructor arg > APPRENTICE_BASE_URL env > hosted default.
+        # Override only for self-hosting or tests; the hosted API is the default.
+        self.base_url = (
+            base_url or os.environ.get("APPRENTICE_BASE_URL") or _DEFAULT_BASE_URL
+        ).rstrip("/")
+        api_key = api_key or os.environ.get("APPRENTICE_API_KEY")
+        headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        self._http = httpx.Client(base_url=self.base_url, headers=headers, timeout=timeout)
+        # Capture defaults to async: traces are buffered and delivered off the caller's
+        # thread with retry, so a slow/down backend never adds latency to the customer's
+        # LLM call. Set capture_async=False for synchronous delivery (tests, short scripts).
+        self._dispatcher = (
+            CaptureDispatcher(self._send_trace_attempt, self._warn_capture_once)
+            if capture_async
+            else None
+        )
+        self.datasets = _DatasetsAPI(self)
+        self.tasks = _TasksAPI(self)
+        self.prompts = _PromptsAPI(self)
+
+    # -- control plane (raises on failure) ------------------------------------
+
+    def _request(self, method: str, path: str, json: Any | None = None) -> Any:
+        t0 = time.monotonic()
+        try:
+            resp = self._http.request(method, path, json=json)
+        except httpx.HTTPError as exc:
+            logger.error("%s %s failed after %.0fms: %s", method, path, _ms(t0), exc)
+            raise ApprenticeError(
+                f"{method} {path} failed: {exc}. "
+                f"Is the Apprentice backend reachable at {self.base_url}?"
+            ) from exc
+        logger.debug("%s %s -> %s (%.0fms)", method, path, resp.status_code, _ms(t0))
+        if resp.status_code >= 400:
+            logger.error("%s %s -> %s: %s", method, path, resp.status_code, resp.text[:500])
+            raise ApprenticeError(f"{method} {path} -> {resp.status_code}: {resp.text[:500]}")
+        return resp.json()
+
+    def optimize(self, task: str) -> Job:
+        data = self._request("POST", "/v1/optimize", json={"task": task})
+        return Job(self, JobInfo.model_validate(data))
+
+    def job(self, job_id: str) -> Job:
+        data = self._request("GET", f"/v1/jobs/{job_id}")
+        return Job(self, JobInfo.model_validate(data))
+
+    def feedback(
+        self,
+        trace_id: str,
+        good: bool | None = None,
+        score: float | None = None,
+        note: str | None = None,
+    ) -> None:
+        self._request(
+            "POST",
+            "/v1/feedback",
+            json={"trace_id": trace_id, "good": good, "score": score, "note": note},
+        )
+
+    # -- capture path (never raises) -------------------------------------------
+
+    def post_trace_failopen(self, record: TraceRecord) -> str | None:
+        """Capture one trace. MUST NOT raise — runs in the customer's request path.
+
+        Async mode (default): enqueues and returns None immediately; delivery happens
+        off-thread with retry. Sync mode (capture_async=False): delivers inline and
+        returns the trace_id (or None on failure).
+        """
+        if self._dispatcher is not None:
+            self._dispatcher.submit(record)
+            return None
+        try:
+            return self._deliver_trace(record)
+        except Exception as exc:  # noqa: BLE001 — fail-open is the contract here
+            self._warn_capture_once(f"capture failed: {exc}")
+            return None
+
+    def capture(
+        self,
+        task: str,
+        output: str,
+        *,
+        input: str | None = None,  # noqa: A002 - public API mirrors dataset field
+        inputs: dict[str, str] | None = None,
+        model: str | None = None,
+        latency_ms: float | None = None,
+        prompt_tokens: int | None = None,
+        completion_tokens: int | None = None,
+        error: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> str | None:
+        """Capture one completed LLM row without raising into application code.
+
+        For RAG, pass the EXACT context string sent to the model in `inputs`;
+        do not pass only the final rendered prompt:
+        `inputs={"question": question, "context": formatted_context}`.
+        """
+        try:
+            display_input = input
+            if display_input is None:
+                if inputs is None:
+                    raise ValueError("capture requires input or inputs")
+                display_input = _display_inputs(inputs)
+            return self.post_trace_failopen(
+                TraceRecord(
+                    task=task,
+                    input=display_input,
+                    inputs=inputs,
+                    output=output,
+                    model=model,
+                    latency_ms=latency_ms,
+                    prompt_tokens=prompt_tokens,
+                    completion_tokens=completion_tokens,
+                    error=error,
+                    metadata=metadata or {},
+                )
+            )
+        except Exception as exc:  # noqa: BLE001 — fail-open is the contract here
+            self._warn_capture_once(f"capture failed: {exc}")
+            return None
+
+    def flush_captures(self, timeout: float = 2.0) -> None:
+        """Block until buffered traces are delivered (or timeout). No-op in sync mode."""
+        if self._dispatcher is not None:
+            self._dispatcher.flush(timeout)
+
+    def _send_trace_attempt(self, record: TraceRecord) -> bool:
+        """One delivery attempt for the dispatcher.
+
+        Returns True on success, False on a permanent (4xx) rejection, and RAISES on a
+        transient failure (network error / 5xx) so the dispatcher retries it.
+        """
+        return self._deliver_trace(record) is not None
+
+    def _deliver_trace(self, record: TraceRecord) -> str | None:
+        """POST one trace. Raises httpx errors and ApprenticeError on 5xx (retryable);
+        returns trace_id on success, None on a 4xx rejection (not retryable)."""
+        t0 = time.monotonic()
+        resp = self._http.post(
+            "/v1/traces",
+            json=record.model_dump(mode="json"),
+            timeout=_CAPTURE_TIMEOUT,
+        )
+        if resp.status_code < 400:
+            trace_id = resp.json().get("trace_id")
+            logger.debug("captured trace task=%s id=%s (%.0fms)", record.task, trace_id, _ms(t0))
+            self._capture_warned = False
+            return str(trace_id) if trace_id is not None else None
+        if resp.status_code >= 500:
+            raise ApprenticeError(f"backend {resp.status_code}: {resp.text[:200]}")
+        self._warn_capture_once(f"backend rejected trace: {resp.status_code} {resp.text[:200]}")
+        return None
+
+    def _warn_capture_once(self, message: str) -> None:
+        if not getattr(self, "_capture_warned", False):
+            logger.warning(
+                "runapprentice: %s, traces are being DROPPED (app unaffected; "
+                "further drops logged at DEBUG)",
+                message,
+            )
+            self._capture_warned = True
+        else:
+            logger.debug("runapprentice: %s", message)
+
+    def close(self) -> None:
+        if self._dispatcher is not None:
+            self._dispatcher.flush()
+            self._dispatcher.close()
+        self._http.close()
+
+
+class Job:
+    def __init__(self, client: Apprentice, info: JobInfo) -> None:
+        self._client = client
+        self.info = info
+
+    @property
+    def job_id(self) -> str:
+        return self.info.job_id
+
+    @property
+    def status(self) -> str:
+        return self.info.status
+
+    def refresh(self) -> Job:
+        self.info = self._client.job(self.info.job_id).info
+        return self
+
+    def wait(self, poll_seconds: float = 5.0, timeout_seconds: float = 3600.0) -> Job:
+        deadline = time.monotonic() + timeout_seconds
+        while self.refresh().info.status in ("queued", "running"):
+            if time.monotonic() > deadline:
+                raise ApprenticeError(f"job {self.info.job_id} timed out after {timeout_seconds}s")
+            time.sleep(poll_seconds)
+        return self
+
+    def report(self) -> OptimizationReport:
+        if self.info.report is None:
+            raise ApprenticeError(
+                f"job {self.info.job_id} has no report (status={self.info.status})"
+            )
+        return self.info.report
+
+
+class _DatasetsAPI:
+    def __init__(self, client: Apprentice) -> None:
+        self._client = client
+
+    def upload(
+        self,
+        task: str,
+        path: str | Path | None = None,
+        rows: list[dict[str, Any]] | None = None,
+        input_col: str = "input",
+        output_col: str = "output",
+        prompt: str | None = None,
+    ) -> DatasetStatus:
+        """Upload rows.
+
+        Rows may use legacy `input`/`output` or structured `inputs`/`output`.
+        LCEL auto-capture of prompt variables is future work; v1 stays explicit.
+        """
+        if (path is None) == (rows is None):
+            raise ApprenticeError("provide exactly one of path= or rows=")
+        if path is not None:
+            with open(path, newline="", encoding="utf-8") as f:
+                rows = list(csv.DictReader(f))
+        assert rows is not None
+        payload_rows = [_dataset_row_payload(r, input_col, output_col) for r in rows]
+        data = self._client._request(
+            "POST",
+            f"/v1/datasets/{task}/rows",
+            json={"rows": payload_rows, "prompt": prompt},
+        )
+        return DatasetStatus.model_validate(data)
+
+    def status(self, task: str) -> DatasetStatus:
+        data = self._client._request("GET", f"/v1/datasets/{task}/status")
+        return DatasetStatus.model_validate(data)
+
+
+class _TasksAPI:
+    def __init__(self, client: Apprentice) -> None:
+        self._client = client
+
+    def create(self, task: str, metric: str = "auto") -> dict[str, Any]:
+        result = self._client._request("POST", "/v1/tasks", json={"name": task, "metric": metric})
+        return dict(result)
+
+
+class _PromptsAPI:
+    def __init__(self, client: Apprentice) -> None:
+        self._client = client
+
+    def register(self, task: str, template: Any) -> dict[str, Any]:
+        artifact = template if isinstance(template, dict) else _serialize_langchain_prompt(template)
+        return dict(self._client._request("POST", f"/v1/templates/{task}", json=artifact))
+
+    def to_langchain(self, task: str | None = None, artifact: dict[str, Any] | None = None) -> Any:
+        if artifact is None:
+            if task is None:
+                raise ApprenticeError("provide task= or artifact=")
+            artifact = dict(self._client._request("GET", f"/v1/templates/{task}"))
+        return _artifact_to_langchain(artifact)
+
+    def get(self, task: str, version: int | None = None) -> PromptVersion:
+        path = f"/v1/prompts/{task}" + (f"?version={version}" if version is not None else "")
+        return PromptVersion.model_validate(self._client._request("GET", path))
+
+    def history(self, task: str) -> list[PromptVersion]:
+        data = self._client._request("GET", f"/v1/prompts/{task}/history")
+        return [PromptVersion.model_validate(p) for p in data]
+
+
+def new_trace_id() -> str:
+    return uuid.uuid4().hex
+
+
+def _display_inputs(inputs: dict[str, str]) -> str:
+    return json.dumps(inputs, sort_keys=True, ensure_ascii=False)
+
+
+def _dataset_row_payload(
+    row: dict[str, Any], input_col: str, output_col: str
+) -> dict[str, Any]:
+    payload: dict[str, Any] = {"output": str(row[output_col])}
+    if "inputs" in row:
+        inputs = row["inputs"]
+        if not isinstance(inputs, dict):
+            raise ApprenticeError("row['inputs'] must be a dict")
+        payload["inputs"] = inputs
+        if input_col in row:
+            payload["input"] = str(row[input_col])
+    else:
+        payload["input"] = str(row[input_col])
+    return payload
+
+
+def _serialize_langchain_prompt(template: Any) -> dict[str, Any]:
+    try:
+        from runapprentice.langchain import serialize_prompt
+    except ImportError as exc:
+        raise ImportError(
+            "LangChain support requires the optional extra: "
+            "`uv add 'runapprentice[langchain]'` or install `langchain-core>=0.3`."
+        ) from exc
+    return serialize_prompt(template)
+
+
+def _artifact_to_langchain(artifact: dict[str, Any]) -> Any:
+    try:
+        from runapprentice.langchain import artifact_to_langchain
+    except ImportError as exc:
+        raise ImportError(
+            "LangChain support requires the optional extra: "
+            "`uv add 'runapprentice[langchain]'` or install `langchain-core>=0.3`."
+        ) from exc
+    return artifact_to_langchain(artifact)
+
+
+def _ms(t0: float) -> float:
+    return (time.monotonic() - t0) * 1000