proofstep 0.1.0__tar.gz

proofstep-0.1.0/.gitignore

@@ -0,0 +1,57 @@
+
+__pycache__/
+.coverage
+.coverage.*
+.docker-data/
+.DS_Store
+.e2e-api.log
+.env
+.env.*
+.env.prod
+# Exceptions, and they must come *after* the patterns above: git takes the last matching rule, so a
+# negation written earlier in the file is silently overridden. That is not a hypothetical — the `!`
+# line used to sit at the top, `.env.*` re-ignored it, and `.env.prod.example` was never committed.
+# `scripts/init_secrets.sh` reads that file, so the first documented step of self-hosting failed for
+# anyone who cloned the repository. It was caught by CI running the same step.
+!.env.example
+!.env.prod.example
+.hypothesis/
+.idea/
+.mypy_cache/
+.next/
+.proofstep/
+.pytest_cache/
+.ruff_cache/
+.turbo/
+.venv/
+.vscode/
+*.egg-info/
+*.key
+*.pem
+*.proofstep.local.yaml
+*.py[cod]
+*.swp
+*.tsbuildinfo
+# Database dumps. Never committed: they contain every tenant's data, and a backup in a git history
+# Docker volumes
+# Editors / OS
+# is a backup with no access control and no retention.
+# Node
+# Proofstep local state
+# Python
+# Reports a local run drops in the working tree. The name comes from the suite, so the pattern has
+# Secrets and local config — never commit these
+# The e2e stack's server log, written next to the repo so a CI failure can print it.
+# to cover all of them rather than the default filename only.
+~/.proofstep/
+backups/
+build/
+coverage.xml
+credentials.json
+dist/
+htmlcov/
+node_modules/
+out/
+proofstep-*.json
+secrets/
+venv/

proofstep-0.1.0/PKG-INFO

@@ -0,0 +1,58 @@
+Metadata-Version: 2.5
+Name: proofstep
+Version: 0.1.0
+Summary: Proofstep Python SDK — tracing and evaluation for AI applications and agents
+Project-URL: Homepage, https://github.com/IlaKhan17/proofstep
+Project-URL: Documentation, https://github.com/IlaKhan17/proofstep/tree/main/docs
+Project-URL: Repository, https://github.com/IlaKhan17/proofstep
+Project-URL: Issues, https://github.com/IlaKhan17/proofstep/issues
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27
+Requires-Dist: proofstep-core
+Requires-Dist: proofstep-types
+Requires-Dist: pydantic>=2.9
+Provides-Extra: eval
+Requires-Dist: proofstep-trajectory; extra == 'eval'
+Description-Content-Type: text/markdown
+
+# proofstep
+
+**Tracing SDK** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+Instrument an AI application or agent, and send the trace to a Proofstep server.
+
+```python
+import proofstep
+
+proofstep.init(endpoint="https://proofstep.internal", api_key="ps_prod_…")
+
+with proofstep.capture("outbound") as captured:
+    proofstep.set_state(unsubscribed=False)
+    with proofstep.start_span("gmail.send", span_type="tool", tool_name="gmail.send") as span:
+        span.set_args({"to": recipient, "thread_id": thread})
+```
+
+`set_args` is not decoration: trajectory policies match on `args.*`, so a check whose result never
+reaches the trace cannot be audited later.
+
+Secrets are redacted in this process, before export. Access tokens, refresh tokens, API keys,
+passwords, session cookies, and `Authorization` headers are never intentionally stored.
+
+Install the evaluation extras with `pip install "proofstep[eval]"`, or the CLI with
+`pip install proofstep-cli`.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep-0.1.0/README.md

@@ -0,0 +1,32 @@
+# proofstep
+
+**Tracing SDK** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+Instrument an AI application or agent, and send the trace to a Proofstep server.
+
+```python
+import proofstep
+
+proofstep.init(endpoint="https://proofstep.internal", api_key="ps_prod_…")
+
+with proofstep.capture("outbound") as captured:
+    proofstep.set_state(unsubscribed=False)
+    with proofstep.start_span("gmail.send", span_type="tool", tool_name="gmail.send") as span:
+        span.set_args({"to": recipient, "thread_id": thread})
+```
+
+`set_args` is not decoration: trajectory policies match on `args.*`, so a check whose result never
+reaches the trace cannot be audited later.
+
+Secrets are redacted in this process, before export. Access tokens, refresh tokens, API keys,
+passwords, session cookies, and `Authorization` headers are never intentionally stored.
+
+Install the evaluation extras with `pip install "proofstep[eval]"`, or the CLI with
+`pip install proofstep-cli`.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "proofstep"
+version = "0.1.0"
+description = "Proofstep Python SDK — tracing and evaluation for AI applications and agents"
+readme = "README.md"
+# Deliberately wider than the rest of the workspace: this installs into *users'*
+# applications, and forcing an interpreter upgrade to adopt a tracing library
+# is a non-starter. See ADR-002.
+requires-python = ">=3.11"
+license = "Apache-2.0"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Quality Assurance",
+    "Typing :: Typed",
+]
+
+dependencies = [
+    "proofstep-types",
+    # For the redaction pipeline, which the server shares — see proofstep/redaction.py.
+    "proofstep-core",
+    "httpx>=0.27",
+    "pydantic>=2.9",
+]
+
+[project.optional-dependencies]
+# `proofstep-core` used to be here; it is a hard dependency now (redaction lives in it), so the
+# extra is down to the trajectory engine. Kept named `eval` so `pip install proofstep[eval]`
+# keeps working for anyone who already wrote it.
+eval = ["proofstep-trajectory"]
+
+[project.urls]
+Homepage = "https://github.com/IlaKhan17/proofstep"
+Documentation = "https://github.com/IlaKhan17/proofstep/tree/main/docs"
+Repository = "https://github.com/IlaKhan17/proofstep"
+Issues = "https://github.com/IlaKhan17/proofstep/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/proofstep"]
+
+[tool.uv.sources]
+proofstep-types = { workspace = true }
+proofstep-core = { workspace = true }
+proofstep-trajectory = { workspace = true }

proofstep-0.1.0/src/proofstep/__init__.py

@@ -0,0 +1,192 @@
+"""Proofstep Python SDK — tracing for AI applications and tool-using agents.
+
+    import proofstep
+
+    proofstep.init(project="my-app")
+
+    @proofstep.trace("generate_outreach")
+    async def generate_outreach(prospect_id: str) -> Email:
+        ...
+
+    @proofstep.tool("gmail.send")
+    async def send_email(to: str, subject: str, body: str) -> str:
+        ...
+
+Two guarantees hold everywhere in this package:
+
+- **It never raises into your application.** Every public entry point is wrapped;
+  internal failures are logged once per window and return a no-op.
+- **It never blocks your application.** Export is a non-blocking enqueue onto a
+  bounded buffer drained by a background thread. When the buffer is full it drops
+  the oldest trace and counts it, because visible loss beats an invisible stall.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Iterator
+from importlib import metadata as _metadata
+from typing import Any
+
+from proofstep import redaction
+from proofstep.client import Captured, Client, sampled
+from proofstep.config import Config
+from proofstep.context import current_span, current_trace, propagate
+from proofstep.decorators import make_span, make_tool, make_trace
+from proofstep.propagation import extract, inject
+from proofstep.recorder import SpanRecorder, TraceRecorder
+from proofstep.safety import NOOP
+from proofstep_types import CaptureMode, SpanType, Status, Trace
+
+# Read from the installed distribution rather than written here twice. A hand-maintained
+# copy drifts the first time a release bumps one and not the other — which it already did,
+# reporting 0.1.0.dev0 from a 0.1.0 wheel.
+__version__ = _metadata.version("proofstep")
+
+_client: Client | None = None
+
+
+def init(**settings: Any) -> Client:
+    """Configure the SDK. Safe to call more than once; the last call wins."""
+    global _client  # noqa: PLW0603 — one process-wide client is the intended shape
+    _client = Client(Config.from_env(**settings))
+    return _client
+
+
+def get_client() -> Client:
+    """The active client, created from the environment on first use.
+
+    Implicit initialization is deliberate: an unconfigured import must still work,
+    so that adding a decorator never breaks a script that has not called `init`.
+    """
+    global _client  # noqa: PLW0603
+    if _client is None:
+        _client = Client(Config.from_env())
+    return _client
+
+
+def configure(**settings: Any) -> None:
+    """Update settings on the existing client without replacing it."""
+    client = get_client()
+    for key, value in settings.items():
+        if not hasattr(client.config, key):
+            msg = f"unknown Proofstep setting {key!r}"
+            raise TypeError(msg)
+        setattr(client.config, key, value)
+
+
+def reset() -> None:
+    """Drop the active client. For tests."""
+    global _client  # noqa: PLW0603
+    if _client is not None:
+        _client.shutdown(0.1)
+    _client = None
+
+
+trace = make_trace(get_client)
+span = make_span(get_client)
+tool = make_tool(span)
+
+
+@contextlib.contextmanager
+def start_trace(name: str, **kwargs: Any) -> Iterator[Any]:
+    """Context-manager form of `@trace`."""
+    with get_client().trace(name, **kwargs) as recorder:
+        yield recorder
+
+
+@contextlib.contextmanager
+def start_span(name: str, **kwargs: Any) -> Iterator[Any]:
+    """Context-manager form of `@span`."""
+    with get_client().span(name, **kwargs) as recorder:
+        yield recorder
+
+
+@contextlib.contextmanager
+def capture(name: str = "task", **kwargs: Any) -> Iterator[list[Trace]]:
+    """Record a trace and hand it back rather than only exporting it.
+
+    This is how an instrumented task feeds the local evaluation engine: the captured
+    `Trace` is what trajectory policies are evaluated against.
+
+        with proofstep.capture("classify") as captured:
+            result = await classify(example.input)
+        return proofstep.Captured(output=result, trace=captured[0])
+    """
+    sink: list[Trace] = []
+    with get_client().trace(name, **kwargs) as recorder:
+        yield sink
+        if recorder is not NOOP:
+            sink.append(recorder.snapshot())
+
+
+def set_metadata(**values: Any) -> None:
+    if (active := current_trace()) is not None:
+        active.set_metadata(**values)
+
+
+def set_tags(**values: str) -> None:
+    if (active := current_trace()) is not None:
+        active.set_tags(**values)
+
+
+def set_state(**values: Any) -> None:
+    """Record explicit workflow state for `final_state` policy rules."""
+    if (active := current_trace()) is not None:
+        active.set_state(**values)
+
+
+def record_event(name: str, **attributes: Any) -> None:
+    if (active := current_span()) is not None:
+        active.record_event(name, **attributes)
+
+
+def set_attributes(**values: Any) -> None:
+    if (active := current_span()) is not None:
+        active.set_attributes(**values)
+
+
+def flush(timeout: float | None = None) -> bool:
+    return bool(get_client().flush(timeout))
+
+
+def shutdown(timeout: float | None = None) -> None:
+    get_client().shutdown(timeout)
+
+
+__all__ = [
+    "NOOP",
+    "CaptureMode",
+    "Captured",
+    "Client",
+    "Config",
+    "SpanRecorder",
+    "SpanType",
+    "Status",
+    "Trace",
+    "TraceRecorder",
+    "capture",
+    "configure",
+    "current_span",
+    "current_trace",
+    "extract",
+    "flush",
+    "get_client",
+    "init",
+    "inject",
+    "propagate",
+    "record_event",
+    "redaction",
+    "reset",
+    "sampled",
+    "set_attributes",
+    "set_metadata",
+    "set_state",
+    "set_tags",
+    "shutdown",
+    "span",
+    "start_span",
+    "start_trace",
+    "tool",
+    "trace",
+]

proofstep-0.1.0/src/proofstep/client.py

@@ -0,0 +1,209 @@
+"""The client: owns configuration, sampling, the exporter, and span creation."""
+
+from __future__ import annotations
+
+import atexit
+import contextlib
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Any
+
+from proofstep import context as ctx
+from proofstep.config import Config
+from proofstep.exporter import Exporter
+from proofstep.recorder import SpanRecorder, TraceRecorder, new_trace_id
+from proofstep.safety import NOOP, log_once, never_raises
+from proofstep_types import SpanType, Status, Trace
+
+
+@dataclass(frozen=True, slots=True)
+class Captured:
+    """A task's return value paired with the trace it produced.
+
+    Shaped to match what the evaluation runner already looks for (`.output` and
+    `.trace`), so an instrumented task drops into a suite with no adapter and no
+    dependency from the engine back to the SDK.
+    """
+
+    output: Any
+    trace: Trace
+
+
+def sampled(trace_id: str, rate: float) -> bool:
+    """Deterministic head sampling on the trace id.
+
+    Hash-mod rather than a coin flip per span, so a sampled trace is captured
+    *whole*. A half-recorded trajectory is worse than none: the policy engine would
+    read the gaps as evidence.
+    """
+    if rate >= 1.0:
+        return True
+    if rate <= 0.0:
+        return False
+    return (int(trace_id[:8], 16) / 0xFFFFFFFF) < rate
+
+
+class Client:
+    def __init__(
+        self, config: Config | None = None, *, transport: Any = None, **overrides: object
+    ) -> None:
+        self.config = config or Config.from_env(**overrides)
+        self.exporter = Exporter(self.config, transport=transport)
+        self._atexit_registered = False
+
+    # ------------------------------------------------------------------- tracing
+
+    @never_raises(default=NOOP)
+    def start_trace(
+        self,
+        name: str,
+        *,
+        trace_id: str | None = None,
+        parent_span_id: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> TraceRecorder | Any:
+        if not self.config.records:
+            return NOOP
+        identifier = trace_id or new_trace_id()
+        recorder = TraceRecorder(
+            name,
+            config=self.config,
+            trace_id=identifier,
+            parent_span_id=parent_span_id,
+            sampled=sampled(identifier, self.config.sample_rate),
+        )
+        if metadata:
+            recorder.set_metadata(**metadata)
+        self._register_atexit()
+        return recorder
+
+    @never_raises(default=NOOP)
+    def start_span(
+        self,
+        name: str,
+        *,
+        span_type: SpanType | str = SpanType.CUSTOM,
+        tool_name: str | None = None,
+        trace: TraceRecorder | None = None,
+        parent: SpanRecorder | None = None,
+    ) -> SpanRecorder | Any:
+        if not self.config.records:
+            return NOOP
+
+        active_trace = trace or ctx.current_trace()
+        if active_trace is None:
+            # An orphan span is more confusing than a synthetic root, and losing it
+            # entirely is worse than both. Create a trace so the span has somewhere
+            # to live, then emit it when the span closes.
+            #
+            # The common cause is a raw ThreadPoolExecutor: Python does not copy
+            # contextvars across threads, so the worker sees no active trace. Say so,
+            # because the fix (`proofstep.propagate`) is not discoverable otherwise.
+            log_once(
+                "client.orphan_span",
+                f"span {name!r} was created with no active trace and has been recorded "
+                "as its own trace. If this is a thread, wrap the callable with "
+                "proofstep.propagate() to keep it attached to its parent.",
+            )
+            active_trace = self.start_trace(name)
+            if active_trace is NOOP:
+                return NOOP
+            ctx.set_trace(active_trace)
+
+        active_parent = parent or ctx.current_span()
+        recorder = SpanRecorder(
+            name,
+            trace=active_trace,
+            span_type=SpanType(span_type),
+            parent_span_id=active_parent.span_id if active_parent else None,
+            tool_name=tool_name,
+            depth=(active_parent.depth + 1) if active_parent else 0,
+        )
+        if not active_trace.register(recorder):
+            return NOOP
+        return recorder
+
+    @contextlib.contextmanager
+    def trace(self, name: str, **kwargs: Any) -> Iterator[Any]:
+        recorder = self.start_trace(name, **kwargs)
+        if recorder is NOOP:
+            yield NOOP
+            return
+
+        token = ctx.set_trace(recorder)
+        try:
+            yield recorder
+        except BaseException as exc:
+            recorder.status = Status.ERROR
+            recorder.set_metadata(error=f"{type(exc).__name__}: {exc}"[:500])
+            raise
+        finally:
+            recorder.end()
+            ctx.reset_trace(token)
+            self.emit(recorder)
+
+    @contextlib.contextmanager
+    def span(self, name: str, **kwargs: Any) -> Iterator[Any]:
+        had_trace = ctx.current_trace() is not None
+        recorder = self.start_span(name, **kwargs)
+        if recorder is NOOP:
+            yield NOOP
+            return
+
+        token = ctx.set_span(recorder)
+        try:
+            yield recorder
+        except BaseException as exc:
+            # Record and re-raise, untouched. The traceback the user sees must be
+            # exactly the one their code produced.
+            if isinstance(exc, Exception):
+                recorder.set_error(exc)
+            else:
+                recorder.status = Status.ERROR
+            raise
+        finally:
+            recorder.end()
+            ctx.reset_span(token)
+            if not had_trace:
+                self._close_implicit_trace()
+
+    def _close_implicit_trace(self) -> None:
+        """Emit a trace `start_span` created on the caller's behalf.
+
+        Without this the span is recorded into a trace nobody ever ends, and the
+        data vanishes silently — the exact failure mode the SDK promises to avoid.
+        """
+        implicit = ctx.current_trace()
+        if implicit is None:
+            return
+        implicit.end()
+        self.emit(implicit)
+        ctx.set_trace(None)
+
+    # -------------------------------------------------------------------- export
+
+    @never_raises()
+    def emit(self, recorder: TraceRecorder) -> None:
+        """Finish a trace: snapshot it and hand it to the exporter."""
+        if not self.config.records:
+            return
+        keep = recorder.sampled or (
+            self.config.always_sample_on_error and recorder.status is Status.ERROR
+        )
+        if not keep:
+            return
+        self.exporter.submit(recorder.snapshot())
+
+    @never_raises(default=False)
+    def flush(self, timeout: float | None = None) -> bool:
+        return bool(self.exporter.flush(timeout))
+
+    @never_raises()
+    def shutdown(self, timeout: float | None = None) -> None:
+        self.exporter.shutdown(timeout)
+
+    def _register_atexit(self) -> None:
+        if self._atexit_registered:
+            return
+        self._atexit_registered = True
+        atexit.register(self.shutdown, self.config.shutdown_timeout_s)