proofstep 0.1.0__py3-none-any.whl

proofstep/recorder.py

@@ -0,0 +1,366 @@
+"""Mutable span and trace recorders that produce the frozen shared types.
+
+The recorder is what user code touches; `Span`/`Trace` from `proofstep-types` are
+the immutable snapshots it emits. Keeping those two apart is what lets the wire
+format stay frozen and validated while the in-flight object stays cheap to mutate.
+"""
+
+from __future__ import annotations
+
+import secrets
+import threading
+from datetime import UTC, datetime
+from decimal import Decimal
+from typing import TYPE_CHECKING, Any
+
+from proofstep.safety import log_once, never_raises
+from proofstep_core.redaction import RedactionPipeline
+from proofstep_types import CaptureMode, Span, SpanEvent, SpanType, Status, TokenUsage, Trace
+
+if TYPE_CHECKING:
+    from proofstep.config import Config
+
+
+def new_trace_id() -> str:
+    return secrets.token_hex(16)  # 32 hex chars, W3C-compatible
+
+
+def new_span_id() -> str:
+    return secrets.token_hex(8)  # 16 hex chars
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+class SpanRecorder:
+    """One operation in progress."""
+
+    __slots__ = (
+        "_ended",
+        "_trace",
+        "args",
+        "attributes",
+        "cost",
+        "depth",
+        "ended_at",
+        "error_type",
+        "events",
+        "input",
+        "model",
+        "name",
+        "output",
+        "parent_span_id",
+        "provider",
+        "sequence_index",
+        "span_id",
+        "span_type",
+        "started_at",
+        "status",
+        "status_message",
+        "tokens",
+        "tool_name",
+        "trace_id",
+    )
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        trace: TraceRecorder,
+        span_type: SpanType = SpanType.CUSTOM,
+        parent_span_id: str | None = None,
+        span_id: str | None = None,
+        tool_name: str | None = None,
+        depth: int = 0,
+    ) -> None:
+        self._trace = trace
+        self._ended = False
+        self.span_id = span_id or new_span_id()
+        self.trace_id = trace.trace_id
+        self.parent_span_id = parent_span_id
+        self.name = name
+        self.span_type = span_type
+        self.tool_name = tool_name
+        self.depth = depth
+        self.started_at = _now()
+        self.ended_at: datetime | None = None
+        self.status = Status.OK
+        self.status_message: str | None = None
+        self.attributes: dict[str, Any] = {}
+        self.input: Any = None
+        self.output: Any = None
+        self.args: dict[str, Any] | None = None
+        self.events: list[SpanEvent] = []
+        self.model: str | None = None
+        self.provider: str | None = None
+        self.tokens: TokenUsage | None = None
+        self.cost: Decimal | None = None
+        self.error_type: str | None = None
+        self.sequence_index = trace.next_sequence()
+
+    # ------------------------------------------------------------------- user API
+
+    @never_raises()
+    def set_input(self, value: Any) -> None:
+        self.input = value
+
+    @never_raises()
+    def set_output(self, value: Any) -> None:
+        self.output = value
+
+    @never_raises()
+    def set_args(self, value: dict[str, Any]) -> None:
+        self.args = value
+
+    @never_raises()
+    def set_attribute(self, key: str, value: Any) -> None:
+        self.attributes[key] = value
+
+    @never_raises()
+    def set_attributes(self, **values: Any) -> None:
+        self.attributes.update(values)
+
+    @never_raises()
+    def set_model(
+        self,
+        model: str,
+        *,
+        provider: str | None = None,
+        prompt_tokens: int = 0,
+        completion_tokens: int = 0,
+        cost: Decimal | float | None = None,
+    ) -> None:
+        self.model = model
+        self.provider = provider
+        self.tokens = TokenUsage(
+            prompt=prompt_tokens,
+            completion=completion_tokens,
+            total=prompt_tokens + completion_tokens,
+        )
+        if cost is not None:
+            # str() first: Decimal(float) inherits the float's imprecision, and cost
+            # totals that drift are worse than no cost at all.
+            self.cost = cost if isinstance(cost, Decimal) else Decimal(str(cost))
+
+    @never_raises()
+    def record_event(self, name: str, **attributes: Any) -> None:
+        self.events.append(SpanEvent(name=name, timestamp=_now(), attributes=attributes))
+
+    @never_raises()
+    def set_error(self, exc: BaseException) -> None:
+        self.status = Status.ERROR
+        self.error_type = type(exc).__name__
+        self.status_message = str(exc)[:1000]
+
+    @never_raises()
+    def end(self, *, status: Status | None = None) -> None:
+        if self._ended:
+            return
+        self._ended = True
+        self.ended_at = _now()
+        if status is not None:
+            self.status = status
+        self._trace.finish_span(self)
+
+    # ---------------------------------------------------------------- conversion
+
+    def snapshot(self, pipeline: RedactionPipeline) -> Span:
+        redactions = 0
+        payloads: dict[str, Any] = {}
+
+        for field in ("input", "output", "args"):
+            value = getattr(self, field)
+            if value is None:
+                continue
+            payloads[field] = pipeline.apply(value, path=field)
+            redactions += pipeline.count
+
+        attributes = pipeline.apply(self.attributes, path="attributes") or {}
+        redactions += pipeline.count
+
+        return Span(
+            span_id=self.span_id,
+            trace_id=self.trace_id,
+            parent_span_id=self.parent_span_id,
+            name=self.name,
+            span_type=self.span_type,
+            status=self.status,
+            status_message=self.status_message,
+            started_at=self.started_at,
+            ended_at=self.ended_at,
+            attributes=attributes,
+            input=payloads.get("input"),
+            output=payloads.get("output"),
+            events=self.events,
+            model=self.model,
+            provider=self.provider,
+            tokens=self.tokens,
+            cost=self.cost,
+            tool_name=self.tool_name,
+            tool_args=payloads.get("args"),
+            error_type=self.error_type,
+            sequence_index=self.sequence_index,
+            redaction_count=redactions,
+        )
+
+    def __repr__(self) -> str:
+        return f"<Span {self.name!r} {self.span_id} {self.status.value}>"
+
+
+class TraceRecorder:
+    """One workflow execution in progress."""
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        config: Config,
+        trace_id: str | None = None,
+        parent_span_id: str | None = None,
+        sampled: bool = True,
+    ) -> None:
+        self.trace_id = trace_id or new_trace_id()
+        self.name = name
+        self.config = config
+        self.sampled = sampled
+        self.root_parent_span_id = parent_span_id
+        self.started_at = _now()
+        self.ended_at: datetime | None = None
+        self.status = Status.OK
+        self.metadata: dict[str, Any] = {}
+        self.tags: dict[str, str] = {}
+        self.state: dict[str, Any] = {}
+        self.dropped_span_count = 0
+
+        self._spans: list[SpanRecorder] = []
+        self._open = 0
+        self._sequence = 0
+        self._lock = threading.Lock()
+
+    def next_sequence(self) -> int:
+        with self._lock:
+            self._sequence += 1
+            return self._sequence
+
+    def register(self, span: SpanRecorder) -> bool:
+        """Accept a span unless the trace is already at its cap."""
+        with self._lock:
+            if len(self._spans) >= self.config.max_spans_per_trace:
+                self.dropped_span_count += 1
+                log_once(
+                    "trace.span_cap",
+                    f"trace {self.name!r} hit max_spans_per_trace "
+                    f"({self.config.max_spans_per_trace}); further spans are dropped "
+                    "and counted in dropped_span_count",
+                )
+                return False
+            self._spans.append(span)
+            self._open += 1
+            return True
+
+    def finish_span(self, _span: SpanRecorder) -> None:
+        with self._lock:
+            self._open = max(0, self._open - 1)
+
+    @never_raises()
+    def set_metadata(self, **values: Any) -> None:
+        self.metadata.update(values)
+
+    @never_raises()
+    def set_tags(self, **values: str) -> None:
+        self.tags.update({k: str(v) for k, v in values.items()})
+
+    @never_raises()
+    def set_state(self, **values: Any) -> None:
+        """Explicit workflow state.
+
+        Exists so `final_state` and `conditional` policy rules have a defined data
+        source instead of scraping outputs for something that looks like a status.
+        """
+        self.state.update(values)
+
+    @never_raises()
+    def end(self, *, status: Status | None = None) -> None:
+        if self.ended_at is not None:
+            return
+        self.ended_at = _now()
+        if status is not None:
+            self.status = status
+        elif any(s.status is Status.ERROR for s in self._spans):
+            self.status = Status.ERROR
+
+    @property
+    def open_span_count(self) -> int:
+        return self._open
+
+    def snapshot(self) -> Trace:
+        """Freeze into the shared `Trace` type, applying redaction."""
+        pipeline = RedactionPipeline(
+            redactors=[*_configured_redactors(self.config)],
+            capture_mode=self.config.capture_mode,
+            max_field_bytes=self.config.max_field_bytes,
+        )
+        spans = [s.snapshot(pipeline) for s in list(self._spans)]
+
+        # Trace-level fields go through the same pipeline as span payloads. They are
+        # user-supplied values like any other, and a credential passed to
+        # `set_metadata` is exactly as much of a leak as one in a span input.
+        #
+        # `metadata_only` capture is the exception: it suppresses payloads, but
+        # metadata, tags and state *are* the metadata, so they survive with secrets
+        # still stripped.
+        keep_structure = pipeline.capture_mode is CaptureMode.METADATA_ONLY
+        scrub = _MetadataScrubber(self.config, keep_structure=keep_structure)
+
+        return Trace(
+            trace_id=self.trace_id,
+            name=self.name,
+            status=self.status,
+            started_at=self.started_at,
+            ended_at=self.ended_at,
+            spans=spans,
+            metadata=scrub(self.metadata, "metadata"),
+            tags={k: str(v) for k, v in scrub(self.tags, "tags").items()},
+            state=scrub(self.state, "state"),
+            environment=self.config.environment,
+            git_commit=self.config.git_commit,
+            dropped_span_count=self.dropped_span_count,
+        )
+
+    def __repr__(self) -> str:
+        return f"<Trace {self.name!r} {self.trace_id} spans={len(self._spans)}>"
+
+
+def _configured_redactors(config: Config) -> list[Any]:
+    from proofstep import redaction  # noqa: PLC0415 — breaks an import cycle
+
+    # Always includes the secret redactors. `full` capture means full *payloads*,
+    # never full credentials, and returning an empty list here previously left
+    # trace-level metadata unredacted in that mode.
+    extra = list(redaction.default())
+    if config.redact_keys:
+        extra.append(redaction.keys(config.redact_keys))
+    return extra
+
+
+class _MetadataScrubber:
+    """Redacts trace-level dictionaries.
+
+    Uses a pipeline pinned to REDACTED rather than the project's capture mode, so
+    that `metadata_only` still keeps its metadata while `disabled` drops everything.
+    """
+
+    def __init__(self, config: Config, *, keep_structure: bool) -> None:
+        mode = CaptureMode.REDACTED if keep_structure else config.capture_mode
+        self._pipeline = RedactionPipeline(
+            redactors=[*_configured_redactors(config)],
+            capture_mode=CaptureMode.REDACTED if mode.stores_payloads or keep_structure else mode,
+            max_field_bytes=config.max_field_bytes,
+        )
+
+    def __call__(self, value: dict[str, Any], path: str) -> dict[str, Any]:
+        if not value:
+            return {}
+        result = self._pipeline.apply(value, path=path)
+        return result if isinstance(result, dict) else {}

proofstep/redaction.py

@@ -0,0 +1,22 @@
+"""Redaction, re-exported from `proofstep_core`.
+
+The pipeline used to live here, in the client SDK. That was the wrong home the moment the server
+started using it too: redaction has to produce *identical* output on both sides — a field the SDK
+strips and the server keeps is a secret that leaks — and the only way to guarantee that is one
+implementation, not two that agree today. But a server importing its own client SDK is backwards,
+and it pulled `httpx` into the API image to get a module that needs nothing but `re` and `math`.
+
+So it now lives in `proofstep_core`, which both sides already depend on and which is a pure library
+by contract (see `.importlinter`). This module stays as a re-export because
+`from proofstep.redaction import RedactionPipeline` is a documented import that users have written.
+"""
+
+from __future__ import annotations
+
+from proofstep_core import redaction as _redaction
+from proofstep_core.redaction import *  # noqa: F403
+
+# `import *` skips anything the source module did not name in `__all__`, and it does not carry the
+# module's own `__all__` across. Deriving it here means this shim exports exactly what the real
+# module exports, and keeps doing so when a name is added there.
+__all__ = [name for name in vars(_redaction) if not name.startswith("_")]

proofstep/safety.py

@@ -0,0 +1,119 @@
+"""The never-raise guarantee.
+
+A telemetry library that can crash the host application is unusable, and this is
+non-negotiable. Every public entry point is wrapped: an internal error is logged
+once per window and the call returns a harmless value.
+
+Logging is rate-limited for the same reason. A library that logs once per span
+during an outage produces more damage than the outage — it fills the disk, floods
+the aggregator, and buries the real error.
+"""
+
+from __future__ import annotations
+
+import functools
+import logging
+import os
+import threading
+import time
+from collections.abc import Callable
+from typing import Any, Literal, TypeVar
+
+logger = logging.getLogger("proofstep")
+
+T = TypeVar("T")
+
+# One message per (site, window). 60s is long enough that a sustained failure is
+# visible roughly once a minute rather than once a span.
+_LOG_WINDOW_S = 60.0
+_last_logged: dict[str, float] = {}
+_lock = threading.Lock()
+
+_STRICT = os.environ.get("PROOFSTEP_STRICT", "").strip().lower() in ("1", "true", "yes")
+"""Re-raise internal errors instead of swallowing them.
+
+For our own test suite only. Swallowing bugs is correct in production and
+catastrophic in tests, where it would let the SDK silently record nothing while
+every assertion about behaviour still passed.
+"""
+
+
+def log_once(key: str, message: str, *, exc_info: bool = False) -> None:
+    """Log at most once per key per window."""
+    now = time.monotonic()
+    with _lock:
+        last = _last_logged.get(key)
+        if last is not None and now - last < _LOG_WINDOW_S:
+            return
+        _last_logged[key] = now
+    logger.warning("proofstep: %s", message, exc_info=exc_info)
+
+
+def reset_log_throttle() -> None:
+    """Test helper."""
+    with _lock:
+        _last_logged.clear()
+
+
+def never_raises(default: Any = None, *, key: str | None = None) -> Callable[..., Any]:
+    """Decorator: swallow every exception and return `default`.
+
+    `BaseException` subclasses that are not `Exception` — KeyboardInterrupt,
+    SystemExit, and crucially `asyncio.CancelledError` — are deliberately not
+    caught. Swallowing cancellation would break the caller's control flow, which is
+    the opposite of staying out of the way.
+    """
+
+    def decorate(fn: Callable[..., Any]) -> Callable[..., Any]:
+        site = key or f"{fn.__module__}.{fn.__qualname__}"
+
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            try:
+                return fn(*args, **kwargs)
+            except Exception as exc:
+                if _STRICT:
+                    raise
+                log_once(site, f"{site} failed: {type(exc).__name__}: {exc}", exc_info=True)
+                return default() if callable(default) else default
+
+        return wrapper
+
+    return decorate
+
+
+class _NoOp:
+    """Returned when the SDK cannot produce a real object.
+
+    Absorbs every attribute access, call, and context-manager use, so user code
+    written against a working SDK keeps running against a broken one.
+    """
+
+    __slots__ = ()
+
+    def __getattr__(self, _name: str) -> _NoOp:
+        return self
+
+    def __call__(self, *_args: Any, **_kwargs: Any) -> _NoOp:
+        return self
+
+    def __enter__(self) -> _NoOp:
+        return self
+
+    def __exit__(self, *_exc: object) -> Literal[False]:
+        return False  # never suppress the caller's exception
+
+    async def __aenter__(self) -> _NoOp:
+        return self
+
+    async def __aexit__(self, *_exc: object) -> Literal[False]:
+        return False
+
+    def __bool__(self) -> bool:
+        return False
+
+    def __repr__(self) -> str:
+        return "<proofstep disabled>"
+
+
+NOOP = _NoOp()

proofstep-0.1.0.dist-info/METADATA

@@ -0,0 +1,58 @@
+Metadata-Version: 2.5
+Name: proofstep
+Version: 0.1.0
+Summary: Proofstep Python SDK — tracing and evaluation for AI applications and agents
+Project-URL: Homepage, https://github.com/IlaKhan17/proofstep
+Project-URL: Documentation, https://github.com/IlaKhan17/proofstep/tree/main/docs
+Project-URL: Repository, https://github.com/IlaKhan17/proofstep
+Project-URL: Issues, https://github.com/IlaKhan17/proofstep/issues
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27
+Requires-Dist: proofstep-core
+Requires-Dist: proofstep-types
+Requires-Dist: pydantic>=2.9
+Provides-Extra: eval
+Requires-Dist: proofstep-trajectory; extra == 'eval'
+Description-Content-Type: text/markdown
+
+# proofstep
+
+**Tracing SDK** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+Instrument an AI application or agent, and send the trace to a Proofstep server.
+
+```python
+import proofstep
+
+proofstep.init(endpoint="https://proofstep.internal", api_key="ps_prod_…")
+
+with proofstep.capture("outbound") as captured:
+    proofstep.set_state(unsubscribed=False)
+    with proofstep.start_span("gmail.send", span_type="tool", tool_name="gmail.send") as span:
+        span.set_args({"to": recipient, "thread_id": thread})
+```
+
+`set_args` is not decoration: trajectory policies match on `args.*`, so a check whose result never
+reaches the trace cannot be audited later.
+
+Secrets are redacted in this process, before export. Access tokens, refresh tokens, API keys,
+passwords, session cookies, and `Authorization` headers are never intentionally stored.
+
+Install the evaluation extras with `pip install "proofstep[eval]"`, or the CLI with
+`pip install proofstep-cli`.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+proofstep/__init__.py,sha256=_vIQXcrEplF_NlQIfOwzZQXjsgsVbvQyj60ALs_tKjU,5621
+proofstep/client.py,sha256=-QNYUsJeXGLUIK669zcF-A8dkdtfhUXqXHWGkn8Owlo,7208
+proofstep/config.py,sha256=giJYU2OeYZfBPlm3O6Cywddmp0Uw_GPlzcsKBomgBxM,5401
+proofstep/context.py,sha256=3aPEruyQImyf1z3hzMfLxod18V2Egsl93TUTqFtGMcI,2309
+proofstep/decorators.py,sha256=pT5OuZUUlMrNZ8FmHkp70s3xkDrCF0HUmwgvs3zPuAc,5823
+proofstep/exporter.py,sha256=YQKFgrxpGXM_j89jIMcGTs4MmoAmDHmWheUAOCInlcA,10348
+proofstep/propagation.py,sha256=dim9s14-1kw6czGKw3eSS1ZnYERKaRns7eOdSDEEIkQ,1641
+proofstep/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+proofstep/recorder.py,sha256=S4HW8zrfPTZVOrIul3JWgsmwIFftteQiE-HHTXaOKek,12115
+proofstep/redaction.py,sha256=bsqW9SSN9FiKYG4bVahZ4zf01Jr5RTwsEJoEj7AtuIE,1296
+proofstep/safety.py,sha256=gvOksXMWu_wRFxCvH4puyaCYIJQeTr4KcC10HzCuBlw,3641
+proofstep-0.1.0.dist-info/METADATA,sha256=DCJ0DnMQCb9B3vFike71v8uYLCITREwcO1bMF9Ko5As,2293
+proofstep-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+proofstep-0.1.0.dist-info/RECORD,,

proofstep-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any