proofstep-types 0.1.0__tar.gz

proofstep_types-0.1.0/.gitignore

@@ -0,0 +1,57 @@
+
+__pycache__/
+.coverage
+.coverage.*
+.docker-data/
+.DS_Store
+.e2e-api.log
+.env
+.env.*
+.env.prod
+# Exceptions, and they must come *after* the patterns above: git takes the last matching rule, so a
+# negation written earlier in the file is silently overridden. That is not a hypothetical — the `!`
+# line used to sit at the top, `.env.*` re-ignored it, and `.env.prod.example` was never committed.
+# `scripts/init_secrets.sh` reads that file, so the first documented step of self-hosting failed for
+# anyone who cloned the repository. It was caught by CI running the same step.
+!.env.example
+!.env.prod.example
+.hypothesis/
+.idea/
+.mypy_cache/
+.next/
+.proofstep/
+.pytest_cache/
+.ruff_cache/
+.turbo/
+.venv/
+.vscode/
+*.egg-info/
+*.key
+*.pem
+*.proofstep.local.yaml
+*.py[cod]
+*.swp
+*.tsbuildinfo
+# Database dumps. Never committed: they contain every tenant's data, and a backup in a git history
+# Docker volumes
+# Editors / OS
+# is a backup with no access control and no retention.
+# Node
+# Proofstep local state
+# Python
+# Reports a local run drops in the working tree. The name comes from the suite, so the pattern has
+# Secrets and local config — never commit these
+# The e2e stack's server log, written next to the repo so a CI failure can print it.
+# to cover all of them rather than the default filename only.
+~/.proofstep/
+backups/
+build/
+coverage.xml
+credentials.json
+dist/
+htmlcov/
+node_modules/
+out/
+proofstep-*.json
+secrets/
+venv/

proofstep_types-0.1.0/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.5
+Name: proofstep-types
+Version: 0.1.0
+Summary: Shared Pydantic models for Proofstep — the wire and domain contract
+Project-URL: Homepage, https://github.com/IlaKhan17/proofstep
+Project-URL: Documentation, https://github.com/IlaKhan17/proofstep/tree/main/docs
+Project-URL: Repository, https://github.com/IlaKhan17/proofstep
+Project-URL: Issues, https://github.com/IlaKhan17/proofstep/issues
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.9
+Description-Content-Type: text/markdown
+
+# proofstep-types
+
+**Shared models** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+Pydantic models shared by every Proofstep package: examples, scores, metrics, gates, spans, and
+traces.
+
+Depends on nothing of ours, by design — it is the bottom of the dependency graph, and an
+import-linter contract keeps it there.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep_types-0.1.0/README.md

@@ -0,0 +1,16 @@
+# proofstep-types
+
+**Shared models** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+Pydantic models shared by every Proofstep package: examples, scores, metrics, gates, spans, and
+traces.
+
+Depends on nothing of ours, by design — it is the bottom of the dependency graph, and an
+import-linter contract keeps it there.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep_types-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "proofstep-types"
+version = "0.1.0"
+description = "Shared Pydantic models for Proofstep — the wire and domain contract"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Quality Assurance",
+    "Typing :: Typed",
+]
+
+dependencies = ["pydantic>=2.9"]
+
+[project.urls]
+Homepage = "https://github.com/IlaKhan17/proofstep"
+Documentation = "https://github.com/IlaKhan17/proofstep/tree/main/docs"
+Repository = "https://github.com/IlaKhan17/proofstep"
+Issues = "https://github.com/IlaKhan17/proofstep/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/proofstep_types"]

proofstep_types-0.1.0/src/proofstep_types/__init__.py

@@ -0,0 +1,61 @@
+"""Shared Pydantic models for Proofstep — the wire and domain contract.
+
+This package is a leaf: it depends on nothing else of ours, which is what lets the
+SDK, the pure libraries, the API, and the generated TypeScript types all agree on one
+definition. Enforced by the `types-are-leaf` contract in `.importlinter`.
+"""
+
+from importlib import metadata as _metadata
+
+from proofstep_types.common import (
+    CaptureMode,
+    ExitCode,
+    OutputKind,
+    ResultStatus,
+    Severity,
+    SpanType,
+    Status,
+    Verdict,
+)
+from proofstep_types.dataset import Example, content_hash
+from proofstep_types.gates import (
+    CalibrationRequirementSpec,
+    CalibrationStatus,
+    GateRule,
+    GateSet,
+)
+from proofstep_types.results import ExampleResult, TaskError
+from proofstep_types.score import GateResult, Metric, MetricDelta, Score
+from proofstep_types.trace import Span, SpanEvent, TokenUsage, Trace
+
+# Read from the installed distribution rather than written here twice. A hand-maintained
+# copy drifts the first time a release bumps one and not the other — which it already did,
+# reporting 0.1.0.dev0 from a 0.1.0 wheel.
+__version__ = _metadata.version("proofstep-types")
+
+__all__ = [
+    "CalibrationRequirementSpec",
+    "CalibrationStatus",
+    "CaptureMode",
+    "Example",
+    "ExampleResult",
+    "ExitCode",
+    "GateResult",
+    "GateRule",
+    "GateSet",
+    "Metric",
+    "MetricDelta",
+    "OutputKind",
+    "ResultStatus",
+    "Score",
+    "Severity",
+    "Span",
+    "SpanEvent",
+    "SpanType",
+    "Status",
+    "TaskError",
+    "TokenUsage",
+    "Trace",
+    "Verdict",
+    "content_hash",
+]

proofstep_types-0.1.0/src/proofstep_types/common.py

@@ -0,0 +1,136 @@
+"""Enumerations shared across the wire contract.
+
+These are plain string enums rather than PostgreSQL enum types: adding a value to a
+PG enum inside a transaction alongside other DDL is a recurring migration hazard, so
+the database stores text with a CHECK constraint instead (docs/DATABASE_DESIGN.md §0).
+
+Clients must tolerate unknown values. Deserializers degrade rather than raise — a
+server that learns a new span type must not break older SDKs (docs/API_DESIGN.md §4).
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+
+class SpanType(StrEnum):
+    """What kind of operation a span represents."""
+
+    AGENT = "agent"
+    WORKFLOW = "workflow"
+    LLM = "llm"
+    TOOL = "tool"
+    RETRIEVER = "retriever"
+    EMBEDDING = "embedding"
+    GUARDRAIL = "guardrail"
+    EVALUATOR = "evaluator"
+    CUSTOM = "custom"
+
+    @classmethod
+    def _missing_(cls, value: object) -> SpanType:  # noqa: ARG003 — signature fixed by Enum
+        """Degrade unknown span types to CUSTOM instead of raising."""
+        return cls.CUSTOM
+
+
+class Status(StrEnum):
+    """Terminal status of a span, trace, or task execution."""
+
+    OK = "ok"
+    ERROR = "error"
+    TIMEOUT = "timeout"
+    UNSET = "unset"
+
+
+class Severity(StrEnum):
+    """How a rule or gate failure should be treated."""
+
+    BLOCK = "block"
+    WARN = "warn"
+
+
+class Verdict(StrEnum):
+    """Outcome of a gate, a gate set, or a whole run.
+
+    ERROR is distinct from FAIL on purpose: a metric that could not be computed is
+    not the same as a metric that came out bad, and collapsing the two is how a
+    broken evaluator silently reports success (docs/EVALUATION_ENGINE.md §1).
+    """
+
+    PASS = "pass"  # noqa: S105 — a verdict, not a credential
+    WARN = "warn"
+    FAIL = "fail"
+    ERROR = "error"
+
+    @property
+    def is_blocking(self) -> bool:
+        return self in (Verdict.FAIL, Verdict.ERROR)
+
+
+class CaptureMode(StrEnum):
+    """How much payload data is retained.
+
+    Ordered most- to least-permissive; `resolve()` picks the most restrictive of
+    several settings, which is how project, environment, SDK, and per-span settings
+    combine (docs/SECURITY.md §8).
+    """
+
+    FULL = "full"
+    REDACTED = "redacted"
+    METADATA_ONLY = "metadata_only"
+    DISABLED = "disabled"
+
+    @property
+    def rank(self) -> int:
+        return _CAPTURE_RANK[self]
+
+    @property
+    def stores_payloads(self) -> bool:
+        return self in (CaptureMode.FULL, CaptureMode.REDACTED)
+
+    @classmethod
+    def resolve(cls, *modes: CaptureMode | None) -> CaptureMode:
+        """Return the most restrictive of the given modes.
+
+        Defaults to REDACTED when nothing is specified: the safe default is not
+        collecting, because data that is never stored cannot leak.
+        """
+        present = [m for m in modes if m is not None]
+        if not present:
+            return cls.REDACTED
+        return max(present, key=lambda m: m.rank)
+
+
+_CAPTURE_RANK: dict[CaptureMode, int] = {
+    CaptureMode.FULL: 0,
+    CaptureMode.REDACTED: 1,
+    CaptureMode.METADATA_ONLY: 2,
+    CaptureMode.DISABLED: 3,
+}
+
+
+class OutputKind(StrEnum):
+    """The shape of the value an evaluator produces."""
+
+    BINARY = "binary"
+    SCORE = "score"
+    CATEGORICAL = "categorical"
+    NUMERIC = "numeric"
+
+
+class ResultStatus(StrEnum):
+    """Outcome of running the task against a single example."""
+
+    OK = "ok"
+    ERROR = "error"
+    TIMEOUT = "timeout"
+    SKIPPED = "skipped"
+
+
+class ExitCode:
+    """Process exit codes for the CLI (docs/EVALUATION_ENGINE.md §7)."""
+
+    PASS = 0
+    BLOCKING_FAILURE = 1
+    EXECUTION_ERROR = 2
+    CONFIGURATION_ERROR = 3
+    CANCELLED = 130

proofstep_types-0.1.0/src/proofstep_types/dataset.py

@@ -0,0 +1,57 @@
+"""Dataset examples — the input side of an evaluation."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class Example(BaseModel):
+    """One evaluation case: an input, an optional expected result, and metadata.
+
+    ``id`` is the *stable external identifier*. Comparison between experiments
+    matches on it rather than on position, because datasets gain and lose examples
+    between versions and ordinal matching would silently compare unrelated rows
+    (docs/EVALUATION_ENGINE.md §6).
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    id: str
+    input: dict[str, Any]
+    expected: dict[str, Any] | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    source_trace_id: str | None = None
+    source_span_id: str | None = None
+
+    def canonical_json(self) -> str:
+        """Deterministic serialization used for dataset content hashing."""
+        return json.dumps(
+            {
+                "id": self.id,
+                "input": self.input,
+                "expected": self.expected,
+                "metadata": self.metadata,
+            },
+            sort_keys=True,
+            separators=(",", ":"),
+            ensure_ascii=False,
+            default=str,
+        )
+
+
+def content_hash(examples: list[Example]) -> str:
+    """SHA-256 over the canonically serialized examples, in order.
+
+    This is what makes an experiment provably reproducible: the hash is recorded
+    alongside the dataset version id, so a later run can prove it used identical
+    data rather than merely claiming the same version label (ADR-012).
+    """
+    digest = hashlib.sha256()
+    for example in examples:
+        digest.update(example.canonical_json().encode("utf-8"))
+        digest.update(b"\n")
+    return digest.hexdigest()

proofstep_types-0.1.0/src/proofstep_types/gates.py

@@ -0,0 +1,213 @@
+"""Quality gate rules — the contract between a suite file and the gate engine."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from proofstep_types.common import Severity
+
+
+class GateRule(BaseModel):
+    """One threshold applied to one metric.
+
+    ``slice`` is what makes protected-class gating possible. A rule can target
+    ``per_class_recall`` for ``unsubscribe`` specifically rather than the macro
+    average — which matters because a 3%-prevalence class can collapse from 0.99
+    to 0.20 recall while macro accuracy moves 0.3%, passing every aggregate gate
+    (docs/EVALUATION_ENGINE.md §7).
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    metric_key: str
+    minimum: float | None = None
+    maximum: float | None = None
+    max_absolute_regression: float | None = None
+    max_relative_regression: float | None = None
+    severity: Severity = Severity.BLOCK
+    slice: dict[str, str] | None = None
+    require_baseline: bool = Field(
+        default=False,
+        description="Fail when no baseline exists, rather than skipping regression checks",
+    )
+    max_error_rate: float = Field(
+        default=0.05,
+        description="Above this share of errored evaluations the gate reports ERROR, not PASS",
+    )
+    significance: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description=(
+            "Alpha for a paired significance test. When set, a regression must be both larger "
+            "than the threshold *and* distinguishable from noise before it fails the build. A "
+            "threshold alone says what size of change matters; it cannot say whether the change "
+            "is real, and at small sample sizes most measured 'regressions' are neither."
+        ),
+    )
+    require_power: bool = Field(
+        default=False,
+        description=(
+            "Report ERROR when the run was too small to have detected the regression this rule "
+            "guards against. A green check from a test that could never have failed is worse than "
+            "no check, because it is believed."
+        ),
+    )
+
+    @property
+    def needs_significance(self) -> bool:
+        """Whether this rule wants a paired test computed for it.
+
+        `require_power` counts: reporting that a run could not have detected the regression it
+        guards needs the same paired data as testing whether one happened.
+        """
+        return self.significance is not None or self.require_power
+
+    @property
+    def blocking(self) -> bool:
+        return self.severity is Severity.BLOCK
+
+    @property
+    def full_key(self) -> str:
+        if not self.slice:
+            return self.metric_key
+        inner = ",".join(f"{k}={v}" for k, v in sorted(self.slice.items()))
+        return f"{self.metric_key}[{inner}]"
+
+    @property
+    def needs_baseline(self) -> bool:
+        return self.max_absolute_regression is not None or self.max_relative_regression is not None
+
+    @model_validator(mode="after")
+    def _at_least_one_condition(self) -> GateRule:
+        if not any(
+            v is not None
+            for v in (
+                self.minimum,
+                self.maximum,
+                self.max_absolute_regression,
+                self.max_relative_regression,
+            )
+        ):
+            msg = (
+                f"Gate on {self.metric_key!r} declares no condition; a gate that "
+                "cannot fail is a gate that gives false assurance"
+            )
+            raise ValueError(msg)
+        return self
+
+    @model_validator(mode="after")
+    def _bounds_are_ordered(self) -> GateRule:
+        if self.minimum is not None and self.maximum is not None and self.minimum > self.maximum:
+            msg = (
+                f"Gate on {self.metric_key!r} has minimum {self.minimum} above "
+                f"maximum {self.maximum}; no value can satisfy it"
+            )
+            raise ValueError(msg)
+        return self
+
+
+class GateSet(BaseModel):
+    """A named collection of gate rules, versioned with the suite that declares it."""
+
+    model_config = ConfigDict(frozen=True)
+
+    name: str = "default"
+    rules: list[GateRule] = Field(default_factory=list)
+    require_dataset_match: bool = Field(
+        default=True,
+        description=(
+            "Refuse to gate when candidate and baseline used different dataset "
+            "content hashes. Comparing across datasets is a silent source of "
+            "confidently wrong conclusions."
+        ),
+    )
+    require_calibration: bool | CalibrationRequirementSpec = Field(
+        default=False,
+        description=(
+            "Turn the uncalibrated-judge warning into a hard error. `true` uses the "
+            "recommended thresholds; a mapping overrides them."
+        ),
+    )
+
+    @property
+    def calibration_requirement(self) -> CalibrationRequirementSpec | None:
+        """Normalize the bool-or-mapping form into one thing the gate engine reads.
+
+        `None` means no requirement, so an uncalibrated judge warns rather than blocks.
+        Warning rather than silence matters: a gate on a judge nobody has checked is
+        the failure this whole subsystem exists to make visible.
+        """
+        if self.require_calibration is False:
+            return None
+        if self.require_calibration is True:
+            return CalibrationRequirementSpec()
+        return self.require_calibration
+
+    def rules_for(self, metric_key: str) -> list[GateRule]:
+        return [r for r in self.rules if r.metric_key == metric_key]
+
+
+class CalibrationRequirementSpec(BaseModel):
+    """What a gate set demands before it will trust a judge.
+
+    ``require_calibration: true`` in YAML means "these defaults", and a mapping means
+    "these defaults with overrides". The defaults are the recommended values for a
+    safety-relevant metric (docs/EVALUATION_ENGINE.md §5).
+
+    ``max_false_pass_rate`` is deliberately four times tighter than
+    ``max_false_fail_rate``. A judge that passes work a human rejected ships a defect;
+    one that fails acceptable work annoys somebody. Treating those as the same error is
+    how a gate ends up either useless or bypassed.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    required: bool = Field(
+        default=True,
+        description="Fail the run rather than warn when the requirement is unmet",
+    )
+    min_agreement: float = Field(default=0.8, ge=0, le=1)
+    min_kappa: float | None = Field(default=0.6, ge=-1, le=1)
+    max_false_pass_rate: float | None = Field(default=0.05, ge=0, le=1)
+    max_false_fail_rate: float | None = Field(default=0.20, ge=0, le=1)
+    min_examples: int = Field(default=100, ge=1)
+    min_per_class: int = Field(default=50, ge=1)
+    max_error_rate: float = Field(default=0.05, ge=0, le=1)
+    allow_position_bias: bool = False
+
+
+class CalibrationStatus(BaseModel):
+    """What is known about one judge metric's calibration at gate time.
+
+    ``calibrated=False`` and "calibrated but failing" are different states with
+    different fixes — go and calibrate it, versus go and fix the judge — so they are
+    reported separately rather than folded into one boolean.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    metric_key: str
+    evaluator_name: str = ""
+    #: Config hash of the calibrated evaluator version. A judge whose rubric, model, or
+    #: parameters changed has a different hash, and an old calibration does not apply to
+    #: it — that is the whole point of versioning by config hash.
+    evaluator_version_hash: str | None = None
+    calibrated: bool = False
+    satisfied: bool | None = None
+    failures: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    n_examples: int = 0
+    agreement: float | None = None
+    kappa: float | None = None
+    false_pass_rate: float | None = None
+    at_human_ceiling: bool = False
+    calibrated_at: datetime | None = None
+    #: Set when a calibration exists but for a different version of the evaluator.
+    stale_for_version: str | None = None
+
+    @property
+    def is_stale(self) -> bool:
+        return self.stale_for_version is not None

proofstep_types-0.1.0/src/proofstep_types/results.py

@@ -0,0 +1,64 @@
+"""Per-example execution results."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from decimal import Decimal
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from proofstep_types.common import ResultStatus
+from proofstep_types.score import Score
+from proofstep_types.trace import Trace
+
+
+class TaskError(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: str
+    message: str
+    traceback: str | None = None
+
+
+class ExampleResult(BaseModel):
+    """The outcome of running the task against one example, with its scores.
+
+    Serialized one-per-line into the run journal as each example completes, so a
+    crash at example 190/200 loses nothing and `--resume` can skip what finished
+    (docs/EVALUATION_ENGINE.md §4).
+    """
+
+    example_id: str
+    status: ResultStatus = ResultStatus.OK
+    output: Any = None
+    scores: list[Score] = Field(default_factory=list)
+    trace: Trace | None = None
+
+    latency_ms: int = 0
+    cost: Decimal = Decimal(0)
+    tokens: int = 0
+    retry_count: int = 0
+    error: TaskError | None = None
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+
+    # Retained so a comparison can slice by any dimension the dataset carried.
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    expected: dict[str, Any] | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.status is ResultStatus.OK
+
+    @property
+    def total_cost(self) -> Decimal:
+        """Task cost plus the cost of evaluating it.
+
+        Judge spend is part of what a suite costs to run, and hiding it in a
+        separate number is how teams end up surprised by the bill.
+        """
+        return self.cost + sum((s.cost for s in self.scores), Decimal(0))
+
+    def score_for(self, metric: str) -> Score | None:
+        return next((s for s in self.scores if s.metric == metric), None)

proofstep_types-0.1.0/src/proofstep_types/score.py

@@ -0,0 +1,150 @@
+"""Scores and metrics — what an evaluator produces and what a gate consumes."""
+
+from __future__ import annotations
+
+from decimal import Decimal
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from proofstep_types.common import Severity
+
+
+class Score(BaseModel):
+    """The result of one evaluator applied to one example.
+
+    The single most important invariant in this model: ``error`` is not ``value=0``.
+    A judge that timed out is not a failing example. Aggregation excludes errored
+    scores from the mean and counts them separately, and a metric with too high an
+    error rate gates as ERROR rather than PASS. Silently scoring infrastructure
+    failures as zero is the fastest way to make a gate untrustworthy
+    (docs/EVALUATION_ENGINE.md §1).
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    metric: str = Field(description="Metric key this score contributes to")
+    value: float | None = Field(
+        default=None, description="Normalized to [0,1] where meaningful; None if not scalar"
+    )
+    passed: bool | None = Field(default=None, description="Binary verdict, if applicable")
+    label: str | None = Field(default=None, description="Categorical evaluators")
+    raw: Any = Field(default=None, description="Non-scalar payload (matrix, list, …)")
+    reasoning: str | None = Field(default=None, description="Judge rationale, if any")
+    confidence: float | None = None
+    cost: Decimal = Decimal(0)
+    latency_ms: int = 0
+    error: str | None = Field(
+        default=None, description="The evaluator itself failed — never conflate with value=0"
+    )
+    slice: dict[str, str] | None = Field(
+        default=None, description="Slice this score belongs to, e.g. {'class': 'unsubscribe'}"
+    )
+
+    @property
+    def errored(self) -> bool:
+        return self.error is not None
+
+    @property
+    def counts_toward_mean(self) -> bool:
+        return self.error is None and self.value is not None
+
+    @model_validator(mode="after")
+    def _check_error_has_no_value(self) -> Score:
+        if self.error is not None and self.value is not None:
+            msg = (
+                f"Score for {self.metric!r} has both error and value; an errored "
+                "evaluation must not contribute a score"
+            )
+            raise ValueError(msg)
+        return self
+
+    @classmethod
+    def failure(cls, metric: str, error: str, *, latency_ms: int = 0) -> Score:
+        """Construct an errored score. Prefer this over `Score(value=0.0)`."""
+        return cls(metric=metric, error=error, latency_ms=latency_ms)
+
+    @classmethod
+    def binary(cls, metric: str, passed: bool, **kw: Any) -> Score:
+        return cls(metric=metric, value=1.0 if passed else 0.0, passed=passed, **kw)
+
+
+class Metric(BaseModel):
+    """An aggregate over many scores — one row of a comparison table."""
+
+    model_config = ConfigDict(frozen=True)
+
+    key: str
+    value: float
+    count: int = Field(description="Number of scores that contributed")
+    error_count: int = Field(default=0, description="Scores excluded because they errored")
+    stddev: float | None = None
+    ci_low: float | None = None
+    ci_high: float | None = None
+    unit: str | None = Field(default=None, description="ms, usd, ratio, count, …")
+    slice: dict[str, str] | None = None
+    aggregation: str = "mean"
+
+    @property
+    def full_key(self) -> str:
+        """Key including its slice, e.g. ``per_class_recall[class=unsubscribe]``."""
+        if not self.slice:
+            return self.key
+        inner = ",".join(f"{k}={v}" for k, v in sorted(self.slice.items()))
+        return f"{self.key}[{inner}]"
+
+    @property
+    def error_rate(self) -> float:
+        total = self.count + self.error_count
+        return self.error_count / total if total else 0.0
+
+
+class MetricDelta(BaseModel):
+    """A candidate metric compared against a baseline."""
+
+    model_config = ConfigDict(frozen=True)
+
+    key: str
+    slice: dict[str, str] | None = None
+    baseline: float | None
+    candidate: float | None
+    absolute_delta: float | None = None
+    relative_delta: float | None = None
+    count: int = 0
+    ci_low: float | None = None
+    ci_high: float | None = None
+    significant: bool | None = Field(
+        default=None,
+        description=(
+            "Bootstrap CI on the delta excludes zero. Advisory only — gates use "
+            "thresholds, because at n=200 most real regressions are not significant "
+            "and gating on p-values would let them through."
+        ),
+    )
+
+    @property
+    def full_key(self) -> str:
+        if not self.slice:
+            return self.key
+        inner = ",".join(f"{k}={v}" for k, v in sorted(self.slice.items()))
+        return f"{self.key}[{inner}]"
+
+
+class GateResult(BaseModel):
+    """The verdict of one gate rule against one metric."""
+
+    model_config = ConfigDict(frozen=True)
+
+    metric_key: str
+    slice: dict[str, str] | None = None
+    verdict: str
+    severity: Severity = Severity.BLOCK
+    rule: str | None = Field(default=None, description="Which clause fired: minimum, maximum, …")
+    threshold: float | None = None
+    actual: float | None = None
+    baseline: float | None = None
+    message: str = ""
+
+    @property
+    def blocking(self) -> bool:
+        return self.severity is Severity.BLOCK

proofstep_types-0.1.0/src/proofstep_types/trace.py

@@ -0,0 +1,144 @@
+"""Traces and spans — the captured record of one workflow execution."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from decimal import Decimal
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from proofstep_types.common import SpanType, Status
+
+
+class TokenUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    prompt: int = 0
+    completion: int = 0
+    total: int = 0
+
+
+class SpanEvent(BaseModel):
+    """A point-in-time occurrence inside a span: a retry, a guardrail trigger.
+
+    Kept as a distinct type rather than a JSON array on the span because retries and
+    guardrail triggers must be independently queryable for operational and trajectory
+    evaluators (docs/DATABASE_DESIGN.md §2.2).
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    timestamp: datetime
+    attributes: dict[str, Any] = Field(default_factory=dict)
+
+
+class Span(BaseModel):
+    """One operation inside a trace."""
+
+    model_config = ConfigDict(frozen=True)
+
+    span_id: str
+    trace_id: str
+    parent_span_id: str | None = None
+    name: str
+    span_type: SpanType = SpanType.CUSTOM
+    status: Status = Status.OK
+    status_message: str | None = None
+    started_at: datetime
+    ended_at: datetime | None = None
+
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    input: Any = None
+    output: Any = None
+    events: list[SpanEvent] = Field(default_factory=list)
+
+    # Denormalized hot-path fields — these are real columns in the database rather
+    # than JSONB keys because they are filtered and aggregated constantly.
+    model: str | None = None
+    provider: str | None = None
+    tokens: TokenUsage | None = None
+    cost: Decimal | None = None
+    tool_name: str | None = None
+    tool_args: dict[str, Any] | None = None
+    error_type: str | None = None
+
+    sequence_index: int = Field(
+        default=0,
+        description=(
+            "Monotonic counter from the SDK. Breaks ordering ties when two spans "
+            "share a start timestamp, which clock granularity makes common."
+        ),
+    )
+    redaction_count: int = 0
+
+    @property
+    def duration_ms(self) -> int | None:
+        if self.ended_at is None:
+            return None
+        return int((self.ended_at - self.started_at).total_seconds() * 1000)
+
+    @property
+    def is_open(self) -> bool:
+        return self.ended_at is None
+
+
+class Trace(BaseModel):
+    """One complete AI workflow execution."""
+
+    model_config = ConfigDict(frozen=True)
+
+    trace_id: str
+    name: str
+    status: Status = Status.OK
+    started_at: datetime
+    ended_at: datetime | None = None
+    spans: list[Span] = Field(default_factory=list)
+
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    tags: dict[str, str] = Field(default_factory=dict)
+    state: dict[str, Any] = Field(
+        default_factory=dict,
+        description=(
+            "Explicit workflow state set via `set_state`. Gives `final_state` and "
+            "`conditional` policy rules a defined data source instead of scraping outputs."
+        ),
+    )
+
+    environment: str | None = None
+    git_commit: str | None = None
+    dropped_span_count: int = Field(
+        default=0,
+        description=(
+            "Spans the exporter dropped under backpressure. Non-zero makes the "
+            "trajectory `incomplete`, which turns `required_*` policy rules into "
+            "`inconclusive` rather than `fail` — asserting absence over incomplete "
+            "data is unsound (docs/TRAJECTORY_POLICIES.md §4)."
+        ),
+    )
+
+    @property
+    def duration_ms(self) -> int | None:
+        if self.ended_at is None:
+            return None
+        return int((self.ended_at - self.started_at).total_seconds() * 1000)
+
+    @property
+    def is_complete(self) -> bool:
+        return self.dropped_span_count == 0 and not any(s.is_open for s in self.spans)
+
+    @property
+    def total_cost(self) -> Decimal:
+        return sum((s.cost for s in self.spans if s.cost is not None), Decimal(0))
+
+    @property
+    def total_tokens(self) -> int:
+        return sum(s.tokens.total for s in self.spans if s.tokens is not None)
+
+    def spans_by_type(self, *types: SpanType) -> list[Span]:
+        wanted = set(types)
+        return [s for s in self.spans if s.span_type in wanted]
+
+    def find_span(self, span_id: str) -> Span | None:
+        return next((s for s in self.spans if s.span_id == span_id), None)