proofstep-trajectory 0.1.0__tar.gz

proofstep_trajectory-0.1.0/.gitignore

@@ -0,0 +1,57 @@
+
+__pycache__/
+.coverage
+.coverage.*
+.docker-data/
+.DS_Store
+.e2e-api.log
+.env
+.env.*
+.env.prod
+# Exceptions, and they must come *after* the patterns above: git takes the last matching rule, so a
+# negation written earlier in the file is silently overridden. That is not a hypothetical — the `!`
+# line used to sit at the top, `.env.*` re-ignored it, and `.env.prod.example` was never committed.
+# `scripts/init_secrets.sh` reads that file, so the first documented step of self-hosting failed for
+# anyone who cloned the repository. It was caught by CI running the same step.
+!.env.example
+!.env.prod.example
+.hypothesis/
+.idea/
+.mypy_cache/
+.next/
+.proofstep/
+.pytest_cache/
+.ruff_cache/
+.turbo/
+.venv/
+.vscode/
+*.egg-info/
+*.key
+*.pem
+*.proofstep.local.yaml
+*.py[cod]
+*.swp
+*.tsbuildinfo
+# Database dumps. Never committed: they contain every tenant's data, and a backup in a git history
+# Docker volumes
+# Editors / OS
+# is a backup with no access control and no retention.
+# Node
+# Proofstep local state
+# Python
+# Reports a local run drops in the working tree. The name comes from the suite, so the pattern has
+# Secrets and local config — never commit these
+# The e2e stack's server log, written next to the repo so a CI failure can print it.
+# to cover all of them rather than the default filename only.
+~/.proofstep/
+backups/
+build/
+coverage.xml
+credentials.json
+dist/
+htmlcov/
+node_modules/
+out/
+proofstep-*.json
+secrets/
+venv/

proofstep_trajectory-0.1.0/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.5
+Name: proofstep-trajectory
+Version: 0.1.0
+Summary: Proofstep trajectory policy engine — policy-as-code for tool-using agents
+Project-URL: Homepage, https://github.com/IlaKhan17/proofstep
+Project-URL: Documentation, https://github.com/IlaKhan17/proofstep/tree/main/docs
+Project-URL: Repository, https://github.com/IlaKhan17/proofstep
+Project-URL: Issues, https://github.com/IlaKhan17/proofstep/issues
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: proofstep-types
+Requires-Dist: pydantic>=2.9
+Requires-Dist: pyyaml>=6.0
+Description-Content-Type: text/markdown
+
+# proofstep-trajectory
+
+**Trajectory policy engine** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+Evaluates what an agent *did* — the ordered sequence of tool calls, their arguments, and the state
+they left behind — against policies written as reviewable YAML.
+
+```yaml
+rules:
+  - id: no-send-before-approval
+    kind: forbidden_before
+    severity: block
+    action: gmail.send
+    before: approval_received
+```
+
+An agent can produce a flawless email and still have sent it before approval. No output evaluator
+can detect that; this is what does.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep_trajectory-0.1.0/README.md

@@ -0,0 +1,25 @@
+# proofstep-trajectory
+
+**Trajectory policy engine** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+Evaluates what an agent *did* — the ordered sequence of tool calls, their arguments, and the state
+they left behind — against policies written as reviewable YAML.
+
+```yaml
+rules:
+  - id: no-send-before-approval
+    kind: forbidden_before
+    severity: block
+    action: gmail.send
+    before: approval_received
+```
+
+An agent can produce a flawless email and still have sent it before approval. No output evaluator
+can detect that; this is what does.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep_trajectory-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "proofstep-trajectory"
+version = "0.1.0"
+description = "Proofstep trajectory policy engine — policy-as-code for tool-using agents"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Quality Assurance",
+    "Typing :: Typed",
+]
+
+dependencies = [
+    "proofstep-types",
+    "pydantic>=2.9",
+    "pyyaml>=6.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/IlaKhan17/proofstep"
+Documentation = "https://github.com/IlaKhan17/proofstep/tree/main/docs"
+Repository = "https://github.com/IlaKhan17/proofstep"
+Issues = "https://github.com/IlaKhan17/proofstep/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/proofstep_trajectory"]
+
+[tool.uv.sources]
+proofstep-types = { workspace = true }

proofstep_trajectory-0.1.0/src/proofstep_trajectory/__init__.py

@@ -0,0 +1,53 @@
+"""Proofstep trajectory policy engine (pure library — no I/O).
+
+Policy-as-code for tool-using agents: `(policy, trace) -> list[PolicyFailure]`.
+
+Output evaluation asks whether the answer was good. This asks whether the agent
+behaved legitimately on the way there — an agent can produce a flawless email and
+still have sent it before approval, which no output evaluator can detect.
+"""
+
+from importlib import metadata as _metadata
+
+from proofstep_trajectory.engine import evaluate_policy
+from proofstep_trajectory.events import (
+    EventRef,
+    PolicyFailure,
+    PolicyResult,
+    TrajectoryEvent,
+)
+from proofstep_trajectory.normalize import Normalized, args_hash, normalize
+from proofstep_trajectory.parser import (
+    LoadedPolicy,
+    PolicyError,
+    check_actions,
+    load_policy,
+    load_policy_file,
+)
+from proofstep_trajectory.predicates import PredicateError, compile_predicate
+from proofstep_trajectory.schema import Include, Policy
+
+# Read from the installed distribution rather than written here twice. A hand-maintained
+# copy drifts the first time a release bumps one and not the other — which it already did,
+# reporting 0.1.0.dev0 from a 0.1.0 wheel.
+__version__ = _metadata.version("proofstep-trajectory")
+
+__all__ = [
+    "EventRef",
+    "Include",
+    "LoadedPolicy",
+    "Normalized",
+    "Policy",
+    "PolicyError",
+    "PolicyFailure",
+    "PolicyResult",
+    "PredicateError",
+    "TrajectoryEvent",
+    "args_hash",
+    "check_actions",
+    "compile_predicate",
+    "evaluate_policy",
+    "load_policy",
+    "load_policy_file",
+    "normalize",
+]

proofstep_trajectory-0.1.0/src/proofstep_trajectory/engine.py

@@ -0,0 +1,71 @@
+"""Top-level evaluation: (policy, trace) -> PolicyResult."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Any
+
+from proofstep_trajectory.events import PolicyFailure, PolicyResult
+from proofstep_trajectory.matchers import MATCHERS, REQUIRES_COMPLETE
+from proofstep_trajectory.normalize import Normalized, normalize
+from proofstep_trajectory.parser import LoadedPolicy, check_actions
+from proofstep_trajectory.predicates import PredicateError, evaluate
+from proofstep_types import Trace
+
+
+def evaluate_policy(loaded: LoadedPolicy, trace: Trace) -> PolicyResult:
+    """Evaluate every rule in the policy against one trace."""
+    policy = loaded.policy
+    norm = normalize(trace, policy)
+
+    failures: list[PolicyFailure] = []
+    inconclusive: list[str] = []
+    warnings = [*norm.warnings, *check_actions(loaded, norm.actions)]
+
+    for rule in policy.rules:
+        # Absence of evidence is not evidence of absence: a rule that asserts
+        # something *did* happen cannot be judged on a trajectory that lost spans.
+        # Rules that observe a forbidden action still run, because seeing it is
+        # valid evidence no matter what is missing.
+        if norm.incomplete and rule.kind in REQUIRES_COMPLETE:
+            inconclusive.append(rule.id)
+            continue
+
+        if rule.when is not None and rule.kind != "conditional":
+            if not _guard(rule, norm, warnings):
+                continue
+        elif rule.kind == "conditional" and not _guard(rule, norm, warnings):
+            continue
+
+        for failure in MATCHERS[rule.kind](rule, norm):
+            failures.append(
+                failure if failure.policy_line else _with_line(failure, loaded.line_for(rule.id))
+            )
+
+    return PolicyResult(
+        policy_name=policy.name,
+        failures=failures,
+        events=norm.events,
+        incomplete=norm.incomplete,
+        inconclusive_rules=inconclusive,
+        warnings=warnings,
+    )
+
+
+def _guard(rule: Any, norm: Normalized, warnings: list[str]) -> bool:
+    """Evaluate a rule's `when` predicate. A broken predicate must not pass silently."""
+    namespace = {
+        "metadata": norm.state,
+        "state": norm.state,
+        "actions": dict.fromkeys(norm.actions, True),
+        "counts": {action: norm.counts(action) for action in norm.actions},
+    }
+    try:
+        return evaluate(rule.when, namespace)
+    except PredicateError as exc:
+        warnings.append(f"rule {rule.id!r}: `when` could not be evaluated ({exc}); rule skipped")
+        return False
+
+
+def _with_line(failure: PolicyFailure, line: int | None) -> PolicyFailure:
+    return replace(failure, policy_line=line)

proofstep_trajectory-0.1.0/src/proofstep_trajectory/evaluator.py

@@ -0,0 +1,93 @@
+"""Adapter exposing a policy as an evaluator.
+
+Deliberately placed here rather than in `proofstep-core`. The two packages are
+siblings in the layering contract and neither may import the other; this works
+because `Evaluator` is a *structural* protocol, so an object with the right shape
+satisfies it without any import. The only shared dependency is `proofstep-types`.
+
+That is the payoff of protocol-based extension points: the integration costs nothing
+architecturally.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from proofstep_trajectory.engine import evaluate_policy
+from proofstep_trajectory.parser import LoadedPolicy, load_policy_file
+from proofstep_types import Score, Severity
+
+
+class TrajectoryEvaluator:
+    """Scores an example by evaluating its captured trajectory against a policy.
+
+    Emits 1.0 when the policy holds and 0.0 when a blocking rule fails. An
+    incomplete trajectory produces an *errored* score rather than a zero: the run
+    failed to observe enough to judge, which is a different fact from observing a
+    violation, and gating on the difference is the whole point.
+    """
+
+    version = 1
+
+    def __init__(
+        self,
+        policy: LoadedPolicy | str | Path,
+        *,
+        name: str | None = None,
+        fail_on_warnings: bool = False,
+    ) -> None:
+        self.loaded = policy if isinstance(policy, LoadedPolicy) else load_policy_file(policy)
+        self.name = name or self.loaded.policy.name.replace("-", "_")
+        self.fail_on_warnings = fail_on_warnings
+        self.requires_trace = True
+
+    async def evaluate(self, ctx: Any) -> Score:
+        trace = getattr(ctx, "trace", None)
+        if trace is None:
+            return Score.failure(
+                self.name,
+                f"policy {self.loaded.policy.name!r} needs a captured trajectory but the "
+                "task produced no trace; is the task instrumented with the SDK?",
+            )
+
+        result = evaluate_policy(self.loaded, trace)
+
+        if result.inconclusive_rules:
+            return Score.failure(
+                self.name,
+                f"trajectory is incomplete, so {len(result.inconclusive_rules)} rule(s) "
+                f"could not be judged: {', '.join(result.inconclusive_rules)}",
+            )
+
+        failing = [
+            f
+            for f in result.failures
+            if f.severity is Severity.BLOCK or (self.fail_on_warnings and f.blocking is False)
+        ]
+        passed = not failing
+
+        return Score(
+            metric=self.name,
+            value=1.0 if passed else 0.0,
+            passed=passed,
+            reasoning=result.format(policy_path=self.loaded.path) if failing else None,
+            raw={
+                "policy": self.loaded.policy.name,
+                "policy_hash": self.loaded.content_hash,
+                "event_count": len(result.events),
+                "failures": [
+                    {
+                        "rule_id": f.rule_id,
+                        "rule_kind": f.rule_kind,
+                        "severity": f.severity.value,
+                        "message": f.message,
+                        "span_id": f.offending_span_id,
+                        "event_index": f.offending_event_index,
+                        "policy_line": f.policy_line,
+                    }
+                    for f in result.failures
+                ],
+                "warnings": result.warnings,
+            },
+        )

proofstep_trajectory-0.1.0/src/proofstep_trajectory/events.py

@@ -0,0 +1,150 @@
+"""The normalized trajectory event, and the result of evaluating a policy.
+
+Rules never touch spans. They operate on this flat, ordered event list, so a
+normalization fix corrects every rule at once instead of twelve times.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+from proofstep_types import Severity, Status
+
+
+@dataclass(frozen=True, slots=True)
+class TrajectoryEvent:
+    index: int
+    action: str
+    span_id: str
+    parent_span_id: str | None
+    depth: int
+    started_at: datetime
+    ended_at: datetime | None
+    status: Status
+    args: dict[str, Any] = field(default_factory=dict)
+    args_hash: str = ""
+    attempt: int = 1
+    is_retry: bool = False
+    parallel_group: str | None = None
+    result_summary: Any = None
+
+    @property
+    def failed(self) -> bool:
+        return self.status in (Status.ERROR, Status.TIMEOUT)
+
+    def key(self, parts: list[str]) -> tuple[Any, ...]:
+        """Composite identity used by unique_action and no_loop."""
+        values: list[Any] = []
+        for part in parts:
+            if part == "action":
+                values.append(self.action)
+            elif part == "args_hash":
+                values.append(self.args_hash)
+            elif part.startswith("args."):
+                values.append(_dig(self.args, part[5:]))
+            else:
+                values.append(getattr(self, part, None))
+        return tuple(values)
+
+
+def _dig(source: dict[str, Any], path: str) -> Any:
+    current: Any = source
+    for part in path.split("."):
+        if not isinstance(current, dict) or part not in current:
+            return None
+        current = current[part]
+    return current
+
+
+@dataclass(frozen=True, slots=True)
+class EventRef:
+    """A pointer back to a span, so every finding is clickable."""
+
+    index: int
+    span_id: str
+    action: str
+    at: datetime
+
+
+@dataclass(frozen=True, slots=True)
+class PolicyFailure:
+    rule_id: str
+    rule_kind: str
+    severity: Severity
+    message: str
+    offending_span_id: str | None = None
+    offending_event_index: int | None = None
+    offending_action: str = ""
+    expected: Any = None
+    actual: Any = None
+    evidence: list[EventRef] = field(default_factory=list)
+    policy_line: int | None = None
+
+    @property
+    def blocking(self) -> bool:
+        return self.severity is Severity.BLOCK
+
+    def format(self, *, policy_path: str | None = None) -> str:
+        """Human-readable rendering.
+
+        Message quality is a feature. Every failure states what happened, where,
+        what was expected, and how to go look — "policy violation" is banned.
+        """
+        mark = "✗" if self.blocking else "⚠"
+        lines = [f"{mark} {self.rule_id}  [{self.severity.value}]", f"  {self.message}"]
+        if self.offending_span_id:
+            label = f"{self.offending_action} " if self.offending_action else ""
+            where = f"    offending  : {label}span {self.offending_span_id}"
+            if self.offending_event_index is not None:
+                where += f" (event #{self.offending_event_index})"
+            lines.append(where)
+        if self.expected is not None:
+            lines.append(f"    expected   : {self.expected}")
+        if self.actual is not None:
+            lines.append(f"    observed   : {self.actual}")
+        if policy_path:
+            suffix = f":{self.policy_line}" if self.policy_line else ""
+            lines.append(f"    policy     : {policy_path}{suffix}")
+        return "\n".join(lines)
+
+
+@dataclass(frozen=True, slots=True)
+class PolicyResult:
+    policy_name: str
+    failures: list[PolicyFailure] = field(default_factory=list)
+    events: list[TrajectoryEvent] = field(default_factory=list)
+    incomplete: bool = False
+    inconclusive_rules: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        """Inconclusive is not passed.
+
+        A trajectory that lost spans cannot support an assertion about what did not
+        happen, and reporting that as compliance would be a false negative in the
+        direction that matters.
+        """
+        return not any(f.blocking for f in self.failures) and not self.inconclusive_rules
+
+    @property
+    def score(self) -> float:
+        return 1.0 if self.passed else 0.0
+
+    @property
+    def blocking_failures(self) -> list[PolicyFailure]:
+        return [f for f in self.failures if f.blocking]
+
+    def format(self, *, policy_path: str | None = None) -> str:
+        if not self.failures and not self.inconclusive_rules:
+            return f"✓ {self.policy_name}: {len(self.events)} events, no violations"
+        blocks = [f.format(policy_path=policy_path) for f in self.failures]
+        for rule_id in self.inconclusive_rules:
+            blocks.append(
+                f"? {rule_id}  [inconclusive]\n"
+                "  The trajectory is incomplete (spans were dropped or left open), so "
+                "this rule cannot assert that something did not happen."
+            )
+        return "\n\n".join(blocks)