stepfork 0.1.0a2__py3-none-any.whl

stepfork/diff/engine.py

@@ -0,0 +1,267 @@
+"""Semantic comparison of two recorded agent behaviors.
+
+Timestamps, event IDs, run IDs, and wall-clock durations are ignored by
+default. Comparison covers dependency call order, tool names, LLM models,
+inputs, outputs, error types, state changes, and run status.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from stepfork.diff.compare import collect_field_changes, values_equal
+from stepfork.diff.models import DiffResult, FieldChange, StepDiff, TraceSummary
+from stepfork.replay.plan import extract_recorded_calls
+from stepfork.trace import (
+    ErrorEvent,
+    JsonValue,
+    LLMRequest,
+    RunEnd,
+    StateChange,
+    ToolCall,
+    Trace,
+)
+from stepfork.trace.redaction import redact_json
+
+TraceInput = str | Path | Trace
+
+
+@dataclass(frozen=True)
+class BehaviorStep:
+    """One comparable unit of agent behavior extracted from a trace."""
+
+    step_type: str
+    label: str
+    payload: dict[str, JsonValue]
+
+
+def diff_traces(
+    baseline: TraceInput,
+    candidate: TraceInput,
+) -> DiffResult:
+    """Compare baseline and candidate traces as structured behavior.
+
+    Accepts `.sftrace` paths or already-loaded :class:`Trace` objects and
+    raises `TraceStorageError` for unreadable bundles.
+    """
+    baseline_trace = _load(baseline)
+    candidate_trace = _load(candidate)
+
+    baseline_steps = extract_behavior(baseline_trace)
+    candidate_steps = extract_behavior(candidate_trace)
+    steps = _diff_steps(baseline_steps, candidate_steps)
+
+    added = sum(step.kind == "added" for step in steps)
+    removed = sum(step.kind == "removed" for step in steps)
+    changed = sum(step.kind == "changed" for step in steps)
+    unchanged = sum(step.kind == "unchanged" for step in steps)
+
+    return DiffResult(
+        equivalent=added == 0 and removed == 0 and changed == 0,
+        baseline=_summary(baseline_trace, baseline),
+        candidate=_summary(candidate_trace, candidate),
+        steps=steps,
+        added=added,
+        removed=removed,
+        changed=changed,
+        unchanged=unchanged,
+    )
+
+
+def extract_behavior(trace: Trace) -> list[BehaviorStep]:
+    """Extract the ordered, sanitized behavior steps from a trace."""
+    calls = {call.call_event_id: call for call in extract_recorded_calls(trace)}
+    steps: list[BehaviorStep] = []
+
+    for event in trace.events:
+        if isinstance(event, ToolCall | LLMRequest):
+            call = calls.get(event.id)
+            if call is None:  # pragma: no cover - defensive
+                continue
+            steps.append(
+                BehaviorStep(
+                    step_type=call.kind,
+                    label=call.label,
+                    payload=_sanitized_payload(
+                        input=call.input,
+                        output=call.output,
+                        status=call.status.value,
+                    ),
+                )
+            )
+        elif isinstance(event, StateChange):
+            steps.append(
+                BehaviorStep(
+                    step_type="state",
+                    label=event.key or "state",
+                    payload={
+                        "before": _sanitize(event.before),
+                        "after": _sanitize(event.after),
+                    },
+                )
+            )
+        elif isinstance(event, ErrorEvent):
+            steps.append(
+                BehaviorStep(
+                    step_type="error",
+                    label=event.error_type,
+                    payload={"message": _sanitize(event.message)},
+                )
+            )
+        elif isinstance(event, RunEnd):
+            steps.append(
+                BehaviorStep(
+                    step_type="run",
+                    label="run",
+                    payload={
+                        "run_status": event.run_status.value,
+                        "output": _sanitize(event.output),
+                    },
+                )
+            )
+
+    return steps
+
+
+def _diff_steps(
+    baseline_steps: list[BehaviorStep],
+    candidate_steps: list[BehaviorStep],
+) -> list[StepDiff]:
+    diffs: list[StepDiff] = []
+    total = max(len(baseline_steps), len(candidate_steps))
+
+    for index in range(total):
+        if index >= len(baseline_steps):
+            step = candidate_steps[index]
+            diffs.append(
+                StepDiff(
+                    index=index,
+                    kind="added",
+                    step_type=step.step_type,
+                    label=step.label,
+                    changes=[],
+                )
+            )
+            continue
+        if index >= len(candidate_steps):
+            step = baseline_steps[index]
+            diffs.append(
+                StepDiff(
+                    index=index,
+                    kind="removed",
+                    step_type=step.step_type,
+                    label=step.label,
+                    changes=[],
+                )
+            )
+            continue
+
+        baseline_step = baseline_steps[index]
+        candidate_step = candidate_steps[index]
+        changes: list[FieldChange] = []
+
+        if baseline_step.step_type != candidate_step.step_type:
+            changes.append(
+                FieldChange(
+                    path="type",
+                    kind="changed",
+                    expected=baseline_step.step_type,
+                    actual=candidate_step.step_type,
+                )
+            )
+        if baseline_step.label != candidate_step.label:
+            changes.append(
+                FieldChange(
+                    path="label",
+                    kind="changed",
+                    expected=baseline_step.label,
+                    actual=candidate_step.label,
+                )
+            )
+
+        keys = sorted(set(baseline_step.payload) | set(candidate_step.payload))
+        for key in keys:
+            if key not in baseline_step.payload:
+                changes.append(
+                    FieldChange(
+                        path=key,
+                        kind="added",
+                        expected=None,
+                        actual=candidate_step.payload[key],
+                    )
+                )
+            elif key not in candidate_step.payload:
+                changes.append(
+                    FieldChange(
+                        path=key,
+                        kind="removed",
+                        expected=baseline_step.payload[key],
+                        actual=None,
+                    )
+                )
+            else:
+                collect_field_changes(
+                    baseline_step.payload[key],
+                    candidate_step.payload[key],
+                    path=key,
+                    output=changes,
+                )
+
+        diffs.append(
+            StepDiff(
+                index=index,
+                kind="changed" if changes else "unchanged",
+                step_type=baseline_step.step_type,
+                label=baseline_step.label,
+                fields=keys,
+                changes=changes,
+            )
+        )
+
+    return diffs
+
+
+def _sanitized_payload(
+    *,
+    input: JsonValue,
+    output: JsonValue | None,
+    status: str,
+) -> dict[str, JsonValue]:
+    return {
+        "input": _sanitize(input),
+        "output": _sanitize(output),
+        "status": status,
+    }
+
+
+def _sanitize(value: JsonValue | None) -> JsonValue:
+    return redact_json(value).value
+
+
+def _summary(trace: Trace, source: TraceInput) -> TraceSummary:
+    path = str(source) if isinstance(source, str | Path) else None
+    return TraceSummary(
+        path=path,
+        agent_name=trace.agent_name,
+        run_id=trace.run_id,
+        status=trace.status.value,
+        events=len(trace.events),
+    )
+
+
+def _load(source: TraceInput) -> Trace:
+    if isinstance(source, Trace):
+        return source
+    return Trace.load(Path(source))
+
+
+def steps_equivalent(left: BehaviorStep, right: BehaviorStep) -> bool:
+    """Return True when two behavior steps describe identical behavior."""
+    if left.step_type != right.step_type or left.label != right.label:
+        return False
+    if set(left.payload) != set(right.payload):
+        return False
+    return all(
+        values_equal(left.payload[key], right.payload[key]) for key in left.payload
+    )

stepfork/diff/models.py

@@ -0,0 +1,63 @@
+"""Structured, machine-readable behavioral diff models."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stepfork.trace.models import JsonValue
+
+ChangeKind = Literal["added", "removed", "changed", "unchanged"]
+LeafChangeKind = Literal["added", "removed", "changed"]
+
+
+class FieldChange(BaseModel):
+    """One changed field difference between baseline and candidate."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    path: str
+    kind: LeafChangeKind
+    expected: JsonValue | None = None
+    actual: JsonValue | None = None
+
+
+class StepDiff(BaseModel):
+    """Comparison result for one aligned behavior step."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    index: int
+    kind: ChangeKind
+    step_type: str
+    label: str
+    fields: list[str] = Field(default_factory=list)
+    changes: list[FieldChange] = Field(default_factory=list)
+
+
+class TraceSummary(BaseModel):
+    """Identity metadata for one side of the comparison."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    path: str | None = None
+    agent_name: str
+    run_id: str
+    status: str
+    events: int
+
+
+class DiffResult(BaseModel):
+    """Structured behavioral comparison between two traces."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    equivalent: bool
+    baseline: TraceSummary
+    candidate: TraceSummary
+    steps: list[StepDiff] = Field(default_factory=list)
+    added: int = 0
+    removed: int = 0
+    changed: int = 0
+    unchanged: int = 0

stepfork/export/__init__.py

@@ -0,0 +1,20 @@
+"""Regression-test exporters."""
+
+from stepfork.export.entrypoint import EntrypointError, resolve_entrypoint
+from stepfork.export.generator import (
+    ExportError,
+    ExportExistsError,
+    export_pytest_test,
+    generate_pytest_source,
+)
+from stepfork.export.runtime import run_regression_case
+
+__all__ = [
+    "EntrypointError",
+    "ExportError",
+    "ExportExistsError",
+    "export_pytest_test",
+    "generate_pytest_source",
+    "resolve_entrypoint",
+    "run_regression_case",
+]

stepfork/export/entrypoint.py

@@ -0,0 +1,58 @@
+"""Trusted resolution of ``MODULE:FUNCTION`` entrypoints.
+
+Stepfork only imports entrypoints the user passes explicitly on the command
+line or in API calls. Trace bundles are data and are never imported, evaluated,
+or executed.
+"""
+
+from __future__ import annotations
+
+import importlib
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, cast
+
+
+class EntrypointError(ValueError):
+    """Raised when an entrypoint specification cannot be resolved."""
+
+
+def resolve_entrypoint(
+    spec: str,
+    *,
+    import_root: Path | None = None,
+) -> Callable[[], Any]:
+    """Import ``MODULE:FUNCTION`` and return the zero-argument callable.
+
+    ``import_root`` (when given) is prepended to ``sys.path`` so local
+    project modules such as ``examples.booking_agent`` resolve regardless of
+    how Python was started.
+    """
+    module_name, separator, attribute_path = spec.partition(":")
+    if not separator or not module_name or not attribute_path:
+        raise EntrypointError(
+            f"invalid entrypoint {spec!r}; expected the form MODULE:FUNCTION"
+        )
+
+    if import_root is not None:
+        root = str(import_root)
+        if root not in sys.path:
+            sys.path.insert(0, root)
+
+    try:
+        module = importlib.import_module(module_name)
+    except Exception as exc:
+        raise EntrypointError(
+            f"cannot import module {module_name!r}: {type(exc).__name__}: {exc}"
+        ) from exc
+
+    target: Any = module
+    for part in attribute_path.split("."):
+        if not hasattr(target, part):
+            raise EntrypointError(f"module {module_name!r} has no attribute {part!r}")
+        target = getattr(target, part)
+
+    if not callable(target):
+        raise EntrypointError(f"entrypoint {spec!r} is not callable")
+    return cast("Callable[[], Any]", target)

stepfork/export/generator.py

@@ -0,0 +1,178 @@
+"""Generation of executable pytest regression tests from `.sftrace` bundles.
+
+Generated files come from a fixed, reviewed template. Trace payloads are never
+interpolated into executable code: only sanitized JSON literals, paths, and
+the user-supplied entrypoint specification are embedded.
+"""
+
+from __future__ import annotations
+
+import os
+import pprint
+import re
+from pathlib import Path
+
+from stepfork.trace import JsonValue
+from stepfork.trace.redaction import redact_json
+from stepfork.version import __version__
+
+MAX_RELATIVE_PARENTS = 3
+
+TEMPLATE = '''"""Stepfork regression test for {title}.
+
+Generated by stepfork {version} from the trace bundle `{trace_name}`.
+Regenerate with the same `stepfork export` command plus `--overwrite`.
+
+This test executes the trusted local entrypoint `{entrypoint}` under frozen
+Stepfork replay and asserts the behavior recorded in EXPECTATION. Stepfork
+never executes code embedded in a trace bundle.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from stepfork.export.runtime import run_regression_case
+
+TRACE_PATH = {trace_expr}
+IMPORT_ROOT = {import_root_expr}
+ENTRYPOINT = {entrypoint_literal}
+MODE = {mode_literal}
+EXPECTATION = {expectation_literal}
+HAS_EXPECTATION = {has_expectation_literal}
+
+
+def {function_name}() -> None:
+    """Fail when the agent diverges from the expected outcome."""
+    run_regression_case(
+        trace_path=TRACE_PATH,
+        import_root=IMPORT_ROOT,
+        entrypoint=ENTRYPOINT,
+        expectation=EXPECTATION,
+        has_expectation=HAS_EXPECTATION,
+        mode=MODE,
+    )
+'''
+
+
+class ExportError(ValueError):
+    """Raised when a regression test cannot be generated."""
+
+
+class ExportExistsError(ExportError):
+    """Raised when the output file exists and overwrite was not requested."""
+
+
+def generate_pytest_source(
+    *,
+    trace_path: Path,
+    output_path: Path,
+    entrypoint: str,
+    expectation: JsonValue | None = None,
+    has_expectation: bool = True,
+    mode: str = "frozen",
+    import_root: Path | None = None,
+) -> str:
+    """Return the source code for a pytest regression test."""
+    trace = trace_path.resolve()
+    root = (import_root if import_root is not None else Path.cwd()).resolve()
+
+    return TEMPLATE.format(
+        title=_docstring_text(_trace_title(trace)),
+        version=_docstring_text(__version__),
+        trace_name=_docstring_text(trace.name),
+        entrypoint=_docstring_text(entrypoint),
+        trace_expr=_path_expression(trace, output_path),
+        import_root_expr=_path_expression(root, output_path),
+        entrypoint_literal=repr(entrypoint),
+        mode_literal=repr(mode),
+        expectation_literal=pprint.pformat(
+            redact_json(expectation).value if has_expectation else None,
+            width=88,
+            sort_dicts=True,
+        ),
+        has_expectation_literal=repr(has_expectation),
+        function_name=_test_function_name(trace),
+    )
+
+
+def export_pytest_test(
+    *,
+    trace_path: Path,
+    output_path: Path,
+    entrypoint: str,
+    expectation: JsonValue | None = None,
+    has_expectation: bool = True,
+    mode: str = "frozen",
+    import_root: Path | None = None,
+    overwrite: bool = False,
+) -> Path:
+    """Write a pytest regression test and return its path."""
+    if output_path.exists() and not overwrite:
+        raise ExportExistsError(
+            f"{output_path} already exists; pass --overwrite to replace it"
+        )
+    if output_path.exists() and output_path.is_dir():
+        raise ExportError(f"{output_path} is a directory, not a file")
+
+    source = generate_pytest_source(
+        trace_path=trace_path,
+        output_path=output_path,
+        entrypoint=entrypoint,
+        expectation=expectation,
+        has_expectation=has_expectation,
+        mode=mode,
+        import_root=import_root,
+    )
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(source, encoding="utf-8")
+    return output_path
+
+
+def _path_expression(target: Path, output_path: Path) -> str:
+    output_dir = output_path.parent.resolve()
+    try:
+        relative = os.path.relpath(target, output_dir)
+    except ValueError:  # different drives on Windows
+        return f"Path({str(target)!r})"
+
+    parts = Path(relative).parts
+    if parts.count("..") > MAX_RELATIVE_PARENTS:
+        return f"Path({str(target)!r})"
+    if not parts:
+        return "Path(__file__).resolve().parent"
+    suffix = "".join(f" / {part!r}" for part in parts)
+    return f"Path(__file__).resolve().parent{suffix}"
+
+
+def _safe_stem(trace_path: Path) -> str:
+    stem = re.sub(r"\W+", "_", trace_path.stem).strip("_")
+    return stem
+
+
+def _docstring_text(value: str, *, limit: int = 200) -> str:
+    """Return text that is safe to embed inside a triple-quoted docstring.
+
+    Trace directory names and entrypoint specs are user- or attacker-supplied.
+    Interpolating them verbatim lets a triple double-quote or backslash
+    sequence terminate the module docstring early and turn the remainder into
+    code. Collapse whitespace, neutralize quotes and backslashes, and bound
+    the length.
+    """
+    collapsed = " ".join(value.split())
+    escaped = collapsed.replace("\\", "\\\\").replace('"', '\\"')
+    if len(escaped) > limit:
+        escaped = escaped[: limit - 1] + "…"
+    return escaped
+
+
+def _trace_title(trace_path: Path) -> str:
+    stem = _safe_stem(trace_path)
+    return stem or "agent run"
+
+
+def _test_function_name(trace_path: Path) -> str:
+    stem = _safe_stem(trace_path)
+    if not stem:
+        return "test_stepfork_regression"
+    return f"test_stepfork_regression_{stem}"

stepfork/export/runtime.py

@@ -0,0 +1,101 @@
+"""Runtime helper executed by Stepfork-generated pytest regression tests.
+
+This module is deliberately small, fixed, and reviewed. Generated tests only
+assemble literals and call :func:`run_regression_case`; no generated code is
+ever derived from trace payloads.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from stepfork.diff.compare import first_difference
+from stepfork.export.entrypoint import EntrypointError, resolve_entrypoint
+from stepfork.replay import (
+    ReplayError,
+    ReplayMismatchError,
+    ReplayPolicyError,
+    ReplaySession,
+)
+from stepfork.trace import (
+    JsonValue,
+    Trace,
+    TraceSerializationError,
+    TraceStorageError,
+    canonical_json_bytes,
+    to_json_value,
+)
+
+PREVIEW_LIMIT = 300
+
+
+def run_regression_case(
+    *,
+    trace_path: Path,
+    import_root: Path | None,
+    entrypoint: str,
+    expectation: JsonValue | None = None,
+    has_expectation: bool = True,
+    mode: str = "frozen",
+) -> JsonValue:
+    """Replay a trusted entrypoint and assert the expected behavior.
+
+    Fails with an actionable ``AssertionError`` when the trace is missing, the
+    entrypoint cannot load, replay diverges from the recorded dependency
+    sequence, or the returned value differs from ``expectation``.
+    """
+    bundle = Path(trace_path)
+    if not bundle.exists():
+        raise AssertionError(f"stepfork: trace bundle not found: {bundle}")
+    try:
+        recorded = Trace.load(bundle)
+    except TraceStorageError as exc:
+        raise AssertionError(f"stepfork: trace bundle unreadable: {exc}") from exc
+
+    try:
+        entry = resolve_entrypoint(entrypoint, import_root=import_root)
+    except EntrypointError as exc:
+        raise AssertionError(f"stepfork: entrypoint failed: {exc}") from exc
+
+    with ReplaySession.from_trace(recorded, mode=mode) as replay:
+        try:
+            actual = entry()
+        except ReplayMismatchError as exc:
+            raise AssertionError(f"stepfork replay divergence: {exc}") from exc
+        except ReplayPolicyError as exc:
+            raise AssertionError(f"stepfork replay policy violation: {exc}") from exc
+        try:
+            replay.verify_complete()
+        except ReplayError as exc:
+            raise AssertionError(f"stepfork replay divergence: {exc}") from exc
+
+    try:
+        actual_json = to_json_value(actual)
+    except TraceSerializationError as exc:
+        raise AssertionError(
+            f"stepfork: entrypoint returned a non-serializable value: {exc}"
+        ) from exc
+
+    if has_expectation:
+        difference = first_difference(expectation, actual_json)
+        if difference is not None:
+            path, expected_value, actual_value = difference
+            raise AssertionError(
+                f"behavior mismatch at '{path}': "
+                f"expected {_preview(expected_value)}, "
+                f"got {_preview(actual_value)}"
+            )
+
+    return actual_json
+
+
+def _preview(value: JsonValue | None) -> str:
+    if value is None:
+        return "None"
+    try:
+        text = canonical_json_bytes(value).decode("utf-8", errors="replace")
+    except TypeError:
+        return "<unserializable>"
+    if len(text) > PREVIEW_LIMIT:
+        return f"{text[: PREVIEW_LIMIT - 1]}…"
+    return text

stepfork/fork/__init__.py

@@ -0,0 +1,4 @@
+"""Trace forking.
+
+Planned for Stepfork v0.2.
+"""

stepfork/inspect/__init__.py

@@ -0,0 +1,11 @@
+"""Trace inspection helpers."""
+
+from stepfork.inspect.inspector import inspect_bundle, inspect_trace
+from stepfork.inspect.models import EventSummary, TraceInspection
+
+__all__ = [
+    "EventSummary",
+    "TraceInspection",
+    "inspect_bundle",
+    "inspect_trace",
+]