stepfork 0.1.0a2__py3-none-any.whl

stepfork/__init__.py

@@ -0,0 +1,32 @@
+"""Stepfork: behavioral regression testing for AI agents."""
+
+from stepfork.diff import diff_traces
+from stepfork.recorder import RecordingSession, llm_request, record, trace_tool
+from stepfork.replay import (
+    RecordedDependencyError,
+    ReplayError,
+    ReplayExhaustedError,
+    ReplayMismatchError,
+    ReplayPolicyError,
+    ReplaySession,
+)
+from stepfork.trace import FailureInfo, ToolCall, Trace
+from stepfork.version import __version__
+
+__all__ = [
+    "FailureInfo",
+    "RecordedDependencyError",
+    "RecordingSession",
+    "ReplayError",
+    "ReplayExhaustedError",
+    "ReplayMismatchError",
+    "ReplayPolicyError",
+    "ReplaySession",
+    "ToolCall",
+    "Trace",
+    "__version__",
+    "diff_traces",
+    "llm_request",
+    "record",
+    "trace_tool",
+]

stepfork/__main__.py

@@ -0,0 +1,6 @@
+"""Allow `python -m stepfork` to run the CLI."""
+
+from stepfork.cli.main import app
+
+if __name__ == "__main__":
+    app()

stepfork/cli/__init__.py

@@ -0,0 +1 @@
+"""Command-line interface for Stepfork."""

stepfork/cli/diff.py

@@ -0,0 +1,156 @@
+"""`stepfork diff` command."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+from rich.console import Console
+
+from stepfork.diff import DiffResult, FieldChange, StepDiff, diff_traces
+from stepfork.trace import TraceStorageError
+from stepfork.trace.redaction import redact_json
+
+console = Console()
+VALUE_LIMIT = 120
+
+RESULT_EQUIVALENT = "BEHAVIOR EQUIVALENT"
+RESULT_CHANGED = "BEHAVIOR CHANGED"
+
+
+def diff_command(
+    baseline: Annotated[
+        Path,
+        typer.Argument(
+            help="Baseline .sftrace directory bundle.",
+            exists=False,
+            file_okay=False,
+            dir_okay=True,
+        ),
+    ],
+    candidate: Annotated[
+        Path,
+        typer.Argument(
+            help="Candidate .sftrace directory bundle.",
+            exists=False,
+            file_okay=False,
+            dir_okay=True,
+        ),
+    ],
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Emit machine-readable JSON (no Rich markup)."),
+    ] = False,
+) -> None:
+    """Compare the behavior of two recorded agent runs.
+
+    Exit codes: 0 behavior equivalent, 1 meaningful behavioral difference,
+    2 invalid or unreadable input.
+    """
+    try:
+        result = diff_traces(baseline, candidate)
+    except (TraceStorageError, OSError) as exc:
+        if json_output:
+            _write_json(
+                {
+                    "error": "unreadable_trace",
+                    "message": _sanitize_text(str(exc)),
+                }
+            )
+        else:
+            console.print("[red]Unable to compare traces.[/red]")
+            console.print(_sanitize_text(str(exc)))
+        raise typer.Exit(2) from exc
+
+    if json_output:
+        sys.stdout.write(result.model_dump_json())
+        sys.stdout.write("\n")
+        raise typer.Exit(0 if result.equivalent else 1)
+
+    _print_diff(result)
+    raise typer.Exit(0 if result.equivalent else 1)
+
+
+def _print_diff(result: DiffResult) -> None:
+    console.print("[bold]Stepfork Behavioral Diff[/bold]")
+    console.print()
+    console.print(f"Baseline:  {result.baseline.path or '(in-memory trace)'}")
+    console.print(f"Candidate: {result.candidate.path or '(in-memory trace)'}")
+    console.print()
+
+    for step in result.steps:
+        _print_step(step)
+
+    if result.changed or result.added or result.removed:
+        console.print(
+            f"Steps: {len(result.steps)} total "
+            f"({result.changed} changed, {result.added} added, "
+            f"{result.removed} removed, {result.unchanged} unchanged)"
+        )
+        console.print()
+        console.print(f"[red]Result: {RESULT_CHANGED}[/red]")
+    else:
+        console.print(f"Steps: {len(result.steps)} total (all unchanged)")
+        console.print()
+        console.print(f"[green]Result: {RESULT_EQUIVALENT}[/green]")
+
+
+def _print_step(step: StepDiff) -> None:
+    header = f"{step.step_type}: {step.label}"
+    if step.kind == "added":
+        console.print(f"[green]+[/green] {header} [green](added)[/green]")
+        return
+    if step.kind == "removed":
+        console.print(f"[red]-[/red] {header} [red](removed)[/red]")
+        return
+
+    console.print(header)
+    if not step.changes:
+        console.print("  unchanged")
+        return
+
+    groups: dict[str, list[FieldChange]] = {}
+    for change in step.changes:
+        key = change.path.split(".", 1)[0].split("[", 1)[0]
+        groups.setdefault(key, []).append(change)
+    for field in step.fields:
+        groups.setdefault(field, [])
+
+    for key, changes in groups.items():
+        if not changes:
+            console.print(f"  {key}: unchanged")
+            continue
+        console.print(f"  {key}:")
+        for change in changes:
+            console.print(f"    expected: {_short(change.expected, absent=True)}")
+            console.print(f"    actual:   {_short(change.actual, absent=True)}")
+            if change.path != key:
+                console.print(f"    at:       {change.path}")
+
+
+def _short(value: Any, *, absent: bool = False) -> str:
+    if value is None:
+        return "(absent)" if absent else "None"
+    sanitized = redact_json(value).value
+    try:
+        text = json.dumps(sanitized, ensure_ascii=False, sort_keys=True)
+    except (TypeError, ValueError):
+        return "<unserializable>"
+    if len(text) > VALUE_LIMIT:
+        return f"{text[: VALUE_LIMIT - 1]}…"
+    return text
+
+
+def _sanitize_text(value: str) -> str:
+    sanitized = redact_json(value).value
+    if isinstance(sanitized, str):
+        return sanitized
+    return str(sanitized)
+
+
+def _write_json(payload: dict[str, Any]) -> None:
+    sys.stdout.write(json.dumps(payload, sort_keys=True))
+    sys.stdout.write("\n")

stepfork/cli/export.py

@@ -0,0 +1,210 @@
+"""`stepfork export` command."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from stepfork.export.entrypoint import EntrypointError, resolve_entrypoint
+from stepfork.export.generator import (
+    ExportError,
+    ExportExistsError,
+    export_pytest_test,
+)
+from stepfork.trace import (
+    JsonValue,
+    RunEnd,
+    RunStatus,
+    Trace,
+    TraceStorageError,
+    validate_bundle,
+)
+from stepfork.trace.redaction import redact_json
+from stepfork.trace.replay_policy import ReplayPolicy
+
+console = Console()
+RECORDED_OUTPUT_WARNING = (
+    "[yellow]warning:[/yellow] no --expect-output given; the test will "
+    "assert the recorded run output. That passes against the code that "
+    "produced the recording. Pass --expect-output to assert the corrected "
+    "behavior instead."
+)
+
+
+def export_command(
+    path: Annotated[
+        Path,
+        typer.Argument(
+            help="Path to a .sftrace directory bundle.",
+            exists=False,
+            file_okay=False,
+            dir_okay=True,
+        ),
+    ],
+    pytest_format: Annotated[
+        bool,
+        typer.Option(
+            "--pytest/--no-pytest",
+            help="Export a pytest test (the only v0.1 format).",
+        ),
+    ] = True,
+    entrypoint: Annotated[
+        str | None,
+        typer.Option(
+            "--entrypoint",
+            help="Trusted local MODULE:FUNCTION the generated test executes.",
+        ),
+    ] = None,
+    expect_output: Annotated[
+        Path | None,
+        typer.Option(
+            "--expect-output",
+            help="JSON file defining the expected regression outcome.",
+        ),
+    ] = None,
+    output: Annotated[
+        Path | None,
+        typer.Option(
+            "--output",
+            help="Destination test file (default: ./test_<trace>_regression.py).",
+        ),
+    ] = None,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            help="Replace the output file if it already exists.",
+        ),
+    ] = False,
+    mode: Annotated[
+        str,
+        typer.Option(
+            "--mode",
+            help="Replay mode used by the generated test (frozen recommended).",
+        ),
+    ] = "frozen",
+) -> None:
+    """Export an executable pytest regression test from a trace.
+
+    The generated test loads the frozen trace, executes your trusted
+    entrypoint under replay, and fails when behavior diverges from the
+    expected outcome.
+    """
+    if not pytest_format:
+        console.print("[red]v0.1 supports pytest export only (--pytest).[/red]")
+        raise typer.Exit(2)
+
+    try:
+        ReplayPolicy(mode)
+    except ValueError:
+        console.print(f"[red]Invalid replay mode {mode!r}.[/red]")
+        raise typer.Exit(2) from None
+
+    if entrypoint is None:
+        console.print(
+            "[red]An executable regression test needs --entrypoint "
+            "MODULE:FUNCTION.[/red]"
+        )
+        raise typer.Exit(2)
+
+    result = validate_bundle(path, strict=True)
+    if not result.valid:
+        console.print("[red]Trace failed validation.[/red]")
+        for issue in result.issues:
+            console.print(f"  {issue.code}: {issue.message}")
+        raise typer.Exit(2)
+
+    try:
+        trace = Trace.load(path)
+    except TraceStorageError as exc:
+        console.print("[red]Unable to read trace.[/red]")
+        console.print(_sanitize_text(str(exc)))
+        raise typer.Exit(2) from exc
+
+    try:
+        resolve_entrypoint(entrypoint, import_root=Path.cwd())
+    except EntrypointError as exc:
+        console.print("[red]Unable to resolve entrypoint.[/red]")
+        console.print(_sanitize_text(str(exc)))
+        raise typer.Exit(2) from exc
+
+    warning: str | None = None
+    if expect_output is not None:
+        expectation = _load_expectation(expect_output)
+        has_expectation = True
+    else:
+        expectation = _recorded_output(trace)
+        has_expectation = expectation is not None
+        warning = RECORDED_OUTPUT_WARNING
+
+    destination = output or Path(f"test_{_safe_stem(path)}_regression.py")
+    if destination.exists() and not overwrite:
+        console.print(
+            f"[red]{destination} already exists; pass --overwrite to replace it.[/red]"
+        )
+        raise typer.Exit(1)
+
+    try:
+        written = export_pytest_test(
+            trace_path=path,
+            output_path=destination,
+            entrypoint=entrypoint,
+            expectation=expectation,
+            has_expectation=has_expectation,
+            mode=mode,
+            import_root=Path.cwd(),
+            overwrite=overwrite,
+        )
+    except ExportExistsError as exc:
+        console.print(f"[red]{_sanitize_text(str(exc))}[/red]")
+        raise typer.Exit(1) from exc
+    except ExportError as exc:
+        console.print(f"[red]Export failed: {_sanitize_text(str(exc))}[/red]")
+        raise typer.Exit(1) from exc
+
+    if warning:
+        console.print(warning)
+    console.print(f"[green]Wrote[/green] {written}")
+    console.print(f"Entrypoint: {entrypoint}")
+    console.print(f"Mode:       {mode}")
+    if has_expectation:
+        console.print(f"Expectation: {_sanitize_text(json.dumps(expectation))}")
+    else:
+        console.print("Expectation: none (replay fidelity only)")
+    raise typer.Exit(0)
+
+
+def _load_expectation(path: Path) -> JsonValue:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except OSError as exc:
+        console.print(f"[red]Cannot read --expect-output:[/red] {exc}")
+        raise typer.Exit(2) from exc
+    except json.JSONDecodeError as exc:
+        console.print(f"[red]--expect-output is not valid JSON:[/red] {exc.msg}")
+        raise typer.Exit(2) from exc
+    return redact_json(payload).value
+
+
+def _recorded_output(trace: Trace) -> JsonValue | None:
+    for event in reversed(trace.events):
+        if isinstance(event, RunEnd) and event.run_status is RunStatus.COMPLETED:
+            return event.output
+    return None
+
+
+def _safe_stem(path: Path) -> str:
+    stem = re.sub(r"\W+", "_", path.stem).strip("_")
+    return stem or "trace"
+
+
+def _sanitize_text(value: str) -> str:
+    sanitized = redact_json(value).value
+    if isinstance(sanitized, str):
+        return sanitized
+    return str(sanitized)

stepfork/cli/inspect.py

@@ -0,0 +1,155 @@
+"""`stepfork inspect` command."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from stepfork.inspect.inspector import filter_timeline, inspect_bundle
+from stepfork.inspect.models import TraceInspection
+from stepfork.trace import TraceStorageError
+from stepfork.trace.redaction import redact_json
+
+console = Console()
+MAX_DETAIL_LENGTH = 180
+
+
+def _sanitize_text(value: str) -> str:
+    sanitized = redact_json(value).value
+    if isinstance(sanitized, str):
+        return sanitized
+    return str(sanitized)
+
+
+def inspect_command(
+    path: Annotated[
+        Path,
+        typer.Argument(
+            help="Path to a .sftrace directory bundle.",
+            exists=False,
+            file_okay=False,
+            dir_okay=True,
+            readable=True,
+        ),
+    ],
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Emit sanitized machine-readable JSON."),
+    ] = False,
+    events: Annotated[
+        bool,
+        typer.Option("--events", help="Show sanitized event details."),
+    ] = False,
+    errors_only: Annotated[
+        bool,
+        typer.Option("--errors-only", help="Show only error events."),
+    ] = False,
+    step: Annotated[
+        int | None,
+        typer.Option("--step", help="Show events at a specific logical step."),
+    ] = None,
+) -> None:
+    """Inspect a `.sftrace` bundle."""
+    try:
+        inspection = filter_timeline(
+            inspect_bundle(path),
+            errors_only=errors_only,
+            step=step,
+        )
+    except TraceStorageError as exc:
+        message = _sanitize_text(str(exc))
+        if json_output:
+            _write_json(
+                {
+                    "error": "unreadable_trace",
+                    "message": message,
+                }
+            )
+        else:
+            console.print("[red]Unable to inspect trace.[/red]")
+            console.print(message)
+        raise typer.Exit(2) from exc
+
+    if step is not None and not inspection.timeline:
+        message = f"No events found at step {step}."
+        if json_output:
+            _write_json({"error": "missing_step", "message": message})
+        else:
+            console.print(f"[red]{message}[/red]")
+        raise typer.Exit(1)
+
+    if json_output:
+        sys.stdout.write(inspection.model_dump_json())
+        sys.stdout.write("\n")
+        raise typer.Exit(0)
+
+    _print_summary(inspection)
+    _print_timeline(inspection, show_details=events)
+    raise typer.Exit(0)
+
+
+def _print_summary(inspection: TraceInspection) -> None:
+    console.print("[bold]Stepfork Trace Inspector[/bold]")
+    console.print()
+
+    table = Table.grid(padding=(0, 4))
+    table.add_column(style="bold")
+    table.add_column()
+    table.add_row("Agent:", inspection.agent_name)
+    table.add_row("Run:", inspection.run_id)
+    table.add_row("Status:", inspection.status.upper())
+    table.add_row("Events:", str(inspection.events))
+    table.add_row("Schema:", inspection.schema_version)
+    table.add_row("Created:", inspection.created_at)
+    table.add_row("LLM calls:", str(inspection.llm_calls))
+    table.add_row("Tool calls:", str(inspection.tool_calls))
+    if inspection.duration_ms is not None:
+        table.add_row("Duration:", f"{inspection.duration_ms} ms")
+    table.add_row("Integrity:", inspection.integrity.upper())
+    if inspection.failure:
+        table.add_row("Failure:", _compact(inspection.failure))
+    console.print(table)
+    console.print()
+
+
+def _print_timeline(inspection: TraceInspection, *, show_details: bool) -> None:
+    console.print("[bold]Event Timeline[/bold]")
+    console.print()
+    table = Table(show_lines=show_details)
+    table.add_column("Step", justify="right")
+    table.add_column("Type")
+    table.add_column("Name/Model")
+    table.add_column("Status")
+    if show_details:
+        table.add_column("Details")
+
+    for event in inspection.timeline:
+        row = [
+            str(event.step),
+            event.type,
+            event.label,
+            event.status,
+        ]
+        if show_details:
+            row.append(_compact(event.details))
+        table.add_row(*row)
+
+    console.print(table)
+
+
+def _compact(value: Any) -> str:
+    text = json.dumps(value, ensure_ascii=False, sort_keys=True)
+    if len(text) <= MAX_DETAIL_LENGTH:
+        return text
+    return f"{text[: MAX_DETAIL_LENGTH - 1]}…"
+
+
+def _write_json(value: dict[str, Any]) -> None:
+    sys.stdout.write(json.dumps(value, sort_keys=True))
+    sys.stdout.write("\n")

stepfork/cli/main.py

@@ -0,0 +1,54 @@
+"""Stepfork command-line interface."""
+
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from stepfork import __version__
+from stepfork.cli.diff import diff_command
+from stepfork.cli.export import export_command
+from stepfork.cli.inspect import inspect_command
+from stepfork.cli.replay import replay_command
+from stepfork.cli.validate import validate_command
+
+app = typer.Typer(
+    name="stepfork",
+    help="Behavioral regression testing for AI agents.",
+    invoke_without_command=True,
+)
+
+console = Console()
+
+app.command("diff")(diff_command)
+app.command("export")(export_command)
+app.command("inspect")(inspect_command)
+app.command("replay")(replay_command)
+app.command("validate")(validate_command)
+
+
+def version_callback(value: bool) -> None:
+    """Print the Stepfork version and exit."""
+    if value:
+        console.print(f"stepfork {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    ctx: typer.Context,
+    version: Annotated[
+        bool | None,
+        typer.Option(
+            "--version",
+            "-V",
+            callback=version_callback,
+            is_eager=True,
+            help="Show the Stepfork version and exit.",
+        ),
+    ] = None,
+) -> None:
+    """Turn failed AI-agent runs into reproducible regression tests."""
+    if ctx.invoked_subcommand is None:
+        console.print(ctx.get_help())
+        raise typer.Exit()