failstep 0.1.0__py3-none-any.whl

failstep/cli.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import sys
+from enum import StrEnum
+from pathlib import Path
+
+import typer
+
+from failstep import __version__
+from failstep.compare import compare_reports
+from failstep.diagnose import diagnose as diagnose_run
+from failstep.errors import ParseError
+from failstep.models import Report, Severity
+from failstep.parser import load_run
+from failstep.report import (
+    format_compare_json,
+    format_compare_markdown,
+    format_compare_terminal,
+    format_diagnose_json,
+    format_diagnose_markdown,
+    format_diagnose_terminal,
+    format_error_json,
+    format_error_terminal,
+    format_fix_json,
+    format_fix_markdown,
+    format_fix_terminal,
+    format_inspect_json,
+    format_inspect_markdown,
+    format_inspect_terminal,
+    format_internal_json,
+    format_internal_terminal,
+)
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    pretty_exceptions_enable=False,
+    pretty_exceptions_show_locals=False,
+    help="Lint one finished AI agent run from a file.",
+)
+
+
+class OutputFormat(StrEnum):
+    terminal = "terminal"
+    json = "json"
+    markdown = "markdown"
+
+
+class FailOn(StrEnum):
+    error = "error"
+    warning = "warning"
+
+
+def _display_path(path: Path) -> str:
+    try:
+        relative = path.resolve().relative_to(Path.cwd().resolve())
+        return relative.as_posix()
+    except ValueError:
+        return path.as_posix()
+
+
+def _emit_error(error: ParseError, output_format: OutputFormat) -> None:
+    if output_format is OutputFormat.json:
+        sys.stdout.write(format_error_json(error, __version__))
+    else:
+        sys.stdout.write(format_error_terminal(error))
+    raise typer.Exit(2)
+
+
+def _emit_internal(output_format: OutputFormat) -> None:
+    if output_format is OutputFormat.json:
+        sys.stdout.write(format_internal_json(__version__))
+    else:
+        sys.stdout.write(format_internal_terminal())
+    raise typer.Exit(3)
+
+
+def _load(path: Path, output_format: OutputFormat):
+    try:
+        return load_run(path)
+    except ParseError as error:
+        _emit_error(error, output_format)
+    except OSError as exc:
+        _emit_error(ParseError(str(exc), _display_path(path)), output_format)
+    raise RuntimeError("unreachable")
+
+
+def _write_diagnose(report: Report, output_format: OutputFormat) -> None:
+    if output_format is OutputFormat.json:
+        sys.stdout.write(format_diagnose_json(report, __version__))
+    elif output_format is OutputFormat.markdown:
+        sys.stdout.write(format_diagnose_markdown(report, __version__))
+    else:
+        sys.stdout.write(format_diagnose_terminal(report, __version__))
+
+
+def _exit_for(report: Report, fail_on: FailOn) -> int:
+    if not report.findings:
+        return 0
+    if fail_on is FailOn.warning:
+        return 1
+    if any(item.severity is Severity.error for item in report.findings):
+        return 1
+    return 0
+
+
+@app.command()
+def inspect(
+    trace: Path = typer.Argument(..., help="Path to a native JSON or JSONL trace."),
+    output_format: OutputFormat = typer.Option(
+        OutputFormat.terminal,
+        "--format",
+        help="terminal, json, or markdown.",
+    ),
+) -> None:
+    """Print the run. No verdict."""
+    run = _load(trace, output_format)
+    display = _display_path(trace)
+    if output_format is OutputFormat.json:
+        sys.stdout.write(format_inspect_json(run, display, __version__))
+    elif output_format is OutputFormat.markdown:
+        sys.stdout.write(format_inspect_markdown(run, display, __version__))
+    else:
+        sys.stdout.write(format_inspect_terminal(run, display, __version__))
+
+
+@app.command()
+def diagnose(
+    trace: Path = typer.Argument(..., help="Path to a native JSON or JSONL trace."),
+    output_format: OutputFormat = typer.Option(
+        OutputFormat.terminal,
+        "--format",
+        help="terminal, json, or markdown.",
+    ),
+    fail_on: FailOn = typer.Option(
+        FailOn.error,
+        "--fail-on",
+        help="Exit 1 on this severity or higher.",
+    ),
+    no_llm: bool = typer.Option(
+        False,
+        "--no-llm",
+        help="Skip LLM leftover (default path never calls a model).",
+    ),
+    no_redact: bool = typer.Option(
+        False,
+        "--no-redact",
+        help="Warn; secrets are still redacted before leftover requests.",
+    ),
+) -> None:
+    """Print a root cause from deterministic detectors."""
+    if no_redact:
+        sys.stderr.write(
+            "Secrets are still redacted before any leftover request.\n"
+        )
+    run = _load(trace, output_format)
+    try:
+        report = diagnose_run(
+            run, _display_path(trace), no_llm=no_llm
+        )
+    except Exception:
+        _emit_internal(output_format)
+    _write_diagnose(report, output_format)
+    code = _exit_for(report, fail_on)
+    if code:
+        raise typer.Exit(code)
+
+
+@app.command()
+def compare(
+    old: Path = typer.Argument(..., help="Older trace file."),
+    new: Path = typer.Argument(..., help="Newer trace file."),
+    output_format: OutputFormat = typer.Option(
+        OutputFormat.terminal,
+        "--format",
+        help="terminal, json, or markdown.",
+    ),
+) -> None:
+    """Count diffs between two diagnosed runs. Never calls leftover."""
+    old_run = _load(old, output_format)
+    new_run = _load(new, output_format)
+    try:
+        result = compare_reports(
+            diagnose_run(old_run, _display_path(old), no_llm=True),
+            diagnose_run(new_run, _display_path(new), no_llm=True),
+        )
+    except Exception:
+        _emit_internal(output_format)
+    if output_format is OutputFormat.json:
+        sys.stdout.write(format_compare_json(result, __version__))
+    elif output_format is OutputFormat.markdown:
+        sys.stdout.write(format_compare_markdown(result, __version__))
+    else:
+        sys.stdout.write(format_compare_terminal(result, __version__))
+    if result.has_diff():
+        raise typer.Exit(1)
+
+
+@app.command()
+def fix(
+    trace: Path = typer.Argument(..., help="Path to a native JSON or JSONL trace."),
+    output_format: OutputFormat = typer.Option(
+        OutputFormat.terminal,
+        "--format",
+        help="terminal, json, or markdown.",
+    ),
+) -> None:
+    """Print a suggested patch. Does not write files. Never calls leftover."""
+    run = _load(trace, output_format)
+    try:
+        report = diagnose_run(run, _display_path(trace), no_llm=True)
+    except Exception:
+        _emit_internal(output_format)
+    if output_format is OutputFormat.json:
+        sys.stdout.write(format_fix_json(report, __version__))
+    elif output_format is OutputFormat.markdown:
+        sys.stdout.write(format_fix_markdown(report, __version__))
+    else:
+        sys.stdout.write(format_fix_terminal(report, __version__))
+    if report.root_cause is not None:
+        raise typer.Exit(1)
+
+
+@app.command()
+def version() -> None:
+    """Print the tool version."""
+    sys.stdout.write(f"failstep {__version__}\n")

failstep/compare.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from failstep.diagnose import diagnose
+from failstep.models import Finding, Report, Run
+
+_RUN_INT_KEYS = ("steps", "duration_ms", "tokens_in", "tokens_out")
+
+
+@dataclass(frozen=True)
+class FieldDelta:
+    key: str
+    old: Any
+    new: Any
+    delta: int | None = None
+
+
+@dataclass(frozen=True)
+class CompareResult:
+    old_file: str
+    new_file: str
+    old_run: Run
+    new_run: Run
+    old_root: Finding | None
+    new_root: Finding | None
+    old_ids: tuple[str, ...]
+    new_ids: tuple[str, ...]
+    gone: tuple[str, ...]
+    added: tuple[str, ...]
+    same: tuple[str, ...]
+    run: tuple[FieldDelta, ...]
+
+    def has_diff(self) -> bool:
+        return bool(self.gone or self.added or self.run)
+
+
+def compare_reports(old: Report, new: Report) -> CompareResult:
+    old_ids = _finding_ids(old)
+    new_ids = _finding_ids(new)
+    old_set = set(old_ids)
+    new_set = set(new_ids)
+    gone = tuple(item for item in old_ids if item not in new_set)
+    added = tuple(item for item in new_ids if item not in old_set)
+    same = tuple(item for item in old_ids if item in new_set)
+    return CompareResult(
+        old_file=old.file,
+        new_file=new.file,
+        old_run=old.run,
+        new_run=new.run,
+        old_root=old.root_cause,
+        new_root=new.root_cause,
+        old_ids=old_ids,
+        new_ids=new_ids,
+        gone=gone,
+        added=added,
+        same=same,
+        run=_run_deltas(old.run, new.run),
+    )
+
+
+def compare_runs(
+    old_run: Run,
+    new_run: Run,
+    old_file: str,
+    new_file: str,
+) -> CompareResult:
+    old = diagnose(old_run, old_file, no_llm=True)
+    new = diagnose(new_run, new_file, no_llm=True)
+    return compare_reports(old, new)
+
+
+def _finding_ids(report: Report) -> tuple[str, ...]:
+    seen: list[str] = []
+    for item in report.findings:
+        if item.id not in seen:
+            seen.append(item.id)
+    return tuple(seen)
+
+
+def _run_value(run: Run, key: str) -> Any:
+    if key == "status":
+        return run.status.value
+    if key == "steps":
+        return len(run.steps)
+    if key == "duration_ms":
+        return run.duration_ms
+    if key == "tokens_in":
+        return run.tokens_in
+    if key == "tokens_out":
+        return run.tokens_out
+    raise KeyError(key)
+
+
+def _run_deltas(old: Run, new: Run) -> tuple[FieldDelta, ...]:
+    out: list[FieldDelta] = []
+    old_status = old.status.value
+    new_status = new.status.value
+    if old_status != new_status:
+        out.append(FieldDelta(key="status", old=old_status, new=new_status))
+    for key in _RUN_INT_KEYS:
+        old_value = _run_value(old, key)
+        new_value = _run_value(new, key)
+        if old_value is None or new_value is None:
+            continue
+        if old_value == new_value:
+            continue
+        if not isinstance(old_value, int) or not isinstance(new_value, int):
+            continue
+        out.append(
+            FieldDelta(
+                key=key,
+                old=old_value,
+                new=new_value,
+                delta=new_value - old_value,
+            )
+        )
+    return tuple(out)

failstep/detectors/__init__.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from failstep.detectors.malformed import detect as detect_malformed
+from failstep.detectors.retrieval import detect as detect_retrieval
+from failstep.detectors.retry import detect as detect_retry
+from failstep.detectors.schema import detect as detect_schema
+from failstep.detectors.timeout import detect as detect_timeout
+from failstep.detectors.tool_error import detect as detect_tool_error
+from failstep.models import Finding, Run
+
+Detector = Callable[[Run], list[Finding]]
+
+DETECTORS: list[Detector] = [
+    detect_malformed,
+    detect_schema,
+    detect_tool_error,
+    detect_retry,
+    detect_timeout,
+    detect_retrieval,
+]
+
+
+def run_detectors(run: Run) -> list[Finding]:
+    findings: list[Finding] = []
+    for detector in DETECTORS:
+        findings.extend(detector(run))
+    return findings

failstep/detectors/malformed.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from failstep.evidence import finding
+from failstep.models import Finding, Run, Step
+
+_JSON_START = "{["
+
+
+def detect(run: Run) -> list[Finding]:
+    bad: list[Step] = []
+    reasons: list[str] = []
+    samples: list[str] = []
+    for step in run.steps:
+        reason = _malformed(step)
+        if reason is None:
+            continue
+        bad.append(step)
+        reasons.append(reason)
+        samples.append(_sample(step.output))
+    if not bad:
+        return []
+    return [
+        finding(
+            code="FS001",
+            detector="malformed",
+            title="malformed output",
+            steps=bad,
+            evidence=[
+                ("steps", len(bad)),
+                ("reason", reasons[0]),
+                ("output", samples[0]),
+                ("tool", bad[0].name),
+            ],
+            recommendation=(
+                "Return complete JSON from the tool. Do not truncate the payload."
+            ),
+        )
+    ]
+
+
+def _malformed(step: Step) -> str | None:
+    output = step.output
+    if isinstance(output, str):
+        text = output.strip()
+        if not text:
+            return None
+        if text[0] in _JSON_START:
+            try:
+                json.loads(text)
+            except json.JSONDecodeError:
+                if text.endswith("...") or not _balanced(text):
+                    return "truncated json"
+                return "invalid json"
+        if text.endswith("...") and len(text) > 3:
+            return "truncated payload"
+        return None
+
+    schema = _output_schema(step)
+    if schema is None or not isinstance(output, dict):
+        return None
+    required = schema.get("required")
+    if not isinstance(required, list):
+        return None
+    missing = [key for key in required if isinstance(key, str) and key not in output]
+    if missing:
+        return "missing required output fields"
+    return None
+
+
+def _output_schema(step: Step) -> dict[str, Any] | None:
+    raw = step.metadata.get("output_schema")
+    if isinstance(raw, dict):
+        return raw
+    if step.schema_ and isinstance(step.schema_.get("output"), dict):
+        return step.schema_["output"]
+    return None
+
+
+def _balanced(text: str) -> bool:
+    curly = 0
+    square = 0
+    in_string = False
+    escape = False
+    for char in text:
+        if in_string:
+            if escape:
+                escape = False
+            elif char == "\\":
+                escape = True
+            elif char == '"':
+                in_string = False
+            continue
+        if char == '"':
+            in_string = True
+        elif char == "{":
+            curly += 1
+        elif char == "}":
+            curly -= 1
+        elif char == "[":
+            square += 1
+        elif char == "]":
+            square -= 1
+        if curly < 0 or square < 0:
+            return False
+    return curly == 0 and square == 0 and not in_string
+
+
+def _sample(output: Any) -> str:
+    if isinstance(output, str):
+        text = output.replace("\n", " ")
+        return text if len(text) <= 80 else text[:77] + "..."
+    return str(output)