crashlab 0.0.1__py3-none-any.whl

crashlab/__init__.py

@@ -0,0 +1,144 @@
+"""crashlab: crash-test your AI agents.
+
+Record every LLM and tool call from your existing agent loop into a local store
+(``.crashlab/``), inspect runs with ``crashlab ls`` / ``crashlab show``, replay them
+offline with ``crashlab.replay(run)``, judge them with ``@crashlab.check`` oracles, and
+ask "what if" with ``crashlab.fork(run, at_step=N, patch=...)``. Built-in runtime checks
+(loops, tool error streaks, bad tool arguments, suspicious tool results, budgets, length
+cut-offs) run while recording and store :class:`Finding` s on the run.
+
+    import crashlab
+    from openai import OpenAI
+
+    client = crashlab.wrap(OpenAI())          # or anthropic.Anthropic()
+
+    @crashlab.tool()
+    def get_order(order_id: str) -> dict: ...
+
+    @crashlab.agent(name="refund-bot")
+    def run_agent(user_msg: str) -> str: ...
+
+    @crashlab.check("refund-bot")
+    def refund_not_above_total(run): ...
+
+    run = crashlab.load("last")
+    crashlab.replay(run)                      # offline; DivergenceError if it differs
+    branch = crashlab.fork(run, at_step=0,
+                           patch=crashlab.patch.SystemPrompt(op="append", text="..."))
+    crashlab.diff(run, branch).summary()
+"""
+
+from __future__ import annotations
+
+from . import checks, patch, pricing
+from ._version import __version__
+from .capture import agent, record, tool, wrap
+from .checks import ChecksConfig, CircuitBreakerError
+from .compare import RunDiff, diff
+from .context import (
+    ContextThreadPoolExecutor,
+    bind_context,
+    copy_context_to_thread,
+    is_replaying,
+    run_in_thread,
+)
+from .diagnose import DiagnoseError, diagnose, propose_fixes, try_fix
+from .models import (
+    CheckResult,
+    Diagnosis,
+    ErrorInfo,
+    Evidence,
+    Finding,
+    Fix,
+    Flag,
+    Run,
+    Step,
+    TaxonomyLabel,
+    Usage,
+    Verification,
+)
+from .patch import (
+    InjectMessagePatch,
+    MessagePatch,
+    ModelParamPatch,
+    PatchError,
+    PatchSet,
+    SystemPromptPatch,
+    ToolDescriptionPatch,
+    ToolResultPatch,
+    ToolSchemaPatch,
+)
+from .replay import (
+    DivergenceError,
+    ForkError,
+    PatchedToolError,
+    ReplayError,
+    ReplayNetworkError,
+    SideEffectBlockedError,
+    fork,
+    replay,
+)
+from .store import RunNotFoundError, Store, load
+from .store import redact as redaction
+from .verify import VerifyError, check, run_checks, verify
+
+__all__ = [
+    "DiagnoseError",
+    "diagnose",
+    "propose_fixes",
+    "try_fix",
+    "verify",
+    "VerifyError",
+    "CheckResult",
+    "ChecksConfig",
+    "CircuitBreakerError",
+    "ContextThreadPoolExecutor",
+    "Diagnosis",
+    "DivergenceError",
+    "ErrorInfo",
+    "Evidence",
+    "Finding",
+    "Fix",
+    "Flag",
+    "ForkError",
+    "InjectMessagePatch",
+    "MessagePatch",
+    "ModelParamPatch",
+    "PatchError",
+    "PatchSet",
+    "PatchedToolError",
+    "ReplayError",
+    "ReplayNetworkError",
+    "Run",
+    "RunDiff",
+    "RunNotFoundError",
+    "SideEffectBlockedError",
+    "Step",
+    "Store",
+    "SystemPromptPatch",
+    "TaxonomyLabel",
+    "ToolDescriptionPatch",
+    "ToolResultPatch",
+    "ToolSchemaPatch",
+    "Usage",
+    "Verification",
+    "__version__",
+    "agent",
+    "bind_context",
+    "check",
+    "checks",
+    "copy_context_to_thread",
+    "diff",
+    "fork",
+    "is_replaying",
+    "load",
+    "patch",
+    "pricing",
+    "record",
+    "redaction",
+    "replay",
+    "run_checks",
+    "run_in_thread",
+    "tool",
+    "wrap",
+]

crashlab/__main__.py

@@ -0,0 +1,3 @@
+from .cli import app
+
+app()

crashlab/_ids.py

@@ -0,0 +1,51 @@
+"""Small id helpers: ULID-like sortable run ids and OTel-style span ids (no extra deps)."""
+
+from __future__ import annotations
+
+import os
+import threading
+import time
+
+_CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+_lock = threading.Lock()
+_last_ms = -1
+_last_rand = 0
+
+
+def _encode(value: int, length: int) -> str:
+    out = []
+    for _ in range(length):
+        value, rem = divmod(value, 32)
+        out.append(_CROCKFORD[rem])
+    return "".join(reversed(out))
+
+
+def new_ulid() -> str:
+    """Return a 26-char Crockford base32 id: 48-bit ms timestamp + 80 random bits.
+
+    Ids are lexicographically sortable by creation time. Within the same millisecond
+    the random part is incremented, so ids stay strictly increasing in one process.
+    """
+    global _last_ms, _last_rand
+    with _lock:
+        ms = time.time_ns() // 1_000_000
+        if ms <= _last_ms:
+            ms = _last_ms
+            rand = (_last_rand + 1) & ((1 << 80) - 1)
+        else:
+            rand = int.from_bytes(os.urandom(10), "big")
+        _last_ms, _last_rand = ms, rand
+    return _encode(ms, 10) + _encode(rand, 16)
+
+
+def ulid_timestamp_ms(ulid: str) -> int:
+    """Decode the millisecond timestamp from a ULID-like id."""
+    value = 0
+    for ch in ulid[:10].upper():
+        value = value * 32 + _CROCKFORD.index(ch)
+    return value
+
+
+def new_span_id() -> str:
+    """Return a 16-hex-char span id (same size as an OpenTelemetry span id)."""
+    return os.urandom(8).hex()

crashlab/_version.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

crashlab/bench/__init__.py

@@ -0,0 +1,18 @@
+"""The seed fault-injection benchmark (``crashlab bench``)."""
+
+from .catalog import CONDITIONS, FAULTS, TASKS, Fault, Task
+from .harness import load_runs_dir, render_md, run_live, score_run, score_runs, write_results
+
+__all__ = [
+    "CONDITIONS",
+    "FAULTS",
+    "TASKS",
+    "Fault",
+    "Task",
+    "load_runs_dir",
+    "render_md",
+    "run_live",
+    "score_run",
+    "score_runs",
+    "write_results",
+]

crashlab/bench/catalog.py

@@ -0,0 +1,117 @@
+"""The seed benchmark's fault catalog and tasks.
+
+Labels and culprit-step rules were fixed *before* the first benchmark run (pre-registered
+in bench/README.md). ``split``: refund-bot is the **dev** agent (the heuristics and the
+diagnosis prompt were developed on its runs); sql-analyst is the held-out **test** agent
+(written for Day 10, never used for tuning).
+"""
+
+from __future__ import annotations
+
+import importlib
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Fault:
+    name: str
+    description: str
+    labels: frozenset[str]  # accepted top-1 labels
+    reference_fix: str
+
+
+FAULTS: dict[str, Fault] = {
+    f.name: f
+    for f in (
+        Fault(
+            "spec_drop",
+            "a rule is removed from the system prompt (refund window / revenue definition)",
+            frozenset({"TASK_SPEC_VIOLATION"}),
+            "restore or strengthen the instruction (system_prompt patch)",
+        ),
+        Fault(
+            "tool_5xx",
+            "a tool returns an HTML 502 page instead of data, every call",
+            frozenset({"EXTERNAL_FAILURE", "TOOL_ERROR_MISHANDLED", "STEP_REPETITION"}),
+            "tell the agent to stop and escalate on non-JSON / 5xx results",
+        ),
+        Fault(
+            "schema_drift",
+            "a field/column is renamed upstream (total -> amount_total)",
+            frozenset({"TOOL_ERROR_MISHANDLED", "BAD_TOOL_ARGUMENTS"}),
+            "update the tool description / prompt to the new field name",
+        ),
+        Fault(
+            "misleading_desc",
+            "a tool description is wrong (restocking fee / amounts in cents)",
+            frozenset({"BAD_TOOL_ARGUMENTS", "TASK_SPEC_VIOLATION"}),
+            "correct the tool description (tool_description patch)",
+        ),
+    )
+}
+
+
+@dataclass(frozen=True)
+class Task:
+    name: str  # the @crashlab.agent name
+    module: str
+    split: str  # "dev" or "test"
+    prompts: dict[str, list[str]] = field(default_factory=dict)  # condition -> prompts
+    default_prompts: tuple[str, ...] = ()
+
+    def load(self) -> Any:
+        """The agent module; the example agents live in the repo checkout, so the
+        current directory is tried too (the ``crashlab`` script does not put it on
+        sys.path)."""
+        try:
+            return importlib.import_module(self.module)
+        except ModuleNotFoundError:
+            if os.getcwd() in sys.path:
+                raise
+            sys.path.insert(0, os.getcwd())
+            return importlib.import_module(self.module)
+
+    def prompts_for(self, condition: str) -> list[str]:
+        return list(self.prompts.get(condition) or self.default_prompts)
+
+    def agent_faults(self, condition: str) -> list[str]:
+        if condition == "clean":
+            return []
+        mapping = getattr(self.load(), "BENCH_FAULTS", None) or {condition: condition}
+        return [mapping[condition]]
+
+    def run(self, prompt: str, model: str, condition: str) -> Any:
+        mod = self.load()
+        faults = self.agent_faults(condition)
+        if self.name == "refund-bot":
+            return mod.run_agent(prompt, model=model, provider="openai", faults=faults)
+        return mod.run_agent(prompt, model=model, faults=faults)
+
+    def ground_truth(self, run: Any, condition: str) -> tuple[Any, Any]:
+        return self.load().ground_truth(run, condition)
+
+
+A17 = "Hi, I'd like a refund for order A17 please, it arrived broken."
+C03 = "Please refund order C03, the shoes fell apart."
+SQL_QS = (
+    "What was ShopCo's revenue in March 2026?",
+    "How much revenue came from customers in Germany in 2026?",
+    "Which country generated the most revenue in Q1 2026 (January to March)?",
+)
+
+TASKS: dict[str, Task] = {
+    "refund-bot": Task(
+        "refund-bot",
+        "examples.refund_bot.agent",
+        "dev",
+        prompts={"spec_drop": [C03], "clean": [A17, C03]},
+        default_prompts=(A17,),
+    ),
+    "sql-analyst": Task(
+        "sql-analyst", "examples.sql_analyst.agent", "test", default_prompts=SQL_QS
+    ),
+}
+CONDITIONS = ("clean", *FAULTS)