agentidem 1.0.0__py3-none-any.whl

agentidem/__init__.py

@@ -0,0 +1,33 @@
+from agentidem.invariants import invariant
+from agentidem.runtime import (
+    read,
+    run_traced,
+    run_traced_async,
+    write,
+)
+from agentidem.testing import TestReport, test_agent, test_agent_async
+from agentidem.trace import (
+    ObservationStatus,
+    OperationKind,
+    OperationStatus,
+    Trace,
+    TraceOperation,
+)
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "ObservationStatus",
+    "OperationKind",
+    "OperationStatus",
+    "TestReport",
+    "Trace",
+    "TraceOperation",
+    "invariant",
+    "read",
+    "run_traced",
+    "run_traced_async",
+    "test_agent",
+    "test_agent_async",
+    "write",
+]

agentidem/cli/__init__.py

@@ -0,0 +1,6 @@
+from agentidem.cli.app import app
+
+
+__all__ = [
+    "app",
+]

agentidem/cli/app.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import typer
+
+from agentidem.cli.commands.record import record_command
+from agentidem.cli.commands.test import test_command
+from agentidem.cli.commands.replay import replay_command
+from agentidem.cli.commands.version import version_command
+
+app = typer.Typer(
+    name="agentidem",
+    help="Reliability testing for state-changing AI agents.",
+    no_args_is_help=True,
+)
+
+
+app.command(
+    "test",
+    help="Test an agent for unsafe side effects.",
+)(test_command)
+
+
+app.command(
+    "record",
+    help="Run an agent once and record its execution trace.",
+)(record_command)
+app.command(
+    "version",
+    help="Show the AgentIdem CLI version.",
+)(version_command)
+
+
+app.command(
+    "replay",
+    help="Re-run a recorded trace and compare the execution.",
+)(replay_command)
+
+
+@app.callback()
+def main() -> None:
+    """
+    AgentIdem command-line interface.
+    """

agentidem/cli/commands/record.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+from collections.abc import Awaitable, Callable
+from pathlib import Path
+from typing import Any, cast
+
+import typer
+
+from agentidem.cli.loader import (
+    TargetLoadError,
+    load_target,
+)
+from agentidem.cli.output import (
+    render_target_error,
+)
+from agentidem.errors import TracedExecutionError
+from agentidem.runtime.runner import (
+    run_traced,
+    run_traced_async,
+)
+from agentidem.trace.models import Trace
+from agentidem.trace.serialization import save_trace
+
+
+def _record_target(
+        target: str,
+        func: Callable[..., Any],
+) -> Trace:
+    if inspect.iscoroutinefunction(func):
+        async_func = cast(
+            Callable[..., Awaitable[Any]],
+            func,
+        )
+
+        _, trace = asyncio.run(
+            run_traced_async(
+                target,
+                async_func,
+            )
+        )
+
+        return trace
+
+    _, trace = run_traced(
+        target,
+        func,
+    )
+
+    return trace
+
+
+def record_command(
+        target: str = typer.Argument(
+            ...,
+            help="Python target in module.path:function format.",
+        ),
+        out: Path = typer.Option(
+            Path("trace.json"),
+            "--out",
+            help="Write the recorded trace to a JSON file.",
+        ),
+) -> None:
+    """
+    Run an agent once and record its execution trace.
+    """
+    try:
+        func = load_target(target)
+
+    except TargetLoadError as exc:
+        render_target_error(str(exc))
+        raise typer.Exit(code=2) from exc
+
+    try:
+        trace = _record_target(
+            target,
+            func,
+        )
+
+    except TracedExecutionError as exc:
+        save_trace(
+            exc.trace,
+            out,
+        )
+
+        render_target_error(
+            f"{type(exc.cause).__name__}: {exc.cause}"
+        )
+
+        raise typer.Exit(code=2) from exc
+
+    save_trace(
+        trace,
+        out,
+    )
+
+    typer.echo(
+        f"Recorded trace to {out}"
+    )

agentidem/cli/commands/replay.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+from collections.abc import Awaitable, Callable
+from pathlib import Path
+from typing import Any, cast
+
+import typer
+
+from agentidem.cli.loader import (
+    TargetLoadError,
+    load_target,
+)
+from agentidem.cli.output import render_target_error
+from agentidem.errors import TracedExecutionError
+from agentidem.runtime.runner import (
+    run_traced,
+    run_traced_async,
+)
+from agentidem.trace.models import Trace, TraceOperation
+from agentidem.trace.serialization import load_trace
+
+
+def _operation_signature(
+        operation: TraceOperation,
+) -> tuple[Any, ...]:
+    return (
+        operation.name,
+        operation.kind,
+        operation.args,
+        operation.identity,
+        operation.result,
+        operation.error,
+        operation.status,
+        operation.observation,
+    )
+
+
+def _traces_match(
+        expected: Trace,
+        actual: Trace,
+) -> bool:
+    if expected.target != actual.target:
+        return False
+
+    if len(expected.operations) != len(actual.operations):
+        return False
+
+    return all(
+        _operation_signature(expected_operation)
+        == _operation_signature(actual_operation)
+        for expected_operation, actual_operation in zip(
+            expected.operations,
+            actual.operations,
+            strict=True,
+        )
+    )
+
+
+def _run_replay_target(
+        target: str,
+        func: Callable[..., Any],
+) -> Trace:
+    if inspect.iscoroutinefunction(func):
+        async_func = cast(
+            Callable[..., Awaitable[Any]],
+            func,
+        )
+
+        try:
+            _, trace = asyncio.run(
+                run_traced_async(
+                    target,
+                    async_func,
+                )
+            )
+        except TracedExecutionError as exc:
+            return exc.trace
+
+        return trace
+
+    try:
+        _, trace = run_traced(
+            target,
+            func,
+        )
+    except TracedExecutionError as exc:
+        return exc.trace
+
+    return trace
+
+
+def replay_command(
+        trace_path: Path = typer.Argument(
+            ...,
+            help="Path to a recorded AgentIdem trace.",
+        ),
+) -> None:
+    """
+    Re-run a recorded target and compare its execution trace.
+    """
+    try:
+        expected_trace = load_trace(trace_path)
+    except Exception as exc:
+        render_target_error(
+            f"Failed to load trace '{trace_path}': {exc}"
+        )
+        raise typer.Exit(code=2) from exc
+
+    try:
+        func = load_target(expected_trace.target)
+    except TargetLoadError as exc:
+        render_target_error(str(exc))
+        raise typer.Exit(code=2) from exc
+
+    actual_trace = _run_replay_target(
+        expected_trace.target,
+        func,
+    )
+
+    if _traces_match(
+            expected_trace,
+            actual_trace,
+    ):
+        typer.echo(
+            f"REPLAY MATCH: {expected_trace.target}"
+        )
+        raise typer.Exit(code=0)
+
+    typer.echo(
+        f"REPLAY MISMATCH: {expected_trace.target}"
+    )
+    raise typer.Exit(code=1)

agentidem/cli/commands/test.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+import json
+from collections.abc import Awaitable, Callable
+from pathlib import Path
+from typing import Any, cast
+
+import typer
+
+from agentidem import (
+    test_agent,
+    test_agent_async,
+)
+from agentidem.cli.loader import (
+    TargetLoadError,
+    load_target,
+)
+from agentidem.cli.output import (
+    render_target_error,
+    render_test_report,
+)
+from agentidem.testing.models import TestReport
+from agentidem.testing.serialization import (
+    report_to_dict,
+    save_report,
+)
+
+
+def _run_target(
+        target: str,
+        func: Callable[..., Any],
+) -> TestReport:
+    if inspect.iscoroutinefunction(func):
+        async_func = cast(
+            Callable[..., Awaitable[Any]],
+            func,
+        )
+
+        return asyncio.run(
+            test_agent_async(
+                target,
+                async_func,
+            )
+        )
+
+    return test_agent(
+        target,
+        func,
+    )
+
+
+def test_command(
+        target: str = typer.Argument(
+            ...,
+            help="Python target in module.path:function format.",
+        ),
+        out: Path | None = typer.Option(
+            None,
+            "--out",
+            help="Write the test report to a JSON file.",
+        ),
+        json_output: bool = typer.Option(
+            False,
+            "--json",
+            help="Print the test report as JSON.",
+        ),
+) -> None:
+    """
+    Test an agent for retry and side effect safety.
+    """
+    try:
+        func = load_target(target)
+
+    except TargetLoadError as exc:
+        render_target_error(str(exc))
+        raise typer.Exit(code=2) from exc
+
+    report = _run_target(
+        target,
+        func,
+    )
+
+    if out is not None:
+        save_report(
+            report,
+            out,
+        )
+
+    if json_output:
+        typer.echo(
+            json.dumps(
+                report_to_dict(report),
+                indent=2,
+            )
+        )
+    else:
+        render_test_report(
+            target,
+            report,
+        )
+
+    if report.baseline_failure is not None:
+        raise typer.Exit(code=2)
+
+    if report.safe:
+        raise typer.Exit(code=0)
+
+    raise typer.Exit(code=1)

agentidem/cli/commands/version.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+import typer
+
+from agentidem.cli.constants import AGENTIDEM_CLI_VERSION
+
+
+def version_command() -> None:
+    """
+    Show the AgentIdem CLI version.
+    """
+    typer.echo(AGENTIDEM_CLI_VERSION)

agentidem/cli/constants.py

@@ -0,0 +1 @@
+AGENTIDEM_CLI_VERSION = "v1.0.0"

agentidem/cli/loader.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import importlib
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+
+class TargetLoadError(RuntimeError):
+    pass
+
+
+def load_target(target: str) -> Callable[..., Any]:
+    if ":" not in target:
+        raise TargetLoadError(
+            "Target must use the format 'module.path:function'."
+        )
+
+    module_name, object_name = target.split(":", 1)
+
+    if not module_name:
+        raise TargetLoadError("Target module cannot be empty.")
+
+    if not object_name:
+        raise TargetLoadError("Target function cannot be empty.")
+
+    cwd = str(Path.cwd())
+
+    if cwd not in sys.path:
+        sys.path.insert(0, cwd)
+
+    try:
+        module = importlib.import_module(module_name)
+    except Exception as exc:
+        raise TargetLoadError(
+            f"Failed to import module '{module_name}': {exc}"
+        ) from exc
+
+    try:
+        obj = getattr(module, object_name)
+    except AttributeError as exc:
+        raise TargetLoadError(
+            f"Module '{module_name}' has no attribute '{object_name}'."
+        ) from exc
+
+    if not callable(obj):
+        raise TargetLoadError(
+            f"Target '{target}' is not callable."
+        )
+
+    return obj

agentidem/cli/output/__init__.py

@@ -0,0 +1,12 @@
+from agentidem.cli.output.console import (
+    render_baseline_failure,
+    render_target_error,
+    render_test_report,
+)
+
+
+__all__ = [
+    "render_baseline_failure",
+    "render_target_error",
+    "render_test_report",
+]

agentidem/cli/output/console.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from rich.console import Console
+from rich.table import Table
+
+from agentidem.testing.models import TestReport
+
+
+console = Console()
+error_console = Console(stderr=True)
+
+
+def render_test_report(
+        target: str,
+        report: TestReport,
+) -> None:
+    if report.baseline_failure is not None:
+        render_baseline_failure(
+            target,
+            report,
+        )
+        return
+
+    status = "SAFE" if report.safe else "UNSAFE"
+
+    console.print()
+    console.print(
+        f"[bold]{status}[/bold]  {target}"
+    )
+
+    console.print(
+        f"Fault cases: {report.fault_count}  "
+        f"Unsafe: {report.unsafe_count}"
+    )
+
+    if not report.results:
+        return
+
+    table = Table(
+        title="Fault scenarios",
+        show_header=True,
+        header_style="bold",
+    )
+
+    table.add_column("Scenario")
+    table.add_column("Status")
+    table.add_column("Execution")
+    table.add_column("Findings", justify="right")
+
+    for result in report.results:
+        scenario_status = (
+            "SAFE"
+            if result.safe
+            else "UNSAFE"
+        )
+
+        execution_status = (
+            "FAILED"
+            if result.failed
+            else "COMPLETED"
+        )
+
+        table.add_row(
+            result.case.scenario,
+            scenario_status,
+            execution_status,
+            str(len(result.findings)),
+        )
+
+    console.print(table)
+
+
+def render_baseline_failure(
+        target: str,
+        report: TestReport,
+) -> None:
+    failure = report.baseline_failure
+
+    if failure is None:
+        return
+
+    error_console.print()
+    error_console.print(
+        f"[bold]BASELINE FAILED[/bold]  {target}"
+    )
+
+    error_console.print(
+        f"{failure.error_type}: {failure.message}"
+    )
+
+
+def render_target_error(
+        message: str,
+) -> None:
+    error_console.print(
+        f"[bold]ERROR[/bold]  {message}"
+    )

agentidem/detection/__init__.py

@@ -0,0 +1,19 @@
+from agentidem.detection.duplicates import (
+    DuplicateWrite,
+    detect_duplicate_write_findings,
+    detect_duplicate_writes,
+    duplicate_write_to_finding,
+)
+from agentidem.detection.models import (
+    Finding,
+    FindingSeverity,
+)
+
+__all__ = [
+    "DuplicateWrite",
+    "Finding",
+    "FindingSeverity",
+    "detect_duplicate_write_findings",
+    "detect_duplicate_writes",
+    "duplicate_write_to_finding",
+]