semantic-trace 0.1.0__py3-none-any.whl

agent_trace/__init__.py

@@ -0,0 +1,78 @@
+"""agent-trace: Semantic tracing primitive for AI agents.
+
+Minimal, composable, and zero-bloat by design.
+
+Quick start:
+    from agent_trace import Trace, IntentInvariant, InvariantType, semantic_replay
+
+    invariants = [
+        IntentInvariant(
+            id="valid-json",
+            description="Output must be valid JSON",
+            invariant_type=InvariantType.SUBSTRING_CHECK,
+            config={"substring": '"summary"'},
+            fidelity_threshold=1.0,
+        ),
+    ]
+
+    with Trace(name="my-agent", invariants=invariants, output_file="traces/run.jsonl") as trace:
+        trace.add_span(span)
+
+    report = semantic_replay("traces/run.jsonl")
+    print(report.summary())
+"""
+
+from agent_trace.core.schema import (
+    ActionType,
+    IntentInvariant,
+    InvariantResult,
+    InvariantType,
+    ReplayReport,
+    Span,
+    Trace,
+    TraceMetadata,
+    TraceModel,
+)
+from agent_trace.core.serializer import (
+    read_trace_from_jsonl,
+    write_metadata_to_jsonl,
+    write_span_to_jsonl,
+)
+from agent_trace.engine.invariants import (
+    BaseInvariantChecker,
+    InvariantViolation,
+    LLMAsJudgeChecker,
+    SchemaInvariantChecker,
+    SubstringInvariantChecker,
+)
+from agent_trace.engine.replay import (
+    mechanical_replay,
+    semantic_replay,
+    validate_trace,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ActionType",
+    "BaseInvariantChecker",
+    "IntentInvariant",
+    "InvariantResult",
+    "InvariantType",
+    "InvariantViolation",
+    "LLMAsJudgeChecker",
+    "ReplayReport",
+    "SchemaInvariantChecker",
+    "Span",
+    "SubstringInvariantChecker",
+    "Trace",
+    "TraceMetadata",
+    "TraceModel",
+    "__version__",
+    "mechanical_replay",
+    "read_trace_from_jsonl",
+    "semantic_replay",
+    "validate_trace",
+    "write_metadata_to_jsonl",
+    "write_span_to_jsonl",
+]

agent_trace/cli.py

@@ -0,0 +1,158 @@
+"""CLI entry point for inspecting and validating trace files.
+
+Usage:
+    trace <command> <trace_file> [options]
+
+Commands:
+    info       Show trace metadata summary
+    validate   Run mechanical (structural) validation
+    replay     Run full semantic replay with invariant checks
+    spans      List all spans with their durations and invariant results
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from agent_trace.core.serializer import read_trace_from_jsonl
+from agent_trace.engine.replay import mechanical_replay, validate_trace
+
+
+def cmd_info(args: argparse.Namespace) -> None:
+    """Print trace metadata summary to stdout."""
+    trace = read_trace_from_jsonl(args.trace_file)
+    if args.json:
+        print(json.dumps(trace.metadata.model_dump(mode="json"), indent=2))
+        return
+
+    print(f"Trace ID:    {trace.metadata.trace_id}")
+    print(f"Session ID:  {trace.metadata.session_id}")
+    print(f"Agent:       {trace.metadata.agent_name}")
+    print(f"Start:       {trace.metadata.start_time.isoformat()}")
+    end = trace.metadata.end_time.isoformat() if trace.metadata.end_time else "N/A"
+    print(f"End:         {end}")
+    print(f"Spans:       {len(trace.spans)}")
+
+
+def cmd_validate(args: argparse.Namespace) -> None:
+    """Run mechanical validation and exit non-zero on errors."""
+    errors = mechanical_replay(args.trace_file)
+    if args.json:
+        print(json.dumps({"valid": len(errors) == 0, "errors": errors}, indent=2))
+        if errors:
+            sys.exit(1)
+        return
+
+    if errors:
+        print("Structural errors:")
+        for err in errors:
+            print(f"  - {err}")
+        sys.exit(1)
+    print("Structural validation passed.")
+
+
+def cmd_replay(args: argparse.Namespace) -> None:
+    """Run full semantic replay and exit non-zero on violations."""
+    report = validate_trace(args.trace_file)
+
+    if args.json:
+        output = {
+            "clean": report.is_clean,
+            "trace_id": str(report.trace_id),
+            "agent_name": report.agent_name,
+            "total_spans": report.total_spans,
+            "total_invariants": report.total_invariants,
+            "pass_rate": report.pass_rate,
+            "structural_errors": report.structural_errors,
+            "violations": [
+                {
+                    "span_id": v.span_id,
+                    "invariant_id": v.invariant_id,
+                    "expected_score": v.expected_score,
+                    "actual_score": v.actual_score,
+                }
+                for v in report.violations
+            ],
+        }
+        print(json.dumps(output, indent=2))
+        if not report.is_clean:
+            sys.exit(1)
+        return
+
+    print(report.summary())
+    if not report.is_clean:
+        report.print_violations()
+        sys.exit(1)
+
+
+def cmd_spans(args: argparse.Namespace) -> None:
+    """List all spans with action type, UUID, duration, and invariant results."""
+    trace = read_trace_from_jsonl(args.trace_file)
+
+    if args.json:
+        spans = []
+        for span in trace.spans:
+            span_data = span.model_dump(mode="json")
+            spans.append(span_data)
+        print(json.dumps(spans, indent=2))
+        return
+
+    for span in trace.spans:
+        if span.duration_ms is not None:
+            print(f"[{span.action_type}] {span.span_id} ({span.duration_ms:.0f}ms)")
+        else:
+            print(f"[{span.action_type}] {span.span_id}")
+        if span.invariant_results:
+            for inv_id, score in span.invariant_results.items():
+                status = "PASS" if score >= 1.0 else "FAIL"
+                print(f"    {inv_id}: {score:.2f} [{status}]")
+
+
+def main() -> None:
+    """CLI entry point. Parses arguments and dispatches to the appropriate command."""
+    parser = argparse.ArgumentParser(
+        prog="trace",
+        description="Semantic tracing CLI for AI agents. Inspect, validate, and replay trace files.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="agent-trace 0.1.0",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    for name, help_text, handler in [
+        ("info", "Show trace metadata summary", cmd_info),
+        ("validate", "Run structural validation", cmd_validate),
+        ("replay", "Run full semantic replay", cmd_replay),
+        ("spans", "List all spans", cmd_spans),
+    ]:
+        sub = subparsers.add_parser(name, help=help_text)
+        sub.add_argument("trace_file", type=str, help="Path to the JSONL trace file")
+        sub.add_argument(
+            "--json",
+            action="store_true",
+            help="Output in JSON format for machine consumption",
+        )
+        sub.set_defaults(handler=handler)
+
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        sys.exit(1)
+
+    trace_path = Path(args.trace_file)
+    if not trace_path.exists():
+        print(f"Error: file not found: {args.trace_file}", file=sys.stderr)
+        sys.exit(1)
+
+    args.handler(args)
+
+
+if __name__ == "__main__":
+    main()

agent_trace/core/__init__.py

@@ -0,0 +1,39 @@
+"""Core data models and serialization for agent-trace.
+
+Provides Pydantic models (TraceModel, Span, IntentInvariant), the Trace
+context manager, and JSONL serialization utilities.
+"""
+
+from agent_trace.core.schema import (
+    ActionType,
+    IntentInvariant,
+    InvariantResult,
+    InvariantType,
+    ReplayReport,
+    Span,
+    Trace,
+    TraceMetadata,
+    TraceModel,
+)
+from agent_trace.core.serializer import (
+    read_trace_from_jsonl,
+    write_metadata_to_jsonl,
+    write_span_to_jsonl,
+)
+
+__all__ = [
+    # Models
+    "ActionType",
+    "IntentInvariant",
+    "InvariantResult",
+    "InvariantType",
+    "ReplayReport",
+    "Span",
+    "Trace",
+    "TraceMetadata",
+    "TraceModel",
+    # Serialization
+    "read_trace_from_jsonl",
+    "write_metadata_to_jsonl",
+    "write_span_to_jsonl",
+]

agent_trace/core/schema.py

@@ -0,0 +1,401 @@
+"""Pydantic v2 data models and Trace context manager for agent-trace.
+
+All models use strict typing and Pydantic v2 validation. No mutable defaults.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel, Field
+
+if TYPE_CHECKING:
+    from agent_trace.engine.invariants import InvariantViolation
+
+
+class InvariantType(str, Enum):
+    """Enumeration of built-in invariant checker types.
+
+    Attributes:
+        SCHEMA_MATCH: Validates output against a Pydantic-compatible schema.
+        SUBSTRING_CHECK: Checks for a target substring in the JSON-serialized output.
+        LLM_AS_JUDGE: Uses an LLM to semantically evaluate the output.
+        CUSTOM: Placeholder for user-defined checkers via the ABC.
+    """
+
+    SCHEMA_MATCH = "SCHEMA_MATCH"
+    SUBSTRING_CHECK = "SUBSTRING_CHECK"
+    LLM_AS_JUDGE = "LLM_AS_JUDGE"
+    CUSTOM = "CUSTOM"
+
+
+class ActionType(str, Enum):
+    """Enumeration of span action types.
+
+    Attributes:
+        LLM_CALL: A call to a language model.
+        TOOL_CALL: A call to an external tool.
+        AGENT_STEP: A high-level agent reasoning step.
+        CUSTOM: A user-defined action type.
+    """
+
+    LLM_CALL = "llm_call"
+    TOOL_CALL = "tool_call"
+    AGENT_STEP = "agent_step"
+    CUSTOM = "custom"
+
+
+class TraceMetadata(BaseModel):
+    """Metadata header for a trace.
+
+    Written as the first line of every JSONL trace file.
+
+    Attributes:
+        trace_id: Unique identifier for this trace.
+        session_id: Identifier for the agent session that produced this trace.
+        agent_name: Human-readable name of the agent.
+        start_time: UTC timestamp when the trace was created.
+        end_time: UTC timestamp when the trace was finalized. None until closed.
+    """
+
+    trace_id: uuid.UUID = Field(default_factory=uuid.uuid4)
+    session_id: str
+    agent_name: str
+    start_time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    end_time: datetime | None = None
+
+
+class IntentInvariant(BaseModel):
+    """A declarative intent assertion attached to a span.
+
+    Invariants express what the span's output *should* satisfy. They are
+    evaluated during semantic replay.
+
+    Attributes:
+        id: Unique identifier for this invariant.
+        description: Human-readable description of the intent.
+        invariant_type: The checker type to use.
+        config: Rule parameters passed to the checker (e.g., ``{"substring": "..."}``).
+        fidelity_threshold: Minimum score (0.0-1.0) for the check to pass.
+    """
+
+    id: str
+    description: str
+    invariant_type: InvariantType
+    config: dict[str, Any] = Field(default_factory=dict)
+    fidelity_threshold: float = Field(ge=0.0, le=1.0, default=1.0)
+
+
+class Span(BaseModel):
+    """A single unit of execution within a trace.
+
+    Each span represents one atomic action (LLM call, tool invocation, etc.)
+    and may carry attached invariants for later verification.
+
+    Attributes:
+        span_id: Unique identifier for this span.
+        parent_id: Optional parent span UUID for nested execution.
+        trace_id: The trace this span belongs to.
+        timestamp: UTC timestamp when the span started.
+        action_type: The kind of action this span represents.
+        input_data: The input payload for the action.
+        output_data: The output payload produced by the action.
+        duration_ms: Execution duration in milliseconds, if available.
+        attached_invariants: Intent invariants to verify during replay.
+        invariant_results: Post-execution scores, populated by semantic replay.
+    """
+
+    span_id: uuid.UUID = Field(default_factory=uuid.uuid4)
+    parent_id: uuid.UUID | None = None
+    trace_id: uuid.UUID
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    action_type: ActionType
+    input_data: dict[str, Any] = Field(default_factory=dict)
+    output_data: dict[str, Any] = Field(default_factory=dict)
+    duration_ms: float | None = None
+    attached_invariants: list[IntentInvariant] = Field(default_factory=list)
+    invariant_results: dict[str, float] | None = None
+
+
+class TraceModel(BaseModel):
+    """A complete trace consisting of metadata and ordered spans.
+
+    This is the core Pydantic model. For the high-level context manager API,
+    use ``Trace`` instead.
+
+    Attributes:
+        metadata: The trace header with session and agent information.
+        spans: Ordered list of spans captured during execution.
+    """
+
+    metadata: TraceMetadata
+    spans: list[Span] = Field(default_factory=list)
+
+
+@dataclass
+class InvariantResult:
+    """Result of a single invariant check during replay.
+
+    Attributes:
+        invariant_id: Identifier of the invariant that was checked.
+        span_id: UUID of the span that was evaluated.
+        score: The actual score returned by the checker (0.0-1.0).
+        threshold: The minimum score required for the check to pass.
+        passed: Whether the score meets or exceeds the threshold.
+    """
+
+    invariant_id: str
+    span_id: str
+    score: float
+    threshold: float
+    passed: bool
+
+
+@dataclass
+class ReplayReport:
+    """Comprehensive report from semantic or mechanical replay.
+
+    Provides summary statistics, violation details, and a human-readable
+    summary string. Iterable over violations for backward compatibility.
+
+    Attributes:
+        trace_file: Path to the trace file that was replayed.
+        trace_id: UUID of the trace.
+        agent_name: Name of the agent that produced the trace.
+        total_spans: Number of spans in the trace.
+        total_invariants: Total number of invariant checks performed.
+        violations: List of invariant violations (score below threshold).
+        results: All invariant check results (passing and failing).
+        structural_errors: Structural errors from mechanical validation.
+    """
+
+    trace_file: Path
+    trace_id: uuid.UUID
+    agent_name: str
+    total_spans: int
+    total_invariants: int
+    violations: list[InvariantViolation]
+    results: list[InvariantResult]
+    structural_errors: list[str] = field(default_factory=list)
+
+    def __iter__(self):
+        return iter(self.violations)
+
+    def __len__(self) -> int:
+        return len(self.violations)
+
+    def __getitem__(self, index: int):
+        return self.violations[index]
+
+    def __bool__(self) -> bool:
+        return bool(self.violations) or bool(self.structural_errors)
+
+    @property
+    def is_clean(self) -> bool:
+        """True if no violations and no structural errors."""
+        return not self.violations and not self.structural_errors
+
+    @property
+    def pass_rate(self) -> float:
+        """Fraction of invariant checks that passed."""
+        if self.total_invariants == 0:
+            return 1.0
+        passed = sum(1 for r in self.results if r.passed)
+        return passed / self.total_invariants
+
+    def summary(self) -> str:
+        """Return a human-readable summary string."""
+        lines = [
+            f"Replay Report: {self.agent_name}",
+            f"  Trace: {self.trace_id}",
+            f"  File: {self.trace_file}",
+            f"  Spans: {self.total_spans}",
+            f"  Invariants checked: {self.total_invariants}",
+            f"  Pass rate: {self.pass_rate:.0%}",
+        ]
+        if self.structural_errors:
+            lines.append(f"  Structural errors: {len(self.structural_errors)}")
+        if self.violations:
+            lines.append(f"  Violations: {len(self.violations)}")
+        else:
+            lines.append("  Status: ALL CLEAR")
+        return "\n".join(lines)
+
+    def print_violations(self) -> None:
+        """Print all violations and structural errors to stdout."""
+        if not self.violations and not self.structural_errors:
+            print("No violations found.")
+            return
+
+        if self.structural_errors:
+            print("Structural errors:")
+            for err in self.structural_errors:
+                print(f"  - {err}")
+
+        if self.violations:
+            print("Invariant violations:")
+            for v in self.violations:
+                print(
+                    f"  Span {v.span_id}: invariant {v.invariant_id} "
+                    f"(expected>={v.expected_score:.2f}, got {v.actual_score:.2f})"
+                )
+
+
+class Trace:
+    """Context manager for capturing agent traces.
+
+    Creates a trace header, collects spans, and writes to a JSONL file
+    in real-time. On exit, finalizes the trace with an end timestamp.
+
+    Usage:
+        with Trace(name="my-agent", output_file="traces/run.jsonl") as trace:
+            trace.add_span(span)
+
+    Or with default invariants attached to every span:
+        invariants = [IntentInvariant(...)]
+        with Trace(name="my-agent", invariants=invariants, output_file="traces/run.jsonl") as trace:
+            # spans automatically get the invariants
+            trace.add_span(span)
+
+    For backward compatibility, can also be initialized with a metadata object:
+        trace = Trace(metadata=TraceMetadata(...))
+        trace.add_span(span)
+        trace.save("traces/run.jsonl")
+
+    Args:
+        name: Human-readable name of the agent (required unless metadata is provided).
+        metadata: Pre-built TraceMetadata (alternative to name).
+        invariants: Default invariants to attach to every span.
+        output_file: Path to the JSONL file for real-time streaming.
+        session_id: Identifier for the agent session. Auto-generated if omitted.
+        trace_id: Explicit trace UUID. Auto-generated if omitted.
+    """
+
+    def __init__(
+        self,
+        name: str | None = None,
+        *,
+        metadata: TraceMetadata | None = None,
+        invariants: list[IntentInvariant] | None = None,
+        output_file: str | Path | None = None,
+        session_id: str | None = None,
+        trace_id: uuid.UUID | None = None,
+    ) -> None:
+        if name is not None and metadata is not None:
+            raise ValueError("Provide either 'name' or 'metadata', not both")
+
+        if metadata is not None:
+            self._model = TraceModel(metadata=metadata, spans=[])
+            self._output_file: Path | None = None
+            self._default_invariants: list[IntentInvariant] = []
+        elif name is not None:
+            self._model = TraceModel(
+                metadata=TraceMetadata(
+                    trace_id=trace_id or uuid.uuid4(),
+                    session_id=session_id or f"session-{uuid.uuid4().hex[:8]}",
+                    agent_name=name,
+                ),
+                spans=[],
+            )
+            self._output_file = Path(output_file).resolve() if output_file else None
+            self._default_invariants = list(invariants) if invariants else []
+        else:
+            raise ValueError("Either 'name' or 'metadata' must be provided")
+
+        self._active = False
+
+    def __enter__(self) -> Trace:
+        if self._active:
+            raise RuntimeError("Trace is already active. Create a new instance.")
+        self._active = True
+        if self._output_file:
+            from agent_trace.core.serializer import write_metadata_to_jsonl
+
+            write_metadata_to_jsonl(self._output_file, self._model)
+        return self
+
+    def __exit__(self, *args: Any) -> bool:
+        self._active = False
+        self._model.metadata.end_time = datetime.now(timezone.utc)
+        if self._output_file:
+            from agent_trace.core.serializer import write_metadata_to_jsonl
+
+            write_metadata_to_jsonl(self._output_file, self._model)
+        return False
+
+    def __repr__(self) -> str:
+        status = "active" if self._active else "inactive"
+        return (
+            f"Trace(name={self._model.metadata.agent_name!r}, "
+            f"trace_id={self.trace_id!r}, "
+            f"spans={len(self.spans)}, "
+            f"status={status!r})"
+        )
+
+    @property
+    def model(self) -> TraceModel:
+        """The underlying TraceModel. Modifications affect the trace."""
+        return self._model
+
+    @property
+    def trace_id(self) -> uuid.UUID:
+        return self._model.metadata.trace_id
+
+    @property
+    def session_id(self) -> str:
+        return self._model.metadata.session_id
+
+    @property
+    def spans(self) -> list[Span]:
+        return list(self._model.spans)
+
+    def add_span(self, span: Span) -> None:
+        """Add a span to the trace.
+
+        Attaches default invariants and writes to the output file if
+        the context manager is active.
+
+        Args:
+            span: The span to add. Must have a matching trace_id.
+
+        Raises:
+            ValueError: If the span's trace_id does not match this trace.
+        """
+        if span.trace_id != self.trace_id:
+            raise ValueError(
+                f"Span trace_id {span.trace_id} does not match trace {self.trace_id}"
+            )
+        span.attached_invariants.extend(self._default_invariants)
+        self._model.spans.append(span)
+        if self._active and self._output_file:
+            from agent_trace.core.serializer import write_span_to_jsonl
+
+            write_span_to_jsonl(self._output_file, span)
+
+    def save(self, output_file: str | Path | None = None) -> None:
+        """Save the complete trace to a JSONL file.
+
+        Args:
+            output_file: Path to write. Uses the constructor value if omitted.
+
+        Raises:
+            ValueError: If no output file is specified.
+        """
+        from agent_trace.core.serializer import (
+            write_metadata_to_jsonl,
+            write_span_to_jsonl,
+        )
+
+        path = Path(output_file).resolve() if output_file else self._output_file
+        if not path:
+            raise ValueError("No output file specified")
+
+        self._model.metadata.end_time = datetime.now(timezone.utc)
+        write_metadata_to_jsonl(path, self._model)
+        for span in self._model.spans:
+            write_span_to_jsonl(path, span)