spooled-ai 0.4.0__tar.gz

spooled_ai-0.4.0/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2026 Spooled AI. All rights reserved.
+
+This software and associated documentation files (the "Software") are the
+proprietary property of Spooled AI. You may use the Software only in
+accordance with the terms of your agreement with Spooled AI or, in the
+absence of such agreement, subject to the following conditions:
+
+1. You may install and use the Software for your internal business purposes.
+2. You may not redistribute, sublicense, sell, or otherwise make the Software
+   available to third parties, in whole or in part.
+3. You may not modify, reverse-engineer, decompile, or create derivative
+   works based on the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+SPOOLED AI BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

spooled_ai-0.4.0/MANIFEST.in

@@ -0,0 +1,28 @@
+# Include only what belongs in the pip package
+include LICENSE
+include README.md
+include pyproject.toml
+
+# SDK and CLI source
+recursive-include spooled *.py
+include spooled/py.typed
+recursive-include cli *.py
+
+# Exclude everything else
+prune action
+prune backend
+prune tests
+prune tools
+prune docs
+prune examples
+prune schemas
+prune .github
+prune cdk.out
+prune htmlcov
+
+# Exclude dev/config files
+exclude .env
+exclude .env.*
+exclude .pre-commit-config.yaml
+exclude Makefile
+exclude setup.py

spooled_ai-0.4.0/PKG-INFO

@@ -0,0 +1,247 @@
+Metadata-Version: 2.4
+Name: spooled-ai
+Version: 0.4.0
+Summary: CI for AI agents - behavioral fingerprinting and drift detection
+Author: Spooled Team
+License: Proprietary
+Project-URL: Homepage, https://spooled.ai
+Project-URL: Documentation, https://spooled.ai/docs
+Project-URL: Changelog, https://spooled.ai/docs/changelog
+Keywords: ai,agents,tracing,debugging,replay,ci,behavioral-testing,llm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: structlog>=23.0.0
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: requests>=2.31.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: mypy>=1.5.0; extra == "dev"
+Requires-Dist: pre-commit>=3.5.0; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Provides-Extra: cli
+Provides-Extra: backend
+Requires-Dist: aws-cdk-lib>=2.100.0; extra == "backend"
+Requires-Dist: constructs>=10.0.0; extra == "backend"
+Requires-Dist: boto3>=1.28.0; extra == "backend"
+Provides-Extra: tools
+Requires-Dist: langchain>=0.3.0; extra == "tools"
+Requires-Dist: langchain-community>=0.3.0; extra == "tools"
+Requires-Dist: langchain-openai>=0.2.0; extra == "tools"
+Requires-Dist: langgraph>=0.2.0; extra == "tools"
+Requires-Dist: crewai>=0.80.0; extra == "tools"
+Requires-Dist: llama-index>=0.10.0; extra == "tools"
+Requires-Dist: pyautogen>=0.2.0; extra == "tools"
+Requires-Dist: aiohttp>=3.9.0; extra == "tools"
+Requires-Dist: boto3>=1.28.0; extra == "tools"
+Provides-Extra: metrics
+Requires-Dist: prometheus-client>=0.20.0; extra == "metrics"
+Provides-Extra: otel
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "otel"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "otel"
+Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.20.0; extra == "otel"
+Dynamic: license-file
+
+# Spooled — Behavioral CI for AI Agents
+
+> **The diff for agent behavior.**
+> Capture what your agent does, detect when it changes, gate the PR.
+
+AI agents are non-deterministic. The same code, prompt, and model produce different tool-calling behavior on every run. A one-word prompt edit can silently drop a compliance check. A model upgrade can change which tools get called. A KB refresh can alter the agent's decision path. Unit tests pass. Eval suites pass. Nobody notices until production.
+
+Spooled catches it on the PR.
+
+## What It Does
+
+**Capture** — wraps your LLM client and records the structural fingerprint of every agent run: which tools were called, in what order, how many times. Content-blind by architecture — prompts, customer data, and AI responses never leave your infrastructure.
+
+**Compare** — diffs the current run against a committed baseline. Shows exactly what changed: tools added, tools removed, sequence reordered, token usage shifted.
+
+**Gate** — posts a PR comment with a behavioral score. Blocks the merge if the policy says so. Resolution instructions included.
+
+## Install
+
+```bash
+pip install spooled-ai
+```
+
+## Quick Start
+
+```python
+import spooled
+from spooled.wrappers import wrap_openai
+from openai import OpenAI
+
+spooled.init(agent_id="my_agent")
+client = wrap_openai(OpenAI())
+
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Analyze this deal"}],
+    tools=MY_TOOLS,
+)
+
+spooled.shutdown()
+```
+
+That's it. Every tool call is captured. The trace is saved to `.spooled/traces/`. The hash chain signs every interaction at capture time.
+
+## CI Integration
+
+```yaml
+# .github/workflows/spooled.yml
+- name: Generate traces
+  run: python ci_runner.py
+  env:
+    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+- name: Spooled behavioral check
+  run: |
+    pip install spooled-ai
+    spooled ci compare .spooled/traces/*.jsonl \
+      --baseline .github/baselines \
+      --policy spooled-policy.yml \
+      --enable-blocking
+```
+
+The action compares traces against committed baselines and posts a PR comment:
+
+```
+❌ Spooled Behavioral CI: FAIL
+Spooled Score: 59/100 (D) 🔴
+
+| Agent      | Status              | Score | Tokens        |
+|------------|---------------------|-------|---------------|
+| deal_agent | ⚠️ Variant · Tooling change | 59 | 🔻 14198 (-32%) |
+
+🔧 Tool Changes:
+  ➖ sanctions_screening removed
+  ➖ ip_patent_search removed
+```
+
+## What Spooled Catches
+
+Validated across 50 real-world test scenarios with ~316 OpenAI API calls:
+
+| Change type | Example | Unit tests | Spooled |
+|-------------|---------|:----------:|:-------:|
+| Prompt tweak | "Be concise" drops compliance tools | ✅ Pass | **VARIANT** |
+| Model swap | gpt-4o drops sanctions screening | ✅ Pass | **VARIANT** |
+| Tool deprecation | Agent proceeds on sanctioned entity without sanctions data | ✅ Pass | **VARIANT** |
+| KB refresh | Fraud tickets lose customer response | ✅ Pass | **VARIANT** |
+| Schema migration | Field rename stops international detection | ✅ Pass | **VARIANT** |
+| Prompt reordering | Same words, sections reordered | ✅ Pass | **VARIANT** |
+| Tool description edit | Better docs change model decisions | ✅ Pass | **VARIANT** |
+| Upstream degradation | Retry paths appear in fingerprint | ✅ Pass | **VARIANT** |
+
+## Content-Blind Architecture
+
+Spooled never captures prompts, customer data, or AI responses. Only structural metadata: tool names, call sequence, token counts, timing. This is enforced in code — content is stripped before the trace reaches disk.
+
+Verified: we injected SSNs, credit cards, API keys, and email addresses into tool outputs. Scanned the trace file. **Zero PII found.** Structural data (tool names, model, usage) fully preserved.
+
+This opens regulated markets (healthcare, finance, government) where competitors who capture content cannot operate.
+
+## Multi-Agent Support
+
+```python
+# Supervisor starts
+spooled.init(agent_id="supervisor")
+# ... supervisor work ...
+
+# Child inherits parent linkage automatically
+spooled.init(agent_id="worker")  # auto-detects parent, inherits session
+# ... worker work ...
+spooled.shutdown()  # stops worker only, supervisor stays active
+```
+
+Each agent gets its own trace, its own fingerprint, its own baseline. Concurrent execution is safe — tested with 3 agents in parallel threads, zero cross-contamination.
+
+## Policy Rules
+
+```yaml
+# spooled-policy.yml
+name: "Production gate"
+enabled: true
+block_merges: true
+rules:
+  - name: "Block behavioral variants"
+    fail_if:
+      on_variant: true        # Block structural changes
+      on_new_behavior: true   # Block new intent patterns
+```
+
+Separate `on_variant` (structural change to a known intent) from `on_new_behavior` (entirely new intent bucket). Gate each independently.
+
+## Actionable Findings
+
+From the 50-scenario assessment:
+
+- **Set `seed=42`** on OpenAI tool-calling agents — achieves 100% fingerprint stability on gpt-4o-mini, 75% on gpt-4o
+- **Guardrails need "non-negotiable" language** — medium-strength prompts erode 100% under conversational pressure; strong prompts with explicit override-rejection hold 100%
+- **Run 20+ inputs** before shipping prompt changes — 10-input samples produce noise; 20+ reveals the real distribution shift
+
+## Supported Libraries
+
+**LLM Providers (explicit wrappers):**
+- OpenAI (sync/async, streaming)
+- Anthropic (sync/async, streaming)
+
+**HTTP & Cloud (auto-instrumented via hooks):**
+- AWS Bedrock
+- requests, httpx, aiohttp
+
+**Frameworks (callback handlers):**
+- LangChain, LlamaIndex, AutoGen, CrewAI, LangGraph
+
+## Commands
+
+```bash
+spooled verify trace <run_id>          # Hash chain integrity check
+spooled diff traces <a> <b>            # Structural comparison
+spooled ci compare <trace> --baseline  # CI comparison
+spooled ci update-baseline --from dir  # Generate baselines
+spooled analyze --agent-id my_agent    # Analyze trace patterns
+spooled policy init                    # Create a policy file
+```
+
+## How It Works
+
+1. `wrap_openai()` intercepts `chat.completions.create()` calls
+2. Each tool call is recorded with a SHA-256 hash of the previous interaction (Merkle chain)
+3. Content is stripped to structural metadata before saving (privacy by architecture)
+4. A `structural_hash` of the saved data enables post-save tampering detection
+5. The trace is saved as append-only JSONL in `.spooled/traces/`
+6. `spooled ci compare` extracts the fingerprint (tool graph) and diffs against the baseline
+7. Policy rules determine PASS/FAIL; the PR comment shows what changed
+
+## Documentation
+
+- [CI Integration Guide](docs/CI_INTEGRATION.md)
+- [Architecture & Trust Model](docs/ARCHITECTURE.md)
+- [Data Schema](docs/DATA_SCHEMA.md)
+- [Deployment Guide](docs/DEPLOYMENT.md)
+
+## License
+
+MIT

spooled_ai-0.4.0/README.md

@@ -0,0 +1,183 @@
+# Spooled — Behavioral CI for AI Agents
+
+> **The diff for agent behavior.**
+> Capture what your agent does, detect when it changes, gate the PR.
+
+AI agents are non-deterministic. The same code, prompt, and model produce different tool-calling behavior on every run. A one-word prompt edit can silently drop a compliance check. A model upgrade can change which tools get called. A KB refresh can alter the agent's decision path. Unit tests pass. Eval suites pass. Nobody notices until production.
+
+Spooled catches it on the PR.
+
+## What It Does
+
+**Capture** — wraps your LLM client and records the structural fingerprint of every agent run: which tools were called, in what order, how many times. Content-blind by architecture — prompts, customer data, and AI responses never leave your infrastructure.
+
+**Compare** — diffs the current run against a committed baseline. Shows exactly what changed: tools added, tools removed, sequence reordered, token usage shifted.
+
+**Gate** — posts a PR comment with a behavioral score. Blocks the merge if the policy says so. Resolution instructions included.
+
+## Install
+
+```bash
+pip install spooled-ai
+```
+
+## Quick Start
+
+```python
+import spooled
+from spooled.wrappers import wrap_openai
+from openai import OpenAI
+
+spooled.init(agent_id="my_agent")
+client = wrap_openai(OpenAI())
+
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Analyze this deal"}],
+    tools=MY_TOOLS,
+)
+
+spooled.shutdown()
+```
+
+That's it. Every tool call is captured. The trace is saved to `.spooled/traces/`. The hash chain signs every interaction at capture time.
+
+## CI Integration
+
+```yaml
+# .github/workflows/spooled.yml
+- name: Generate traces
+  run: python ci_runner.py
+  env:
+    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+- name: Spooled behavioral check
+  run: |
+    pip install spooled-ai
+    spooled ci compare .spooled/traces/*.jsonl \
+      --baseline .github/baselines \
+      --policy spooled-policy.yml \
+      --enable-blocking
+```
+
+The action compares traces against committed baselines and posts a PR comment:
+
+```
+❌ Spooled Behavioral CI: FAIL
+Spooled Score: 59/100 (D) 🔴
+
+| Agent      | Status              | Score | Tokens        |
+|------------|---------------------|-------|---------------|
+| deal_agent | ⚠️ Variant · Tooling change | 59 | 🔻 14198 (-32%) |
+
+🔧 Tool Changes:
+  ➖ sanctions_screening removed
+  ➖ ip_patent_search removed
+```
+
+## What Spooled Catches
+
+Validated across 50 real-world test scenarios with ~316 OpenAI API calls:
+
+| Change type | Example | Unit tests | Spooled |
+|-------------|---------|:----------:|:-------:|
+| Prompt tweak | "Be concise" drops compliance tools | ✅ Pass | **VARIANT** |
+| Model swap | gpt-4o drops sanctions screening | ✅ Pass | **VARIANT** |
+| Tool deprecation | Agent proceeds on sanctioned entity without sanctions data | ✅ Pass | **VARIANT** |
+| KB refresh | Fraud tickets lose customer response | ✅ Pass | **VARIANT** |
+| Schema migration | Field rename stops international detection | ✅ Pass | **VARIANT** |
+| Prompt reordering | Same words, sections reordered | ✅ Pass | **VARIANT** |
+| Tool description edit | Better docs change model decisions | ✅ Pass | **VARIANT** |
+| Upstream degradation | Retry paths appear in fingerprint | ✅ Pass | **VARIANT** |
+
+## Content-Blind Architecture
+
+Spooled never captures prompts, customer data, or AI responses. Only structural metadata: tool names, call sequence, token counts, timing. This is enforced in code — content is stripped before the trace reaches disk.
+
+Verified: we injected SSNs, credit cards, API keys, and email addresses into tool outputs. Scanned the trace file. **Zero PII found.** Structural data (tool names, model, usage) fully preserved.
+
+This opens regulated markets (healthcare, finance, government) where competitors who capture content cannot operate.
+
+## Multi-Agent Support
+
+```python
+# Supervisor starts
+spooled.init(agent_id="supervisor")
+# ... supervisor work ...
+
+# Child inherits parent linkage automatically
+spooled.init(agent_id="worker")  # auto-detects parent, inherits session
+# ... worker work ...
+spooled.shutdown()  # stops worker only, supervisor stays active
+```
+
+Each agent gets its own trace, its own fingerprint, its own baseline. Concurrent execution is safe — tested with 3 agents in parallel threads, zero cross-contamination.
+
+## Policy Rules
+
+```yaml
+# spooled-policy.yml
+name: "Production gate"
+enabled: true
+block_merges: true
+rules:
+  - name: "Block behavioral variants"
+    fail_if:
+      on_variant: true        # Block structural changes
+      on_new_behavior: true   # Block new intent patterns
+```
+
+Separate `on_variant` (structural change to a known intent) from `on_new_behavior` (entirely new intent bucket). Gate each independently.
+
+## Actionable Findings
+
+From the 50-scenario assessment:
+
+- **Set `seed=42`** on OpenAI tool-calling agents — achieves 100% fingerprint stability on gpt-4o-mini, 75% on gpt-4o
+- **Guardrails need "non-negotiable" language** — medium-strength prompts erode 100% under conversational pressure; strong prompts with explicit override-rejection hold 100%
+- **Run 20+ inputs** before shipping prompt changes — 10-input samples produce noise; 20+ reveals the real distribution shift
+
+## Supported Libraries
+
+**LLM Providers (explicit wrappers):**
+- OpenAI (sync/async, streaming)
+- Anthropic (sync/async, streaming)
+
+**HTTP & Cloud (auto-instrumented via hooks):**
+- AWS Bedrock
+- requests, httpx, aiohttp
+
+**Frameworks (callback handlers):**
+- LangChain, LlamaIndex, AutoGen, CrewAI, LangGraph
+
+## Commands
+
+```bash
+spooled verify trace <run_id>          # Hash chain integrity check
+spooled diff traces <a> <b>            # Structural comparison
+spooled ci compare <trace> --baseline  # CI comparison
+spooled ci update-baseline --from dir  # Generate baselines
+spooled analyze --agent-id my_agent    # Analyze trace patterns
+spooled policy init                    # Create a policy file
+```
+
+## How It Works
+
+1. `wrap_openai()` intercepts `chat.completions.create()` calls
+2. Each tool call is recorded with a SHA-256 hash of the previous interaction (Merkle chain)
+3. Content is stripped to structural metadata before saving (privacy by architecture)
+4. A `structural_hash` of the saved data enables post-save tampering detection
+5. The trace is saved as append-only JSONL in `.spooled/traces/`
+6. `spooled ci compare` extracts the fingerprint (tool graph) and diffs against the baseline
+7. Policy rules determine PASS/FAIL; the PR comment shows what changed
+
+## Documentation
+
+- [CI Integration Guide](docs/CI_INTEGRATION.md)
+- [Architecture & Trust Model](docs/ARCHITECTURE.md)
+- [Data Schema](docs/DATA_SCHEMA.md)
+- [Deployment Guide](docs/DEPLOYMENT.md)
+
+## License
+
+MIT

spooled_ai-0.4.0/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Spooled CLI - Command-line tool for replaying traces."""
+
+from __future__ import annotations
+
+__version__ = "0.4.0"

spooled_ai-0.4.0/cli/commands/__init__.py

@@ -0,0 +1,53 @@
+"""CLI command modules."""
+
+from __future__ import annotations
+
+from cli.commands import (
+    analyze,
+    attest,
+    baseline,
+    ci,
+    demo,
+    diff,
+    doctor,
+    fingerprint,
+    fleet,
+    ingest,
+    init,
+    keys,
+    list,
+    new_agent,
+    policy,
+    pull,
+    replay,
+    session,
+    traces,
+    verify,
+    view,
+    watch,
+)
+
+__all__ = [
+    "pull",
+    "replay",
+    "diff",
+    "view",
+    "list",
+    "init",
+    "verify",
+    "baseline",
+    "fingerprint",
+    "ci",
+    "fleet",
+    "policy",
+    "keys",
+    "session",
+    "doctor",
+    "attest",
+    "traces",
+    "new_agent",
+    "demo",
+    "watch",
+    "ingest",
+    "analyze",
+]