spandrift 0.1.0__tar.gz

spandrift-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: Spandrift CI Gate
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    name: Run Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Run pytest
+        run: pytest -v
+
+  trace-regression-gate:
+    name: Trace Regression Gate (spandrift diff)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install spandrift
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e "."
+      - name: Run Benchmark / Generate Traces
+        run: |
+          # Generate base and head traces or download benchmark artifacts
+          python demo/run_demo.py
+      - name: Check Cost and Latency Regressions
+        run: |
+          spandrift diff \
+            demo/base_trace.json \
+            demo/head_trace.json \
+            --cost-threshold 0.10 \
+            --latency-threshold 0.20 \
+            --exit-code

spandrift-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,67 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build, test, and package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tools and test dependencies
+        run: |
+          python -m pip install --upgrade pip build
+          pip install -e ".[dev]"
+
+      - name: Run test suite
+        run: python -m pytest tests/ -v
+
+      - name: Build package
+        run: python -m build
+
+      - name: Verify wheel installs cleanly
+        run: |
+          python -m venv /tmp/verify-venv
+          /tmp/verify-venv/bin/pip install dist/spandrift-*.whl
+          /tmp/verify-venv/bin/spandrift --help
+          /tmp/verify-venv/bin/python -c "from spandrift import __version__; print(f'spandrift {__version__}')"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # Required for PyPI Trusted Publishing (OIDC)
+    environment:
+      name: pypi
+      url: https://pypi.org/p/spandrift
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # Uses Trusted Publishing (OIDC) — no PYPI_API_TOKEN needed.
+        # Configure the trusted publisher on PyPI's side:
+        #   Owner: arv20
+        #   Repository: spandrift
+        #   Workflow: publish.yml
+        #   Environment: pypi

spandrift-0.1.0/.gitignore

@@ -0,0 +1,51 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# macOS
+.DS_Store
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+
+# Nested sub-repos and temporary state
+/test/
+/tech club/
+/the team/
+node_modules/
+.freebuff/
+*.db
+*.db-shm
+*.db-wal
+

spandrift-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Aarav Kolgaonkar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

spandrift-0.1.0/PKG-INFO

@@ -0,0 +1,256 @@
+Metadata-Version: 2.5
+Name: spandrift
+Version: 0.1.0
+Summary: Local-first trace analysis and diagnostic engine for multi-agent LLM systems.
+Project-URL: Homepage, https://github.com/arv20/spandrift
+Project-URL: Repository, https://github.com/arv20/spandrift
+Project-URL: Issues, https://github.com/arv20/spandrift/issues
+Project-URL: Documentation, https://github.com/arv20/spandrift#readme
+Author-email: Aarav Kolgaonkar <arv20@users.noreply.github.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai-agents,diagnostics,langchain,langgraph,llm,multi-agent,observability,opentelemetry,smolagents,tracing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: System :: Monitoring
+Requires-Python: >=3.11
+Requires-Dist: click>=8.0
+Requires-Dist: polars>=1.0
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: opentelemetry-api>=1.20; extra == 'dev'
+Requires-Dist: opentelemetry-sdk>=1.20; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: otel
+Requires-Dist: opentelemetry-api>=1.20; extra == 'otel'
+Requires-Dist: opentelemetry-sdk>=1.20; extra == 'otel'
+Provides-Extra: smolagents
+Requires-Dist: openinference-instrumentation-smolagents>=0.1; extra == 'smolagents'
+Requires-Dist: smolagents>=1.0; extra == 'smolagents'
+Description-Content-Type: text/markdown
+
+# spandrift
+
+Local-first trace analysis for multi-agent LLM systems.
+
+Basically, spandrift reads the OpenTelemetry spans your agents are probably already producing and tries to answer a few questions that normal tracing dashboards don't really focus on:
+
+- which agent actually cost the money?
+- did the same tool/agent accidentally get called twice?
+- are retries going crazy?
+- did this PR make the workflow slower or more expensive?
+
+It's not meant to replace Langfuse, Phoenix, Logfire, etc. If you're already sending traces to one of those, keep doing that.
+
+spandrift just reads the same OTel spans and runs a smaller set of checks that are more specific to agent workflows.
+
+Right now it works with spans from `openinference-instrumentation-smolagents` for smolagents, Pydantic AI's built-in instrumentation, and its own decorators for code that isn't instrumented already.
+
+## Install
+
+```bash
+pip install spandrift
+```
+
+Python 3.11+ is required.
+
+The concurrency-safe attribution stuff depends on `asyncio.TaskGroup` and the `context=` argument on `create_task`, which were both added in 3.11.
+
+## Quickstart
+
+```bash
+spandrift analyze demo/trace.json
+```
+
+```text
+╭──────────────────── Spandrift Analysis: head_trace.json ─────────────────────╮
+│ Total spans: 15    Total cost: $0.0152                                       │
+│ Wall-clock: 234ms   Compute time: 775ms                                      │
+╰──────────────────────────────────────────────────────────────────────────────╯
+                                Per-Agent Rollup
+┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
+┃               ┃           ┃            ┃         ┃           ┃        Tokens ┃
+┃ Agent         ┃      Cost ┃ Wall-clock ┃ Compute ┃ LLM calls ┃      (in/out) ┃
+┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
+│ FactChecker   │ $0.002075 │       85ms │   159ms │         1 │       350/120 │
+│ Orchestrator  │ $0.000000 │      234ms │   234ms │         0 │           0/0 │
+│ ResearchAgent │ $0.006800 │      149ms │   255ms │         2 │      1.0k/420 │
+│ WriterAgent   │ $0.006325 │       74ms │   126ms │         1 │       850/420 │
+└───────────────┴───────────┴────────────┴─────────┴───────────┴───────────────┘
+
+                      Execution Waterfall & Concurrency
+ Span / Task         Timeline (░=TTFT, █=Exec)     Duration  TTFT       Cost
+ Orchestrator        ████████████████████████████     234ms     —          —
+  ├─ ResearchAgent   ████████                          74ms     —          —
+  │  ├─ chat gpt-4o    ░░░██                           42ms  21ms  $0.003400
+  │  └─ web_search          █                          11ms     —          —
+  ├─ FactChecker     ██████████                        85ms     —          —
+  │  ├─ chat gpt-4o   ░░██                             31ms  21ms  $0.002075
+  │  ├─ web_search        █                            11ms     —          —
+  │  ├─ web_search         █                           10ms     —          —
+  │  ├─ web_search          █                          11ms     —          —
+  │  └─ web_search           ██                        11ms     —          —
+  ├─ ResearchAgent             █████████               75ms     —          —
+  │  ├─ chat gpt-4o              ░░░██                 42ms  21ms  $0.003400
+  │  └─ web_search                    ██               11ms     —          —
+  └─ WriterAgent                        ████████       74ms     —          —
+     └─ chat gpt-4o                       ░░░░██       52ms  31ms  $0.006325
+
+⚠ Duplicate Calls
+  ResearchAgent called 2× with identical input  wasted: $0.003400
+
+⚠ Retry/Loop Storms
+  web_search: 4 calls with identical input
+```
+
+You can also compare two runs:
+
+```bash
+spandrift diff base.json head.json --cost-threshold 0.10 --exit-code
+```
+
+That makes it usable in CI too. If the cost delta crosses the threshold, the command can fail the step.
+
+There's a composite GitHub Action in `action.yml` that wraps this. If you want it to post the diff as a PR comment, the calling workflow needs:
+
+```yaml
+pull-requests: write
+```
+
+There's also:
+
+```bash
+spandrift listen
+```
+
+which starts a local OTLP receiver so you don't have to write a trace file first. It binds to `127.0.0.1` by default.
+
+## What it catches
+
+### Cost + latency per agent
+
+Costs and timing are rolled up per agent, including workflows with concurrent branches.
+
+This matters because if two sub-agents run at the same time through `asyncio.gather`, you obviously don't want to just add both wall-clock durations together and pretend they ran sequentially.
+
+### Duplicate calls
+
+If the same agent or tool gets called twice with the same input in one run, it'll flag it.
+
+This has caught more dumb orchestration bugs than I expected.
+
+### Retry storms
+
+It also looks for repeated calls with identical or near-identical input.
+
+Those get chained using exact matches or token-level similarity on the captured input text, so something retrying over and over with tiny changes doesn't just disappear into the trace.
+
+### TTFT outliers
+
+TTFT = time to first token.
+
+spandrift tracks that separately from total request duration and flags outliers per model. Mostly useful for figuring out whether the actual agent got slower or whether a provider/model was just having a bad time.
+
+## How it works
+
+For code that isn't already OTel-instrumented, there are two decorators:
+
+```python
+@profile_agent
+@profile_tool
+```
+
+They wrap async functions and emit spans in the same general shape as the other supported instrumentation.
+
+A surprisingly annoying part of building this was getting attribution right with concurrency.
+
+A global "current span" works fine until two agents run at the same time. Then child spans can start getting attributed to whichever agent most recently changed the global value, which is obviously bad.
+
+spandrift uses `contextvars.ContextVar` instead.
+
+asyncio copies context when tasks are created, so each concurrent task keeps the right agent context without having to manually pass IDs everywhere.
+
+There's a test specifically for this too: two agents execute concurrently and both produce child spans. I wanted a test that would actually break the naive implementation instead of just testing the easy sequential case.
+
+The analysis side uses Polars.
+
+Most of the work is grouped batch aggregation anyway — cost by agent, latency percentiles by model, retries, duplicates, etc. A run can also get into the thousands of spans pretty quickly once agents start retrying, so using a columnar dataframe library made sense.
+
+## A bug I found while building it
+
+The cost calculation ended up being slightly more annoying than expected because of cached tokens.
+
+OpenTelemetry's:
+
+```text
+gen_ai.usage.input_tokens
+```
+
+is the total input token count, including cached input.
+
+So if you take that value and then add cache-read/cache-write token counts separately, you're double-counting part of the bill.
+
+I originally did exactly that.
+
+I caught it by comparing spandrift's numbers against `genai-prices` on a few actual Anthropic/OpenAI caching scenarios. The fix is in `cost_engine.py` and there's a regression test for it now.
+
+## Known trade-offs
+
+Duplicate/retry detection needs the input text to actually exist in the span.
+
+Usually that's either:
+
+```text
+input.value
+```
+
+or:
+
+```text
+gen_ai.input.messages
+```
+
+The `@profile_agent` path always captures it, but bare LLM spans from really minimal OTel instrumentation sometimes don't.
+
+If the input isn't there, spandrift doesn't try to guess. That span just isn't checked for duplicate/retry similarity.
+
+Pricing is also a static table right now, not a live API lookup.
+
+`cost_engine.py` includes the date the bundled prices were last verified. If something changes before I update it, you can override the table with:
+
+```bash
+SPANDRIFT_PRICES_PATH
+```
+
+There's also a LangChain/LangGraph adapter under `spandrift.adapters`.
+
+That part is experimental and definitely less tested than the core path. Also, if your entire stack is LangChain, LangSmith already does a lot, so I'm not trying to pretend this replaces it.
+
+## Why I made this
+
+I mostly wanted something I could run locally on traces after changing an agent workflow and immediately answer stuff like:
+
+> why did this run suddenly cost 40% more?
+
+or
+
+> why did this tool execute 6 times?
+
+General observability tools are way more capable overall, but sometimes I just want those answers without digging through a tracing UI.
+
+That's basically the point of spandrift.
+
+It's still a young project, so issues and PRs are very welcome. If something looks wrong, there's a non-zero chance it actually is.
+
+## License
+
+MIT © Aarav Kolgaonkar

spandrift-0.1.0/README.md

@@ -0,0 +1,216 @@
+# spandrift
+
+Local-first trace analysis for multi-agent LLM systems.
+
+Basically, spandrift reads the OpenTelemetry spans your agents are probably already producing and tries to answer a few questions that normal tracing dashboards don't really focus on:
+
+- which agent actually cost the money?
+- did the same tool/agent accidentally get called twice?
+- are retries going crazy?
+- did this PR make the workflow slower or more expensive?
+
+It's not meant to replace Langfuse, Phoenix, Logfire, etc. If you're already sending traces to one of those, keep doing that.
+
+spandrift just reads the same OTel spans and runs a smaller set of checks that are more specific to agent workflows.
+
+Right now it works with spans from `openinference-instrumentation-smolagents` for smolagents, Pydantic AI's built-in instrumentation, and its own decorators for code that isn't instrumented already.
+
+## Install
+
+```bash
+pip install spandrift
+```
+
+Python 3.11+ is required.
+
+The concurrency-safe attribution stuff depends on `asyncio.TaskGroup` and the `context=` argument on `create_task`, which were both added in 3.11.
+
+## Quickstart
+
+```bash
+spandrift analyze demo/trace.json
+```
+
+```text
+╭──────────────────── Spandrift Analysis: head_trace.json ─────────────────────╮
+│ Total spans: 15    Total cost: $0.0152                                       │
+│ Wall-clock: 234ms   Compute time: 775ms                                      │
+╰──────────────────────────────────────────────────────────────────────────────╯
+                                Per-Agent Rollup
+┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
+┃               ┃           ┃            ┃         ┃           ┃        Tokens ┃
+┃ Agent         ┃      Cost ┃ Wall-clock ┃ Compute ┃ LLM calls ┃      (in/out) ┃
+┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
+│ FactChecker   │ $0.002075 │       85ms │   159ms │         1 │       350/120 │
+│ Orchestrator  │ $0.000000 │      234ms │   234ms │         0 │           0/0 │
+│ ResearchAgent │ $0.006800 │      149ms │   255ms │         2 │      1.0k/420 │
+│ WriterAgent   │ $0.006325 │       74ms │   126ms │         1 │       850/420 │
+└───────────────┴───────────┴────────────┴─────────┴───────────┴───────────────┘
+
+                      Execution Waterfall & Concurrency
+ Span / Task         Timeline (░=TTFT, █=Exec)     Duration  TTFT       Cost
+ Orchestrator        ████████████████████████████     234ms     —          —
+  ├─ ResearchAgent   ████████                          74ms     —          —
+  │  ├─ chat gpt-4o    ░░░██                           42ms  21ms  $0.003400
+  │  └─ web_search          █                          11ms     —          —
+  ├─ FactChecker     ██████████                        85ms     —          —
+  │  ├─ chat gpt-4o   ░░██                             31ms  21ms  $0.002075
+  │  ├─ web_search        █                            11ms     —          —
+  │  ├─ web_search         █                           10ms     —          —
+  │  ├─ web_search          █                          11ms     —          —
+  │  └─ web_search           ██                        11ms     —          —
+  ├─ ResearchAgent             █████████               75ms     —          —
+  │  ├─ chat gpt-4o              ░░░██                 42ms  21ms  $0.003400
+  │  └─ web_search                    ██               11ms     —          —
+  └─ WriterAgent                        ████████       74ms     —          —
+     └─ chat gpt-4o                       ░░░░██       52ms  31ms  $0.006325
+
+⚠ Duplicate Calls
+  ResearchAgent called 2× with identical input  wasted: $0.003400
+
+⚠ Retry/Loop Storms
+  web_search: 4 calls with identical input
+```
+
+You can also compare two runs:
+
+```bash
+spandrift diff base.json head.json --cost-threshold 0.10 --exit-code
+```
+
+That makes it usable in CI too. If the cost delta crosses the threshold, the command can fail the step.
+
+There's a composite GitHub Action in `action.yml` that wraps this. If you want it to post the diff as a PR comment, the calling workflow needs:
+
+```yaml
+pull-requests: write
+```
+
+There's also:
+
+```bash
+spandrift listen
+```
+
+which starts a local OTLP receiver so you don't have to write a trace file first. It binds to `127.0.0.1` by default.
+
+## What it catches
+
+### Cost + latency per agent
+
+Costs and timing are rolled up per agent, including workflows with concurrent branches.
+
+This matters because if two sub-agents run at the same time through `asyncio.gather`, you obviously don't want to just add both wall-clock durations together and pretend they ran sequentially.
+
+### Duplicate calls
+
+If the same agent or tool gets called twice with the same input in one run, it'll flag it.
+
+This has caught more dumb orchestration bugs than I expected.
+
+### Retry storms
+
+It also looks for repeated calls with identical or near-identical input.
+
+Those get chained using exact matches or token-level similarity on the captured input text, so something retrying over and over with tiny changes doesn't just disappear into the trace.
+
+### TTFT outliers
+
+TTFT = time to first token.
+
+spandrift tracks that separately from total request duration and flags outliers per model. Mostly useful for figuring out whether the actual agent got slower or whether a provider/model was just having a bad time.
+
+## How it works
+
+For code that isn't already OTel-instrumented, there are two decorators:
+
+```python
+@profile_agent
+@profile_tool
+```
+
+They wrap async functions and emit spans in the same general shape as the other supported instrumentation.
+
+A surprisingly annoying part of building this was getting attribution right with concurrency.
+
+A global "current span" works fine until two agents run at the same time. Then child spans can start getting attributed to whichever agent most recently changed the global value, which is obviously bad.
+
+spandrift uses `contextvars.ContextVar` instead.
+
+asyncio copies context when tasks are created, so each concurrent task keeps the right agent context without having to manually pass IDs everywhere.
+
+There's a test specifically for this too: two agents execute concurrently and both produce child spans. I wanted a test that would actually break the naive implementation instead of just testing the easy sequential case.
+
+The analysis side uses Polars.
+
+Most of the work is grouped batch aggregation anyway — cost by agent, latency percentiles by model, retries, duplicates, etc. A run can also get into the thousands of spans pretty quickly once agents start retrying, so using a columnar dataframe library made sense.
+
+## A bug I found while building it
+
+The cost calculation ended up being slightly more annoying than expected because of cached tokens.
+
+OpenTelemetry's:
+
+```text
+gen_ai.usage.input_tokens
+```
+
+is the total input token count, including cached input.
+
+So if you take that value and then add cache-read/cache-write token counts separately, you're double-counting part of the bill.
+
+I originally did exactly that.
+
+I caught it by comparing spandrift's numbers against `genai-prices` on a few actual Anthropic/OpenAI caching scenarios. The fix is in `cost_engine.py` and there's a regression test for it now.
+
+## Known trade-offs
+
+Duplicate/retry detection needs the input text to actually exist in the span.
+
+Usually that's either:
+
+```text
+input.value
+```
+
+or:
+
+```text
+gen_ai.input.messages
+```
+
+The `@profile_agent` path always captures it, but bare LLM spans from really minimal OTel instrumentation sometimes don't.
+
+If the input isn't there, spandrift doesn't try to guess. That span just isn't checked for duplicate/retry similarity.
+
+Pricing is also a static table right now, not a live API lookup.
+
+`cost_engine.py` includes the date the bundled prices were last verified. If something changes before I update it, you can override the table with:
+
+```bash
+SPANDRIFT_PRICES_PATH
+```
+
+There's also a LangChain/LangGraph adapter under `spandrift.adapters`.
+
+That part is experimental and definitely less tested than the core path. Also, if your entire stack is LangChain, LangSmith already does a lot, so I'm not trying to pretend this replaces it.
+
+## Why I made this
+
+I mostly wanted something I could run locally on traces after changing an agent workflow and immediately answer stuff like:
+
+> why did this run suddenly cost 40% more?
+
+or
+
+> why did this tool execute 6 times?
+
+General observability tools are way more capable overall, but sometimes I just want those answers without digging through a tracing UI.
+
+That's basically the point of spandrift.
+
+It's still a young project, so issues and PRs are very welcome. If something looks wrong, there's a non-zero chance it actually is.
+
+## License
+
+MIT © Aarav Kolgaonkar