traceops 0.5.0__tar.gz

traceops-0.5.0/.github/actions/agent-replay/action.yml

@@ -0,0 +1,138 @@
+name: "TraceOps"
+description: "Record and replay LLM agent traces for regression testing in CI"
+branding:
+  icon: "refresh-cw"
+  color: "blue"
+
+inputs:
+  cassette-dir:
+    description: "Directory containing cassette files"
+    required: false
+    default: "cassettes"
+  record-mode:
+    description: "Recording mode: none, once, new, all"
+    required: false
+    default: "once"
+  python-version:
+    description: "Python version to use"
+    required: false
+    default: "3.12"
+  requirements:
+    description: "Extra pip requirements to install (space-separated or path to requirements.txt)"
+    required: false
+    default: ""
+  pytest-args:
+    description: "Additional arguments to pass to pytest"
+    required: false
+    default: ""
+  post-diff-comment:
+    description: "Post a PR comment with trace diff on failure (requires GITHUB_TOKEN)"
+    required: false
+    default: "true"
+
+outputs:
+  result:
+    description: "Test result: passed, failed, or error"
+    value: ${{ steps.test.outputs.result }}
+  diff-summary:
+    description: "Trace diff summary (if regression detected)"
+    value: ${{ steps.diff.outputs.summary }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install TraceOps
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install traceops[all]
+
+    - name: Install extra requirements
+      shell: bash
+      if: inputs.requirements != ''
+      run: |
+        if [ -f "${{ inputs.requirements }}" ]; then
+          pip install -r "${{ inputs.requirements }}"
+        else
+          pip install ${{ inputs.requirements }}
+        fi
+
+    - name: Run tests with TraceOps
+      id: test
+      shell: bash
+      continue-on-error: true
+      run: |
+        set +e
+        pytest --record-mode=${{ inputs.record-mode }} ${{ inputs.pytest-args }} 2>&1 | tee test_output.txt
+        EXIT_CODE=${PIPESTATUS[0]}
+        if [ $EXIT_CODE -eq 0 ]; then
+          echo "result=passed" >> $GITHUB_OUTPUT
+        else
+          echo "result=failed" >> $GITHUB_OUTPUT
+        fi
+        exit $EXIT_CODE
+
+    - name: Generate diff on failure
+      id: diff
+      shell: bash
+      if: steps.test.outputs.result == 'failed'
+      run: |
+        echo "## 🔁 TraceOps Regression Report" > diff_report.md
+        echo "" >> diff_report.md
+
+        # Find cassettes that have changed
+        CASSETTE_DIR="${{ inputs.cassette-dir }}"
+        if [ -d "$CASSETTE_DIR" ]; then
+          CHANGED=$(git diff --name-only HEAD -- "$CASSETTE_DIR" 2>/dev/null || echo "")
+          if [ -n "$CHANGED" ]; then
+            echo "### Changed Cassettes" >> diff_report.md
+            echo '```' >> diff_report.md
+            echo "$CHANGED" >> diff_report.md
+            echo '```' >> diff_report.md
+          fi
+        fi
+
+        echo "" >> diff_report.md
+        echo "### Test Output (last 50 lines)" >> diff_report.md
+        echo '```' >> diff_report.md
+        tail -50 test_output.txt >> diff_report.md
+        echo '```' >> diff_report.md
+
+        SUMMARY=$(cat diff_report.md)
+        # Escape for GitHub output
+        SUMMARY="${SUMMARY//'%'/'%25'}"
+        SUMMARY="${SUMMARY//$'\n'/'%0A'}"
+        SUMMARY="${SUMMARY//$'\r'/'%0D'}"
+        echo "summary=$SUMMARY" >> $GITHUB_OUTPUT
+
+    - name: Post PR comment
+      if: >
+        steps.test.outputs.result == 'failed' &&
+        inputs.post-diff-comment == 'true' &&
+        github.event_name == 'pull_request'
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const fs = require('fs');
+          let body = '## 🔁 TraceOps Regression Detected\n\n';
+          try {
+            body += fs.readFileSync('diff_report.md', 'utf8');
+          } catch (e) {
+            body += 'See test output for details.';
+          }
+          await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+            body: body.slice(0, 65536),
+          });
+
+    - name: Fail if tests failed
+      shell: bash
+      if: steps.test.outputs.result == 'failed'
+      run: exit 1

traceops-0.5.0/.github/workflows/ci.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint
+        run: ruff check src/ tests/
+
+      - name: Test
+        run: pytest --tb=short -q
+
+      - name: Type check
+        run: mypy src/trace_ops/ --ignore-missing-imports
+        if: matrix.python-version == '3.12'

traceops-0.5.0/.gitignore

@@ -0,0 +1,46 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.egg
+*.egg-info/
+dist/
+build/
+.eggs/
+*.whl
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Pytest / coverage
+.pytest_cache/
+.coverage
+htmlcov/
+coverage.xml
+
+# Mypy
+.mypy_cache/
+
+# Ruff
+.ruff_cache/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Hatch
+.hatch/
+
+# Dist artifacts
+dist/
+*.tar.gz

traceops-0.5.0/.pypirc.template

@@ -0,0 +1,14 @@
+[distutils]
+index-servers =
+    pypi
+    testpypi
+
+[pypi]
+repository = https://upload.pypi.org/legacy/
+username = __token__
+password = pypi-REPLACE_WITH_YOUR_PRODUCTION_TOKEN
+
+[testpypi]
+repository = https://test.pypi.org/legacy/
+username = __token__
+password = pypi-REPLACE_WITH_YOUR_TEST_TOKEN

traceops-0.5.0/CHANGELOG.md

@@ -0,0 +1,78 @@
+# Changelog
+
+All notable changes to **TraceOps** will be documented in this file.
+
+The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+---
+
+## [0.4.0] — 2025-01-27
+
+### Added
+- **Cost Dashboard** — new `replay costs [dir]` CLI command aggregating spend
+  across cassettes, with per-model and per-cassette breakdowns.  Also available
+  programmatically via `CostDashboard` / `CostSummary`.
+- **Enhanced `replay diff`** — `--detailed` flag for per-event side-by-side
+  comparison and `--output` flag to write an HTML diff report.
+- **Auto-record mode** — `--record-mode auto` (and `AGENT_REPLAY_RECORD` /
+  `AGENT_REPLAY_MODE` environment variables) for zero-config "record if
+  missing, replay if exists" workflow.
+- Exported `CostDashboard` and `CostSummary` from the top-level package.
+- 100 new tests (529 total), raising coverage from 75 % → 77 %.
+
+### Changed
+- Harmonised version strings across CLI, HTML reporter, types, and
+  `pyproject.toml` (all now `0.4.0`).
+
+### Fixed
+- `replay diff` no longer silently swallows HTML-output errors.
+
+---
+
+## [0.3.0] — 2025-01-26
+
+### Added
+- **Framework interceptors** — first-class LangChain, LangGraph, and CrewAI
+  support via `interceptors/` sub-package.
+- **Integration test suite** — end-to-end tests for OpenAI, Anthropic,
+  LangChain, and LangGraph recording/replay.
+- **pytest plugin tests** — full coverage of `--record`, `--record-mode`,
+  `--replay-strict`, `cassette` fixture, `trace_snapshot` fixture, and the
+  `@pytest.mark.budget` marker.
+- Example scripts under `examples/`.
+- README refresh with quick-start, architecture diagram, and badge row.
+
+---
+
+## [0.2.0] — 2025-01-25
+
+### Added
+- **Async & streaming support** — `async with Recorder()` / `Replayer()`,
+  plus transparent streaming-response replay.
+- **Normalization engine** — `normalize_response()` and
+  `normalize_for_comparison()` for provider-agnostic diffing.
+- **Budget assertions** — `assert_cost_under`, `assert_tokens_under`,
+  `assert_max_llm_calls`, `assert_no_loops`.
+- **HTML trace reports** — `generate_html_report()` with inline CSS/JS,
+  dark theme, token bar chart, and expandable event details.
+- **Terminal debugger** — interactive time-travel debugger (`replay debug`).
+- **CLI commands** — `replay inspect`, `diff`, `export`, `debug`, `report`,
+  `ls`, `prune`, `stats`, `validate`.
+- **Cassette management** — `save_cassette` / `load_cassette` with YAML
+  storage, automatic redaction of API keys.
+- **Diff engine** — `diff_traces` / `assert_trace_unchanged` with DeepDiff
+  integration.
+
+---
+
+## [0.1.0] — 2025-01-24
+
+### Added
+- Initial release.
+- `Recorder` context-manager with monkey-patching for OpenAI, Anthropic,
+  and LiteLLM (sync only).
+- `Replayer` context-manager returning stored responses.
+- Core data model: `Trace`, `TraceEvent`, `TraceMetadata`, `EventType`.
+- YAML cassette serialization.
+- pytest plugin with `--record` flag and `agent_cassette` marker.

traceops-0.5.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Joshua
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.