backstop-ai 0.6.0__tar.gz

backstop_ai-0.6.0/.github/ISSUE_TEMPLATE/benchmark_report.yml

@@ -0,0 +1,26 @@
+name: Benchmark report
+description: Share benchmark results or report benchmark issues
+title: "[Benchmark]: "
+labels: ["benchmark"]
+body:
+  - type: textarea
+    id: command
+    attributes:
+      label: Command
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, Python version, CPU, provider SDK versions, and whether a local mock provider or real provider was used.
+    validations:
+      required: true
+  - type: textarea
+    id: results
+    attributes:
+      label: Results
+      render: text
+    validations:
+      required: true

backstop_ai-0.6.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,42 @@
+name: Bug report
+description: Report a Backstop bug
+title: "[Bug]: "
+labels: ["bug"]
+body:
+  - type: textarea
+    id: summary
+    attributes:
+      label: Summary
+      description: What happened?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Reproduction
+      description: Minimal code or commands that reproduce the issue.
+      render: python
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Backstop version or commit
+    validations:
+      required: true
+  - type: input
+    id: provider
+    attributes:
+      label: Provider SDK and version
+      placeholder: openai 1.x, anthropic 0.x
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+    validations:
+      required: true
+  - type: textarea
+    id: notes
+    attributes:
+      label: Notes
+      description: Do not include API keys, real prompts, real responses, or customer data.

backstop_ai-0.6.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,37 @@
+name: Feature request
+description: Suggest an improvement to Backstop
+title: "[Feature]: "
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What LLM spend, latency, reliability, or provider-control problem does this solve?
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposal
+      description: What should Backstop do?
+    validations:
+      required: true
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      options:
+        - Budget enforcement
+        - Distributed state
+        - Provider compatibility
+        - Metrics or telemetry
+        - Retry or circuit breaking
+        - Documentation
+        - Other
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered

backstop_ai-0.6.0/.github/ISSUE_TEMPLATE/provider_compatibility.yml

@@ -0,0 +1,37 @@
+name: Provider compatibility
+description: Report a provider SDK compatibility issue
+title: "[Provider]: "
+labels: ["provider-compatibility"]
+body:
+  - type: input
+    id: provider
+    attributes:
+      label: Provider
+      placeholder: OpenAI, Anthropic
+    validations:
+      required: true
+  - type: input
+    id: sdk_version
+    attributes:
+      label: SDK version
+    validations:
+      required: true
+  - type: textarea
+    id: code
+    attributes:
+      label: Minimal code
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: behavior
+    attributes:
+      label: Behavior
+      description: What failed, and what did you expect?
+    validations:
+      required: true
+  - type: textarea
+    id: privacy
+    attributes:
+      label: Privacy note
+      description: Do not include API keys, real prompts, real responses, or customer data.

backstop_ai-0.6.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+## Summary
+
+Describe the change and why it is needed.
+
+## Tests
+
+- [ ] `pytest`
+- [ ] Real OpenAI smoke test, if OpenAI SDK behavior changed
+- [ ] Real Anthropic smoke test, if Anthropic SDK behavior changed
+- [ ] Benchmark run, if latency or throughput changed
+
+## Compatibility
+
+- Provider SDK impact:
+- Public API impact:
+- Migration notes:
+
+## Security And Privacy
+
+- [ ] Does not send prompt payloads to external services by default
+- [ ] Does not log secrets, prompts, or responses
+- [ ] Budget enforcement failure behavior is explicit

backstop_ai-0.6.0/.github/workflows/ci.yml

@@ -0,0 +1,88 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        # Pins must (a) be compatible with httpx>=0.28, which removed the
+        # `proxies` kwarg (openai<1.90 / anthropic<0.40 crash on it), and
+        # (b) stop relabelling transport exceptions: below openai 2.37 /
+        # anthropic 0.98 the SDK catches Backstop's BudgetExceededError and
+        # re-raises it as APIConnectionError, so the guardrail is invisible
+        # to user code. Floors in src/backstop/wrapper.py must match.
+        openai-version: ["2.37.0", "3.14.0"]
+        anthropic-version: ["0.99.0", "1.6.0"]
+        # Cross-check: newest openai against the oldest supported anthropic.
+        include:
+          - python-version: "3.12"
+            openai-version: "latest"
+            anthropic-version: "0.99.0"
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install -e ".[test,metrics,anthropic]"
+          if [ "${{ matrix.openai-version }}" != "latest" ]; then
+            pip install "openai==${{ matrix.openai-version }}"
+          fi
+          if [ "${{ matrix.anthropic-version }}" != "latest" ]; then
+            pip install "anthropic==${{ matrix.anthropic-version }}"
+          fi
+      - name: Run tests
+        run: pytest
+      - name: Run benchmark smoke
+        run: python -m backstop benchmark
+
+  build:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install build tools
+        run: pip install build twine
+      - name: Build package
+        run: python -m build
+      - name: Validate package
+        run: python -m twine check dist/*
+      # Supply chain hardening: cosign signing, SLSA provenance (slsa-framework), SBOM generation
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Create GitHub Release
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          gh release create "${{ github.ref_name }}" \
+            --repo "${{ github.repository }}" \
+            --notes "Backstop ${{ github.ref_name }}" \
+            dist/*
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

backstop_ai-0.6.0/.gitignore

@@ -0,0 +1,31 @@
+.venv/
+.pytest_cache/
+__pycache__/
+*.py[cod]
+
+# Secrets / environment files — NEVER commit (see .agents/AGENTS.md)
+.env
+.env.*
+*.env
+!.env.example
+.coverage
+htmlcov/
+dist/
+build/
+*.egg-info/
+.kilo/
+node_modules/
+*.tsbuildinfo
+ts/backstop/dist/
+wedge_report.md
+benchmarks/competitive/compose/
+benchmarks/competitive/*.json
+benchmarks/competitive/__pycache__/
+
+# Internal working artifacts — never publish (see PLAN.md §0 rule 9)
+artifacts/
+agent/
+.claude/
+.agents/skills/
+skills-lock.json
+docs/internal/

backstop_ai-0.6.0/CHANGELOG.md

@@ -0,0 +1,190 @@
+<div align="center">
+  <p>
+    <a href="README.md">README</a> •
+    <a href="CODE_OF_CONDUCT.md">Code of Conduct</a>
+  </p>
+</div>
+
+# Changelog
+
+All notable changes to Backstop should be documented in this file.
+
+## 0.6.0 — Unreleased
+
+- Renamed the Python distribution to `backstop-ai`; `import backstop` and the
+  `backstop` / `wedge` console commands are unchanged.
+- Added repository metadata, Python classifiers, and package keywords; aligned
+  package and installer versions at `0.6.0`.
+- Corrected install examples and removed unsupported PyPI verification claims.
+  Historical install strings below are retained as history, not current guidance;
+  see [the install guide](docs/install.md).
+- Release preparation only: PyPI publication and clean-environment installation
+  from PyPI are pending. Provider SDK compatibility and enforcement-error
+  propagation remain unresolved; SDK dependency bounds are unchanged.
+
+## Unreleased
+
+- **Custom dashboard hardened (visual audit follow-up).** Fixed the topbar
+  collision between the mode badge and wordmark at phone widths (≤640px: the
+  brand row now wraps onto its own line); fixed keyboard focus being dropped to
+  `<body>` on every table re-render (row focus is restored across polls); grew
+  the refresh control and badges to comfortable touch-target sizes under
+  `pointer: coarse`.
+- **Dashboard theme control.** `backstop dashboard --theme auto|dark|light`
+  sets the initial theme (default `auto` follows the OS), and a new **theme**
+  button in the topbar cycles auto → dark → light per browser, persisted in
+  `localStorage`.
+- **Grafana export removed.** `backstop.dashboard` (the JSON spec module),
+  `tests/test_dashboard.py`, and `observability/grafana/` are gone — the
+  built-in dashboard is the single custom ops surface. Prometheus metrics and
+  `observability/prometheus-alerts.yml` are unchanged.
+
+## v0.5.0 — 10× Better
+
+Closes the gap between Backstop and proxy gateways (LiteLLM / BricksLLM) while
+staying in-process and drop-in. All new infrastructure is opt-in; existing
+installs keep their default behavior.
+
+- **Competitive benchmark** — Firecrawl-sourced feature matrix vs LiteLLM /
+  BricksLLM and the "10× better" wedge (`docs/competitive-benchmark-2026-07-20.md`).
+- **Deep Research: 10× Better** — exhaustive 6-agent Firecrawl synthesis
+  (gateways, observability, frameworks, clouds, in-process techniques, risks)
+  with a prioritized 13-item roadmap (`docs/deep-research-10x-better-2026-07-20.md`).
+
+- **P0#1 Semantic cache (near-duplicate)** — opt-in pluggable embedder +
+  cosine `cache_similarity_threshold`. Exact match stays the zero-cost fast path;
+  on a miss, the prompt embedding is compared against cached entries and a
+  `>= threshold` match is short-circuited. Biggest single cost lever (50–80%
+  savings at typical hit rates) — `BackstopConfig(cache_enabled=True,
+  cache_semantic=True, cache_embedder=<callable>, cache_similarity_threshold=0.95)`.
+- **P0#2 Fallback chain + priority routing** — the single `fallback_model` is
+  promoted to an ordered `fallback_chain` (list of `{model, base_url?}`) walked
+  in-process on circuit-open; `fallback_chain_for_priority` gives critical traffic
+  its own chain. Also fixes a latent bug where the single-model fallback silently
+  no-op'd (`httpx.Request` has no `copy()` in this httpx version).
+- **P1 Shared (Redis) budget** — one token budget enforced across processes and
+  replicas via atomic Lua scripts. `BackstopConfig(shared_budget=True, redis_url=...)`
+  with `pip install "backstop[redis]"`.
+- **P2 OpenTelemetry export** — mirrors the Prometheus series to a vendor-neutral
+  OTel meter. `BackstopConfig(otel_enabled=True)` with `pip install "backstop[otel]"`.
+- **P3 In-process fallback** — retries once against a backup model/deployment when
+  the circuit opens, no proxy required. `BackstopConfig(fallback_model=...)`.
+- **P4 Wedge semantic diff v2** — token/line-normalized similarity (identifier
+  Dice + line ratio) that is format/whitespace/removal-immune; report now carries
+  per-runner budget-isolation evidence and a cost estimate.
+- **P5 Deterministic benchmarks** — seeded, reproducible `backstop benchmark`
+  (seed `0xC0FFEE`) with a `--publish` flag; published results in
+  `docs/benchmark-results-2026-07-20.md`.
+- **P6 Accurate pricing** — maintained 2026 price table (Claude 4/3.5/3,
+  GPT 4.1/4o/o1/o3) with prefix+family resolution and offline-cached
+  `refresh_pricing()`.
+- **P8 CLI ergonomics** — `backstop doctor` validates install/SDKs/keys and runs a
+  wrap smoke test; `backstop benchmark` produces reproducible proof.
+- **P7 Doc drift fixes** — README benchmarks, features, and "what this is NOT"
+  corrected against current behavior.
+- **P9 TypeScript SDK scaffold** — `@ravanish/backstop` mirrors `wrap()` (budget +
+  circuit breaker + retry + fallback) for Node.js agents (`ts/backstop`).
+- **P10 Concurrency ceiling** — configurable `max_wrap_sessions` soft cap with a
+  warning, plus `docs/concurrency.md` guidance on the GIL ceiling and when to use
+  a proxy gateway.
+- **Housekeeping** — deleted the superseded `docs/research` deep-research dump;
+  added `redis`/`otel` install extras.
+
+### 10× Better — roadmap follow-up (this pass)
+
+- **P1#3 Virtual keys + hierarchical budgets** — `virtual_keys` maps an API key
+  header (`virtual_key_header`) to a tenant, and `TenantBudget(parent=...)` rolls
+  spend up a team/org tree so a child budget can never exceed its parent.
+- **P1#4 True per-tenant circuit breaker** — `per_tenant_circuit` keeps a separate
+  breaker per tenant (falls back to the global one); failures from one tenant no
+  longer trip the breaker for everyone.
+- **P1#5 Cloud-quota-aware auto-tuning** — ingests provider `x-ratelimit-*` /
+  `anthropic-ratelimit-*` headers and proactively clamps the AIMD concurrency
+  limit (`apply_external_decrease`) before 429s hit, instead of reacting to them.
+- **P1#6 Framework adapters** — `backstop.adapters` lazily bridges LangChain /
+  LlamaIndex callbacks to Backstop's hooks/metrics/tenant scoping so Backstop
+  becomes the guardrail *inside* the framework. Framework imports are deferred, so
+  importing the module never requires the framework installed.
+- **P1#7 Cost forecasting + anomaly detection** — `backstop.forecast` projects
+  budget exhaustion from a measured burn rate and flags spend anomalies, turning
+  ledger data into an enforcement-triggering signal.
+- **P1#12 Pluggable rate limiter + tiktoken pre-estimation** — `rate_limiter`
+  accepts any `allow(tokens) -> bool` object (e.g. `TokenBucketLimiter`, a
+  variable-cost token bucket charged by estimated tokens); `auto_token_count`
+  (opt-in) switches pre-dispatch estimates from the chars/4 heuristic to tiktoken;
+  `compress` is a pre-send `callable(body, model) -> body` hook.
+- **P2#8 Tamper-evident audit log** — `audit_enabled` writes a chained,
+  HMAC-verifiable JSONL of every enforcement decision (`deny` / `fallback` /
+  `downgrade` / `shadow`) via `AuditLog.verify()`; the enterprise "escape hatch"
+  that makes in-process enforcement audit-ready.
+- **P2#9 Secret provider** — `secret_provider` resolves virtual keys / tenant ids
+  to provider secrets at call time (env + static ships; cloud vaults implement the
+  same interface), avoiding plaintext key handling in-process.
+- **P2#10 Gateway / sidecar mode** — `backstop gateway` (`backstop serve`) runs an
+  OpenAI-compatible reverse proxy so policy is non-bypassable for non-Python
+  services; `pip install "backstop[fastapi]"`.
+- **P2#11 Agent guardrails** — `agent_guard` (`AgentGuard`) fences runaway agent
+  loops per agent id via sliding-window call/token ceilings.
+- **P2#13 Safe rollout: shadow / canary** — `shadow_policy` (`ShadowPolicy` /
+  `CanaryRouter`) samples traffic to a candidate config/policy and records
+  reason-coded decisions before any hard cutover.
+
+### v0.5.0 — Launch Readiness (2026-08-10)
+
+Production-readiness pass: every claim is now backed by proof, every stale
+doc is corrected, and the TypeScript SDK reaches full feature parity.
+
+- **Proof infrastructure** — runnable proof scripts (`proofs/`) demonstrate
+  budget exhaustion prevention, multi-agent isolation, real-provider overhead,
+  and semantic cache savings against live providers.
+- **Wedge fixed** — patch extraction now handles markdown code blocks, `FILE:`
+  headers, and multi-line LLM output (was unified-diff-only). Agent loop with
+  retry/fix on test failure. Per-runner test timeout.
+- **Bundled test fixture** — `wedge-test-fixture/` gives `wedge run task.yaml`
+  a real repo to run against out of the box.
+- **Docs corrected** — README benchmark table regenerated dynamically (was
+  stale), benchmark overhead measured live, version strings unified at `0.5.0`
+  across `pyproject.toml`, `wedge/__init__.py`, `install.sh`, and `docs/install.md`.
+- **Gateway hardened** — auth (`--api-keys`), per-key rate limiting, 1 MB body
+  size guard, path traversal rejection, and error handling.
+- **Cost forecasting → enforcement** — `forecast_horizon_seconds` proactively
+  tightens AIMD concurrency when burn rate projects exhaustion.
+- **Secret provider wired** — virtual keys resolve to provider secrets at call
+  time in the transport, not just at wrap time.
+- **TypeScript SDK v0.5.0** — full parity: priority admission, AIMD concurrency,
+  fallback chains, response cache (exact + semantic), hooks, audit log, agent
+  guardrails, quota-aware auto-tuning, cost forecasting, and `shadow/canary`.
+- **Concurrent budget stress tests** — 20-thread hammer proves the no-overspend
+  invariant. CI matrix tests multiple SDK versions.
+
+## v0.4.0
+
+- Integrated the `wedge` tool directly into the `backstop` codebase as an executable package script (`wedge run task.yaml`).
+- **Isolation Harness**: Built per-runner Git worktree simulation and task execution.
+- **Diff Engine**: Added `difflib`-based patch similarity scoring (`CONVERGED`, `PARTIAL`, `DIVERGED`).
+- **Reporting Engine**: Added terminal summary and Markdown report generation (`wedge_report.md`).
+- Fully un-mocked and configured the Anthropic and OpenAI wrapper handlers for production-ready API integration.
+- Added Phase 0 trust assets: security policy, contribution guide, code of conduct, public docs, examples, benchmark scaffold, and GitHub templates.
+
+## v0.3.0
+
+- Added per-tenant budget buckets.
+- Added cost estimation helpers.
+- Added downgrade behavior for exhausted tenant budgets.
+
+## v0.2.0
+
+- Added latency metadata.
+- Added streaming handling.
+- Added middleware hooks.
+- Added request timeout support.
+- Added response caching.
+- Added token counting improvements.
+- Removed dead `priority_weights` behavior.
+
+## v0.1.0
+
+- Added the initial Backstop package.
+- Added OpenAI SDK wrapping.
+- Added Anthropic SDK compatibility.
+- Added budget enforcement, priority admission, AIMD concurrency, retry handling, circuit breaking, metrics, and CLI harness support.

backstop_ai-0.6.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,29 @@
+# Code Of Conduct
+
+Backstop follows the Contributor Covenant Code of Conduct, version 2.1.
+
+## Our Pledge
+
+We pledge to make participation in this project a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
+
+## Expected Behavior
+
+- Be respectful and direct.
+- Focus criticism on technical work and project outcomes.
+- Assume maintainers and contributors are balancing tradeoffs.
+- Keep security-sensitive details out of public threads.
+
+## Unacceptable Behavior
+
+- Harassment, insults, or personal attacks.
+- Publishing private information without permission.
+- Sexualized language or attention.
+- Trolling or sustained disruption.
+
+## Enforcement
+
+Maintainers may remove comments, close issues, reject contributions, or ban participants who violate this code of conduct.
+
+## Attribution
+
+This Code of Conduct is adapted from the Contributor Covenant, version 2.1.

backstop_ai-0.6.0/CONTRIBUTING.md

@@ -0,0 +1,43 @@
+# Contributing
+
+Thanks for helping improve Backstop. This project is optimized for practical LLM spend control: budgets, backpressure, retries, circuit breaking, fallbacks, metrics, and provider SDK compatibility.
+
+## Local Setup
+
+```bash
+python3 -m venv .venv
+. .venv/bin/activate
+pip install -e ".[test,metrics,anthropic]"
+pytest
+```
+
+## Development Guidelines
+
+- Keep the local SDK path useful without a hosted service.
+- Do not send prompt payloads to external services by default.
+- Prefer provider SDK compatibility over custom HTTP API abstractions.
+- Add tests for budget, retry, circuit, streaming, metrics, or wrapper behavior when changing those paths.
+- Keep public APIs typed and documented.
+- Avoid broad refactors in feature PRs.
+
+## Pull Request Checklist
+
+- Tests pass with `pytest`.
+- New behavior has tests or a clear reason tests are not practical.
+- Docs or examples are updated when public behavior changes.
+- Security and privacy implications are called out.
+- Provider SDK compatibility impact is described.
+
+## Real Provider Tests
+
+Real API tests are opt-in and require credentials:
+
+```bash
+export OPENAI_API_KEY="..."
+pytest -m real_openai
+
+export ANTHROPIC_API_KEY="..."
+pytest -m real_anthropic
+```
+
+Do not include real API keys, prompts, responses, or customer data in issues, tests, fixtures, screenshots, or logs.

backstop_ai-0.6.0/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Backstop Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.