tokenjam-bench 0.1.0__tar.gz

tokenjam_bench-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,25 @@
+---
+name: Bug report
+about: Something is wrong, broken, or a number looks off
+title: ""
+labels: bug
+assignees: ""
+---
+
+**What happened**
+A clear description of the bug.
+
+**Expected**
+What you expected instead.
+
+**Reproduce**
+Steps or the exact command, e.g. `tjb run --benchmark humaneval --limit 50`.
+
+**Evidence (for a wrong/surprising number)**
+The artifact filename under `docs/evidence/` or `results/`, or paste the
+headline line, so it's reproducible.
+
+**Environment**
+- OS:
+- Python (`python --version`): 3.10 / 3.11 / 3.12
+- `tjb version` (tokenjam build):

tokenjam_bench-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Propose a new benchmark, provider, or capability
+title: ""
+labels: enhancement
+assignees: ""
+---
+
+**Problem**
+What can't you measure or prove today?
+
+**Proposal**
+What you'd like added — a benchmark, model client, statistic, or dashboard view.
+
+**Honesty check**
+How would the result stay measured and hedged (real ground truth, CI + p-value,
+no placeholder pricing, no overclaim)? See CONTRIBUTING.md.
+
+**Alternatives**
+Anything you considered or worked around.

tokenjam_bench-0.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,21 @@
+<!-- Thanks for contributing to TokenJam Bench. -->
+
+## What & why
+
+<!-- What does this change do, and why? Link any related issue. -->
+
+## Checklist
+
+- [ ] `ruff check .` passes
+- [ ] `pytest` passes locally (includes the honesty guard)
+- [ ] Docs updated if behavior or CLI changed
+- [ ] No placeholder-priced run is surfaced as headline/dashboard evidence
+      (`priced_with_defaults=true` stays under `docs/evidence/archive/`)
+- [ ] No banned overclaim strings added (use Wilson CI + McNemar p + the hedged
+      verdicts, never "quality preserved" / a single `confidence = NN%` scalar /
+      ROI extrapolation)
+
+## Evidence (if numbers changed)
+
+<!-- Paste the relevant artifact filename(s) under docs/evidence/ or results/,
+     or the headline line, so reviewers can reproduce. -->

tokenjam_bench-0.1.0/.github/workflows/benchmark.yml

@@ -0,0 +1,56 @@
+name: benchmark
+
+# Continuous benchmarking against the LATEST TokenJam release.
+# - nightly cron picks up new TokenJam releases automatically (pip install -U)
+# - manual dispatch for on-demand runs
+on:
+  schedule:
+    - cron: "0 6 * * *"        # 06:00 UTC nightly
+  workflow_dispatch:
+    inputs:
+      live:
+        description: "Run LIVE benchmarks (needs the DEEPSEEK_API_KEY secret)"
+        type: boolean
+        default: true
+      limit:
+        description: "Task limit for the live HumanEval run"
+        type: string
+        default: "10"
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    env:
+      # Key-gated: live proofs only run if this secret is configured on the repo.
+      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+      TJBENCH_JUDGE: deepseek
+      TJBENCH_JUDGE_METRIC: correctness
+      HF_HUB_DISABLE_PROGRESS_BARS: "1"
+      DEEPEVAL_TELEMETRY_OPT_OUT: "YES"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install latest TokenJam + bench extras
+        run: |
+          pip install -U tokenjam
+          pip install -e ".[dev,providers,datasets,judge]"
+
+      - name: Show the TokenJam version under test
+        run: python -c "import importlib.metadata as m; print('tokenjam', m.version('tokenjam'))"
+
+      - name: Run benchmarks (offline always; live if the key secret is set)
+        run: python -m tjbench.ci_benchmark
+
+      - name: Upload version-stamped artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: results/
+          if-no-files-found: warn

tokenjam_bench-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: ci
+
+# Always-on gate: lint + tests + an offline proof, across supported Pythons.
+# No keys, no spend. (Branch-protection / required-review rules are a GitHub-UI
+# setting configured at the public flip — not expressible here.)
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    name: test (py${{ matrix.python-version }})
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install (consumes tokenjam exactly like an external user)
+        run: pip install -e ".[dev]"
+      - name: Lint
+        run: ruff check .
+      # `pytest -q` runs the full offline suite, which INCLUDES the Brief C
+      # honesty guard (tests/test_honesty_guard.py): no placeholder-priced
+      # headlines, no banned overclaim strings in README/docs/dashboard.
+      - name: Tests (all offline / mock, incl. honesty guard)
+        run: pytest -q
+      - name: Offline proof smoke (pipeline produces a valid stamped proof)
+        run: python -m tjbench.ci_benchmark

tokenjam_bench-0.1.0/.github/workflows/publish-pypi.yml

@@ -0,0 +1,34 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+# Trusted Publishing (OIDC) — no API token. The PyPI project must have a
+# matching publisher configured: owner Metabuilder-Labs, repo tokenjam-bench,
+# workflow publish-pypi.yml, environment pypi.
+jobs:
+  publish-pypi:
+    if: startsWith(github.ref_name, 'v')
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment: pypi
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build + dev deps
+        run: pip install build ".[dev]"
+
+      - name: Run tests (offline)
+        run: pytest -q
+
+      - name: Build sdist + wheel
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

tokenjam_bench-0.1.0/.gitignore

@@ -0,0 +1,32 @@
+__pycache__/
+*.pyc
+.pytest_cache/
+.ruff_cache/
+*.egg-info/
+.venv/
+# Proof artifacts are generated; keep the dir, not the runs.
+results/*.json
+results/*.html
+results/*.duckdb
+results/*.duckdb.wal
+!results/.gitkeep
+
+# `tjb serve` writes a local history index next to whatever dir it serves
+# (e.g. docs/evidence/…); never commit that byproduct.
+history.duckdb
+history.duckdb.wal
+
+# DeepEval local cache/config (created on a live judge run)
+.deepeval/
+.deepeval-cache.json
+
+# Secrets — never commit. Live provider keys land here for real benchmark runs.
+.env
+.env.local
+.env.*.local
+
+# Local agent scratch (briefs, notes) — not part of the published repo.
+.claude/
+
+# macOS
+.DS_Store

tokenjam_bench-0.1.0/CHANGELOG.md

@@ -0,0 +1,81 @@
+# Changelog
+
+## 0.1.0 (2025-06-24)
+
+### Added
+
+#### Agent Evaluation Framework
+- **AgentRunner** — Multi-turn agent execution loop with max-turns guard
+- **ToolRegistry** — Register and execute tools with JSON-schema advertisement
+- **AgentTrace** — Observable record of every turn, tool call, and result
+- **Safety Gate** — `validate_tools()` catches forbidden tools even with correct answers
+- **ToolValidation** — Reports expected tools, ordering, safety, and error rate
+
+#### Agent Benchmarks
+- **sample-agent** — 3 offline tool-use tasks with safety gate validation
+- **swe-bench-lite** — experimental scaffold (dataset loader + developer tools); fix-verification is NOT implemented, so it does not produce a pass-rate
+
+#### SWE-Bench Tools
+- **view** — Read file contents with line numbers
+- **view_range** — Read specific line range
+- **str_replace** — Exact-match string replacement (must match exactly once)
+- **create** — Create new file
+- **insert** — Insert text after specific line
+- **bash** — Run shell commands with timeout
+- **Path traversal blocking** — Prevents escaping workspace
+- **Exact-match enforcement** — Prevents accidental mass-replace
+
+#### Model Clients
+- **AnthropicAgentClient** — Live tool-calling client for Anthropic
+- **MockAgentClient** — Deterministic offline tool-calling client
+- **ToolCallingClient** protocol — Multi-turn chat with tool use
+
+#### Pipelines
+- **Agent Proof Pipeline** — `run_agent_proof()` for multi-turn agent evaluation
+- **Token summation** — Aggregates token usage across all turns for pricing
+- **Tool validation scoring** — Safety gate + ordering + expected tools
+
+#### Documentation (14 files)
+- docs/README.md — Master documentation index
+- docs/overview.md — Project overview and design principles
+- docs/architecture.md — System design, data flow, module relationships
+- docs/quickstart.md — 5-minute quickstart guide
+- docs/cli-reference.md — Complete `tjbench` command reference
+- docs/pipelines.md — Single-shot and agent proof pipeline deep dive
+- docs/models.md — Model client adapters and protocols
+- docs/benchmarks.md — Available benchmarks and scoring
+- docs/agents.md — Multi-turn agent execution framework
+- docs/statistics.md — Statistical methods used for proof
+- docs/cost-pricing.md — How costs are computed
+- docs/tokenjam-integration.md — How we consume TokenJam
+- docs/development.md — Contributing, testing, extending
+- docs/api-reference.md — Module-level API documentation
+- docs/swe-bench-lite.md — SWE-Bench Lite integration guide
+- docs/tests.md — Complete test suite inventory
+
+#### Tests
+- 55 total tests (20 new for SWE-Bench Lite)
+- Mock scoring tests for SWE-Bench Lite
+- Tool operation tests (view, replace, create, insert, bash)
+- Path traversal safety tests
+- Patch parsing tests
+
+### Design Principles
+
+- **Black-box consumer** of TokenJam — imports as pip dependency, never vendored
+- **Offline-first** — All tests run without API keys using mock clients
+- **Objective ground truth** — Code execution and exact-match scoring, not LLM-as-judge
+- **Statistical honesty** — Wilson CIs, McNemar exact tests, never claim significance on small samples
+- **Safety-first** — Agent benchmarks include safety gate for dangerous tool calls
+
+### Integration Points
+
+| Feature | TokenJam API | Module |
+|---------|-------------|--------|
+| Candidate recommendation | `tokenjam.core.optimize.DOWNGRADE_CANDIDATES` | `recommend.py` |
+| Cost pricing | `tokenjam.core.pricing.get_rates` | `cost.py` |
+| Version stamp | `importlib.metadata.version("tokenjam")` | `version.py` |
+
+---
+
+*This changelog documents the initial release of tokenjam-bench as an agent evaluation framework.*

tokenjam_bench-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,63 @@
+# Contributing to TokenJam Bench
+
+Thanks for helping. TokenJam Bench is honesty-branded: its credibility is its
+evidence. The bar for a change is that every number it shows traces to a real,
+reproducible measurement.
+
+## Setup
+
+```bash
+pip install -e ".[dev]"
+pytest          # full offline suite — no keys, no spend
+ruff check .    # lint
+```
+
+Supported Python: 3.10, 3.11, 3.12. CI runs the suite on all three.
+
+Run the app to see your change:
+
+```bash
+tjb run         # zero-flag offline proof
+tjb serve       # dashboard over the bundled real evidence
+```
+
+## The honesty rules (enforced in CI)
+
+`tests/test_honesty_guard.py` fails CI if a change reintroduces a dishonest
+surface. Before you open a PR:
+
+- No headline/dashboard number may come from a placeholder-priced run
+  (`priced_with_defaults=true`). Re-run with real rates, or keep legacy runs
+  under `docs/evidence/archive/` (non-headline).
+- No banned overclaim strings in README, docs, or the dashboard: no
+  "quality preserved", no "safe to replace", no single `confidence = NN%`
+  scalar, no ROI extrapolation ("at 10x", "annual savings"). The honest forms —
+  Wilson CI, McNemar p-value, and the three hedged verdicts
+  (`no_significant_regression` / `significant_regression` /
+  `insufficient_evidence`) — are what to use instead.
+- Accuracy is the pass-rate on a named suite. It is never a general quality
+  claim.
+
+## How it's built
+
+- **Offline-first.** Tests, lint, and the default `tjb run` work with no
+  provider keys and no network. Live providers are opt-in via a key in the env.
+- **Flat layout.** Top-level modules and subpackages live under `tjbench/`.
+- **TokenJam is a published dependency**, consumed like any external user — never
+  vendored. Every artifact is stamped with the exact `tokenjam_version`.
+
+See [docs/development.md](docs/development.md) for adding a benchmark or model
+client, and [docs/architecture.md](docs/architecture.md) for the data flow.
+
+## Pull requests
+
+1. Branch off `main`.
+2. Keep the change focused; update docs when behavior changes.
+3. Make sure `ruff check .` and `pytest` pass locally (the honesty guard runs
+   inside `pytest`).
+4. Fill out the PR template. CI must be green before review.
+
+## Reporting issues
+
+Use the issue templates. For a wrong or surprising number, include the artifact
+JSON (or its filename under `docs/evidence/` or `results/`) so it's reproducible.

tokenjam_bench-0.1.0/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Metabuilder Labs
+Copyright (c) 2026 Hooman Digital
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tokenjam_bench-0.1.0/Makefile

@@ -0,0 +1,37 @@
+.PHONY: install update-tokenjam test lint bench-smoke version serve dashboard
+
+# Use the python where the deps (tokenjam, click, rich) are installed.
+PY ?= python3
+
+# Install the bench (editable) + dev tooling.
+install:
+	pip install -e ".[dev]"
+
+# THE daily-pull command: upgrade to the latest published TokenJam and show the
+# version every subsequent proof will be stamped with.
+update-tokenjam:
+	pip install -U tokenjam
+	@$(PY) -c "import importlib.metadata as m; print('tokenjam now at', m.version('tokenjam'))"
+
+# All commands go through run.py so they work without relying on the installed
+# console script (flat layout → `cli` collides with the `cli` PyPI package).
+version:
+	$(PY) run.py version
+
+# Live proof dashboard (offline, auto-refreshing) at http://127.0.0.1:7392/
+serve dashboard:
+	$(PY) run.py serve --open
+
+# Offline end-to-end smoke (no keys, no spend).
+bench-smoke:
+	$(PY) run.py run --benchmark samples --original anthropic:claude-opus-4-7 --mock
+
+test:
+	pytest -q
+
+lint:
+	ruff check .
+
+# Continuous-benchmark set: offline always; live if a provider key is exported.
+ci-bench:
+	$(PY) -m tjbench.ci_benchmark

tokenjam_bench-0.1.0/NOTICE

@@ -0,0 +1,7 @@
+tokenjam-bench
+Copyright (c) 2026 Metabuilder Labs
+
+This project was originally created by Piyush (Hooman Digital) and is maintained
+by Metabuilder Labs as the official benchmark for TokenJam
+(https://github.com/Metabuilder-Labs/tokenjam). Licensed under the MIT License;
+the original Hooman Digital copyright is retained in LICENSE.