pasr-bench 0.2.0__tar.gz

pasr_bench-0.2.0/.gitignore

@@ -0,0 +1,226 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# PASR local artifacts — receipts are ignored, Context Packs are committable
+.pasr/*
+!.pasr/packs/
+eval/runs/
+eval/pasr_eval/runs/
+pasr-bench-runs/
+.eval-checkouts/

pasr_bench-0.2.0/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.5
+Name: pasr-bench
+Version: 0.2.0
+Summary: PASR-Bench: a pre-registered protocol for measuring a code-context retriever against grep, a repo-map, semantic search, and a whole-repo dump.
+Project-URL: Homepage, https://github.com/Apheironn/pasr
+Project-URL: Source, https://github.com/Apheironn/pasr/tree/main/eval
+Author: Apheironn
+License-Expression: Apache-2.0
+Keywords: benchmark,coding-agent,context,evaluation,non-inferiority,retrieval
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: pasr-mcp>=0.2
+Provides-Extra: dev
+Requires-Dist: pytest<9,>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: llm
+Requires-Dist: anthropic>=0.40; extra == 'llm'
+Provides-Extra: plots
+Requires-Dist: matplotlib>=3.7; extra == 'plots'
+Description-Content-Type: text/markdown
+
+# PASR-Bench
+
+A small, **pre-registered** protocol for measuring a code-context retriever, and the
+harness that runs it. Packaged separately from `pasr-mcp` (`pip install
+./eval`, distribution name `pasr-bench`). Results for PASR itself:
+[`RESULTS.md`](RESULTS.md) and [`../docs/competitors-benchmark.md`](../docs/competitors-benchmark.md).
+
+## What it measures
+
+**Real-agent run** — four arms per task, each producing the same `ArmResult`:
+
+1. **native_search** — deterministic grep + read-files (stand-in for an agent's own tools)
+2. **broad** — the whole repo, source-first, truncated to a cap
+3. **pasr** — `select_context` at a fixed budget
+4. **pasr_fallback** — `pasr`, then one budget widening if confidence is low
+
+A model answers each task from **only** that arm's context (`UNKNOWN` if absent); a
+second model judges the answer against the task's expected identifiers;
+`critical_source_hit` is still required. Metrics: task success, model input tokens
+(incl. a flat per-tool-call overhead), round trips, critical-source miss rate, fallback
+rate. Then a **paired non-inferiority** test of `pasr` / `pasr_fallback` vs the
+`baseline_arm` at the plan's `margin_task_success` (point estimate + 10k-resample
+bootstrap CI).
+
+**Bake-off** — an offline, no-API retrieval comparison at a shared budget: `grep`,
+`repomap` (aider-style signatures), `embed_lex` (a no-setup semantic floor), `pasr`,
+`pasr_hash`, `pasr_map`. Scored on critical-file hit ∧ keyword coverage, split by task
+kind.
+
+**No GPU.** Arms run on CPU; the only model use is one answer + one judge call per
+(task, arm) — ~400 Anthropic calls for the 50-task plan. Full Sonnet ≈ $12–15; a cheap
+`--model` with a strong `--judge-model` ≈ $4–6.
+
+## Install & run
+
+```bash
+pip install ./eval               # the pasr-bench distribution (deps: pasr-mcp)
+pip install "./eval[llm,plots]"  # + anthropic (real agent) + matplotlib (report.png)
+
+pasr-bench plans                             # the packaged plan(s)
+pasr-bench run --agent keyword               # offline smoke: full matrix + report + validation
+pasr-bench bakeoff --budget 6000             # offline retrieval bake-off
+
+export ANTHROPIC_API_KEY=sk-ant-...
+pasr-bench run --agent claude --max-tasks 4  # cheap trial (~$0.4)
+pasr-bench run --agent claude \
+  --model claude-haiku-4-5 --judge-model claude-sonnet-5 \
+  --checkout-dir .eval-checkouts             # budget-safe full run (~$4–6)
+```
+
+Each finished `(task, arm)` row is appended to `matrix.jsonl` and flushed, so a crash
+keeps every completed row; `--resume <run_dir>` reloads the partial matrix and finishes
+into the same delivery. `--checkout-dir DIR` reuses clones. `PASR_EVAL_BROAD_CAP=30000`
+shrinks the `broad` arm. A run writes `<out>/<plan>_<utc>/` with `matrix.jsonl`,
+`report.{json,md,png}`, `validation.json`, `resolved_commits.json`.
+
+`python eval/run_eval.py …` and `python eval/bakeoff.py …` still work as thin shims for
+the two sub-commands.
+
+## Bring your own retriever
+
+The protocol is retriever-agnostic. To measure a different context tool against the
+same 50 tasks and the same baselines:
+
+1. Add an arm in [`pasr_eval/arms.py`](pasr_eval/arms.py): extend `ARMS` and add a
+   branch in `_build(...)` that returns `(context, sources, tool_calls, round_trips,
+   fallback)` for your retriever. Everything downstream — grading, metrics,
+   non-inferiority, the validator — is arm-agnostic.
+2. For a bake-off arm, add a function in [`pasr_eval/bakeoff.py`](pasr_eval/bakeoff.py)
+   and list it in that file's `ARMS`.
+3. Keep the plan (`pasr_eval/plans/pilot.json`) fixed so numbers stay comparable, or
+   register a new plan and cite it.
+
+The validator (`validate_matrix`) rejects synthetic rows, query→answer leaks, and an
+unmatched task×arm matrix, so a submitted result is checkable.
+
+## Files
+
+| Path | What |
+|---|---|
+| `pasr_eval/spec.py` | `RepoSpec` / `TaskSpec` / `EvalPlan`, `load_plan`; the leak + kind guards |
+| `pasr_eval/arms.py` | the four real-agent arms → `ArmResult` |
+| `pasr_eval/bakeoff.py` | the six offline bake-off arms |
+| `pasr_eval/agents.py` | `AgentRunner` protocol, `KeywordAgent` (offline proxy) |
+| `pasr_eval/llm_agent.py` | `LlmAgent` — answer + judge via the Anthropic API (`[llm]` extra) |
+| `pasr_eval/metrics.py` | grade, aggregate, paired bootstrap CI, non-inferiority, `full_report` |
+| `pasr_eval/runner.py` | `resolve_repos`, `run_plan` (`skip=` / `on_row=`), `write_matrix` |
+| `pasr_eval/validate.py` | `validate_matrix` — no synthetic rows, no leaks, matched matrix |
+| `pasr_eval/run.py` | end-to-end orchestrator: clone → streamed matrix → report → validate |
+| `pasr_eval/plans/pilot.json` | the registered plan — 10 pinned repos, 50 tasks |
+| `RESULTS.md` | pre-registration + n=15 pilot + keyword-50 + the n=50 real-agent headline |

pasr_bench-0.2.0/README.md

@@ -0,0 +1,91 @@
+# PASR-Bench
+
+A small, **pre-registered** protocol for measuring a code-context retriever, and the
+harness that runs it. Packaged separately from `pasr-mcp` (`pip install
+./eval`, distribution name `pasr-bench`). Results for PASR itself:
+[`RESULTS.md`](RESULTS.md) and [`../docs/competitors-benchmark.md`](../docs/competitors-benchmark.md).
+
+## What it measures
+
+**Real-agent run** — four arms per task, each producing the same `ArmResult`:
+
+1. **native_search** — deterministic grep + read-files (stand-in for an agent's own tools)
+2. **broad** — the whole repo, source-first, truncated to a cap
+3. **pasr** — `select_context` at a fixed budget
+4. **pasr_fallback** — `pasr`, then one budget widening if confidence is low
+
+A model answers each task from **only** that arm's context (`UNKNOWN` if absent); a
+second model judges the answer against the task's expected identifiers;
+`critical_source_hit` is still required. Metrics: task success, model input tokens
+(incl. a flat per-tool-call overhead), round trips, critical-source miss rate, fallback
+rate. Then a **paired non-inferiority** test of `pasr` / `pasr_fallback` vs the
+`baseline_arm` at the plan's `margin_task_success` (point estimate + 10k-resample
+bootstrap CI).
+
+**Bake-off** — an offline, no-API retrieval comparison at a shared budget: `grep`,
+`repomap` (aider-style signatures), `embed_lex` (a no-setup semantic floor), `pasr`,
+`pasr_hash`, `pasr_map`. Scored on critical-file hit ∧ keyword coverage, split by task
+kind.
+
+**No GPU.** Arms run on CPU; the only model use is one answer + one judge call per
+(task, arm) — ~400 Anthropic calls for the 50-task plan. Full Sonnet ≈ $12–15; a cheap
+`--model` with a strong `--judge-model` ≈ $4–6.
+
+## Install & run
+
+```bash
+pip install ./eval               # the pasr-bench distribution (deps: pasr-mcp)
+pip install "./eval[llm,plots]"  # + anthropic (real agent) + matplotlib (report.png)
+
+pasr-bench plans                             # the packaged plan(s)
+pasr-bench run --agent keyword               # offline smoke: full matrix + report + validation
+pasr-bench bakeoff --budget 6000             # offline retrieval bake-off
+
+export ANTHROPIC_API_KEY=sk-ant-...
+pasr-bench run --agent claude --max-tasks 4  # cheap trial (~$0.4)
+pasr-bench run --agent claude \
+  --model claude-haiku-4-5 --judge-model claude-sonnet-5 \
+  --checkout-dir .eval-checkouts             # budget-safe full run (~$4–6)
+```
+
+Each finished `(task, arm)` row is appended to `matrix.jsonl` and flushed, so a crash
+keeps every completed row; `--resume <run_dir>` reloads the partial matrix and finishes
+into the same delivery. `--checkout-dir DIR` reuses clones. `PASR_EVAL_BROAD_CAP=30000`
+shrinks the `broad` arm. A run writes `<out>/<plan>_<utc>/` with `matrix.jsonl`,
+`report.{json,md,png}`, `validation.json`, `resolved_commits.json`.
+
+`python eval/run_eval.py …` and `python eval/bakeoff.py …` still work as thin shims for
+the two sub-commands.
+
+## Bring your own retriever
+
+The protocol is retriever-agnostic. To measure a different context tool against the
+same 50 tasks and the same baselines:
+
+1. Add an arm in [`pasr_eval/arms.py`](pasr_eval/arms.py): extend `ARMS` and add a
+   branch in `_build(...)` that returns `(context, sources, tool_calls, round_trips,
+   fallback)` for your retriever. Everything downstream — grading, metrics,
+   non-inferiority, the validator — is arm-agnostic.
+2. For a bake-off arm, add a function in [`pasr_eval/bakeoff.py`](pasr_eval/bakeoff.py)
+   and list it in that file's `ARMS`.
+3. Keep the plan (`pasr_eval/plans/pilot.json`) fixed so numbers stay comparable, or
+   register a new plan and cite it.
+
+The validator (`validate_matrix`) rejects synthetic rows, query→answer leaks, and an
+unmatched task×arm matrix, so a submitted result is checkable.
+
+## Files
+
+| Path | What |
+|---|---|
+| `pasr_eval/spec.py` | `RepoSpec` / `TaskSpec` / `EvalPlan`, `load_plan`; the leak + kind guards |
+| `pasr_eval/arms.py` | the four real-agent arms → `ArmResult` |
+| `pasr_eval/bakeoff.py` | the six offline bake-off arms |
+| `pasr_eval/agents.py` | `AgentRunner` protocol, `KeywordAgent` (offline proxy) |
+| `pasr_eval/llm_agent.py` | `LlmAgent` — answer + judge via the Anthropic API (`[llm]` extra) |
+| `pasr_eval/metrics.py` | grade, aggregate, paired bootstrap CI, non-inferiority, `full_report` |
+| `pasr_eval/runner.py` | `resolve_repos`, `run_plan` (`skip=` / `on_row=`), `write_matrix` |
+| `pasr_eval/validate.py` | `validate_matrix` — no synthetic rows, no leaks, matched matrix |
+| `pasr_eval/run.py` | end-to-end orchestrator: clone → streamed matrix → report → validate |
+| `pasr_eval/plans/pilot.json` | the registered plan — 10 pinned repos, 50 tasks |
+| `RESULTS.md` | pre-registration + n=15 pilot + keyword-50 + the n=50 real-agent headline |

pasr_bench-0.2.0/RESULTS.md

@@ -0,0 +1,151 @@
+# PASR real-agent evaluation — results
+
+> **50-task real-agent run done** (`eval/deliveries/realagent50_20260902T221528Z/`,
+> 10 pinned repos, Claude answer + judge). PASR gives the model **5.8k targeted
+> tokens in one tool call** and it answers **48%** of the tasks; the **59k-token
+> source-first repo dump answers 38%** — PASR is **+0.10** on paired task success
+> (`pasr_fallback` +0.12), and the **point estimate clears the -0.05
+> non-inferiority margin** (95% CI still crosses it: `pasr` [-0.08, +0.30]).
+> PASR puts the critical file in context **46/50** times vs the dump's **33/50**.
+>
+> Reading: on localized code questions PASR matches — slightly beats — a 10×-larger
+> whole-repo dump, at **~90% fewer input tokens and one round trip**. `native_search`
+> (grep + read 6 files, 23k tokens, 6 round trips) is the raw-success leader at 0.52
+> but misses the critical file 30% of the time. This is a **bounded efficiency
+> result**, not a superiority claim — one more batch of 50 would settle the interval.
+
+## Pre-registration
+
+- **Plan:** `pasr_eval/plans/pilot.json` — 10 pinned public Python repos, 50 source-grounded
+  `locate` / `trace` / `explain` tasks (5/repo). `pasr-bench run` records each repo's
+  resolved commit SHA and the answer/judge model IDs into `matrix.jsonl`'s `_meta` line.
+- **Arms:** `native_search`, `broad`, `pasr`, `pasr_fallback`.
+- **Baseline:** `broad`.
+- **Primary claim (non-inferiority):** on paired tasks, `pasr_fallback` task success
+  is non-inferior to `broad` within a **-0.05** margin (both the point estimate and
+  the lower bound of a 10k-resample paired bootstrap 95% CI ≥ margin).
+- **Secondary:** `pasr` / `pasr_fallback` reduce mean `tokens_in` and `round_trips`
+  vs `broad`; `critical_source_miss_rate` reported per arm; `fallback_rate` reported.
+- **No synthetic rows, no reference leak:** enforced by `validate_matrix` (every row
+  traces to a registered task + repo; the query never contains the full answer;
+  exactly one row per task×arm).
+
+## The two graders
+
+- **Keyword proxy** (`--agent keyword`, offline, no API): task success = every expected
+  identifier appears verbatim in the arm's context **and** the critical file is present.
+  A harsh literal proxy for retrieval quality — machinery + token-story check only.
+- **Real agent** (`--agent claude`): a model answers from **only** the arm's context
+  (`UNKNOWN` if not present), then a second model judges the answer against the task's
+  expected identifiers; `critical_source_hit` is still required for a pass.
+
+The n=15 pilot and the 50-task keyword re-run below are the supporting runs; the
+[50-task real-agent run](#50-task-real-agent-run--claude-answer--judge-10-repos---headline)
+is the headline.
+
+## Pilot run — `claude-sonnet-5`, 15 tasks, 5 repos
+
+| arm | task success | context tokens | round trips | crit. miss | fallback rate |
+|---|---:|---:|---:|---:|---:|
+| broad (58k source-first repo dump) | 0.467 | 58 146 | 1 | 0.067 | – |
+| native_search (grep + 6 files, 4k each) | 0.400 | 22 771 | 6 | 0.267 | – |
+| **pasr** | **0.533** | **5 764** | **1** | 0.133 | – |
+| pasr_fallback | 0.600 | 5 764 | 1 | 0.133 | 0.000 |
+
+- **Token savings vs `broad`: pasr / pasr_fallback +90.0%**; native_search +60.4%.
+- **Non-inferiority (`pasr_fallback` vs `broad`, task success, margin -0.05):**
+  delta **+0.133**, 95% CI **[-0.20, +0.47]** — **point estimate PASSES**, the CI
+  lower bound (-0.20) does **not** clear the margin. `pasr` vs `broad`: delta +0.067,
+  CI [-0.33, +0.47], same picture.
+- `pasr` critical-source miss: **2/15** (`req-02`, `star-03`; `star-03` was also
+  missed by `broad` → likely a task-spec problem, not retrieval).
+- The `pasr_fallback` threshold (`confidence < 0.5`) **never triggered** — all 15
+  PASR selections were confident. `pasr_fallback` ran on the same contexts as `pasr`;
+  its +0.067 over `pasr` is answer/judge sampling noise, not a fallback effect.
+
+### Reading it
+
+- `broad` puts the critical file in context 14/15 times (58k tokens) yet the model
+  answers only 7/15 — a large-haystack utilisation effect. `pasr` gives the model
+  5.8k targeted tokens and it answers 8/15. **Same answer quality band, one order of
+  magnitude fewer tokens, one tool call instead of the model chewing 58k.**
+- `native_search` is worst: brittle grep (27% critical miss) plus six partial files.
+- **What this does not show:** inferential non-inferiority (n=15 CI is ±0.3). The
+  bounded positive claim is a cross-repo efficiency direction, mirroring `researchv2`'s
+  LongBench Pro finding.
+
+Delivery: `eval/deliveries/pilot_20260902T212108Z/`.
+
+## 50-task keyword-proxy re-run (no API — `pasr-bench run --agent keyword`)
+
+`eval/deliveries/keyword50_20260902T213744Z/`. 10 repos, 50 source-grounded tasks;
+`_FALLBACK_CONFIDENCE` raised to 0.65.
+
+| arm | task success | context tokens | round trips | crit. miss | fallback rate |
+|---|---:|---:|---:|---:|---:|
+| broad (58k source-first) | 0.660 | 59 075 | 1 | **0.340** | – |
+| native_search (6 files, 4k) | 0.640 | 22 901 | 6 | 0.300 | – |
+| **pasr** | **0.700** | **5 777** | 1 | **0.080** | – |
+| pasr_fallback | 0.700 | 5 854 | 1.02 | 0.060 | 0.020 |
+
+- **PASR beats both baselines on the literal grader** (0.70 vs 0.66 / 0.64) at **+90%
+  tokens**, and its critical-source miss (0.08 = 4/50) is **4× lower than broad's
+  0.34** — a 60k source-first dump still fails to include the right file for 17/50
+  real-repo tasks; PASR's targeted retrieval gets it in 92% of the time.
+- Non-inferiority (`pasr` vs `broad`, keyword grader): delta **+0.04**, 95% CI
+  **[-0.14, +0.22]** — point PASSES, CI is now ±0.18 (was ±0.30 at n=15) and just
+  misses clearing the -0.05 margin.
+- Fallback engaged on 1/50 (crit-miss 0.08 → 0.06).
+
+## 50-task real-agent run — Claude answer + judge, 10 repos  ★ headline
+
+`eval/deliveries/realagent50_20260902T221528Z/`. Models recorded in `matrix.jsonl`'s
+`_meta`. `validate_matrix` clean (no synthetic rows, no leak, matched 4×50 matrix).
+
+| arm | task success | context tokens | tokens_in | round trips | crit. miss | fallback |
+|---|---:|---:|---:|---:|---:|---:|
+| broad (59k source-first repo dump) | 0.380 | 59 075 | 59 115 | 1 | **0.340** | – |
+| native_search (grep + 6 files, 4k each) | **0.520** | 22 901 | 23 181 | 6 | 0.300 | – |
+| **pasr** | 0.480 | **5 777** | **5 817** | **1** | **0.080** | – |
+| pasr_fallback | 0.500 | 5 854 | 5 895 | 1.02 | 0.060 | 0.020 |
+
+- **Token savings vs `broad`: pasr +90.2%, pasr_fallback +90.0%**; native_search +60.8%.
+- **Non-inferiority (task success, margin -0.05):** `pasr` delta **+0.100**, 95% CI
+  **[-0.08, +0.30]**; `pasr_fallback` delta **+0.120**, CI **[-0.06, +0.30]**. Both
+  **point estimates PASS**; both CI lower bounds miss the margin by ≈0.03 (half-width
+  ±0.18–0.19 at n=50, down from ±0.30 at n=15).
+- **Critical-source hit: `pasr` 46/50 (92%), `pasr_fallback` 47/50** vs **`broad`
+  33/50 (66%)**. A 59k source-first dump *still* omits the answer's file for 17/50
+  real-repo tasks — concentrated in the large repos (`typer` 5/5 missed, `jinja` 4/5,
+  `packaging` 4/5, `anyio` 3/5); those are exactly the repos where `broad` scores
+  0–1/5. Where the file *does* fit (`requests`, `pluggy`, `httpx`) `broad` reaches
+  4/5. **`broad`'s 0.38 is a truncation + large-haystack failure, not a grading one.**
+- Head-to-head: `pasr` wins **15** tasks `broad` loses, loses **10** `broad` wins
+  (net +5 of 50 = the +0.10). `pasr_fallback` widened once (`pkg`-family), turning one
+  loss into a win.
+- **PASR's weak spot — `typer` (0/5, both PASR arms).** The slice reaches the right
+  file 3/5 but the answering lines aren't in the selected window: `typer` leans on
+  re-exports and decorator plumbing that the current chunker + 6k budget don't
+  resolve. Drop `typer` and `pasr` is 24/45 = **0.53**. Logged for a chunker follow-up.
+- `native_search` is the raw-success leader (0.52) but pays **4× the tokens, 6 round
+  trips**, and a **30% critical-source miss** — brittle when the query terms don't
+  literally appear near the answer.
+
+### Reading it
+
+The claim PASR makes is a **profile trade**, and the run supports it: at parity-ish
+answer quality with a whole-repo dump (−0 to +0.12 depending on arm/margin), PASR
+costs **one order of magnitude fewer input tokens, one tool call instead of the model
+chewing 59k, and a 92% critical-file hit rate with per-line provenance**. It is *not*
+an inferential non-inferiority pass (CI lower bound −0.08) and *not* a raw-accuracy
+win over an agent's own grep. It mirrors `researchv2`'s LongBench Pro finding: a
+bounded cross-repo efficiency direction.
+
+### Next
+
+1. **Second batch of 50 tasks** to close the CI (projected half-width ≈±0.13 at
+   n=100 would clear the −0.05 margin if the point estimate holds).
+2. **Chunker follow-up for re-export/decorator-heavy repos** (`typer`): the pass that
+   turns a critical-file hit into an answerable window.
+3. Consider a relevance-ranked `broad` truncation so it isn't a strawman past ~40k —
+   though the point of this arm is precisely "what a naive big-context dump gets you".

pasr_bench-0.2.0/pasr_eval/__init__.py

@@ -0,0 +1,34 @@
+"""Real-agent evaluation harness for the PASR MCP (roadmap M11 / research Track 2).
+
+Not shipped with ``pasr-mcp``. Compares four arms per task — the agent's native
+search, full/broad context, PASR selected context, PASR + one controlled fallback —
+and reports paired metrics with a pre-registered non-inferiority margin.
+
+The dry-run path (``KeywordAgent`` + local-mode plan) exercises the whole matrix
+offline; the real run swaps in ``LlmAgent`` (answer + judge over the Anthropic API).
+"""
+
+from pasr_eval.arms import ARMS, ArmResult, run_arm
+from pasr_eval.metrics import aggregate, bootstrap_ci, grade, non_inferiority, paired_delta
+from pasr_eval.runner import run_plan, write_matrix
+from pasr_eval.spec import EvalPlan, RepoSpec, TaskSpec, load_plan, plan_from_dict
+from pasr_eval.validate import validate_matrix
+
+__all__ = [
+    "ARMS",
+    "ArmResult",
+    "run_arm",
+    "run_plan",
+    "write_matrix",
+    "EvalPlan",
+    "RepoSpec",
+    "TaskSpec",
+    "load_plan",
+    "plan_from_dict",
+    "aggregate",
+    "paired_delta",
+    "bootstrap_ci",
+    "non_inferiority",
+    "grade",
+    "validate_matrix",
+]

pasr_bench-0.2.0/pasr_eval/__main__.py

@@ -0,0 +1,47 @@
+"""``pasr-bench`` console entry: a thin dispatcher over the two orchestrators.
+
+    pasr-bench run [args...]       # the 4-arm real-agent / keyword evaluation
+    pasr-bench bakeoff [args...]   # the offline retrieval bake-off
+    pasr-bench plans               # list the packaged plans
+
+Everything after the sub-command is forwarded verbatim, so
+``pasr-bench run --agent claude --resume ...`` works exactly like the old
+``python eval/run_eval.py``.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_USAGE = "usage: pasr-bench {run|bakeoff|plans} [args...]"
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    if not args or args[0] in ("-h", "--help"):
+        print(_USAGE)
+        print("\n  run      end-to-end evaluation -> a timestamped delivery")
+        print("  bakeoff  offline retrieval bake-off (grep / repo-map / semantic / PASR)")
+        print("  plans    list the plan files bundled with this package")
+        return 0 if args else 2
+
+    cmd, rest = args[0], args[1:]
+    if cmd == "run":
+        from pasr_eval.run import main as run_main
+
+        return run_main(rest)
+    if cmd == "bakeoff":
+        from pasr_eval.bakeoff import main as bakeoff_main
+
+        return bakeoff_main(rest)
+    if cmd == "plans":
+        for path in sorted((Path(__file__).parent / "plans").glob("*.json")):
+            print(path)
+        return 0
+    print(f"unknown sub-command {cmd!r}\n{_USAGE}", file=sys.stderr)
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())