compactbench 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compactbench-0.1.0 → compactbench-0.3.0}/.gitignore +9 -4
- compactbench-0.3.0/CHANGELOG.md +498 -0
- compactbench-0.3.0/PKG-INFO +198 -0
- compactbench-0.3.0/README.md +136 -0
- compactbench-0.3.0/THIRD_PARTY_NOTICES.md +50 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/pyproject.toml +36 -14
- compactbench-0.3.0/src/compactbench/check.py +238 -0
- compactbench-0.3.0/src/compactbench/cli.py +922 -0
- compactbench-0.3.0/src/compactbench/compactors/__init__.py +89 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/compactors/_state_parser.py +43 -23
- compactbench-0.3.0/src/compactbench/compactors/_utils.py +158 -0
- compactbench-0.3.0/src/compactbench/compactors/agent_defaults.py +333 -0
- compactbench-0.3.0/src/compactbench/compactors/base.py +58 -0
- compactbench-0.3.0/src/compactbench/compactors/controls.py +110 -0
- compactbench-0.3.0/src/compactbench/compactors/extractive.py +73 -0
- compactbench-0.3.0/src/compactbench/compactors/hierarchical_summary.py +136 -0
- compactbench-0.3.0/src/compactbench/compactors/hybrid_ledger.py +121 -0
- compactbench-0.3.0/src/compactbench/compactors/naive_summary.py +68 -0
- compactbench-0.3.0/src/compactbench/compactors/provider_native.py +86 -0
- compactbench-0.3.0/src/compactbench/compactors/sectioned_summary.py +74 -0
- compactbench-0.3.0/src/compactbench/compactors/structured_state.py +71 -0
- compactbench-0.3.0/src/compactbench/config.py +84 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/contracts/__init__.py +26 -10
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/contracts/artifact.py +10 -4
- compactbench-0.3.0/src/compactbench/contracts/case.py +216 -0
- compactbench-0.3.0/src/compactbench/contracts/request.py +38 -0
- compactbench-0.3.0/src/compactbench/contracts/result.py +198 -0
- compactbench-0.3.0/src/compactbench/integrations/__init__.py +6 -0
- compactbench-0.3.0/src/compactbench/integrations/langchain.py +346 -0
- compactbench-0.3.0/src/compactbench/integrations/llamaindex.py +356 -0
- compactbench-0.3.0/src/compactbench/jsonutil.py +99 -0
- compactbench-0.3.0/src/compactbench/leaderboard/__init__.py +35 -0
- compactbench-0.3.0/src/compactbench/leaderboard/projection.py +211 -0
- compactbench-0.3.0/src/compactbench/leaderboard/qualification.py +73 -0
- compactbench-0.3.0/src/compactbench/leaderboard/ranking.py +39 -0
- compactbench-0.3.0/src/compactbench/leaderboard/season.py +80 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/__init__.py +13 -0
- compactbench-0.3.0/src/compactbench/providers/_retry.py +113 -0
- compactbench-0.3.0/src/compactbench/providers/anthropic.py +248 -0
- compactbench-0.3.0/src/compactbench/providers/base.py +106 -0
- compactbench-0.3.0/src/compactbench/providers/compatible.py +120 -0
- compactbench-0.3.0/src/compactbench/providers/counting.py +117 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/google_ai_studio.py +20 -7
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/groq.py +25 -5
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/mock.py +6 -1
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/ollama.py +29 -7
- compactbench-0.3.0/src/compactbench/providers/openai.py +213 -0
- compactbench-0.3.0/src/compactbench/proxy.py +447 -0
- compactbench-0.3.0/src/compactbench/redaction.py +76 -0
- compactbench-0.3.0/src/compactbench/rendering.py +73 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/runner/__init__.py +19 -18
- compactbench-0.3.0/src/compactbench/runner/costs.py +120 -0
- compactbench-0.3.0/src/compactbench/runner/cycle.py +151 -0
- compactbench-0.3.0/src/compactbench/runner/estimate.py +167 -0
- compactbench-0.3.0/src/compactbench/runner/persistence.py +226 -0
- compactbench-0.3.0/src/compactbench/runner/reader.py +163 -0
- compactbench-0.3.0/src/compactbench/runner/run.py +375 -0
- compactbench-0.3.0/src/compactbench/runner/traces.py +70 -0
- compactbench-0.3.0/src/compactbench/sandbox.py +328 -0
- compactbench-0.3.0/src/compactbench/scenarios/__init__.py +74 -0
- compactbench-0.3.0/src/compactbench/scenarios/builder.py +229 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/__init__.py +47 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/base.py +135 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/buried_constraint.py +452 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/conditional_scope.py +297 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/decision_override.py +358 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/entity_confusion.py +303 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/mixed.py +91 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/numeric_precision.py +265 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/provenance_authority.py +276 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/reference_resolution.py +333 -0
- compactbench-0.3.0/src/compactbench/scenarios/families/task_continuity.py +276 -0
- compactbench-0.3.0/src/compactbench/scenarios/generate.py +214 -0
- compactbench-0.3.0/src/compactbench/scenarios/lexicon.py +1032 -0
- compactbench-0.3.0/src/compactbench/scenarios/noise.py +362 -0
- compactbench-0.3.0/src/compactbench/scenarios/probes.py +146 -0
- compactbench-0.3.0/src/compactbench/scenarios/profile.py +101 -0
- compactbench-0.3.0/src/compactbench/scenarios/rng.py +67 -0
- compactbench-0.3.0/src/compactbench/scenarios/world.py +102 -0
- compactbench-0.3.0/src/compactbench/scoring/__init__.py +24 -0
- compactbench-0.3.0/src/compactbench/scoring/answers.py +282 -0
- compactbench-0.3.0/src/compactbench/scoring/metrics.py +205 -0
- compactbench-0.3.0/src/compactbench/sealing.py +132 -0
- compactbench-0.3.0/src/compactbench/submission.py +198 -0
- compactbench-0.3.0/src/compactbench/tokens.py +46 -0
- compactbench-0.1.0/CHANGELOG.md +0 -136
- compactbench-0.1.0/PKG-INFO +0 -161
- compactbench-0.1.0/README.md +0 -111
- compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_1.yaml +0 -66
- compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_2.yaml +0 -61
- compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_3.yaml +0 -71
- compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_4.yaml +0 -61
- compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_5.yaml +0 -67
- compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_1.yaml +0 -63
- compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_2.yaml +0 -76
- compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_3.yaml +0 -72
- compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_4.yaml +0 -71
- compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_5.yaml +0 -63
- compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_1.yaml +0 -64
- compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_2.yaml +0 -69
- compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_3.yaml +0 -61
- compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_4.yaml +0 -66
- compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_5.yaml +0 -71
- compactbench-0.1.0/benchmarks/public/hard/.gitkeep +0 -0
- compactbench-0.1.0/benchmarks/public/starter/buried_constraint_starter_v1.yaml +0 -68
- compactbench-0.1.0/benchmarks/public/starter/decision_override_starter_v1.yaml +0 -63
- compactbench-0.1.0/benchmarks/public/starter/entity_confusion_starter_v1.yaml +0 -67
- compactbench-0.1.0/benchmarks/schemas/template-v1.json +0 -135
- compactbench-0.1.0/docs/architecture/_legacy-saas/README.md +0 -111
- compactbench-0.1.0/src/compactbench/cli.py +0 -343
- compactbench-0.1.0/src/compactbench/compactors/__init__.py +0 -47
- compactbench-0.1.0/src/compactbench/compactors/_utils.py +0 -37
- compactbench-0.1.0/src/compactbench/compactors/base.py +0 -38
- compactbench-0.1.0/src/compactbench/compactors/hierarchical_summary.py +0 -113
- compactbench-0.1.0/src/compactbench/compactors/hybrid_ledger.py +0 -115
- compactbench-0.1.0/src/compactbench/compactors/naive_summary.py +0 -54
- compactbench-0.1.0/src/compactbench/compactors/structured_state.py +0 -64
- compactbench-0.1.0/src/compactbench/config.py +0 -36
- compactbench-0.1.0/src/compactbench/contracts/case.py +0 -81
- compactbench-0.1.0/src/compactbench/contracts/result.py +0 -79
- compactbench-0.1.0/src/compactbench/dsl/__init__.py +0 -66
- compactbench-0.1.0/src/compactbench/dsl/errors.py +0 -23
- compactbench-0.1.0/src/compactbench/dsl/generators.py +0 -243
- compactbench-0.1.0/src/compactbench/dsl/models.py +0 -107
- compactbench-0.1.0/src/compactbench/dsl/parser.py +0 -55
- compactbench-0.1.0/src/compactbench/dsl/substitution.py +0 -32
- compactbench-0.1.0/src/compactbench/dsl/validator.py +0 -97
- compactbench-0.1.0/src/compactbench/engine/__init__.py +0 -29
- compactbench-0.1.0/src/compactbench/engine/difficulty.py +0 -54
- compactbench-0.1.0/src/compactbench/engine/distractors.py +0 -69
- compactbench-0.1.0/src/compactbench/engine/errors.py +0 -11
- compactbench-0.1.0/src/compactbench/engine/generation.py +0 -96
- compactbench-0.1.0/src/compactbench/engine/seeds.py +0 -31
- compactbench-0.1.0/src/compactbench/engine/transcript.py +0 -84
- compactbench-0.1.0/src/compactbench/leaderboard/__init__.py +0 -38
- compactbench-0.1.0/src/compactbench/leaderboard/projection.py +0 -103
- compactbench-0.1.0/src/compactbench/leaderboard/qualification.py +0 -90
- compactbench-0.1.0/src/compactbench/leaderboard/ranking.py +0 -71
- compactbench-0.1.0/src/compactbench/providers/_retry.py +0 -48
- compactbench-0.1.0/src/compactbench/providers/base.py +0 -44
- compactbench-0.1.0/src/compactbench/runner/continuation.py +0 -78
- compactbench-0.1.0/src/compactbench/runner/cycle.py +0 -67
- compactbench-0.1.0/src/compactbench/runner/evaluation.py +0 -71
- compactbench-0.1.0/src/compactbench/runner/persistence.py +0 -227
- compactbench-0.1.0/src/compactbench/runner/run.py +0 -257
- compactbench-0.1.0/src/compactbench/scoring/__init__.py +0 -48
- compactbench-0.1.0/src/compactbench/scoring/checks.py +0 -92
- compactbench-0.1.0/src/compactbench/scoring/compression.py +0 -66
- compactbench-0.1.0/src/compactbench/scoring/contradictions.py +0 -70
- compactbench-0.1.0/src/compactbench/scoring/drift.py +0 -30
- compactbench-0.1.0/src/compactbench/scoring/scorer.py +0 -77
- compactbench-0.1.0/submissions/README.md +0 -34
- compactbench-0.1.0/submissions/_template/README.md +0 -17
- {compactbench-0.1.0 → compactbench-0.3.0}/LICENSE +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/__init__.py +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/__main__.py +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/compactors/errors.py +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/leaderboard/errors.py +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/errors.py +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/py.typed +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/runner/_resolver.py +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/runner/errors.py +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/scoring/errors.py +0 -0
- {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/utils/__init__.py +0 -0
|
@@ -80,12 +80,17 @@ site/
|
|
|
80
80
|
Thumbs.db
|
|
81
81
|
desktop.ini
|
|
82
82
|
|
|
83
|
-
# Local state
|
|
84
|
-
data/
|
|
85
|
-
results/
|
|
86
|
-
runs/
|
|
83
|
+
# Local state (anchored to repo root — docs/data/ must remain tracked)
|
|
84
|
+
/data/
|
|
85
|
+
/results/
|
|
86
|
+
/runs/
|
|
87
|
+
/.smoke-runs/
|
|
87
88
|
*.jsonl.gz
|
|
88
89
|
|
|
90
|
+
# Maintainer-only local tooling (kept local, never shipped)
|
|
91
|
+
/.local/
|
|
92
|
+
/smoke.bat
|
|
93
|
+
|
|
89
94
|
# Hidden benchmark content — never commit
|
|
90
95
|
benchmarks/hidden/
|
|
91
96
|
benchmarks/**/_private/
|
|
@@ -0,0 +1,498 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.3.0] — 2026-11-16
|
|
11
|
+
|
|
12
|
+
Benchmark v2 — **scorer_version 2.0.0 → 3.0.0**. Case generation, the compaction loop and
|
|
13
|
+
scoring are rewritten. Scores are not comparable with 0.2; v1 leaderboard rows are retired
|
|
14
|
+
to `docs/data/archive/` and v1 baseline results to `baselines/archive/`.
|
|
15
|
+
|
|
16
|
+
0.2.0 was never published to PyPI, so PyPI goes from 0.1.0 straight to 0.3.0; its changes
|
|
17
|
+
are listed under 0.2.0 below and included here.
|
|
18
|
+
|
|
19
|
+
### Breaking
|
|
20
|
+
|
|
21
|
+
- **Compactor interface.** `compact(self, request: CompactionRequest)` replaces
|
|
22
|
+
`compact(self, transcript, config=None, previous_artifact=None)`. After the first cycle,
|
|
23
|
+
`request.turns` holds only the turns since the last compaction and
|
|
24
|
+
`request.previous_artifact` holds the method's previous output — the earlier turns are
|
|
25
|
+
never resent. `config` moved to `self.config`. Classes with the old signature fail to
|
|
26
|
+
load with a migration message. LangChain and LlamaIndex adapters follow the same shape
|
|
27
|
+
(the previous artifact arrives as a leading system message).
|
|
28
|
+
- **YAML templates and the DSL are removed.** Cases come from scenario families
|
|
29
|
+
(`compactbench.scenarios`); suites are `starter` and `practice`. `generate` takes
|
|
30
|
+
`--family`; `run` takes `--cases` instead of `--case-count`; `--benchmarks-dir` is gone.
|
|
31
|
+
- **Results files use a new format** (`results_schema: "2"`). 0.2 files can't be read.
|
|
32
|
+
- **Submissions move to the website.** Ranked runs are submitted at compactbench.com/submit
|
|
33
|
+
instead of by pull request; `submissions/<handle>/<method>/config.yaml` is gone, and the
|
|
34
|
+
season pins every evaluation setting.
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
- **Hugging Face export** (`scripts/hf_dataset.py`): the public `practice`, `starter` and
|
|
39
|
+
`drills` suites, exactly the cases `compactbench run` uses, as JSONL configs with a dataset
|
|
40
|
+
card (seeds stored as strings so Arrow can hold them). Nothing from the hidden set.
|
|
41
|
+
- **Claude Code plugin** (`plugins/compactbench`, marketplace in `.claude-plugin/`): the
|
|
42
|
+
`forget-less` skill writes compaction instructions from the Season 1 findings for Claude
|
|
43
|
+
Code, Codex CLI, LangChain or LlamaIndex; the `measure-compaction` skill wraps a method,
|
|
44
|
+
estimates the cost, runs `compactbench check` against a baseline on the user's keys, and can
|
|
45
|
+
add the GitHub Action.
|
|
46
|
+
- **`compactbench check`** — runs a small, fixed Practice sample (4 cases, seed group `ci`,
|
|
47
|
+
Season 1 budget and reader by default) on your keys, prints a Markdown report (score and
|
|
48
|
+
interval, score after each compaction, per family) and compares it with a saved baseline:
|
|
49
|
+
`--max-drop` and `--min-score` fail the run on a regression, `--update-baseline` records a
|
|
50
|
+
new one. Baselines from different settings are refused rather than compared.
|
|
51
|
+
- **Agents' default compaction** as built-ins, reproduced from their open-source code at
|
|
52
|
+
pinned commits: `codex-cli` (Codex CLI's handoff-summary prompt, recent user messages
|
|
53
|
+
kept verbatim ahead of the prefixed summary) and `langchain-summarization` (LangChain's
|
|
54
|
+
`SummarizationMiddleware` defaults: keep the last 20 messages, summarize the rest after
|
|
55
|
+
trimming it to its last ~4,000 tokens). Under a budget, verbatim messages get at most a
|
|
56
|
+
quarter of it. Attribution in `THIRD_PARTY_NOTICES.md`.
|
|
57
|
+
- **"Forget less" guide** (`docs/forget-less.md`): what Season 1 shows about compaction
|
|
58
|
+
instructions, with paste-in instructions for Claude Code, Codex CLI, LangChain and
|
|
59
|
+
LlamaIndex and a recipe to measure them.
|
|
60
|
+
- **GitHub Action** (`action.yml`) wrapping `compactbench check`: job summary, an optional
|
|
61
|
+
pull-request comment kept up to date, and `score` / `change` / `passed` outputs.
|
|
62
|
+
- **Scenario families**: `buried_constraint`, `decision_override`, `entity_confusion`,
|
|
63
|
+
`reference_resolution`, `numeric_precision`, `conditional_scope`, `task_continuity` and
|
|
64
|
+
`provenance_authority` — long, multi-cycle conversations (up to ~32k-token openings and
|
|
65
|
+
4 drift cycles at `elite`) where later cycles add and lift rules, override and revert
|
|
66
|
+
decisions, hand over ownership, switch choices by reference, correct exact figures,
|
|
67
|
+
change a rule's scope, move tickets across a board, and slip contradicting claims in
|
|
68
|
+
through fetched documents. On-topic procedural
|
|
69
|
+
filler (code, tests, logs, API responses, git history, metrics, notes), decoys built from
|
|
70
|
+
the case, and "never settled" questions. Deterministic and platform-stable.
|
|
71
|
+
- **Structured probes, scored mechanically** — single choice, select-all, exact value
|
|
72
|
+
(money, date, number, version, identifier, text normalization) and list. No LLM judge;
|
|
73
|
+
hedging can't pay.
|
|
74
|
+
- **A separate reader model** (`--reader-provider` / `--reader-model`) answers each cycle's
|
|
75
|
+
probes in batches, from the artifact only, fenced as untrusted data, with a JSON schema
|
|
76
|
+
where the provider supports one.
|
|
77
|
+
- **Metrics**: family-balanced score with a 95% bootstrap interval, score by cycle, drift
|
|
78
|
+
resistance and decay on carried probes, stale-answer rate, trap rate (decoy-only
|
|
79
|
+
answers: other projects' rules, rejected proposals, injected values), hallucination rate,
|
|
80
|
+
per-family and per-category breakdowns, artifact tokens. Abstention probes count only
|
|
81
|
+
when missed, so an empty artifact scores 0.
|
|
82
|
+
- **Controls on the same cases** (`--controls`): oracle and null scores per cycle and a
|
|
83
|
+
normalized score per family. The `oracle`, `null` and `truncate-last-n` built-ins are
|
|
84
|
+
ported to the new loop (`truncate-last-n` now keeps as many recent turns as fit the budget).
|
|
85
|
+
- **Token budgets** (`--budget`, default 2048): over-budget artifacts are cut before the
|
|
86
|
+
reader sees them and flagged; ranked runs reject them.
|
|
87
|
+
- **Traces** (`--traces`): every artifact, question and raw reader reply, per cycle.
|
|
88
|
+
- **Seasons** (`--season`): a file pinning suite, difficulty, cycles, cases, budget, reader
|
|
89
|
+
and controls, with fixed or fresh-per-run sampling; `qualify --season` checks a run
|
|
90
|
+
against one.
|
|
91
|
+
- **Suite packs** (`--suite-pack`): load private families and worlds — vocabulary,
|
|
92
|
+
filler and the wording of every sentence the families write (`Lexicon.phrases`) — and
|
|
93
|
+
spread a suite's cases across several worlds (`Suite.settings`).
|
|
94
|
+
- **Mixed sessions** (`mixed`): every family in one conversation, sharing people,
|
|
95
|
+
components and one token budget; probes are tagged with their family, so scores still
|
|
96
|
+
break down per family. `practice` (and ranked Elite) are mixed sessions; the new
|
|
97
|
+
`drills` suite keeps one family per case for focused work. Families that draw facts
|
|
98
|
+
from the same vocabulary pool get disjoint shares of it.
|
|
99
|
+
- **Sandboxed ranked evaluation.** `run --sandbox-image` runs the method in a container
|
|
100
|
+
(no network but the model proxy, read-only filesystem, no capabilities, resource
|
|
101
|
+
limits) and talks to it one request at a time over stdin/stdout; submitted code is never
|
|
102
|
+
imported by the harness. `compactbench proxy` holds the provider key, allows only
|
|
103
|
+
declared models and model-call endpoints, refuses streaming, enforces a token cap and
|
|
104
|
+
logs metadata only, and refuses requests that would have the provider reach the
|
|
105
|
+
internet for the method (server-side tools, MCP servers, content by URL). Keys are
|
|
106
|
+
sealed to the season's public key (X25519, HKDF-SHA256, AES-256-GCM), bound to their
|
|
107
|
+
submission. The container runs under gVisor in CI. Dockerfiles, submission terms,
|
|
108
|
+
privacy policy and a datasheet.
|
|
109
|
+
- **Self-serve ranked submissions.** Sign in with GitHub on the site, upload `method.py`,
|
|
110
|
+
pick the model and add your keys; the Worker (D1 + R2) validates the submission, seals
|
|
111
|
+
the keys on arrival, applies per-account and daily limits (with an optional Turnstile
|
|
112
|
+
check and a kill switch) and starts the evaluation. Qualifying runs
|
|
113
|
+
publish themselves — method source, public settings, scores and a commitment to the
|
|
114
|
+
sealed traces under `submissions/<handle>/<method>/` — and the site redeploys. A status
|
|
115
|
+
page follows each run and says why a run didn't qualify. Runs are recorded under the
|
|
116
|
+
name and version given on the form, and method pages include the submitter's handle. Submitters pay for their method's calls and the reader.
|
|
117
|
+
- **Published results are redacted** (`compactbench.redaction`). Results files from a
|
|
118
|
+
hidden suite carry probe keys named after the private world's people, rules and
|
|
119
|
+
tickets, and each cycle carries the method's own warning text. Published copies (ranked
|
|
120
|
+
submissions and baselines) replace keys with salted hashes — equal within a case, so
|
|
121
|
+
every metric is unchanged — and drop method text; the unredacted results are sealed
|
|
122
|
+
with the traces. A test fails if an unredacted hidden-suite file is committed.
|
|
123
|
+
- **Any model.** Hosted OpenAI-compatible providers `openrouter`, `deepseek`, `mistral`,
|
|
124
|
+
`together`, `fireworks`, `xai` and `gemini` (keys from `COMPACTBENCH_<NAME>_API_KEY`),
|
|
125
|
+
and `custom`: a submitter's own OpenAI-compatible HTTPS endpoint, reached only through
|
|
126
|
+
the proxy and only at a public address, with an optional display name shown on the
|
|
127
|
+
leaderboard next to a "custom endpoint" label.
|
|
128
|
+
- `Suite.probes_per_case`: keep a fixed, family-balanced subset of each case's probes,
|
|
129
|
+
asked at every compaction, so a run can afford many more cases. The score interval
|
|
130
|
+
resamples whole cases, so a mixed case's families move together.
|
|
131
|
+
- **Owner dashboard** (`admin/`, Cloudflare Worker behind Cloudflare Access): website
|
|
132
|
+
submissions with stuck-run flags, ranked evaluation runs, the published leaderboard,
|
|
133
|
+
per-family difficulty, review flags for unusual submissions and a CSV export; read-only. The Worker
|
|
134
|
+
verifies the Access token itself (owner email, audience, issuer, expiry, signature).
|
|
135
|
+
- **Remote MCP server** at `/mcp` on the site: `get_leaderboard`, `get_method`,
|
|
136
|
+
`compare_methods`, `list_families`, `explain_metric`, and a self-test for AIs
|
|
137
|
+
(`selftest_list`, `selftest_turns`, `selftest_questions`, `selftest_score`): compact a
|
|
138
|
+
public case with your own model and get a mechanical score (`scripts/selftest_cases.py`).
|
|
139
|
+
- `scripts/run_baselines.py` (season roster, `--parallel`, `--estimate`) and
|
|
140
|
+
`scripts/season_calibration.py` (per-family ceiling and floor for the Seasons page).
|
|
141
|
+
- Maintainer tools: `scripts/item_analysis.py` (per-template validity, difficulty and
|
|
142
|
+
discrimination), `scripts/qa_sheet.py` (a human review sheet for a suite's questions),
|
|
143
|
+
`scripts/reader_robustness.py` (re-read traced artifacts with a second reader and
|
|
144
|
+
compare method orderings) and `scripts/failure_gallery.py`.
|
|
145
|
+
- Per-probe control scores (`oracle_score`, `null_score` on each probe result), so any
|
|
146
|
+
grouping has its own ceiling and floor.
|
|
147
|
+
- **Website** (`web/`, Astro, deployed to Cloudflare Workers static assets): home,
|
|
148
|
+
leaderboard (sortable, per-family heat map, score-by-compaction sparklines, intervals),
|
|
149
|
+
methodology, families, submit, privacy and terms pages rendered from `docs/`, a page per
|
|
150
|
+
method (score by compaction, by family, run details), a two-method comparison, and a
|
|
151
|
+
score-versus-cost chart, plus
|
|
152
|
+
`llms.txt`, `llms-full.txt`, a Markdown twin of every page, JSON endpoints
|
|
153
|
+
(`/api/leaderboard.json`, `/api/methods/<slug>.json`, `/api/families.json`,
|
|
154
|
+
`/api/season.json`) with an OpenAPI description, a CSV download, JSON-LD, share images
|
|
155
|
+
(one per method, with its score and score by compaction), security headers with a strict
|
|
156
|
+
Content-Security-Policy, the developer docs under `/docs`, and a failure gallery of
|
|
157
|
+
real failures from public Practice runs.
|
|
158
|
+
`scripts/export_site_data.py` feeds it from the package so the site never drifts.
|
|
159
|
+
- **Season 1** (`seasons/s1.yaml`): Elite 2.0.1, 4 drift cycles, 48 mixed cases of 20 probes,
|
|
160
|
+
1,024-token budget (Tight), `claude-haiku-5-5` reader at high effort, fresh sampling;
|
|
161
|
+
controls are measured once per season on a calibration sample rather than on every run.
|
|
162
|
+
- **`sectioned-summary`** built-in: the multi-section summary format coding agents
|
|
163
|
+
commonly compact into, with a generic prompt. **`extractive`**: keeps pattern-matched
|
|
164
|
+
state sentences, newest first within the budget, with no model calls.
|
|
165
|
+
- Larger public vocabulary (26 rules, 25 decision topics, 14 facts, 15 ticket titles) for
|
|
166
|
+
more varied cases.
|
|
167
|
+
- **`anthropic-compaction`** built-in: Anthropic's server-side on-demand compaction
|
|
168
|
+
(beta `compact-2026-09-04`) run as shipped — default prompt, compacting again from the
|
|
169
|
+
signed block each cycle. OpenAI's Responses compaction returns an encrypted item that a
|
|
170
|
+
reader from another provider can't read, so it has no built-in.
|
|
171
|
+
- **Reader effort** (`--reader-effort`, and `reader.effort` in seasons) is recorded with
|
|
172
|
+
the run and is part of the leaderboard segment.
|
|
173
|
+
- Built-in methods size their output for the budget (with room for thinking), trim
|
|
174
|
+
their own artifact to fit it, oldest entries first, and use `low` reasoning effort
|
|
175
|
+
unless `effort` is set in their config.
|
|
176
|
+
- `run --method-config` passes a JSON object to the method as `self.config`.
|
|
177
|
+
- `compactbench.compactors.fit_to_budget` is public: trims an artifact to the token budget
|
|
178
|
+
the way the built-ins do. The submission template uses it, asks for low reasoning effort
|
|
179
|
+
and leaves output room for thinking, so it qualifies out of the box.
|
|
180
|
+
- `families list`; `generate --text` prints a readable case.
|
|
181
|
+
|
|
182
|
+
### Changed
|
|
183
|
+
|
|
184
|
+
- Qualification: control arms never ranked; incomplete runs, runs without drift cycles or
|
|
185
|
+
a budget, over-budget cycles, more than 5% unusable reader answers, and readers on a
|
|
186
|
+
custom endpoint are rejected; season settings must match.
|
|
187
|
+
- Ranking: by score within suite, scorer, reader and budget; ties on drift resistance,
|
|
188
|
+
then stale-answer rate. A rank is one plus the number of rows whose 95% interval lies
|
|
189
|
+
entirely above, so rows that can't be told apart share a rank. Leaderboard rows carry CIs, per-family scores and score by cycle.
|
|
190
|
+
- `--estimate` projects the method, reader and control costs separately.
|
|
191
|
+
- Rate-limited calls (HTTP 429) get up to 8 attempts and wait at least the server's
|
|
192
|
+
`retry-after`.
|
|
193
|
+
- Providers no longer send sampling parameters by default, and gain `json_schema` and
|
|
194
|
+
`reasoning_effort` on `CompletionRequest`. The Anthropic SDK moves to 2.x (API version
|
|
195
|
+
`2026-10-21`); 529 responses are now retried; refusals and empty token-limit stops raise
|
|
196
|
+
clear errors. Cost catalogue refreshed.
|
|
197
|
+
- The ranked evaluation workflow is rebuilt around the sandbox and started by the site:
|
|
198
|
+
an untrusted build with no secrets, an evaluation in a protected environment, and a
|
|
199
|
+
publish job that commits the results and redeploys the site. The site's internal
|
|
200
|
+
endpoints accept only that workflow's OIDC token from `main`.
|
|
201
|
+
- Dependabot uses the `uv` ecosystem and batches security updates.
|
|
202
|
+
|
|
203
|
+
### Removed
|
|
204
|
+
|
|
205
|
+
- `compactbench.dsl`, `compactbench.engine`, the v1 scorer (`checks`, `contradictions`,
|
|
206
|
+
`drift`, `compression`, `diagnostics`), model-generated continuation turns, and
|
|
207
|
+
`benchmarks/public` / `benchmarks/schemas`.
|
|
208
|
+
- The pull-request submission flow: `submissions/_template/`, `validate_submissions.py`,
|
|
209
|
+
`validate_submission_config.py` and the separate publish workflow.
|
|
210
|
+
|
|
211
|
+
## 0.2.0 — 2026-08-24 (not published to PyPI)
|
|
212
|
+
|
|
213
|
+
### Scoring integrity — **scorer_version 1.0.0 → 2.0.0**
|
|
214
|
+
|
|
215
|
+
Scores produced by scorer 1.0.0 are **not comparable** to these. The leaderboard
|
|
216
|
+
segments on `scorer_version`, so old and new rows never mix. Four defects meant a
|
|
217
|
+
method that did nothing intelligent would have topped the first leaderboard.
|
|
218
|
+
|
|
219
|
+
- **Compaction methods no longer receive the answer key.** Generated turns carry
|
|
220
|
+
`tags` — `distractor` on filler, `critical_constraint` on the turn holding the
|
|
221
|
+
answer — and the runner handed them straight to the method. A five-line
|
|
222
|
+
compactor with no model call that keeps the non-`distractor` turns recovered
|
|
223
|
+
100% of the recall items at ~9x compression. Tags are now stripped at the
|
|
224
|
+
compaction boundary (this also closes the same leak through the LangChain and
|
|
225
|
+
LlamaIndex adapters); they remain on the case for diagnostics.
|
|
226
|
+
- **`drift_resistance` no longer rewards consistent failure.** It was
|
|
227
|
+
`clamp(1 + mean(score[n] - score[0]))` — a measure of whether scores *change*,
|
|
228
|
+
not whether they are any good — so a method scoring 0.0 on every cycle was
|
|
229
|
+
perfectly stable and scored **1.0**, collecting 30% of `elite_score` for the
|
|
230
|
+
worst possible result. It is now a retention ratio,
|
|
231
|
+
`mean(later) / first`, and is 0.0 when unmeasurable (fewer than two cycles, or
|
|
232
|
+
a zero baseline) rather than a free 1.0. Empty and crashed runs aggregate to
|
|
233
|
+
0.0 drift resistance instead of 1.0.
|
|
234
|
+
- **The JSON state parser recovers objects wrapped in prose.** It required the
|
|
235
|
+
response to be bare JSON or a fenced block, so `Here is the state: {...}`, a
|
|
236
|
+
trailing `Hope that helps!`, and `<think>` preambles all produced a fully
|
|
237
|
+
empty state that scored as total information loss — measuring the parser
|
|
238
|
+
rather than the method, and worst on exactly the small and local models the
|
|
239
|
+
benchmark should be free to run on. Brace matching is string- and
|
|
240
|
+
escape-aware, so quoted code containing `}` no longer truncates the object.
|
|
241
|
+
- **Qualification closes four routes to an unearned rank.** Control arms are
|
|
242
|
+
never ranked; compression above 50x is rejected as a degenerate/empty artifact
|
|
243
|
+
(the `null` arm measures ~450x and previously cleared every tier and maxed the
|
|
244
|
+
compression bonus); runs configured with zero drift cycles are unrankable,
|
|
245
|
+
since drift is 30% of the score and they never measured it; and a run whose
|
|
246
|
+
results file has no `run_end` event is rejected, so a crashed run cannot
|
|
247
|
+
publish a partial score and stopping early is not a way to cherry-pick.
|
|
248
|
+
|
|
249
|
+
### Security — submission evaluation pipeline
|
|
250
|
+
|
|
251
|
+
- **The evaluation workflow could never succeed, and reported it as the submitter's fault.**
|
|
252
|
+
It checked out the PR head at the default `fetch-depth: 1`, then ran
|
|
253
|
+
`git diff <base> <head>`. The base commit does not exist in a shallow clone, so
|
|
254
|
+
the diff failed, the failure was swallowed by `|| true`, and the job exited with
|
|
255
|
+
"PR does not touch any submissions/<handle>/<method>/ directory". Every
|
|
256
|
+
submission would have been turned away with a misleading error. Now `fetch-depth: 0`.
|
|
257
|
+
- **The hidden ranked set was exfiltrable with no code execution.** The config step
|
|
258
|
+
interpolated five unvalidated submitter-controlled YAML values into a file and
|
|
259
|
+
appended it to `$GITHUB_ENV`. A newline in `runtime.model` injected arbitrary
|
|
260
|
+
job-wide environment variables; since `OllamaProvider` passes
|
|
261
|
+
`COMPACTBENCH_OLLAMA_BASE_URL` straight to its client, a submission with
|
|
262
|
+
`provider: ollama` would have caused the runner to POST **every hidden ranked
|
|
263
|
+
case to an attacker-controlled host as a prompt**. No `method.py` runs on that
|
|
264
|
+
path, so sandboxing submitter code would not have closed it. All values now go
|
|
265
|
+
through `scripts/validate_submission_config.py` — allow-listed provider,
|
|
266
|
+
whitespace-free model pattern, bounded integers, control characters rejected,
|
|
267
|
+
and `class_path` required to sit inside the submission directory the PR
|
|
268
|
+
actually touched. Values never transit `$GITHUB_ENV`; the runner reads them
|
|
269
|
+
from validated JSON. 35 tests cover it, including the concrete attack payload.
|
|
270
|
+
- **Script injection in the score comment.** `body: \`${{ steps.summary.outputs.body }}\``
|
|
271
|
+
placed submitter-influenced text (the method name reaches it via
|
|
272
|
+
`compactbench score` output) inside a JS template literal, where a backtick or
|
|
273
|
+
`${...}` escapes into arbitrary JavaScript in a step holding a write-scoped
|
|
274
|
+
`GITHUB_TOKEN`. Passed via `process.env` instead.
|
|
275
|
+
- **The hidden-set token no longer persists on disk.** It was embedded in the
|
|
276
|
+
clone URL, which git writes verbatim into `/tmp/hidden/.git/config` — leaving a
|
|
277
|
+
credential that regenerates the entire hidden set readable by submitter code
|
|
278
|
+
executing later in the same job. Now passed via `GIT_ASKPASS`, with the `.git`
|
|
279
|
+
directory removed once the working tree exists.
|
|
280
|
+
- `scripts/validate_submissions.py` now validates every submission config, not
|
|
281
|
+
just greps for leftover `FILL_ME` placeholders, so contributors get the error
|
|
282
|
+
on their PR instead of after a maintainer applies the `evaluate` label.
|
|
283
|
+
- `ollama` and `mock` are no longer accepted as ranked-evaluation providers. Both
|
|
284
|
+
can be pointed at an arbitrary host, and the evaluator must control where calls go.
|
|
285
|
+
|
|
286
|
+
### Added — control arms
|
|
287
|
+
- `oracle`, `null`, and `truncate-last-n` built-ins. None makes a model call, so
|
|
288
|
+
they are free and deterministic to run alongside any experiment. A score in
|
|
289
|
+
isolation is uninterpretable: the oracle bounds what is achievable when
|
|
290
|
+
compaction loses nothing, and the null arm exposes how many points are
|
|
291
|
+
available with no information at all. Measured against a real model, the
|
|
292
|
+
oracle scores **0.650**, not 1.0 — so a method scoring 0.5 has not lost half
|
|
293
|
+
the information, it is at 77% of the achievable ceiling.
|
|
294
|
+
- `compactbench qualify --results <file>` — runs the leaderboard floors locally
|
|
295
|
+
and prints every reason a run would be rejected. Previously a submitter could
|
|
296
|
+
only discover a disqualification by opening a PR and waiting for a maintainer,
|
|
297
|
+
after having already spent the API budget on the run.
|
|
298
|
+
- `CompactionArtifact` accepts either Python field names or the camelCase wire
|
|
299
|
+
aliases (`populate_by_name`). `summary_text=` previously raised
|
|
300
|
+
"Extra inputs are not permitted" while `summaryText=` worked — a confusing
|
|
301
|
+
first failure for anyone writing a method, which the docs' own example
|
|
302
|
+
half-tripped over by mixing the two styles.
|
|
303
|
+
|
|
304
|
+
Four months of merged work that was never released, plus the packaging fix that
|
|
305
|
+
makes it reachable. **If you installed 0.1.0, upgrade** — that wheel shipped
|
|
306
|
+
without any benchmark content, so `compactbench suites list` failed on a clean
|
|
307
|
+
install and no benchmark could be run from PyPI at all.
|
|
308
|
+
|
|
309
|
+
### Fixed
|
|
310
|
+
- **The published wheel now contains the benchmark suites.** The `force-include`
|
|
311
|
+
mapping that bundles `benchmarks/public` landed in #21, *after* the v0.1.0 tag,
|
|
312
|
+
so the released artifact had zero templates and the documented first command
|
|
313
|
+
exited 1. CI and the release workflow now install the built wheel into a clean
|
|
314
|
+
environment and assert the CLI can see `starter` and `elite_practice`, so this
|
|
315
|
+
class of packaging regression cannot ship again.
|
|
316
|
+
- `.env` is now actually read. Every provider reads `os.environ` directly, and
|
|
317
|
+
the `Settings` object that parsed `.env` was imported by nothing — the
|
|
318
|
+
documented way to supply API keys silently did nothing. The CLI now loads
|
|
319
|
+
`COMPACTBENCH_*` variables from `.env` before any provider is constructed.
|
|
320
|
+
A real environment variable still wins over the file.
|
|
321
|
+
- An empty `COMPACTBENCH_OPENAI_BASE_URL` no longer marks a genuine OpenAI run
|
|
322
|
+
as self-hosted. The client was gated on truthiness and the provenance flag on
|
|
323
|
+
identity, so `""` produced a run stamped as having gone somewhere it hadn't.
|
|
324
|
+
- A base URL without an `http://` or `https://` scheme is now rejected at
|
|
325
|
+
construction instead of failing much later with an opaque connection error.
|
|
326
|
+
- `uv.lock` regenerated (90 → 145 packages; it had not been touched since the
|
|
327
|
+
initial scaffold and was missing the `anthropic` and `openai` entries
|
|
328
|
+
entirely). All workflows now install with `--locked`, so CI fails on lock
|
|
329
|
+
drift rather than silently re-resolving against live PyPI on every run — which
|
|
330
|
+
matters for a project whose central claim is reproducibility.
|
|
331
|
+
- The test suite no longer inherits the developer's environment: an autouse
|
|
332
|
+
fixture clears `COMPACTBENCH_*`, fixing `test_requires_api_key` failing for
|
|
333
|
+
anyone with a base URL exported (i.e. exactly the local-model users).
|
|
334
|
+
|
|
335
|
+
### Added
|
|
336
|
+
- **Endpoint provenance.** `run_start` and `RunResult` now carry `endpoint_kind`
|
|
337
|
+
(`"default"` or `"custom"`), sourced from the provider, and it is part of the
|
|
338
|
+
leaderboard's ranking segment. Without it a locally-served model submitted as
|
|
339
|
+
`--provider openai --model gpt-4o` was ranked against genuine gpt-4o — the
|
|
340
|
+
provider and model fields are both caller-supplied strings. `--resume` also
|
|
341
|
+
checks it, so a run cannot be half-executed against a hosted model and
|
|
342
|
+
finished against a local one under the same alias. The URL itself is never
|
|
343
|
+
stored; results files get shared and the host is often internal.
|
|
344
|
+
- `OpenAIProvider` accepts a `base_url` (constructor argument or
|
|
345
|
+
`COMPACTBENCH_OPENAI_BASE_URL`), so the `openai` provider also serves any
|
|
346
|
+
OpenAI-compatible endpoint — vLLM, llama.cpp server, LM Studio, Together,
|
|
347
|
+
Fireworks, OpenRouter (#37, thanks @jaaabir). When a base URL is set the API
|
|
348
|
+
key becomes optional, because self-hosted servers generally do not check one.
|
|
349
|
+
- Anthropic and OpenAI providers (#19).
|
|
350
|
+
- LangChain (#16) and LlamaIndex (#17) integration adapters.
|
|
351
|
+
- Zero-install Colab notebook (#18) with a CI smoke test that rebuilds it from
|
|
352
|
+
its generator and executes every embedded command against the mock provider.
|
|
353
|
+
- Case-level parallelism and a `--estimate` cost projector (#22).
|
|
354
|
+
- Prompt caching via `cached_prefix` on `CompletionRequest` (#23).
|
|
355
|
+
- `reference_resolution` template family (#27) — a fourth family, 5 templates —
|
|
356
|
+
and a per-item-type diagnostic breakdown in `compactbench score`.
|
|
357
|
+
- Per-cycle / per-case / per-run `TokenUsage` telemetry (#28).
|
|
358
|
+
|
|
359
|
+
### Changed
|
|
360
|
+
- Cost catalogue refreshed and made self-policing: `--estimate` now prints the
|
|
361
|
+
price vintage alongside the dollar figure, and a test fails once the catalogue
|
|
362
|
+
is more than 120 days old. Anthropic entries updated to the current lineup;
|
|
363
|
+
the previous table priced models that have since been superseded and had no
|
|
364
|
+
entry for any current one, so `--estimate` either quoted stale rates or
|
|
365
|
+
refused to answer.
|
|
366
|
+
- Pinned GitHub Action SHAs bumped (#34).
|
|
367
|
+
- Docs corrected to describe the project that actually exists: four template
|
|
368
|
+
families and 20 practice templates (not three and 15), five real providers
|
|
369
|
+
(not three), and `~900` calls for a full Elite practice run (not `~450`).
|
|
370
|
+
|
|
371
|
+
## [0.1.0] — 2026-04-17
|
|
372
|
+
|
|
373
|
+
First public release. The full v1 stack: DSL parser, case-generation engine,
|
|
374
|
+
scoring engine, mock + real providers, built-in compactors, end-to-end runner,
|
|
375
|
+
submission pipeline with public leaderboard, and 15 Elite practice templates
|
|
376
|
+
across three launch families.
|
|
377
|
+
|
|
378
|
+
### Added
|
|
379
|
+
- Initial project scaffold: Python package layout, CLI entrypoint, core ABCs
|
|
380
|
+
- Apache 2.0 license
|
|
381
|
+
- Contributor guide and code of conduct
|
|
382
|
+
- MkDocs documentation skeleton
|
|
383
|
+
- GitHub Actions CI workflow
|
|
384
|
+
- Template DSL (WO-002):
|
|
385
|
+
- YAML + Handlebars-style `{{variable}}` substitution
|
|
386
|
+
- Pydantic models for `TemplateDefinition` and sub-structures
|
|
387
|
+
- Seeded generator registry: `person_name`, `action_phrase`, `project_noun`,
|
|
388
|
+
`org_name`, `date_iso`, `amount_usd`, `product_sku`
|
|
389
|
+
- SHA-256-based sub-seed derivation for reordering-stable determinism
|
|
390
|
+
- Semantic validator for references, generators, and duplicate names
|
|
391
|
+
- JSON Schema at `benchmarks/schemas/template-v1.json`
|
|
392
|
+
- Three starter templates: `buried_constraint_starter_v1`,
|
|
393
|
+
`decision_override_starter_v1`, `entity_confusion_starter_v1`
|
|
394
|
+
- `compactbench suites list` command wired up
|
|
395
|
+
- Elite v1 public practice templates (WO-009):
|
|
396
|
+
- 15 templates in `benchmarks/public/elite_practice/`, 5 variations each for
|
|
397
|
+
the three launch families: `buried_constraint_v1`, `decision_override_v1`,
|
|
398
|
+
`entity_confusion_v1`
|
|
399
|
+
- Variations within each family probe different failure modes (e.g.,
|
|
400
|
+
indirect phrasing, double constraints, negative examples, sandwich
|
|
401
|
+
layouts for `buried_constraint`; simple/three-way/partial/late/meta-reset
|
|
402
|
+
overrides for `decision_override`; 2-person/3-person/same-role/
|
|
403
|
+
observer-vs-owner/shared-project for `entity_confusion`)
|
|
404
|
+
- Difficulty policy scales distractor turns 4 → 8 → 16 → 32 from easy to
|
|
405
|
+
elite, matching the ranked set's expected adversarial load
|
|
406
|
+
- `compactbench suites list` now shows `elite_practice` with 15 templates
|
|
407
|
+
across 3 families
|
|
408
|
+
- `docs/governance/elite-versions.md` — active-version table, version
|
|
409
|
+
policy, retirement policy, anti-gaming reminders, v1 changelog
|
|
410
|
+
- 9 new unit tests verify every template parses, validates, and generates
|
|
411
|
+
at every configured difficulty
|
|
412
|
+
- Hidden ranked set (20 per family in `compactbench/compactbench-hidden`)
|
|
413
|
+
lands in a follow-up so it doesn't commingle public and hidden content
|
|
414
|
+
- Submission pipeline + leaderboard (WO-008):
|
|
415
|
+
- `src/compactbench/leaderboard/` — `elite_score` computation, tie-breakers,
|
|
416
|
+
qualification floors (with rejection reasons), `RunResult` → public
|
|
417
|
+
`LeaderboardRow` projection, best-first rank assignment
|
|
418
|
+
- `.github/workflows/evaluate-submission.yml` — runs on GitHub-hosted
|
|
419
|
+
`ubuntu-latest`, gated on a maintainer-applied `evaluate` label; clones PR
|
|
420
|
+
head, installs submission deps, runs `compactbench run`, posts score
|
|
421
|
+
comment, uploads results artifact
|
|
422
|
+
- `.github/workflows/update-leaderboard.yml` — on push under `submissions/**`
|
|
423
|
+
rebuilds `docs/data/leaderboard.json` via `scripts/rebuild_leaderboard.py`
|
|
424
|
+
and commits; triggers the docs Pages redeploy
|
|
425
|
+
- `docs/leaderboard.md` — live table rendered client-side from
|
|
426
|
+
`data/leaderboard.json`, grouped by `(benchmark_version, target_model)`
|
|
427
|
+
- `submissions/_template/` — copy-paste scaffold (method.py, config.yaml,
|
|
428
|
+
requirements.txt, README.md)
|
|
429
|
+
- `compactbench submit` prints a submission checklist (full PR automation
|
|
430
|
+
stays server-side since hidden-set evaluation requires repo secrets)
|
|
431
|
+
- Runner infra decision updated: v1 uses GitHub-hosted runners, not
|
|
432
|
+
self-hosted; migration path preserved
|
|
433
|
+
- End-to-end runner (WO-007):
|
|
434
|
+
- `compactbench run --method <spec> --suite <key> --provider <k> --model <m>`
|
|
435
|
+
orchestrates: load suite → generate cases → per case iterate drift cycles
|
|
436
|
+
(compact → evaluate → score) → aggregate → persist results.jsonl
|
|
437
|
+
- Event-log JSONL format: `run_start`, one `case_complete` per case
|
|
438
|
+
streamed, final `run_end` with aggregates
|
|
439
|
+
- `--resume` continues from an existing output file; rejects with
|
|
440
|
+
`ResumeError` if run parameters differ
|
|
441
|
+
- `compactbench score --results` now pretty-prints the run summary from the
|
|
442
|
+
persisted events (reads, not re-scores)
|
|
443
|
+
- `--method` supports `built-in:<key>` and `<path.py>:<ClassName>`
|
|
444
|
+
(loads user compactors from file paths, verifies `Compactor` subclass)
|
|
445
|
+
- Drift cycles extend the transcript with seeded continuation turns; the
|
|
446
|
+
model's continuation response uses only the previous artifact as context
|
|
447
|
+
(that is the drift vector measured)
|
|
448
|
+
- Per-case `CaseResult` captures all cycles, case-level score, and
|
|
449
|
+
`drift_resistance`; run-level aggregates cover overall / drift /
|
|
450
|
+
constraint retention / contradiction / compression
|
|
451
|
+
- Real model providers (WO-006):
|
|
452
|
+
- `GroqProvider` — Llama 3.3 70B, Kimi K2, etc. via the `groq` SDK
|
|
453
|
+
- `GoogleAIStudioProvider` — Gemini 2.0 Flash etc. via the `google-genai` SDK
|
|
454
|
+
- `OllamaProvider` — local models via the `ollama` SDK
|
|
455
|
+
- Shared `retry_with_backoff` async helper with exponential + capped delay
|
|
456
|
+
- Per-provider retry predicates: Groq retries `RateLimitError` /
|
|
457
|
+
`APITimeoutError` / `APIConnectionError`; Google retries 429s and 5xxs;
|
|
458
|
+
Ollama retries `httpx.TimeoutException` / `ConnectError` + 429/5xx
|
|
459
|
+
`ResponseError`
|
|
460
|
+
- Config via `COMPACTBENCH_GROQ_API_KEY`,
|
|
461
|
+
`COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY`, `COMPACTBENCH_OLLAMA_BASE_URL`
|
|
462
|
+
- Cross-provider contract tests ensure identical response shape
|
|
463
|
+
- Real provider SDKs stay optional (`pip install 'compactbench[providers]'`);
|
|
464
|
+
importing a provider without its SDK raises a clean `ProviderError`
|
|
465
|
+
- Built-in compactors + mock provider (WO-005):
|
|
466
|
+
- Four baselines in `compactbench.compactors`: `naive-summary`,
|
|
467
|
+
`structured-state`, `hierarchical-summary`, `hybrid-ledger`
|
|
468
|
+
- All inherit async `Compactor` ABC bound to `(provider, model)` at
|
|
469
|
+
construction
|
|
470
|
+
- Shared JSON → `StructuredState` parser with code-fence stripping and
|
|
471
|
+
lenient field coercion
|
|
472
|
+
- `MockProvider` with scripted-sequence + default-response modes and call
|
|
473
|
+
recording for tests
|
|
474
|
+
- Registries: `list_built_ins()` / `get_built_in()` for compactors and
|
|
475
|
+
`list_providers()` / `get_provider_cls()` for providers
|
|
476
|
+
- `compactbench providers list` wired up
|
|
477
|
+
- Scoring engine (WO-004):
|
|
478
|
+
- Per-item checks: `contains_normalized`, `forbidden_absent`, `exact`, `set_match`
|
|
479
|
+
- Weighted cycle aggregation per decisions.md §B3 item weights
|
|
480
|
+
- Contradiction detection (item-aware: recall items excluded)
|
|
481
|
+
- Compression ratio via `cl100k_base` tokenizer
|
|
482
|
+
- Drift resistance from cross-cycle scores (clamped to `[0, 1]`)
|
|
483
|
+
- `compactbench score --results results.jsonl` wired up (reads JSONL of
|
|
484
|
+
`{case, artifact, responses}` records, prints per-case + aggregate summary)
|
|
485
|
+
- Case generation engine (WO-003):
|
|
486
|
+
- `compactbench.engine.generate_case` — pure function of
|
|
487
|
+
(template, seed, difficulty) → `GeneratedCase`
|
|
488
|
+
- Case-seed derivation from `(suite_version, seed_group, case_slot)` via SHA-256
|
|
489
|
+
- Difficulty policy application (distractor count, paraphrase depth,
|
|
490
|
+
override timing exposed as `difficulty.*` bindings)
|
|
491
|
+
- Distractor-turn generator (seeded, alternating user/assistant)
|
|
492
|
+
- Variable substitution across transcript, ground truth, and evaluation items
|
|
493
|
+
- `compactbench generate --template <key> --seed <int>` command wired up
|
|
494
|
+
- Three regression fixtures pinning the starter templates at seed=42, medium
|
|
495
|
+
|
|
496
|
+
[Unreleased]: https://github.com/compactbench/compactbench/compare/v0.3.0...HEAD
|
|
497
|
+
[0.3.0]: https://github.com/compactbench/compactbench/compare/v0.1.0...v0.3.0
|
|
498
|
+
[0.1.0]: https://github.com/compactbench/compactbench/releases/tag/v0.1.0
|