compactbench 0.1.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. {compactbench-0.1.0 → compactbench-0.3.0}/.gitignore +9 -4
  2. compactbench-0.3.0/CHANGELOG.md +498 -0
  3. compactbench-0.3.0/PKG-INFO +198 -0
  4. compactbench-0.3.0/README.md +136 -0
  5. compactbench-0.3.0/THIRD_PARTY_NOTICES.md +50 -0
  6. {compactbench-0.1.0 → compactbench-0.3.0}/pyproject.toml +36 -14
  7. compactbench-0.3.0/src/compactbench/check.py +238 -0
  8. compactbench-0.3.0/src/compactbench/cli.py +922 -0
  9. compactbench-0.3.0/src/compactbench/compactors/__init__.py +89 -0
  10. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/compactors/_state_parser.py +43 -23
  11. compactbench-0.3.0/src/compactbench/compactors/_utils.py +158 -0
  12. compactbench-0.3.0/src/compactbench/compactors/agent_defaults.py +333 -0
  13. compactbench-0.3.0/src/compactbench/compactors/base.py +58 -0
  14. compactbench-0.3.0/src/compactbench/compactors/controls.py +110 -0
  15. compactbench-0.3.0/src/compactbench/compactors/extractive.py +73 -0
  16. compactbench-0.3.0/src/compactbench/compactors/hierarchical_summary.py +136 -0
  17. compactbench-0.3.0/src/compactbench/compactors/hybrid_ledger.py +121 -0
  18. compactbench-0.3.0/src/compactbench/compactors/naive_summary.py +68 -0
  19. compactbench-0.3.0/src/compactbench/compactors/provider_native.py +86 -0
  20. compactbench-0.3.0/src/compactbench/compactors/sectioned_summary.py +74 -0
  21. compactbench-0.3.0/src/compactbench/compactors/structured_state.py +71 -0
  22. compactbench-0.3.0/src/compactbench/config.py +84 -0
  23. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/contracts/__init__.py +26 -10
  24. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/contracts/artifact.py +10 -4
  25. compactbench-0.3.0/src/compactbench/contracts/case.py +216 -0
  26. compactbench-0.3.0/src/compactbench/contracts/request.py +38 -0
  27. compactbench-0.3.0/src/compactbench/contracts/result.py +198 -0
  28. compactbench-0.3.0/src/compactbench/integrations/__init__.py +6 -0
  29. compactbench-0.3.0/src/compactbench/integrations/langchain.py +346 -0
  30. compactbench-0.3.0/src/compactbench/integrations/llamaindex.py +356 -0
  31. compactbench-0.3.0/src/compactbench/jsonutil.py +99 -0
  32. compactbench-0.3.0/src/compactbench/leaderboard/__init__.py +35 -0
  33. compactbench-0.3.0/src/compactbench/leaderboard/projection.py +211 -0
  34. compactbench-0.3.0/src/compactbench/leaderboard/qualification.py +73 -0
  35. compactbench-0.3.0/src/compactbench/leaderboard/ranking.py +39 -0
  36. compactbench-0.3.0/src/compactbench/leaderboard/season.py +80 -0
  37. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/__init__.py +13 -0
  38. compactbench-0.3.0/src/compactbench/providers/_retry.py +113 -0
  39. compactbench-0.3.0/src/compactbench/providers/anthropic.py +248 -0
  40. compactbench-0.3.0/src/compactbench/providers/base.py +106 -0
  41. compactbench-0.3.0/src/compactbench/providers/compatible.py +120 -0
  42. compactbench-0.3.0/src/compactbench/providers/counting.py +117 -0
  43. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/google_ai_studio.py +20 -7
  44. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/groq.py +25 -5
  45. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/mock.py +6 -1
  46. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/ollama.py +29 -7
  47. compactbench-0.3.0/src/compactbench/providers/openai.py +213 -0
  48. compactbench-0.3.0/src/compactbench/proxy.py +447 -0
  49. compactbench-0.3.0/src/compactbench/redaction.py +76 -0
  50. compactbench-0.3.0/src/compactbench/rendering.py +73 -0
  51. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/runner/__init__.py +19 -18
  52. compactbench-0.3.0/src/compactbench/runner/costs.py +120 -0
  53. compactbench-0.3.0/src/compactbench/runner/cycle.py +151 -0
  54. compactbench-0.3.0/src/compactbench/runner/estimate.py +167 -0
  55. compactbench-0.3.0/src/compactbench/runner/persistence.py +226 -0
  56. compactbench-0.3.0/src/compactbench/runner/reader.py +163 -0
  57. compactbench-0.3.0/src/compactbench/runner/run.py +375 -0
  58. compactbench-0.3.0/src/compactbench/runner/traces.py +70 -0
  59. compactbench-0.3.0/src/compactbench/sandbox.py +328 -0
  60. compactbench-0.3.0/src/compactbench/scenarios/__init__.py +74 -0
  61. compactbench-0.3.0/src/compactbench/scenarios/builder.py +229 -0
  62. compactbench-0.3.0/src/compactbench/scenarios/families/__init__.py +47 -0
  63. compactbench-0.3.0/src/compactbench/scenarios/families/base.py +135 -0
  64. compactbench-0.3.0/src/compactbench/scenarios/families/buried_constraint.py +452 -0
  65. compactbench-0.3.0/src/compactbench/scenarios/families/conditional_scope.py +297 -0
  66. compactbench-0.3.0/src/compactbench/scenarios/families/decision_override.py +358 -0
  67. compactbench-0.3.0/src/compactbench/scenarios/families/entity_confusion.py +303 -0
  68. compactbench-0.3.0/src/compactbench/scenarios/families/mixed.py +91 -0
  69. compactbench-0.3.0/src/compactbench/scenarios/families/numeric_precision.py +265 -0
  70. compactbench-0.3.0/src/compactbench/scenarios/families/provenance_authority.py +276 -0
  71. compactbench-0.3.0/src/compactbench/scenarios/families/reference_resolution.py +333 -0
  72. compactbench-0.3.0/src/compactbench/scenarios/families/task_continuity.py +276 -0
  73. compactbench-0.3.0/src/compactbench/scenarios/generate.py +214 -0
  74. compactbench-0.3.0/src/compactbench/scenarios/lexicon.py +1032 -0
  75. compactbench-0.3.0/src/compactbench/scenarios/noise.py +362 -0
  76. compactbench-0.3.0/src/compactbench/scenarios/probes.py +146 -0
  77. compactbench-0.3.0/src/compactbench/scenarios/profile.py +101 -0
  78. compactbench-0.3.0/src/compactbench/scenarios/rng.py +67 -0
  79. compactbench-0.3.0/src/compactbench/scenarios/world.py +102 -0
  80. compactbench-0.3.0/src/compactbench/scoring/__init__.py +24 -0
  81. compactbench-0.3.0/src/compactbench/scoring/answers.py +282 -0
  82. compactbench-0.3.0/src/compactbench/scoring/metrics.py +205 -0
  83. compactbench-0.3.0/src/compactbench/sealing.py +132 -0
  84. compactbench-0.3.0/src/compactbench/submission.py +198 -0
  85. compactbench-0.3.0/src/compactbench/tokens.py +46 -0
  86. compactbench-0.1.0/CHANGELOG.md +0 -136
  87. compactbench-0.1.0/PKG-INFO +0 -161
  88. compactbench-0.1.0/README.md +0 -111
  89. compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_1.yaml +0 -66
  90. compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_2.yaml +0 -61
  91. compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_3.yaml +0 -71
  92. compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_4.yaml +0 -61
  93. compactbench-0.1.0/benchmarks/public/elite_practice/buried_constraint_v1_5.yaml +0 -67
  94. compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_1.yaml +0 -63
  95. compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_2.yaml +0 -76
  96. compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_3.yaml +0 -72
  97. compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_4.yaml +0 -71
  98. compactbench-0.1.0/benchmarks/public/elite_practice/decision_override_v1_5.yaml +0 -63
  99. compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_1.yaml +0 -64
  100. compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_2.yaml +0 -69
  101. compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_3.yaml +0 -61
  102. compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_4.yaml +0 -66
  103. compactbench-0.1.0/benchmarks/public/elite_practice/entity_confusion_v1_5.yaml +0 -71
  104. compactbench-0.1.0/benchmarks/public/hard/.gitkeep +0 -0
  105. compactbench-0.1.0/benchmarks/public/starter/buried_constraint_starter_v1.yaml +0 -68
  106. compactbench-0.1.0/benchmarks/public/starter/decision_override_starter_v1.yaml +0 -63
  107. compactbench-0.1.0/benchmarks/public/starter/entity_confusion_starter_v1.yaml +0 -67
  108. compactbench-0.1.0/benchmarks/schemas/template-v1.json +0 -135
  109. compactbench-0.1.0/docs/architecture/_legacy-saas/README.md +0 -111
  110. compactbench-0.1.0/src/compactbench/cli.py +0 -343
  111. compactbench-0.1.0/src/compactbench/compactors/__init__.py +0 -47
  112. compactbench-0.1.0/src/compactbench/compactors/_utils.py +0 -37
  113. compactbench-0.1.0/src/compactbench/compactors/base.py +0 -38
  114. compactbench-0.1.0/src/compactbench/compactors/hierarchical_summary.py +0 -113
  115. compactbench-0.1.0/src/compactbench/compactors/hybrid_ledger.py +0 -115
  116. compactbench-0.1.0/src/compactbench/compactors/naive_summary.py +0 -54
  117. compactbench-0.1.0/src/compactbench/compactors/structured_state.py +0 -64
  118. compactbench-0.1.0/src/compactbench/config.py +0 -36
  119. compactbench-0.1.0/src/compactbench/contracts/case.py +0 -81
  120. compactbench-0.1.0/src/compactbench/contracts/result.py +0 -79
  121. compactbench-0.1.0/src/compactbench/dsl/__init__.py +0 -66
  122. compactbench-0.1.0/src/compactbench/dsl/errors.py +0 -23
  123. compactbench-0.1.0/src/compactbench/dsl/generators.py +0 -243
  124. compactbench-0.1.0/src/compactbench/dsl/models.py +0 -107
  125. compactbench-0.1.0/src/compactbench/dsl/parser.py +0 -55
  126. compactbench-0.1.0/src/compactbench/dsl/substitution.py +0 -32
  127. compactbench-0.1.0/src/compactbench/dsl/validator.py +0 -97
  128. compactbench-0.1.0/src/compactbench/engine/__init__.py +0 -29
  129. compactbench-0.1.0/src/compactbench/engine/difficulty.py +0 -54
  130. compactbench-0.1.0/src/compactbench/engine/distractors.py +0 -69
  131. compactbench-0.1.0/src/compactbench/engine/errors.py +0 -11
  132. compactbench-0.1.0/src/compactbench/engine/generation.py +0 -96
  133. compactbench-0.1.0/src/compactbench/engine/seeds.py +0 -31
  134. compactbench-0.1.0/src/compactbench/engine/transcript.py +0 -84
  135. compactbench-0.1.0/src/compactbench/leaderboard/__init__.py +0 -38
  136. compactbench-0.1.0/src/compactbench/leaderboard/projection.py +0 -103
  137. compactbench-0.1.0/src/compactbench/leaderboard/qualification.py +0 -90
  138. compactbench-0.1.0/src/compactbench/leaderboard/ranking.py +0 -71
  139. compactbench-0.1.0/src/compactbench/providers/_retry.py +0 -48
  140. compactbench-0.1.0/src/compactbench/providers/base.py +0 -44
  141. compactbench-0.1.0/src/compactbench/runner/continuation.py +0 -78
  142. compactbench-0.1.0/src/compactbench/runner/cycle.py +0 -67
  143. compactbench-0.1.0/src/compactbench/runner/evaluation.py +0 -71
  144. compactbench-0.1.0/src/compactbench/runner/persistence.py +0 -227
  145. compactbench-0.1.0/src/compactbench/runner/run.py +0 -257
  146. compactbench-0.1.0/src/compactbench/scoring/__init__.py +0 -48
  147. compactbench-0.1.0/src/compactbench/scoring/checks.py +0 -92
  148. compactbench-0.1.0/src/compactbench/scoring/compression.py +0 -66
  149. compactbench-0.1.0/src/compactbench/scoring/contradictions.py +0 -70
  150. compactbench-0.1.0/src/compactbench/scoring/drift.py +0 -30
  151. compactbench-0.1.0/src/compactbench/scoring/scorer.py +0 -77
  152. compactbench-0.1.0/submissions/README.md +0 -34
  153. compactbench-0.1.0/submissions/_template/README.md +0 -17
  154. {compactbench-0.1.0 → compactbench-0.3.0}/LICENSE +0 -0
  155. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/__init__.py +0 -0
  156. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/__main__.py +0 -0
  157. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/compactors/errors.py +0 -0
  158. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/leaderboard/errors.py +0 -0
  159. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/providers/errors.py +0 -0
  160. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/py.typed +0 -0
  161. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/runner/_resolver.py +0 -0
  162. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/runner/errors.py +0 -0
  163. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/scoring/errors.py +0 -0
  164. {compactbench-0.1.0 → compactbench-0.3.0}/src/compactbench/utils/__init__.py +0 -0
@@ -80,12 +80,17 @@ site/
80
80
  Thumbs.db
81
81
  desktop.ini
82
82
 
83
- # Local state
84
- data/
85
- results/
86
- runs/
83
+ # Local state (anchored to repo root — docs/data/ must remain tracked)
84
+ /data/
85
+ /results/
86
+ /runs/
87
+ /.smoke-runs/
87
88
  *.jsonl.gz
88
89
 
90
+ # Maintainer-only local tooling (kept local, never shipped)
91
+ /.local/
92
+ /smoke.bat
93
+
89
94
  # Hidden benchmark content — never commit
90
95
  benchmarks/hidden/
91
96
  benchmarks/**/_private/
@@ -0,0 +1,498 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.3.0] — 2026-11-16
11
+
12
+ Benchmark v2 — **scorer_version 2.0.0 → 3.0.0**. Case generation, the compaction loop and
13
+ scoring are rewritten. Scores are not comparable with 0.2; v1 leaderboard rows are retired
14
+ to `docs/data/archive/` and v1 baseline results to `baselines/archive/`.
15
+
16
+ 0.2.0 was never published to PyPI, so PyPI goes from 0.1.0 straight to 0.3.0; its changes
17
+ are listed under 0.2.0 below and included here.
18
+
19
+ ### Breaking
20
+
21
+ - **Compactor interface.** `compact(self, request: CompactionRequest)` replaces
22
+ `compact(self, transcript, config=None, previous_artifact=None)`. After the first cycle,
23
+ `request.turns` holds only the turns since the last compaction and
24
+ `request.previous_artifact` holds the method's previous output — the earlier turns are
25
+ never resent. `config` moved to `self.config`. Classes with the old signature fail to
26
+ load with a migration message. LangChain and LlamaIndex adapters follow the same shape
27
+ (the previous artifact arrives as a leading system message).
28
+ - **YAML templates and the DSL are removed.** Cases come from scenario families
29
+ (`compactbench.scenarios`); suites are `starter` and `practice`. `generate` takes
30
+ `--family`; `run` takes `--cases` instead of `--case-count`; `--benchmarks-dir` is gone.
31
+ - **Results files use a new format** (`results_schema: "2"`). 0.2 files can't be read.
32
+ - **Submissions move to the website.** Ranked runs are submitted at compactbench.com/submit
33
+ instead of by pull request; `submissions/<handle>/<method>/config.yaml` is gone, and the
34
+ season pins every evaluation setting.
35
+
36
+ ### Added
37
+
38
+ - **Hugging Face export** (`scripts/hf_dataset.py`): the public `practice`, `starter` and
39
+ `drills` suites, exactly the cases `compactbench run` uses, as JSONL configs with a dataset
40
+ card (seeds stored as strings so Arrow can hold them). Nothing from the hidden set.
41
+ - **Claude Code plugin** (`plugins/compactbench`, marketplace in `.claude-plugin/`): the
42
+ `forget-less` skill writes compaction instructions from the Season 1 findings for Claude
43
+ Code, Codex CLI, LangChain or LlamaIndex; the `measure-compaction` skill wraps a method,
44
+ estimates the cost, runs `compactbench check` against a baseline on the user's keys, and can
45
+ add the GitHub Action.
46
+ - **`compactbench check`** — runs a small, fixed Practice sample (4 cases, seed group `ci`,
47
+ Season 1 budget and reader by default) on your keys, prints a Markdown report (score and
48
+ interval, score after each compaction, per family) and compares it with a saved baseline:
49
+ `--max-drop` and `--min-score` fail the run on a regression, `--update-baseline` records a
50
+ new one. Baselines from different settings are refused rather than compared.
51
+ - **Agents' default compaction** as built-ins, reproduced from their open-source code at
52
+ pinned commits: `codex-cli` (Codex CLI's handoff-summary prompt, recent user messages
53
+ kept verbatim ahead of the prefixed summary) and `langchain-summarization` (LangChain's
54
+ `SummarizationMiddleware` defaults: keep the last 20 messages, summarize the rest after
55
+ trimming it to its last ~4,000 tokens). Under a budget, verbatim messages get at most a
56
+ quarter of it. Attribution in `THIRD_PARTY_NOTICES.md`.
57
+ - **"Forget less" guide** (`docs/forget-less.md`): what Season 1 shows about compaction
58
+ instructions, with paste-in instructions for Claude Code, Codex CLI, LangChain and
59
+ LlamaIndex and a recipe to measure them.
60
+ - **GitHub Action** (`action.yml`) wrapping `compactbench check`: job summary, an optional
61
+ pull-request comment kept up to date, and `score` / `change` / `passed` outputs.
62
+ - **Scenario families**: `buried_constraint`, `decision_override`, `entity_confusion`,
63
+ `reference_resolution`, `numeric_precision`, `conditional_scope`, `task_continuity` and
64
+ `provenance_authority` — long, multi-cycle conversations (up to ~32k-token openings and
65
+ 4 drift cycles at `elite`) where later cycles add and lift rules, override and revert
66
+ decisions, hand over ownership, switch choices by reference, correct exact figures,
67
+ change a rule's scope, move tickets across a board, and slip contradicting claims in
68
+ through fetched documents. On-topic procedural
69
+ filler (code, tests, logs, API responses, git history, metrics, notes), decoys built from
70
+ the case, and "never settled" questions. Deterministic and platform-stable.
71
+ - **Structured probes, scored mechanically** — single choice, select-all, exact value
72
+ (money, date, number, version, identifier, text normalization) and list. No LLM judge;
73
+ hedging can't pay.
74
+ - **A separate reader model** (`--reader-provider` / `--reader-model`) answers each cycle's
75
+ probes in batches, from the artifact only, fenced as untrusted data, with a JSON schema
76
+ where the provider supports one.
77
+ - **Metrics**: family-balanced score with a 95% bootstrap interval, score by cycle, drift
78
+ resistance and decay on carried probes, stale-answer rate, trap rate (decoy-only
79
+ answers: other projects' rules, rejected proposals, injected values), hallucination rate,
80
+ per-family and per-category breakdowns, artifact tokens. Abstention probes count only
81
+ when missed, so an empty artifact scores 0.
82
+ - **Controls on the same cases** (`--controls`): oracle and null scores per cycle and a
83
+ normalized score per family. The `oracle`, `null` and `truncate-last-n` built-ins are
84
+ ported to the new loop (`truncate-last-n` now keeps as many recent turns as fit the budget).
85
+ - **Token budgets** (`--budget`, default 2048): over-budget artifacts are cut before the
86
+ reader sees them and flagged; ranked runs reject them.
87
+ - **Traces** (`--traces`): every artifact, question and raw reader reply, per cycle.
88
+ - **Seasons** (`--season`): a file pinning suite, difficulty, cycles, cases, budget, reader
89
+ and controls, with fixed or fresh-per-run sampling; `qualify --season` checks a run
90
+ against one.
91
+ - **Suite packs** (`--suite-pack`): load private families and worlds — vocabulary,
92
+ filler and the wording of every sentence the families write (`Lexicon.phrases`) — and
93
+ spread a suite's cases across several worlds (`Suite.settings`).
94
+ - **Mixed sessions** (`mixed`): every family in one conversation, sharing people,
95
+ components and one token budget; probes are tagged with their family, so scores still
96
+ break down per family. `practice` (and ranked Elite) are mixed sessions; the new
97
+ `drills` suite keeps one family per case for focused work. Families that draw facts
98
+ from the same vocabulary pool get disjoint shares of it.
99
+ - **Sandboxed ranked evaluation.** `run --sandbox-image` runs the method in a container
100
+ (no network but the model proxy, read-only filesystem, no capabilities, resource
101
+ limits) and talks to it one request at a time over stdin/stdout; submitted code is never
102
+ imported by the harness. `compactbench proxy` holds the provider key, allows only
103
+ declared models and model-call endpoints, refuses streaming, enforces a token cap and
104
+ logs metadata only, and refuses requests that would have the provider reach the
105
+ internet for the method (server-side tools, MCP servers, content by URL). Keys are
106
+ sealed to the season's public key (X25519, HKDF-SHA256, AES-256-GCM), bound to their
107
+ submission. The container runs under gVisor in CI. Dockerfiles, submission terms,
108
+ privacy policy and a datasheet.
109
+ - **Self-serve ranked submissions.** Sign in with GitHub on the site, upload `method.py`,
110
+ pick the model and add your keys; the Worker (D1 + R2) validates the submission, seals
111
+ the keys on arrival, applies per-account and daily limits (with an optional Turnstile
112
+ check and a kill switch) and starts the evaluation. Qualifying runs
113
+ publish themselves — method source, public settings, scores and a commitment to the
114
+ sealed traces under `submissions/<handle>/<method>/` — and the site redeploys. A status
115
+ page follows each run and says why a run didn't qualify. Runs are recorded under the
116
+ name and version given on the form, and method pages include the submitter's handle. Submitters pay for their method's calls and the reader.
117
+ - **Published results are redacted** (`compactbench.redaction`). Results files from a
118
+ hidden suite carry probe keys named after the private world's people, rules and
119
+ tickets, and each cycle carries the method's own warning text. Published copies (ranked
120
+ submissions and baselines) replace keys with salted hashes — equal within a case, so
121
+ every metric is unchanged — and drop method text; the unredacted results are sealed
122
+ with the traces. A test fails if an unredacted hidden-suite file is committed.
123
+ - **Any model.** Hosted OpenAI-compatible providers `openrouter`, `deepseek`, `mistral`,
124
+ `together`, `fireworks`, `xai` and `gemini` (keys from `COMPACTBENCH_<NAME>_API_KEY`),
125
+ and `custom`: a submitter's own OpenAI-compatible HTTPS endpoint, reached only through
126
+ the proxy and only at a public address, with an optional display name shown on the
127
+ leaderboard next to a "custom endpoint" label.
128
+ - `Suite.probes_per_case`: keep a fixed, family-balanced subset of each case's probes,
129
+ asked at every compaction, so a run can afford many more cases. The score interval
130
+ resamples whole cases, so a mixed case's families move together.
131
+ - **Owner dashboard** (`admin/`, Cloudflare Worker behind Cloudflare Access): website
132
+ submissions with stuck-run flags, ranked evaluation runs, the published leaderboard,
133
+ per-family difficulty, review flags for unusual submissions and a CSV export; read-only. The Worker
134
+ verifies the Access token itself (owner email, audience, issuer, expiry, signature).
135
+ - **Remote MCP server** at `/mcp` on the site: `get_leaderboard`, `get_method`,
136
+ `compare_methods`, `list_families`, `explain_metric`, and a self-test for AIs
137
+ (`selftest_list`, `selftest_turns`, `selftest_questions`, `selftest_score`): compact a
138
+ public case with your own model and get a mechanical score (`scripts/selftest_cases.py`).
139
+ - `scripts/run_baselines.py` (season roster, `--parallel`, `--estimate`) and
140
+ `scripts/season_calibration.py` (per-family ceiling and floor for the Seasons page).
141
+ - Maintainer tools: `scripts/item_analysis.py` (per-template validity, difficulty and
142
+ discrimination), `scripts/qa_sheet.py` (a human review sheet for a suite's questions),
143
+ `scripts/reader_robustness.py` (re-read traced artifacts with a second reader and
144
+ compare method orderings) and `scripts/failure_gallery.py`.
145
+ - Per-probe control scores (`oracle_score`, `null_score` on each probe result), so any
146
+ grouping has its own ceiling and floor.
147
+ - **Website** (`web/`, Astro, deployed to Cloudflare Workers static assets): home,
148
+ leaderboard (sortable, per-family heat map, score-by-compaction sparklines, intervals),
149
+ methodology, families, submit, privacy and terms pages rendered from `docs/`, a page per
150
+ method (score by compaction, by family, run details), a two-method comparison, and a
151
+ score-versus-cost chart, plus
152
+ `llms.txt`, `llms-full.txt`, a Markdown twin of every page, JSON endpoints
153
+ (`/api/leaderboard.json`, `/api/methods/<slug>.json`, `/api/families.json`,
154
+ `/api/season.json`) with an OpenAPI description, a CSV download, JSON-LD, share images
155
+ (one per method, with its score and score by compaction), security headers with a strict
156
+ Content-Security-Policy, the developer docs under `/docs`, and a failure gallery of
157
+ real failures from public Practice runs.
158
+ `scripts/export_site_data.py` feeds it from the package so the site never drifts.
159
+ - **Season 1** (`seasons/s1.yaml`): Elite 2.0.1, 4 drift cycles, 48 mixed cases of 20 probes,
160
+ 1,024-token budget (Tight), `claude-haiku-5-5` reader at high effort, fresh sampling;
161
+ controls are measured once per season on a calibration sample rather than on every run.
162
+ - **`sectioned-summary`** built-in: the multi-section summary format coding agents
163
+ commonly compact into, with a generic prompt. **`extractive`**: keeps pattern-matched
164
+ state sentences, newest first within the budget, with no model calls.
165
+ - Larger public vocabulary (26 rules, 25 decision topics, 14 facts, 15 ticket titles) for
166
+ more varied cases.
167
+ - **`anthropic-compaction`** built-in: Anthropic's server-side on-demand compaction
168
+ (beta `compact-2026-09-04`) run as shipped — default prompt, compacting again from the
169
+ signed block each cycle. OpenAI's Responses compaction returns an encrypted item that a
170
+ reader from another provider can't read, so it has no built-in.
171
+ - **Reader effort** (`--reader-effort`, and `reader.effort` in seasons) is recorded with
172
+ the run and is part of the leaderboard segment.
173
+ - Built-in methods size their output for the budget (with room for thinking), trim
174
+ their own artifact to fit it, oldest entries first, and use `low` reasoning effort
175
+ unless `effort` is set in their config.
176
+ - `run --method-config` passes a JSON object to the method as `self.config`.
177
+ - `compactbench.compactors.fit_to_budget` is public: trims an artifact to the token budget
178
+ the way the built-ins do. The submission template uses it, asks for low reasoning effort
179
+ and leaves output room for thinking, so it qualifies out of the box.
180
+ - `families list`; `generate --text` prints a readable case.
181
+
182
+ ### Changed
183
+
184
+ - Qualification: control arms never ranked; incomplete runs, runs without drift cycles or
185
+ a budget, over-budget cycles, more than 5% unusable reader answers, and readers on a
186
+ custom endpoint are rejected; season settings must match.
187
+ - Ranking: by score within suite, scorer, reader and budget; ties on drift resistance,
188
+ then stale-answer rate. A rank is one plus the number of rows whose 95% interval lies
189
+ entirely above, so rows that can't be told apart share a rank. Leaderboard rows carry CIs, per-family scores and score by cycle.
190
+ - `--estimate` projects the method, reader and control costs separately.
191
+ - Rate-limited calls (HTTP 429) get up to 8 attempts and wait at least the server's
192
+ `retry-after`.
193
+ - Providers no longer send sampling parameters by default, and gain `json_schema` and
194
+ `reasoning_effort` on `CompletionRequest`. The Anthropic SDK moves to 2.x (API version
195
+ `2026-10-21`); 529 responses are now retried; refusals and empty token-limit stops raise
196
+ clear errors. Cost catalogue refreshed.
197
+ - The ranked evaluation workflow is rebuilt around the sandbox and started by the site:
198
+ an untrusted build with no secrets, an evaluation in a protected environment, and a
199
+ publish job that commits the results and redeploys the site. The site's internal
200
+ endpoints accept only that workflow's OIDC token from `main`.
201
+ - Dependabot uses the `uv` ecosystem and batches security updates.
202
+
203
+ ### Removed
204
+
205
+ - `compactbench.dsl`, `compactbench.engine`, the v1 scorer (`checks`, `contradictions`,
206
+ `drift`, `compression`, `diagnostics`), model-generated continuation turns, and
207
+ `benchmarks/public` / `benchmarks/schemas`.
208
+ - The pull-request submission flow: `submissions/_template/`, `validate_submissions.py`,
209
+ `validate_submission_config.py` and the separate publish workflow.
210
+
211
+ ## 0.2.0 — 2026-08-24 (not published to PyPI)
212
+
213
+ ### Scoring integrity — **scorer_version 1.0.0 → 2.0.0**
214
+
215
+ Scores produced by scorer 1.0.0 are **not comparable** to these. The leaderboard
216
+ segments on `scorer_version`, so old and new rows never mix. Four defects meant a
217
+ method that did nothing intelligent would have topped the first leaderboard.
218
+
219
+ - **Compaction methods no longer receive the answer key.** Generated turns carry
220
+ `tags` — `distractor` on filler, `critical_constraint` on the turn holding the
221
+ answer — and the runner handed them straight to the method. A five-line
222
+ compactor with no model call that keeps the non-`distractor` turns recovered
223
+ 100% of the recall items at ~9x compression. Tags are now stripped at the
224
+ compaction boundary (this also closes the same leak through the LangChain and
225
+ LlamaIndex adapters); they remain on the case for diagnostics.
226
+ - **`drift_resistance` no longer rewards consistent failure.** It was
227
+ `clamp(1 + mean(score[n] - score[0]))` — a measure of whether scores *change*,
228
+ not whether they are any good — so a method scoring 0.0 on every cycle was
229
+ perfectly stable and scored **1.0**, collecting 30% of `elite_score` for the
230
+ worst possible result. It is now a retention ratio,
231
+ `mean(later) / first`, and is 0.0 when unmeasurable (fewer than two cycles, or
232
+ a zero baseline) rather than a free 1.0. Empty and crashed runs aggregate to
233
+ 0.0 drift resistance instead of 1.0.
234
+ - **The JSON state parser recovers objects wrapped in prose.** It required the
235
+ response to be bare JSON or a fenced block, so `Here is the state: {...}`, a
236
+ trailing `Hope that helps!`, and `<think>` preambles all produced a fully
237
+ empty state that scored as total information loss — measuring the parser
238
+ rather than the method, and worst on exactly the small and local models the
239
+ benchmark should be free to run on. Brace matching is string- and
240
+ escape-aware, so quoted code containing `}` no longer truncates the object.
241
+ - **Qualification closes four routes to an unearned rank.** Control arms are
242
+ never ranked; compression above 50x is rejected as a degenerate/empty artifact
243
+ (the `null` arm measures ~450x and previously cleared every tier and maxed the
244
+ compression bonus); runs configured with zero drift cycles are unrankable,
245
+ since drift is 30% of the score and they never measured it; and a run whose
246
+ results file has no `run_end` event is rejected, so a crashed run cannot
247
+ publish a partial score and stopping early is not a way to cherry-pick.
248
+
249
+ ### Security — submission evaluation pipeline
250
+
251
+ - **The evaluation workflow could never succeed, and reported it as the submitter's fault.**
252
+ It checked out the PR head at the default `fetch-depth: 1`, then ran
253
+ `git diff <base> <head>`. The base commit does not exist in a shallow clone, so
254
+ the diff failed, the failure was swallowed by `|| true`, and the job exited with
255
+ "PR does not touch any submissions/<handle>/<method>/ directory". Every
256
+ submission would have been turned away with a misleading error. Now `fetch-depth: 0`.
257
+ - **The hidden ranked set was exfiltrable with no code execution.** The config step
258
+ interpolated five unvalidated submitter-controlled YAML values into a file and
259
+ appended it to `$GITHUB_ENV`. A newline in `runtime.model` injected arbitrary
260
+ job-wide environment variables; since `OllamaProvider` passes
261
+ `COMPACTBENCH_OLLAMA_BASE_URL` straight to its client, a submission with
262
+ `provider: ollama` would have caused the runner to POST **every hidden ranked
263
+ case to an attacker-controlled host as a prompt**. No `method.py` runs on that
264
+ path, so sandboxing submitter code would not have closed it. All values now go
265
+ through `scripts/validate_submission_config.py` — allow-listed provider,
266
+ whitespace-free model pattern, bounded integers, control characters rejected,
267
+ and `class_path` required to sit inside the submission directory the PR
268
+ actually touched. Values never transit `$GITHUB_ENV`; the runner reads them
269
+ from validated JSON. 35 tests cover it, including the concrete attack payload.
270
+ - **Script injection in the score comment.** `body: \`${{ steps.summary.outputs.body }}\``
271
+ placed submitter-influenced text (the method name reaches it via
272
+ `compactbench score` output) inside a JS template literal, where a backtick or
273
+ `${...}` escapes into arbitrary JavaScript in a step holding a write-scoped
274
+ `GITHUB_TOKEN`. Passed via `process.env` instead.
275
+ - **The hidden-set token no longer persists on disk.** It was embedded in the
276
+ clone URL, which git writes verbatim into `/tmp/hidden/.git/config` — leaving a
277
+ credential that regenerates the entire hidden set readable by submitter code
278
+ executing later in the same job. Now passed via `GIT_ASKPASS`, with the `.git`
279
+ directory removed once the working tree exists.
280
+ - `scripts/validate_submissions.py` now validates every submission config, not
281
+ just greps for leftover `FILL_ME` placeholders, so contributors get the error
282
+ on their PR instead of after a maintainer applies the `evaluate` label.
283
+ - `ollama` and `mock` are no longer accepted as ranked-evaluation providers. Both
284
+ can be pointed at an arbitrary host, and the evaluator must control where calls go.
285
+
286
+ ### Added — control arms
287
+ - `oracle`, `null`, and `truncate-last-n` built-ins. None makes a model call, so
288
+ they are free and deterministic to run alongside any experiment. A score in
289
+ isolation is uninterpretable: the oracle bounds what is achievable when
290
+ compaction loses nothing, and the null arm exposes how many points are
291
+ available with no information at all. Measured against a real model, the
292
+ oracle scores **0.650**, not 1.0 — so a method scoring 0.5 has not lost half
293
+ the information, it is at 77% of the achievable ceiling.
294
+ - `compactbench qualify --results <file>` — runs the leaderboard floors locally
295
+ and prints every reason a run would be rejected. Previously a submitter could
296
+ only discover a disqualification by opening a PR and waiting for a maintainer,
297
+ after having already spent the API budget on the run.
298
+ - `CompactionArtifact` accepts either Python field names or the camelCase wire
299
+ aliases (`populate_by_name`). `summary_text=` previously raised
300
+ "Extra inputs are not permitted" while `summaryText=` worked — a confusing
301
+ first failure for anyone writing a method, which the docs' own example
302
+ half-tripped over by mixing the two styles.
303
+
304
+ Four months of merged work that was never released, plus the packaging fix that
305
+ makes it reachable. **If you installed 0.1.0, upgrade** — that wheel shipped
306
+ without any benchmark content, so `compactbench suites list` failed on a clean
307
+ install and no benchmark could be run from PyPI at all.
308
+
309
+ ### Fixed
310
+ - **The published wheel now contains the benchmark suites.** The `force-include`
311
+ mapping that bundles `benchmarks/public` landed in #21, *after* the v0.1.0 tag,
312
+ so the released artifact had zero templates and the documented first command
313
+ exited 1. CI and the release workflow now install the built wheel into a clean
314
+ environment and assert the CLI can see `starter` and `elite_practice`, so this
315
+ class of packaging regression cannot ship again.
316
+ - `.env` is now actually read. Every provider reads `os.environ` directly, and
317
+ the `Settings` object that parsed `.env` was imported by nothing — the
318
+ documented way to supply API keys silently did nothing. The CLI now loads
319
+ `COMPACTBENCH_*` variables from `.env` before any provider is constructed.
320
+ A real environment variable still wins over the file.
321
+ - An empty `COMPACTBENCH_OPENAI_BASE_URL` no longer marks a genuine OpenAI run
322
+ as self-hosted. The client was gated on truthiness and the provenance flag on
323
+ identity, so `""` produced a run stamped as having gone somewhere it hadn't.
324
+ - A base URL without an `http://` or `https://` scheme is now rejected at
325
+ construction instead of failing much later with an opaque connection error.
326
+ - `uv.lock` regenerated (90 → 145 packages; it had not been touched since the
327
+ initial scaffold and was missing the `anthropic` and `openai` entries
328
+ entirely). All workflows now install with `--locked`, so CI fails on lock
329
+ drift rather than silently re-resolving against live PyPI on every run — which
330
+ matters for a project whose central claim is reproducibility.
331
+ - The test suite no longer inherits the developer's environment: an autouse
332
+ fixture clears `COMPACTBENCH_*`, fixing `test_requires_api_key` failing for
333
+ anyone with a base URL exported (i.e. exactly the local-model users).
334
+
335
+ ### Added
336
+ - **Endpoint provenance.** `run_start` and `RunResult` now carry `endpoint_kind`
337
+ (`"default"` or `"custom"`), sourced from the provider, and it is part of the
338
+ leaderboard's ranking segment. Without it a locally-served model submitted as
339
+ `--provider openai --model gpt-4o` was ranked against genuine gpt-4o — the
340
+ provider and model fields are both caller-supplied strings. `--resume` also
341
+ checks it, so a run cannot be half-executed against a hosted model and
342
+ finished against a local one under the same alias. The URL itself is never
343
+ stored; results files get shared and the host is often internal.
344
+ - `OpenAIProvider` accepts a `base_url` (constructor argument or
345
+ `COMPACTBENCH_OPENAI_BASE_URL`), so the `openai` provider also serves any
346
+ OpenAI-compatible endpoint — vLLM, llama.cpp server, LM Studio, Together,
347
+ Fireworks, OpenRouter (#37, thanks @jaaabir). When a base URL is set the API
348
+ key becomes optional, because self-hosted servers generally do not check one.
349
+ - Anthropic and OpenAI providers (#19).
350
+ - LangChain (#16) and LlamaIndex (#17) integration adapters.
351
+ - Zero-install Colab notebook (#18) with a CI smoke test that rebuilds it from
352
+ its generator and executes every embedded command against the mock provider.
353
+ - Case-level parallelism and a `--estimate` cost projector (#22).
354
+ - Prompt caching via `cached_prefix` on `CompletionRequest` (#23).
355
+ - `reference_resolution` template family (#27) — a fourth family, 5 templates —
356
+ and a per-item-type diagnostic breakdown in `compactbench score`.
357
+ - Per-cycle / per-case / per-run `TokenUsage` telemetry (#28).
358
+
359
+ ### Changed
360
+ - Cost catalogue refreshed and made self-policing: `--estimate` now prints the
361
+ price vintage alongside the dollar figure, and a test fails once the catalogue
362
+ is more than 120 days old. Anthropic entries updated to the current lineup;
363
+ the previous table priced models that have since been superseded and had no
364
+ entry for any current one, so `--estimate` either quoted stale rates or
365
+ refused to answer.
366
+ - Pinned GitHub Action SHAs bumped (#34).
367
+ - Docs corrected to describe the project that actually exists: four template
368
+ families and 20 practice templates (not three and 15), five real providers
369
+ (not three), and `~900` calls for a full Elite practice run (not `~450`).
370
+
371
+ ## [0.1.0] — 2026-04-17
372
+
373
+ First public release. The full v1 stack: DSL parser, case-generation engine,
374
+ scoring engine, mock + real providers, built-in compactors, end-to-end runner,
375
+ submission pipeline with public leaderboard, and 15 Elite practice templates
376
+ across three launch families.
377
+
378
+ ### Added
379
+ - Initial project scaffold: Python package layout, CLI entrypoint, core ABCs
380
+ - Apache 2.0 license
381
+ - Contributor guide and code of conduct
382
+ - MkDocs documentation skeleton
383
+ - GitHub Actions CI workflow
384
+ - Template DSL (WO-002):
385
+ - YAML + Handlebars-style `{{variable}}` substitution
386
+ - Pydantic models for `TemplateDefinition` and sub-structures
387
+ - Seeded generator registry: `person_name`, `action_phrase`, `project_noun`,
388
+ `org_name`, `date_iso`, `amount_usd`, `product_sku`
389
+ - SHA-256-based sub-seed derivation for reordering-stable determinism
390
+ - Semantic validator for references, generators, and duplicate names
391
+ - JSON Schema at `benchmarks/schemas/template-v1.json`
392
+ - Three starter templates: `buried_constraint_starter_v1`,
393
+ `decision_override_starter_v1`, `entity_confusion_starter_v1`
394
+ - `compactbench suites list` command wired up
395
+ - Elite v1 public practice templates (WO-009):
396
+ - 15 templates in `benchmarks/public/elite_practice/`, 5 variations each for
397
+ the three launch families: `buried_constraint_v1`, `decision_override_v1`,
398
+ `entity_confusion_v1`
399
+ - Variations within each family probe different failure modes (e.g.,
400
+ indirect phrasing, double constraints, negative examples, sandwich
401
+ layouts for `buried_constraint`; simple/three-way/partial/late/meta-reset
402
+ overrides for `decision_override`; 2-person/3-person/same-role/
403
+ observer-vs-owner/shared-project for `entity_confusion`)
404
+ - Difficulty policy scales distractor turns 4 → 8 → 16 → 32 from easy to
405
+ elite, matching the ranked set's expected adversarial load
406
+ - `compactbench suites list` now shows `elite_practice` with 15 templates
407
+ across 3 families
408
+ - `docs/governance/elite-versions.md` — active-version table, version
409
+ policy, retirement policy, anti-gaming reminders, v1 changelog
410
+ - 9 new unit tests verify every template parses, validates, and generates
411
+ at every configured difficulty
412
+ - Hidden ranked set (20 per family in `compactbench/compactbench-hidden`)
413
+ lands in a follow-up so it doesn't commingle public and hidden content
414
+ - Submission pipeline + leaderboard (WO-008):
415
+ - `src/compactbench/leaderboard/` — `elite_score` computation, tie-breakers,
416
+ qualification floors (with rejection reasons), `RunResult` → public
417
+ `LeaderboardRow` projection, best-first rank assignment
418
+ - `.github/workflows/evaluate-submission.yml` — runs on GitHub-hosted
419
+ `ubuntu-latest`, gated on a maintainer-applied `evaluate` label; clones PR
420
+ head, installs submission deps, runs `compactbench run`, posts score
421
+ comment, uploads results artifact
422
+ - `.github/workflows/update-leaderboard.yml` — on push under `submissions/**`
423
+ rebuilds `docs/data/leaderboard.json` via `scripts/rebuild_leaderboard.py`
424
+ and commits; triggers the docs Pages redeploy
425
+ - `docs/leaderboard.md` — live table rendered client-side from
426
+ `data/leaderboard.json`, grouped by `(benchmark_version, target_model)`
427
+ - `submissions/_template/` — copy-paste scaffold (method.py, config.yaml,
428
+ requirements.txt, README.md)
429
+ - `compactbench submit` prints a submission checklist (full PR automation
430
+ stays server-side since hidden-set evaluation requires repo secrets)
431
+ - Runner infra decision updated: v1 uses GitHub-hosted runners, not
432
+ self-hosted; migration path preserved
433
+ - End-to-end runner (WO-007):
434
+ - `compactbench run --method <spec> --suite <key> --provider <k> --model <m>`
435
+ orchestrates: load suite → generate cases → per case iterate drift cycles
436
+ (compact → evaluate → score) → aggregate → persist results.jsonl
437
+ - Event-log JSONL format: `run_start`, one `case_complete` per case
438
+ streamed, final `run_end` with aggregates
439
+ - `--resume` continues from an existing output file; rejects with
440
+ `ResumeError` if run parameters differ
441
+ - `compactbench score --results` now pretty-prints the run summary from the
442
+ persisted events (reads, not re-scores)
443
+ - `--method` supports `built-in:<key>` and `<path.py>:<ClassName>`
444
+ (loads user compactors from file paths, verifies `Compactor` subclass)
445
+ - Drift cycles extend the transcript with seeded continuation turns; the
446
+ model's continuation response uses only the previous artifact as context
447
+ (that is the drift vector measured)
448
+ - Per-case `CaseResult` captures all cycles, case-level score, and
449
+ `drift_resistance`; run-level aggregates cover overall / drift /
450
+ constraint retention / contradiction / compression
451
+ - Real model providers (WO-006):
452
+ - `GroqProvider` — Llama 3.3 70B, Kimi K2, etc. via the `groq` SDK
453
+ - `GoogleAIStudioProvider` — Gemini 2.0 Flash etc. via the `google-genai` SDK
454
+ - `OllamaProvider` — local models via the `ollama` SDK
455
+ - Shared `retry_with_backoff` async helper with exponential + capped delay
456
+ - Per-provider retry predicates: Groq retries `RateLimitError` /
457
+ `APITimeoutError` / `APIConnectionError`; Google retries 429s and 5xxs;
458
+ Ollama retries `httpx.TimeoutException` / `ConnectError` + 429/5xx
459
+ `ResponseError`
460
+ - Config via `COMPACTBENCH_GROQ_API_KEY`,
461
+ `COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY`, `COMPACTBENCH_OLLAMA_BASE_URL`
462
+ - Cross-provider contract tests ensure identical response shape
463
+ - Real provider SDKs stay optional (`pip install 'compactbench[providers]'`);
464
+ importing a provider without its SDK raises a clean `ProviderError`
465
+ - Built-in compactors + mock provider (WO-005):
466
+ - Four baselines in `compactbench.compactors`: `naive-summary`,
467
+ `structured-state`, `hierarchical-summary`, `hybrid-ledger`
468
+ - All inherit async `Compactor` ABC bound to `(provider, model)` at
469
+ construction
470
+ - Shared JSON → `StructuredState` parser with code-fence stripping and
471
+ lenient field coercion
472
+ - `MockProvider` with scripted-sequence + default-response modes and call
473
+ recording for tests
474
+ - Registries: `list_built_ins()` / `get_built_in()` for compactors and
475
+ `list_providers()` / `get_provider_cls()` for providers
476
+ - `compactbench providers list` wired up
477
+ - Scoring engine (WO-004):
478
+ - Per-item checks: `contains_normalized`, `forbidden_absent`, `exact`, `set_match`
479
+ - Weighted cycle aggregation per decisions.md §B3 item weights
480
+ - Contradiction detection (item-aware: recall items excluded)
481
+ - Compression ratio via `cl100k_base` tokenizer
482
+ - Drift resistance from cross-cycle scores (clamped to `[0, 1]`)
483
+ - `compactbench score --results results.jsonl` wired up (reads JSONL of
484
+ `{case, artifact, responses}` records, prints per-case + aggregate summary)
485
+ - Case generation engine (WO-003):
486
+ - `compactbench.engine.generate_case` — pure function of
487
+ (template, seed, difficulty) → `GeneratedCase`
488
+ - Case-seed derivation from `(suite_version, seed_group, case_slot)` via SHA-256
489
+ - Difficulty policy application (distractor count, paraphrase depth,
490
+ override timing exposed as `difficulty.*` bindings)
491
+ - Distractor-turn generator (seeded, alternating user/assistant)
492
+ - Variable substitution across transcript, ground truth, and evaluation items
493
+ - `compactbench generate --template <key> --seed <int>` command wired up
494
+ - Three regression fixtures pinning the starter templates at seed=42, medium
495
+
496
+ [Unreleased]: https://github.com/compactbench/compactbench/compare/v0.3.0...HEAD
497
+ [0.3.0]: https://github.com/compactbench/compactbench/compare/v0.1.0...v0.3.0
498
+ [0.1.0]: https://github.com/compactbench/compactbench/releases/tag/v0.1.0