compactbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compactbench/__init__.py +10 -0
- compactbench/__main__.py +6 -0
- compactbench/cli.py +343 -0
- compactbench/compactors/__init__.py +47 -0
- compactbench/compactors/_state_parser.py +118 -0
- compactbench/compactors/_utils.py +37 -0
- compactbench/compactors/base.py +38 -0
- compactbench/compactors/errors.py +11 -0
- compactbench/compactors/hierarchical_summary.py +113 -0
- compactbench/compactors/hybrid_ledger.py +115 -0
- compactbench/compactors/naive_summary.py +54 -0
- compactbench/compactors/structured_state.py +64 -0
- compactbench/config.py +36 -0
- compactbench/contracts/__init__.py +41 -0
- compactbench/contracts/artifact.py +49 -0
- compactbench/contracts/case.py +81 -0
- compactbench/contracts/result.py +79 -0
- compactbench/dsl/__init__.py +66 -0
- compactbench/dsl/errors.py +23 -0
- compactbench/dsl/generators.py +243 -0
- compactbench/dsl/models.py +107 -0
- compactbench/dsl/parser.py +55 -0
- compactbench/dsl/substitution.py +32 -0
- compactbench/dsl/validator.py +97 -0
- compactbench/engine/__init__.py +29 -0
- compactbench/engine/difficulty.py +54 -0
- compactbench/engine/distractors.py +69 -0
- compactbench/engine/errors.py +11 -0
- compactbench/engine/generation.py +96 -0
- compactbench/engine/seeds.py +31 -0
- compactbench/engine/transcript.py +84 -0
- compactbench/leaderboard/__init__.py +38 -0
- compactbench/leaderboard/errors.py +11 -0
- compactbench/leaderboard/projection.py +103 -0
- compactbench/leaderboard/qualification.py +90 -0
- compactbench/leaderboard/ranking.py +71 -0
- compactbench/providers/__init__.py +52 -0
- compactbench/providers/_retry.py +48 -0
- compactbench/providers/base.py +44 -0
- compactbench/providers/errors.py +15 -0
- compactbench/providers/google_ai_studio.py +112 -0
- compactbench/providers/groq.py +95 -0
- compactbench/providers/mock.py +53 -0
- compactbench/providers/ollama.py +100 -0
- compactbench/py.typed +0 -0
- compactbench/runner/__init__.py +53 -0
- compactbench/runner/_resolver.py +57 -0
- compactbench/runner/continuation.py +78 -0
- compactbench/runner/cycle.py +67 -0
- compactbench/runner/errors.py +15 -0
- compactbench/runner/evaluation.py +71 -0
- compactbench/runner/persistence.py +227 -0
- compactbench/runner/run.py +257 -0
- compactbench/scoring/__init__.py +48 -0
- compactbench/scoring/checks.py +92 -0
- compactbench/scoring/compression.py +66 -0
- compactbench/scoring/contradictions.py +70 -0
- compactbench/scoring/drift.py +30 -0
- compactbench/scoring/errors.py +7 -0
- compactbench/scoring/scorer.py +77 -0
- compactbench/utils/__init__.py +1 -0
- compactbench-0.1.0.dist-info/METADATA +161 -0
- compactbench-0.1.0.dist-info/RECORD +66 -0
- compactbench-0.1.0.dist-info/WHEEL +4 -0
- compactbench-0.1.0.dist-info/entry_points.txt +2 -0
- compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
"""Top-level run orchestrator.
|
|
2
|
+
|
|
3
|
+
Loads a suite, generates one case per (template, seed slot), executes each
|
|
4
|
+
case through all drift cycles, scores everything, and writes a streamed
|
|
5
|
+
``results.jsonl`` event log.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import uuid
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from datetime import UTC, datetime
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from compactbench.compactors import Compactor
|
|
16
|
+
from compactbench.contracts import CaseResult, CycleResult, GeneratedCase
|
|
17
|
+
from compactbench.dsl import (
|
|
18
|
+
DifficultyLevel,
|
|
19
|
+
TemplateDefinition,
|
|
20
|
+
load_suite,
|
|
21
|
+
validate_template,
|
|
22
|
+
)
|
|
23
|
+
from compactbench.engine import derive_case_seed, generate_case
|
|
24
|
+
from compactbench.providers import Provider, get_provider_cls
|
|
25
|
+
from compactbench.runner._resolver import resolve_compactor_class
|
|
26
|
+
from compactbench.runner.cycle import execute_cycle
|
|
27
|
+
from compactbench.runner.errors import ResumeError, RunnerError
|
|
28
|
+
from compactbench.runner.persistence import (
|
|
29
|
+
SCORER_VERSION,
|
|
30
|
+
CaseCompleteEvent,
|
|
31
|
+
ResultsWriter,
|
|
32
|
+
RunEndEvent,
|
|
33
|
+
RunStartEvent,
|
|
34
|
+
aggregate_run_metrics,
|
|
35
|
+
completed_case_ids,
|
|
36
|
+
read_run_start,
|
|
37
|
+
)
|
|
38
|
+
from compactbench.scoring import drift_resistance
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class RunArgs:
|
|
43
|
+
method_spec: str
|
|
44
|
+
suite_key: str
|
|
45
|
+
provider_key: str
|
|
46
|
+
model: str
|
|
47
|
+
difficulty: DifficultyLevel
|
|
48
|
+
drift_cycles: int
|
|
49
|
+
case_count_per_template: int
|
|
50
|
+
seed_group: str
|
|
51
|
+
benchmarks_dir: Path
|
|
52
|
+
output_path: Path
|
|
53
|
+
resume: bool
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
async def run_experiment(args: RunArgs) -> Path:
|
|
57
|
+
"""Execute a full run per ``args`` and return the path to the written results file."""
|
|
58
|
+
suite_dir = args.benchmarks_dir / args.suite_key
|
|
59
|
+
if not suite_dir.is_dir():
|
|
60
|
+
raise RunnerError(f"suite directory not found: {suite_dir}")
|
|
61
|
+
|
|
62
|
+
templates = load_suite(suite_dir)
|
|
63
|
+
if not templates:
|
|
64
|
+
raise RunnerError(f"no templates in suite {args.suite_key!r}")
|
|
65
|
+
for template in templates:
|
|
66
|
+
validate_template(template)
|
|
67
|
+
|
|
68
|
+
compactor_cls = resolve_compactor_class(args.method_spec)
|
|
69
|
+
provider = _instantiate_provider(args.provider_key)
|
|
70
|
+
|
|
71
|
+
already_done: set[str] = set()
|
|
72
|
+
run_id: str
|
|
73
|
+
started_at: datetime
|
|
74
|
+
mode: str = "write"
|
|
75
|
+
|
|
76
|
+
if args.resume and args.output_path.exists():
|
|
77
|
+
existing = read_run_start(args.output_path)
|
|
78
|
+
if existing is None:
|
|
79
|
+
raise ResumeError(f"--resume passed but {args.output_path} has no run_start event")
|
|
80
|
+
_assert_resume_compatible(existing, args, compactor_cls)
|
|
81
|
+
already_done = completed_case_ids(args.output_path)
|
|
82
|
+
run_id = existing.run_id
|
|
83
|
+
started_at = existing.started_at
|
|
84
|
+
mode = "append"
|
|
85
|
+
else:
|
|
86
|
+
run_id = str(uuid.uuid4())
|
|
87
|
+
started_at = datetime.now(UTC)
|
|
88
|
+
|
|
89
|
+
suite_version = _suite_version(templates)
|
|
90
|
+
writer = ResultsWriter(args.output_path, mode=mode) # type: ignore[arg-type]
|
|
91
|
+
try:
|
|
92
|
+
if mode == "write":
|
|
93
|
+
writer.write(
|
|
94
|
+
RunStartEvent(
|
|
95
|
+
run_id=run_id,
|
|
96
|
+
method_name=compactor_cls.name,
|
|
97
|
+
method_version=compactor_cls.version,
|
|
98
|
+
suite_key=args.suite_key,
|
|
99
|
+
suite_version=suite_version,
|
|
100
|
+
scorer_version=SCORER_VERSION,
|
|
101
|
+
target_provider=args.provider_key,
|
|
102
|
+
target_model=args.model,
|
|
103
|
+
difficulty=args.difficulty.value,
|
|
104
|
+
drift_cycles=args.drift_cycles,
|
|
105
|
+
seed_group=args.seed_group,
|
|
106
|
+
case_count_per_template=args.case_count_per_template,
|
|
107
|
+
started_at=started_at,
|
|
108
|
+
)
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
completed_cases: list[CaseResult] = []
|
|
112
|
+
for template in templates:
|
|
113
|
+
for slot in range(args.case_count_per_template):
|
|
114
|
+
case_seed = derive_case_seed(
|
|
115
|
+
f"{args.suite_key}@{suite_version}", args.seed_group, slot
|
|
116
|
+
)
|
|
117
|
+
case = generate_case(template, case_seed, args.difficulty)
|
|
118
|
+
if case.case_id in already_done:
|
|
119
|
+
continue
|
|
120
|
+
|
|
121
|
+
compactor = compactor_cls(provider, args.model)
|
|
122
|
+
case_result = await _execute_case(
|
|
123
|
+
case=case,
|
|
124
|
+
compactor=compactor,
|
|
125
|
+
provider=provider,
|
|
126
|
+
model=args.model,
|
|
127
|
+
drift_cycles=args.drift_cycles,
|
|
128
|
+
case_seed=case_seed,
|
|
129
|
+
)
|
|
130
|
+
writer.write(CaseCompleteEvent(case_result=case_result))
|
|
131
|
+
completed_cases.append(case_result)
|
|
132
|
+
|
|
133
|
+
# For run-end aggregates, combine newly-executed cases with any that
|
|
134
|
+
# were already persisted from a resumed run.
|
|
135
|
+
from compactbench.runner.persistence import iter_events
|
|
136
|
+
|
|
137
|
+
all_cases: list[CaseResult] = list(completed_cases)
|
|
138
|
+
if already_done:
|
|
139
|
+
for event in iter_events(args.output_path):
|
|
140
|
+
if (
|
|
141
|
+
isinstance(event, CaseCompleteEvent)
|
|
142
|
+
and event.case_result.case_id in already_done
|
|
143
|
+
):
|
|
144
|
+
all_cases.append(event.case_result)
|
|
145
|
+
|
|
146
|
+
agg = aggregate_run_metrics(all_cases)
|
|
147
|
+
writer.write(
|
|
148
|
+
RunEndEvent(
|
|
149
|
+
completed_at=datetime.now(UTC),
|
|
150
|
+
overall_score=agg["overall_score"],
|
|
151
|
+
drift_resistance=agg["drift_resistance"],
|
|
152
|
+
constraint_retention=agg["constraint_retention"],
|
|
153
|
+
contradiction_rate=agg["contradiction_rate"],
|
|
154
|
+
compression_ratio=agg["compression_ratio"],
|
|
155
|
+
)
|
|
156
|
+
)
|
|
157
|
+
finally:
|
|
158
|
+
writer.close()
|
|
159
|
+
|
|
160
|
+
return args.output_path
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
async def _execute_case(
|
|
164
|
+
*,
|
|
165
|
+
case: GeneratedCase,
|
|
166
|
+
compactor: Compactor,
|
|
167
|
+
provider: Provider,
|
|
168
|
+
model: str,
|
|
169
|
+
drift_cycles: int,
|
|
170
|
+
case_seed: int,
|
|
171
|
+
) -> CaseResult:
|
|
172
|
+
from compactbench.contracts import CompactionArtifact, Transcript
|
|
173
|
+
|
|
174
|
+
transcript: Transcript = case.transcript
|
|
175
|
+
previous_artifact: CompactionArtifact | None = None
|
|
176
|
+
cycles: list[CycleResult] = []
|
|
177
|
+
cycle_scores: list[float] = []
|
|
178
|
+
|
|
179
|
+
for cycle_num in range(drift_cycles + 1):
|
|
180
|
+
result = await execute_cycle(
|
|
181
|
+
case=case,
|
|
182
|
+
transcript=transcript,
|
|
183
|
+
cycle_number=cycle_num,
|
|
184
|
+
previous_artifact=previous_artifact,
|
|
185
|
+
compactor=compactor,
|
|
186
|
+
provider=provider,
|
|
187
|
+
model=model,
|
|
188
|
+
case_seed=case_seed,
|
|
189
|
+
)
|
|
190
|
+
drift_delta = (
|
|
191
|
+
result.scorecard.penalized_cycle_score - cycle_scores[0]
|
|
192
|
+
if cycle_num > 0 and cycle_scores
|
|
193
|
+
else None
|
|
194
|
+
)
|
|
195
|
+
cycles.append(
|
|
196
|
+
CycleResult(
|
|
197
|
+
cycle_number=cycle_num,
|
|
198
|
+
scorecard=result.scorecard,
|
|
199
|
+
drift_delta=drift_delta,
|
|
200
|
+
latency_ms=result.latency_ms,
|
|
201
|
+
)
|
|
202
|
+
)
|
|
203
|
+
cycle_scores.append(result.scorecard.penalized_cycle_score)
|
|
204
|
+
transcript = result.extended_transcript
|
|
205
|
+
previous_artifact = result.artifact
|
|
206
|
+
|
|
207
|
+
case_score = sum(cycle_scores) / len(cycle_scores) if cycle_scores else 0.0
|
|
208
|
+
return CaseResult(
|
|
209
|
+
case_id=case.case_id,
|
|
210
|
+
template_key=case.template_key,
|
|
211
|
+
seed=case.seed,
|
|
212
|
+
cycles=cycles,
|
|
213
|
+
case_score=case_score,
|
|
214
|
+
drift_resistance=drift_resistance(cycle_scores),
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _instantiate_provider(provider_key: str) -> Provider:
|
|
219
|
+
cls = get_provider_cls(provider_key)
|
|
220
|
+
# All real providers accept keyword-only args; mock accepts no args.
|
|
221
|
+
return cls()
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _suite_version(templates: list[TemplateDefinition]) -> str:
|
|
225
|
+
versions = {t.version for t in templates}
|
|
226
|
+
if len(versions) == 1:
|
|
227
|
+
return next(iter(versions))
|
|
228
|
+
return "mixed"
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _assert_resume_compatible(
|
|
232
|
+
existing: RunStartEvent, args: RunArgs, compactor_cls: type[Compactor]
|
|
233
|
+
) -> None:
|
|
234
|
+
mismatches: list[str] = []
|
|
235
|
+
if existing.method_name != compactor_cls.name:
|
|
236
|
+
mismatches.append(f"method_name: {existing.method_name!r} vs {compactor_cls.name!r}")
|
|
237
|
+
if existing.suite_key != args.suite_key:
|
|
238
|
+
mismatches.append(f"suite_key: {existing.suite_key!r} vs {args.suite_key!r}")
|
|
239
|
+
if existing.target_provider != args.provider_key:
|
|
240
|
+
mismatches.append(f"provider: {existing.target_provider!r} vs {args.provider_key!r}")
|
|
241
|
+
if existing.target_model != args.model:
|
|
242
|
+
mismatches.append(f"model: {existing.target_model!r} vs {args.model!r}")
|
|
243
|
+
if existing.difficulty != args.difficulty.value:
|
|
244
|
+
mismatches.append(f"difficulty: {existing.difficulty!r} vs {args.difficulty.value!r}")
|
|
245
|
+
if existing.drift_cycles != args.drift_cycles:
|
|
246
|
+
mismatches.append(f"drift_cycles: {existing.drift_cycles} vs {args.drift_cycles}")
|
|
247
|
+
if existing.seed_group != args.seed_group:
|
|
248
|
+
mismatches.append(f"seed_group: {existing.seed_group!r} vs {args.seed_group!r}")
|
|
249
|
+
if existing.case_count_per_template != args.case_count_per_template:
|
|
250
|
+
mismatches.append(
|
|
251
|
+
f"case_count_per_template: "
|
|
252
|
+
f"{existing.case_count_per_template} vs {args.case_count_per_template}"
|
|
253
|
+
)
|
|
254
|
+
if mismatches:
|
|
255
|
+
raise ResumeError(
|
|
256
|
+
"existing run is incompatible with requested arguments: " + "; ".join(mismatches)
|
|
257
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Scoring engine: checks, weighted aggregation, drift, contradictions, compression.
|
|
2
|
+
|
|
3
|
+
See docs/architecture/decisions.md §B3 for formulas and weights.
|
|
4
|
+
Implemented in WO-004.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from compactbench.scoring.checks import (
|
|
8
|
+
contains_normalized,
|
|
9
|
+
exact,
|
|
10
|
+
forbidden_absent,
|
|
11
|
+
run_check,
|
|
12
|
+
set_match,
|
|
13
|
+
)
|
|
14
|
+
from compactbench.scoring.compression import (
|
|
15
|
+
artifact_tokens,
|
|
16
|
+
compression_ratio,
|
|
17
|
+
count_tokens,
|
|
18
|
+
transcript_tokens,
|
|
19
|
+
)
|
|
20
|
+
from compactbench.scoring.contradictions import (
|
|
21
|
+
contradiction_rate,
|
|
22
|
+
count_violations,
|
|
23
|
+
response_violates,
|
|
24
|
+
)
|
|
25
|
+
from compactbench.scoring.drift import drift_deltas, drift_resistance
|
|
26
|
+
from compactbench.scoring.errors import ScoringError
|
|
27
|
+
from compactbench.scoring.scorer import WEIGHTS, score_cycle, score_item
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"WEIGHTS",
|
|
31
|
+
"ScoringError",
|
|
32
|
+
"artifact_tokens",
|
|
33
|
+
"compression_ratio",
|
|
34
|
+
"contains_normalized",
|
|
35
|
+
"contradiction_rate",
|
|
36
|
+
"count_tokens",
|
|
37
|
+
"count_violations",
|
|
38
|
+
"drift_deltas",
|
|
39
|
+
"drift_resistance",
|
|
40
|
+
"exact",
|
|
41
|
+
"forbidden_absent",
|
|
42
|
+
"response_violates",
|
|
43
|
+
"run_check",
|
|
44
|
+
"score_cycle",
|
|
45
|
+
"score_item",
|
|
46
|
+
"set_match",
|
|
47
|
+
"transcript_tokens",
|
|
48
|
+
]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Individual check implementations used by the scorer.
|
|
2
|
+
|
|
3
|
+
Each check maps a (spec, response) pair to a score in ``[0.0, 1.0]``.
|
|
4
|
+
Higher is better. Strings are normalized by lowercasing and collapsing
|
|
5
|
+
runs of whitespace so minor formatting differences don't spuriously fail.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from compactbench.scoring.errors import ScoringError
|
|
14
|
+
|
|
15
|
+
_WHITESPACE = re.compile(r"\s+")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _normalize(text: str) -> str:
|
|
19
|
+
"""Lowercase + collapse whitespace for tolerant substring comparison."""
|
|
20
|
+
return _WHITESPACE.sub(" ", text.strip().lower())
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def contains_normalized(expected: str, response: str) -> float:
|
|
24
|
+
"""1.0 if the normalized ``expected`` is a substring of the normalized ``response``."""
|
|
25
|
+
if not expected:
|
|
26
|
+
return 1.0
|
|
27
|
+
return 1.0 if _normalize(expected) in _normalize(response) else 0.0
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def forbidden_absent(forbidden: str, response: str) -> float:
|
|
31
|
+
"""1.0 if ``forbidden`` does NOT appear in ``response`` (normalized)."""
|
|
32
|
+
if not forbidden:
|
|
33
|
+
return 1.0
|
|
34
|
+
return 0.0 if _normalize(forbidden) in _normalize(response) else 1.0
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def exact(expected: str, response: str) -> float:
|
|
38
|
+
"""1.0 if stripped ``expected`` equals stripped ``response`` (case-sensitive)."""
|
|
39
|
+
return 1.0 if expected.strip() == response.strip() else 0.0
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def set_match(expected_items: list[str], response: str) -> float:
|
|
43
|
+
"""Fraction of ``expected_items`` that appear (normalized) in ``response``."""
|
|
44
|
+
if not expected_items:
|
|
45
|
+
return 1.0
|
|
46
|
+
norm = _normalize(response)
|
|
47
|
+
matches = sum(1 for item in expected_items if _normalize(item) in norm)
|
|
48
|
+
return matches / len(expected_items)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def run_check(spec: dict[str, Any], response: str) -> float:
|
|
52
|
+
"""Dispatch on ``spec['check']`` and run the corresponding check.
|
|
53
|
+
|
|
54
|
+
Raises :class:`ScoringError` for missing or unknown check types.
|
|
55
|
+
"""
|
|
56
|
+
check_type = spec.get("check")
|
|
57
|
+
if not check_type:
|
|
58
|
+
raise ScoringError("expected spec is missing a 'check' key")
|
|
59
|
+
|
|
60
|
+
if check_type == "contains_normalized":
|
|
61
|
+
value = spec.get("value", "")
|
|
62
|
+
if not isinstance(value, str):
|
|
63
|
+
raise ScoringError("'contains_normalized' requires 'value' to be a string")
|
|
64
|
+
return contains_normalized(value, response)
|
|
65
|
+
|
|
66
|
+
if check_type == "forbidden_absent":
|
|
67
|
+
value = spec.get("value", "")
|
|
68
|
+
if not isinstance(value, str):
|
|
69
|
+
raise ScoringError("'forbidden_absent' requires 'value' to be a string")
|
|
70
|
+
return forbidden_absent(value, response)
|
|
71
|
+
|
|
72
|
+
if check_type == "exact":
|
|
73
|
+
value = spec.get("value", "")
|
|
74
|
+
if not isinstance(value, str):
|
|
75
|
+
raise ScoringError("'exact' requires 'value' to be a string")
|
|
76
|
+
return exact(value, response)
|
|
77
|
+
|
|
78
|
+
if check_type == "set_match":
|
|
79
|
+
raw = spec.get("values", [])
|
|
80
|
+
if not isinstance(raw, list):
|
|
81
|
+
raise ScoringError("'set_match' requires 'values' to be a list of strings")
|
|
82
|
+
values: list[str] = []
|
|
83
|
+
for v in raw: # pyright: ignore[reportUnknownVariableType]
|
|
84
|
+
if not isinstance(v, str):
|
|
85
|
+
raise ScoringError("'set_match' requires 'values' to be a list of strings")
|
|
86
|
+
values.append(v)
|
|
87
|
+
return set_match(values, response)
|
|
88
|
+
|
|
89
|
+
raise ScoringError(
|
|
90
|
+
f"unknown check type {check_type!r}. "
|
|
91
|
+
f"Known: ['contains_normalized', 'forbidden_absent', 'exact', 'set_match']"
|
|
92
|
+
)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Compression-ratio computation using the ``cl100k_base`` tokenizer.
|
|
2
|
+
|
|
3
|
+
Using a single canonical tokenizer regardless of the target model keeps
|
|
4
|
+
compression ratios directly comparable across methods and leaderboard
|
|
5
|
+
versions (see docs/architecture/decisions.md §B3).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import tiktoken
|
|
13
|
+
|
|
14
|
+
from compactbench.contracts import CompactionArtifact, Transcript
|
|
15
|
+
|
|
16
|
+
_ENCODER_CACHE: dict[str, Any] = {}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _encoder(name: str = "cl100k_base") -> Any:
|
|
20
|
+
enc = _ENCODER_CACHE.get(name)
|
|
21
|
+
if enc is None:
|
|
22
|
+
enc = tiktoken.get_encoding(name)
|
|
23
|
+
_ENCODER_CACHE[name] = enc
|
|
24
|
+
return enc
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def count_tokens(text: str, encoding: str = "cl100k_base") -> int:
|
|
28
|
+
"""Return the number of tokens in ``text`` under the given ``encoding``."""
|
|
29
|
+
if not text:
|
|
30
|
+
return 0
|
|
31
|
+
return len(_encoder(encoding).encode(text))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def transcript_tokens(transcript: Transcript) -> int:
|
|
35
|
+
"""Sum of token counts across every turn's content."""
|
|
36
|
+
return sum(count_tokens(t.content) for t in transcript.turns)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def artifact_tokens(artifact: CompactionArtifact) -> int:
|
|
40
|
+
"""Sum of tokens across the artifact's summary text and structured state values."""
|
|
41
|
+
total = count_tokens(artifact.summary_text)
|
|
42
|
+
state = artifact.structured_state
|
|
43
|
+
for section in (
|
|
44
|
+
state.immutable_facts,
|
|
45
|
+
state.locked_decisions,
|
|
46
|
+
state.deferred_items,
|
|
47
|
+
state.forbidden_behaviors,
|
|
48
|
+
state.unresolved_items,
|
|
49
|
+
):
|
|
50
|
+
for s in section:
|
|
51
|
+
total += count_tokens(s)
|
|
52
|
+
for k, v in state.entity_map.items():
|
|
53
|
+
total += count_tokens(k) + count_tokens(v)
|
|
54
|
+
return total
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def compression_ratio(transcript: Transcript, artifact: CompactionArtifact) -> float:
|
|
58
|
+
"""Return ``tokens(transcript) / tokens(artifact)``.
|
|
59
|
+
|
|
60
|
+
If the artifact has zero tokens, the denominator is floored at 1 to keep
|
|
61
|
+
the ratio finite; the caller can detect this by checking that
|
|
62
|
+
:func:`artifact_tokens` is zero.
|
|
63
|
+
"""
|
|
64
|
+
src = transcript_tokens(transcript)
|
|
65
|
+
dst = artifact_tokens(artifact)
|
|
66
|
+
return src / max(dst, 1)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Detect responses that contradict a case's ground truth.
|
|
2
|
+
|
|
3
|
+
A response "contradicts" when it contains a forbidden-behavior phrase *in a
|
|
4
|
+
context where it shouldn't*. Retention-recall items (those whose item type is
|
|
5
|
+
``forbidden_behavior_retention`` or ``locked_decision_retention``) are
|
|
6
|
+
**excluded** from the denominator — their correct answer names the forbidden
|
|
7
|
+
or locked thing, and naming it there is not a contradiction.
|
|
8
|
+
|
|
9
|
+
This is a conservative lexical detector; genuine semantic contradictions are
|
|
10
|
+
caught by the per-item scoring.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
|
|
17
|
+
from compactbench.contracts import EvaluationItem, EvaluationItemType, GroundTruth
|
|
18
|
+
|
|
19
|
+
_WHITESPACE = re.compile(r"\s+")
|
|
20
|
+
|
|
21
|
+
_RECALL_ITEM_TYPES = frozenset(
|
|
22
|
+
{
|
|
23
|
+
EvaluationItemType.FORBIDDEN_BEHAVIOR_RETENTION,
|
|
24
|
+
EvaluationItemType.LOCKED_DECISION_RETENTION,
|
|
25
|
+
}
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _normalize(text: str) -> str:
|
|
30
|
+
return _WHITESPACE.sub(" ", text.strip().lower())
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def response_violates(response: str, ground_truth: GroundTruth) -> bool:
|
|
34
|
+
"""True if ``response`` contains any forbidden behavior from ``ground_truth``."""
|
|
35
|
+
if not ground_truth.forbidden_behaviors:
|
|
36
|
+
return False
|
|
37
|
+
norm = _normalize(response)
|
|
38
|
+
return any(_normalize(f) in norm for f in ground_truth.forbidden_behaviors)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def count_violations(
|
|
42
|
+
items: list[EvaluationItem],
|
|
43
|
+
responses: dict[str, str],
|
|
44
|
+
ground_truth: GroundTruth,
|
|
45
|
+
) -> int:
|
|
46
|
+
"""Count responses that violate a forbidden behavior (excluding recall items)."""
|
|
47
|
+
violations = 0
|
|
48
|
+
for item in items:
|
|
49
|
+
if item.item_type in _RECALL_ITEM_TYPES:
|
|
50
|
+
continue
|
|
51
|
+
response = responses.get(item.key, "")
|
|
52
|
+
if response_violates(response, ground_truth):
|
|
53
|
+
violations += 1
|
|
54
|
+
return violations
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def contradiction_rate(
|
|
58
|
+
items: list[EvaluationItem],
|
|
59
|
+
responses: dict[str, str],
|
|
60
|
+
ground_truth: GroundTruth,
|
|
61
|
+
) -> float:
|
|
62
|
+
"""Fraction of non-recall responses that violate a forbidden behavior.
|
|
63
|
+
|
|
64
|
+
Denominator is the number of non-recall items. Returns 0.0 when no items
|
|
65
|
+
qualify (e.g., a case made up entirely of recall items).
|
|
66
|
+
"""
|
|
67
|
+
non_recall = [i for i in items if i.item_type not in _RECALL_ITEM_TYPES]
|
|
68
|
+
if not non_recall:
|
|
69
|
+
return 0.0
|
|
70
|
+
return count_violations(items, responses, ground_truth) / len(non_recall)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Cross-cycle drift metrics.
|
|
2
|
+
|
|
3
|
+
Given per-cycle scores ``[c0, c1, c2, ...]``, we compute how much scores
|
|
4
|
+
degrade after the initial cycle. A method that holds steady scores 1.0;
|
|
5
|
+
a method that degrades scores proportionally lower.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def drift_deltas(cycle_scores: list[float]) -> list[float]:
|
|
12
|
+
"""Return ``[c_n - c_0 for n >= 1]``. Empty list for ``len < 2``."""
|
|
13
|
+
if len(cycle_scores) < 2:
|
|
14
|
+
return []
|
|
15
|
+
baseline = cycle_scores[0]
|
|
16
|
+
return [s - baseline for s in cycle_scores[1:]]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def drift_resistance(cycle_scores: list[float]) -> float:
|
|
20
|
+
"""Clamp of ``1 + mean(drift_deltas)`` into ``[0.0, 1.0]``.
|
|
21
|
+
|
|
22
|
+
A method with stable scores returns 1.0. A method that degrades 20%
|
|
23
|
+
per cycle on average returns 0.8. A method that improves returns 1.0
|
|
24
|
+
(clamped — improvement does not extend beyond perfect drift resistance).
|
|
25
|
+
"""
|
|
26
|
+
deltas = drift_deltas(cycle_scores)
|
|
27
|
+
if not deltas:
|
|
28
|
+
return 1.0
|
|
29
|
+
avg = sum(deltas) / len(deltas)
|
|
30
|
+
return max(0.0, min(1.0, 1.0 + avg))
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Orchestrate per-cycle scoring: checks → weighted aggregate → penalty → scorecard.
|
|
2
|
+
|
|
3
|
+
Weights are locked in docs/architecture/decisions.md §B3. Changing them is a
|
|
4
|
+
scorer-version bump.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from compactbench.contracts import (
|
|
10
|
+
CompactionArtifact,
|
|
11
|
+
EvaluationItem,
|
|
12
|
+
EvaluationItemType,
|
|
13
|
+
GeneratedCase,
|
|
14
|
+
ItemScore,
|
|
15
|
+
Scorecard,
|
|
16
|
+
)
|
|
17
|
+
from compactbench.scoring.checks import run_check
|
|
18
|
+
from compactbench.scoring.compression import compression_ratio
|
|
19
|
+
from compactbench.scoring.contradictions import contradiction_rate
|
|
20
|
+
|
|
21
|
+
WEIGHTS: dict[EvaluationItemType, float] = {
|
|
22
|
+
EvaluationItemType.LOCKED_DECISION_RETENTION: 3.0,
|
|
23
|
+
EvaluationItemType.FORBIDDEN_BEHAVIOR_RETENTION: 3.0,
|
|
24
|
+
EvaluationItemType.IMMUTABLE_FACT_RECALL: 2.0,
|
|
25
|
+
EvaluationItemType.UNRESOLVED_TASK_CONTINUITY: 2.0,
|
|
26
|
+
EvaluationItemType.ENTITY_INTEGRITY: 1.0,
|
|
27
|
+
EvaluationItemType.PLANNING_SOUNDNESS: 1.0,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def score_item(item: EvaluationItem, response: str) -> ItemScore:
|
|
32
|
+
"""Score a single evaluation item against a response."""
|
|
33
|
+
check_type = str(item.expected.get("check", "unknown"))
|
|
34
|
+
raw = run_check(item.expected, response)
|
|
35
|
+
return ItemScore(
|
|
36
|
+
item_key=item.key,
|
|
37
|
+
item_type=item.item_type.value,
|
|
38
|
+
score=raw,
|
|
39
|
+
weight=WEIGHTS.get(item.item_type, 1.0),
|
|
40
|
+
check_type=check_type,
|
|
41
|
+
details={},
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _weighted_cycle_score(item_scores: list[ItemScore]) -> float:
|
|
46
|
+
if not item_scores:
|
|
47
|
+
return 0.0
|
|
48
|
+
total_weight = sum(i.weight for i in item_scores)
|
|
49
|
+
if total_weight <= 0:
|
|
50
|
+
return 0.0
|
|
51
|
+
return sum(i.score * i.weight for i in item_scores) / total_weight
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def score_cycle(
|
|
55
|
+
case: GeneratedCase,
|
|
56
|
+
artifact: CompactionArtifact,
|
|
57
|
+
responses: dict[str, str],
|
|
58
|
+
cycle_number: int = 0,
|
|
59
|
+
) -> Scorecard:
|
|
60
|
+
"""Score one cycle of a compaction run and return a :class:`Scorecard`."""
|
|
61
|
+
if cycle_number < 0:
|
|
62
|
+
raise ValueError(f"cycle_number must be >= 0, got {cycle_number}")
|
|
63
|
+
|
|
64
|
+
item_scores = [score_item(item, responses.get(item.key, "")) for item in case.evaluation_items]
|
|
65
|
+
cycle_score = _weighted_cycle_score(item_scores)
|
|
66
|
+
contra = contradiction_rate(case.evaluation_items, responses, case.ground_truth)
|
|
67
|
+
penalized = max(0.0, min(1.0, cycle_score * (1.0 - contra)))
|
|
68
|
+
comp_ratio = compression_ratio(case.transcript, artifact)
|
|
69
|
+
|
|
70
|
+
return Scorecard(
|
|
71
|
+
cycle_number=cycle_number,
|
|
72
|
+
cycle_score=cycle_score,
|
|
73
|
+
penalized_cycle_score=penalized,
|
|
74
|
+
contradiction_rate=contra,
|
|
75
|
+
compression_ratio=comp_ratio,
|
|
76
|
+
item_scores=item_scores,
|
|
77
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Shared utilities: tokenizer wrapper, structured logging."""
|