compactbench 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. compactbench/__init__.py +10 -0
  2. compactbench/__main__.py +6 -0
  3. compactbench/cli.py +343 -0
  4. compactbench/compactors/__init__.py +47 -0
  5. compactbench/compactors/_state_parser.py +118 -0
  6. compactbench/compactors/_utils.py +37 -0
  7. compactbench/compactors/base.py +38 -0
  8. compactbench/compactors/errors.py +11 -0
  9. compactbench/compactors/hierarchical_summary.py +113 -0
  10. compactbench/compactors/hybrid_ledger.py +115 -0
  11. compactbench/compactors/naive_summary.py +54 -0
  12. compactbench/compactors/structured_state.py +64 -0
  13. compactbench/config.py +36 -0
  14. compactbench/contracts/__init__.py +41 -0
  15. compactbench/contracts/artifact.py +49 -0
  16. compactbench/contracts/case.py +81 -0
  17. compactbench/contracts/result.py +79 -0
  18. compactbench/dsl/__init__.py +66 -0
  19. compactbench/dsl/errors.py +23 -0
  20. compactbench/dsl/generators.py +243 -0
  21. compactbench/dsl/models.py +107 -0
  22. compactbench/dsl/parser.py +55 -0
  23. compactbench/dsl/substitution.py +32 -0
  24. compactbench/dsl/validator.py +97 -0
  25. compactbench/engine/__init__.py +29 -0
  26. compactbench/engine/difficulty.py +54 -0
  27. compactbench/engine/distractors.py +69 -0
  28. compactbench/engine/errors.py +11 -0
  29. compactbench/engine/generation.py +96 -0
  30. compactbench/engine/seeds.py +31 -0
  31. compactbench/engine/transcript.py +84 -0
  32. compactbench/leaderboard/__init__.py +38 -0
  33. compactbench/leaderboard/errors.py +11 -0
  34. compactbench/leaderboard/projection.py +103 -0
  35. compactbench/leaderboard/qualification.py +90 -0
  36. compactbench/leaderboard/ranking.py +71 -0
  37. compactbench/providers/__init__.py +52 -0
  38. compactbench/providers/_retry.py +48 -0
  39. compactbench/providers/base.py +44 -0
  40. compactbench/providers/errors.py +15 -0
  41. compactbench/providers/google_ai_studio.py +112 -0
  42. compactbench/providers/groq.py +95 -0
  43. compactbench/providers/mock.py +53 -0
  44. compactbench/providers/ollama.py +100 -0
  45. compactbench/py.typed +0 -0
  46. compactbench/runner/__init__.py +53 -0
  47. compactbench/runner/_resolver.py +57 -0
  48. compactbench/runner/continuation.py +78 -0
  49. compactbench/runner/cycle.py +67 -0
  50. compactbench/runner/errors.py +15 -0
  51. compactbench/runner/evaluation.py +71 -0
  52. compactbench/runner/persistence.py +227 -0
  53. compactbench/runner/run.py +257 -0
  54. compactbench/scoring/__init__.py +48 -0
  55. compactbench/scoring/checks.py +92 -0
  56. compactbench/scoring/compression.py +66 -0
  57. compactbench/scoring/contradictions.py +70 -0
  58. compactbench/scoring/drift.py +30 -0
  59. compactbench/scoring/errors.py +7 -0
  60. compactbench/scoring/scorer.py +77 -0
  61. compactbench/utils/__init__.py +1 -0
  62. compactbench-0.1.0.dist-info/METADATA +161 -0
  63. compactbench-0.1.0.dist-info/RECORD +66 -0
  64. compactbench-0.1.0.dist-info/WHEEL +4 -0
  65. compactbench-0.1.0.dist-info/entry_points.txt +2 -0
  66. compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,257 @@
1
+ """Top-level run orchestrator.
2
+
3
+ Loads a suite, generates one case per (template, seed slot), executes each
4
+ case through all drift cycles, scores everything, and writes a streamed
5
+ ``results.jsonl`` event log.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import uuid
11
+ from dataclasses import dataclass
12
+ from datetime import UTC, datetime
13
+ from pathlib import Path
14
+
15
+ from compactbench.compactors import Compactor
16
+ from compactbench.contracts import CaseResult, CycleResult, GeneratedCase
17
+ from compactbench.dsl import (
18
+ DifficultyLevel,
19
+ TemplateDefinition,
20
+ load_suite,
21
+ validate_template,
22
+ )
23
+ from compactbench.engine import derive_case_seed, generate_case
24
+ from compactbench.providers import Provider, get_provider_cls
25
+ from compactbench.runner._resolver import resolve_compactor_class
26
+ from compactbench.runner.cycle import execute_cycle
27
+ from compactbench.runner.errors import ResumeError, RunnerError
28
+ from compactbench.runner.persistence import (
29
+ SCORER_VERSION,
30
+ CaseCompleteEvent,
31
+ ResultsWriter,
32
+ RunEndEvent,
33
+ RunStartEvent,
34
+ aggregate_run_metrics,
35
+ completed_case_ids,
36
+ read_run_start,
37
+ )
38
+ from compactbench.scoring import drift_resistance
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class RunArgs:
43
+ method_spec: str
44
+ suite_key: str
45
+ provider_key: str
46
+ model: str
47
+ difficulty: DifficultyLevel
48
+ drift_cycles: int
49
+ case_count_per_template: int
50
+ seed_group: str
51
+ benchmarks_dir: Path
52
+ output_path: Path
53
+ resume: bool
54
+
55
+
56
+ async def run_experiment(args: RunArgs) -> Path:
57
+ """Execute a full run per ``args`` and return the path to the written results file."""
58
+ suite_dir = args.benchmarks_dir / args.suite_key
59
+ if not suite_dir.is_dir():
60
+ raise RunnerError(f"suite directory not found: {suite_dir}")
61
+
62
+ templates = load_suite(suite_dir)
63
+ if not templates:
64
+ raise RunnerError(f"no templates in suite {args.suite_key!r}")
65
+ for template in templates:
66
+ validate_template(template)
67
+
68
+ compactor_cls = resolve_compactor_class(args.method_spec)
69
+ provider = _instantiate_provider(args.provider_key)
70
+
71
+ already_done: set[str] = set()
72
+ run_id: str
73
+ started_at: datetime
74
+ mode: str = "write"
75
+
76
+ if args.resume and args.output_path.exists():
77
+ existing = read_run_start(args.output_path)
78
+ if existing is None:
79
+ raise ResumeError(f"--resume passed but {args.output_path} has no run_start event")
80
+ _assert_resume_compatible(existing, args, compactor_cls)
81
+ already_done = completed_case_ids(args.output_path)
82
+ run_id = existing.run_id
83
+ started_at = existing.started_at
84
+ mode = "append"
85
+ else:
86
+ run_id = str(uuid.uuid4())
87
+ started_at = datetime.now(UTC)
88
+
89
+ suite_version = _suite_version(templates)
90
+ writer = ResultsWriter(args.output_path, mode=mode) # type: ignore[arg-type]
91
+ try:
92
+ if mode == "write":
93
+ writer.write(
94
+ RunStartEvent(
95
+ run_id=run_id,
96
+ method_name=compactor_cls.name,
97
+ method_version=compactor_cls.version,
98
+ suite_key=args.suite_key,
99
+ suite_version=suite_version,
100
+ scorer_version=SCORER_VERSION,
101
+ target_provider=args.provider_key,
102
+ target_model=args.model,
103
+ difficulty=args.difficulty.value,
104
+ drift_cycles=args.drift_cycles,
105
+ seed_group=args.seed_group,
106
+ case_count_per_template=args.case_count_per_template,
107
+ started_at=started_at,
108
+ )
109
+ )
110
+
111
+ completed_cases: list[CaseResult] = []
112
+ for template in templates:
113
+ for slot in range(args.case_count_per_template):
114
+ case_seed = derive_case_seed(
115
+ f"{args.suite_key}@{suite_version}", args.seed_group, slot
116
+ )
117
+ case = generate_case(template, case_seed, args.difficulty)
118
+ if case.case_id in already_done:
119
+ continue
120
+
121
+ compactor = compactor_cls(provider, args.model)
122
+ case_result = await _execute_case(
123
+ case=case,
124
+ compactor=compactor,
125
+ provider=provider,
126
+ model=args.model,
127
+ drift_cycles=args.drift_cycles,
128
+ case_seed=case_seed,
129
+ )
130
+ writer.write(CaseCompleteEvent(case_result=case_result))
131
+ completed_cases.append(case_result)
132
+
133
+ # For run-end aggregates, combine newly-executed cases with any that
134
+ # were already persisted from a resumed run.
135
+ from compactbench.runner.persistence import iter_events
136
+
137
+ all_cases: list[CaseResult] = list(completed_cases)
138
+ if already_done:
139
+ for event in iter_events(args.output_path):
140
+ if (
141
+ isinstance(event, CaseCompleteEvent)
142
+ and event.case_result.case_id in already_done
143
+ ):
144
+ all_cases.append(event.case_result)
145
+
146
+ agg = aggregate_run_metrics(all_cases)
147
+ writer.write(
148
+ RunEndEvent(
149
+ completed_at=datetime.now(UTC),
150
+ overall_score=agg["overall_score"],
151
+ drift_resistance=agg["drift_resistance"],
152
+ constraint_retention=agg["constraint_retention"],
153
+ contradiction_rate=agg["contradiction_rate"],
154
+ compression_ratio=agg["compression_ratio"],
155
+ )
156
+ )
157
+ finally:
158
+ writer.close()
159
+
160
+ return args.output_path
161
+
162
+
163
+ async def _execute_case(
164
+ *,
165
+ case: GeneratedCase,
166
+ compactor: Compactor,
167
+ provider: Provider,
168
+ model: str,
169
+ drift_cycles: int,
170
+ case_seed: int,
171
+ ) -> CaseResult:
172
+ from compactbench.contracts import CompactionArtifact, Transcript
173
+
174
+ transcript: Transcript = case.transcript
175
+ previous_artifact: CompactionArtifact | None = None
176
+ cycles: list[CycleResult] = []
177
+ cycle_scores: list[float] = []
178
+
179
+ for cycle_num in range(drift_cycles + 1):
180
+ result = await execute_cycle(
181
+ case=case,
182
+ transcript=transcript,
183
+ cycle_number=cycle_num,
184
+ previous_artifact=previous_artifact,
185
+ compactor=compactor,
186
+ provider=provider,
187
+ model=model,
188
+ case_seed=case_seed,
189
+ )
190
+ drift_delta = (
191
+ result.scorecard.penalized_cycle_score - cycle_scores[0]
192
+ if cycle_num > 0 and cycle_scores
193
+ else None
194
+ )
195
+ cycles.append(
196
+ CycleResult(
197
+ cycle_number=cycle_num,
198
+ scorecard=result.scorecard,
199
+ drift_delta=drift_delta,
200
+ latency_ms=result.latency_ms,
201
+ )
202
+ )
203
+ cycle_scores.append(result.scorecard.penalized_cycle_score)
204
+ transcript = result.extended_transcript
205
+ previous_artifact = result.artifact
206
+
207
+ case_score = sum(cycle_scores) / len(cycle_scores) if cycle_scores else 0.0
208
+ return CaseResult(
209
+ case_id=case.case_id,
210
+ template_key=case.template_key,
211
+ seed=case.seed,
212
+ cycles=cycles,
213
+ case_score=case_score,
214
+ drift_resistance=drift_resistance(cycle_scores),
215
+ )
216
+
217
+
218
+ def _instantiate_provider(provider_key: str) -> Provider:
219
+ cls = get_provider_cls(provider_key)
220
+ # All real providers accept keyword-only args; mock accepts no args.
221
+ return cls()
222
+
223
+
224
+ def _suite_version(templates: list[TemplateDefinition]) -> str:
225
+ versions = {t.version for t in templates}
226
+ if len(versions) == 1:
227
+ return next(iter(versions))
228
+ return "mixed"
229
+
230
+
231
+ def _assert_resume_compatible(
232
+ existing: RunStartEvent, args: RunArgs, compactor_cls: type[Compactor]
233
+ ) -> None:
234
+ mismatches: list[str] = []
235
+ if existing.method_name != compactor_cls.name:
236
+ mismatches.append(f"method_name: {existing.method_name!r} vs {compactor_cls.name!r}")
237
+ if existing.suite_key != args.suite_key:
238
+ mismatches.append(f"suite_key: {existing.suite_key!r} vs {args.suite_key!r}")
239
+ if existing.target_provider != args.provider_key:
240
+ mismatches.append(f"provider: {existing.target_provider!r} vs {args.provider_key!r}")
241
+ if existing.target_model != args.model:
242
+ mismatches.append(f"model: {existing.target_model!r} vs {args.model!r}")
243
+ if existing.difficulty != args.difficulty.value:
244
+ mismatches.append(f"difficulty: {existing.difficulty!r} vs {args.difficulty.value!r}")
245
+ if existing.drift_cycles != args.drift_cycles:
246
+ mismatches.append(f"drift_cycles: {existing.drift_cycles} vs {args.drift_cycles}")
247
+ if existing.seed_group != args.seed_group:
248
+ mismatches.append(f"seed_group: {existing.seed_group!r} vs {args.seed_group!r}")
249
+ if existing.case_count_per_template != args.case_count_per_template:
250
+ mismatches.append(
251
+ f"case_count_per_template: "
252
+ f"{existing.case_count_per_template} vs {args.case_count_per_template}"
253
+ )
254
+ if mismatches:
255
+ raise ResumeError(
256
+ "existing run is incompatible with requested arguments: " + "; ".join(mismatches)
257
+ )
@@ -0,0 +1,48 @@
1
+ """Scoring engine: checks, weighted aggregation, drift, contradictions, compression.
2
+
3
+ See docs/architecture/decisions.md §B3 for formulas and weights.
4
+ Implemented in WO-004.
5
+ """
6
+
7
+ from compactbench.scoring.checks import (
8
+ contains_normalized,
9
+ exact,
10
+ forbidden_absent,
11
+ run_check,
12
+ set_match,
13
+ )
14
+ from compactbench.scoring.compression import (
15
+ artifact_tokens,
16
+ compression_ratio,
17
+ count_tokens,
18
+ transcript_tokens,
19
+ )
20
+ from compactbench.scoring.contradictions import (
21
+ contradiction_rate,
22
+ count_violations,
23
+ response_violates,
24
+ )
25
+ from compactbench.scoring.drift import drift_deltas, drift_resistance
26
+ from compactbench.scoring.errors import ScoringError
27
+ from compactbench.scoring.scorer import WEIGHTS, score_cycle, score_item
28
+
29
+ __all__ = [
30
+ "WEIGHTS",
31
+ "ScoringError",
32
+ "artifact_tokens",
33
+ "compression_ratio",
34
+ "contains_normalized",
35
+ "contradiction_rate",
36
+ "count_tokens",
37
+ "count_violations",
38
+ "drift_deltas",
39
+ "drift_resistance",
40
+ "exact",
41
+ "forbidden_absent",
42
+ "response_violates",
43
+ "run_check",
44
+ "score_cycle",
45
+ "score_item",
46
+ "set_match",
47
+ "transcript_tokens",
48
+ ]
@@ -0,0 +1,92 @@
1
+ """Individual check implementations used by the scorer.
2
+
3
+ Each check maps a (spec, response) pair to a score in ``[0.0, 1.0]``.
4
+ Higher is better. Strings are normalized by lowercasing and collapsing
5
+ runs of whitespace so minor formatting differences don't spuriously fail.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from typing import Any
12
+
13
+ from compactbench.scoring.errors import ScoringError
14
+
15
+ _WHITESPACE = re.compile(r"\s+")
16
+
17
+
18
+ def _normalize(text: str) -> str:
19
+ """Lowercase + collapse whitespace for tolerant substring comparison."""
20
+ return _WHITESPACE.sub(" ", text.strip().lower())
21
+
22
+
23
+ def contains_normalized(expected: str, response: str) -> float:
24
+ """1.0 if the normalized ``expected`` is a substring of the normalized ``response``."""
25
+ if not expected:
26
+ return 1.0
27
+ return 1.0 if _normalize(expected) in _normalize(response) else 0.0
28
+
29
+
30
+ def forbidden_absent(forbidden: str, response: str) -> float:
31
+ """1.0 if ``forbidden`` does NOT appear in ``response`` (normalized)."""
32
+ if not forbidden:
33
+ return 1.0
34
+ return 0.0 if _normalize(forbidden) in _normalize(response) else 1.0
35
+
36
+
37
+ def exact(expected: str, response: str) -> float:
38
+ """1.0 if stripped ``expected`` equals stripped ``response`` (case-sensitive)."""
39
+ return 1.0 if expected.strip() == response.strip() else 0.0
40
+
41
+
42
+ def set_match(expected_items: list[str], response: str) -> float:
43
+ """Fraction of ``expected_items`` that appear (normalized) in ``response``."""
44
+ if not expected_items:
45
+ return 1.0
46
+ norm = _normalize(response)
47
+ matches = sum(1 for item in expected_items if _normalize(item) in norm)
48
+ return matches / len(expected_items)
49
+
50
+
51
+ def run_check(spec: dict[str, Any], response: str) -> float:
52
+ """Dispatch on ``spec['check']`` and run the corresponding check.
53
+
54
+ Raises :class:`ScoringError` for missing or unknown check types.
55
+ """
56
+ check_type = spec.get("check")
57
+ if not check_type:
58
+ raise ScoringError("expected spec is missing a 'check' key")
59
+
60
+ if check_type == "contains_normalized":
61
+ value = spec.get("value", "")
62
+ if not isinstance(value, str):
63
+ raise ScoringError("'contains_normalized' requires 'value' to be a string")
64
+ return contains_normalized(value, response)
65
+
66
+ if check_type == "forbidden_absent":
67
+ value = spec.get("value", "")
68
+ if not isinstance(value, str):
69
+ raise ScoringError("'forbidden_absent' requires 'value' to be a string")
70
+ return forbidden_absent(value, response)
71
+
72
+ if check_type == "exact":
73
+ value = spec.get("value", "")
74
+ if not isinstance(value, str):
75
+ raise ScoringError("'exact' requires 'value' to be a string")
76
+ return exact(value, response)
77
+
78
+ if check_type == "set_match":
79
+ raw = spec.get("values", [])
80
+ if not isinstance(raw, list):
81
+ raise ScoringError("'set_match' requires 'values' to be a list of strings")
82
+ values: list[str] = []
83
+ for v in raw: # pyright: ignore[reportUnknownVariableType]
84
+ if not isinstance(v, str):
85
+ raise ScoringError("'set_match' requires 'values' to be a list of strings")
86
+ values.append(v)
87
+ return set_match(values, response)
88
+
89
+ raise ScoringError(
90
+ f"unknown check type {check_type!r}. "
91
+ f"Known: ['contains_normalized', 'forbidden_absent', 'exact', 'set_match']"
92
+ )
@@ -0,0 +1,66 @@
1
+ """Compression-ratio computation using the ``cl100k_base`` tokenizer.
2
+
3
+ Using a single canonical tokenizer regardless of the target model keeps
4
+ compression ratios directly comparable across methods and leaderboard
5
+ versions (see docs/architecture/decisions.md §B3).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any
11
+
12
+ import tiktoken
13
+
14
+ from compactbench.contracts import CompactionArtifact, Transcript
15
+
16
+ _ENCODER_CACHE: dict[str, Any] = {}
17
+
18
+
19
+ def _encoder(name: str = "cl100k_base") -> Any:
20
+ enc = _ENCODER_CACHE.get(name)
21
+ if enc is None:
22
+ enc = tiktoken.get_encoding(name)
23
+ _ENCODER_CACHE[name] = enc
24
+ return enc
25
+
26
+
27
+ def count_tokens(text: str, encoding: str = "cl100k_base") -> int:
28
+ """Return the number of tokens in ``text`` under the given ``encoding``."""
29
+ if not text:
30
+ return 0
31
+ return len(_encoder(encoding).encode(text))
32
+
33
+
34
+ def transcript_tokens(transcript: Transcript) -> int:
35
+ """Sum of token counts across every turn's content."""
36
+ return sum(count_tokens(t.content) for t in transcript.turns)
37
+
38
+
39
+ def artifact_tokens(artifact: CompactionArtifact) -> int:
40
+ """Sum of tokens across the artifact's summary text and structured state values."""
41
+ total = count_tokens(artifact.summary_text)
42
+ state = artifact.structured_state
43
+ for section in (
44
+ state.immutable_facts,
45
+ state.locked_decisions,
46
+ state.deferred_items,
47
+ state.forbidden_behaviors,
48
+ state.unresolved_items,
49
+ ):
50
+ for s in section:
51
+ total += count_tokens(s)
52
+ for k, v in state.entity_map.items():
53
+ total += count_tokens(k) + count_tokens(v)
54
+ return total
55
+
56
+
57
+ def compression_ratio(transcript: Transcript, artifact: CompactionArtifact) -> float:
58
+ """Return ``tokens(transcript) / tokens(artifact)``.
59
+
60
+ If the artifact has zero tokens, the denominator is floored at 1 to keep
61
+ the ratio finite; the caller can detect this by checking that
62
+ :func:`artifact_tokens` is zero.
63
+ """
64
+ src = transcript_tokens(transcript)
65
+ dst = artifact_tokens(artifact)
66
+ return src / max(dst, 1)
@@ -0,0 +1,70 @@
1
+ """Detect responses that contradict a case's ground truth.
2
+
3
+ A response "contradicts" when it contains a forbidden-behavior phrase *in a
4
+ context where it shouldn't*. Retention-recall items (those whose item type is
5
+ ``forbidden_behavior_retention`` or ``locked_decision_retention``) are
6
+ **excluded** from the denominator — their correct answer names the forbidden
7
+ or locked thing, and naming it there is not a contradiction.
8
+
9
+ This is a conservative lexical detector; genuine semantic contradictions are
10
+ caught by the per-item scoring.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import re
16
+
17
+ from compactbench.contracts import EvaluationItem, EvaluationItemType, GroundTruth
18
+
19
+ _WHITESPACE = re.compile(r"\s+")
20
+
21
+ _RECALL_ITEM_TYPES = frozenset(
22
+ {
23
+ EvaluationItemType.FORBIDDEN_BEHAVIOR_RETENTION,
24
+ EvaluationItemType.LOCKED_DECISION_RETENTION,
25
+ }
26
+ )
27
+
28
+
29
+ def _normalize(text: str) -> str:
30
+ return _WHITESPACE.sub(" ", text.strip().lower())
31
+
32
+
33
+ def response_violates(response: str, ground_truth: GroundTruth) -> bool:
34
+ """True if ``response`` contains any forbidden behavior from ``ground_truth``."""
35
+ if not ground_truth.forbidden_behaviors:
36
+ return False
37
+ norm = _normalize(response)
38
+ return any(_normalize(f) in norm for f in ground_truth.forbidden_behaviors)
39
+
40
+
41
+ def count_violations(
42
+ items: list[EvaluationItem],
43
+ responses: dict[str, str],
44
+ ground_truth: GroundTruth,
45
+ ) -> int:
46
+ """Count responses that violate a forbidden behavior (excluding recall items)."""
47
+ violations = 0
48
+ for item in items:
49
+ if item.item_type in _RECALL_ITEM_TYPES:
50
+ continue
51
+ response = responses.get(item.key, "")
52
+ if response_violates(response, ground_truth):
53
+ violations += 1
54
+ return violations
55
+
56
+
57
+ def contradiction_rate(
58
+ items: list[EvaluationItem],
59
+ responses: dict[str, str],
60
+ ground_truth: GroundTruth,
61
+ ) -> float:
62
+ """Fraction of non-recall responses that violate a forbidden behavior.
63
+
64
+ Denominator is the number of non-recall items. Returns 0.0 when no items
65
+ qualify (e.g., a case made up entirely of recall items).
66
+ """
67
+ non_recall = [i for i in items if i.item_type not in _RECALL_ITEM_TYPES]
68
+ if not non_recall:
69
+ return 0.0
70
+ return count_violations(items, responses, ground_truth) / len(non_recall)
@@ -0,0 +1,30 @@
1
+ """Cross-cycle drift metrics.
2
+
3
+ Given per-cycle scores ``[c0, c1, c2, ...]``, we compute how much scores
4
+ degrade after the initial cycle. A method that holds steady scores 1.0;
5
+ a method that degrades scores proportionally lower.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+
11
+ def drift_deltas(cycle_scores: list[float]) -> list[float]:
12
+ """Return ``[c_n - c_0 for n >= 1]``. Empty list for ``len < 2``."""
13
+ if len(cycle_scores) < 2:
14
+ return []
15
+ baseline = cycle_scores[0]
16
+ return [s - baseline for s in cycle_scores[1:]]
17
+
18
+
19
+ def drift_resistance(cycle_scores: list[float]) -> float:
20
+ """Clamp of ``1 + mean(drift_deltas)`` into ``[0.0, 1.0]``.
21
+
22
+ A method with stable scores returns 1.0. A method that degrades 20%
23
+ per cycle on average returns 0.8. A method that improves returns 1.0
24
+ (clamped — improvement does not extend beyond perfect drift resistance).
25
+ """
26
+ deltas = drift_deltas(cycle_scores)
27
+ if not deltas:
28
+ return 1.0
29
+ avg = sum(deltas) / len(deltas)
30
+ return max(0.0, min(1.0, 1.0 + avg))
@@ -0,0 +1,7 @@
1
+ """Scoring-specific exceptions."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class ScoringError(Exception):
7
+ """Raised when scoring inputs are malformed or a check cannot run."""
@@ -0,0 +1,77 @@
1
+ """Orchestrate per-cycle scoring: checks → weighted aggregate → penalty → scorecard.
2
+
3
+ Weights are locked in docs/architecture/decisions.md §B3. Changing them is a
4
+ scorer-version bump.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from compactbench.contracts import (
10
+ CompactionArtifact,
11
+ EvaluationItem,
12
+ EvaluationItemType,
13
+ GeneratedCase,
14
+ ItemScore,
15
+ Scorecard,
16
+ )
17
+ from compactbench.scoring.checks import run_check
18
+ from compactbench.scoring.compression import compression_ratio
19
+ from compactbench.scoring.contradictions import contradiction_rate
20
+
21
+ WEIGHTS: dict[EvaluationItemType, float] = {
22
+ EvaluationItemType.LOCKED_DECISION_RETENTION: 3.0,
23
+ EvaluationItemType.FORBIDDEN_BEHAVIOR_RETENTION: 3.0,
24
+ EvaluationItemType.IMMUTABLE_FACT_RECALL: 2.0,
25
+ EvaluationItemType.UNRESOLVED_TASK_CONTINUITY: 2.0,
26
+ EvaluationItemType.ENTITY_INTEGRITY: 1.0,
27
+ EvaluationItemType.PLANNING_SOUNDNESS: 1.0,
28
+ }
29
+
30
+
31
+ def score_item(item: EvaluationItem, response: str) -> ItemScore:
32
+ """Score a single evaluation item against a response."""
33
+ check_type = str(item.expected.get("check", "unknown"))
34
+ raw = run_check(item.expected, response)
35
+ return ItemScore(
36
+ item_key=item.key,
37
+ item_type=item.item_type.value,
38
+ score=raw,
39
+ weight=WEIGHTS.get(item.item_type, 1.0),
40
+ check_type=check_type,
41
+ details={},
42
+ )
43
+
44
+
45
+ def _weighted_cycle_score(item_scores: list[ItemScore]) -> float:
46
+ if not item_scores:
47
+ return 0.0
48
+ total_weight = sum(i.weight for i in item_scores)
49
+ if total_weight <= 0:
50
+ return 0.0
51
+ return sum(i.score * i.weight for i in item_scores) / total_weight
52
+
53
+
54
+ def score_cycle(
55
+ case: GeneratedCase,
56
+ artifact: CompactionArtifact,
57
+ responses: dict[str, str],
58
+ cycle_number: int = 0,
59
+ ) -> Scorecard:
60
+ """Score one cycle of a compaction run and return a :class:`Scorecard`."""
61
+ if cycle_number < 0:
62
+ raise ValueError(f"cycle_number must be >= 0, got {cycle_number}")
63
+
64
+ item_scores = [score_item(item, responses.get(item.key, "")) for item in case.evaluation_items]
65
+ cycle_score = _weighted_cycle_score(item_scores)
66
+ contra = contradiction_rate(case.evaluation_items, responses, case.ground_truth)
67
+ penalized = max(0.0, min(1.0, cycle_score * (1.0 - contra)))
68
+ comp_ratio = compression_ratio(case.transcript, artifact)
69
+
70
+ return Scorecard(
71
+ cycle_number=cycle_number,
72
+ cycle_score=cycle_score,
73
+ penalized_cycle_score=penalized,
74
+ contradiction_rate=contra,
75
+ compression_ratio=comp_ratio,
76
+ item_scores=item_scores,
77
+ )
@@ -0,0 +1 @@
1
+ """Shared utilities: tokenizer wrapper, structured logging."""