compactbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compactbench/__init__.py +10 -0
- compactbench/__main__.py +6 -0
- compactbench/cli.py +343 -0
- compactbench/compactors/__init__.py +47 -0
- compactbench/compactors/_state_parser.py +118 -0
- compactbench/compactors/_utils.py +37 -0
- compactbench/compactors/base.py +38 -0
- compactbench/compactors/errors.py +11 -0
- compactbench/compactors/hierarchical_summary.py +113 -0
- compactbench/compactors/hybrid_ledger.py +115 -0
- compactbench/compactors/naive_summary.py +54 -0
- compactbench/compactors/structured_state.py +64 -0
- compactbench/config.py +36 -0
- compactbench/contracts/__init__.py +41 -0
- compactbench/contracts/artifact.py +49 -0
- compactbench/contracts/case.py +81 -0
- compactbench/contracts/result.py +79 -0
- compactbench/dsl/__init__.py +66 -0
- compactbench/dsl/errors.py +23 -0
- compactbench/dsl/generators.py +243 -0
- compactbench/dsl/models.py +107 -0
- compactbench/dsl/parser.py +55 -0
- compactbench/dsl/substitution.py +32 -0
- compactbench/dsl/validator.py +97 -0
- compactbench/engine/__init__.py +29 -0
- compactbench/engine/difficulty.py +54 -0
- compactbench/engine/distractors.py +69 -0
- compactbench/engine/errors.py +11 -0
- compactbench/engine/generation.py +96 -0
- compactbench/engine/seeds.py +31 -0
- compactbench/engine/transcript.py +84 -0
- compactbench/leaderboard/__init__.py +38 -0
- compactbench/leaderboard/errors.py +11 -0
- compactbench/leaderboard/projection.py +103 -0
- compactbench/leaderboard/qualification.py +90 -0
- compactbench/leaderboard/ranking.py +71 -0
- compactbench/providers/__init__.py +52 -0
- compactbench/providers/_retry.py +48 -0
- compactbench/providers/base.py +44 -0
- compactbench/providers/errors.py +15 -0
- compactbench/providers/google_ai_studio.py +112 -0
- compactbench/providers/groq.py +95 -0
- compactbench/providers/mock.py +53 -0
- compactbench/providers/ollama.py +100 -0
- compactbench/py.typed +0 -0
- compactbench/runner/__init__.py +53 -0
- compactbench/runner/_resolver.py +57 -0
- compactbench/runner/continuation.py +78 -0
- compactbench/runner/cycle.py +67 -0
- compactbench/runner/errors.py +15 -0
- compactbench/runner/evaluation.py +71 -0
- compactbench/runner/persistence.py +227 -0
- compactbench/runner/run.py +257 -0
- compactbench/scoring/__init__.py +48 -0
- compactbench/scoring/checks.py +92 -0
- compactbench/scoring/compression.py +66 -0
- compactbench/scoring/contradictions.py +70 -0
- compactbench/scoring/drift.py +30 -0
- compactbench/scoring/errors.py +7 -0
- compactbench/scoring/scorer.py +77 -0
- compactbench/utils/__init__.py +1 -0
- compactbench-0.1.0.dist-info/METADATA +161 -0
- compactbench-0.1.0.dist-info/RECORD +66 -0
- compactbench-0.1.0.dist-info/WHEEL +4 -0
- compactbench-0.1.0.dist-info/entry_points.txt +2 -0
- compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Resolve difficulty knobs from a ``DifficultyPolicy`` at generation time."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from compactbench.dsl import DifficultyLevel, DifficultyPolicy
|
|
6
|
+
from compactbench.engine.errors import GenerationError
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def distractor_count_for(policy: DifficultyPolicy, difficulty: DifficultyLevel) -> int:
|
|
10
|
+
"""Return the distractor-turn count for ``difficulty``.
|
|
11
|
+
|
|
12
|
+
If the policy configures no distractor levels, returns 0. If the policy is
|
|
13
|
+
configured but does not include the chosen level, raises
|
|
14
|
+
:class:`GenerationError`.
|
|
15
|
+
"""
|
|
16
|
+
mapping = policy.distractor_turns
|
|
17
|
+
if not mapping:
|
|
18
|
+
return 0
|
|
19
|
+
if difficulty not in mapping:
|
|
20
|
+
raise GenerationError(
|
|
21
|
+
f"template does not configure distractor_turns for difficulty "
|
|
22
|
+
f"{difficulty.value!r}. Configured levels: "
|
|
23
|
+
f"{sorted(lvl.value for lvl in mapping)}"
|
|
24
|
+
)
|
|
25
|
+
return mapping[difficulty]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def paraphrase_depth_for(policy: DifficultyPolicy, difficulty: DifficultyLevel) -> int:
|
|
29
|
+
"""Return the paraphrase depth for ``difficulty`` (0 when unconfigured).
|
|
30
|
+
|
|
31
|
+
Actual paraphrasing is a no-op in v1; this value is surfaced as a binding
|
|
32
|
+
so templates can reference it, but no transform is applied yet.
|
|
33
|
+
"""
|
|
34
|
+
return policy.paraphrase_depth.get(difficulty, 0)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def override_timing_for(policy: DifficultyPolicy, difficulty: DifficultyLevel) -> str:
|
|
38
|
+
"""Return the override timing for ``difficulty`` (``""`` when unconfigured).
|
|
39
|
+
|
|
40
|
+
Like paraphrase depth, this is a v1 no-op passthrough.
|
|
41
|
+
"""
|
|
42
|
+
return policy.override_timing.get(difficulty, "")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def difficulty_bindings(policy: DifficultyPolicy, difficulty: DifficultyLevel) -> dict[str, str]:
|
|
46
|
+
"""Return bindings for ``difficulty.*`` references in template strings."""
|
|
47
|
+
bindings: dict[str, str] = {
|
|
48
|
+
"difficulty.paraphrase_depth": str(paraphrase_depth_for(policy, difficulty)),
|
|
49
|
+
"difficulty.override_timing": override_timing_for(policy, difficulty),
|
|
50
|
+
}
|
|
51
|
+
# distractor_turns may not be configured; bind the literal string form when it is.
|
|
52
|
+
if policy.distractor_turns:
|
|
53
|
+
bindings["difficulty.distractor_turns"] = str(distractor_count_for(policy, difficulty))
|
|
54
|
+
return bindings
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Distractor turn generation.
|
|
2
|
+
|
|
3
|
+
Distractor blocks expand into plausible-sounding off-topic conversational
|
|
4
|
+
turns. Their purpose is to act as noise between critical turns so the
|
|
5
|
+
compactor has to filter signal from filler.
|
|
6
|
+
|
|
7
|
+
Content is drawn from bounded lexicons and alternates user/assistant.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import random
|
|
13
|
+
|
|
14
|
+
from compactbench.contracts import Turn, TurnRole
|
|
15
|
+
|
|
16
|
+
_USER_DISTRACTORS: tuple[str, ...] = (
|
|
17
|
+
"By the way, how long have you been working with me today?",
|
|
18
|
+
"Quick question unrelated to this: what time is it where you are?",
|
|
19
|
+
"I just remembered I need to send someone an email later.",
|
|
20
|
+
"Random thought: do you have opinions on coffee versus tea?",
|
|
21
|
+
"Oh, did you see that article about electric cars?",
|
|
22
|
+
"Side note — my internet has been slow today.",
|
|
23
|
+
"What's the weather like lately?",
|
|
24
|
+
"Unrelated, but what do you think of remote work in general?",
|
|
25
|
+
"By the way, do you read much fiction?",
|
|
26
|
+
"Quick tangent: is it true that pandas have thumbs?",
|
|
27
|
+
"I was thinking about lunch earlier.",
|
|
28
|
+
"Off-topic, but any thoughts on daylight saving time?",
|
|
29
|
+
"Random: I keep hearing about this one podcast.",
|
|
30
|
+
"Unrelated question — do you know anything about gardening?",
|
|
31
|
+
"Do you happen to know what a quokka is?",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
_ASSISTANT_DISTRACTORS: tuple[str, ...] = (
|
|
35
|
+
"Happy to chat about that briefly — anything else on the main topic?",
|
|
36
|
+
"I can answer, but let me know when you want to return to the main question.",
|
|
37
|
+
"Quick aside, sure. Anything else pressing?",
|
|
38
|
+
"Got it. Let me know if you want to switch focus back.",
|
|
39
|
+
"Sure, I can address that. Is there anything more on the main thread?",
|
|
40
|
+
"Understood. Shall we continue where we left off?",
|
|
41
|
+
"Acknowledged. Ready when you are to move forward.",
|
|
42
|
+
"Fair point. Want me to refocus on the earlier question?",
|
|
43
|
+
"Noted. Do you want to get back to the plan?",
|
|
44
|
+
"I hear you. Anything else to cover on the primary question?",
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def generate_distractor_turns(count: int, starting_turn_id: int, seed: int) -> list[Turn]:
|
|
49
|
+
"""Produce ``count`` distractor turns alternating user/assistant.
|
|
50
|
+
|
|
51
|
+
IDs start at ``starting_turn_id`` and increment by one. Role alternation
|
|
52
|
+
starts with ``user``.
|
|
53
|
+
"""
|
|
54
|
+
if count < 0:
|
|
55
|
+
raise ValueError(f"count must be >= 0, got {count}")
|
|
56
|
+
if count == 0:
|
|
57
|
+
return []
|
|
58
|
+
rng = random.Random(seed)
|
|
59
|
+
turns: list[Turn] = []
|
|
60
|
+
for i in range(count):
|
|
61
|
+
turn_id = starting_turn_id + i
|
|
62
|
+
if i % 2 == 0:
|
|
63
|
+
role = TurnRole.USER
|
|
64
|
+
content = rng.choice(_USER_DISTRACTORS)
|
|
65
|
+
else:
|
|
66
|
+
role = TurnRole.ASSISTANT
|
|
67
|
+
content = rng.choice(_ASSISTANT_DISTRACTORS)
|
|
68
|
+
turns.append(Turn(id=turn_id, role=role, content=content, tags=["distractor"]))
|
|
69
|
+
return turns
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Exceptions raised by the case generation engine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class EngineError(Exception):
|
|
7
|
+
"""Base exception for anything engine-related."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class GenerationError(EngineError):
|
|
11
|
+
"""Case generation failed (unsupported expression, missing config, etc.)."""
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Top-level case generation: ``TemplateDefinition`` + seed + difficulty -> ``GeneratedCase``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from compactbench.contracts import (
|
|
8
|
+
EvaluationItem,
|
|
9
|
+
EvaluationItemType,
|
|
10
|
+
GeneratedCase,
|
|
11
|
+
GroundTruth,
|
|
12
|
+
)
|
|
13
|
+
from compactbench.dsl import (
|
|
14
|
+
DifficultyLevel,
|
|
15
|
+
EvaluationItemTemplate,
|
|
16
|
+
GroundTruthTemplate,
|
|
17
|
+
TemplateDefinition,
|
|
18
|
+
resolve_variables,
|
|
19
|
+
substitute,
|
|
20
|
+
)
|
|
21
|
+
from compactbench.engine.difficulty import difficulty_bindings
|
|
22
|
+
from compactbench.engine.errors import GenerationError
|
|
23
|
+
from compactbench.engine.transcript import build_transcript
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def generate_case(
|
|
27
|
+
template: TemplateDefinition,
|
|
28
|
+
seed: int,
|
|
29
|
+
difficulty: DifficultyLevel = DifficultyLevel.MEDIUM,
|
|
30
|
+
*,
|
|
31
|
+
case_id: str | None = None,
|
|
32
|
+
) -> GeneratedCase:
|
|
33
|
+
"""Generate a concrete :class:`GeneratedCase` from a template, seed, and difficulty.
|
|
34
|
+
|
|
35
|
+
Pure function of its inputs. No filesystem, network, or global state is
|
|
36
|
+
touched beyond the module-level generator registry in
|
|
37
|
+
:mod:`compactbench.dsl.generators`.
|
|
38
|
+
"""
|
|
39
|
+
bindings = resolve_variables(list(template.variables), seed)
|
|
40
|
+
bindings.update(difficulty_bindings(template.difficulty_policy, difficulty))
|
|
41
|
+
|
|
42
|
+
transcript = build_transcript(template, bindings, seed, difficulty)
|
|
43
|
+
ground_truth = _build_ground_truth(template.ground_truth, bindings)
|
|
44
|
+
evaluation_items = _build_evaluation_items(template.evaluation_items, bindings)
|
|
45
|
+
|
|
46
|
+
return GeneratedCase(
|
|
47
|
+
case_id=case_id or f"{template.key}:s{seed}:d{difficulty.value}",
|
|
48
|
+
template_key=template.key,
|
|
49
|
+
template_version=template.version,
|
|
50
|
+
seed=seed,
|
|
51
|
+
difficulty=difficulty.value,
|
|
52
|
+
transcript=transcript,
|
|
53
|
+
ground_truth=ground_truth,
|
|
54
|
+
evaluation_items=evaluation_items,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _build_ground_truth(template: GroundTruthTemplate, bindings: dict[str, str]) -> GroundTruth:
|
|
59
|
+
def _sub_list(items: list[str]) -> list[str]:
|
|
60
|
+
return [substitute(s, bindings) for s in items]
|
|
61
|
+
|
|
62
|
+
return GroundTruth(
|
|
63
|
+
immutable_facts=_sub_list(template.immutable_facts),
|
|
64
|
+
locked_decisions=_sub_list(template.locked_decisions),
|
|
65
|
+
forbidden_behaviors=_sub_list(template.forbidden_behaviors),
|
|
66
|
+
unresolved_items=_sub_list(template.unresolved_items),
|
|
67
|
+
entity_map={
|
|
68
|
+
substitute(k, bindings): substitute(v, bindings) for k, v in template.entity_map.items()
|
|
69
|
+
},
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _build_evaluation_items(
|
|
74
|
+
templates: list[EvaluationItemTemplate], bindings: dict[str, str]
|
|
75
|
+
) -> list[EvaluationItem]:
|
|
76
|
+
results: list[EvaluationItem] = []
|
|
77
|
+
for item in templates:
|
|
78
|
+
try:
|
|
79
|
+
item_type = EvaluationItemType(item.type)
|
|
80
|
+
except ValueError as exc:
|
|
81
|
+
raise GenerationError(
|
|
82
|
+
f"evaluation item {item.key!r}: unknown type {item.type!r}. "
|
|
83
|
+
f"Known: {[t.value for t in EvaluationItemType]}"
|
|
84
|
+
) from exc
|
|
85
|
+
expected: dict[str, Any] = {}
|
|
86
|
+
for k, v in item.expected.items():
|
|
87
|
+
expected[k] = substitute(v, bindings) if isinstance(v, str) else v
|
|
88
|
+
results.append(
|
|
89
|
+
EvaluationItem(
|
|
90
|
+
key=item.key,
|
|
91
|
+
item_type=item_type,
|
|
92
|
+
prompt=substitute(item.prompt, bindings),
|
|
93
|
+
expected=expected,
|
|
94
|
+
)
|
|
95
|
+
)
|
|
96
|
+
return results
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Seed derivation for case generation.
|
|
2
|
+
|
|
3
|
+
A *case seed* is derived from ``(suite_version, seed_group, case_slot)`` via
|
|
4
|
+
SHA-256. This is the seed that downstream :mod:`compactbench.dsl.generators`
|
|
5
|
+
uses to resolve template variables.
|
|
6
|
+
|
|
7
|
+
Stable across Python versions and platforms: the derivation uses UTF-8 byte
|
|
8
|
+
encoding, SHA-256, and a big-endian integer decode of the leading 8 bytes.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def derive_case_seed(suite_version: str, seed_group: str, case_slot: int) -> int:
|
|
17
|
+
"""Derive a deterministic seed for a specific case slot.
|
|
18
|
+
|
|
19
|
+
Parameters
|
|
20
|
+
----------
|
|
21
|
+
suite_version:
|
|
22
|
+
Identifier of the benchmark suite version (e.g. ``"starter@1.0.0"``).
|
|
23
|
+
seed_group:
|
|
24
|
+
Identifier of the seed group within this suite version
|
|
25
|
+
(e.g. ``"default"`` or ``"elite_ranked_q1"``).
|
|
26
|
+
case_slot:
|
|
27
|
+
Zero-based case slot index.
|
|
28
|
+
"""
|
|
29
|
+
payload = f"{suite_version}:{seed_group}:{case_slot}".encode()
|
|
30
|
+
digest = hashlib.sha256(payload).digest()
|
|
31
|
+
return int.from_bytes(digest[:8], "big", signed=False)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Build a concrete :class:`Transcript` from a template and bindings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from compactbench.contracts import Transcript, Turn, TurnRole
|
|
6
|
+
from compactbench.dsl import (
|
|
7
|
+
DifficultyLevel,
|
|
8
|
+
TemplateDefinition,
|
|
9
|
+
TemplateTurnRole,
|
|
10
|
+
derive_seed,
|
|
11
|
+
extract_references,
|
|
12
|
+
substitute,
|
|
13
|
+
)
|
|
14
|
+
from compactbench.engine.difficulty import distractor_count_for
|
|
15
|
+
from compactbench.engine.distractors import generate_distractor_turns
|
|
16
|
+
from compactbench.engine.errors import GenerationError
|
|
17
|
+
|
|
18
|
+
_TEMPLATE_TO_TURN_ROLE: dict[TemplateTurnRole, TurnRole] = {
|
|
19
|
+
TemplateTurnRole.USER: TurnRole.USER,
|
|
20
|
+
TemplateTurnRole.ASSISTANT: TurnRole.ASSISTANT,
|
|
21
|
+
TemplateTurnRole.SYSTEM: TurnRole.SYSTEM,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def build_transcript(
|
|
26
|
+
template: TemplateDefinition,
|
|
27
|
+
bindings: dict[str, str],
|
|
28
|
+
case_seed: int,
|
|
29
|
+
difficulty: DifficultyLevel,
|
|
30
|
+
) -> Transcript:
|
|
31
|
+
"""Expand a template transcript into concrete turns with variables resolved."""
|
|
32
|
+
turns: list[Turn] = []
|
|
33
|
+
next_id = 0
|
|
34
|
+
|
|
35
|
+
for index, template_turn in enumerate(template.transcript.turns):
|
|
36
|
+
if template_turn.role is TemplateTurnRole.DISTRACTOR_BLOCK:
|
|
37
|
+
count = _resolve_count(
|
|
38
|
+
template_turn.count,
|
|
39
|
+
template,
|
|
40
|
+
difficulty,
|
|
41
|
+
location=f"transcript.turns[{index}].count",
|
|
42
|
+
)
|
|
43
|
+
sub_seed = derive_seed(case_seed, f"distractor_block_{index}")
|
|
44
|
+
turns.extend(generate_distractor_turns(count, next_id, sub_seed))
|
|
45
|
+
next_id += count
|
|
46
|
+
continue
|
|
47
|
+
|
|
48
|
+
if template_turn.template is None:
|
|
49
|
+
raise GenerationError(
|
|
50
|
+
f"transcript.turns[{index}] has role {template_turn.role.value!r} "
|
|
51
|
+
f"but no 'template' field"
|
|
52
|
+
)
|
|
53
|
+
content = substitute(template_turn.template, bindings)
|
|
54
|
+
turns.append(
|
|
55
|
+
Turn(
|
|
56
|
+
id=next_id,
|
|
57
|
+
role=_TEMPLATE_TO_TURN_ROLE[template_turn.role],
|
|
58
|
+
content=content,
|
|
59
|
+
tags=list(template_turn.tags),
|
|
60
|
+
)
|
|
61
|
+
)
|
|
62
|
+
next_id += 1
|
|
63
|
+
|
|
64
|
+
return Transcript(turns=turns)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _resolve_count(
|
|
68
|
+
count: str | int | None,
|
|
69
|
+
template: TemplateDefinition,
|
|
70
|
+
difficulty: DifficultyLevel,
|
|
71
|
+
*,
|
|
72
|
+
location: str,
|
|
73
|
+
) -> int:
|
|
74
|
+
if count is None:
|
|
75
|
+
raise GenerationError(f"{location}: missing count for distractor_block")
|
|
76
|
+
if isinstance(count, int):
|
|
77
|
+
if count < 0:
|
|
78
|
+
raise GenerationError(f"{location}: negative count {count}")
|
|
79
|
+
return count
|
|
80
|
+
|
|
81
|
+
refs = extract_references(count)
|
|
82
|
+
if refs == ["difficulty.distractor_turns"]:
|
|
83
|
+
return distractor_count_for(template.difficulty_policy, difficulty)
|
|
84
|
+
raise GenerationError(f"{location}: unsupported count expression {count!r}")
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Leaderboard ranking, qualification, and public projection.
|
|
2
|
+
|
|
3
|
+
Formulas locked in docs/architecture/decisions.md §B4.
|
|
4
|
+
Implemented in WO-008.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from compactbench.leaderboard.errors import LeaderboardError, QualificationError
|
|
8
|
+
from compactbench.leaderboard.projection import LeaderboardRow, project_row, rank_rows
|
|
9
|
+
from compactbench.leaderboard.qualification import (
|
|
10
|
+
MAX_CONTRADICTION_RATE,
|
|
11
|
+
MIN_FAMILY_PASS_RATE,
|
|
12
|
+
QualificationResult,
|
|
13
|
+
qualify,
|
|
14
|
+
)
|
|
15
|
+
from compactbench.leaderboard.ranking import (
|
|
16
|
+
TIER_FLOORS,
|
|
17
|
+
CompressionTier,
|
|
18
|
+
compression_bonus,
|
|
19
|
+
elite_score,
|
|
20
|
+
rank_key,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"MAX_CONTRADICTION_RATE",
|
|
25
|
+
"MIN_FAMILY_PASS_RATE",
|
|
26
|
+
"TIER_FLOORS",
|
|
27
|
+
"CompressionTier",
|
|
28
|
+
"LeaderboardError",
|
|
29
|
+
"LeaderboardRow",
|
|
30
|
+
"QualificationError",
|
|
31
|
+
"QualificationResult",
|
|
32
|
+
"compression_bonus",
|
|
33
|
+
"elite_score",
|
|
34
|
+
"project_row",
|
|
35
|
+
"qualify",
|
|
36
|
+
"rank_key",
|
|
37
|
+
"rank_rows",
|
|
38
|
+
]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Leaderboard-specific exceptions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class LeaderboardError(Exception):
|
|
7
|
+
"""Base exception for leaderboard operations."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class QualificationError(LeaderboardError):
|
|
11
|
+
"""Raised (or returned as a failure reason) when a run fails qualification floors."""
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Project run results into public leaderboard rows.
|
|
2
|
+
|
|
3
|
+
The public leaderboard never exposes hidden template content — only aggregate
|
|
4
|
+
metrics, method identity, and versions. Raw case outputs are not projected.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from typing import TypedDict
|
|
11
|
+
|
|
12
|
+
from compactbench.contracts import RunResult
|
|
13
|
+
from compactbench.leaderboard.ranking import (
|
|
14
|
+
CompressionTier,
|
|
15
|
+
elite_score,
|
|
16
|
+
rank_key,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LeaderboardRow(TypedDict):
|
|
21
|
+
"""Public row shape. Keep this stable — it is the public contract."""
|
|
22
|
+
|
|
23
|
+
rank: int | None
|
|
24
|
+
method_name: str
|
|
25
|
+
method_version: str
|
|
26
|
+
handle: str | None
|
|
27
|
+
org: str | None
|
|
28
|
+
tier: str
|
|
29
|
+
benchmark_version: str
|
|
30
|
+
target_provider: str
|
|
31
|
+
target_model: str
|
|
32
|
+
scorer_version: str
|
|
33
|
+
elite_score: float
|
|
34
|
+
overall_score: float
|
|
35
|
+
drift_resistance: float
|
|
36
|
+
constraint_retention: float
|
|
37
|
+
contradiction_rate: float
|
|
38
|
+
compression_ratio: float
|
|
39
|
+
published_at: str
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def project_row(
|
|
43
|
+
run_result: RunResult,
|
|
44
|
+
*,
|
|
45
|
+
tier: CompressionTier,
|
|
46
|
+
handle: str | None,
|
|
47
|
+
org: str | None,
|
|
48
|
+
published_at: datetime,
|
|
49
|
+
) -> LeaderboardRow:
|
|
50
|
+
"""Turn a qualified :class:`RunResult` into a :class:`LeaderboardRow`.
|
|
51
|
+
|
|
52
|
+
``rank`` is set to ``None`` here; caller assigns ranks after sorting.
|
|
53
|
+
"""
|
|
54
|
+
score = elite_score(
|
|
55
|
+
overall_score=run_result.overall_score,
|
|
56
|
+
drift_resistance=run_result.drift_resistance,
|
|
57
|
+
constraint_retention=run_result.constraint_retention,
|
|
58
|
+
compression_ratio=run_result.compression_ratio,
|
|
59
|
+
tier=tier,
|
|
60
|
+
)
|
|
61
|
+
return LeaderboardRow(
|
|
62
|
+
rank=None,
|
|
63
|
+
method_name=run_result.method_name,
|
|
64
|
+
method_version=run_result.method_version,
|
|
65
|
+
handle=handle,
|
|
66
|
+
org=org,
|
|
67
|
+
tier=tier,
|
|
68
|
+
benchmark_version=f"{run_result.suite_key}@{run_result.suite_version}",
|
|
69
|
+
target_provider=run_result.target_provider,
|
|
70
|
+
target_model=run_result.target_model,
|
|
71
|
+
scorer_version=run_result.scorer_version,
|
|
72
|
+
elite_score=score,
|
|
73
|
+
overall_score=run_result.overall_score,
|
|
74
|
+
drift_resistance=run_result.drift_resistance,
|
|
75
|
+
constraint_retention=run_result.constraint_retention,
|
|
76
|
+
contradiction_rate=run_result.contradiction_rate,
|
|
77
|
+
compression_ratio=run_result.compression_ratio,
|
|
78
|
+
published_at=published_at.isoformat(),
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def rank_rows(rows: list[LeaderboardRow]) -> list[LeaderboardRow]:
|
|
83
|
+
"""Return ``rows`` sorted best-first with ``rank`` assigned.
|
|
84
|
+
|
|
85
|
+
Segmentation by ``(benchmark_version, target_model)`` is the caller's
|
|
86
|
+
responsibility — this function only sorts and numbers whatever it is given.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
def _key(row: LeaderboardRow) -> tuple[float, float, float, float, float]:
|
|
90
|
+
return rank_key(
|
|
91
|
+
elite_score_value=row["elite_score"],
|
|
92
|
+
drift_resistance=row["drift_resistance"],
|
|
93
|
+
constraint_retention=row["constraint_retention"],
|
|
94
|
+
contradiction_rate=row["contradiction_rate"],
|
|
95
|
+
published_at=datetime.fromisoformat(row["published_at"]),
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
ranked: list[LeaderboardRow] = []
|
|
99
|
+
for i, row in enumerate(sorted(rows, key=_key), start=1):
|
|
100
|
+
ranked_row: LeaderboardRow = dict(row) # pyright: ignore[reportAssignmentType]
|
|
101
|
+
ranked_row["rank"] = i
|
|
102
|
+
ranked.append(ranked_row)
|
|
103
|
+
return ranked
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Qualification floor checks for leaderboard entries.
|
|
2
|
+
|
|
3
|
+
Floors locked in docs/architecture/decisions.md §B4.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
|
|
10
|
+
from compactbench.contracts import RunResult
|
|
11
|
+
from compactbench.leaderboard.ranking import TIER_FLOORS, CompressionTier
|
|
12
|
+
|
|
13
|
+
MAX_CONTRADICTION_RATE: float = 0.10
|
|
14
|
+
MIN_FAMILY_PASS_RATE: float = 0.40
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class QualificationResult:
|
|
19
|
+
"""Outcome of checking a run against leaderboard qualification floors."""
|
|
20
|
+
|
|
21
|
+
qualified: bool
|
|
22
|
+
reasons: list[str] = field(default_factory=list[str])
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def qualify(
|
|
26
|
+
run_result: RunResult,
|
|
27
|
+
*,
|
|
28
|
+
tier: CompressionTier,
|
|
29
|
+
expected_drift_cycles: int,
|
|
30
|
+
) -> QualificationResult:
|
|
31
|
+
"""Check a :class:`RunResult` against the leaderboard floors.
|
|
32
|
+
|
|
33
|
+
Returns a :class:`QualificationResult` — inspect ``reasons`` when
|
|
34
|
+
``qualified`` is ``False`` to learn why.
|
|
35
|
+
"""
|
|
36
|
+
reasons: list[str] = []
|
|
37
|
+
|
|
38
|
+
tier_floor = TIER_FLOORS[tier]
|
|
39
|
+
if run_result.compression_ratio < tier_floor:
|
|
40
|
+
reasons.append(
|
|
41
|
+
f"compression {run_result.compression_ratio:.2f}x is below the {tier} "
|
|
42
|
+
f"floor of {tier_floor:.1f}x"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
if run_result.contradiction_rate > MAX_CONTRADICTION_RATE:
|
|
46
|
+
reasons.append(
|
|
47
|
+
f"contradiction_rate {run_result.contradiction_rate:.3f} exceeds the "
|
|
48
|
+
f"{MAX_CONTRADICTION_RATE:.2f} maximum"
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
if not run_result.cases:
|
|
52
|
+
reasons.append("no cases completed")
|
|
53
|
+
else:
|
|
54
|
+
expected_cycles = expected_drift_cycles + 1
|
|
55
|
+
for case in run_result.cases:
|
|
56
|
+
if len(case.cycles) < expected_cycles:
|
|
57
|
+
reasons.append(
|
|
58
|
+
f"case {case.case_id!r} completed only {len(case.cycles)} of "
|
|
59
|
+
f"{expected_cycles} configured cycles"
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
# Per-family pass-rate guard only applies when the run covers more than one family.
|
|
63
|
+
family_rates = _family_pass_rates(run_result)
|
|
64
|
+
if len(family_rates) > 1:
|
|
65
|
+
for family, pass_rate in family_rates.items():
|
|
66
|
+
if pass_rate < MIN_FAMILY_PASS_RATE:
|
|
67
|
+
reasons.append(
|
|
68
|
+
f"family {family!r} pass rate {pass_rate:.2f} is below the "
|
|
69
|
+
f"{MIN_FAMILY_PASS_RATE:.2f} minimum (category-diversity guard)"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
return QualificationResult(qualified=not reasons, reasons=reasons)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _family_pass_rates(run_result: RunResult) -> dict[str, float]:
|
|
76
|
+
"""Mean case_score grouped by benchmark family inferred from template_key."""
|
|
77
|
+
groups: dict[str, list[float]] = {}
|
|
78
|
+
for case in run_result.cases:
|
|
79
|
+
family = _infer_family(case.template_key)
|
|
80
|
+
groups.setdefault(family, []).append(case.case_score)
|
|
81
|
+
return {family: sum(scores) / len(scores) for family, scores in groups.items() if scores}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _infer_family(template_key: str) -> str:
|
|
85
|
+
"""Infer family name from template key by stripping trailing ``_starter_v<N>`` or ``_v<N>``."""
|
|
86
|
+
for suffix_prefix in ("_starter_v", "_elite_v", "_v"):
|
|
87
|
+
idx = template_key.rfind(suffix_prefix)
|
|
88
|
+
if idx > 0:
|
|
89
|
+
return template_key[:idx]
|
|
90
|
+
return template_key
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Elite score and tie-breaker computation.
|
|
2
|
+
|
|
3
|
+
Formulas are locked in docs/architecture/decisions.md §B4. Any change is a
|
|
4
|
+
scorer-version bump.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from typing import Literal
|
|
11
|
+
|
|
12
|
+
CompressionTier = Literal["Elite-Light", "Elite-Mid", "Elite-Aggressive"]
|
|
13
|
+
|
|
14
|
+
TIER_FLOORS: dict[CompressionTier, float] = {
|
|
15
|
+
"Elite-Light": 2.0,
|
|
16
|
+
"Elite-Mid": 4.0,
|
|
17
|
+
"Elite-Aggressive": 8.0,
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def compression_bonus(compression_ratio: float, tier: CompressionTier) -> float:
|
|
22
|
+
"""Normalized bonus ``[0, 1]`` for exceeding the tier's compression floor.
|
|
23
|
+
|
|
24
|
+
0 at exactly the floor, 1 at 2x the floor, capped.
|
|
25
|
+
"""
|
|
26
|
+
floor = TIER_FLOORS[tier]
|
|
27
|
+
if compression_ratio <= floor:
|
|
28
|
+
return 0.0
|
|
29
|
+
raw = (compression_ratio - floor) / floor
|
|
30
|
+
return max(0.0, min(1.0, raw))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def elite_score(
|
|
34
|
+
*,
|
|
35
|
+
overall_score: float,
|
|
36
|
+
drift_resistance: float,
|
|
37
|
+
constraint_retention: float,
|
|
38
|
+
compression_ratio: float,
|
|
39
|
+
tier: CompressionTier,
|
|
40
|
+
) -> float:
|
|
41
|
+
"""Compute ``elite_score`` per decisions.md §B4.
|
|
42
|
+
|
|
43
|
+
``elite_score = 0.40 * overall + 0.30 * drift + 0.20 * constraint + 0.10 * compression_bonus``
|
|
44
|
+
"""
|
|
45
|
+
bonus = compression_bonus(compression_ratio, tier)
|
|
46
|
+
return (
|
|
47
|
+
0.40 * overall_score + 0.30 * drift_resistance + 0.20 * constraint_retention + 0.10 * bonus
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def rank_key(
|
|
52
|
+
*,
|
|
53
|
+
elite_score_value: float,
|
|
54
|
+
drift_resistance: float,
|
|
55
|
+
constraint_retention: float,
|
|
56
|
+
contradiction_rate: float,
|
|
57
|
+
published_at: datetime,
|
|
58
|
+
) -> tuple[float, float, float, float, float]:
|
|
59
|
+
"""Return a sort key that orders better runs FIRST when sorted ascending.
|
|
60
|
+
|
|
61
|
+
Primary: higher elite_score. Tie-breakers in order:
|
|
62
|
+
higher drift_resistance → higher constraint_retention → lower contradiction_rate
|
|
63
|
+
→ earlier published_at.
|
|
64
|
+
"""
|
|
65
|
+
return (
|
|
66
|
+
-elite_score_value,
|
|
67
|
+
-drift_resistance,
|
|
68
|
+
-constraint_retention,
|
|
69
|
+
contradiction_rate,
|
|
70
|
+
published_at.timestamp(),
|
|
71
|
+
)
|