compactbench 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. compactbench/__init__.py +10 -0
  2. compactbench/__main__.py +6 -0
  3. compactbench/cli.py +343 -0
  4. compactbench/compactors/__init__.py +47 -0
  5. compactbench/compactors/_state_parser.py +118 -0
  6. compactbench/compactors/_utils.py +37 -0
  7. compactbench/compactors/base.py +38 -0
  8. compactbench/compactors/errors.py +11 -0
  9. compactbench/compactors/hierarchical_summary.py +113 -0
  10. compactbench/compactors/hybrid_ledger.py +115 -0
  11. compactbench/compactors/naive_summary.py +54 -0
  12. compactbench/compactors/structured_state.py +64 -0
  13. compactbench/config.py +36 -0
  14. compactbench/contracts/__init__.py +41 -0
  15. compactbench/contracts/artifact.py +49 -0
  16. compactbench/contracts/case.py +81 -0
  17. compactbench/contracts/result.py +79 -0
  18. compactbench/dsl/__init__.py +66 -0
  19. compactbench/dsl/errors.py +23 -0
  20. compactbench/dsl/generators.py +243 -0
  21. compactbench/dsl/models.py +107 -0
  22. compactbench/dsl/parser.py +55 -0
  23. compactbench/dsl/substitution.py +32 -0
  24. compactbench/dsl/validator.py +97 -0
  25. compactbench/engine/__init__.py +29 -0
  26. compactbench/engine/difficulty.py +54 -0
  27. compactbench/engine/distractors.py +69 -0
  28. compactbench/engine/errors.py +11 -0
  29. compactbench/engine/generation.py +96 -0
  30. compactbench/engine/seeds.py +31 -0
  31. compactbench/engine/transcript.py +84 -0
  32. compactbench/leaderboard/__init__.py +38 -0
  33. compactbench/leaderboard/errors.py +11 -0
  34. compactbench/leaderboard/projection.py +103 -0
  35. compactbench/leaderboard/qualification.py +90 -0
  36. compactbench/leaderboard/ranking.py +71 -0
  37. compactbench/providers/__init__.py +52 -0
  38. compactbench/providers/_retry.py +48 -0
  39. compactbench/providers/base.py +44 -0
  40. compactbench/providers/errors.py +15 -0
  41. compactbench/providers/google_ai_studio.py +112 -0
  42. compactbench/providers/groq.py +95 -0
  43. compactbench/providers/mock.py +53 -0
  44. compactbench/providers/ollama.py +100 -0
  45. compactbench/py.typed +0 -0
  46. compactbench/runner/__init__.py +53 -0
  47. compactbench/runner/_resolver.py +57 -0
  48. compactbench/runner/continuation.py +78 -0
  49. compactbench/runner/cycle.py +67 -0
  50. compactbench/runner/errors.py +15 -0
  51. compactbench/runner/evaluation.py +71 -0
  52. compactbench/runner/persistence.py +227 -0
  53. compactbench/runner/run.py +257 -0
  54. compactbench/scoring/__init__.py +48 -0
  55. compactbench/scoring/checks.py +92 -0
  56. compactbench/scoring/compression.py +66 -0
  57. compactbench/scoring/contradictions.py +70 -0
  58. compactbench/scoring/drift.py +30 -0
  59. compactbench/scoring/errors.py +7 -0
  60. compactbench/scoring/scorer.py +77 -0
  61. compactbench/utils/__init__.py +1 -0
  62. compactbench-0.1.0.dist-info/METADATA +161 -0
  63. compactbench-0.1.0.dist-info/RECORD +66 -0
  64. compactbench-0.1.0.dist-info/WHEEL +4 -0
  65. compactbench-0.1.0.dist-info/entry_points.txt +2 -0
  66. compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,54 @@
1
+ """Resolve difficulty knobs from a ``DifficultyPolicy`` at generation time."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from compactbench.dsl import DifficultyLevel, DifficultyPolicy
6
+ from compactbench.engine.errors import GenerationError
7
+
8
+
9
+ def distractor_count_for(policy: DifficultyPolicy, difficulty: DifficultyLevel) -> int:
10
+ """Return the distractor-turn count for ``difficulty``.
11
+
12
+ If the policy configures no distractor levels, returns 0. If the policy is
13
+ configured but does not include the chosen level, raises
14
+ :class:`GenerationError`.
15
+ """
16
+ mapping = policy.distractor_turns
17
+ if not mapping:
18
+ return 0
19
+ if difficulty not in mapping:
20
+ raise GenerationError(
21
+ f"template does not configure distractor_turns for difficulty "
22
+ f"{difficulty.value!r}. Configured levels: "
23
+ f"{sorted(lvl.value for lvl in mapping)}"
24
+ )
25
+ return mapping[difficulty]
26
+
27
+
28
+ def paraphrase_depth_for(policy: DifficultyPolicy, difficulty: DifficultyLevel) -> int:
29
+ """Return the paraphrase depth for ``difficulty`` (0 when unconfigured).
30
+
31
+ Actual paraphrasing is a no-op in v1; this value is surfaced as a binding
32
+ so templates can reference it, but no transform is applied yet.
33
+ """
34
+ return policy.paraphrase_depth.get(difficulty, 0)
35
+
36
+
37
+ def override_timing_for(policy: DifficultyPolicy, difficulty: DifficultyLevel) -> str:
38
+ """Return the override timing for ``difficulty`` (``""`` when unconfigured).
39
+
40
+ Like paraphrase depth, this is a v1 no-op passthrough.
41
+ """
42
+ return policy.override_timing.get(difficulty, "")
43
+
44
+
45
+ def difficulty_bindings(policy: DifficultyPolicy, difficulty: DifficultyLevel) -> dict[str, str]:
46
+ """Return bindings for ``difficulty.*`` references in template strings."""
47
+ bindings: dict[str, str] = {
48
+ "difficulty.paraphrase_depth": str(paraphrase_depth_for(policy, difficulty)),
49
+ "difficulty.override_timing": override_timing_for(policy, difficulty),
50
+ }
51
+ # distractor_turns may not be configured; bind the literal string form when it is.
52
+ if policy.distractor_turns:
53
+ bindings["difficulty.distractor_turns"] = str(distractor_count_for(policy, difficulty))
54
+ return bindings
@@ -0,0 +1,69 @@
1
+ """Distractor turn generation.
2
+
3
+ Distractor blocks expand into plausible-sounding off-topic conversational
4
+ turns. Their purpose is to act as noise between critical turns so the
5
+ compactor has to filter signal from filler.
6
+
7
+ Content is drawn from bounded lexicons and alternates user/assistant.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import random
13
+
14
+ from compactbench.contracts import Turn, TurnRole
15
+
16
+ _USER_DISTRACTORS: tuple[str, ...] = (
17
+ "By the way, how long have you been working with me today?",
18
+ "Quick question unrelated to this: what time is it where you are?",
19
+ "I just remembered I need to send someone an email later.",
20
+ "Random thought: do you have opinions on coffee versus tea?",
21
+ "Oh, did you see that article about electric cars?",
22
+ "Side note — my internet has been slow today.",
23
+ "What's the weather like lately?",
24
+ "Unrelated, but what do you think of remote work in general?",
25
+ "By the way, do you read much fiction?",
26
+ "Quick tangent: is it true that pandas have thumbs?",
27
+ "I was thinking about lunch earlier.",
28
+ "Off-topic, but any thoughts on daylight saving time?",
29
+ "Random: I keep hearing about this one podcast.",
30
+ "Unrelated question — do you know anything about gardening?",
31
+ "Do you happen to know what a quokka is?",
32
+ )
33
+
34
+ _ASSISTANT_DISTRACTORS: tuple[str, ...] = (
35
+ "Happy to chat about that briefly — anything else on the main topic?",
36
+ "I can answer, but let me know when you want to return to the main question.",
37
+ "Quick aside, sure. Anything else pressing?",
38
+ "Got it. Let me know if you want to switch focus back.",
39
+ "Sure, I can address that. Is there anything more on the main thread?",
40
+ "Understood. Shall we continue where we left off?",
41
+ "Acknowledged. Ready when you are to move forward.",
42
+ "Fair point. Want me to refocus on the earlier question?",
43
+ "Noted. Do you want to get back to the plan?",
44
+ "I hear you. Anything else to cover on the primary question?",
45
+ )
46
+
47
+
48
+ def generate_distractor_turns(count: int, starting_turn_id: int, seed: int) -> list[Turn]:
49
+ """Produce ``count`` distractor turns alternating user/assistant.
50
+
51
+ IDs start at ``starting_turn_id`` and increment by one. Role alternation
52
+ starts with ``user``.
53
+ """
54
+ if count < 0:
55
+ raise ValueError(f"count must be >= 0, got {count}")
56
+ if count == 0:
57
+ return []
58
+ rng = random.Random(seed)
59
+ turns: list[Turn] = []
60
+ for i in range(count):
61
+ turn_id = starting_turn_id + i
62
+ if i % 2 == 0:
63
+ role = TurnRole.USER
64
+ content = rng.choice(_USER_DISTRACTORS)
65
+ else:
66
+ role = TurnRole.ASSISTANT
67
+ content = rng.choice(_ASSISTANT_DISTRACTORS)
68
+ turns.append(Turn(id=turn_id, role=role, content=content, tags=["distractor"]))
69
+ return turns
@@ -0,0 +1,11 @@
1
+ """Exceptions raised by the case generation engine."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class EngineError(Exception):
7
+ """Base exception for anything engine-related."""
8
+
9
+
10
+ class GenerationError(EngineError):
11
+ """Case generation failed (unsupported expression, missing config, etc.)."""
@@ -0,0 +1,96 @@
1
+ """Top-level case generation: ``TemplateDefinition`` + seed + difficulty -> ``GeneratedCase``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from compactbench.contracts import (
8
+ EvaluationItem,
9
+ EvaluationItemType,
10
+ GeneratedCase,
11
+ GroundTruth,
12
+ )
13
+ from compactbench.dsl import (
14
+ DifficultyLevel,
15
+ EvaluationItemTemplate,
16
+ GroundTruthTemplate,
17
+ TemplateDefinition,
18
+ resolve_variables,
19
+ substitute,
20
+ )
21
+ from compactbench.engine.difficulty import difficulty_bindings
22
+ from compactbench.engine.errors import GenerationError
23
+ from compactbench.engine.transcript import build_transcript
24
+
25
+
26
+ def generate_case(
27
+ template: TemplateDefinition,
28
+ seed: int,
29
+ difficulty: DifficultyLevel = DifficultyLevel.MEDIUM,
30
+ *,
31
+ case_id: str | None = None,
32
+ ) -> GeneratedCase:
33
+ """Generate a concrete :class:`GeneratedCase` from a template, seed, and difficulty.
34
+
35
+ Pure function of its inputs. No filesystem, network, or global state is
36
+ touched beyond the module-level generator registry in
37
+ :mod:`compactbench.dsl.generators`.
38
+ """
39
+ bindings = resolve_variables(list(template.variables), seed)
40
+ bindings.update(difficulty_bindings(template.difficulty_policy, difficulty))
41
+
42
+ transcript = build_transcript(template, bindings, seed, difficulty)
43
+ ground_truth = _build_ground_truth(template.ground_truth, bindings)
44
+ evaluation_items = _build_evaluation_items(template.evaluation_items, bindings)
45
+
46
+ return GeneratedCase(
47
+ case_id=case_id or f"{template.key}:s{seed}:d{difficulty.value}",
48
+ template_key=template.key,
49
+ template_version=template.version,
50
+ seed=seed,
51
+ difficulty=difficulty.value,
52
+ transcript=transcript,
53
+ ground_truth=ground_truth,
54
+ evaluation_items=evaluation_items,
55
+ )
56
+
57
+
58
+ def _build_ground_truth(template: GroundTruthTemplate, bindings: dict[str, str]) -> GroundTruth:
59
+ def _sub_list(items: list[str]) -> list[str]:
60
+ return [substitute(s, bindings) for s in items]
61
+
62
+ return GroundTruth(
63
+ immutable_facts=_sub_list(template.immutable_facts),
64
+ locked_decisions=_sub_list(template.locked_decisions),
65
+ forbidden_behaviors=_sub_list(template.forbidden_behaviors),
66
+ unresolved_items=_sub_list(template.unresolved_items),
67
+ entity_map={
68
+ substitute(k, bindings): substitute(v, bindings) for k, v in template.entity_map.items()
69
+ },
70
+ )
71
+
72
+
73
+ def _build_evaluation_items(
74
+ templates: list[EvaluationItemTemplate], bindings: dict[str, str]
75
+ ) -> list[EvaluationItem]:
76
+ results: list[EvaluationItem] = []
77
+ for item in templates:
78
+ try:
79
+ item_type = EvaluationItemType(item.type)
80
+ except ValueError as exc:
81
+ raise GenerationError(
82
+ f"evaluation item {item.key!r}: unknown type {item.type!r}. "
83
+ f"Known: {[t.value for t in EvaluationItemType]}"
84
+ ) from exc
85
+ expected: dict[str, Any] = {}
86
+ for k, v in item.expected.items():
87
+ expected[k] = substitute(v, bindings) if isinstance(v, str) else v
88
+ results.append(
89
+ EvaluationItem(
90
+ key=item.key,
91
+ item_type=item_type,
92
+ prompt=substitute(item.prompt, bindings),
93
+ expected=expected,
94
+ )
95
+ )
96
+ return results
@@ -0,0 +1,31 @@
1
+ """Seed derivation for case generation.
2
+
3
+ A *case seed* is derived from ``(suite_version, seed_group, case_slot)`` via
4
+ SHA-256. This is the seed that downstream :mod:`compactbench.dsl.generators`
5
+ uses to resolve template variables.
6
+
7
+ Stable across Python versions and platforms: the derivation uses UTF-8 byte
8
+ encoding, SHA-256, and a big-endian integer decode of the leading 8 bytes.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import hashlib
14
+
15
+
16
+ def derive_case_seed(suite_version: str, seed_group: str, case_slot: int) -> int:
17
+ """Derive a deterministic seed for a specific case slot.
18
+
19
+ Parameters
20
+ ----------
21
+ suite_version:
22
+ Identifier of the benchmark suite version (e.g. ``"starter@1.0.0"``).
23
+ seed_group:
24
+ Identifier of the seed group within this suite version
25
+ (e.g. ``"default"`` or ``"elite_ranked_q1"``).
26
+ case_slot:
27
+ Zero-based case slot index.
28
+ """
29
+ payload = f"{suite_version}:{seed_group}:{case_slot}".encode()
30
+ digest = hashlib.sha256(payload).digest()
31
+ return int.from_bytes(digest[:8], "big", signed=False)
@@ -0,0 +1,84 @@
1
+ """Build a concrete :class:`Transcript` from a template and bindings."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from compactbench.contracts import Transcript, Turn, TurnRole
6
+ from compactbench.dsl import (
7
+ DifficultyLevel,
8
+ TemplateDefinition,
9
+ TemplateTurnRole,
10
+ derive_seed,
11
+ extract_references,
12
+ substitute,
13
+ )
14
+ from compactbench.engine.difficulty import distractor_count_for
15
+ from compactbench.engine.distractors import generate_distractor_turns
16
+ from compactbench.engine.errors import GenerationError
17
+
18
+ _TEMPLATE_TO_TURN_ROLE: dict[TemplateTurnRole, TurnRole] = {
19
+ TemplateTurnRole.USER: TurnRole.USER,
20
+ TemplateTurnRole.ASSISTANT: TurnRole.ASSISTANT,
21
+ TemplateTurnRole.SYSTEM: TurnRole.SYSTEM,
22
+ }
23
+
24
+
25
+ def build_transcript(
26
+ template: TemplateDefinition,
27
+ bindings: dict[str, str],
28
+ case_seed: int,
29
+ difficulty: DifficultyLevel,
30
+ ) -> Transcript:
31
+ """Expand a template transcript into concrete turns with variables resolved."""
32
+ turns: list[Turn] = []
33
+ next_id = 0
34
+
35
+ for index, template_turn in enumerate(template.transcript.turns):
36
+ if template_turn.role is TemplateTurnRole.DISTRACTOR_BLOCK:
37
+ count = _resolve_count(
38
+ template_turn.count,
39
+ template,
40
+ difficulty,
41
+ location=f"transcript.turns[{index}].count",
42
+ )
43
+ sub_seed = derive_seed(case_seed, f"distractor_block_{index}")
44
+ turns.extend(generate_distractor_turns(count, next_id, sub_seed))
45
+ next_id += count
46
+ continue
47
+
48
+ if template_turn.template is None:
49
+ raise GenerationError(
50
+ f"transcript.turns[{index}] has role {template_turn.role.value!r} "
51
+ f"but no 'template' field"
52
+ )
53
+ content = substitute(template_turn.template, bindings)
54
+ turns.append(
55
+ Turn(
56
+ id=next_id,
57
+ role=_TEMPLATE_TO_TURN_ROLE[template_turn.role],
58
+ content=content,
59
+ tags=list(template_turn.tags),
60
+ )
61
+ )
62
+ next_id += 1
63
+
64
+ return Transcript(turns=turns)
65
+
66
+
67
+ def _resolve_count(
68
+ count: str | int | None,
69
+ template: TemplateDefinition,
70
+ difficulty: DifficultyLevel,
71
+ *,
72
+ location: str,
73
+ ) -> int:
74
+ if count is None:
75
+ raise GenerationError(f"{location}: missing count for distractor_block")
76
+ if isinstance(count, int):
77
+ if count < 0:
78
+ raise GenerationError(f"{location}: negative count {count}")
79
+ return count
80
+
81
+ refs = extract_references(count)
82
+ if refs == ["difficulty.distractor_turns"]:
83
+ return distractor_count_for(template.difficulty_policy, difficulty)
84
+ raise GenerationError(f"{location}: unsupported count expression {count!r}")
@@ -0,0 +1,38 @@
1
+ """Leaderboard ranking, qualification, and public projection.
2
+
3
+ Formulas locked in docs/architecture/decisions.md §B4.
4
+ Implemented in WO-008.
5
+ """
6
+
7
+ from compactbench.leaderboard.errors import LeaderboardError, QualificationError
8
+ from compactbench.leaderboard.projection import LeaderboardRow, project_row, rank_rows
9
+ from compactbench.leaderboard.qualification import (
10
+ MAX_CONTRADICTION_RATE,
11
+ MIN_FAMILY_PASS_RATE,
12
+ QualificationResult,
13
+ qualify,
14
+ )
15
+ from compactbench.leaderboard.ranking import (
16
+ TIER_FLOORS,
17
+ CompressionTier,
18
+ compression_bonus,
19
+ elite_score,
20
+ rank_key,
21
+ )
22
+
23
+ __all__ = [
24
+ "MAX_CONTRADICTION_RATE",
25
+ "MIN_FAMILY_PASS_RATE",
26
+ "TIER_FLOORS",
27
+ "CompressionTier",
28
+ "LeaderboardError",
29
+ "LeaderboardRow",
30
+ "QualificationError",
31
+ "QualificationResult",
32
+ "compression_bonus",
33
+ "elite_score",
34
+ "project_row",
35
+ "qualify",
36
+ "rank_key",
37
+ "rank_rows",
38
+ ]
@@ -0,0 +1,11 @@
1
+ """Leaderboard-specific exceptions."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class LeaderboardError(Exception):
7
+ """Base exception for leaderboard operations."""
8
+
9
+
10
+ class QualificationError(LeaderboardError):
11
+ """Raised (or returned as a failure reason) when a run fails qualification floors."""
@@ -0,0 +1,103 @@
1
+ """Project run results into public leaderboard rows.
2
+
3
+ The public leaderboard never exposes hidden template content — only aggregate
4
+ metrics, method identity, and versions. Raw case outputs are not projected.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from datetime import datetime
10
+ from typing import TypedDict
11
+
12
+ from compactbench.contracts import RunResult
13
+ from compactbench.leaderboard.ranking import (
14
+ CompressionTier,
15
+ elite_score,
16
+ rank_key,
17
+ )
18
+
19
+
20
+ class LeaderboardRow(TypedDict):
21
+ """Public row shape. Keep this stable — it is the public contract."""
22
+
23
+ rank: int | None
24
+ method_name: str
25
+ method_version: str
26
+ handle: str | None
27
+ org: str | None
28
+ tier: str
29
+ benchmark_version: str
30
+ target_provider: str
31
+ target_model: str
32
+ scorer_version: str
33
+ elite_score: float
34
+ overall_score: float
35
+ drift_resistance: float
36
+ constraint_retention: float
37
+ contradiction_rate: float
38
+ compression_ratio: float
39
+ published_at: str
40
+
41
+
42
+ def project_row(
43
+ run_result: RunResult,
44
+ *,
45
+ tier: CompressionTier,
46
+ handle: str | None,
47
+ org: str | None,
48
+ published_at: datetime,
49
+ ) -> LeaderboardRow:
50
+ """Turn a qualified :class:`RunResult` into a :class:`LeaderboardRow`.
51
+
52
+ ``rank`` is set to ``None`` here; caller assigns ranks after sorting.
53
+ """
54
+ score = elite_score(
55
+ overall_score=run_result.overall_score,
56
+ drift_resistance=run_result.drift_resistance,
57
+ constraint_retention=run_result.constraint_retention,
58
+ compression_ratio=run_result.compression_ratio,
59
+ tier=tier,
60
+ )
61
+ return LeaderboardRow(
62
+ rank=None,
63
+ method_name=run_result.method_name,
64
+ method_version=run_result.method_version,
65
+ handle=handle,
66
+ org=org,
67
+ tier=tier,
68
+ benchmark_version=f"{run_result.suite_key}@{run_result.suite_version}",
69
+ target_provider=run_result.target_provider,
70
+ target_model=run_result.target_model,
71
+ scorer_version=run_result.scorer_version,
72
+ elite_score=score,
73
+ overall_score=run_result.overall_score,
74
+ drift_resistance=run_result.drift_resistance,
75
+ constraint_retention=run_result.constraint_retention,
76
+ contradiction_rate=run_result.contradiction_rate,
77
+ compression_ratio=run_result.compression_ratio,
78
+ published_at=published_at.isoformat(),
79
+ )
80
+
81
+
82
+ def rank_rows(rows: list[LeaderboardRow]) -> list[LeaderboardRow]:
83
+ """Return ``rows`` sorted best-first with ``rank`` assigned.
84
+
85
+ Segmentation by ``(benchmark_version, target_model)`` is the caller's
86
+ responsibility — this function only sorts and numbers whatever it is given.
87
+ """
88
+
89
+ def _key(row: LeaderboardRow) -> tuple[float, float, float, float, float]:
90
+ return rank_key(
91
+ elite_score_value=row["elite_score"],
92
+ drift_resistance=row["drift_resistance"],
93
+ constraint_retention=row["constraint_retention"],
94
+ contradiction_rate=row["contradiction_rate"],
95
+ published_at=datetime.fromisoformat(row["published_at"]),
96
+ )
97
+
98
+ ranked: list[LeaderboardRow] = []
99
+ for i, row in enumerate(sorted(rows, key=_key), start=1):
100
+ ranked_row: LeaderboardRow = dict(row) # pyright: ignore[reportAssignmentType]
101
+ ranked_row["rank"] = i
102
+ ranked.append(ranked_row)
103
+ return ranked
@@ -0,0 +1,90 @@
1
+ """Qualification floor checks for leaderboard entries.
2
+
3
+ Floors locked in docs/architecture/decisions.md §B4.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from dataclasses import dataclass, field
9
+
10
+ from compactbench.contracts import RunResult
11
+ from compactbench.leaderboard.ranking import TIER_FLOORS, CompressionTier
12
+
13
+ MAX_CONTRADICTION_RATE: float = 0.10
14
+ MIN_FAMILY_PASS_RATE: float = 0.40
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class QualificationResult:
19
+ """Outcome of checking a run against leaderboard qualification floors."""
20
+
21
+ qualified: bool
22
+ reasons: list[str] = field(default_factory=list[str])
23
+
24
+
25
+ def qualify(
26
+ run_result: RunResult,
27
+ *,
28
+ tier: CompressionTier,
29
+ expected_drift_cycles: int,
30
+ ) -> QualificationResult:
31
+ """Check a :class:`RunResult` against the leaderboard floors.
32
+
33
+ Returns a :class:`QualificationResult` — inspect ``reasons`` when
34
+ ``qualified`` is ``False`` to learn why.
35
+ """
36
+ reasons: list[str] = []
37
+
38
+ tier_floor = TIER_FLOORS[tier]
39
+ if run_result.compression_ratio < tier_floor:
40
+ reasons.append(
41
+ f"compression {run_result.compression_ratio:.2f}x is below the {tier} "
42
+ f"floor of {tier_floor:.1f}x"
43
+ )
44
+
45
+ if run_result.contradiction_rate > MAX_CONTRADICTION_RATE:
46
+ reasons.append(
47
+ f"contradiction_rate {run_result.contradiction_rate:.3f} exceeds the "
48
+ f"{MAX_CONTRADICTION_RATE:.2f} maximum"
49
+ )
50
+
51
+ if not run_result.cases:
52
+ reasons.append("no cases completed")
53
+ else:
54
+ expected_cycles = expected_drift_cycles + 1
55
+ for case in run_result.cases:
56
+ if len(case.cycles) < expected_cycles:
57
+ reasons.append(
58
+ f"case {case.case_id!r} completed only {len(case.cycles)} of "
59
+ f"{expected_cycles} configured cycles"
60
+ )
61
+
62
+ # Per-family pass-rate guard only applies when the run covers more than one family.
63
+ family_rates = _family_pass_rates(run_result)
64
+ if len(family_rates) > 1:
65
+ for family, pass_rate in family_rates.items():
66
+ if pass_rate < MIN_FAMILY_PASS_RATE:
67
+ reasons.append(
68
+ f"family {family!r} pass rate {pass_rate:.2f} is below the "
69
+ f"{MIN_FAMILY_PASS_RATE:.2f} minimum (category-diversity guard)"
70
+ )
71
+
72
+ return QualificationResult(qualified=not reasons, reasons=reasons)
73
+
74
+
75
+ def _family_pass_rates(run_result: RunResult) -> dict[str, float]:
76
+ """Mean case_score grouped by benchmark family inferred from template_key."""
77
+ groups: dict[str, list[float]] = {}
78
+ for case in run_result.cases:
79
+ family = _infer_family(case.template_key)
80
+ groups.setdefault(family, []).append(case.case_score)
81
+ return {family: sum(scores) / len(scores) for family, scores in groups.items() if scores}
82
+
83
+
84
+ def _infer_family(template_key: str) -> str:
85
+ """Infer family name from template key by stripping trailing ``_starter_v<N>`` or ``_v<N>``."""
86
+ for suffix_prefix in ("_starter_v", "_elite_v", "_v"):
87
+ idx = template_key.rfind(suffix_prefix)
88
+ if idx > 0:
89
+ return template_key[:idx]
90
+ return template_key
@@ -0,0 +1,71 @@
1
+ """Elite score and tie-breaker computation.
2
+
3
+ Formulas are locked in docs/architecture/decisions.md §B4. Any change is a
4
+ scorer-version bump.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from datetime import datetime
10
+ from typing import Literal
11
+
12
+ CompressionTier = Literal["Elite-Light", "Elite-Mid", "Elite-Aggressive"]
13
+
14
+ TIER_FLOORS: dict[CompressionTier, float] = {
15
+ "Elite-Light": 2.0,
16
+ "Elite-Mid": 4.0,
17
+ "Elite-Aggressive": 8.0,
18
+ }
19
+
20
+
21
+ def compression_bonus(compression_ratio: float, tier: CompressionTier) -> float:
22
+ """Normalized bonus ``[0, 1]`` for exceeding the tier's compression floor.
23
+
24
+ 0 at exactly the floor, 1 at 2x the floor, capped.
25
+ """
26
+ floor = TIER_FLOORS[tier]
27
+ if compression_ratio <= floor:
28
+ return 0.0
29
+ raw = (compression_ratio - floor) / floor
30
+ return max(0.0, min(1.0, raw))
31
+
32
+
33
+ def elite_score(
34
+ *,
35
+ overall_score: float,
36
+ drift_resistance: float,
37
+ constraint_retention: float,
38
+ compression_ratio: float,
39
+ tier: CompressionTier,
40
+ ) -> float:
41
+ """Compute ``elite_score`` per decisions.md §B4.
42
+
43
+ ``elite_score = 0.40 * overall + 0.30 * drift + 0.20 * constraint + 0.10 * compression_bonus``
44
+ """
45
+ bonus = compression_bonus(compression_ratio, tier)
46
+ return (
47
+ 0.40 * overall_score + 0.30 * drift_resistance + 0.20 * constraint_retention + 0.10 * bonus
48
+ )
49
+
50
+
51
+ def rank_key(
52
+ *,
53
+ elite_score_value: float,
54
+ drift_resistance: float,
55
+ constraint_retention: float,
56
+ contradiction_rate: float,
57
+ published_at: datetime,
58
+ ) -> tuple[float, float, float, float, float]:
59
+ """Return a sort key that orders better runs FIRST when sorted ascending.
60
+
61
+ Primary: higher elite_score. Tie-breakers in order:
62
+ higher drift_resistance → higher constraint_retention → lower contradiction_rate
63
+ → earlier published_at.
64
+ """
65
+ return (
66
+ -elite_score_value,
67
+ -drift_resistance,
68
+ -constraint_retention,
69
+ contradiction_rate,
70
+ published_at.timestamp(),
71
+ )