compactbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compactbench/__init__.py +10 -0
- compactbench/__main__.py +6 -0
- compactbench/cli.py +343 -0
- compactbench/compactors/__init__.py +47 -0
- compactbench/compactors/_state_parser.py +118 -0
- compactbench/compactors/_utils.py +37 -0
- compactbench/compactors/base.py +38 -0
- compactbench/compactors/errors.py +11 -0
- compactbench/compactors/hierarchical_summary.py +113 -0
- compactbench/compactors/hybrid_ledger.py +115 -0
- compactbench/compactors/naive_summary.py +54 -0
- compactbench/compactors/structured_state.py +64 -0
- compactbench/config.py +36 -0
- compactbench/contracts/__init__.py +41 -0
- compactbench/contracts/artifact.py +49 -0
- compactbench/contracts/case.py +81 -0
- compactbench/contracts/result.py +79 -0
- compactbench/dsl/__init__.py +66 -0
- compactbench/dsl/errors.py +23 -0
- compactbench/dsl/generators.py +243 -0
- compactbench/dsl/models.py +107 -0
- compactbench/dsl/parser.py +55 -0
- compactbench/dsl/substitution.py +32 -0
- compactbench/dsl/validator.py +97 -0
- compactbench/engine/__init__.py +29 -0
- compactbench/engine/difficulty.py +54 -0
- compactbench/engine/distractors.py +69 -0
- compactbench/engine/errors.py +11 -0
- compactbench/engine/generation.py +96 -0
- compactbench/engine/seeds.py +31 -0
- compactbench/engine/transcript.py +84 -0
- compactbench/leaderboard/__init__.py +38 -0
- compactbench/leaderboard/errors.py +11 -0
- compactbench/leaderboard/projection.py +103 -0
- compactbench/leaderboard/qualification.py +90 -0
- compactbench/leaderboard/ranking.py +71 -0
- compactbench/providers/__init__.py +52 -0
- compactbench/providers/_retry.py +48 -0
- compactbench/providers/base.py +44 -0
- compactbench/providers/errors.py +15 -0
- compactbench/providers/google_ai_studio.py +112 -0
- compactbench/providers/groq.py +95 -0
- compactbench/providers/mock.py +53 -0
- compactbench/providers/ollama.py +100 -0
- compactbench/py.typed +0 -0
- compactbench/runner/__init__.py +53 -0
- compactbench/runner/_resolver.py +57 -0
- compactbench/runner/continuation.py +78 -0
- compactbench/runner/cycle.py +67 -0
- compactbench/runner/errors.py +15 -0
- compactbench/runner/evaluation.py +71 -0
- compactbench/runner/persistence.py +227 -0
- compactbench/runner/run.py +257 -0
- compactbench/scoring/__init__.py +48 -0
- compactbench/scoring/checks.py +92 -0
- compactbench/scoring/compression.py +66 -0
- compactbench/scoring/contradictions.py +70 -0
- compactbench/scoring/drift.py +30 -0
- compactbench/scoring/errors.py +7 -0
- compactbench/scoring/scorer.py +77 -0
- compactbench/utils/__init__.py +1 -0
- compactbench-0.1.0.dist-info/METADATA +161 -0
- compactbench-0.1.0.dist-info/RECORD +66 -0
- compactbench-0.1.0.dist-info/WHEEL +4 -0
- compactbench-0.1.0.dist-info/entry_points.txt +2 -0
- compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Extend a transcript with continuation turns for drift-cycle testing.
|
|
2
|
+
|
|
3
|
+
At each drift cycle we:
|
|
4
|
+
1. Append a seeded user continuation prompt.
|
|
5
|
+
2. Ask the target model for an assistant response *using only the previous
|
|
6
|
+
compacted artifact as context* — that is the drift vector we care about.
|
|
7
|
+
3. Append both turns to the transcript and hand it back for re-compaction.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import random
|
|
13
|
+
|
|
14
|
+
from compactbench.contracts import CompactionArtifact, Transcript, Turn, TurnRole
|
|
15
|
+
from compactbench.dsl import derive_seed
|
|
16
|
+
from compactbench.providers import CompletionRequest, Provider
|
|
17
|
+
from compactbench.runner.evaluation import render_artifact_for_prompt
|
|
18
|
+
|
|
19
|
+
_CONTINUATION_PROMPTS: tuple[str, ...] = (
|
|
20
|
+
"What should we tackle next?",
|
|
21
|
+
"Given everything we've discussed, what's the priority?",
|
|
22
|
+
"Any considerations I might be missing?",
|
|
23
|
+
"What could go wrong with the current plan?",
|
|
24
|
+
"Anything else to wire in before we move on?",
|
|
25
|
+
"Walk me through the immediate next step.",
|
|
26
|
+
"Which risk concerns you most right now?",
|
|
27
|
+
"Who needs to be looped in before we proceed?",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def select_continuation_prompt(case_seed: int, cycle_number: int) -> str:
|
|
32
|
+
"""Pick a deterministic user prompt for the given cycle."""
|
|
33
|
+
sub_seed = derive_seed(case_seed, f"continuation_prompt_{cycle_number}")
|
|
34
|
+
rng = random.Random(sub_seed)
|
|
35
|
+
return rng.choice(_CONTINUATION_PROMPTS)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def build_continuation_prompt(artifact: CompactionArtifact, user_content: str) -> str:
|
|
39
|
+
"""Build the prompt asking the target model for an assistant response."""
|
|
40
|
+
context = render_artifact_for_prompt(artifact)
|
|
41
|
+
return (
|
|
42
|
+
"You have the following summary of a prior conversation:\n\n"
|
|
43
|
+
f"{context}\n\n"
|
|
44
|
+
"The user now says:\n\n"
|
|
45
|
+
f"USER: {user_content}\n\n"
|
|
46
|
+
"Respond as the assistant. Your response will be treated as the next "
|
|
47
|
+
"assistant turn in the ongoing conversation.\n\n"
|
|
48
|
+
"ASSISTANT:"
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def extend_with_continuation(
|
|
53
|
+
transcript: Transcript,
|
|
54
|
+
previous_artifact: CompactionArtifact,
|
|
55
|
+
provider: Provider,
|
|
56
|
+
model: str,
|
|
57
|
+
case_seed: int,
|
|
58
|
+
cycle_number: int,
|
|
59
|
+
) -> Transcript:
|
|
60
|
+
"""Return a transcript extended with one (user, assistant) continuation pair."""
|
|
61
|
+
user_content = select_continuation_prompt(case_seed, cycle_number)
|
|
62
|
+
assistant_prompt = build_continuation_prompt(previous_artifact, user_content)
|
|
63
|
+
response = await provider.complete(CompletionRequest(model=model, prompt=assistant_prompt))
|
|
64
|
+
|
|
65
|
+
next_id = (max((t.id for t in transcript.turns), default=-1)) + 1
|
|
66
|
+
user_turn = Turn(
|
|
67
|
+
id=next_id,
|
|
68
|
+
role=TurnRole.USER,
|
|
69
|
+
content=user_content,
|
|
70
|
+
tags=["continuation"],
|
|
71
|
+
)
|
|
72
|
+
assistant_turn = Turn(
|
|
73
|
+
id=next_id + 1,
|
|
74
|
+
role=TurnRole.ASSISTANT,
|
|
75
|
+
content=response.text.strip(),
|
|
76
|
+
tags=["continuation"],
|
|
77
|
+
)
|
|
78
|
+
return Transcript(turns=[*transcript.turns, user_turn, assistant_turn])
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Execute a single drift cycle for one case.
|
|
2
|
+
|
|
3
|
+
One cycle = extend transcript (cycles >=1) → compact → evaluate → score.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import time
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
|
|
11
|
+
from compactbench.compactors import Compactor
|
|
12
|
+
from compactbench.contracts import CompactionArtifact, GeneratedCase, Scorecard, Transcript
|
|
13
|
+
from compactbench.providers import Provider
|
|
14
|
+
from compactbench.runner.continuation import extend_with_continuation
|
|
15
|
+
from compactbench.runner.evaluation import evaluate_items
|
|
16
|
+
from compactbench.scoring import score_cycle
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class CycleExecutionResult:
|
|
21
|
+
scorecard: Scorecard
|
|
22
|
+
artifact: CompactionArtifact
|
|
23
|
+
extended_transcript: Transcript
|
|
24
|
+
latency_ms: int
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
async def execute_cycle(
|
|
28
|
+
*,
|
|
29
|
+
case: GeneratedCase,
|
|
30
|
+
transcript: Transcript,
|
|
31
|
+
cycle_number: int,
|
|
32
|
+
previous_artifact: CompactionArtifact | None,
|
|
33
|
+
compactor: Compactor,
|
|
34
|
+
provider: Provider,
|
|
35
|
+
model: str,
|
|
36
|
+
case_seed: int,
|
|
37
|
+
) -> CycleExecutionResult:
|
|
38
|
+
"""Compact, evaluate, and score a single cycle.
|
|
39
|
+
|
|
40
|
+
For ``cycle_number >= 1`` with a ``previous_artifact``, the transcript is
|
|
41
|
+
first extended with one user/assistant continuation pair so the method
|
|
42
|
+
must survive repeated compact → continue → compact loops.
|
|
43
|
+
"""
|
|
44
|
+
started = time.perf_counter()
|
|
45
|
+
|
|
46
|
+
working_transcript = transcript
|
|
47
|
+
if cycle_number >= 1 and previous_artifact is not None:
|
|
48
|
+
working_transcript = await extend_with_continuation(
|
|
49
|
+
working_transcript,
|
|
50
|
+
previous_artifact,
|
|
51
|
+
provider,
|
|
52
|
+
model,
|
|
53
|
+
case_seed,
|
|
54
|
+
cycle_number,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
artifact = await compactor.compact(working_transcript, previous_artifact=previous_artifact)
|
|
58
|
+
responses = await evaluate_items(case.evaluation_items, artifact, provider, model)
|
|
59
|
+
scorecard = score_cycle(case, artifact, responses, cycle_number=cycle_number)
|
|
60
|
+
|
|
61
|
+
latency_ms = int((time.perf_counter() - started) * 1000)
|
|
62
|
+
return CycleExecutionResult(
|
|
63
|
+
scorecard=scorecard,
|
|
64
|
+
artifact=artifact,
|
|
65
|
+
extended_transcript=working_transcript,
|
|
66
|
+
latency_ms=latency_ms,
|
|
67
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Runner-specific exceptions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RunnerError(Exception):
|
|
7
|
+
"""Base exception for runner operations."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ResumeError(RunnerError):
|
|
11
|
+
"""Resume requested but the existing results file is incompatible."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MethodResolutionError(RunnerError):
|
|
15
|
+
"""The ``--method`` spec could not be resolved to a Compactor class."""
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Build evaluation prompts and issue them against the target model.
|
|
2
|
+
|
|
3
|
+
After compaction, the target model answers each ``EvaluationItem`` using
|
|
4
|
+
*only* the compacted artifact as context. That is the core measurement of
|
|
5
|
+
the benchmark: how well state survives compaction.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from compactbench.contracts import CompactionArtifact, EvaluationItem, StructuredState
|
|
11
|
+
from compactbench.providers import CompletionRequest, Provider
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def render_artifact_for_prompt(artifact: CompactionArtifact) -> str:
|
|
15
|
+
"""Render a compaction artifact as plain text for inclusion in a prompt."""
|
|
16
|
+
blocks: list[str] = []
|
|
17
|
+
if artifact.summary_text:
|
|
18
|
+
blocks.append(f"SUMMARY:\n{artifact.summary_text}")
|
|
19
|
+
|
|
20
|
+
state_block = _render_state(artifact.structured_state)
|
|
21
|
+
if state_block:
|
|
22
|
+
blocks.append("STRUCTURED STATE:\n" + state_block)
|
|
23
|
+
|
|
24
|
+
return "\n\n".join(blocks) if blocks else "(no summary or structured state)"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _render_state(state: StructuredState) -> str:
|
|
28
|
+
sections: list[str] = []
|
|
29
|
+
|
|
30
|
+
def _list_section(label: str, items: list[str]) -> None:
|
|
31
|
+
if items:
|
|
32
|
+
sections.append(f"{label}:\n" + "\n".join(f"- {s}" for s in items))
|
|
33
|
+
|
|
34
|
+
_list_section("Immutable facts", list(state.immutable_facts))
|
|
35
|
+
_list_section("Locked decisions", list(state.locked_decisions))
|
|
36
|
+
_list_section("Forbidden behaviors (NEVER do)", list(state.forbidden_behaviors))
|
|
37
|
+
_list_section("Deferred items", list(state.deferred_items))
|
|
38
|
+
_list_section("Unresolved items", list(state.unresolved_items))
|
|
39
|
+
|
|
40
|
+
if state.entity_map:
|
|
41
|
+
entity_lines = "\n".join(f"- {k}: {v}" for k, v in state.entity_map.items())
|
|
42
|
+
sections.append(f"Entity roles:\n{entity_lines}")
|
|
43
|
+
|
|
44
|
+
return "\n\n".join(sections)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def build_evaluation_prompt(artifact: CompactionArtifact, item: EvaluationItem) -> str:
|
|
48
|
+
"""Compose the prompt sent to the target model for a single evaluation item."""
|
|
49
|
+
context = render_artifact_for_prompt(artifact)
|
|
50
|
+
return (
|
|
51
|
+
"You have the following summary of a prior conversation:\n\n"
|
|
52
|
+
f"{context}\n\n"
|
|
53
|
+
"Based only on this summary, answer the following question concisely.\n\n"
|
|
54
|
+
f"QUESTION: {item.prompt}\n\n"
|
|
55
|
+
"ANSWER:"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def evaluate_items(
|
|
60
|
+
items: list[EvaluationItem],
|
|
61
|
+
artifact: CompactionArtifact,
|
|
62
|
+
provider: Provider,
|
|
63
|
+
model: str,
|
|
64
|
+
) -> dict[str, str]:
|
|
65
|
+
"""Invoke the target model on every evaluation item, returning ``{item_key: response}``."""
|
|
66
|
+
responses: dict[str, str] = {}
|
|
67
|
+
for item in items:
|
|
68
|
+
prompt = build_evaluation_prompt(artifact, item)
|
|
69
|
+
response = await provider.complete(CompletionRequest(model=model, prompt=prompt))
|
|
70
|
+
responses[item.key] = response.text.strip()
|
|
71
|
+
return responses
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""JSONL event log for run results.
|
|
2
|
+
|
|
3
|
+
Each run writes a sequence of events to a single ``results.jsonl`` file:
|
|
4
|
+
|
|
5
|
+
- one ``run_start`` event at the top with run metadata
|
|
6
|
+
- one ``case_complete`` event per completed case (streamed as they finish)
|
|
7
|
+
- one ``run_end`` event at the bottom with run-level aggregates
|
|
8
|
+
|
|
9
|
+
Writer flushes after every event so partial results survive crashes and
|
|
10
|
+
``--resume`` can pick up where we left off. Reader aggregates the stream into
|
|
11
|
+
a :class:`RunResult`.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections.abc import Iterator
|
|
17
|
+
from datetime import datetime
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from types import TracebackType
|
|
20
|
+
from typing import IO, Literal
|
|
21
|
+
|
|
22
|
+
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
|
23
|
+
|
|
24
|
+
from compactbench.contracts import CaseResult, RunResult
|
|
25
|
+
|
|
26
|
+
SCORER_VERSION: str = "1.0.0"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class RunStartEvent(BaseModel):
|
|
30
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
31
|
+
|
|
32
|
+
event: Literal["run_start"] = "run_start"
|
|
33
|
+
run_id: str
|
|
34
|
+
method_name: str
|
|
35
|
+
method_version: str
|
|
36
|
+
suite_key: str
|
|
37
|
+
suite_version: str
|
|
38
|
+
scorer_version: str
|
|
39
|
+
target_provider: str
|
|
40
|
+
target_model: str
|
|
41
|
+
difficulty: str
|
|
42
|
+
drift_cycles: int
|
|
43
|
+
seed_group: str
|
|
44
|
+
case_count_per_template: int
|
|
45
|
+
started_at: datetime
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class CaseCompleteEvent(BaseModel):
|
|
49
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
50
|
+
|
|
51
|
+
event: Literal["case_complete"] = "case_complete"
|
|
52
|
+
case_result: CaseResult
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class RunEndEvent(BaseModel):
|
|
56
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
57
|
+
|
|
58
|
+
event: Literal["run_end"] = "run_end"
|
|
59
|
+
completed_at: datetime
|
|
60
|
+
overall_score: float = Field(ge=0.0, le=1.0)
|
|
61
|
+
drift_resistance: float = Field(ge=0.0, le=1.0)
|
|
62
|
+
constraint_retention: float = Field(ge=0.0, le=1.0)
|
|
63
|
+
contradiction_rate: float = Field(ge=0.0, le=1.0)
|
|
64
|
+
compression_ratio: float = Field(ge=0.0)
|
|
65
|
+
notes: list[str] = Field(default_factory=list[str])
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
ResultEvent = RunStartEvent | CaseCompleteEvent | RunEndEvent
|
|
69
|
+
|
|
70
|
+
_EVENT_ADAPTER: TypeAdapter[ResultEvent] = TypeAdapter(ResultEvent)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class ResultsWriter:
|
|
74
|
+
"""Append-only JSONL writer for run events."""
|
|
75
|
+
|
|
76
|
+
def __init__(self, path: Path, mode: Literal["write", "append"] = "write") -> None:
|
|
77
|
+
self._path = path
|
|
78
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
79
|
+
self._fp: IO[str] = path.open("a" if mode == "append" else "w", encoding="utf-8")
|
|
80
|
+
|
|
81
|
+
def write(self, event: ResultEvent) -> None:
|
|
82
|
+
self._fp.write(event.model_dump_json() + "\n")
|
|
83
|
+
self._fp.flush()
|
|
84
|
+
|
|
85
|
+
def close(self) -> None:
|
|
86
|
+
if not self._fp.closed:
|
|
87
|
+
self._fp.close()
|
|
88
|
+
|
|
89
|
+
def __enter__(self) -> ResultsWriter:
|
|
90
|
+
return self
|
|
91
|
+
|
|
92
|
+
def __exit__(
|
|
93
|
+
self,
|
|
94
|
+
exc_type: type[BaseException] | None,
|
|
95
|
+
exc: BaseException | None,
|
|
96
|
+
tb: TracebackType | None,
|
|
97
|
+
) -> None:
|
|
98
|
+
self.close()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def iter_events(path: Path) -> Iterator[ResultEvent]:
|
|
102
|
+
"""Yield every event parsed from ``path``."""
|
|
103
|
+
with path.open(encoding="utf-8") as fp:
|
|
104
|
+
for line_num, line in enumerate(fp, start=1):
|
|
105
|
+
text = line.strip()
|
|
106
|
+
if not text:
|
|
107
|
+
continue
|
|
108
|
+
try:
|
|
109
|
+
yield _EVENT_ADAPTER.validate_json(text)
|
|
110
|
+
except Exception as exc:
|
|
111
|
+
raise ValueError(f"invalid event at {path}:{line_num}: {exc}") from exc
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def read_run_start(path: Path) -> RunStartEvent | None:
|
|
115
|
+
"""Return the ``run_start`` event, or ``None`` if the file has none."""
|
|
116
|
+
for event in iter_events(path):
|
|
117
|
+
if isinstance(event, RunStartEvent):
|
|
118
|
+
return event
|
|
119
|
+
break
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def completed_case_ids(path: Path) -> set[str]:
|
|
124
|
+
"""Return the set of case IDs whose ``case_complete`` event has been persisted."""
|
|
125
|
+
if not path.exists():
|
|
126
|
+
return set()
|
|
127
|
+
return {
|
|
128
|
+
event.case_result.case_id
|
|
129
|
+
for event in iter_events(path)
|
|
130
|
+
if isinstance(event, CaseCompleteEvent)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def to_run_result(path: Path) -> RunResult:
|
|
135
|
+
"""Aggregate every event in ``path`` into a :class:`RunResult`.
|
|
136
|
+
|
|
137
|
+
Raises :class:`ValueError` if there is no ``run_start`` event.
|
|
138
|
+
"""
|
|
139
|
+
run_start: RunStartEvent | None = None
|
|
140
|
+
cases: list[CaseResult] = []
|
|
141
|
+
run_end: RunEndEvent | None = None
|
|
142
|
+
|
|
143
|
+
for event in iter_events(path):
|
|
144
|
+
if isinstance(event, RunStartEvent):
|
|
145
|
+
run_start = event
|
|
146
|
+
elif isinstance(event, CaseCompleteEvent):
|
|
147
|
+
cases.append(event.case_result)
|
|
148
|
+
else:
|
|
149
|
+
run_end = event
|
|
150
|
+
|
|
151
|
+
if run_start is None:
|
|
152
|
+
raise ValueError(f"{path}: no run_start event found")
|
|
153
|
+
|
|
154
|
+
if run_end is not None:
|
|
155
|
+
end: RunEndEvent = run_end
|
|
156
|
+
completed_at = end.completed_at
|
|
157
|
+
overall_score = end.overall_score
|
|
158
|
+
drift_resistance = end.drift_resistance
|
|
159
|
+
constraint_retention = end.constraint_retention
|
|
160
|
+
contradiction_rate = end.contradiction_rate
|
|
161
|
+
compression_ratio = end.compression_ratio
|
|
162
|
+
notes = list(end.notes)
|
|
163
|
+
else:
|
|
164
|
+
# Run was interrupted — derive aggregates from whatever cases landed.
|
|
165
|
+
agg = aggregate_run_metrics(cases)
|
|
166
|
+
completed_at = run_start.started_at
|
|
167
|
+
overall_score = agg["overall_score"]
|
|
168
|
+
drift_resistance = agg["drift_resistance"]
|
|
169
|
+
constraint_retention = agg["constraint_retention"]
|
|
170
|
+
contradiction_rate = agg["contradiction_rate"]
|
|
171
|
+
compression_ratio = agg["compression_ratio"]
|
|
172
|
+
notes = ["run_end event missing: results may be incomplete"]
|
|
173
|
+
|
|
174
|
+
return RunResult(
|
|
175
|
+
run_id=run_start.run_id,
|
|
176
|
+
method_name=run_start.method_name,
|
|
177
|
+
method_version=run_start.method_version,
|
|
178
|
+
suite_key=run_start.suite_key,
|
|
179
|
+
suite_version=run_start.suite_version,
|
|
180
|
+
scorer_version=run_start.scorer_version,
|
|
181
|
+
target_provider=run_start.target_provider,
|
|
182
|
+
target_model=run_start.target_model,
|
|
183
|
+
started_at=run_start.started_at,
|
|
184
|
+
completed_at=completed_at,
|
|
185
|
+
cases=cases,
|
|
186
|
+
overall_score=overall_score,
|
|
187
|
+
drift_resistance=drift_resistance,
|
|
188
|
+
constraint_retention=constraint_retention,
|
|
189
|
+
contradiction_rate=contradiction_rate,
|
|
190
|
+
compression_ratio=compression_ratio,
|
|
191
|
+
notes=notes,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def aggregate_run_metrics(case_results: list[CaseResult]) -> dict[str, float]:
|
|
196
|
+
"""Compute run-level aggregate metrics from a list of case results."""
|
|
197
|
+
if not case_results:
|
|
198
|
+
return {
|
|
199
|
+
"overall_score": 0.0,
|
|
200
|
+
"drift_resistance": 1.0,
|
|
201
|
+
"constraint_retention": 0.0,
|
|
202
|
+
"contradiction_rate": 0.0,
|
|
203
|
+
"compression_ratio": 0.0,
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
def _mean(values: list[float], default: float = 0.0) -> float:
|
|
207
|
+
return sum(values) / len(values) if values else default
|
|
208
|
+
|
|
209
|
+
all_cycles = [cycle for cr in case_results for cycle in cr.cycles]
|
|
210
|
+
|
|
211
|
+
constraint_scores: list[float] = []
|
|
212
|
+
for cr in case_results:
|
|
213
|
+
for cycle in cr.cycles:
|
|
214
|
+
for item_score in cycle.scorecard.item_scores:
|
|
215
|
+
if item_score.item_type in (
|
|
216
|
+
"locked_decision_retention",
|
|
217
|
+
"forbidden_behavior_retention",
|
|
218
|
+
):
|
|
219
|
+
constraint_scores.append(item_score.score)
|
|
220
|
+
|
|
221
|
+
return {
|
|
222
|
+
"overall_score": _mean([cr.case_score for cr in case_results]),
|
|
223
|
+
"drift_resistance": _mean([cr.drift_resistance for cr in case_results], default=1.0),
|
|
224
|
+
"constraint_retention": _mean(constraint_scores),
|
|
225
|
+
"contradiction_rate": _mean([cycle.scorecard.contradiction_rate for cycle in all_cycles]),
|
|
226
|
+
"compression_ratio": _mean([cycle.scorecard.compression_ratio for cycle in all_cycles]),
|
|
227
|
+
}
|