compactbench 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. compactbench/__init__.py +10 -0
  2. compactbench/__main__.py +6 -0
  3. compactbench/cli.py +343 -0
  4. compactbench/compactors/__init__.py +47 -0
  5. compactbench/compactors/_state_parser.py +118 -0
  6. compactbench/compactors/_utils.py +37 -0
  7. compactbench/compactors/base.py +38 -0
  8. compactbench/compactors/errors.py +11 -0
  9. compactbench/compactors/hierarchical_summary.py +113 -0
  10. compactbench/compactors/hybrid_ledger.py +115 -0
  11. compactbench/compactors/naive_summary.py +54 -0
  12. compactbench/compactors/structured_state.py +64 -0
  13. compactbench/config.py +36 -0
  14. compactbench/contracts/__init__.py +41 -0
  15. compactbench/contracts/artifact.py +49 -0
  16. compactbench/contracts/case.py +81 -0
  17. compactbench/contracts/result.py +79 -0
  18. compactbench/dsl/__init__.py +66 -0
  19. compactbench/dsl/errors.py +23 -0
  20. compactbench/dsl/generators.py +243 -0
  21. compactbench/dsl/models.py +107 -0
  22. compactbench/dsl/parser.py +55 -0
  23. compactbench/dsl/substitution.py +32 -0
  24. compactbench/dsl/validator.py +97 -0
  25. compactbench/engine/__init__.py +29 -0
  26. compactbench/engine/difficulty.py +54 -0
  27. compactbench/engine/distractors.py +69 -0
  28. compactbench/engine/errors.py +11 -0
  29. compactbench/engine/generation.py +96 -0
  30. compactbench/engine/seeds.py +31 -0
  31. compactbench/engine/transcript.py +84 -0
  32. compactbench/leaderboard/__init__.py +38 -0
  33. compactbench/leaderboard/errors.py +11 -0
  34. compactbench/leaderboard/projection.py +103 -0
  35. compactbench/leaderboard/qualification.py +90 -0
  36. compactbench/leaderboard/ranking.py +71 -0
  37. compactbench/providers/__init__.py +52 -0
  38. compactbench/providers/_retry.py +48 -0
  39. compactbench/providers/base.py +44 -0
  40. compactbench/providers/errors.py +15 -0
  41. compactbench/providers/google_ai_studio.py +112 -0
  42. compactbench/providers/groq.py +95 -0
  43. compactbench/providers/mock.py +53 -0
  44. compactbench/providers/ollama.py +100 -0
  45. compactbench/py.typed +0 -0
  46. compactbench/runner/__init__.py +53 -0
  47. compactbench/runner/_resolver.py +57 -0
  48. compactbench/runner/continuation.py +78 -0
  49. compactbench/runner/cycle.py +67 -0
  50. compactbench/runner/errors.py +15 -0
  51. compactbench/runner/evaluation.py +71 -0
  52. compactbench/runner/persistence.py +227 -0
  53. compactbench/runner/run.py +257 -0
  54. compactbench/scoring/__init__.py +48 -0
  55. compactbench/scoring/checks.py +92 -0
  56. compactbench/scoring/compression.py +66 -0
  57. compactbench/scoring/contradictions.py +70 -0
  58. compactbench/scoring/drift.py +30 -0
  59. compactbench/scoring/errors.py +7 -0
  60. compactbench/scoring/scorer.py +77 -0
  61. compactbench/utils/__init__.py +1 -0
  62. compactbench-0.1.0.dist-info/METADATA +161 -0
  63. compactbench-0.1.0.dist-info/RECORD +66 -0
  64. compactbench-0.1.0.dist-info/WHEEL +4 -0
  65. compactbench-0.1.0.dist-info/entry_points.txt +2 -0
  66. compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,78 @@
1
+ """Extend a transcript with continuation turns for drift-cycle testing.
2
+
3
+ At each drift cycle we:
4
+ 1. Append a seeded user continuation prompt.
5
+ 2. Ask the target model for an assistant response *using only the previous
6
+ compacted artifact as context* — that is the drift vector we care about.
7
+ 3. Append both turns to the transcript and hand it back for re-compaction.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import random
13
+
14
+ from compactbench.contracts import CompactionArtifact, Transcript, Turn, TurnRole
15
+ from compactbench.dsl import derive_seed
16
+ from compactbench.providers import CompletionRequest, Provider
17
+ from compactbench.runner.evaluation import render_artifact_for_prompt
18
+
19
+ _CONTINUATION_PROMPTS: tuple[str, ...] = (
20
+ "What should we tackle next?",
21
+ "Given everything we've discussed, what's the priority?",
22
+ "Any considerations I might be missing?",
23
+ "What could go wrong with the current plan?",
24
+ "Anything else to wire in before we move on?",
25
+ "Walk me through the immediate next step.",
26
+ "Which risk concerns you most right now?",
27
+ "Who needs to be looped in before we proceed?",
28
+ )
29
+
30
+
31
+ def select_continuation_prompt(case_seed: int, cycle_number: int) -> str:
32
+ """Pick a deterministic user prompt for the given cycle."""
33
+ sub_seed = derive_seed(case_seed, f"continuation_prompt_{cycle_number}")
34
+ rng = random.Random(sub_seed)
35
+ return rng.choice(_CONTINUATION_PROMPTS)
36
+
37
+
38
+ def build_continuation_prompt(artifact: CompactionArtifact, user_content: str) -> str:
39
+ """Build the prompt asking the target model for an assistant response."""
40
+ context = render_artifact_for_prompt(artifact)
41
+ return (
42
+ "You have the following summary of a prior conversation:\n\n"
43
+ f"{context}\n\n"
44
+ "The user now says:\n\n"
45
+ f"USER: {user_content}\n\n"
46
+ "Respond as the assistant. Your response will be treated as the next "
47
+ "assistant turn in the ongoing conversation.\n\n"
48
+ "ASSISTANT:"
49
+ )
50
+
51
+
52
+ async def extend_with_continuation(
53
+ transcript: Transcript,
54
+ previous_artifact: CompactionArtifact,
55
+ provider: Provider,
56
+ model: str,
57
+ case_seed: int,
58
+ cycle_number: int,
59
+ ) -> Transcript:
60
+ """Return a transcript extended with one (user, assistant) continuation pair."""
61
+ user_content = select_continuation_prompt(case_seed, cycle_number)
62
+ assistant_prompt = build_continuation_prompt(previous_artifact, user_content)
63
+ response = await provider.complete(CompletionRequest(model=model, prompt=assistant_prompt))
64
+
65
+ next_id = (max((t.id for t in transcript.turns), default=-1)) + 1
66
+ user_turn = Turn(
67
+ id=next_id,
68
+ role=TurnRole.USER,
69
+ content=user_content,
70
+ tags=["continuation"],
71
+ )
72
+ assistant_turn = Turn(
73
+ id=next_id + 1,
74
+ role=TurnRole.ASSISTANT,
75
+ content=response.text.strip(),
76
+ tags=["continuation"],
77
+ )
78
+ return Transcript(turns=[*transcript.turns, user_turn, assistant_turn])
@@ -0,0 +1,67 @@
1
+ """Execute a single drift cycle for one case.
2
+
3
+ One cycle = extend transcript (cycles >=1) → compact → evaluate → score.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import time
9
+ from dataclasses import dataclass
10
+
11
+ from compactbench.compactors import Compactor
12
+ from compactbench.contracts import CompactionArtifact, GeneratedCase, Scorecard, Transcript
13
+ from compactbench.providers import Provider
14
+ from compactbench.runner.continuation import extend_with_continuation
15
+ from compactbench.runner.evaluation import evaluate_items
16
+ from compactbench.scoring import score_cycle
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class CycleExecutionResult:
21
+ scorecard: Scorecard
22
+ artifact: CompactionArtifact
23
+ extended_transcript: Transcript
24
+ latency_ms: int
25
+
26
+
27
+ async def execute_cycle(
28
+ *,
29
+ case: GeneratedCase,
30
+ transcript: Transcript,
31
+ cycle_number: int,
32
+ previous_artifact: CompactionArtifact | None,
33
+ compactor: Compactor,
34
+ provider: Provider,
35
+ model: str,
36
+ case_seed: int,
37
+ ) -> CycleExecutionResult:
38
+ """Compact, evaluate, and score a single cycle.
39
+
40
+ For ``cycle_number >= 1`` with a ``previous_artifact``, the transcript is
41
+ first extended with one user/assistant continuation pair so the method
42
+ must survive repeated compact → continue → compact loops.
43
+ """
44
+ started = time.perf_counter()
45
+
46
+ working_transcript = transcript
47
+ if cycle_number >= 1 and previous_artifact is not None:
48
+ working_transcript = await extend_with_continuation(
49
+ working_transcript,
50
+ previous_artifact,
51
+ provider,
52
+ model,
53
+ case_seed,
54
+ cycle_number,
55
+ )
56
+
57
+ artifact = await compactor.compact(working_transcript, previous_artifact=previous_artifact)
58
+ responses = await evaluate_items(case.evaluation_items, artifact, provider, model)
59
+ scorecard = score_cycle(case, artifact, responses, cycle_number=cycle_number)
60
+
61
+ latency_ms = int((time.perf_counter() - started) * 1000)
62
+ return CycleExecutionResult(
63
+ scorecard=scorecard,
64
+ artifact=artifact,
65
+ extended_transcript=working_transcript,
66
+ latency_ms=latency_ms,
67
+ )
@@ -0,0 +1,15 @@
1
+ """Runner-specific exceptions."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class RunnerError(Exception):
7
+ """Base exception for runner operations."""
8
+
9
+
10
+ class ResumeError(RunnerError):
11
+ """Resume requested but the existing results file is incompatible."""
12
+
13
+
14
+ class MethodResolutionError(RunnerError):
15
+ """The ``--method`` spec could not be resolved to a Compactor class."""
@@ -0,0 +1,71 @@
1
+ """Build evaluation prompts and issue them against the target model.
2
+
3
+ After compaction, the target model answers each ``EvaluationItem`` using
4
+ *only* the compacted artifact as context. That is the core measurement of
5
+ the benchmark: how well state survives compaction.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from compactbench.contracts import CompactionArtifact, EvaluationItem, StructuredState
11
+ from compactbench.providers import CompletionRequest, Provider
12
+
13
+
14
+ def render_artifact_for_prompt(artifact: CompactionArtifact) -> str:
15
+ """Render a compaction artifact as plain text for inclusion in a prompt."""
16
+ blocks: list[str] = []
17
+ if artifact.summary_text:
18
+ blocks.append(f"SUMMARY:\n{artifact.summary_text}")
19
+
20
+ state_block = _render_state(artifact.structured_state)
21
+ if state_block:
22
+ blocks.append("STRUCTURED STATE:\n" + state_block)
23
+
24
+ return "\n\n".join(blocks) if blocks else "(no summary or structured state)"
25
+
26
+
27
+ def _render_state(state: StructuredState) -> str:
28
+ sections: list[str] = []
29
+
30
+ def _list_section(label: str, items: list[str]) -> None:
31
+ if items:
32
+ sections.append(f"{label}:\n" + "\n".join(f"- {s}" for s in items))
33
+
34
+ _list_section("Immutable facts", list(state.immutable_facts))
35
+ _list_section("Locked decisions", list(state.locked_decisions))
36
+ _list_section("Forbidden behaviors (NEVER do)", list(state.forbidden_behaviors))
37
+ _list_section("Deferred items", list(state.deferred_items))
38
+ _list_section("Unresolved items", list(state.unresolved_items))
39
+
40
+ if state.entity_map:
41
+ entity_lines = "\n".join(f"- {k}: {v}" for k, v in state.entity_map.items())
42
+ sections.append(f"Entity roles:\n{entity_lines}")
43
+
44
+ return "\n\n".join(sections)
45
+
46
+
47
+ def build_evaluation_prompt(artifact: CompactionArtifact, item: EvaluationItem) -> str:
48
+ """Compose the prompt sent to the target model for a single evaluation item."""
49
+ context = render_artifact_for_prompt(artifact)
50
+ return (
51
+ "You have the following summary of a prior conversation:\n\n"
52
+ f"{context}\n\n"
53
+ "Based only on this summary, answer the following question concisely.\n\n"
54
+ f"QUESTION: {item.prompt}\n\n"
55
+ "ANSWER:"
56
+ )
57
+
58
+
59
+ async def evaluate_items(
60
+ items: list[EvaluationItem],
61
+ artifact: CompactionArtifact,
62
+ provider: Provider,
63
+ model: str,
64
+ ) -> dict[str, str]:
65
+ """Invoke the target model on every evaluation item, returning ``{item_key: response}``."""
66
+ responses: dict[str, str] = {}
67
+ for item in items:
68
+ prompt = build_evaluation_prompt(artifact, item)
69
+ response = await provider.complete(CompletionRequest(model=model, prompt=prompt))
70
+ responses[item.key] = response.text.strip()
71
+ return responses
@@ -0,0 +1,227 @@
1
+ """JSONL event log for run results.
2
+
3
+ Each run writes a sequence of events to a single ``results.jsonl`` file:
4
+
5
+ - one ``run_start`` event at the top with run metadata
6
+ - one ``case_complete`` event per completed case (streamed as they finish)
7
+ - one ``run_end`` event at the bottom with run-level aggregates
8
+
9
+ Writer flushes after every event so partial results survive crashes and
10
+ ``--resume`` can pick up where we left off. Reader aggregates the stream into
11
+ a :class:`RunResult`.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from collections.abc import Iterator
17
+ from datetime import datetime
18
+ from pathlib import Path
19
+ from types import TracebackType
20
+ from typing import IO, Literal
21
+
22
+ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
23
+
24
+ from compactbench.contracts import CaseResult, RunResult
25
+
26
+ SCORER_VERSION: str = "1.0.0"
27
+
28
+
29
+ class RunStartEvent(BaseModel):
30
+ model_config = ConfigDict(extra="forbid", frozen=True)
31
+
32
+ event: Literal["run_start"] = "run_start"
33
+ run_id: str
34
+ method_name: str
35
+ method_version: str
36
+ suite_key: str
37
+ suite_version: str
38
+ scorer_version: str
39
+ target_provider: str
40
+ target_model: str
41
+ difficulty: str
42
+ drift_cycles: int
43
+ seed_group: str
44
+ case_count_per_template: int
45
+ started_at: datetime
46
+
47
+
48
+ class CaseCompleteEvent(BaseModel):
49
+ model_config = ConfigDict(extra="forbid", frozen=True)
50
+
51
+ event: Literal["case_complete"] = "case_complete"
52
+ case_result: CaseResult
53
+
54
+
55
+ class RunEndEvent(BaseModel):
56
+ model_config = ConfigDict(extra="forbid", frozen=True)
57
+
58
+ event: Literal["run_end"] = "run_end"
59
+ completed_at: datetime
60
+ overall_score: float = Field(ge=0.0, le=1.0)
61
+ drift_resistance: float = Field(ge=0.0, le=1.0)
62
+ constraint_retention: float = Field(ge=0.0, le=1.0)
63
+ contradiction_rate: float = Field(ge=0.0, le=1.0)
64
+ compression_ratio: float = Field(ge=0.0)
65
+ notes: list[str] = Field(default_factory=list[str])
66
+
67
+
68
+ ResultEvent = RunStartEvent | CaseCompleteEvent | RunEndEvent
69
+
70
+ _EVENT_ADAPTER: TypeAdapter[ResultEvent] = TypeAdapter(ResultEvent)
71
+
72
+
73
+ class ResultsWriter:
74
+ """Append-only JSONL writer for run events."""
75
+
76
+ def __init__(self, path: Path, mode: Literal["write", "append"] = "write") -> None:
77
+ self._path = path
78
+ path.parent.mkdir(parents=True, exist_ok=True)
79
+ self._fp: IO[str] = path.open("a" if mode == "append" else "w", encoding="utf-8")
80
+
81
+ def write(self, event: ResultEvent) -> None:
82
+ self._fp.write(event.model_dump_json() + "\n")
83
+ self._fp.flush()
84
+
85
+ def close(self) -> None:
86
+ if not self._fp.closed:
87
+ self._fp.close()
88
+
89
+ def __enter__(self) -> ResultsWriter:
90
+ return self
91
+
92
+ def __exit__(
93
+ self,
94
+ exc_type: type[BaseException] | None,
95
+ exc: BaseException | None,
96
+ tb: TracebackType | None,
97
+ ) -> None:
98
+ self.close()
99
+
100
+
101
+ def iter_events(path: Path) -> Iterator[ResultEvent]:
102
+ """Yield every event parsed from ``path``."""
103
+ with path.open(encoding="utf-8") as fp:
104
+ for line_num, line in enumerate(fp, start=1):
105
+ text = line.strip()
106
+ if not text:
107
+ continue
108
+ try:
109
+ yield _EVENT_ADAPTER.validate_json(text)
110
+ except Exception as exc:
111
+ raise ValueError(f"invalid event at {path}:{line_num}: {exc}") from exc
112
+
113
+
114
+ def read_run_start(path: Path) -> RunStartEvent | None:
115
+ """Return the ``run_start`` event, or ``None`` if the file has none."""
116
+ for event in iter_events(path):
117
+ if isinstance(event, RunStartEvent):
118
+ return event
119
+ break
120
+ return None
121
+
122
+
123
+ def completed_case_ids(path: Path) -> set[str]:
124
+ """Return the set of case IDs whose ``case_complete`` event has been persisted."""
125
+ if not path.exists():
126
+ return set()
127
+ return {
128
+ event.case_result.case_id
129
+ for event in iter_events(path)
130
+ if isinstance(event, CaseCompleteEvent)
131
+ }
132
+
133
+
134
+ def to_run_result(path: Path) -> RunResult:
135
+ """Aggregate every event in ``path`` into a :class:`RunResult`.
136
+
137
+ Raises :class:`ValueError` if there is no ``run_start`` event.
138
+ """
139
+ run_start: RunStartEvent | None = None
140
+ cases: list[CaseResult] = []
141
+ run_end: RunEndEvent | None = None
142
+
143
+ for event in iter_events(path):
144
+ if isinstance(event, RunStartEvent):
145
+ run_start = event
146
+ elif isinstance(event, CaseCompleteEvent):
147
+ cases.append(event.case_result)
148
+ else:
149
+ run_end = event
150
+
151
+ if run_start is None:
152
+ raise ValueError(f"{path}: no run_start event found")
153
+
154
+ if run_end is not None:
155
+ end: RunEndEvent = run_end
156
+ completed_at = end.completed_at
157
+ overall_score = end.overall_score
158
+ drift_resistance = end.drift_resistance
159
+ constraint_retention = end.constraint_retention
160
+ contradiction_rate = end.contradiction_rate
161
+ compression_ratio = end.compression_ratio
162
+ notes = list(end.notes)
163
+ else:
164
+ # Run was interrupted — derive aggregates from whatever cases landed.
165
+ agg = aggregate_run_metrics(cases)
166
+ completed_at = run_start.started_at
167
+ overall_score = agg["overall_score"]
168
+ drift_resistance = agg["drift_resistance"]
169
+ constraint_retention = agg["constraint_retention"]
170
+ contradiction_rate = agg["contradiction_rate"]
171
+ compression_ratio = agg["compression_ratio"]
172
+ notes = ["run_end event missing: results may be incomplete"]
173
+
174
+ return RunResult(
175
+ run_id=run_start.run_id,
176
+ method_name=run_start.method_name,
177
+ method_version=run_start.method_version,
178
+ suite_key=run_start.suite_key,
179
+ suite_version=run_start.suite_version,
180
+ scorer_version=run_start.scorer_version,
181
+ target_provider=run_start.target_provider,
182
+ target_model=run_start.target_model,
183
+ started_at=run_start.started_at,
184
+ completed_at=completed_at,
185
+ cases=cases,
186
+ overall_score=overall_score,
187
+ drift_resistance=drift_resistance,
188
+ constraint_retention=constraint_retention,
189
+ contradiction_rate=contradiction_rate,
190
+ compression_ratio=compression_ratio,
191
+ notes=notes,
192
+ )
193
+
194
+
195
+ def aggregate_run_metrics(case_results: list[CaseResult]) -> dict[str, float]:
196
+ """Compute run-level aggregate metrics from a list of case results."""
197
+ if not case_results:
198
+ return {
199
+ "overall_score": 0.0,
200
+ "drift_resistance": 1.0,
201
+ "constraint_retention": 0.0,
202
+ "contradiction_rate": 0.0,
203
+ "compression_ratio": 0.0,
204
+ }
205
+
206
+ def _mean(values: list[float], default: float = 0.0) -> float:
207
+ return sum(values) / len(values) if values else default
208
+
209
+ all_cycles = [cycle for cr in case_results for cycle in cr.cycles]
210
+
211
+ constraint_scores: list[float] = []
212
+ for cr in case_results:
213
+ for cycle in cr.cycles:
214
+ for item_score in cycle.scorecard.item_scores:
215
+ if item_score.item_type in (
216
+ "locked_decision_retention",
217
+ "forbidden_behavior_retention",
218
+ ):
219
+ constraint_scores.append(item_score.score)
220
+
221
+ return {
222
+ "overall_score": _mean([cr.case_score for cr in case_results]),
223
+ "drift_resistance": _mean([cr.drift_resistance for cr in case_results], default=1.0),
224
+ "constraint_retention": _mean(constraint_scores),
225
+ "contradiction_rate": _mean([cycle.scorecard.contradiction_rate for cycle in all_cycles]),
226
+ "compression_ratio": _mean([cycle.scorecard.compression_ratio for cycle in all_cycles]),
227
+ }