compactbench 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. compactbench/__init__.py +10 -0
  2. compactbench/__main__.py +6 -0
  3. compactbench/cli.py +343 -0
  4. compactbench/compactors/__init__.py +47 -0
  5. compactbench/compactors/_state_parser.py +118 -0
  6. compactbench/compactors/_utils.py +37 -0
  7. compactbench/compactors/base.py +38 -0
  8. compactbench/compactors/errors.py +11 -0
  9. compactbench/compactors/hierarchical_summary.py +113 -0
  10. compactbench/compactors/hybrid_ledger.py +115 -0
  11. compactbench/compactors/naive_summary.py +54 -0
  12. compactbench/compactors/structured_state.py +64 -0
  13. compactbench/config.py +36 -0
  14. compactbench/contracts/__init__.py +41 -0
  15. compactbench/contracts/artifact.py +49 -0
  16. compactbench/contracts/case.py +81 -0
  17. compactbench/contracts/result.py +79 -0
  18. compactbench/dsl/__init__.py +66 -0
  19. compactbench/dsl/errors.py +23 -0
  20. compactbench/dsl/generators.py +243 -0
  21. compactbench/dsl/models.py +107 -0
  22. compactbench/dsl/parser.py +55 -0
  23. compactbench/dsl/substitution.py +32 -0
  24. compactbench/dsl/validator.py +97 -0
  25. compactbench/engine/__init__.py +29 -0
  26. compactbench/engine/difficulty.py +54 -0
  27. compactbench/engine/distractors.py +69 -0
  28. compactbench/engine/errors.py +11 -0
  29. compactbench/engine/generation.py +96 -0
  30. compactbench/engine/seeds.py +31 -0
  31. compactbench/engine/transcript.py +84 -0
  32. compactbench/leaderboard/__init__.py +38 -0
  33. compactbench/leaderboard/errors.py +11 -0
  34. compactbench/leaderboard/projection.py +103 -0
  35. compactbench/leaderboard/qualification.py +90 -0
  36. compactbench/leaderboard/ranking.py +71 -0
  37. compactbench/providers/__init__.py +52 -0
  38. compactbench/providers/_retry.py +48 -0
  39. compactbench/providers/base.py +44 -0
  40. compactbench/providers/errors.py +15 -0
  41. compactbench/providers/google_ai_studio.py +112 -0
  42. compactbench/providers/groq.py +95 -0
  43. compactbench/providers/mock.py +53 -0
  44. compactbench/providers/ollama.py +100 -0
  45. compactbench/py.typed +0 -0
  46. compactbench/runner/__init__.py +53 -0
  47. compactbench/runner/_resolver.py +57 -0
  48. compactbench/runner/continuation.py +78 -0
  49. compactbench/runner/cycle.py +67 -0
  50. compactbench/runner/errors.py +15 -0
  51. compactbench/runner/evaluation.py +71 -0
  52. compactbench/runner/persistence.py +227 -0
  53. compactbench/runner/run.py +257 -0
  54. compactbench/scoring/__init__.py +48 -0
  55. compactbench/scoring/checks.py +92 -0
  56. compactbench/scoring/compression.py +66 -0
  57. compactbench/scoring/contradictions.py +70 -0
  58. compactbench/scoring/drift.py +30 -0
  59. compactbench/scoring/errors.py +7 -0
  60. compactbench/scoring/scorer.py +77 -0
  61. compactbench/utils/__init__.py +1 -0
  62. compactbench-0.1.0.dist-info/METADATA +161 -0
  63. compactbench-0.1.0.dist-info/RECORD +66 -0
  64. compactbench-0.1.0.dist-info/WHEEL +4 -0
  65. compactbench-0.1.0.dist-info/entry_points.txt +2 -0
  66. compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,10 @@
1
+ """CompactBench — open benchmark for AI conversation compaction methods."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ try:
6
+ __version__ = version("compactbench")
7
+ except PackageNotFoundError: # pragma: no cover - during local dev without install
8
+ __version__ = "0.0.0+local"
9
+
10
+ __all__ = ["__version__"]
@@ -0,0 +1,6 @@
1
+ """Entry point for ``python -m compactbench``."""
2
+
3
+ from compactbench.cli import app
4
+
5
+ if __name__ == "__main__":
6
+ app()
compactbench/cli.py ADDED
@@ -0,0 +1,343 @@
1
+ """CompactBench command-line interface.
2
+
3
+ Implemented incrementally. Each command raises ``NotImplementedError`` until its
4
+ backing workorder lands. See docs/architecture/workorders.md.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import typer
13
+ from rich.console import Console
14
+
15
+ from compactbench import __version__
16
+
17
+ app = typer.Typer(
18
+ name="compactbench",
19
+ help="Open benchmark for AI conversation compaction methods.",
20
+ no_args_is_help=True,
21
+ add_completion=False,
22
+ )
23
+
24
+ console = Console()
25
+
26
+
27
+ def _version_callback(value: bool) -> None:
28
+ if value:
29
+ console.print(f"compactbench {__version__}")
30
+ raise typer.Exit()
31
+
32
+
33
+ @app.callback()
34
+ def main(
35
+ _version: bool = typer.Option(
36
+ False,
37
+ "--version",
38
+ "-V",
39
+ callback=_version_callback,
40
+ is_eager=True,
41
+ help="Show version and exit.",
42
+ ),
43
+ ) -> None:
44
+ """CompactBench CLI."""
45
+
46
+
47
+ @app.command()
48
+ def run(
49
+ method: str = typer.Option(
50
+ ..., "--method", "-m", help="Method id: 'built-in:<key>' or '<path.py>:<ClassName>'."
51
+ ),
52
+ suite: str = typer.Option("starter", "--suite", "-s", help="Benchmark suite key."),
53
+ provider: str = typer.Option("mock", "--provider", "-p", help="Model provider key."),
54
+ model: str = typer.Option("mock-deterministic", "--model", help="Provider-specific model key."),
55
+ difficulty: str = typer.Option("medium", "--difficulty"),
56
+ drift_cycles: int = typer.Option(2, "--drift-cycles", min=0, max=5),
57
+ case_count: int = typer.Option(3, "--case-count", min=1, help="Cases per template."),
58
+ seed_group: str = typer.Option("default", "--seed-group"),
59
+ benchmarks_dir: Path = typer.Option(
60
+ Path("benchmarks/public"),
61
+ "--benchmarks-dir",
62
+ help="Directory containing benchmark suites.",
63
+ ),
64
+ output: Path = typer.Option(Path("results.jsonl"), "--output", "-o"),
65
+ resume: bool = typer.Option(
66
+ False, "--resume", help="Continue from existing output file (skip completed cases)."
67
+ ),
68
+ ) -> None:
69
+ """Run a compaction method against a benchmark suite."""
70
+ import asyncio
71
+
72
+ from compactbench.dsl import DifficultyLevel
73
+ from compactbench.runner import RunArgs, RunnerError, run_experiment
74
+
75
+ try:
76
+ diff = DifficultyLevel(difficulty.lower())
77
+ except ValueError as exc:
78
+ console.print(
79
+ f"[red]unknown difficulty {difficulty!r}. "
80
+ f"Valid: {[d.value for d in DifficultyLevel]}[/red]"
81
+ )
82
+ raise typer.Exit(code=1) from exc
83
+
84
+ args = RunArgs(
85
+ method_spec=method,
86
+ suite_key=suite,
87
+ provider_key=provider,
88
+ model=model,
89
+ difficulty=diff,
90
+ drift_cycles=drift_cycles,
91
+ case_count_per_template=case_count,
92
+ seed_group=seed_group,
93
+ benchmarks_dir=benchmarks_dir,
94
+ output_path=output,
95
+ resume=resume,
96
+ )
97
+
98
+ try:
99
+ asyncio.run(run_experiment(args))
100
+ except RunnerError as exc:
101
+ console.print(f"[red]{exc}[/red]")
102
+ raise typer.Exit(code=1) from exc
103
+
104
+ console.print(f"[green]wrote {output}[/green]")
105
+ console.print(f"run 'compactbench score --results {output}' for a summary.")
106
+
107
+
108
+ @app.command()
109
+ def generate(
110
+ template: str = typer.Option(..., "--template", "-t", help="Template key."),
111
+ seed: int = typer.Option(0, "--seed"),
112
+ difficulty: str = typer.Option("medium", "--difficulty"),
113
+ benchmarks_dir: Path = typer.Option(
114
+ Path("benchmarks/public"),
115
+ "--benchmarks-dir",
116
+ help="Directory containing public benchmark suites.",
117
+ ),
118
+ output: Path | None = typer.Option(
119
+ None, "--output", "-o", help="Write JSON to this file (stdout if omitted)."
120
+ ),
121
+ ) -> None:
122
+ """Generate a single benchmark case from a template and seed."""
123
+ from compactbench.dsl import (
124
+ DifficultyLevel,
125
+ TemplateError,
126
+ load_suite,
127
+ validate_template,
128
+ )
129
+ from compactbench.engine import generate_case
130
+
131
+ if not benchmarks_dir.is_dir():
132
+ console.print(f"[red]no benchmarks directory at {benchmarks_dir}[/red]")
133
+ raise typer.Exit(code=1)
134
+
135
+ found: dict[str, Any] = {}
136
+ for suite_dir in sorted(p for p in benchmarks_dir.iterdir() if p.is_dir()):
137
+ try:
138
+ for t in load_suite(suite_dir):
139
+ found[t.key] = t
140
+ except TemplateError as exc:
141
+ console.print(f"[yellow]skipping {suite_dir.name}: {exc}[/yellow]")
142
+
143
+ if template not in found:
144
+ console.print(f"[red]template {template!r} not found. Known: {sorted(found)}[/red]")
145
+ raise typer.Exit(code=1)
146
+
147
+ tmpl = found[template]
148
+ try:
149
+ validate_template(tmpl)
150
+ except TemplateError as exc:
151
+ console.print(f"[red]template {template!r} failed validation: {exc}[/red]")
152
+ raise typer.Exit(code=1) from exc
153
+
154
+ try:
155
+ diff = DifficultyLevel(difficulty.lower())
156
+ except ValueError as exc:
157
+ console.print(
158
+ f"[red]unknown difficulty {difficulty!r}. "
159
+ f"Valid: {[d.value for d in DifficultyLevel]}[/red]"
160
+ )
161
+ raise typer.Exit(code=1) from exc
162
+
163
+ case = generate_case(tmpl, seed, diff)
164
+ json_text = case.model_dump_json(indent=2)
165
+ if output is None:
166
+ console.print_json(json_text)
167
+ else:
168
+ output.write_text(json_text + "\n", encoding="utf-8")
169
+ console.print(f"wrote {output}")
170
+
171
+
172
+ @app.command()
173
+ def score(
174
+ results: Path = typer.Option(..., "--results", "-r", exists=True, readable=True),
175
+ ) -> None:
176
+ """Print a summary of a ``results.jsonl`` file produced by ``compactbench run``."""
177
+ from rich.table import Table
178
+
179
+ from compactbench.runner import to_run_result
180
+
181
+ try:
182
+ run_result = to_run_result(results)
183
+ except ValueError as exc:
184
+ console.print(f"[red]{exc}[/red]")
185
+ raise typer.Exit(code=1) from exc
186
+
187
+ header = Table(title="Run")
188
+ header.add_column("Field")
189
+ header.add_column("Value")
190
+ header.add_row("run_id", run_result.run_id)
191
+ header.add_row("method", f"{run_result.method_name} ({run_result.method_version})")
192
+ header.add_row("suite", f"{run_result.suite_key} ({run_result.suite_version})")
193
+ header.add_row("target", f"{run_result.target_provider} / {run_result.target_model}")
194
+ header.add_row("scorer", run_result.scorer_version)
195
+ header.add_row("started", run_result.started_at.isoformat())
196
+ header.add_row("completed", run_result.completed_at.isoformat())
197
+ console.print(header)
198
+
199
+ per_case = Table(title=f"{len(run_result.cases)} case(s)")
200
+ per_case.add_column("Case ID")
201
+ per_case.add_column("Case score", justify="right")
202
+ per_case.add_column("Drift resistance", justify="right")
203
+ per_case.add_column("Cycles", justify="right")
204
+ for case in run_result.cases:
205
+ per_case.add_row(
206
+ case.case_id,
207
+ f"{case.case_score:.3f}",
208
+ f"{case.drift_resistance:.3f}",
209
+ str(len(case.cycles)),
210
+ )
211
+ console.print(per_case)
212
+
213
+ summary = Table(title="Aggregate")
214
+ summary.add_column("Metric")
215
+ summary.add_column("Value", justify="right")
216
+ summary.add_row("overall_score", f"{run_result.overall_score:.3f}")
217
+ summary.add_row("drift_resistance", f"{run_result.drift_resistance:.3f}")
218
+ summary.add_row("constraint_retention", f"{run_result.constraint_retention:.3f}")
219
+ summary.add_row("contradiction_rate", f"{run_result.contradiction_rate:.3f}")
220
+ summary.add_row("compression_ratio", f"{run_result.compression_ratio:.2f}x")
221
+ console.print(summary)
222
+
223
+ for note in run_result.notes:
224
+ console.print(f"[yellow]note: {note}[/yellow]")
225
+
226
+
227
+ @app.command()
228
+ def submit(
229
+ results: Path = typer.Option(
230
+ ...,
231
+ "--results",
232
+ "-r",
233
+ exists=True,
234
+ readable=True,
235
+ help="Local results.jsonl produced by `compactbench run`.",
236
+ ),
237
+ handle: str = typer.Option(..., "--handle", help="Your GitHub handle."),
238
+ method_name: str = typer.Option(
239
+ ..., "--name", help="Method name in kebab-case, e.g. 'my-ledger-v1'."
240
+ ),
241
+ ) -> None:
242
+ """Print the steps to submit a method for leaderboard scoring.
243
+
244
+ Full automation of PR creation is out of scope for the local CLI — GitHub
245
+ Actions evaluation happens against the hidden test set on a repo secret,
246
+ so submission is always a PR + maintainer review. This command prints a
247
+ checklist so you don't miss a step.
248
+ """
249
+ method_dir = Path("submissions") / handle / method_name
250
+ console.print("[bold]Submit a method to the CompactBench leaderboard[/bold]")
251
+ console.print("")
252
+ console.print(f" 1. Create [cyan]{method_dir}/[/cyan] in your fork.")
253
+ console.print(
254
+ f" 2. Copy the scaffold from [cyan]submissions/_template/[/cyan] "
255
+ f"into [cyan]{method_dir}/[/cyan] and edit every file."
256
+ )
257
+ console.print(
258
+ f" 3. Put your validated [cyan]{results}[/cyan] at "
259
+ f"[cyan]{method_dir}/results.jsonl[/cyan]."
260
+ )
261
+ console.print(
262
+ " 4. Commit, push, and open a PR on https://github.com/compactbench/compactbench"
263
+ )
264
+ console.print(
265
+ " 5. A maintainer will code-review and apply the [cyan]evaluate[/cyan] "
266
+ "label, which runs your method against the hidden set."
267
+ )
268
+ console.print("")
269
+ console.print("Full protocol: https://compactbench.github.io/compactbench/submitting/")
270
+
271
+
272
+ providers_app = typer.Typer(help="Inspect available model providers.", no_args_is_help=True)
273
+ suites_app = typer.Typer(help="Inspect available benchmark suites.", no_args_is_help=True)
274
+ app.add_typer(providers_app, name="providers")
275
+ app.add_typer(suites_app, name="suites")
276
+
277
+
278
+ @providers_app.command("list")
279
+ def providers_list() -> None:
280
+ """List registered model providers."""
281
+ from rich.table import Table
282
+
283
+ from compactbench.providers import list_providers
284
+
285
+ providers = list_providers()
286
+ if not providers:
287
+ console.print("[yellow]no providers registered[/yellow]")
288
+ raise typer.Exit(code=1)
289
+
290
+ table = Table(title="Model providers")
291
+ table.add_column("Key")
292
+ table.add_column("Notes")
293
+ notes: dict[str, str] = {
294
+ "mock": "deterministic, offline — for tests and local dev",
295
+ "groq": "Groq Cloud. Set COMPACTBENCH_GROQ_API_KEY.",
296
+ "google-ai-studio": ("Google AI Studio. Set COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY."),
297
+ "ollama": "Local Ollama. COMPACTBENCH_OLLAMA_BASE_URL (default localhost:11434).",
298
+ }
299
+ for key in providers:
300
+ table.add_row(key, notes.get(key, ""))
301
+ console.print(table)
302
+
303
+
304
+ @suites_app.command("list")
305
+ def suites_list(
306
+ benchmarks_dir: Path = typer.Option(
307
+ Path("benchmarks/public"),
308
+ "--benchmarks-dir",
309
+ help="Directory containing public benchmark suites.",
310
+ ),
311
+ ) -> None:
312
+ """List available benchmark suites."""
313
+ from rich.table import Table
314
+
315
+ from compactbench.dsl import TemplateError, load_suite, validate_template
316
+
317
+ if not benchmarks_dir.is_dir():
318
+ console.print(f"[yellow]no benchmarks directory at {benchmarks_dir}[/yellow]")
319
+ raise typer.Exit(code=1)
320
+
321
+ table = Table(title="Benchmark suites")
322
+ table.add_column("Suite")
323
+ table.add_column("Templates", justify="right")
324
+ table.add_column("Families")
325
+
326
+ found_any = False
327
+ for suite_path in sorted(p for p in benchmarks_dir.iterdir() if p.is_dir()):
328
+ try:
329
+ templates = load_suite(suite_path)
330
+ for t in templates:
331
+ validate_template(t)
332
+ except TemplateError as exc:
333
+ console.print(f"[red]{suite_path.name}: {exc}[/red]")
334
+ continue
335
+ families = sorted({t.family for t in templates})
336
+ table.add_row(suite_path.name, str(len(templates)), ", ".join(families) or "—")
337
+ found_any = True
338
+
339
+ if not found_any:
340
+ console.print(f"[yellow]no suites found under {benchmarks_dir}[/yellow]")
341
+ raise typer.Exit(code=1)
342
+
343
+ console.print(table)
@@ -0,0 +1,47 @@
1
+ """Built-in compaction methods and the ``Compactor`` ABC that custom methods extend.
2
+
3
+ Built-in keys are registered here; external methods subclass :class:`Compactor`
4
+ and are loaded by file path in WO-007 (runner).
5
+ """
6
+
7
+ from compactbench.compactors.base import Compactor
8
+ from compactbench.compactors.errors import CompactorError, UnknownCompactorError
9
+ from compactbench.compactors.hierarchical_summary import HierarchicalSummaryCompactor
10
+ from compactbench.compactors.hybrid_ledger import HybridLedgerCompactor
11
+ from compactbench.compactors.naive_summary import NaiveSummaryCompactor
12
+ from compactbench.compactors.structured_state import StructuredStateCompactor
13
+
14
+ _BUILT_IN: dict[str, type[Compactor]] = {
15
+ NaiveSummaryCompactor.name: NaiveSummaryCompactor,
16
+ StructuredStateCompactor.name: StructuredStateCompactor,
17
+ HierarchicalSummaryCompactor.name: HierarchicalSummaryCompactor,
18
+ HybridLedgerCompactor.name: HybridLedgerCompactor,
19
+ }
20
+
21
+
22
+ def list_built_ins() -> list[str]:
23
+ """Return the sorted list of built-in compactor keys."""
24
+ return sorted(_BUILT_IN)
25
+
26
+
27
+ def get_built_in(key: str) -> type[Compactor]:
28
+ """Return the compactor class for ``key`` or raise :class:`UnknownCompactorError`."""
29
+ cls = _BUILT_IN.get(key)
30
+ if cls is None:
31
+ raise UnknownCompactorError(
32
+ f"unknown built-in compactor {key!r}. Known: {sorted(_BUILT_IN)}"
33
+ )
34
+ return cls
35
+
36
+
37
+ __all__ = [
38
+ "Compactor",
39
+ "CompactorError",
40
+ "HierarchicalSummaryCompactor",
41
+ "HybridLedgerCompactor",
42
+ "NaiveSummaryCompactor",
43
+ "StructuredStateCompactor",
44
+ "UnknownCompactorError",
45
+ "get_built_in",
46
+ "list_built_ins",
47
+ ]
@@ -0,0 +1,118 @@
1
+ """Shared JSON-to-StructuredState parser.
2
+
3
+ Several built-in compactors (structured-state, hierarchical-summary,
4
+ hybrid-ledger) ask the model to return a JSON object shaped like
5
+ :class:`StructuredState`. This module parses those responses leniently:
6
+
7
+ - Strips common LLM wrappers (``\\`\\`\\`json`` fences).
8
+ - Coerces missing sections to empty collections.
9
+ - Filters non-string values out of lists and the entity map.
10
+ - Returns any issues as warnings rather than raising, so a single bad field
11
+ doesn't discard the rest of the state.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from typing import Any, cast
18
+
19
+ from compactbench.contracts import StructuredState
20
+
21
+ _LIST_SECTIONS = (
22
+ "immutable_facts",
23
+ "locked_decisions",
24
+ "deferred_items",
25
+ "forbidden_behaviors",
26
+ "unresolved_items",
27
+ )
28
+
29
+ _MAX_SECTION_ITEMS = 200
30
+ _MAX_STRING_LENGTH = 500
31
+
32
+
33
+ def parse_state(text: str) -> tuple[StructuredState, list[str]]:
34
+ """Parse a JSON-object response into a :class:`StructuredState` and warnings."""
35
+ warnings: list[str] = []
36
+ cleaned = _strip_code_fences(text)
37
+ try:
38
+ data = json.loads(cleaned)
39
+ except json.JSONDecodeError as exc:
40
+ warnings.append(f"response was not valid JSON: {exc}")
41
+ return StructuredState(), warnings
42
+
43
+ if not isinstance(data, dict):
44
+ warnings.append(f"expected JSON object, got {type(data).__name__}; treating as empty state")
45
+ return StructuredState(), warnings
46
+
47
+ obj = cast("dict[str, Any]", data)
48
+ sections: dict[str, Any] = {
49
+ name: _clean_list(obj.get(name), name, warnings) for name in _LIST_SECTIONS
50
+ }
51
+ sections["entity_map"] = _clean_entity_map(obj.get("entity_map"), warnings)
52
+ return StructuredState.model_validate(sections), warnings
53
+
54
+
55
+ def _strip_code_fences(text: str) -> str:
56
+ """Remove ``\\`\\`\\`json`` fences or leading/trailing backticks if present."""
57
+ stripped = text.strip()
58
+ if not stripped.startswith("```"):
59
+ return stripped
60
+ # Find the first newline after the opening fence.
61
+ after_fence = stripped[3:]
62
+ if after_fence.lower().startswith("json"):
63
+ after_fence = after_fence[4:]
64
+ if "\n" in after_fence:
65
+ after_fence = after_fence.split("\n", 1)[1]
66
+ # Strip trailing closing fence if present.
67
+ if after_fence.rstrip().endswith("```"):
68
+ after_fence = after_fence.rstrip()[:-3]
69
+ return after_fence.strip()
70
+
71
+
72
+ def _clean_list(value: Any, name: str, warnings: list[str]) -> list[str]:
73
+ if value is None:
74
+ return []
75
+ if not isinstance(value, list):
76
+ warnings.append(f"{name!r} was not a list (got {type(value).__name__}); dropping")
77
+ return []
78
+ items = cast("list[Any]", value)
79
+ result: list[str] = []
80
+ for i, item in enumerate(items):
81
+ if not isinstance(item, str):
82
+ warnings.append(f"{name}[{i}] was not a string (got {type(item).__name__}); skipping")
83
+ continue
84
+ item = item.strip()
85
+ if not item:
86
+ continue
87
+ if len(item) > _MAX_STRING_LENGTH:
88
+ warnings.append(f"{name}[{i}] exceeded {_MAX_STRING_LENGTH} chars; truncating")
89
+ item = item[:_MAX_STRING_LENGTH]
90
+ result.append(item)
91
+ if len(result) >= _MAX_SECTION_ITEMS:
92
+ warnings.append(f"{name} exceeded {_MAX_SECTION_ITEMS} items; dropping the rest")
93
+ break
94
+ return result
95
+
96
+
97
+ def _clean_entity_map(value: Any, warnings: list[str]) -> dict[str, str]:
98
+ if value is None:
99
+ return {}
100
+ if not isinstance(value, dict):
101
+ warnings.append(f"'entity_map' was not a mapping (got {type(value).__name__}); dropping")
102
+ return {}
103
+ mapping = cast("dict[Any, Any]", value)
104
+ result: dict[str, str] = {}
105
+ for k, v in mapping.items():
106
+ if not isinstance(k, str) or not isinstance(v, str):
107
+ warnings.append(f"entity_map entry {k!r}={v!r} is not string-to-string; skipping")
108
+ continue
109
+ k = k.strip()
110
+ v = v.strip()
111
+ if not k or not v:
112
+ continue
113
+ if len(k) > _MAX_STRING_LENGTH:
114
+ k = k[:_MAX_STRING_LENGTH]
115
+ if len(v) > _MAX_STRING_LENGTH:
116
+ v = v[:_MAX_STRING_LENGTH]
117
+ result[k] = v
118
+ return result
@@ -0,0 +1,37 @@
1
+ """Helpers used across compactor implementations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TypeVar
6
+
7
+ from compactbench.contracts import Transcript, Turn
8
+
9
+ T = TypeVar("T")
10
+
11
+
12
+ def render_transcript(transcript: Transcript) -> str:
13
+ """Render a transcript as a plain text prompt body."""
14
+ return render_turns(list(transcript.turns))
15
+
16
+
17
+ def render_turns(turns: list[Turn]) -> str:
18
+ """Render an arbitrary list of turns as a plain text prompt body."""
19
+ return "\n\n".join(f"{turn.role.value.upper()}: {turn.content}" for turn in turns)
20
+
21
+
22
+ def chunk(items: list[T], size: int) -> list[list[T]]:
23
+ """Split ``items`` into groups of ``size`` (last group may be smaller)."""
24
+ if size <= 0:
25
+ raise ValueError(f"chunk size must be > 0, got {size}")
26
+ return [items[i : i + size] for i in range(0, len(items), size)]
27
+
28
+
29
+ def uniq_preserve_order(items: list[str]) -> list[str]:
30
+ """Return ``items`` with duplicates removed, preserving first-occurrence order."""
31
+ seen: set[str] = set()
32
+ result: list[str] = []
33
+ for item in items:
34
+ if item not in seen:
35
+ seen.add(item)
36
+ result.append(item)
37
+ return result
@@ -0,0 +1,38 @@
1
+ """Compactor ABC.
2
+
3
+ Every built-in and external compaction method subclasses this and returns a
4
+ validated :class:`CompactionArtifact`. A compactor is bound to a provider +
5
+ model at construction; ``compact`` is async because provider calls are.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from abc import ABC, abstractmethod
11
+ from typing import Any, ClassVar
12
+
13
+ from compactbench.contracts import CompactionArtifact, Transcript
14
+ from compactbench.providers import Provider
15
+
16
+
17
+ class Compactor(ABC):
18
+ """Abstract base for compaction methods."""
19
+
20
+ name: ClassVar[str]
21
+ version: ClassVar[str]
22
+
23
+ def __init__(self, provider: Provider, model: str) -> None:
24
+ self.provider = provider
25
+ self.model = model
26
+
27
+ @abstractmethod
28
+ async def compact(
29
+ self,
30
+ transcript: Transcript,
31
+ config: dict[str, Any] | None = None,
32
+ previous_artifact: CompactionArtifact | None = None,
33
+ ) -> CompactionArtifact:
34
+ """Produce a compacted artifact from a transcript.
35
+
36
+ ``previous_artifact`` is provided on drift cycles so ledger-style methods
37
+ can accumulate state; stateless methods should ignore it.
38
+ """
@@ -0,0 +1,11 @@
1
+ """Compactor-specific exceptions."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class CompactorError(Exception):
7
+ """Base exception for compactor operations."""
8
+
9
+
10
+ class UnknownCompactorError(CompactorError):
11
+ """Requested built-in compactor key is not registered."""