compactbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compactbench/__init__.py +10 -0
- compactbench/__main__.py +6 -0
- compactbench/cli.py +343 -0
- compactbench/compactors/__init__.py +47 -0
- compactbench/compactors/_state_parser.py +118 -0
- compactbench/compactors/_utils.py +37 -0
- compactbench/compactors/base.py +38 -0
- compactbench/compactors/errors.py +11 -0
- compactbench/compactors/hierarchical_summary.py +113 -0
- compactbench/compactors/hybrid_ledger.py +115 -0
- compactbench/compactors/naive_summary.py +54 -0
- compactbench/compactors/structured_state.py +64 -0
- compactbench/config.py +36 -0
- compactbench/contracts/__init__.py +41 -0
- compactbench/contracts/artifact.py +49 -0
- compactbench/contracts/case.py +81 -0
- compactbench/contracts/result.py +79 -0
- compactbench/dsl/__init__.py +66 -0
- compactbench/dsl/errors.py +23 -0
- compactbench/dsl/generators.py +243 -0
- compactbench/dsl/models.py +107 -0
- compactbench/dsl/parser.py +55 -0
- compactbench/dsl/substitution.py +32 -0
- compactbench/dsl/validator.py +97 -0
- compactbench/engine/__init__.py +29 -0
- compactbench/engine/difficulty.py +54 -0
- compactbench/engine/distractors.py +69 -0
- compactbench/engine/errors.py +11 -0
- compactbench/engine/generation.py +96 -0
- compactbench/engine/seeds.py +31 -0
- compactbench/engine/transcript.py +84 -0
- compactbench/leaderboard/__init__.py +38 -0
- compactbench/leaderboard/errors.py +11 -0
- compactbench/leaderboard/projection.py +103 -0
- compactbench/leaderboard/qualification.py +90 -0
- compactbench/leaderboard/ranking.py +71 -0
- compactbench/providers/__init__.py +52 -0
- compactbench/providers/_retry.py +48 -0
- compactbench/providers/base.py +44 -0
- compactbench/providers/errors.py +15 -0
- compactbench/providers/google_ai_studio.py +112 -0
- compactbench/providers/groq.py +95 -0
- compactbench/providers/mock.py +53 -0
- compactbench/providers/ollama.py +100 -0
- compactbench/py.typed +0 -0
- compactbench/runner/__init__.py +53 -0
- compactbench/runner/_resolver.py +57 -0
- compactbench/runner/continuation.py +78 -0
- compactbench/runner/cycle.py +67 -0
- compactbench/runner/errors.py +15 -0
- compactbench/runner/evaluation.py +71 -0
- compactbench/runner/persistence.py +227 -0
- compactbench/runner/run.py +257 -0
- compactbench/scoring/__init__.py +48 -0
- compactbench/scoring/checks.py +92 -0
- compactbench/scoring/compression.py +66 -0
- compactbench/scoring/contradictions.py +70 -0
- compactbench/scoring/drift.py +30 -0
- compactbench/scoring/errors.py +7 -0
- compactbench/scoring/scorer.py +77 -0
- compactbench/utils/__init__.py +1 -0
- compactbench-0.1.0.dist-info/METADATA +161 -0
- compactbench-0.1.0.dist-info/RECORD +66 -0
- compactbench-0.1.0.dist-info/WHEEL +4 -0
- compactbench-0.1.0.dist-info/entry_points.txt +2 -0
- compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
compactbench/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""CompactBench — open benchmark for AI conversation compaction methods."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("compactbench")
|
|
7
|
+
except PackageNotFoundError: # pragma: no cover - during local dev without install
|
|
8
|
+
__version__ = "0.0.0+local"
|
|
9
|
+
|
|
10
|
+
__all__ = ["__version__"]
|
compactbench/__main__.py
ADDED
compactbench/cli.py
ADDED
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""CompactBench command-line interface.
|
|
2
|
+
|
|
3
|
+
Implemented incrementally. Each command raises ``NotImplementedError`` until its
|
|
4
|
+
backing workorder lands. See docs/architecture/workorders.md.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import typer
|
|
13
|
+
from rich.console import Console
|
|
14
|
+
|
|
15
|
+
from compactbench import __version__
|
|
16
|
+
|
|
17
|
+
app = typer.Typer(
|
|
18
|
+
name="compactbench",
|
|
19
|
+
help="Open benchmark for AI conversation compaction methods.",
|
|
20
|
+
no_args_is_help=True,
|
|
21
|
+
add_completion=False,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
console = Console()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _version_callback(value: bool) -> None:
|
|
28
|
+
if value:
|
|
29
|
+
console.print(f"compactbench {__version__}")
|
|
30
|
+
raise typer.Exit()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@app.callback()
|
|
34
|
+
def main(
|
|
35
|
+
_version: bool = typer.Option(
|
|
36
|
+
False,
|
|
37
|
+
"--version",
|
|
38
|
+
"-V",
|
|
39
|
+
callback=_version_callback,
|
|
40
|
+
is_eager=True,
|
|
41
|
+
help="Show version and exit.",
|
|
42
|
+
),
|
|
43
|
+
) -> None:
|
|
44
|
+
"""CompactBench CLI."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@app.command()
|
|
48
|
+
def run(
|
|
49
|
+
method: str = typer.Option(
|
|
50
|
+
..., "--method", "-m", help="Method id: 'built-in:<key>' or '<path.py>:<ClassName>'."
|
|
51
|
+
),
|
|
52
|
+
suite: str = typer.Option("starter", "--suite", "-s", help="Benchmark suite key."),
|
|
53
|
+
provider: str = typer.Option("mock", "--provider", "-p", help="Model provider key."),
|
|
54
|
+
model: str = typer.Option("mock-deterministic", "--model", help="Provider-specific model key."),
|
|
55
|
+
difficulty: str = typer.Option("medium", "--difficulty"),
|
|
56
|
+
drift_cycles: int = typer.Option(2, "--drift-cycles", min=0, max=5),
|
|
57
|
+
case_count: int = typer.Option(3, "--case-count", min=1, help="Cases per template."),
|
|
58
|
+
seed_group: str = typer.Option("default", "--seed-group"),
|
|
59
|
+
benchmarks_dir: Path = typer.Option(
|
|
60
|
+
Path("benchmarks/public"),
|
|
61
|
+
"--benchmarks-dir",
|
|
62
|
+
help="Directory containing benchmark suites.",
|
|
63
|
+
),
|
|
64
|
+
output: Path = typer.Option(Path("results.jsonl"), "--output", "-o"),
|
|
65
|
+
resume: bool = typer.Option(
|
|
66
|
+
False, "--resume", help="Continue from existing output file (skip completed cases)."
|
|
67
|
+
),
|
|
68
|
+
) -> None:
|
|
69
|
+
"""Run a compaction method against a benchmark suite."""
|
|
70
|
+
import asyncio
|
|
71
|
+
|
|
72
|
+
from compactbench.dsl import DifficultyLevel
|
|
73
|
+
from compactbench.runner import RunArgs, RunnerError, run_experiment
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
diff = DifficultyLevel(difficulty.lower())
|
|
77
|
+
except ValueError as exc:
|
|
78
|
+
console.print(
|
|
79
|
+
f"[red]unknown difficulty {difficulty!r}. "
|
|
80
|
+
f"Valid: {[d.value for d in DifficultyLevel]}[/red]"
|
|
81
|
+
)
|
|
82
|
+
raise typer.Exit(code=1) from exc
|
|
83
|
+
|
|
84
|
+
args = RunArgs(
|
|
85
|
+
method_spec=method,
|
|
86
|
+
suite_key=suite,
|
|
87
|
+
provider_key=provider,
|
|
88
|
+
model=model,
|
|
89
|
+
difficulty=diff,
|
|
90
|
+
drift_cycles=drift_cycles,
|
|
91
|
+
case_count_per_template=case_count,
|
|
92
|
+
seed_group=seed_group,
|
|
93
|
+
benchmarks_dir=benchmarks_dir,
|
|
94
|
+
output_path=output,
|
|
95
|
+
resume=resume,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
try:
|
|
99
|
+
asyncio.run(run_experiment(args))
|
|
100
|
+
except RunnerError as exc:
|
|
101
|
+
console.print(f"[red]{exc}[/red]")
|
|
102
|
+
raise typer.Exit(code=1) from exc
|
|
103
|
+
|
|
104
|
+
console.print(f"[green]wrote {output}[/green]")
|
|
105
|
+
console.print(f"run 'compactbench score --results {output}' for a summary.")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@app.command()
|
|
109
|
+
def generate(
|
|
110
|
+
template: str = typer.Option(..., "--template", "-t", help="Template key."),
|
|
111
|
+
seed: int = typer.Option(0, "--seed"),
|
|
112
|
+
difficulty: str = typer.Option("medium", "--difficulty"),
|
|
113
|
+
benchmarks_dir: Path = typer.Option(
|
|
114
|
+
Path("benchmarks/public"),
|
|
115
|
+
"--benchmarks-dir",
|
|
116
|
+
help="Directory containing public benchmark suites.",
|
|
117
|
+
),
|
|
118
|
+
output: Path | None = typer.Option(
|
|
119
|
+
None, "--output", "-o", help="Write JSON to this file (stdout if omitted)."
|
|
120
|
+
),
|
|
121
|
+
) -> None:
|
|
122
|
+
"""Generate a single benchmark case from a template and seed."""
|
|
123
|
+
from compactbench.dsl import (
|
|
124
|
+
DifficultyLevel,
|
|
125
|
+
TemplateError,
|
|
126
|
+
load_suite,
|
|
127
|
+
validate_template,
|
|
128
|
+
)
|
|
129
|
+
from compactbench.engine import generate_case
|
|
130
|
+
|
|
131
|
+
if not benchmarks_dir.is_dir():
|
|
132
|
+
console.print(f"[red]no benchmarks directory at {benchmarks_dir}[/red]")
|
|
133
|
+
raise typer.Exit(code=1)
|
|
134
|
+
|
|
135
|
+
found: dict[str, Any] = {}
|
|
136
|
+
for suite_dir in sorted(p for p in benchmarks_dir.iterdir() if p.is_dir()):
|
|
137
|
+
try:
|
|
138
|
+
for t in load_suite(suite_dir):
|
|
139
|
+
found[t.key] = t
|
|
140
|
+
except TemplateError as exc:
|
|
141
|
+
console.print(f"[yellow]skipping {suite_dir.name}: {exc}[/yellow]")
|
|
142
|
+
|
|
143
|
+
if template not in found:
|
|
144
|
+
console.print(f"[red]template {template!r} not found. Known: {sorted(found)}[/red]")
|
|
145
|
+
raise typer.Exit(code=1)
|
|
146
|
+
|
|
147
|
+
tmpl = found[template]
|
|
148
|
+
try:
|
|
149
|
+
validate_template(tmpl)
|
|
150
|
+
except TemplateError as exc:
|
|
151
|
+
console.print(f"[red]template {template!r} failed validation: {exc}[/red]")
|
|
152
|
+
raise typer.Exit(code=1) from exc
|
|
153
|
+
|
|
154
|
+
try:
|
|
155
|
+
diff = DifficultyLevel(difficulty.lower())
|
|
156
|
+
except ValueError as exc:
|
|
157
|
+
console.print(
|
|
158
|
+
f"[red]unknown difficulty {difficulty!r}. "
|
|
159
|
+
f"Valid: {[d.value for d in DifficultyLevel]}[/red]"
|
|
160
|
+
)
|
|
161
|
+
raise typer.Exit(code=1) from exc
|
|
162
|
+
|
|
163
|
+
case = generate_case(tmpl, seed, diff)
|
|
164
|
+
json_text = case.model_dump_json(indent=2)
|
|
165
|
+
if output is None:
|
|
166
|
+
console.print_json(json_text)
|
|
167
|
+
else:
|
|
168
|
+
output.write_text(json_text + "\n", encoding="utf-8")
|
|
169
|
+
console.print(f"wrote {output}")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@app.command()
|
|
173
|
+
def score(
|
|
174
|
+
results: Path = typer.Option(..., "--results", "-r", exists=True, readable=True),
|
|
175
|
+
) -> None:
|
|
176
|
+
"""Print a summary of a ``results.jsonl`` file produced by ``compactbench run``."""
|
|
177
|
+
from rich.table import Table
|
|
178
|
+
|
|
179
|
+
from compactbench.runner import to_run_result
|
|
180
|
+
|
|
181
|
+
try:
|
|
182
|
+
run_result = to_run_result(results)
|
|
183
|
+
except ValueError as exc:
|
|
184
|
+
console.print(f"[red]{exc}[/red]")
|
|
185
|
+
raise typer.Exit(code=1) from exc
|
|
186
|
+
|
|
187
|
+
header = Table(title="Run")
|
|
188
|
+
header.add_column("Field")
|
|
189
|
+
header.add_column("Value")
|
|
190
|
+
header.add_row("run_id", run_result.run_id)
|
|
191
|
+
header.add_row("method", f"{run_result.method_name} ({run_result.method_version})")
|
|
192
|
+
header.add_row("suite", f"{run_result.suite_key} ({run_result.suite_version})")
|
|
193
|
+
header.add_row("target", f"{run_result.target_provider} / {run_result.target_model}")
|
|
194
|
+
header.add_row("scorer", run_result.scorer_version)
|
|
195
|
+
header.add_row("started", run_result.started_at.isoformat())
|
|
196
|
+
header.add_row("completed", run_result.completed_at.isoformat())
|
|
197
|
+
console.print(header)
|
|
198
|
+
|
|
199
|
+
per_case = Table(title=f"{len(run_result.cases)} case(s)")
|
|
200
|
+
per_case.add_column("Case ID")
|
|
201
|
+
per_case.add_column("Case score", justify="right")
|
|
202
|
+
per_case.add_column("Drift resistance", justify="right")
|
|
203
|
+
per_case.add_column("Cycles", justify="right")
|
|
204
|
+
for case in run_result.cases:
|
|
205
|
+
per_case.add_row(
|
|
206
|
+
case.case_id,
|
|
207
|
+
f"{case.case_score:.3f}",
|
|
208
|
+
f"{case.drift_resistance:.3f}",
|
|
209
|
+
str(len(case.cycles)),
|
|
210
|
+
)
|
|
211
|
+
console.print(per_case)
|
|
212
|
+
|
|
213
|
+
summary = Table(title="Aggregate")
|
|
214
|
+
summary.add_column("Metric")
|
|
215
|
+
summary.add_column("Value", justify="right")
|
|
216
|
+
summary.add_row("overall_score", f"{run_result.overall_score:.3f}")
|
|
217
|
+
summary.add_row("drift_resistance", f"{run_result.drift_resistance:.3f}")
|
|
218
|
+
summary.add_row("constraint_retention", f"{run_result.constraint_retention:.3f}")
|
|
219
|
+
summary.add_row("contradiction_rate", f"{run_result.contradiction_rate:.3f}")
|
|
220
|
+
summary.add_row("compression_ratio", f"{run_result.compression_ratio:.2f}x")
|
|
221
|
+
console.print(summary)
|
|
222
|
+
|
|
223
|
+
for note in run_result.notes:
|
|
224
|
+
console.print(f"[yellow]note: {note}[/yellow]")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
@app.command()
|
|
228
|
+
def submit(
|
|
229
|
+
results: Path = typer.Option(
|
|
230
|
+
...,
|
|
231
|
+
"--results",
|
|
232
|
+
"-r",
|
|
233
|
+
exists=True,
|
|
234
|
+
readable=True,
|
|
235
|
+
help="Local results.jsonl produced by `compactbench run`.",
|
|
236
|
+
),
|
|
237
|
+
handle: str = typer.Option(..., "--handle", help="Your GitHub handle."),
|
|
238
|
+
method_name: str = typer.Option(
|
|
239
|
+
..., "--name", help="Method name in kebab-case, e.g. 'my-ledger-v1'."
|
|
240
|
+
),
|
|
241
|
+
) -> None:
|
|
242
|
+
"""Print the steps to submit a method for leaderboard scoring.
|
|
243
|
+
|
|
244
|
+
Full automation of PR creation is out of scope for the local CLI — GitHub
|
|
245
|
+
Actions evaluation happens against the hidden test set on a repo secret,
|
|
246
|
+
so submission is always a PR + maintainer review. This command prints a
|
|
247
|
+
checklist so you don't miss a step.
|
|
248
|
+
"""
|
|
249
|
+
method_dir = Path("submissions") / handle / method_name
|
|
250
|
+
console.print("[bold]Submit a method to the CompactBench leaderboard[/bold]")
|
|
251
|
+
console.print("")
|
|
252
|
+
console.print(f" 1. Create [cyan]{method_dir}/[/cyan] in your fork.")
|
|
253
|
+
console.print(
|
|
254
|
+
f" 2. Copy the scaffold from [cyan]submissions/_template/[/cyan] "
|
|
255
|
+
f"into [cyan]{method_dir}/[/cyan] and edit every file."
|
|
256
|
+
)
|
|
257
|
+
console.print(
|
|
258
|
+
f" 3. Put your validated [cyan]{results}[/cyan] at "
|
|
259
|
+
f"[cyan]{method_dir}/results.jsonl[/cyan]."
|
|
260
|
+
)
|
|
261
|
+
console.print(
|
|
262
|
+
" 4. Commit, push, and open a PR on https://github.com/compactbench/compactbench"
|
|
263
|
+
)
|
|
264
|
+
console.print(
|
|
265
|
+
" 5. A maintainer will code-review and apply the [cyan]evaluate[/cyan] "
|
|
266
|
+
"label, which runs your method against the hidden set."
|
|
267
|
+
)
|
|
268
|
+
console.print("")
|
|
269
|
+
console.print("Full protocol: https://compactbench.github.io/compactbench/submitting/")
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
providers_app = typer.Typer(help="Inspect available model providers.", no_args_is_help=True)
|
|
273
|
+
suites_app = typer.Typer(help="Inspect available benchmark suites.", no_args_is_help=True)
|
|
274
|
+
app.add_typer(providers_app, name="providers")
|
|
275
|
+
app.add_typer(suites_app, name="suites")
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@providers_app.command("list")
|
|
279
|
+
def providers_list() -> None:
|
|
280
|
+
"""List registered model providers."""
|
|
281
|
+
from rich.table import Table
|
|
282
|
+
|
|
283
|
+
from compactbench.providers import list_providers
|
|
284
|
+
|
|
285
|
+
providers = list_providers()
|
|
286
|
+
if not providers:
|
|
287
|
+
console.print("[yellow]no providers registered[/yellow]")
|
|
288
|
+
raise typer.Exit(code=1)
|
|
289
|
+
|
|
290
|
+
table = Table(title="Model providers")
|
|
291
|
+
table.add_column("Key")
|
|
292
|
+
table.add_column("Notes")
|
|
293
|
+
notes: dict[str, str] = {
|
|
294
|
+
"mock": "deterministic, offline — for tests and local dev",
|
|
295
|
+
"groq": "Groq Cloud. Set COMPACTBENCH_GROQ_API_KEY.",
|
|
296
|
+
"google-ai-studio": ("Google AI Studio. Set COMPACTBENCH_GOOGLE_AI_STUDIO_API_KEY."),
|
|
297
|
+
"ollama": "Local Ollama. COMPACTBENCH_OLLAMA_BASE_URL (default localhost:11434).",
|
|
298
|
+
}
|
|
299
|
+
for key in providers:
|
|
300
|
+
table.add_row(key, notes.get(key, ""))
|
|
301
|
+
console.print(table)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
@suites_app.command("list")
|
|
305
|
+
def suites_list(
|
|
306
|
+
benchmarks_dir: Path = typer.Option(
|
|
307
|
+
Path("benchmarks/public"),
|
|
308
|
+
"--benchmarks-dir",
|
|
309
|
+
help="Directory containing public benchmark suites.",
|
|
310
|
+
),
|
|
311
|
+
) -> None:
|
|
312
|
+
"""List available benchmark suites."""
|
|
313
|
+
from rich.table import Table
|
|
314
|
+
|
|
315
|
+
from compactbench.dsl import TemplateError, load_suite, validate_template
|
|
316
|
+
|
|
317
|
+
if not benchmarks_dir.is_dir():
|
|
318
|
+
console.print(f"[yellow]no benchmarks directory at {benchmarks_dir}[/yellow]")
|
|
319
|
+
raise typer.Exit(code=1)
|
|
320
|
+
|
|
321
|
+
table = Table(title="Benchmark suites")
|
|
322
|
+
table.add_column("Suite")
|
|
323
|
+
table.add_column("Templates", justify="right")
|
|
324
|
+
table.add_column("Families")
|
|
325
|
+
|
|
326
|
+
found_any = False
|
|
327
|
+
for suite_path in sorted(p for p in benchmarks_dir.iterdir() if p.is_dir()):
|
|
328
|
+
try:
|
|
329
|
+
templates = load_suite(suite_path)
|
|
330
|
+
for t in templates:
|
|
331
|
+
validate_template(t)
|
|
332
|
+
except TemplateError as exc:
|
|
333
|
+
console.print(f"[red]{suite_path.name}: {exc}[/red]")
|
|
334
|
+
continue
|
|
335
|
+
families = sorted({t.family for t in templates})
|
|
336
|
+
table.add_row(suite_path.name, str(len(templates)), ", ".join(families) or "—")
|
|
337
|
+
found_any = True
|
|
338
|
+
|
|
339
|
+
if not found_any:
|
|
340
|
+
console.print(f"[yellow]no suites found under {benchmarks_dir}[/yellow]")
|
|
341
|
+
raise typer.Exit(code=1)
|
|
342
|
+
|
|
343
|
+
console.print(table)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Built-in compaction methods and the ``Compactor`` ABC that custom methods extend.
|
|
2
|
+
|
|
3
|
+
Built-in keys are registered here; external methods subclass :class:`Compactor`
|
|
4
|
+
and are loaded by file path in WO-007 (runner).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from compactbench.compactors.base import Compactor
|
|
8
|
+
from compactbench.compactors.errors import CompactorError, UnknownCompactorError
|
|
9
|
+
from compactbench.compactors.hierarchical_summary import HierarchicalSummaryCompactor
|
|
10
|
+
from compactbench.compactors.hybrid_ledger import HybridLedgerCompactor
|
|
11
|
+
from compactbench.compactors.naive_summary import NaiveSummaryCompactor
|
|
12
|
+
from compactbench.compactors.structured_state import StructuredStateCompactor
|
|
13
|
+
|
|
14
|
+
_BUILT_IN: dict[str, type[Compactor]] = {
|
|
15
|
+
NaiveSummaryCompactor.name: NaiveSummaryCompactor,
|
|
16
|
+
StructuredStateCompactor.name: StructuredStateCompactor,
|
|
17
|
+
HierarchicalSummaryCompactor.name: HierarchicalSummaryCompactor,
|
|
18
|
+
HybridLedgerCompactor.name: HybridLedgerCompactor,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def list_built_ins() -> list[str]:
|
|
23
|
+
"""Return the sorted list of built-in compactor keys."""
|
|
24
|
+
return sorted(_BUILT_IN)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_built_in(key: str) -> type[Compactor]:
|
|
28
|
+
"""Return the compactor class for ``key`` or raise :class:`UnknownCompactorError`."""
|
|
29
|
+
cls = _BUILT_IN.get(key)
|
|
30
|
+
if cls is None:
|
|
31
|
+
raise UnknownCompactorError(
|
|
32
|
+
f"unknown built-in compactor {key!r}. Known: {sorted(_BUILT_IN)}"
|
|
33
|
+
)
|
|
34
|
+
return cls
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
"Compactor",
|
|
39
|
+
"CompactorError",
|
|
40
|
+
"HierarchicalSummaryCompactor",
|
|
41
|
+
"HybridLedgerCompactor",
|
|
42
|
+
"NaiveSummaryCompactor",
|
|
43
|
+
"StructuredStateCompactor",
|
|
44
|
+
"UnknownCompactorError",
|
|
45
|
+
"get_built_in",
|
|
46
|
+
"list_built_ins",
|
|
47
|
+
]
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Shared JSON-to-StructuredState parser.
|
|
2
|
+
|
|
3
|
+
Several built-in compactors (structured-state, hierarchical-summary,
|
|
4
|
+
hybrid-ledger) ask the model to return a JSON object shaped like
|
|
5
|
+
:class:`StructuredState`. This module parses those responses leniently:
|
|
6
|
+
|
|
7
|
+
- Strips common LLM wrappers (``\\`\\`\\`json`` fences).
|
|
8
|
+
- Coerces missing sections to empty collections.
|
|
9
|
+
- Filters non-string values out of lists and the entity map.
|
|
10
|
+
- Returns any issues as warnings rather than raising, so a single bad field
|
|
11
|
+
doesn't discard the rest of the state.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from typing import Any, cast
|
|
18
|
+
|
|
19
|
+
from compactbench.contracts import StructuredState
|
|
20
|
+
|
|
21
|
+
_LIST_SECTIONS = (
|
|
22
|
+
"immutable_facts",
|
|
23
|
+
"locked_decisions",
|
|
24
|
+
"deferred_items",
|
|
25
|
+
"forbidden_behaviors",
|
|
26
|
+
"unresolved_items",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
_MAX_SECTION_ITEMS = 200
|
|
30
|
+
_MAX_STRING_LENGTH = 500
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def parse_state(text: str) -> tuple[StructuredState, list[str]]:
|
|
34
|
+
"""Parse a JSON-object response into a :class:`StructuredState` and warnings."""
|
|
35
|
+
warnings: list[str] = []
|
|
36
|
+
cleaned = _strip_code_fences(text)
|
|
37
|
+
try:
|
|
38
|
+
data = json.loads(cleaned)
|
|
39
|
+
except json.JSONDecodeError as exc:
|
|
40
|
+
warnings.append(f"response was not valid JSON: {exc}")
|
|
41
|
+
return StructuredState(), warnings
|
|
42
|
+
|
|
43
|
+
if not isinstance(data, dict):
|
|
44
|
+
warnings.append(f"expected JSON object, got {type(data).__name__}; treating as empty state")
|
|
45
|
+
return StructuredState(), warnings
|
|
46
|
+
|
|
47
|
+
obj = cast("dict[str, Any]", data)
|
|
48
|
+
sections: dict[str, Any] = {
|
|
49
|
+
name: _clean_list(obj.get(name), name, warnings) for name in _LIST_SECTIONS
|
|
50
|
+
}
|
|
51
|
+
sections["entity_map"] = _clean_entity_map(obj.get("entity_map"), warnings)
|
|
52
|
+
return StructuredState.model_validate(sections), warnings
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _strip_code_fences(text: str) -> str:
|
|
56
|
+
"""Remove ``\\`\\`\\`json`` fences or leading/trailing backticks if present."""
|
|
57
|
+
stripped = text.strip()
|
|
58
|
+
if not stripped.startswith("```"):
|
|
59
|
+
return stripped
|
|
60
|
+
# Find the first newline after the opening fence.
|
|
61
|
+
after_fence = stripped[3:]
|
|
62
|
+
if after_fence.lower().startswith("json"):
|
|
63
|
+
after_fence = after_fence[4:]
|
|
64
|
+
if "\n" in after_fence:
|
|
65
|
+
after_fence = after_fence.split("\n", 1)[1]
|
|
66
|
+
# Strip trailing closing fence if present.
|
|
67
|
+
if after_fence.rstrip().endswith("```"):
|
|
68
|
+
after_fence = after_fence.rstrip()[:-3]
|
|
69
|
+
return after_fence.strip()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _clean_list(value: Any, name: str, warnings: list[str]) -> list[str]:
|
|
73
|
+
if value is None:
|
|
74
|
+
return []
|
|
75
|
+
if not isinstance(value, list):
|
|
76
|
+
warnings.append(f"{name!r} was not a list (got {type(value).__name__}); dropping")
|
|
77
|
+
return []
|
|
78
|
+
items = cast("list[Any]", value)
|
|
79
|
+
result: list[str] = []
|
|
80
|
+
for i, item in enumerate(items):
|
|
81
|
+
if not isinstance(item, str):
|
|
82
|
+
warnings.append(f"{name}[{i}] was not a string (got {type(item).__name__}); skipping")
|
|
83
|
+
continue
|
|
84
|
+
item = item.strip()
|
|
85
|
+
if not item:
|
|
86
|
+
continue
|
|
87
|
+
if len(item) > _MAX_STRING_LENGTH:
|
|
88
|
+
warnings.append(f"{name}[{i}] exceeded {_MAX_STRING_LENGTH} chars; truncating")
|
|
89
|
+
item = item[:_MAX_STRING_LENGTH]
|
|
90
|
+
result.append(item)
|
|
91
|
+
if len(result) >= _MAX_SECTION_ITEMS:
|
|
92
|
+
warnings.append(f"{name} exceeded {_MAX_SECTION_ITEMS} items; dropping the rest")
|
|
93
|
+
break
|
|
94
|
+
return result
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _clean_entity_map(value: Any, warnings: list[str]) -> dict[str, str]:
|
|
98
|
+
if value is None:
|
|
99
|
+
return {}
|
|
100
|
+
if not isinstance(value, dict):
|
|
101
|
+
warnings.append(f"'entity_map' was not a mapping (got {type(value).__name__}); dropping")
|
|
102
|
+
return {}
|
|
103
|
+
mapping = cast("dict[Any, Any]", value)
|
|
104
|
+
result: dict[str, str] = {}
|
|
105
|
+
for k, v in mapping.items():
|
|
106
|
+
if not isinstance(k, str) or not isinstance(v, str):
|
|
107
|
+
warnings.append(f"entity_map entry {k!r}={v!r} is not string-to-string; skipping")
|
|
108
|
+
continue
|
|
109
|
+
k = k.strip()
|
|
110
|
+
v = v.strip()
|
|
111
|
+
if not k or not v:
|
|
112
|
+
continue
|
|
113
|
+
if len(k) > _MAX_STRING_LENGTH:
|
|
114
|
+
k = k[:_MAX_STRING_LENGTH]
|
|
115
|
+
if len(v) > _MAX_STRING_LENGTH:
|
|
116
|
+
v = v[:_MAX_STRING_LENGTH]
|
|
117
|
+
result[k] = v
|
|
118
|
+
return result
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Helpers used across compactor implementations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TypeVar
|
|
6
|
+
|
|
7
|
+
from compactbench.contracts import Transcript, Turn
|
|
8
|
+
|
|
9
|
+
T = TypeVar("T")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def render_transcript(transcript: Transcript) -> str:
|
|
13
|
+
"""Render a transcript as a plain text prompt body."""
|
|
14
|
+
return render_turns(list(transcript.turns))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def render_turns(turns: list[Turn]) -> str:
|
|
18
|
+
"""Render an arbitrary list of turns as a plain text prompt body."""
|
|
19
|
+
return "\n\n".join(f"{turn.role.value.upper()}: {turn.content}" for turn in turns)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def chunk(items: list[T], size: int) -> list[list[T]]:
|
|
23
|
+
"""Split ``items`` into groups of ``size`` (last group may be smaller)."""
|
|
24
|
+
if size <= 0:
|
|
25
|
+
raise ValueError(f"chunk size must be > 0, got {size}")
|
|
26
|
+
return [items[i : i + size] for i in range(0, len(items), size)]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def uniq_preserve_order(items: list[str]) -> list[str]:
|
|
30
|
+
"""Return ``items`` with duplicates removed, preserving first-occurrence order."""
|
|
31
|
+
seen: set[str] = set()
|
|
32
|
+
result: list[str] = []
|
|
33
|
+
for item in items:
|
|
34
|
+
if item not in seen:
|
|
35
|
+
seen.add(item)
|
|
36
|
+
result.append(item)
|
|
37
|
+
return result
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Compactor ABC.
|
|
2
|
+
|
|
3
|
+
Every built-in and external compaction method subclasses this and returns a
|
|
4
|
+
validated :class:`CompactionArtifact`. A compactor is bound to a provider +
|
|
5
|
+
model at construction; ``compact`` is async because provider calls are.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
from typing import Any, ClassVar
|
|
12
|
+
|
|
13
|
+
from compactbench.contracts import CompactionArtifact, Transcript
|
|
14
|
+
from compactbench.providers import Provider
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Compactor(ABC):
|
|
18
|
+
"""Abstract base for compaction methods."""
|
|
19
|
+
|
|
20
|
+
name: ClassVar[str]
|
|
21
|
+
version: ClassVar[str]
|
|
22
|
+
|
|
23
|
+
def __init__(self, provider: Provider, model: str) -> None:
|
|
24
|
+
self.provider = provider
|
|
25
|
+
self.model = model
|
|
26
|
+
|
|
27
|
+
@abstractmethod
|
|
28
|
+
async def compact(
|
|
29
|
+
self,
|
|
30
|
+
transcript: Transcript,
|
|
31
|
+
config: dict[str, Any] | None = None,
|
|
32
|
+
previous_artifact: CompactionArtifact | None = None,
|
|
33
|
+
) -> CompactionArtifact:
|
|
34
|
+
"""Produce a compacted artifact from a transcript.
|
|
35
|
+
|
|
36
|
+
``previous_artifact`` is provided on drift cycles so ledger-style methods
|
|
37
|
+
can accumulate state; stateless methods should ignore it.
|
|
38
|
+
"""
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Compactor-specific exceptions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CompactorError(Exception):
|
|
7
|
+
"""Base exception for compactor operations."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class UnknownCompactorError(CompactorError):
|
|
11
|
+
"""Requested built-in compactor key is not registered."""
|