compactbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compactbench/__init__.py +10 -0
- compactbench/__main__.py +6 -0
- compactbench/cli.py +343 -0
- compactbench/compactors/__init__.py +47 -0
- compactbench/compactors/_state_parser.py +118 -0
- compactbench/compactors/_utils.py +37 -0
- compactbench/compactors/base.py +38 -0
- compactbench/compactors/errors.py +11 -0
- compactbench/compactors/hierarchical_summary.py +113 -0
- compactbench/compactors/hybrid_ledger.py +115 -0
- compactbench/compactors/naive_summary.py +54 -0
- compactbench/compactors/structured_state.py +64 -0
- compactbench/config.py +36 -0
- compactbench/contracts/__init__.py +41 -0
- compactbench/contracts/artifact.py +49 -0
- compactbench/contracts/case.py +81 -0
- compactbench/contracts/result.py +79 -0
- compactbench/dsl/__init__.py +66 -0
- compactbench/dsl/errors.py +23 -0
- compactbench/dsl/generators.py +243 -0
- compactbench/dsl/models.py +107 -0
- compactbench/dsl/parser.py +55 -0
- compactbench/dsl/substitution.py +32 -0
- compactbench/dsl/validator.py +97 -0
- compactbench/engine/__init__.py +29 -0
- compactbench/engine/difficulty.py +54 -0
- compactbench/engine/distractors.py +69 -0
- compactbench/engine/errors.py +11 -0
- compactbench/engine/generation.py +96 -0
- compactbench/engine/seeds.py +31 -0
- compactbench/engine/transcript.py +84 -0
- compactbench/leaderboard/__init__.py +38 -0
- compactbench/leaderboard/errors.py +11 -0
- compactbench/leaderboard/projection.py +103 -0
- compactbench/leaderboard/qualification.py +90 -0
- compactbench/leaderboard/ranking.py +71 -0
- compactbench/providers/__init__.py +52 -0
- compactbench/providers/_retry.py +48 -0
- compactbench/providers/base.py +44 -0
- compactbench/providers/errors.py +15 -0
- compactbench/providers/google_ai_studio.py +112 -0
- compactbench/providers/groq.py +95 -0
- compactbench/providers/mock.py +53 -0
- compactbench/providers/ollama.py +100 -0
- compactbench/py.typed +0 -0
- compactbench/runner/__init__.py +53 -0
- compactbench/runner/_resolver.py +57 -0
- compactbench/runner/continuation.py +78 -0
- compactbench/runner/cycle.py +67 -0
- compactbench/runner/errors.py +15 -0
- compactbench/runner/evaluation.py +71 -0
- compactbench/runner/persistence.py +227 -0
- compactbench/runner/run.py +257 -0
- compactbench/scoring/__init__.py +48 -0
- compactbench/scoring/checks.py +92 -0
- compactbench/scoring/compression.py +66 -0
- compactbench/scoring/contradictions.py +70 -0
- compactbench/scoring/drift.py +30 -0
- compactbench/scoring/errors.py +7 -0
- compactbench/scoring/scorer.py +77 -0
- compactbench/utils/__init__.py +1 -0
- compactbench-0.1.0.dist-info/METADATA +161 -0
- compactbench-0.1.0.dist-info/RECORD +66 -0
- compactbench-0.1.0.dist-info/WHEEL +4 -0
- compactbench-0.1.0.dist-info/entry_points.txt +2 -0
- compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Hierarchical summary: chunk -> per-chunk summary -> meta-summary -> structured state."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, ClassVar
|
|
6
|
+
|
|
7
|
+
from compactbench.compactors._state_parser import parse_state
|
|
8
|
+
from compactbench.compactors._utils import chunk, render_transcript, render_turns
|
|
9
|
+
from compactbench.compactors.base import Compactor
|
|
10
|
+
from compactbench.contracts import CompactionArtifact, Transcript
|
|
11
|
+
from compactbench.providers import CompletionRequest
|
|
12
|
+
|
|
13
|
+
_CHUNK_PROMPT = (
|
|
14
|
+
"Summarize these {n} conversation turns in under 200 words. Preserve "
|
|
15
|
+
"constraints, decisions, and unresolved items.\n\n"
|
|
16
|
+
"TURNS:\n{turns}\n\n"
|
|
17
|
+
"SUMMARY:"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
_META_PROMPT = (
|
|
21
|
+
"Combine these partial summaries into one coherent summary in under 500 "
|
|
22
|
+
"words. Preserve every constraint, decision, and unresolved item.\n\n"
|
|
23
|
+
"{summaries}\n\n"
|
|
24
|
+
"FINAL SUMMARY:"
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
_STATE_PROMPT = (
|
|
28
|
+
"Extract a strict JSON object summarizing the state of this conversation. "
|
|
29
|
+
"Return ONLY the JSON, no prose.\n\n"
|
|
30
|
+
"Required fields: immutable_facts, locked_decisions, deferred_items, "
|
|
31
|
+
"forbidden_behaviors, entity_map, unresolved_items.\n\n"
|
|
32
|
+
"CONVERSATION:\n{transcript}\n\n"
|
|
33
|
+
"JSON:"
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class HierarchicalSummaryCompactor(Compactor):
|
|
38
|
+
"""Two-level summarization plus a separate structured-state extraction call."""
|
|
39
|
+
|
|
40
|
+
name: ClassVar[str] = "hierarchical-summary"
|
|
41
|
+
version: ClassVar[str] = "1.0.0"
|
|
42
|
+
|
|
43
|
+
_CHUNK_SIZE: ClassVar[int] = 10
|
|
44
|
+
|
|
45
|
+
async def compact(
|
|
46
|
+
self,
|
|
47
|
+
transcript: Transcript,
|
|
48
|
+
config: dict[str, Any] | None = None,
|
|
49
|
+
previous_artifact: CompactionArtifact | None = None,
|
|
50
|
+
) -> CompactionArtifact:
|
|
51
|
+
chunks = chunk(list(transcript.turns), self._CHUNK_SIZE)
|
|
52
|
+
|
|
53
|
+
chunk_summaries: list[str] = []
|
|
54
|
+
total_prompt_tokens = 0
|
|
55
|
+
total_completion_tokens = 0
|
|
56
|
+
|
|
57
|
+
for turns in chunks:
|
|
58
|
+
response = await self.provider.complete(
|
|
59
|
+
CompletionRequest(
|
|
60
|
+
model=self.model,
|
|
61
|
+
prompt=_CHUNK_PROMPT.format(n=len(turns), turns=render_turns(turns)),
|
|
62
|
+
)
|
|
63
|
+
)
|
|
64
|
+
chunk_summaries.append(response.text.strip())
|
|
65
|
+
total_prompt_tokens += response.prompt_tokens
|
|
66
|
+
total_completion_tokens += response.completion_tokens
|
|
67
|
+
|
|
68
|
+
if len(chunk_summaries) == 1:
|
|
69
|
+
final_summary = chunk_summaries[0]
|
|
70
|
+
else:
|
|
71
|
+
meta_response = await self.provider.complete(
|
|
72
|
+
CompletionRequest(
|
|
73
|
+
model=self.model,
|
|
74
|
+
prompt=_META_PROMPT.format(
|
|
75
|
+
summaries="\n\n".join(
|
|
76
|
+
f"PARTIAL {i + 1}: {s}" for i, s in enumerate(chunk_summaries)
|
|
77
|
+
)
|
|
78
|
+
),
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
final_summary = meta_response.text.strip()
|
|
82
|
+
total_prompt_tokens += meta_response.prompt_tokens
|
|
83
|
+
total_completion_tokens += meta_response.completion_tokens
|
|
84
|
+
|
|
85
|
+
state_response = await self.provider.complete(
|
|
86
|
+
CompletionRequest(
|
|
87
|
+
model=self.model,
|
|
88
|
+
prompt=_STATE_PROMPT.format(transcript=render_transcript(transcript)),
|
|
89
|
+
response_format={"type": "json_object"},
|
|
90
|
+
)
|
|
91
|
+
)
|
|
92
|
+
state, warnings = parse_state(state_response.text)
|
|
93
|
+
total_prompt_tokens += state_response.prompt_tokens
|
|
94
|
+
total_completion_tokens += state_response.completion_tokens
|
|
95
|
+
|
|
96
|
+
total_calls = len(chunk_summaries) + (1 if len(chunk_summaries) > 1 else 0) + 1
|
|
97
|
+
|
|
98
|
+
return CompactionArtifact(
|
|
99
|
+
summaryText=final_summary,
|
|
100
|
+
structured_state=state,
|
|
101
|
+
selectedSourceTurnIds=[t.id for t in transcript.turns],
|
|
102
|
+
warnings=warnings,
|
|
103
|
+
methodMetadata={
|
|
104
|
+
"method": self.name,
|
|
105
|
+
"version": self.version,
|
|
106
|
+
"model": self.model,
|
|
107
|
+
"provider": self.provider.key,
|
|
108
|
+
"chunks": len(chunks),
|
|
109
|
+
"calls": total_calls,
|
|
110
|
+
"prompt_tokens": total_prompt_tokens,
|
|
111
|
+
"completion_tokens": total_completion_tokens,
|
|
112
|
+
},
|
|
113
|
+
)
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Hybrid ledger: append-only structured state across cycles + short prose header."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, ClassVar
|
|
6
|
+
|
|
7
|
+
from compactbench.compactors._state_parser import parse_state
|
|
8
|
+
from compactbench.compactors._utils import render_transcript, uniq_preserve_order
|
|
9
|
+
from compactbench.compactors.base import Compactor
|
|
10
|
+
from compactbench.contracts import CompactionArtifact, StructuredState, Transcript
|
|
11
|
+
from compactbench.providers import CompletionRequest
|
|
12
|
+
|
|
13
|
+
_DELTA_PROMPT = (
|
|
14
|
+
"Extract NEW state that appears in this conversation segment as a strict "
|
|
15
|
+
"JSON object. Include only items stated or decided in these turns; do not "
|
|
16
|
+
"infer beyond the text. Return ONLY the JSON, no prose.\n\n"
|
|
17
|
+
"Required fields: immutable_facts, locked_decisions, deferred_items, "
|
|
18
|
+
"forbidden_behaviors, entity_map, unresolved_items.\n\n"
|
|
19
|
+
"CONVERSATION:\n{transcript}\n\n"
|
|
20
|
+
"JSON:"
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
_HEADER_PROMPT = (
|
|
24
|
+
"Write a 2-to-3 sentence header describing the current situation. "
|
|
25
|
+
"Do not restate constraints or decisions (they are tracked separately). "
|
|
26
|
+
"Focus on: what is being worked on, who is involved, what phase we are in.\n\n"
|
|
27
|
+
"CONVERSATION:\n{transcript}\n\n"
|
|
28
|
+
"HEADER:"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class HybridLedgerCompactor(Compactor):
|
|
33
|
+
"""Append-only ledger compactor.
|
|
34
|
+
|
|
35
|
+
- ``locked_decisions``, ``forbidden_behaviors``, ``deferred_items``,
|
|
36
|
+
``unresolved_items`` accumulate across cycles (de-duplicated).
|
|
37
|
+
- ``immutable_facts`` and ``entity_map`` are re-extracted each cycle.
|
|
38
|
+
- ``summaryText`` is a short (2-3 sentence) situational header.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
name: ClassVar[str] = "hybrid-ledger"
|
|
42
|
+
version: ClassVar[str] = "1.0.0"
|
|
43
|
+
|
|
44
|
+
async def compact(
|
|
45
|
+
self,
|
|
46
|
+
transcript: Transcript,
|
|
47
|
+
config: dict[str, Any] | None = None,
|
|
48
|
+
previous_artifact: CompactionArtifact | None = None,
|
|
49
|
+
) -> CompactionArtifact:
|
|
50
|
+
delta_response = await self.provider.complete(
|
|
51
|
+
CompletionRequest(
|
|
52
|
+
model=self.model,
|
|
53
|
+
prompt=_DELTA_PROMPT.format(transcript=render_transcript(transcript)),
|
|
54
|
+
response_format={"type": "json_object"},
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
delta, warnings = parse_state(delta_response.text)
|
|
58
|
+
|
|
59
|
+
header_response = await self.provider.complete(
|
|
60
|
+
CompletionRequest(
|
|
61
|
+
model=self.model,
|
|
62
|
+
prompt=_HEADER_PROMPT.format(transcript=render_transcript(transcript)),
|
|
63
|
+
)
|
|
64
|
+
)
|
|
65
|
+
header = header_response.text.strip()
|
|
66
|
+
|
|
67
|
+
merged = _merge_ledger(delta, previous_artifact)
|
|
68
|
+
cycles_accumulated = _cycles_accumulated(previous_artifact)
|
|
69
|
+
|
|
70
|
+
return CompactionArtifact(
|
|
71
|
+
summaryText=header,
|
|
72
|
+
structured_state=merged,
|
|
73
|
+
selectedSourceTurnIds=[t.id for t in transcript.turns],
|
|
74
|
+
warnings=warnings,
|
|
75
|
+
methodMetadata={
|
|
76
|
+
"method": self.name,
|
|
77
|
+
"version": self.version,
|
|
78
|
+
"model": self.model,
|
|
79
|
+
"provider": self.provider.key,
|
|
80
|
+
"calls": 2,
|
|
81
|
+
"cycles_accumulated": cycles_accumulated,
|
|
82
|
+
"prompt_tokens": delta_response.prompt_tokens + header_response.prompt_tokens,
|
|
83
|
+
"completion_tokens": (
|
|
84
|
+
delta_response.completion_tokens + header_response.completion_tokens
|
|
85
|
+
),
|
|
86
|
+
},
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _merge_ledger(delta: StructuredState, previous: CompactionArtifact | None) -> StructuredState:
|
|
91
|
+
if previous is None:
|
|
92
|
+
return delta
|
|
93
|
+
prev = previous.structured_state
|
|
94
|
+
return StructuredState(
|
|
95
|
+
immutable_facts=list(delta.immutable_facts),
|
|
96
|
+
locked_decisions=uniq_preserve_order(
|
|
97
|
+
list(prev.locked_decisions) + list(delta.locked_decisions)
|
|
98
|
+
),
|
|
99
|
+
deferred_items=uniq_preserve_order(list(prev.deferred_items) + list(delta.deferred_items)),
|
|
100
|
+
forbidden_behaviors=uniq_preserve_order(
|
|
101
|
+
list(prev.forbidden_behaviors) + list(delta.forbidden_behaviors)
|
|
102
|
+
),
|
|
103
|
+
entity_map={**prev.entity_map, **delta.entity_map},
|
|
104
|
+
unresolved_items=uniq_preserve_order(
|
|
105
|
+
list(prev.unresolved_items) + list(delta.unresolved_items)
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _cycles_accumulated(previous: CompactionArtifact | None) -> int:
|
|
111
|
+
if previous is None:
|
|
112
|
+
return 1
|
|
113
|
+
prev_count_raw = previous.method_metadata.get("cycles_accumulated", 0)
|
|
114
|
+
prev_count = prev_count_raw if isinstance(prev_count_raw, int) else 0
|
|
115
|
+
return prev_count + 1
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Naive summary baseline.
|
|
2
|
+
|
|
3
|
+
Single summarization call. Produces prose only — all ``structuredState``
|
|
4
|
+
sections remain empty. Intended as a floor baseline that structured methods
|
|
5
|
+
should outperform on retention metrics.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any, ClassVar
|
|
11
|
+
|
|
12
|
+
from compactbench.compactors._utils import render_transcript
|
|
13
|
+
from compactbench.compactors.base import Compactor
|
|
14
|
+
from compactbench.contracts import CompactionArtifact, StructuredState, Transcript
|
|
15
|
+
from compactbench.providers import CompletionRequest
|
|
16
|
+
|
|
17
|
+
_PROMPT = (
|
|
18
|
+
"Summarize the following conversation in under 500 words. "
|
|
19
|
+
"Preserve all constraints, decisions, unresolved tasks, and anything the "
|
|
20
|
+
"user said must never happen.\n\n"
|
|
21
|
+
"CONVERSATION:\n{transcript}\n\n"
|
|
22
|
+
"SUMMARY:"
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class NaiveSummaryCompactor(Compactor):
|
|
27
|
+
"""Baseline: one call, prose only, no structured state."""
|
|
28
|
+
|
|
29
|
+
name: ClassVar[str] = "naive-summary"
|
|
30
|
+
version: ClassVar[str] = "1.0.0"
|
|
31
|
+
|
|
32
|
+
async def compact(
|
|
33
|
+
self,
|
|
34
|
+
transcript: Transcript,
|
|
35
|
+
config: dict[str, Any] | None = None,
|
|
36
|
+
previous_artifact: CompactionArtifact | None = None,
|
|
37
|
+
) -> CompactionArtifact:
|
|
38
|
+
prompt = _PROMPT.format(transcript=render_transcript(transcript))
|
|
39
|
+
response = await self.provider.complete(CompletionRequest(model=self.model, prompt=prompt))
|
|
40
|
+
return CompactionArtifact(
|
|
41
|
+
summaryText=response.text.strip(),
|
|
42
|
+
structured_state=StructuredState(),
|
|
43
|
+
selectedSourceTurnIds=[t.id for t in transcript.turns],
|
|
44
|
+
warnings=[],
|
|
45
|
+
methodMetadata={
|
|
46
|
+
"method": self.name,
|
|
47
|
+
"version": self.version,
|
|
48
|
+
"model": self.model,
|
|
49
|
+
"provider": self.provider.key,
|
|
50
|
+
"calls": 1,
|
|
51
|
+
"prompt_tokens": response.prompt_tokens,
|
|
52
|
+
"completion_tokens": response.completion_tokens,
|
|
53
|
+
},
|
|
54
|
+
)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Structured-state baseline: JSON-only extraction, no prose."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, ClassVar
|
|
6
|
+
|
|
7
|
+
from compactbench.compactors._state_parser import parse_state
|
|
8
|
+
from compactbench.compactors._utils import render_transcript
|
|
9
|
+
from compactbench.compactors.base import Compactor
|
|
10
|
+
from compactbench.contracts import CompactionArtifact, Transcript
|
|
11
|
+
from compactbench.providers import CompletionRequest
|
|
12
|
+
|
|
13
|
+
_PROMPT = (
|
|
14
|
+
"Extract a strict JSON object summarizing the current state of this "
|
|
15
|
+
"conversation. Return ONLY the JSON, no prose or markdown fencing.\n\n"
|
|
16
|
+
"Required fields (all must be present; use empty arrays or an empty object "
|
|
17
|
+
"when none apply):\n"
|
|
18
|
+
"- immutable_facts: array of strings — facts that must not change\n"
|
|
19
|
+
"- locked_decisions: array of strings — decisions the user committed to\n"
|
|
20
|
+
"- deferred_items: array of strings — items explicitly postponed\n"
|
|
21
|
+
"- forbidden_behaviors: array of strings — things the assistant must never do\n"
|
|
22
|
+
"- entity_map: object mapping entity names to a role label\n"
|
|
23
|
+
"- unresolved_items: array of strings — open tasks or questions\n\n"
|
|
24
|
+
"CONVERSATION:\n{transcript}\n\n"
|
|
25
|
+
"JSON:"
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class StructuredStateCompactor(Compactor):
|
|
30
|
+
"""JSON-forced structured extraction; no ``summaryText``."""
|
|
31
|
+
|
|
32
|
+
name: ClassVar[str] = "structured-state"
|
|
33
|
+
version: ClassVar[str] = "1.0.0"
|
|
34
|
+
|
|
35
|
+
async def compact(
|
|
36
|
+
self,
|
|
37
|
+
transcript: Transcript,
|
|
38
|
+
config: dict[str, Any] | None = None,
|
|
39
|
+
previous_artifact: CompactionArtifact | None = None,
|
|
40
|
+
) -> CompactionArtifact:
|
|
41
|
+
prompt = _PROMPT.format(transcript=render_transcript(transcript))
|
|
42
|
+
response = await self.provider.complete(
|
|
43
|
+
CompletionRequest(
|
|
44
|
+
model=self.model,
|
|
45
|
+
prompt=prompt,
|
|
46
|
+
response_format={"type": "json_object"},
|
|
47
|
+
)
|
|
48
|
+
)
|
|
49
|
+
state, warnings = parse_state(response.text)
|
|
50
|
+
return CompactionArtifact(
|
|
51
|
+
summaryText="",
|
|
52
|
+
structured_state=state,
|
|
53
|
+
selectedSourceTurnIds=[t.id for t in transcript.turns],
|
|
54
|
+
warnings=warnings,
|
|
55
|
+
methodMetadata={
|
|
56
|
+
"method": self.name,
|
|
57
|
+
"version": self.version,
|
|
58
|
+
"model": self.model,
|
|
59
|
+
"provider": self.provider.key,
|
|
60
|
+
"calls": 1,
|
|
61
|
+
"prompt_tokens": response.prompt_tokens,
|
|
62
|
+
"completion_tokens": response.completion_tokens,
|
|
63
|
+
},
|
|
64
|
+
)
|
compactbench/config.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Environment-driven runtime settings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pydantic import Field
|
|
6
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Settings(BaseSettings):
|
|
10
|
+
"""Global settings read from environment variables and ``.env``."""
|
|
11
|
+
|
|
12
|
+
model_config = SettingsConfigDict(
|
|
13
|
+
env_prefix="COMPACTBENCH_",
|
|
14
|
+
env_file=".env",
|
|
15
|
+
env_file_encoding="utf-8",
|
|
16
|
+
extra="ignore",
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# Providers
|
|
20
|
+
groq_api_key: str | None = Field(default=None)
|
|
21
|
+
google_ai_studio_api_key: str | None = Field(default=None)
|
|
22
|
+
ollama_base_url: str = Field(default="http://localhost:11434")
|
|
23
|
+
|
|
24
|
+
# Evaluation defaults
|
|
25
|
+
default_drift_cycles: int = Field(default=2, ge=0, le=5)
|
|
26
|
+
default_provider: str = Field(default="mock")
|
|
27
|
+
default_model: str = Field(default="mock-deterministic")
|
|
28
|
+
|
|
29
|
+
# Determinism
|
|
30
|
+
tokenizer: str = Field(default="cl100k_base")
|
|
31
|
+
|
|
32
|
+
# Paths
|
|
33
|
+
results_dir: str = Field(default="results")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
settings = Settings()
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Canonical data contracts shared across the engine, scorer, and providers."""
|
|
2
|
+
|
|
3
|
+
from compactbench.contracts.artifact import (
|
|
4
|
+
ARTIFACT_SCHEMA_VERSION,
|
|
5
|
+
CompactionArtifact,
|
|
6
|
+
StructuredState,
|
|
7
|
+
)
|
|
8
|
+
from compactbench.contracts.case import (
|
|
9
|
+
EvaluationItem,
|
|
10
|
+
EvaluationItemType,
|
|
11
|
+
GeneratedCase,
|
|
12
|
+
GroundTruth,
|
|
13
|
+
Transcript,
|
|
14
|
+
Turn,
|
|
15
|
+
TurnRole,
|
|
16
|
+
)
|
|
17
|
+
from compactbench.contracts.result import (
|
|
18
|
+
CaseResult,
|
|
19
|
+
CycleResult,
|
|
20
|
+
ItemScore,
|
|
21
|
+
RunResult,
|
|
22
|
+
Scorecard,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"ARTIFACT_SCHEMA_VERSION",
|
|
27
|
+
"CaseResult",
|
|
28
|
+
"CompactionArtifact",
|
|
29
|
+
"CycleResult",
|
|
30
|
+
"EvaluationItem",
|
|
31
|
+
"EvaluationItemType",
|
|
32
|
+
"GeneratedCase",
|
|
33
|
+
"GroundTruth",
|
|
34
|
+
"ItemScore",
|
|
35
|
+
"RunResult",
|
|
36
|
+
"Scorecard",
|
|
37
|
+
"StructuredState",
|
|
38
|
+
"Transcript",
|
|
39
|
+
"Turn",
|
|
40
|
+
"TurnRole",
|
|
41
|
+
]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Compaction artifact contract.
|
|
2
|
+
|
|
3
|
+
Mirrors the JSON Schema locked in docs/architecture/decisions.md §B2.
|
|
4
|
+
Every built-in and external compactor must return an instance of
|
|
5
|
+
``CompactionArtifact`` that validates against this shape.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Annotated, Any
|
|
11
|
+
|
|
12
|
+
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
|
|
13
|
+
|
|
14
|
+
ARTIFACT_SCHEMA_VERSION = "1.0.0"
|
|
15
|
+
|
|
16
|
+
_Bounded = Annotated[str, StringConstraints(min_length=1, max_length=500)]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class StructuredState(BaseModel):
|
|
20
|
+
"""Six required sections that every compaction artifact must carry.
|
|
21
|
+
|
|
22
|
+
Empty arrays / empty object are allowed. Missing keys are not.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
26
|
+
|
|
27
|
+
immutable_facts: list[_Bounded] = Field(default_factory=list, max_length=200)
|
|
28
|
+
locked_decisions: list[_Bounded] = Field(default_factory=list, max_length=200)
|
|
29
|
+
deferred_items: list[_Bounded] = Field(default_factory=list, max_length=200)
|
|
30
|
+
forbidden_behaviors: list[_Bounded] = Field(default_factory=list, max_length=200)
|
|
31
|
+
entity_map: dict[str, _Bounded] = Field(default_factory=dict)
|
|
32
|
+
unresolved_items: list[_Bounded] = Field(default_factory=list, max_length=200)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class CompactionArtifact(BaseModel):
|
|
36
|
+
"""Canonical artifact returned by a compactor for a single compaction step."""
|
|
37
|
+
|
|
38
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
39
|
+
|
|
40
|
+
schema_version: str = Field(default=ARTIFACT_SCHEMA_VERSION, alias="schemaVersion")
|
|
41
|
+
summary_text: Annotated[str, StringConstraints(max_length=8000)] = Field(
|
|
42
|
+
default="", alias="summaryText"
|
|
43
|
+
)
|
|
44
|
+
structured_state: StructuredState = Field(default_factory=StructuredState)
|
|
45
|
+
selected_source_turn_ids: list[int] = Field(
|
|
46
|
+
default_factory=list[int], alias="selectedSourceTurnIds"
|
|
47
|
+
)
|
|
48
|
+
warnings: list[_Bounded] = Field(default_factory=list, max_length=50)
|
|
49
|
+
method_metadata: dict[str, Any] = Field(default_factory=dict, alias="methodMetadata")
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Generated-case contract: transcript, ground truth, and evaluation items."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from enum import StrEnum
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TurnRole(StrEnum):
|
|
12
|
+
USER = "user"
|
|
13
|
+
ASSISTANT = "assistant"
|
|
14
|
+
SYSTEM = "system"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Turn(BaseModel):
|
|
18
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
19
|
+
|
|
20
|
+
id: int = Field(ge=0)
|
|
21
|
+
role: TurnRole
|
|
22
|
+
content: str
|
|
23
|
+
tags: list[str] = Field(default_factory=list)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Transcript(BaseModel):
|
|
27
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
28
|
+
|
|
29
|
+
turns: list[Turn]
|
|
30
|
+
|
|
31
|
+
def tokens_by_role(self) -> dict[TurnRole, int]:
|
|
32
|
+
return {
|
|
33
|
+
role: sum(len(t.content) for t in self.turns if t.role == role) for role in TurnRole
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class GroundTruth(BaseModel):
|
|
38
|
+
"""What the compactor must preserve for the case to score well."""
|
|
39
|
+
|
|
40
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
41
|
+
|
|
42
|
+
immutable_facts: list[str] = Field(default_factory=list)
|
|
43
|
+
locked_decisions: list[str] = Field(default_factory=list)
|
|
44
|
+
forbidden_behaviors: list[str] = Field(default_factory=list)
|
|
45
|
+
unresolved_items: list[str] = Field(default_factory=list)
|
|
46
|
+
entity_map: dict[str, str] = Field(default_factory=dict)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class EvaluationItemType(StrEnum):
|
|
50
|
+
LOCKED_DECISION_RETENTION = "locked_decision_retention"
|
|
51
|
+
FORBIDDEN_BEHAVIOR_RETENTION = "forbidden_behavior_retention"
|
|
52
|
+
IMMUTABLE_FACT_RECALL = "immutable_fact_recall"
|
|
53
|
+
UNRESOLVED_TASK_CONTINUITY = "unresolved_task_continuity"
|
|
54
|
+
ENTITY_INTEGRITY = "entity_integrity"
|
|
55
|
+
PLANNING_SOUNDNESS = "planning_soundness"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class EvaluationItem(BaseModel):
|
|
59
|
+
"""A single scorer question bound to a case."""
|
|
60
|
+
|
|
61
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
62
|
+
|
|
63
|
+
key: str
|
|
64
|
+
item_type: EvaluationItemType
|
|
65
|
+
prompt: str
|
|
66
|
+
expected: dict[str, Any]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class GeneratedCase(BaseModel):
|
|
70
|
+
"""A concrete benchmark case produced by the engine for a given seed."""
|
|
71
|
+
|
|
72
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
73
|
+
|
|
74
|
+
case_id: str
|
|
75
|
+
template_key: str
|
|
76
|
+
template_version: str
|
|
77
|
+
seed: int
|
|
78
|
+
difficulty: str
|
|
79
|
+
transcript: Transcript
|
|
80
|
+
ground_truth: GroundTruth
|
|
81
|
+
evaluation_items: list[EvaluationItem]
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Scoring and run-result contracts.
|
|
2
|
+
|
|
3
|
+
Metric shapes mirror the formulas locked in docs/architecture/decisions.md §B3.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ItemScore(BaseModel):
|
|
15
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
16
|
+
|
|
17
|
+
item_key: str
|
|
18
|
+
item_type: str
|
|
19
|
+
score: float = Field(ge=0.0, le=1.0)
|
|
20
|
+
weight: float = Field(gt=0.0)
|
|
21
|
+
check_type: str
|
|
22
|
+
details: dict[str, Any] = Field(default_factory=dict)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Scorecard(BaseModel):
|
|
26
|
+
"""Per-cycle aggregated scores."""
|
|
27
|
+
|
|
28
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
29
|
+
|
|
30
|
+
cycle_number: int = Field(ge=0)
|
|
31
|
+
cycle_score: float = Field(ge=0.0, le=1.0)
|
|
32
|
+
penalized_cycle_score: float = Field(ge=0.0, le=1.0)
|
|
33
|
+
contradiction_rate: float = Field(ge=0.0, le=1.0)
|
|
34
|
+
compression_ratio: float = Field(ge=0.0)
|
|
35
|
+
item_scores: list[ItemScore]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class CycleResult(BaseModel):
|
|
39
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
40
|
+
|
|
41
|
+
cycle_number: int = Field(ge=0)
|
|
42
|
+
scorecard: Scorecard
|
|
43
|
+
drift_delta: float | None = None
|
|
44
|
+
latency_ms: int | None = None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class CaseResult(BaseModel):
|
|
48
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
49
|
+
|
|
50
|
+
case_id: str
|
|
51
|
+
template_key: str
|
|
52
|
+
seed: int
|
|
53
|
+
cycles: list[CycleResult]
|
|
54
|
+
case_score: float = Field(ge=0.0, le=1.0)
|
|
55
|
+
drift_resistance: float = Field(ge=0.0, le=1.0)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class RunResult(BaseModel):
|
|
59
|
+
"""Top-level run artifact. Serialized one per line in results.jsonl."""
|
|
60
|
+
|
|
61
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
62
|
+
|
|
63
|
+
run_id: str
|
|
64
|
+
method_name: str
|
|
65
|
+
method_version: str
|
|
66
|
+
suite_key: str
|
|
67
|
+
suite_version: str
|
|
68
|
+
scorer_version: str
|
|
69
|
+
target_provider: str
|
|
70
|
+
target_model: str
|
|
71
|
+
started_at: datetime
|
|
72
|
+
completed_at: datetime
|
|
73
|
+
cases: list[CaseResult]
|
|
74
|
+
overall_score: float = Field(ge=0.0, le=1.0)
|
|
75
|
+
drift_resistance: float = Field(ge=0.0, le=1.0)
|
|
76
|
+
constraint_retention: float = Field(ge=0.0, le=1.0)
|
|
77
|
+
contradiction_rate: float = Field(ge=0.0, le=1.0)
|
|
78
|
+
compression_ratio: float = Field(ge=0.0)
|
|
79
|
+
notes: list[str] = Field(default_factory=list)
|