compactbench 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. compactbench/__init__.py +10 -0
  2. compactbench/__main__.py +6 -0
  3. compactbench/cli.py +343 -0
  4. compactbench/compactors/__init__.py +47 -0
  5. compactbench/compactors/_state_parser.py +118 -0
  6. compactbench/compactors/_utils.py +37 -0
  7. compactbench/compactors/base.py +38 -0
  8. compactbench/compactors/errors.py +11 -0
  9. compactbench/compactors/hierarchical_summary.py +113 -0
  10. compactbench/compactors/hybrid_ledger.py +115 -0
  11. compactbench/compactors/naive_summary.py +54 -0
  12. compactbench/compactors/structured_state.py +64 -0
  13. compactbench/config.py +36 -0
  14. compactbench/contracts/__init__.py +41 -0
  15. compactbench/contracts/artifact.py +49 -0
  16. compactbench/contracts/case.py +81 -0
  17. compactbench/contracts/result.py +79 -0
  18. compactbench/dsl/__init__.py +66 -0
  19. compactbench/dsl/errors.py +23 -0
  20. compactbench/dsl/generators.py +243 -0
  21. compactbench/dsl/models.py +107 -0
  22. compactbench/dsl/parser.py +55 -0
  23. compactbench/dsl/substitution.py +32 -0
  24. compactbench/dsl/validator.py +97 -0
  25. compactbench/engine/__init__.py +29 -0
  26. compactbench/engine/difficulty.py +54 -0
  27. compactbench/engine/distractors.py +69 -0
  28. compactbench/engine/errors.py +11 -0
  29. compactbench/engine/generation.py +96 -0
  30. compactbench/engine/seeds.py +31 -0
  31. compactbench/engine/transcript.py +84 -0
  32. compactbench/leaderboard/__init__.py +38 -0
  33. compactbench/leaderboard/errors.py +11 -0
  34. compactbench/leaderboard/projection.py +103 -0
  35. compactbench/leaderboard/qualification.py +90 -0
  36. compactbench/leaderboard/ranking.py +71 -0
  37. compactbench/providers/__init__.py +52 -0
  38. compactbench/providers/_retry.py +48 -0
  39. compactbench/providers/base.py +44 -0
  40. compactbench/providers/errors.py +15 -0
  41. compactbench/providers/google_ai_studio.py +112 -0
  42. compactbench/providers/groq.py +95 -0
  43. compactbench/providers/mock.py +53 -0
  44. compactbench/providers/ollama.py +100 -0
  45. compactbench/py.typed +0 -0
  46. compactbench/runner/__init__.py +53 -0
  47. compactbench/runner/_resolver.py +57 -0
  48. compactbench/runner/continuation.py +78 -0
  49. compactbench/runner/cycle.py +67 -0
  50. compactbench/runner/errors.py +15 -0
  51. compactbench/runner/evaluation.py +71 -0
  52. compactbench/runner/persistence.py +227 -0
  53. compactbench/runner/run.py +257 -0
  54. compactbench/scoring/__init__.py +48 -0
  55. compactbench/scoring/checks.py +92 -0
  56. compactbench/scoring/compression.py +66 -0
  57. compactbench/scoring/contradictions.py +70 -0
  58. compactbench/scoring/drift.py +30 -0
  59. compactbench/scoring/errors.py +7 -0
  60. compactbench/scoring/scorer.py +77 -0
  61. compactbench/utils/__init__.py +1 -0
  62. compactbench-0.1.0.dist-info/METADATA +161 -0
  63. compactbench-0.1.0.dist-info/RECORD +66 -0
  64. compactbench-0.1.0.dist-info/WHEEL +4 -0
  65. compactbench-0.1.0.dist-info/entry_points.txt +2 -0
  66. compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,113 @@
1
+ """Hierarchical summary: chunk -> per-chunk summary -> meta-summary -> structured state."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, ClassVar
6
+
7
+ from compactbench.compactors._state_parser import parse_state
8
+ from compactbench.compactors._utils import chunk, render_transcript, render_turns
9
+ from compactbench.compactors.base import Compactor
10
+ from compactbench.contracts import CompactionArtifact, Transcript
11
+ from compactbench.providers import CompletionRequest
12
+
13
+ _CHUNK_PROMPT = (
14
+ "Summarize these {n} conversation turns in under 200 words. Preserve "
15
+ "constraints, decisions, and unresolved items.\n\n"
16
+ "TURNS:\n{turns}\n\n"
17
+ "SUMMARY:"
18
+ )
19
+
20
+ _META_PROMPT = (
21
+ "Combine these partial summaries into one coherent summary in under 500 "
22
+ "words. Preserve every constraint, decision, and unresolved item.\n\n"
23
+ "{summaries}\n\n"
24
+ "FINAL SUMMARY:"
25
+ )
26
+
27
+ _STATE_PROMPT = (
28
+ "Extract a strict JSON object summarizing the state of this conversation. "
29
+ "Return ONLY the JSON, no prose.\n\n"
30
+ "Required fields: immutable_facts, locked_decisions, deferred_items, "
31
+ "forbidden_behaviors, entity_map, unresolved_items.\n\n"
32
+ "CONVERSATION:\n{transcript}\n\n"
33
+ "JSON:"
34
+ )
35
+
36
+
37
+ class HierarchicalSummaryCompactor(Compactor):
38
+ """Two-level summarization plus a separate structured-state extraction call."""
39
+
40
+ name: ClassVar[str] = "hierarchical-summary"
41
+ version: ClassVar[str] = "1.0.0"
42
+
43
+ _CHUNK_SIZE: ClassVar[int] = 10
44
+
45
+ async def compact(
46
+ self,
47
+ transcript: Transcript,
48
+ config: dict[str, Any] | None = None,
49
+ previous_artifact: CompactionArtifact | None = None,
50
+ ) -> CompactionArtifact:
51
+ chunks = chunk(list(transcript.turns), self._CHUNK_SIZE)
52
+
53
+ chunk_summaries: list[str] = []
54
+ total_prompt_tokens = 0
55
+ total_completion_tokens = 0
56
+
57
+ for turns in chunks:
58
+ response = await self.provider.complete(
59
+ CompletionRequest(
60
+ model=self.model,
61
+ prompt=_CHUNK_PROMPT.format(n=len(turns), turns=render_turns(turns)),
62
+ )
63
+ )
64
+ chunk_summaries.append(response.text.strip())
65
+ total_prompt_tokens += response.prompt_tokens
66
+ total_completion_tokens += response.completion_tokens
67
+
68
+ if len(chunk_summaries) == 1:
69
+ final_summary = chunk_summaries[0]
70
+ else:
71
+ meta_response = await self.provider.complete(
72
+ CompletionRequest(
73
+ model=self.model,
74
+ prompt=_META_PROMPT.format(
75
+ summaries="\n\n".join(
76
+ f"PARTIAL {i + 1}: {s}" for i, s in enumerate(chunk_summaries)
77
+ )
78
+ ),
79
+ )
80
+ )
81
+ final_summary = meta_response.text.strip()
82
+ total_prompt_tokens += meta_response.prompt_tokens
83
+ total_completion_tokens += meta_response.completion_tokens
84
+
85
+ state_response = await self.provider.complete(
86
+ CompletionRequest(
87
+ model=self.model,
88
+ prompt=_STATE_PROMPT.format(transcript=render_transcript(transcript)),
89
+ response_format={"type": "json_object"},
90
+ )
91
+ )
92
+ state, warnings = parse_state(state_response.text)
93
+ total_prompt_tokens += state_response.prompt_tokens
94
+ total_completion_tokens += state_response.completion_tokens
95
+
96
+ total_calls = len(chunk_summaries) + (1 if len(chunk_summaries) > 1 else 0) + 1
97
+
98
+ return CompactionArtifact(
99
+ summaryText=final_summary,
100
+ structured_state=state,
101
+ selectedSourceTurnIds=[t.id for t in transcript.turns],
102
+ warnings=warnings,
103
+ methodMetadata={
104
+ "method": self.name,
105
+ "version": self.version,
106
+ "model": self.model,
107
+ "provider": self.provider.key,
108
+ "chunks": len(chunks),
109
+ "calls": total_calls,
110
+ "prompt_tokens": total_prompt_tokens,
111
+ "completion_tokens": total_completion_tokens,
112
+ },
113
+ )
@@ -0,0 +1,115 @@
1
+ """Hybrid ledger: append-only structured state across cycles + short prose header."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, ClassVar
6
+
7
+ from compactbench.compactors._state_parser import parse_state
8
+ from compactbench.compactors._utils import render_transcript, uniq_preserve_order
9
+ from compactbench.compactors.base import Compactor
10
+ from compactbench.contracts import CompactionArtifact, StructuredState, Transcript
11
+ from compactbench.providers import CompletionRequest
12
+
13
+ _DELTA_PROMPT = (
14
+ "Extract NEW state that appears in this conversation segment as a strict "
15
+ "JSON object. Include only items stated or decided in these turns; do not "
16
+ "infer beyond the text. Return ONLY the JSON, no prose.\n\n"
17
+ "Required fields: immutable_facts, locked_decisions, deferred_items, "
18
+ "forbidden_behaviors, entity_map, unresolved_items.\n\n"
19
+ "CONVERSATION:\n{transcript}\n\n"
20
+ "JSON:"
21
+ )
22
+
23
+ _HEADER_PROMPT = (
24
+ "Write a 2-to-3 sentence header describing the current situation. "
25
+ "Do not restate constraints or decisions (they are tracked separately). "
26
+ "Focus on: what is being worked on, who is involved, what phase we are in.\n\n"
27
+ "CONVERSATION:\n{transcript}\n\n"
28
+ "HEADER:"
29
+ )
30
+
31
+
32
+ class HybridLedgerCompactor(Compactor):
33
+ """Append-only ledger compactor.
34
+
35
+ - ``locked_decisions``, ``forbidden_behaviors``, ``deferred_items``,
36
+ ``unresolved_items`` accumulate across cycles (de-duplicated).
37
+ - ``immutable_facts`` and ``entity_map`` are re-extracted each cycle.
38
+ - ``summaryText`` is a short (2-3 sentence) situational header.
39
+ """
40
+
41
+ name: ClassVar[str] = "hybrid-ledger"
42
+ version: ClassVar[str] = "1.0.0"
43
+
44
+ async def compact(
45
+ self,
46
+ transcript: Transcript,
47
+ config: dict[str, Any] | None = None,
48
+ previous_artifact: CompactionArtifact | None = None,
49
+ ) -> CompactionArtifact:
50
+ delta_response = await self.provider.complete(
51
+ CompletionRequest(
52
+ model=self.model,
53
+ prompt=_DELTA_PROMPT.format(transcript=render_transcript(transcript)),
54
+ response_format={"type": "json_object"},
55
+ )
56
+ )
57
+ delta, warnings = parse_state(delta_response.text)
58
+
59
+ header_response = await self.provider.complete(
60
+ CompletionRequest(
61
+ model=self.model,
62
+ prompt=_HEADER_PROMPT.format(transcript=render_transcript(transcript)),
63
+ )
64
+ )
65
+ header = header_response.text.strip()
66
+
67
+ merged = _merge_ledger(delta, previous_artifact)
68
+ cycles_accumulated = _cycles_accumulated(previous_artifact)
69
+
70
+ return CompactionArtifact(
71
+ summaryText=header,
72
+ structured_state=merged,
73
+ selectedSourceTurnIds=[t.id for t in transcript.turns],
74
+ warnings=warnings,
75
+ methodMetadata={
76
+ "method": self.name,
77
+ "version": self.version,
78
+ "model": self.model,
79
+ "provider": self.provider.key,
80
+ "calls": 2,
81
+ "cycles_accumulated": cycles_accumulated,
82
+ "prompt_tokens": delta_response.prompt_tokens + header_response.prompt_tokens,
83
+ "completion_tokens": (
84
+ delta_response.completion_tokens + header_response.completion_tokens
85
+ ),
86
+ },
87
+ )
88
+
89
+
90
+ def _merge_ledger(delta: StructuredState, previous: CompactionArtifact | None) -> StructuredState:
91
+ if previous is None:
92
+ return delta
93
+ prev = previous.structured_state
94
+ return StructuredState(
95
+ immutable_facts=list(delta.immutable_facts),
96
+ locked_decisions=uniq_preserve_order(
97
+ list(prev.locked_decisions) + list(delta.locked_decisions)
98
+ ),
99
+ deferred_items=uniq_preserve_order(list(prev.deferred_items) + list(delta.deferred_items)),
100
+ forbidden_behaviors=uniq_preserve_order(
101
+ list(prev.forbidden_behaviors) + list(delta.forbidden_behaviors)
102
+ ),
103
+ entity_map={**prev.entity_map, **delta.entity_map},
104
+ unresolved_items=uniq_preserve_order(
105
+ list(prev.unresolved_items) + list(delta.unresolved_items)
106
+ ),
107
+ )
108
+
109
+
110
+ def _cycles_accumulated(previous: CompactionArtifact | None) -> int:
111
+ if previous is None:
112
+ return 1
113
+ prev_count_raw = previous.method_metadata.get("cycles_accumulated", 0)
114
+ prev_count = prev_count_raw if isinstance(prev_count_raw, int) else 0
115
+ return prev_count + 1
@@ -0,0 +1,54 @@
1
+ """Naive summary baseline.
2
+
3
+ Single summarization call. Produces prose only — all ``structuredState``
4
+ sections remain empty. Intended as a floor baseline that structured methods
5
+ should outperform on retention metrics.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any, ClassVar
11
+
12
+ from compactbench.compactors._utils import render_transcript
13
+ from compactbench.compactors.base import Compactor
14
+ from compactbench.contracts import CompactionArtifact, StructuredState, Transcript
15
+ from compactbench.providers import CompletionRequest
16
+
17
+ _PROMPT = (
18
+ "Summarize the following conversation in under 500 words. "
19
+ "Preserve all constraints, decisions, unresolved tasks, and anything the "
20
+ "user said must never happen.\n\n"
21
+ "CONVERSATION:\n{transcript}\n\n"
22
+ "SUMMARY:"
23
+ )
24
+
25
+
26
+ class NaiveSummaryCompactor(Compactor):
27
+ """Baseline: one call, prose only, no structured state."""
28
+
29
+ name: ClassVar[str] = "naive-summary"
30
+ version: ClassVar[str] = "1.0.0"
31
+
32
+ async def compact(
33
+ self,
34
+ transcript: Transcript,
35
+ config: dict[str, Any] | None = None,
36
+ previous_artifact: CompactionArtifact | None = None,
37
+ ) -> CompactionArtifact:
38
+ prompt = _PROMPT.format(transcript=render_transcript(transcript))
39
+ response = await self.provider.complete(CompletionRequest(model=self.model, prompt=prompt))
40
+ return CompactionArtifact(
41
+ summaryText=response.text.strip(),
42
+ structured_state=StructuredState(),
43
+ selectedSourceTurnIds=[t.id for t in transcript.turns],
44
+ warnings=[],
45
+ methodMetadata={
46
+ "method": self.name,
47
+ "version": self.version,
48
+ "model": self.model,
49
+ "provider": self.provider.key,
50
+ "calls": 1,
51
+ "prompt_tokens": response.prompt_tokens,
52
+ "completion_tokens": response.completion_tokens,
53
+ },
54
+ )
@@ -0,0 +1,64 @@
1
+ """Structured-state baseline: JSON-only extraction, no prose."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, ClassVar
6
+
7
+ from compactbench.compactors._state_parser import parse_state
8
+ from compactbench.compactors._utils import render_transcript
9
+ from compactbench.compactors.base import Compactor
10
+ from compactbench.contracts import CompactionArtifact, Transcript
11
+ from compactbench.providers import CompletionRequest
12
+
13
+ _PROMPT = (
14
+ "Extract a strict JSON object summarizing the current state of this "
15
+ "conversation. Return ONLY the JSON, no prose or markdown fencing.\n\n"
16
+ "Required fields (all must be present; use empty arrays or an empty object "
17
+ "when none apply):\n"
18
+ "- immutable_facts: array of strings — facts that must not change\n"
19
+ "- locked_decisions: array of strings — decisions the user committed to\n"
20
+ "- deferred_items: array of strings — items explicitly postponed\n"
21
+ "- forbidden_behaviors: array of strings — things the assistant must never do\n"
22
+ "- entity_map: object mapping entity names to a role label\n"
23
+ "- unresolved_items: array of strings — open tasks or questions\n\n"
24
+ "CONVERSATION:\n{transcript}\n\n"
25
+ "JSON:"
26
+ )
27
+
28
+
29
+ class StructuredStateCompactor(Compactor):
30
+ """JSON-forced structured extraction; no ``summaryText``."""
31
+
32
+ name: ClassVar[str] = "structured-state"
33
+ version: ClassVar[str] = "1.0.0"
34
+
35
+ async def compact(
36
+ self,
37
+ transcript: Transcript,
38
+ config: dict[str, Any] | None = None,
39
+ previous_artifact: CompactionArtifact | None = None,
40
+ ) -> CompactionArtifact:
41
+ prompt = _PROMPT.format(transcript=render_transcript(transcript))
42
+ response = await self.provider.complete(
43
+ CompletionRequest(
44
+ model=self.model,
45
+ prompt=prompt,
46
+ response_format={"type": "json_object"},
47
+ )
48
+ )
49
+ state, warnings = parse_state(response.text)
50
+ return CompactionArtifact(
51
+ summaryText="",
52
+ structured_state=state,
53
+ selectedSourceTurnIds=[t.id for t in transcript.turns],
54
+ warnings=warnings,
55
+ methodMetadata={
56
+ "method": self.name,
57
+ "version": self.version,
58
+ "model": self.model,
59
+ "provider": self.provider.key,
60
+ "calls": 1,
61
+ "prompt_tokens": response.prompt_tokens,
62
+ "completion_tokens": response.completion_tokens,
63
+ },
64
+ )
compactbench/config.py ADDED
@@ -0,0 +1,36 @@
1
+ """Environment-driven runtime settings."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pydantic import Field
6
+ from pydantic_settings import BaseSettings, SettingsConfigDict
7
+
8
+
9
+ class Settings(BaseSettings):
10
+ """Global settings read from environment variables and ``.env``."""
11
+
12
+ model_config = SettingsConfigDict(
13
+ env_prefix="COMPACTBENCH_",
14
+ env_file=".env",
15
+ env_file_encoding="utf-8",
16
+ extra="ignore",
17
+ )
18
+
19
+ # Providers
20
+ groq_api_key: str | None = Field(default=None)
21
+ google_ai_studio_api_key: str | None = Field(default=None)
22
+ ollama_base_url: str = Field(default="http://localhost:11434")
23
+
24
+ # Evaluation defaults
25
+ default_drift_cycles: int = Field(default=2, ge=0, le=5)
26
+ default_provider: str = Field(default="mock")
27
+ default_model: str = Field(default="mock-deterministic")
28
+
29
+ # Determinism
30
+ tokenizer: str = Field(default="cl100k_base")
31
+
32
+ # Paths
33
+ results_dir: str = Field(default="results")
34
+
35
+
36
+ settings = Settings()
@@ -0,0 +1,41 @@
1
+ """Canonical data contracts shared across the engine, scorer, and providers."""
2
+
3
+ from compactbench.contracts.artifact import (
4
+ ARTIFACT_SCHEMA_VERSION,
5
+ CompactionArtifact,
6
+ StructuredState,
7
+ )
8
+ from compactbench.contracts.case import (
9
+ EvaluationItem,
10
+ EvaluationItemType,
11
+ GeneratedCase,
12
+ GroundTruth,
13
+ Transcript,
14
+ Turn,
15
+ TurnRole,
16
+ )
17
+ from compactbench.contracts.result import (
18
+ CaseResult,
19
+ CycleResult,
20
+ ItemScore,
21
+ RunResult,
22
+ Scorecard,
23
+ )
24
+
25
+ __all__ = [
26
+ "ARTIFACT_SCHEMA_VERSION",
27
+ "CaseResult",
28
+ "CompactionArtifact",
29
+ "CycleResult",
30
+ "EvaluationItem",
31
+ "EvaluationItemType",
32
+ "GeneratedCase",
33
+ "GroundTruth",
34
+ "ItemScore",
35
+ "RunResult",
36
+ "Scorecard",
37
+ "StructuredState",
38
+ "Transcript",
39
+ "Turn",
40
+ "TurnRole",
41
+ ]
@@ -0,0 +1,49 @@
1
+ """Compaction artifact contract.
2
+
3
+ Mirrors the JSON Schema locked in docs/architecture/decisions.md §B2.
4
+ Every built-in and external compactor must return an instance of
5
+ ``CompactionArtifact`` that validates against this shape.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Annotated, Any
11
+
12
+ from pydantic import BaseModel, ConfigDict, Field, StringConstraints
13
+
14
+ ARTIFACT_SCHEMA_VERSION = "1.0.0"
15
+
16
+ _Bounded = Annotated[str, StringConstraints(min_length=1, max_length=500)]
17
+
18
+
19
+ class StructuredState(BaseModel):
20
+ """Six required sections that every compaction artifact must carry.
21
+
22
+ Empty arrays / empty object are allowed. Missing keys are not.
23
+ """
24
+
25
+ model_config = ConfigDict(extra="forbid", frozen=True)
26
+
27
+ immutable_facts: list[_Bounded] = Field(default_factory=list, max_length=200)
28
+ locked_decisions: list[_Bounded] = Field(default_factory=list, max_length=200)
29
+ deferred_items: list[_Bounded] = Field(default_factory=list, max_length=200)
30
+ forbidden_behaviors: list[_Bounded] = Field(default_factory=list, max_length=200)
31
+ entity_map: dict[str, _Bounded] = Field(default_factory=dict)
32
+ unresolved_items: list[_Bounded] = Field(default_factory=list, max_length=200)
33
+
34
+
35
+ class CompactionArtifact(BaseModel):
36
+ """Canonical artifact returned by a compactor for a single compaction step."""
37
+
38
+ model_config = ConfigDict(extra="forbid", frozen=True)
39
+
40
+ schema_version: str = Field(default=ARTIFACT_SCHEMA_VERSION, alias="schemaVersion")
41
+ summary_text: Annotated[str, StringConstraints(max_length=8000)] = Field(
42
+ default="", alias="summaryText"
43
+ )
44
+ structured_state: StructuredState = Field(default_factory=StructuredState)
45
+ selected_source_turn_ids: list[int] = Field(
46
+ default_factory=list[int], alias="selectedSourceTurnIds"
47
+ )
48
+ warnings: list[_Bounded] = Field(default_factory=list, max_length=50)
49
+ method_metadata: dict[str, Any] = Field(default_factory=dict, alias="methodMetadata")
@@ -0,0 +1,81 @@
1
+ """Generated-case contract: transcript, ground truth, and evaluation items."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from enum import StrEnum
6
+ from typing import Any
7
+
8
+ from pydantic import BaseModel, ConfigDict, Field
9
+
10
+
11
+ class TurnRole(StrEnum):
12
+ USER = "user"
13
+ ASSISTANT = "assistant"
14
+ SYSTEM = "system"
15
+
16
+
17
+ class Turn(BaseModel):
18
+ model_config = ConfigDict(extra="forbid", frozen=True)
19
+
20
+ id: int = Field(ge=0)
21
+ role: TurnRole
22
+ content: str
23
+ tags: list[str] = Field(default_factory=list)
24
+
25
+
26
+ class Transcript(BaseModel):
27
+ model_config = ConfigDict(extra="forbid", frozen=True)
28
+
29
+ turns: list[Turn]
30
+
31
+ def tokens_by_role(self) -> dict[TurnRole, int]:
32
+ return {
33
+ role: sum(len(t.content) for t in self.turns if t.role == role) for role in TurnRole
34
+ }
35
+
36
+
37
+ class GroundTruth(BaseModel):
38
+ """What the compactor must preserve for the case to score well."""
39
+
40
+ model_config = ConfigDict(extra="forbid", frozen=True)
41
+
42
+ immutable_facts: list[str] = Field(default_factory=list)
43
+ locked_decisions: list[str] = Field(default_factory=list)
44
+ forbidden_behaviors: list[str] = Field(default_factory=list)
45
+ unresolved_items: list[str] = Field(default_factory=list)
46
+ entity_map: dict[str, str] = Field(default_factory=dict)
47
+
48
+
49
+ class EvaluationItemType(StrEnum):
50
+ LOCKED_DECISION_RETENTION = "locked_decision_retention"
51
+ FORBIDDEN_BEHAVIOR_RETENTION = "forbidden_behavior_retention"
52
+ IMMUTABLE_FACT_RECALL = "immutable_fact_recall"
53
+ UNRESOLVED_TASK_CONTINUITY = "unresolved_task_continuity"
54
+ ENTITY_INTEGRITY = "entity_integrity"
55
+ PLANNING_SOUNDNESS = "planning_soundness"
56
+
57
+
58
+ class EvaluationItem(BaseModel):
59
+ """A single scorer question bound to a case."""
60
+
61
+ model_config = ConfigDict(extra="forbid", frozen=True)
62
+
63
+ key: str
64
+ item_type: EvaluationItemType
65
+ prompt: str
66
+ expected: dict[str, Any]
67
+
68
+
69
+ class GeneratedCase(BaseModel):
70
+ """A concrete benchmark case produced by the engine for a given seed."""
71
+
72
+ model_config = ConfigDict(extra="forbid", frozen=True)
73
+
74
+ case_id: str
75
+ template_key: str
76
+ template_version: str
77
+ seed: int
78
+ difficulty: str
79
+ transcript: Transcript
80
+ ground_truth: GroundTruth
81
+ evaluation_items: list[EvaluationItem]
@@ -0,0 +1,79 @@
1
+ """Scoring and run-result contracts.
2
+
3
+ Metric shapes mirror the formulas locked in docs/architecture/decisions.md §B3.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from datetime import datetime
9
+ from typing import Any
10
+
11
+ from pydantic import BaseModel, ConfigDict, Field
12
+
13
+
14
+ class ItemScore(BaseModel):
15
+ model_config = ConfigDict(extra="forbid", frozen=True)
16
+
17
+ item_key: str
18
+ item_type: str
19
+ score: float = Field(ge=0.0, le=1.0)
20
+ weight: float = Field(gt=0.0)
21
+ check_type: str
22
+ details: dict[str, Any] = Field(default_factory=dict)
23
+
24
+
25
+ class Scorecard(BaseModel):
26
+ """Per-cycle aggregated scores."""
27
+
28
+ model_config = ConfigDict(extra="forbid", frozen=True)
29
+
30
+ cycle_number: int = Field(ge=0)
31
+ cycle_score: float = Field(ge=0.0, le=1.0)
32
+ penalized_cycle_score: float = Field(ge=0.0, le=1.0)
33
+ contradiction_rate: float = Field(ge=0.0, le=1.0)
34
+ compression_ratio: float = Field(ge=0.0)
35
+ item_scores: list[ItemScore]
36
+
37
+
38
+ class CycleResult(BaseModel):
39
+ model_config = ConfigDict(extra="forbid", frozen=True)
40
+
41
+ cycle_number: int = Field(ge=0)
42
+ scorecard: Scorecard
43
+ drift_delta: float | None = None
44
+ latency_ms: int | None = None
45
+
46
+
47
+ class CaseResult(BaseModel):
48
+ model_config = ConfigDict(extra="forbid", frozen=True)
49
+
50
+ case_id: str
51
+ template_key: str
52
+ seed: int
53
+ cycles: list[CycleResult]
54
+ case_score: float = Field(ge=0.0, le=1.0)
55
+ drift_resistance: float = Field(ge=0.0, le=1.0)
56
+
57
+
58
+ class RunResult(BaseModel):
59
+ """Top-level run artifact. Serialized one per line in results.jsonl."""
60
+
61
+ model_config = ConfigDict(extra="forbid", frozen=True)
62
+
63
+ run_id: str
64
+ method_name: str
65
+ method_version: str
66
+ suite_key: str
67
+ suite_version: str
68
+ scorer_version: str
69
+ target_provider: str
70
+ target_model: str
71
+ started_at: datetime
72
+ completed_at: datetime
73
+ cases: list[CaseResult]
74
+ overall_score: float = Field(ge=0.0, le=1.0)
75
+ drift_resistance: float = Field(ge=0.0, le=1.0)
76
+ constraint_retention: float = Field(ge=0.0, le=1.0)
77
+ contradiction_rate: float = Field(ge=0.0, le=1.0)
78
+ compression_ratio: float = Field(ge=0.0)
79
+ notes: list[str] = Field(default_factory=list)