compactbench 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. compactbench/__init__.py +10 -0
  2. compactbench/__main__.py +6 -0
  3. compactbench/cli.py +343 -0
  4. compactbench/compactors/__init__.py +47 -0
  5. compactbench/compactors/_state_parser.py +118 -0
  6. compactbench/compactors/_utils.py +37 -0
  7. compactbench/compactors/base.py +38 -0
  8. compactbench/compactors/errors.py +11 -0
  9. compactbench/compactors/hierarchical_summary.py +113 -0
  10. compactbench/compactors/hybrid_ledger.py +115 -0
  11. compactbench/compactors/naive_summary.py +54 -0
  12. compactbench/compactors/structured_state.py +64 -0
  13. compactbench/config.py +36 -0
  14. compactbench/contracts/__init__.py +41 -0
  15. compactbench/contracts/artifact.py +49 -0
  16. compactbench/contracts/case.py +81 -0
  17. compactbench/contracts/result.py +79 -0
  18. compactbench/dsl/__init__.py +66 -0
  19. compactbench/dsl/errors.py +23 -0
  20. compactbench/dsl/generators.py +243 -0
  21. compactbench/dsl/models.py +107 -0
  22. compactbench/dsl/parser.py +55 -0
  23. compactbench/dsl/substitution.py +32 -0
  24. compactbench/dsl/validator.py +97 -0
  25. compactbench/engine/__init__.py +29 -0
  26. compactbench/engine/difficulty.py +54 -0
  27. compactbench/engine/distractors.py +69 -0
  28. compactbench/engine/errors.py +11 -0
  29. compactbench/engine/generation.py +96 -0
  30. compactbench/engine/seeds.py +31 -0
  31. compactbench/engine/transcript.py +84 -0
  32. compactbench/leaderboard/__init__.py +38 -0
  33. compactbench/leaderboard/errors.py +11 -0
  34. compactbench/leaderboard/projection.py +103 -0
  35. compactbench/leaderboard/qualification.py +90 -0
  36. compactbench/leaderboard/ranking.py +71 -0
  37. compactbench/providers/__init__.py +52 -0
  38. compactbench/providers/_retry.py +48 -0
  39. compactbench/providers/base.py +44 -0
  40. compactbench/providers/errors.py +15 -0
  41. compactbench/providers/google_ai_studio.py +112 -0
  42. compactbench/providers/groq.py +95 -0
  43. compactbench/providers/mock.py +53 -0
  44. compactbench/providers/ollama.py +100 -0
  45. compactbench/py.typed +0 -0
  46. compactbench/runner/__init__.py +53 -0
  47. compactbench/runner/_resolver.py +57 -0
  48. compactbench/runner/continuation.py +78 -0
  49. compactbench/runner/cycle.py +67 -0
  50. compactbench/runner/errors.py +15 -0
  51. compactbench/runner/evaluation.py +71 -0
  52. compactbench/runner/persistence.py +227 -0
  53. compactbench/runner/run.py +257 -0
  54. compactbench/scoring/__init__.py +48 -0
  55. compactbench/scoring/checks.py +92 -0
  56. compactbench/scoring/compression.py +66 -0
  57. compactbench/scoring/contradictions.py +70 -0
  58. compactbench/scoring/drift.py +30 -0
  59. compactbench/scoring/errors.py +7 -0
  60. compactbench/scoring/scorer.py +77 -0
  61. compactbench/utils/__init__.py +1 -0
  62. compactbench-0.1.0.dist-info/METADATA +161 -0
  63. compactbench-0.1.0.dist-info/RECORD +66 -0
  64. compactbench-0.1.0.dist-info/WHEEL +4 -0
  65. compactbench-0.1.0.dist-info/entry_points.txt +2 -0
  66. compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,66 @@
1
+ """Template DSL: parser, seeded generators, validator, substitution.
2
+
3
+ See docs/architecture/decisions.md §B1 for the DSL shape.
4
+ Implemented in WO-002.
5
+ """
6
+
7
+ from compactbench.dsl.errors import (
8
+ TemplateError,
9
+ TemplateParseError,
10
+ TemplateValidationError,
11
+ UnknownGeneratorError,
12
+ UnresolvedReferenceError,
13
+ )
14
+ from compactbench.dsl.generators import (
15
+ Generator,
16
+ derive_seed,
17
+ get_generator,
18
+ registered_generators,
19
+ resolve_variables,
20
+ )
21
+ from compactbench.dsl.models import (
22
+ DifficultyLevel,
23
+ DifficultyPolicy,
24
+ EvaluationItemTemplate,
25
+ GroundTruthTemplate,
26
+ TemplateDefinition,
27
+ TemplateTurnRole,
28
+ TranscriptTemplate,
29
+ TurnTemplate,
30
+ VariableDeclaration,
31
+ )
32
+ from compactbench.dsl.parser import (
33
+ load_suite,
34
+ parse_template_file,
35
+ parse_template_string,
36
+ )
37
+ from compactbench.dsl.substitution import extract_references, substitute
38
+ from compactbench.dsl.validator import validate_template
39
+
40
+ __all__ = [
41
+ "DifficultyLevel",
42
+ "DifficultyPolicy",
43
+ "EvaluationItemTemplate",
44
+ "Generator",
45
+ "GroundTruthTemplate",
46
+ "TemplateDefinition",
47
+ "TemplateError",
48
+ "TemplateParseError",
49
+ "TemplateTurnRole",
50
+ "TemplateValidationError",
51
+ "TranscriptTemplate",
52
+ "TurnTemplate",
53
+ "UnknownGeneratorError",
54
+ "UnresolvedReferenceError",
55
+ "VariableDeclaration",
56
+ "derive_seed",
57
+ "extract_references",
58
+ "get_generator",
59
+ "load_suite",
60
+ "parse_template_file",
61
+ "parse_template_string",
62
+ "registered_generators",
63
+ "resolve_variables",
64
+ "substitute",
65
+ "validate_template",
66
+ ]
@@ -0,0 +1,23 @@
1
+ """Exception hierarchy for template DSL operations."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class TemplateError(Exception):
7
+ """Base exception for anything DSL-related."""
8
+
9
+
10
+ class TemplateParseError(TemplateError):
11
+ """YAML structure does not match the template DSL shape."""
12
+
13
+
14
+ class TemplateValidationError(TemplateError):
15
+ """Template is structurally valid but has a semantic error."""
16
+
17
+
18
+ class UnknownGeneratorError(TemplateError):
19
+ """A variable references a generator that is not registered."""
20
+
21
+
22
+ class UnresolvedReferenceError(TemplateError):
23
+ """A ``{{reference}}`` in the template cannot be resolved from the bindings."""
@@ -0,0 +1,243 @@
1
+ """Seeded random generators for template variables.
2
+
3
+ Each generator is a deterministic function of a seed. Sub-seeds are derived
4
+ from ``(base_seed, variable_name)`` via SHA-256 so adding or reordering
5
+ variables in a template never changes the output of unchanged variables.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import random
12
+ from dataclasses import dataclass, field
13
+ from typing import Protocol, runtime_checkable
14
+
15
+ from compactbench.dsl.errors import UnknownGeneratorError
16
+ from compactbench.dsl.models import VariableDeclaration
17
+
18
+
19
+ def derive_seed(base_seed: int, variable_name: str) -> int:
20
+ """Derive a deterministic sub-seed for a named variable.
21
+
22
+ Uses SHA-256 of ``"{base}:{name}"``. Stable across Python versions.
23
+ """
24
+ payload = f"{base_seed}:{variable_name}".encode()
25
+ digest = hashlib.sha256(payload).digest()
26
+ return int.from_bytes(digest[:8], "big", signed=False)
27
+
28
+
29
+ @runtime_checkable
30
+ class Generator(Protocol):
31
+ """Protocol every generator must satisfy."""
32
+
33
+ name: str
34
+
35
+ def generate(self, seed: int) -> str: ...
36
+
37
+
38
+ @dataclass
39
+ class LexiconGenerator:
40
+ """Picks one item from a bounded lexicon, deterministically by seed."""
41
+
42
+ name: str
43
+ lexicon: tuple[str, ...]
44
+
45
+ def generate(self, seed: int) -> str:
46
+ rng = random.Random(seed)
47
+ return rng.choice(self.lexicon)
48
+
49
+
50
+ @dataclass
51
+ class DateGenerator:
52
+ """Generates an ISO-8601 date within a bounded year range."""
53
+
54
+ name: str = "date_iso"
55
+ start_year: int = 2024
56
+ end_year: int = 2026
57
+
58
+ def generate(self, seed: int) -> str:
59
+ rng = random.Random(seed)
60
+ year = rng.randint(self.start_year, self.end_year)
61
+ month = rng.randint(1, 12)
62
+ day = rng.randint(1, 28)
63
+ return f"{year:04d}-{month:02d}-{day:02d}"
64
+
65
+
66
+ @dataclass
67
+ class AmountGenerator:
68
+ """Generates a USD amount string like ``$47,250``."""
69
+
70
+ name: str = "amount_usd"
71
+ magnitudes: tuple[int, ...] = field(default=(2, 3, 4, 5))
72
+
73
+ def generate(self, seed: int) -> str:
74
+ rng = random.Random(seed)
75
+ magnitude = rng.choice(self.magnitudes)
76
+ low = 10 ** (magnitude - 1)
77
+ high = 10**magnitude - 1
78
+ amount = rng.randint(low, high)
79
+ return f"${amount:,}"
80
+
81
+
82
+ @dataclass
83
+ class SkuGenerator:
84
+ """Generates an alphanumeric SKU like ``SKU-AB9421``."""
85
+
86
+ name: str = "product_sku"
87
+
88
+ def generate(self, seed: int) -> str:
89
+ rng = random.Random(seed)
90
+ prefix = rng.choice(("SKU", "PRD", "ITEM", "ART"))
91
+ alpha = "".join(rng.choices("ABCDEFGHJKLMNPQRTUVWXYZ", k=2))
92
+ num = rng.randint(100, 9999)
93
+ return f"{prefix}-{alpha}{num}"
94
+
95
+
96
+ # --- lexicons ------------------------------------------------------------
97
+
98
+ _FIRST_NAMES: tuple[str, ...] = (
99
+ "Alice",
100
+ "Bob",
101
+ "Carol",
102
+ "David",
103
+ "Elena",
104
+ "Farid",
105
+ "Grace",
106
+ "Hassan",
107
+ "Iris",
108
+ "Julia",
109
+ "Kenji",
110
+ "Leo",
111
+ "Maya",
112
+ "Noor",
113
+ "Omar",
114
+ "Priya",
115
+ "Quinn",
116
+ "Rafael",
117
+ "Sofia",
118
+ "Tara",
119
+ "Umar",
120
+ "Vera",
121
+ "Wen",
122
+ "Xiu",
123
+ "Yusuf",
124
+ "Zara",
125
+ "Aditya",
126
+ "Bianca",
127
+ "Chen",
128
+ "Dimitri",
129
+ "Esha",
130
+ "Fatima",
131
+ "Gabriel",
132
+ "Hana",
133
+ "Ivy",
134
+ "Jamal",
135
+ )
136
+
137
+ _ACTION_PHRASES: tuple[str, ...] = (
138
+ "use regex to parse HTML",
139
+ "store passwords in plaintext",
140
+ "deploy on Fridays",
141
+ "skip code review",
142
+ "hardcode API keys in source",
143
+ "use floats for money",
144
+ "ignore rate limits",
145
+ "trust user input without validation",
146
+ "log personally identifiable information to stdout",
147
+ "cache forever without an invalidation strategy",
148
+ "mutate shared global state",
149
+ "catch exceptions silently",
150
+ "call eval() on user input",
151
+ "commit directly to the main branch",
152
+ "build SQL through string concatenation",
153
+ "run schema migrations inside a web request",
154
+ "hold database locks during outbound HTTP calls",
155
+ "retry non-idempotent operations",
156
+ "disable TLS certificate verification",
157
+ "commit credentials to git history",
158
+ "use md5 to hash passwords",
159
+ "trust the X-Forwarded-For header blindly",
160
+ "disable CSRF tokens",
161
+ "return raw exception messages to end users",
162
+ "assume the server timezone is UTC without configuring it",
163
+ )
164
+
165
+ _PROJECT_NOUNS: tuple[str, ...] = (
166
+ "auth rewrite",
167
+ "pricing revamp",
168
+ "dashboard migration",
169
+ "CI overhaul",
170
+ "onboarding redesign",
171
+ "billing audit",
172
+ "observability pass",
173
+ "search relevance upgrade",
174
+ "webhook refactor",
175
+ "legacy deprecation",
176
+ "mobile launch",
177
+ "integration sprint",
178
+ "performance tune-up",
179
+ "schema migration",
180
+ "feature flag cleanup",
181
+ "access control review",
182
+ "data pipeline rebuild",
183
+ "admin tools revamp",
184
+ "notifications unification",
185
+ "reports module",
186
+ )
187
+
188
+ _ORG_NAMES: tuple[str, ...] = (
189
+ "Acme Systems",
190
+ "Blue Harbor Inc",
191
+ "Cobalt Labs",
192
+ "Delta Fleet",
193
+ "Evergreen Co",
194
+ "Fulcrum Group",
195
+ "Granite Works",
196
+ "Harbor Line",
197
+ "Ironmask Ltd",
198
+ "Juniper Partners",
199
+ "Kestrel and Sons",
200
+ "Linden Row",
201
+ "Meridian Works",
202
+ "Nimbus Group",
203
+ "Orion Bearing",
204
+ "Pioneer Cast",
205
+ "Quartz Rail",
206
+ "Redwood Signals",
207
+ "Starboard Inc",
208
+ "Timberline Holdings",
209
+ )
210
+
211
+
212
+ _REGISTRY: dict[str, Generator] = {
213
+ "person_name": LexiconGenerator(name="person_name", lexicon=_FIRST_NAMES),
214
+ "action_phrase": LexiconGenerator(name="action_phrase", lexicon=_ACTION_PHRASES),
215
+ "project_noun": LexiconGenerator(name="project_noun", lexicon=_PROJECT_NOUNS),
216
+ "org_name": LexiconGenerator(name="org_name", lexicon=_ORG_NAMES),
217
+ "date_iso": DateGenerator(),
218
+ "amount_usd": AmountGenerator(),
219
+ "product_sku": SkuGenerator(),
220
+ }
221
+
222
+
223
+ def get_generator(name: str) -> Generator:
224
+ """Return the registered generator with ``name`` or raise."""
225
+ gen = _REGISTRY.get(name)
226
+ if gen is None:
227
+ raise UnknownGeneratorError(f"unknown generator {name!r}. Known: {sorted(_REGISTRY)}")
228
+ return gen
229
+
230
+
231
+ def registered_generators() -> tuple[str, ...]:
232
+ """Sorted tuple of registered generator names."""
233
+ return tuple(sorted(_REGISTRY))
234
+
235
+
236
+ def resolve_variables(declarations: list[VariableDeclaration], base_seed: int) -> dict[str, str]:
237
+ """Resolve declared variables into concrete seeded values."""
238
+ resolved: dict[str, str] = {}
239
+ for decl in declarations:
240
+ gen = get_generator(decl.generator)
241
+ sub_seed = derive_seed(base_seed, decl.name)
242
+ resolved[decl.name] = gen.generate(sub_seed)
243
+ return resolved
@@ -0,0 +1,107 @@
1
+ """Pydantic models for parsed template definitions.
2
+
3
+ These are distinct from ``compactbench.contracts.case`` — template models carry
4
+ ``{{placeholder}}`` strings; contract models carry realized values produced by
5
+ the engine at generation time.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from enum import StrEnum
11
+ from typing import Any
12
+
13
+ from pydantic import BaseModel, ConfigDict, Field, model_validator
14
+
15
+
16
+ class DifficultyLevel(StrEnum):
17
+ EASY = "easy"
18
+ MEDIUM = "medium"
19
+ HARD = "hard"
20
+ ELITE = "elite"
21
+
22
+
23
+ class TemplateTurnRole(StrEnum):
24
+ """Roles permitted in a template turn.
25
+
26
+ ``distractor_block`` is a template-only pseudo-role that the engine expands
27
+ to multiple concrete turns at generation time.
28
+ """
29
+
30
+ USER = "user"
31
+ ASSISTANT = "assistant"
32
+ SYSTEM = "system"
33
+ DISTRACTOR_BLOCK = "distractor_block"
34
+
35
+
36
+ class VariableDeclaration(BaseModel):
37
+ model_config = ConfigDict(extra="forbid", frozen=True)
38
+
39
+ name: str = Field(min_length=1, pattern=r"^[a-zA-Z_]\w*$")
40
+ generator: str = Field(min_length=1)
41
+
42
+
43
+ class DifficultyPolicy(BaseModel):
44
+ model_config = ConfigDict(extra="forbid", frozen=True)
45
+
46
+ distractor_turns: dict[DifficultyLevel, int] = Field(default_factory=dict[DifficultyLevel, int])
47
+ paraphrase_depth: dict[DifficultyLevel, int] = Field(default_factory=dict[DifficultyLevel, int])
48
+ override_timing: dict[DifficultyLevel, str] = Field(default_factory=dict[DifficultyLevel, str])
49
+
50
+
51
+ class TurnTemplate(BaseModel):
52
+ model_config = ConfigDict(extra="forbid", frozen=True)
53
+
54
+ role: TemplateTurnRole
55
+ template: str | None = None
56
+ tags: list[str] = Field(default_factory=list[str])
57
+ count: str | int | None = None
58
+
59
+ @model_validator(mode="after")
60
+ def _check_role_requirements(self) -> TurnTemplate:
61
+ text_roles = {TemplateTurnRole.USER, TemplateTurnRole.ASSISTANT, TemplateTurnRole.SYSTEM}
62
+ if self.role in text_roles and not self.template:
63
+ raise ValueError(f"role {self.role.value!r} requires a non-empty 'template'")
64
+ if self.role is TemplateTurnRole.DISTRACTOR_BLOCK and self.count is None:
65
+ raise ValueError("role 'distractor_block' requires 'count'")
66
+ return self
67
+
68
+
69
+ class TranscriptTemplate(BaseModel):
70
+ model_config = ConfigDict(extra="forbid", frozen=True)
71
+
72
+ turns: list[TurnTemplate]
73
+
74
+
75
+ class GroundTruthTemplate(BaseModel):
76
+ model_config = ConfigDict(extra="forbid", frozen=True)
77
+
78
+ immutable_facts: list[str] = Field(default_factory=list[str])
79
+ locked_decisions: list[str] = Field(default_factory=list[str])
80
+ forbidden_behaviors: list[str] = Field(default_factory=list[str])
81
+ unresolved_items: list[str] = Field(default_factory=list[str])
82
+ deferred_items: list[str] = Field(default_factory=list[str])
83
+ entity_map: dict[str, str] = Field(default_factory=dict[str, str])
84
+
85
+
86
+ class EvaluationItemTemplate(BaseModel):
87
+ model_config = ConfigDict(extra="forbid", frozen=True)
88
+
89
+ key: str = Field(min_length=1, pattern=r"^[a-zA-Z_]\w*$")
90
+ type: str = Field(min_length=1)
91
+ prompt: str = Field(min_length=1)
92
+ expected: dict[str, Any]
93
+
94
+
95
+ class TemplateDefinition(BaseModel):
96
+ """A parsed template before variable resolution or difficulty application."""
97
+
98
+ model_config = ConfigDict(extra="forbid", frozen=True)
99
+
100
+ key: str = Field(min_length=1, pattern=r"^[a-zA-Z_][\w-]*$")
101
+ family: str = Field(min_length=1, pattern=r"^[a-zA-Z_]\w*$")
102
+ version: str = Field(pattern=r"^\d+\.\d+\.\d+$")
103
+ difficulty_policy: DifficultyPolicy
104
+ variables: list[VariableDeclaration] = Field(default_factory=list[VariableDeclaration])
105
+ transcript: TranscriptTemplate
106
+ ground_truth: GroundTruthTemplate
107
+ evaluation_items: list[EvaluationItemTemplate]
@@ -0,0 +1,55 @@
1
+ """Parse YAML template files into :class:`TemplateDefinition` models."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+
7
+ from pydantic import ValidationError
8
+ from ruamel.yaml import YAML
9
+ from ruamel.yaml.error import YAMLError
10
+
11
+ from compactbench.dsl.errors import TemplateParseError
12
+ from compactbench.dsl.models import TemplateDefinition
13
+
14
+ _yaml = YAML(typ="safe")
15
+
16
+
17
+ def parse_template_file(path: Path | str) -> TemplateDefinition:
18
+ """Load a template YAML file from disk and return a validated model."""
19
+ path = Path(path)
20
+ if not path.exists():
21
+ raise TemplateParseError(f"template file not found: {path}")
22
+ try:
23
+ with path.open("r", encoding="utf-8") as f:
24
+ data = _yaml.load(f) # pyright: ignore[reportUnknownMemberType]
25
+ except YAMLError as exc:
26
+ raise TemplateParseError(f"YAML syntax error in {path}: {exc}") from exc
27
+ return _build_definition(data, source=str(path))
28
+
29
+
30
+ def parse_template_string(yaml_text: str) -> TemplateDefinition:
31
+ """Parse a template from a YAML string (used in tests)."""
32
+ try:
33
+ data = _yaml.load(yaml_text) # pyright: ignore[reportUnknownMemberType]
34
+ except YAMLError as exc:
35
+ raise TemplateParseError(f"YAML syntax error: {exc}") from exc
36
+ return _build_definition(data, source="<string>")
37
+
38
+
39
+ def load_suite(suite_dir: Path | str) -> list[TemplateDefinition]:
40
+ """Load every ``*.yaml`` template in a suite directory, sorted by filename."""
41
+ suite_dir = Path(suite_dir)
42
+ if not suite_dir.is_dir():
43
+ raise TemplateParseError(f"not a directory: {suite_dir}")
44
+ return [parse_template_file(p) for p in sorted(suite_dir.glob("*.yaml"))]
45
+
46
+
47
+ def _build_definition(data: object, source: str) -> TemplateDefinition:
48
+ if not isinstance(data, dict):
49
+ raise TemplateParseError(f"expected a YAML mapping at root of {source}")
50
+ if "template" not in data:
51
+ raise TemplateParseError(f"missing 'template' key at root of {source}")
52
+ try:
53
+ return TemplateDefinition.model_validate(data["template"])
54
+ except ValidationError as exc:
55
+ raise TemplateParseError(f"structural validation failed for {source}:\n{exc}") from exc
@@ -0,0 +1,32 @@
1
+ """Handlebars-style ``{{variable}}`` substitution."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+
7
+ from compactbench.dsl.errors import UnresolvedReferenceError
8
+
9
+ # Matches {{ name }} or {{name}} or {{ namespace.name }}. Whitespace optional.
10
+ _PATTERN = re.compile(r"\{\{\s*([\w.]+)\s*\}\}")
11
+
12
+
13
+ def substitute(text: str, bindings: dict[str, str]) -> str:
14
+ """Replace every ``{{ref}}`` in ``text`` with its binding.
15
+
16
+ Raises :class:`UnresolvedReferenceError` if any reference has no binding.
17
+ """
18
+
19
+ def _repl(match: re.Match[str]) -> str:
20
+ ref = match.group(1)
21
+ if ref not in bindings:
22
+ raise UnresolvedReferenceError(
23
+ f"no binding for {{{{{ref}}}}}. Known: {sorted(bindings)}"
24
+ )
25
+ return bindings[ref]
26
+
27
+ return _PATTERN.sub(_repl, text)
28
+
29
+
30
+ def extract_references(text: str) -> list[str]:
31
+ """Return every ``{{ref}}`` name in ``text`` in order of appearance."""
32
+ return [m.group(1) for m in _PATTERN.finditer(text)]
@@ -0,0 +1,97 @@
1
+ """Semantic validation for parsed template definitions.
2
+
3
+ Pydantic handles structural validation (required fields, types, patterns).
4
+ This module catches semantic errors that structure alone cannot: undeclared
5
+ variable references, unknown generators, duplicate names.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Iterator
11
+
12
+ from compactbench.dsl.errors import TemplateValidationError, UnknownGeneratorError
13
+ from compactbench.dsl.generators import get_generator
14
+ from compactbench.dsl.models import TemplateDefinition
15
+ from compactbench.dsl.substitution import extract_references
16
+
17
+ _DIFFICULTY_REFS = frozenset(
18
+ {
19
+ "difficulty.distractor_turns",
20
+ "difficulty.paraphrase_depth",
21
+ "difficulty.override_timing",
22
+ }
23
+ )
24
+
25
+
26
+ def validate_template(template: TemplateDefinition) -> None:
27
+ """Raise :class:`TemplateValidationError` if ``template`` is semantically invalid."""
28
+ _validate_variables(template)
29
+ _validate_evaluation_items(template)
30
+ _validate_references(template)
31
+
32
+
33
+ def _validate_variables(template: TemplateDefinition) -> None:
34
+ seen: set[str] = set()
35
+ for var in template.variables:
36
+ if var.name in seen:
37
+ raise TemplateValidationError(f"duplicate variable name: {var.name!r}")
38
+ seen.add(var.name)
39
+ try:
40
+ get_generator(var.generator)
41
+ except UnknownGeneratorError as exc:
42
+ raise TemplateValidationError(f"variable {var.name!r}: {exc}") from exc
43
+
44
+
45
+ def _validate_evaluation_items(template: TemplateDefinition) -> None:
46
+ seen: set[str] = set()
47
+ for item in template.evaluation_items:
48
+ if item.key in seen:
49
+ raise TemplateValidationError(f"duplicate evaluation item key: {item.key!r}")
50
+ seen.add(item.key)
51
+
52
+
53
+ def _validate_references(template: TemplateDefinition) -> None:
54
+ declared = {v.name for v in template.variables}
55
+
56
+ for source, text in _walk_strings(template):
57
+ for ref in extract_references(text):
58
+ if ref.startswith("difficulty."):
59
+ if ref not in _DIFFICULTY_REFS:
60
+ raise TemplateValidationError(
61
+ f"unknown difficulty reference {{{{{ref}}}}} at {source}. "
62
+ f"Allowed: {sorted(_DIFFICULTY_REFS)}"
63
+ )
64
+ elif ref not in declared:
65
+ raise TemplateValidationError(
66
+ f"reference to undeclared variable {{{{{ref}}}}} at {source}. "
67
+ f"Declared: {sorted(declared)}"
68
+ )
69
+
70
+
71
+ def _walk_strings(template: TemplateDefinition) -> Iterator[tuple[str, str]]:
72
+ """Yield ``(location, text)`` pairs for every template string that may carry refs."""
73
+ for i, turn in enumerate(template.transcript.turns):
74
+ if turn.template is not None:
75
+ yield f"transcript.turns[{i}].template", turn.template
76
+ if isinstance(turn.count, str):
77
+ yield f"transcript.turns[{i}].count", turn.count
78
+
79
+ for gt_field in (
80
+ "immutable_facts",
81
+ "locked_decisions",
82
+ "forbidden_behaviors",
83
+ "unresolved_items",
84
+ "deferred_items",
85
+ ):
86
+ for i, s in enumerate(getattr(template.ground_truth, gt_field)):
87
+ yield f"ground_truth.{gt_field}[{i}]", s
88
+
89
+ for k, v in template.ground_truth.entity_map.items():
90
+ yield f"ground_truth.entity_map[{k!r}].key", k
91
+ yield f"ground_truth.entity_map[{k!r}].value", v
92
+
93
+ for i, item in enumerate(template.evaluation_items):
94
+ yield f"evaluation_items[{i}].prompt", item.prompt
95
+ for k, v in item.expected.items():
96
+ if isinstance(v, str):
97
+ yield f"evaluation_items[{i}].expected[{k!r}]", v
@@ -0,0 +1,29 @@
1
+ """Deterministic case generation from template + seed.
2
+
3
+ Implements WO-003 per docs/architecture/decisions.md §B1.
4
+ """
5
+
6
+ from compactbench.engine.difficulty import (
7
+ difficulty_bindings,
8
+ distractor_count_for,
9
+ override_timing_for,
10
+ paraphrase_depth_for,
11
+ )
12
+ from compactbench.engine.distractors import generate_distractor_turns
13
+ from compactbench.engine.errors import EngineError, GenerationError
14
+ from compactbench.engine.generation import generate_case
15
+ from compactbench.engine.seeds import derive_case_seed
16
+ from compactbench.engine.transcript import build_transcript
17
+
18
+ __all__ = [
19
+ "EngineError",
20
+ "GenerationError",
21
+ "build_transcript",
22
+ "derive_case_seed",
23
+ "difficulty_bindings",
24
+ "distractor_count_for",
25
+ "generate_case",
26
+ "generate_distractor_turns",
27
+ "override_timing_for",
28
+ "paraphrase_depth_for",
29
+ ]