compactbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compactbench/__init__.py +10 -0
- compactbench/__main__.py +6 -0
- compactbench/cli.py +343 -0
- compactbench/compactors/__init__.py +47 -0
- compactbench/compactors/_state_parser.py +118 -0
- compactbench/compactors/_utils.py +37 -0
- compactbench/compactors/base.py +38 -0
- compactbench/compactors/errors.py +11 -0
- compactbench/compactors/hierarchical_summary.py +113 -0
- compactbench/compactors/hybrid_ledger.py +115 -0
- compactbench/compactors/naive_summary.py +54 -0
- compactbench/compactors/structured_state.py +64 -0
- compactbench/config.py +36 -0
- compactbench/contracts/__init__.py +41 -0
- compactbench/contracts/artifact.py +49 -0
- compactbench/contracts/case.py +81 -0
- compactbench/contracts/result.py +79 -0
- compactbench/dsl/__init__.py +66 -0
- compactbench/dsl/errors.py +23 -0
- compactbench/dsl/generators.py +243 -0
- compactbench/dsl/models.py +107 -0
- compactbench/dsl/parser.py +55 -0
- compactbench/dsl/substitution.py +32 -0
- compactbench/dsl/validator.py +97 -0
- compactbench/engine/__init__.py +29 -0
- compactbench/engine/difficulty.py +54 -0
- compactbench/engine/distractors.py +69 -0
- compactbench/engine/errors.py +11 -0
- compactbench/engine/generation.py +96 -0
- compactbench/engine/seeds.py +31 -0
- compactbench/engine/transcript.py +84 -0
- compactbench/leaderboard/__init__.py +38 -0
- compactbench/leaderboard/errors.py +11 -0
- compactbench/leaderboard/projection.py +103 -0
- compactbench/leaderboard/qualification.py +90 -0
- compactbench/leaderboard/ranking.py +71 -0
- compactbench/providers/__init__.py +52 -0
- compactbench/providers/_retry.py +48 -0
- compactbench/providers/base.py +44 -0
- compactbench/providers/errors.py +15 -0
- compactbench/providers/google_ai_studio.py +112 -0
- compactbench/providers/groq.py +95 -0
- compactbench/providers/mock.py +53 -0
- compactbench/providers/ollama.py +100 -0
- compactbench/py.typed +0 -0
- compactbench/runner/__init__.py +53 -0
- compactbench/runner/_resolver.py +57 -0
- compactbench/runner/continuation.py +78 -0
- compactbench/runner/cycle.py +67 -0
- compactbench/runner/errors.py +15 -0
- compactbench/runner/evaluation.py +71 -0
- compactbench/runner/persistence.py +227 -0
- compactbench/runner/run.py +257 -0
- compactbench/scoring/__init__.py +48 -0
- compactbench/scoring/checks.py +92 -0
- compactbench/scoring/compression.py +66 -0
- compactbench/scoring/contradictions.py +70 -0
- compactbench/scoring/drift.py +30 -0
- compactbench/scoring/errors.py +7 -0
- compactbench/scoring/scorer.py +77 -0
- compactbench/utils/__init__.py +1 -0
- compactbench-0.1.0.dist-info/METADATA +161 -0
- compactbench-0.1.0.dist-info/RECORD +66 -0
- compactbench-0.1.0.dist-info/WHEEL +4 -0
- compactbench-0.1.0.dist-info/entry_points.txt +2 -0
- compactbench-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Template DSL: parser, seeded generators, validator, substitution.
|
|
2
|
+
|
|
3
|
+
See docs/architecture/decisions.md §B1 for the DSL shape.
|
|
4
|
+
Implemented in WO-002.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from compactbench.dsl.errors import (
|
|
8
|
+
TemplateError,
|
|
9
|
+
TemplateParseError,
|
|
10
|
+
TemplateValidationError,
|
|
11
|
+
UnknownGeneratorError,
|
|
12
|
+
UnresolvedReferenceError,
|
|
13
|
+
)
|
|
14
|
+
from compactbench.dsl.generators import (
|
|
15
|
+
Generator,
|
|
16
|
+
derive_seed,
|
|
17
|
+
get_generator,
|
|
18
|
+
registered_generators,
|
|
19
|
+
resolve_variables,
|
|
20
|
+
)
|
|
21
|
+
from compactbench.dsl.models import (
|
|
22
|
+
DifficultyLevel,
|
|
23
|
+
DifficultyPolicy,
|
|
24
|
+
EvaluationItemTemplate,
|
|
25
|
+
GroundTruthTemplate,
|
|
26
|
+
TemplateDefinition,
|
|
27
|
+
TemplateTurnRole,
|
|
28
|
+
TranscriptTemplate,
|
|
29
|
+
TurnTemplate,
|
|
30
|
+
VariableDeclaration,
|
|
31
|
+
)
|
|
32
|
+
from compactbench.dsl.parser import (
|
|
33
|
+
load_suite,
|
|
34
|
+
parse_template_file,
|
|
35
|
+
parse_template_string,
|
|
36
|
+
)
|
|
37
|
+
from compactbench.dsl.substitution import extract_references, substitute
|
|
38
|
+
from compactbench.dsl.validator import validate_template
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"DifficultyLevel",
|
|
42
|
+
"DifficultyPolicy",
|
|
43
|
+
"EvaluationItemTemplate",
|
|
44
|
+
"Generator",
|
|
45
|
+
"GroundTruthTemplate",
|
|
46
|
+
"TemplateDefinition",
|
|
47
|
+
"TemplateError",
|
|
48
|
+
"TemplateParseError",
|
|
49
|
+
"TemplateTurnRole",
|
|
50
|
+
"TemplateValidationError",
|
|
51
|
+
"TranscriptTemplate",
|
|
52
|
+
"TurnTemplate",
|
|
53
|
+
"UnknownGeneratorError",
|
|
54
|
+
"UnresolvedReferenceError",
|
|
55
|
+
"VariableDeclaration",
|
|
56
|
+
"derive_seed",
|
|
57
|
+
"extract_references",
|
|
58
|
+
"get_generator",
|
|
59
|
+
"load_suite",
|
|
60
|
+
"parse_template_file",
|
|
61
|
+
"parse_template_string",
|
|
62
|
+
"registered_generators",
|
|
63
|
+
"resolve_variables",
|
|
64
|
+
"substitute",
|
|
65
|
+
"validate_template",
|
|
66
|
+
]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Exception hierarchy for template DSL operations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TemplateError(Exception):
|
|
7
|
+
"""Base exception for anything DSL-related."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TemplateParseError(TemplateError):
|
|
11
|
+
"""YAML structure does not match the template DSL shape."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TemplateValidationError(TemplateError):
|
|
15
|
+
"""Template is structurally valid but has a semantic error."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class UnknownGeneratorError(TemplateError):
|
|
19
|
+
"""A variable references a generator that is not registered."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class UnresolvedReferenceError(TemplateError):
|
|
23
|
+
"""A ``{{reference}}`` in the template cannot be resolved from the bindings."""
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""Seeded random generators for template variables.
|
|
2
|
+
|
|
3
|
+
Each generator is a deterministic function of a seed. Sub-seeds are derived
|
|
4
|
+
from ``(base_seed, variable_name)`` via SHA-256 so adding or reordering
|
|
5
|
+
variables in a template never changes the output of unchanged variables.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import random
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Protocol, runtime_checkable
|
|
14
|
+
|
|
15
|
+
from compactbench.dsl.errors import UnknownGeneratorError
|
|
16
|
+
from compactbench.dsl.models import VariableDeclaration
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def derive_seed(base_seed: int, variable_name: str) -> int:
|
|
20
|
+
"""Derive a deterministic sub-seed for a named variable.
|
|
21
|
+
|
|
22
|
+
Uses SHA-256 of ``"{base}:{name}"``. Stable across Python versions.
|
|
23
|
+
"""
|
|
24
|
+
payload = f"{base_seed}:{variable_name}".encode()
|
|
25
|
+
digest = hashlib.sha256(payload).digest()
|
|
26
|
+
return int.from_bytes(digest[:8], "big", signed=False)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@runtime_checkable
|
|
30
|
+
class Generator(Protocol):
|
|
31
|
+
"""Protocol every generator must satisfy."""
|
|
32
|
+
|
|
33
|
+
name: str
|
|
34
|
+
|
|
35
|
+
def generate(self, seed: int) -> str: ...
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class LexiconGenerator:
|
|
40
|
+
"""Picks one item from a bounded lexicon, deterministically by seed."""
|
|
41
|
+
|
|
42
|
+
name: str
|
|
43
|
+
lexicon: tuple[str, ...]
|
|
44
|
+
|
|
45
|
+
def generate(self, seed: int) -> str:
|
|
46
|
+
rng = random.Random(seed)
|
|
47
|
+
return rng.choice(self.lexicon)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class DateGenerator:
|
|
52
|
+
"""Generates an ISO-8601 date within a bounded year range."""
|
|
53
|
+
|
|
54
|
+
name: str = "date_iso"
|
|
55
|
+
start_year: int = 2024
|
|
56
|
+
end_year: int = 2026
|
|
57
|
+
|
|
58
|
+
def generate(self, seed: int) -> str:
|
|
59
|
+
rng = random.Random(seed)
|
|
60
|
+
year = rng.randint(self.start_year, self.end_year)
|
|
61
|
+
month = rng.randint(1, 12)
|
|
62
|
+
day = rng.randint(1, 28)
|
|
63
|
+
return f"{year:04d}-{month:02d}-{day:02d}"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class AmountGenerator:
|
|
68
|
+
"""Generates a USD amount string like ``$47,250``."""
|
|
69
|
+
|
|
70
|
+
name: str = "amount_usd"
|
|
71
|
+
magnitudes: tuple[int, ...] = field(default=(2, 3, 4, 5))
|
|
72
|
+
|
|
73
|
+
def generate(self, seed: int) -> str:
|
|
74
|
+
rng = random.Random(seed)
|
|
75
|
+
magnitude = rng.choice(self.magnitudes)
|
|
76
|
+
low = 10 ** (magnitude - 1)
|
|
77
|
+
high = 10**magnitude - 1
|
|
78
|
+
amount = rng.randint(low, high)
|
|
79
|
+
return f"${amount:,}"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass
|
|
83
|
+
class SkuGenerator:
|
|
84
|
+
"""Generates an alphanumeric SKU like ``SKU-AB9421``."""
|
|
85
|
+
|
|
86
|
+
name: str = "product_sku"
|
|
87
|
+
|
|
88
|
+
def generate(self, seed: int) -> str:
|
|
89
|
+
rng = random.Random(seed)
|
|
90
|
+
prefix = rng.choice(("SKU", "PRD", "ITEM", "ART"))
|
|
91
|
+
alpha = "".join(rng.choices("ABCDEFGHJKLMNPQRTUVWXYZ", k=2))
|
|
92
|
+
num = rng.randint(100, 9999)
|
|
93
|
+
return f"{prefix}-{alpha}{num}"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# --- lexicons ------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
_FIRST_NAMES: tuple[str, ...] = (
|
|
99
|
+
"Alice",
|
|
100
|
+
"Bob",
|
|
101
|
+
"Carol",
|
|
102
|
+
"David",
|
|
103
|
+
"Elena",
|
|
104
|
+
"Farid",
|
|
105
|
+
"Grace",
|
|
106
|
+
"Hassan",
|
|
107
|
+
"Iris",
|
|
108
|
+
"Julia",
|
|
109
|
+
"Kenji",
|
|
110
|
+
"Leo",
|
|
111
|
+
"Maya",
|
|
112
|
+
"Noor",
|
|
113
|
+
"Omar",
|
|
114
|
+
"Priya",
|
|
115
|
+
"Quinn",
|
|
116
|
+
"Rafael",
|
|
117
|
+
"Sofia",
|
|
118
|
+
"Tara",
|
|
119
|
+
"Umar",
|
|
120
|
+
"Vera",
|
|
121
|
+
"Wen",
|
|
122
|
+
"Xiu",
|
|
123
|
+
"Yusuf",
|
|
124
|
+
"Zara",
|
|
125
|
+
"Aditya",
|
|
126
|
+
"Bianca",
|
|
127
|
+
"Chen",
|
|
128
|
+
"Dimitri",
|
|
129
|
+
"Esha",
|
|
130
|
+
"Fatima",
|
|
131
|
+
"Gabriel",
|
|
132
|
+
"Hana",
|
|
133
|
+
"Ivy",
|
|
134
|
+
"Jamal",
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
_ACTION_PHRASES: tuple[str, ...] = (
|
|
138
|
+
"use regex to parse HTML",
|
|
139
|
+
"store passwords in plaintext",
|
|
140
|
+
"deploy on Fridays",
|
|
141
|
+
"skip code review",
|
|
142
|
+
"hardcode API keys in source",
|
|
143
|
+
"use floats for money",
|
|
144
|
+
"ignore rate limits",
|
|
145
|
+
"trust user input without validation",
|
|
146
|
+
"log personally identifiable information to stdout",
|
|
147
|
+
"cache forever without an invalidation strategy",
|
|
148
|
+
"mutate shared global state",
|
|
149
|
+
"catch exceptions silently",
|
|
150
|
+
"call eval() on user input",
|
|
151
|
+
"commit directly to the main branch",
|
|
152
|
+
"build SQL through string concatenation",
|
|
153
|
+
"run schema migrations inside a web request",
|
|
154
|
+
"hold database locks during outbound HTTP calls",
|
|
155
|
+
"retry non-idempotent operations",
|
|
156
|
+
"disable TLS certificate verification",
|
|
157
|
+
"commit credentials to git history",
|
|
158
|
+
"use md5 to hash passwords",
|
|
159
|
+
"trust the X-Forwarded-For header blindly",
|
|
160
|
+
"disable CSRF tokens",
|
|
161
|
+
"return raw exception messages to end users",
|
|
162
|
+
"assume the server timezone is UTC without configuring it",
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
_PROJECT_NOUNS: tuple[str, ...] = (
|
|
166
|
+
"auth rewrite",
|
|
167
|
+
"pricing revamp",
|
|
168
|
+
"dashboard migration",
|
|
169
|
+
"CI overhaul",
|
|
170
|
+
"onboarding redesign",
|
|
171
|
+
"billing audit",
|
|
172
|
+
"observability pass",
|
|
173
|
+
"search relevance upgrade",
|
|
174
|
+
"webhook refactor",
|
|
175
|
+
"legacy deprecation",
|
|
176
|
+
"mobile launch",
|
|
177
|
+
"integration sprint",
|
|
178
|
+
"performance tune-up",
|
|
179
|
+
"schema migration",
|
|
180
|
+
"feature flag cleanup",
|
|
181
|
+
"access control review",
|
|
182
|
+
"data pipeline rebuild",
|
|
183
|
+
"admin tools revamp",
|
|
184
|
+
"notifications unification",
|
|
185
|
+
"reports module",
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
_ORG_NAMES: tuple[str, ...] = (
|
|
189
|
+
"Acme Systems",
|
|
190
|
+
"Blue Harbor Inc",
|
|
191
|
+
"Cobalt Labs",
|
|
192
|
+
"Delta Fleet",
|
|
193
|
+
"Evergreen Co",
|
|
194
|
+
"Fulcrum Group",
|
|
195
|
+
"Granite Works",
|
|
196
|
+
"Harbor Line",
|
|
197
|
+
"Ironmask Ltd",
|
|
198
|
+
"Juniper Partners",
|
|
199
|
+
"Kestrel and Sons",
|
|
200
|
+
"Linden Row",
|
|
201
|
+
"Meridian Works",
|
|
202
|
+
"Nimbus Group",
|
|
203
|
+
"Orion Bearing",
|
|
204
|
+
"Pioneer Cast",
|
|
205
|
+
"Quartz Rail",
|
|
206
|
+
"Redwood Signals",
|
|
207
|
+
"Starboard Inc",
|
|
208
|
+
"Timberline Holdings",
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
_REGISTRY: dict[str, Generator] = {
|
|
213
|
+
"person_name": LexiconGenerator(name="person_name", lexicon=_FIRST_NAMES),
|
|
214
|
+
"action_phrase": LexiconGenerator(name="action_phrase", lexicon=_ACTION_PHRASES),
|
|
215
|
+
"project_noun": LexiconGenerator(name="project_noun", lexicon=_PROJECT_NOUNS),
|
|
216
|
+
"org_name": LexiconGenerator(name="org_name", lexicon=_ORG_NAMES),
|
|
217
|
+
"date_iso": DateGenerator(),
|
|
218
|
+
"amount_usd": AmountGenerator(),
|
|
219
|
+
"product_sku": SkuGenerator(),
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def get_generator(name: str) -> Generator:
|
|
224
|
+
"""Return the registered generator with ``name`` or raise."""
|
|
225
|
+
gen = _REGISTRY.get(name)
|
|
226
|
+
if gen is None:
|
|
227
|
+
raise UnknownGeneratorError(f"unknown generator {name!r}. Known: {sorted(_REGISTRY)}")
|
|
228
|
+
return gen
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def registered_generators() -> tuple[str, ...]:
|
|
232
|
+
"""Sorted tuple of registered generator names."""
|
|
233
|
+
return tuple(sorted(_REGISTRY))
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def resolve_variables(declarations: list[VariableDeclaration], base_seed: int) -> dict[str, str]:
|
|
237
|
+
"""Resolve declared variables into concrete seeded values."""
|
|
238
|
+
resolved: dict[str, str] = {}
|
|
239
|
+
for decl in declarations:
|
|
240
|
+
gen = get_generator(decl.generator)
|
|
241
|
+
sub_seed = derive_seed(base_seed, decl.name)
|
|
242
|
+
resolved[decl.name] = gen.generate(sub_seed)
|
|
243
|
+
return resolved
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Pydantic models for parsed template definitions.
|
|
2
|
+
|
|
3
|
+
These are distinct from ``compactbench.contracts.case`` — template models carry
|
|
4
|
+
``{{placeholder}}`` strings; contract models carry realized values produced by
|
|
5
|
+
the engine at generation time.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from enum import StrEnum
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DifficultyLevel(StrEnum):
|
|
17
|
+
EASY = "easy"
|
|
18
|
+
MEDIUM = "medium"
|
|
19
|
+
HARD = "hard"
|
|
20
|
+
ELITE = "elite"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class TemplateTurnRole(StrEnum):
|
|
24
|
+
"""Roles permitted in a template turn.
|
|
25
|
+
|
|
26
|
+
``distractor_block`` is a template-only pseudo-role that the engine expands
|
|
27
|
+
to multiple concrete turns at generation time.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
USER = "user"
|
|
31
|
+
ASSISTANT = "assistant"
|
|
32
|
+
SYSTEM = "system"
|
|
33
|
+
DISTRACTOR_BLOCK = "distractor_block"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class VariableDeclaration(BaseModel):
|
|
37
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
38
|
+
|
|
39
|
+
name: str = Field(min_length=1, pattern=r"^[a-zA-Z_]\w*$")
|
|
40
|
+
generator: str = Field(min_length=1)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class DifficultyPolicy(BaseModel):
|
|
44
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
45
|
+
|
|
46
|
+
distractor_turns: dict[DifficultyLevel, int] = Field(default_factory=dict[DifficultyLevel, int])
|
|
47
|
+
paraphrase_depth: dict[DifficultyLevel, int] = Field(default_factory=dict[DifficultyLevel, int])
|
|
48
|
+
override_timing: dict[DifficultyLevel, str] = Field(default_factory=dict[DifficultyLevel, str])
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class TurnTemplate(BaseModel):
|
|
52
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
53
|
+
|
|
54
|
+
role: TemplateTurnRole
|
|
55
|
+
template: str | None = None
|
|
56
|
+
tags: list[str] = Field(default_factory=list[str])
|
|
57
|
+
count: str | int | None = None
|
|
58
|
+
|
|
59
|
+
@model_validator(mode="after")
|
|
60
|
+
def _check_role_requirements(self) -> TurnTemplate:
|
|
61
|
+
text_roles = {TemplateTurnRole.USER, TemplateTurnRole.ASSISTANT, TemplateTurnRole.SYSTEM}
|
|
62
|
+
if self.role in text_roles and not self.template:
|
|
63
|
+
raise ValueError(f"role {self.role.value!r} requires a non-empty 'template'")
|
|
64
|
+
if self.role is TemplateTurnRole.DISTRACTOR_BLOCK and self.count is None:
|
|
65
|
+
raise ValueError("role 'distractor_block' requires 'count'")
|
|
66
|
+
return self
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class TranscriptTemplate(BaseModel):
|
|
70
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
71
|
+
|
|
72
|
+
turns: list[TurnTemplate]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class GroundTruthTemplate(BaseModel):
|
|
76
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
77
|
+
|
|
78
|
+
immutable_facts: list[str] = Field(default_factory=list[str])
|
|
79
|
+
locked_decisions: list[str] = Field(default_factory=list[str])
|
|
80
|
+
forbidden_behaviors: list[str] = Field(default_factory=list[str])
|
|
81
|
+
unresolved_items: list[str] = Field(default_factory=list[str])
|
|
82
|
+
deferred_items: list[str] = Field(default_factory=list[str])
|
|
83
|
+
entity_map: dict[str, str] = Field(default_factory=dict[str, str])
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class EvaluationItemTemplate(BaseModel):
|
|
87
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
88
|
+
|
|
89
|
+
key: str = Field(min_length=1, pattern=r"^[a-zA-Z_]\w*$")
|
|
90
|
+
type: str = Field(min_length=1)
|
|
91
|
+
prompt: str = Field(min_length=1)
|
|
92
|
+
expected: dict[str, Any]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class TemplateDefinition(BaseModel):
|
|
96
|
+
"""A parsed template before variable resolution or difficulty application."""
|
|
97
|
+
|
|
98
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
99
|
+
|
|
100
|
+
key: str = Field(min_length=1, pattern=r"^[a-zA-Z_][\w-]*$")
|
|
101
|
+
family: str = Field(min_length=1, pattern=r"^[a-zA-Z_]\w*$")
|
|
102
|
+
version: str = Field(pattern=r"^\d+\.\d+\.\d+$")
|
|
103
|
+
difficulty_policy: DifficultyPolicy
|
|
104
|
+
variables: list[VariableDeclaration] = Field(default_factory=list[VariableDeclaration])
|
|
105
|
+
transcript: TranscriptTemplate
|
|
106
|
+
ground_truth: GroundTruthTemplate
|
|
107
|
+
evaluation_items: list[EvaluationItemTemplate]
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Parse YAML template files into :class:`TemplateDefinition` models."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
from ruamel.yaml import YAML
|
|
9
|
+
from ruamel.yaml.error import YAMLError
|
|
10
|
+
|
|
11
|
+
from compactbench.dsl.errors import TemplateParseError
|
|
12
|
+
from compactbench.dsl.models import TemplateDefinition
|
|
13
|
+
|
|
14
|
+
_yaml = YAML(typ="safe")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def parse_template_file(path: Path | str) -> TemplateDefinition:
|
|
18
|
+
"""Load a template YAML file from disk and return a validated model."""
|
|
19
|
+
path = Path(path)
|
|
20
|
+
if not path.exists():
|
|
21
|
+
raise TemplateParseError(f"template file not found: {path}")
|
|
22
|
+
try:
|
|
23
|
+
with path.open("r", encoding="utf-8") as f:
|
|
24
|
+
data = _yaml.load(f) # pyright: ignore[reportUnknownMemberType]
|
|
25
|
+
except YAMLError as exc:
|
|
26
|
+
raise TemplateParseError(f"YAML syntax error in {path}: {exc}") from exc
|
|
27
|
+
return _build_definition(data, source=str(path))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_template_string(yaml_text: str) -> TemplateDefinition:
|
|
31
|
+
"""Parse a template from a YAML string (used in tests)."""
|
|
32
|
+
try:
|
|
33
|
+
data = _yaml.load(yaml_text) # pyright: ignore[reportUnknownMemberType]
|
|
34
|
+
except YAMLError as exc:
|
|
35
|
+
raise TemplateParseError(f"YAML syntax error: {exc}") from exc
|
|
36
|
+
return _build_definition(data, source="<string>")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def load_suite(suite_dir: Path | str) -> list[TemplateDefinition]:
|
|
40
|
+
"""Load every ``*.yaml`` template in a suite directory, sorted by filename."""
|
|
41
|
+
suite_dir = Path(suite_dir)
|
|
42
|
+
if not suite_dir.is_dir():
|
|
43
|
+
raise TemplateParseError(f"not a directory: {suite_dir}")
|
|
44
|
+
return [parse_template_file(p) for p in sorted(suite_dir.glob("*.yaml"))]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _build_definition(data: object, source: str) -> TemplateDefinition:
|
|
48
|
+
if not isinstance(data, dict):
|
|
49
|
+
raise TemplateParseError(f"expected a YAML mapping at root of {source}")
|
|
50
|
+
if "template" not in data:
|
|
51
|
+
raise TemplateParseError(f"missing 'template' key at root of {source}")
|
|
52
|
+
try:
|
|
53
|
+
return TemplateDefinition.model_validate(data["template"])
|
|
54
|
+
except ValidationError as exc:
|
|
55
|
+
raise TemplateParseError(f"structural validation failed for {source}:\n{exc}") from exc
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Handlebars-style ``{{variable}}`` substitution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from compactbench.dsl.errors import UnresolvedReferenceError
|
|
8
|
+
|
|
9
|
+
# Matches {{ name }} or {{name}} or {{ namespace.name }}. Whitespace optional.
|
|
10
|
+
_PATTERN = re.compile(r"\{\{\s*([\w.]+)\s*\}\}")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def substitute(text: str, bindings: dict[str, str]) -> str:
|
|
14
|
+
"""Replace every ``{{ref}}`` in ``text`` with its binding.
|
|
15
|
+
|
|
16
|
+
Raises :class:`UnresolvedReferenceError` if any reference has no binding.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def _repl(match: re.Match[str]) -> str:
|
|
20
|
+
ref = match.group(1)
|
|
21
|
+
if ref not in bindings:
|
|
22
|
+
raise UnresolvedReferenceError(
|
|
23
|
+
f"no binding for {{{{{ref}}}}}. Known: {sorted(bindings)}"
|
|
24
|
+
)
|
|
25
|
+
return bindings[ref]
|
|
26
|
+
|
|
27
|
+
return _PATTERN.sub(_repl, text)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def extract_references(text: str) -> list[str]:
|
|
31
|
+
"""Return every ``{{ref}}`` name in ``text`` in order of appearance."""
|
|
32
|
+
return [m.group(1) for m in _PATTERN.finditer(text)]
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Semantic validation for parsed template definitions.
|
|
2
|
+
|
|
3
|
+
Pydantic handles structural validation (required fields, types, patterns).
|
|
4
|
+
This module catches semantic errors that structure alone cannot: undeclared
|
|
5
|
+
variable references, unknown generators, duplicate names.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Iterator
|
|
11
|
+
|
|
12
|
+
from compactbench.dsl.errors import TemplateValidationError, UnknownGeneratorError
|
|
13
|
+
from compactbench.dsl.generators import get_generator
|
|
14
|
+
from compactbench.dsl.models import TemplateDefinition
|
|
15
|
+
from compactbench.dsl.substitution import extract_references
|
|
16
|
+
|
|
17
|
+
_DIFFICULTY_REFS = frozenset(
|
|
18
|
+
{
|
|
19
|
+
"difficulty.distractor_turns",
|
|
20
|
+
"difficulty.paraphrase_depth",
|
|
21
|
+
"difficulty.override_timing",
|
|
22
|
+
}
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def validate_template(template: TemplateDefinition) -> None:
|
|
27
|
+
"""Raise :class:`TemplateValidationError` if ``template`` is semantically invalid."""
|
|
28
|
+
_validate_variables(template)
|
|
29
|
+
_validate_evaluation_items(template)
|
|
30
|
+
_validate_references(template)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _validate_variables(template: TemplateDefinition) -> None:
|
|
34
|
+
seen: set[str] = set()
|
|
35
|
+
for var in template.variables:
|
|
36
|
+
if var.name in seen:
|
|
37
|
+
raise TemplateValidationError(f"duplicate variable name: {var.name!r}")
|
|
38
|
+
seen.add(var.name)
|
|
39
|
+
try:
|
|
40
|
+
get_generator(var.generator)
|
|
41
|
+
except UnknownGeneratorError as exc:
|
|
42
|
+
raise TemplateValidationError(f"variable {var.name!r}: {exc}") from exc
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _validate_evaluation_items(template: TemplateDefinition) -> None:
|
|
46
|
+
seen: set[str] = set()
|
|
47
|
+
for item in template.evaluation_items:
|
|
48
|
+
if item.key in seen:
|
|
49
|
+
raise TemplateValidationError(f"duplicate evaluation item key: {item.key!r}")
|
|
50
|
+
seen.add(item.key)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _validate_references(template: TemplateDefinition) -> None:
|
|
54
|
+
declared = {v.name for v in template.variables}
|
|
55
|
+
|
|
56
|
+
for source, text in _walk_strings(template):
|
|
57
|
+
for ref in extract_references(text):
|
|
58
|
+
if ref.startswith("difficulty."):
|
|
59
|
+
if ref not in _DIFFICULTY_REFS:
|
|
60
|
+
raise TemplateValidationError(
|
|
61
|
+
f"unknown difficulty reference {{{{{ref}}}}} at {source}. "
|
|
62
|
+
f"Allowed: {sorted(_DIFFICULTY_REFS)}"
|
|
63
|
+
)
|
|
64
|
+
elif ref not in declared:
|
|
65
|
+
raise TemplateValidationError(
|
|
66
|
+
f"reference to undeclared variable {{{{{ref}}}}} at {source}. "
|
|
67
|
+
f"Declared: {sorted(declared)}"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _walk_strings(template: TemplateDefinition) -> Iterator[tuple[str, str]]:
|
|
72
|
+
"""Yield ``(location, text)`` pairs for every template string that may carry refs."""
|
|
73
|
+
for i, turn in enumerate(template.transcript.turns):
|
|
74
|
+
if turn.template is not None:
|
|
75
|
+
yield f"transcript.turns[{i}].template", turn.template
|
|
76
|
+
if isinstance(turn.count, str):
|
|
77
|
+
yield f"transcript.turns[{i}].count", turn.count
|
|
78
|
+
|
|
79
|
+
for gt_field in (
|
|
80
|
+
"immutable_facts",
|
|
81
|
+
"locked_decisions",
|
|
82
|
+
"forbidden_behaviors",
|
|
83
|
+
"unresolved_items",
|
|
84
|
+
"deferred_items",
|
|
85
|
+
):
|
|
86
|
+
for i, s in enumerate(getattr(template.ground_truth, gt_field)):
|
|
87
|
+
yield f"ground_truth.{gt_field}[{i}]", s
|
|
88
|
+
|
|
89
|
+
for k, v in template.ground_truth.entity_map.items():
|
|
90
|
+
yield f"ground_truth.entity_map[{k!r}].key", k
|
|
91
|
+
yield f"ground_truth.entity_map[{k!r}].value", v
|
|
92
|
+
|
|
93
|
+
for i, item in enumerate(template.evaluation_items):
|
|
94
|
+
yield f"evaluation_items[{i}].prompt", item.prompt
|
|
95
|
+
for k, v in item.expected.items():
|
|
96
|
+
if isinstance(v, str):
|
|
97
|
+
yield f"evaluation_items[{i}].expected[{k!r}]", v
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Deterministic case generation from template + seed.
|
|
2
|
+
|
|
3
|
+
Implements WO-003 per docs/architecture/decisions.md §B1.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from compactbench.engine.difficulty import (
|
|
7
|
+
difficulty_bindings,
|
|
8
|
+
distractor_count_for,
|
|
9
|
+
override_timing_for,
|
|
10
|
+
paraphrase_depth_for,
|
|
11
|
+
)
|
|
12
|
+
from compactbench.engine.distractors import generate_distractor_turns
|
|
13
|
+
from compactbench.engine.errors import EngineError, GenerationError
|
|
14
|
+
from compactbench.engine.generation import generate_case
|
|
15
|
+
from compactbench.engine.seeds import derive_case_seed
|
|
16
|
+
from compactbench.engine.transcript import build_transcript
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"EngineError",
|
|
20
|
+
"GenerationError",
|
|
21
|
+
"build_transcript",
|
|
22
|
+
"derive_case_seed",
|
|
23
|
+
"difficulty_bindings",
|
|
24
|
+
"distractor_count_for",
|
|
25
|
+
"generate_case",
|
|
26
|
+
"generate_distractor_turns",
|
|
27
|
+
"override_timing_for",
|
|
28
|
+
"paraphrase_depth_for",
|
|
29
|
+
]
|