eq-layer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eq_layer/__init__.py +68 -0
- eq_layer/ab_eval.py +302 -0
- eq_layer/actions.py +242 -0
- eq_layer/affect.py +227 -0
- eq_layer/auditor.py +321 -0
- eq_layer/cli.py +174 -0
- eq_layer/coarse_discourse.py +107 -0
- eq_layer/data/intent_train.jsonl +299 -0
- eq_layer/dbdc3.py +197 -0
- eq_layer/dialogbank_iso.py +272 -0
- eq_layer/dialogue_reference.py +393 -0
- eq_layer/intent.py +199 -0
- eq_layer/intent_belief.py +276 -0
- eq_layer/interaction.py +314 -0
- eq_layer/policies.py +295 -0
- eq_layer/preferences.py +125 -0
- eq_layer/response_experiment.py +606 -0
- eq_layer/steer.py +281 -0
- eq_layer/tracker.py +111 -0
- eq_layer/trained_affect.py +246 -0
- eq_layer/trained_breakdown.py +143 -0
- eq_layer/trained_disagreement.py +148 -0
- eq_layer/trained_intent.py +291 -0
- eq_layer/trained_subtext.py +436 -0
- eq_layer-0.1.0.dist-info/METADATA +527 -0
- eq_layer-0.1.0.dist-info/RECORD +29 -0
- eq_layer-0.1.0.dist-info/WHEEL +4 -0
- eq_layer-0.1.0.dist-info/entry_points.txt +2 -0
- eq_layer-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,606 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import random
|
|
6
|
+
import shlex
|
|
7
|
+
import subprocess
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Iterable
|
|
11
|
+
|
|
12
|
+
from .auditor import audit_from_generation_metadata, regeneration_instruction
|
|
13
|
+
from .policies import Selector
|
|
14
|
+
from .steer import Steer
|
|
15
|
+
from .tracker import ConversationTracker
|
|
16
|
+
from .trained_affect import TrainedAffect
|
|
17
|
+
from .trained_intent import TrainedIntent
|
|
18
|
+
from .trained_subtext import TrainedSubtext
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
LEAKAGE_FIELDS = {
|
|
23
|
+
"expected_policy",
|
|
24
|
+
"expected_intent",
|
|
25
|
+
"expected_concepts",
|
|
26
|
+
"annotated",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def validate_experiment_cases(cases: list[dict], *, final: bool = False) -> dict:
|
|
31
|
+
errors: list[str] = []
|
|
32
|
+
seen: set[str] = set()
|
|
33
|
+
role_counts = {"user": 0, "assistant": 0, "system": 0, "other": 0}
|
|
34
|
+
|
|
35
|
+
for index, case in enumerate(cases, start=1):
|
|
36
|
+
case_id = str(case.get("id", "")).strip()
|
|
37
|
+
if not case_id:
|
|
38
|
+
errors.append(f"line {index}: missing id")
|
|
39
|
+
continue
|
|
40
|
+
if case_id in seen:
|
|
41
|
+
errors.append(f"{case_id}: duplicate id")
|
|
42
|
+
seen.add(case_id)
|
|
43
|
+
|
|
44
|
+
messages = case.get("transcript") or case.get("context")
|
|
45
|
+
if not isinstance(messages, list) or not messages:
|
|
46
|
+
errors.append(f"{case_id}: missing transcript/context")
|
|
47
|
+
continue
|
|
48
|
+
if not any(
|
|
49
|
+
isinstance(message, dict) and message.get("role") == "user"
|
|
50
|
+
for message in messages
|
|
51
|
+
):
|
|
52
|
+
errors.append(f"{case_id}: transcript contains no user turn")
|
|
53
|
+
|
|
54
|
+
for message in messages:
|
|
55
|
+
if not isinstance(message, dict):
|
|
56
|
+
errors.append(f"{case_id}: non-object message")
|
|
57
|
+
continue
|
|
58
|
+
role = str(message.get("role", ""))
|
|
59
|
+
role_counts[role if role in role_counts else "other"] += 1
|
|
60
|
+
if not str(message.get("content", "")).strip():
|
|
61
|
+
errors.append(f"{case_id}: empty message content")
|
|
62
|
+
|
|
63
|
+
if final:
|
|
64
|
+
leaked = sorted(LEAKAGE_FIELDS.intersection(case))
|
|
65
|
+
if leaked:
|
|
66
|
+
errors.append(
|
|
67
|
+
f"{case_id}: final held-out case contains development/gold fields: "
|
|
68
|
+
+ ", ".join(leaked)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
if final and len(cases) < 120:
|
|
72
|
+
errors.append(
|
|
73
|
+
f"final run has {len(cases)} cases; protocol target is at least 120 "
|
|
74
|
+
"total cases to make ~90+ non-ties plausible"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
"n_cases": len(cases),
|
|
79
|
+
"unique_ids": len(seen),
|
|
80
|
+
"role_counts": role_counts,
|
|
81
|
+
"final_mode": final,
|
|
82
|
+
"errors": errors,
|
|
83
|
+
"ok": not errors,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def select_stratified_cases(cases: list[dict], limit: int) -> list[dict]:
|
|
88
|
+
"""Select a deterministic, approximately balanced pilot across source strata.
|
|
89
|
+
|
|
90
|
+
Strata are read from case["source"]["stratum"]. Selection preserves the
|
|
91
|
+
within-stratum source order and interleaves strata round-robin so a simple
|
|
92
|
+
prefix cannot accidentally become single-domain.
|
|
93
|
+
"""
|
|
94
|
+
if limit < 1:
|
|
95
|
+
raise ValueError("Stratified limit must be >= 1.")
|
|
96
|
+
|
|
97
|
+
groups: dict[str, list[dict]] = {}
|
|
98
|
+
for case in cases:
|
|
99
|
+
source = case.get("source")
|
|
100
|
+
stratum = source.get("stratum") if isinstance(source, dict) else None
|
|
101
|
+
if not stratum:
|
|
102
|
+
raise ValueError(
|
|
103
|
+
"Stratified selection requires source.stratum on every case."
|
|
104
|
+
)
|
|
105
|
+
groups.setdefault(str(stratum), []).append(case)
|
|
106
|
+
|
|
107
|
+
if not groups:
|
|
108
|
+
raise ValueError("No strata available for stratified selection.")
|
|
109
|
+
|
|
110
|
+
strata = sorted(groups)
|
|
111
|
+
target = min(limit, len(cases))
|
|
112
|
+
base = target // len(strata)
|
|
113
|
+
remainder = target % len(strata)
|
|
114
|
+
quotas = {
|
|
115
|
+
stratum: base + (1 if index < remainder else 0)
|
|
116
|
+
for index, stratum in enumerate(strata)
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
# If a stratum is too small, redistribute its unused quota deterministically.
|
|
120
|
+
deficit = 0
|
|
121
|
+
for stratum in strata:
|
|
122
|
+
available = len(groups[stratum])
|
|
123
|
+
if quotas[stratum] > available:
|
|
124
|
+
deficit += quotas[stratum] - available
|
|
125
|
+
quotas[stratum] = available
|
|
126
|
+
|
|
127
|
+
while deficit > 0:
|
|
128
|
+
progressed = False
|
|
129
|
+
for stratum in strata:
|
|
130
|
+
if quotas[stratum] < len(groups[stratum]):
|
|
131
|
+
quotas[stratum] += 1
|
|
132
|
+
deficit -= 1
|
|
133
|
+
progressed = True
|
|
134
|
+
if deficit == 0:
|
|
135
|
+
break
|
|
136
|
+
if not progressed:
|
|
137
|
+
break
|
|
138
|
+
|
|
139
|
+
selected: list[dict] = []
|
|
140
|
+
max_quota = max(quotas.values(), default=0)
|
|
141
|
+
for index in range(max_quota):
|
|
142
|
+
for stratum in strata:
|
|
143
|
+
if index < quotas[stratum]:
|
|
144
|
+
selected.append(groups[stratum][index])
|
|
145
|
+
|
|
146
|
+
return selected[:target]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def stratum_counts(cases: list[dict]) -> dict[str, int]:
|
|
150
|
+
counts: dict[str, int] = {}
|
|
151
|
+
for case in cases:
|
|
152
|
+
source = case.get("source")
|
|
153
|
+
stratum = source.get("stratum") if isinstance(source, dict) else None
|
|
154
|
+
key = str(stratum) if stratum else "unstratified"
|
|
155
|
+
counts[key] = counts.get(key, 0) + 1
|
|
156
|
+
return counts
|
|
157
|
+
|
|
158
|
+
def stable_case_seed(global_seed: int, case_id: str) -> int:
|
|
159
|
+
digest = hashlib.sha256(f"{global_seed}:{case_id}".encode("utf-8")).digest()
|
|
160
|
+
return int.from_bytes(digest[:4], "big", signed=False)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def file_sha256(path: str | Path) -> str:
|
|
164
|
+
digest = hashlib.sha256()
|
|
165
|
+
with open(path, "rb") as fh:
|
|
166
|
+
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
|
167
|
+
digest.update(chunk)
|
|
168
|
+
return digest.hexdigest()
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@dataclass
|
|
172
|
+
class CommandModel:
|
|
173
|
+
"""Provider-neutral model adapter.
|
|
174
|
+
|
|
175
|
+
The configured command receives one JSON object on stdin and must write
|
|
176
|
+
either plain response text or a JSON object containing `text`/`response`
|
|
177
|
+
on stdout. The same command/model id is used for both experiment arms.
|
|
178
|
+
"""
|
|
179
|
+
|
|
180
|
+
command: tuple[str, ...]
|
|
181
|
+
model_id: str
|
|
182
|
+
temperature: float = 0.0
|
|
183
|
+
timeout_seconds: int = 120
|
|
184
|
+
persistent: bool = False
|
|
185
|
+
_process: subprocess.Popen | None = field(default=None, init=False, repr=False)
|
|
186
|
+
|
|
187
|
+
@classmethod
|
|
188
|
+
def from_shell(
|
|
189
|
+
cls,
|
|
190
|
+
command: str,
|
|
191
|
+
*,
|
|
192
|
+
model_id: str,
|
|
193
|
+
temperature: float = 0.0,
|
|
194
|
+
timeout_seconds: int = 120,
|
|
195
|
+
persistent: bool = False,
|
|
196
|
+
) -> "CommandModel":
|
|
197
|
+
parsed = tuple(shlex.split(command))
|
|
198
|
+
if not parsed:
|
|
199
|
+
raise ValueError("Model command cannot be empty.")
|
|
200
|
+
return cls(
|
|
201
|
+
command=parsed,
|
|
202
|
+
model_id=model_id,
|
|
203
|
+
temperature=temperature,
|
|
204
|
+
timeout_seconds=timeout_seconds,
|
|
205
|
+
persistent=persistent,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
def generate(
|
|
209
|
+
self,
|
|
210
|
+
messages: list[dict],
|
|
211
|
+
*,
|
|
212
|
+
seed: int,
|
|
213
|
+
condition: str,
|
|
214
|
+
) -> str:
|
|
215
|
+
payload = {
|
|
216
|
+
"model_id": self.model_id,
|
|
217
|
+
"seed": seed,
|
|
218
|
+
"temperature": self.temperature,
|
|
219
|
+
"messages": messages,
|
|
220
|
+
}
|
|
221
|
+
if self.persistent:
|
|
222
|
+
raw = self._persistent_generate(payload)
|
|
223
|
+
else:
|
|
224
|
+
completed = subprocess.run(
|
|
225
|
+
list(self.command),
|
|
226
|
+
input=json.dumps(payload, ensure_ascii=False),
|
|
227
|
+
text=True,
|
|
228
|
+
capture_output=True,
|
|
229
|
+
timeout=self.timeout_seconds,
|
|
230
|
+
check=False,
|
|
231
|
+
)
|
|
232
|
+
if completed.returncode != 0:
|
|
233
|
+
raise RuntimeError(
|
|
234
|
+
f"Model command failed ({completed.returncode}): "
|
|
235
|
+
f"{completed.stderr.strip()}"
|
|
236
|
+
)
|
|
237
|
+
raw = completed.stdout.strip()
|
|
238
|
+
|
|
239
|
+
return self._decode_response(raw)
|
|
240
|
+
|
|
241
|
+
def _persistent_generate(self, payload: dict) -> str:
|
|
242
|
+
if self._process is None:
|
|
243
|
+
self._process = subprocess.Popen(
|
|
244
|
+
list(self.command),
|
|
245
|
+
stdin=subprocess.PIPE,
|
|
246
|
+
stdout=subprocess.PIPE,
|
|
247
|
+
stderr=subprocess.PIPE,
|
|
248
|
+
text=True,
|
|
249
|
+
bufsize=1,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
process = self._process
|
|
253
|
+
if process.stdin is None or process.stdout is None:
|
|
254
|
+
raise RuntimeError("Persistent model command pipes are unavailable.")
|
|
255
|
+
if process.poll() is not None:
|
|
256
|
+
stderr = process.stderr.read().strip() if process.stderr else ""
|
|
257
|
+
raise RuntimeError(
|
|
258
|
+
f"Persistent model command exited ({process.returncode}): {stderr}"
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
process.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
|
262
|
+
process.stdin.flush()
|
|
263
|
+
raw = process.stdout.readline().strip()
|
|
264
|
+
if not raw:
|
|
265
|
+
stderr = process.stderr.read().strip() if process.stderr else ""
|
|
266
|
+
raise RuntimeError(
|
|
267
|
+
"Persistent model command returned an empty response. "
|
|
268
|
+
f"stderr={stderr}"
|
|
269
|
+
)
|
|
270
|
+
return raw
|
|
271
|
+
|
|
272
|
+
@staticmethod
|
|
273
|
+
def _decode_response(raw: str) -> str:
|
|
274
|
+
if not raw:
|
|
275
|
+
raise RuntimeError("Model command returned an empty response.")
|
|
276
|
+
|
|
277
|
+
try:
|
|
278
|
+
decoded = json.loads(raw)
|
|
279
|
+
except json.JSONDecodeError:
|
|
280
|
+
return raw
|
|
281
|
+
|
|
282
|
+
if isinstance(decoded, dict):
|
|
283
|
+
for key in ("text", "response"):
|
|
284
|
+
value = decoded.get(key)
|
|
285
|
+
if isinstance(value, str) and value.strip():
|
|
286
|
+
return value.strip()
|
|
287
|
+
|
|
288
|
+
raise RuntimeError(
|
|
289
|
+
"Model command JSON must contain a non-empty 'text' or 'response'."
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
def close(self) -> None:
|
|
293
|
+
if self._process is None:
|
|
294
|
+
return
|
|
295
|
+
process = self._process
|
|
296
|
+
self._process = None
|
|
297
|
+
if process.stdin is not None:
|
|
298
|
+
process.stdin.close()
|
|
299
|
+
try:
|
|
300
|
+
process.wait(timeout=10)
|
|
301
|
+
except subprocess.TimeoutExpired:
|
|
302
|
+
process.terminate()
|
|
303
|
+
try:
|
|
304
|
+
process.wait(timeout=5)
|
|
305
|
+
except subprocess.TimeoutExpired:
|
|
306
|
+
process.kill()
|
|
307
|
+
process.wait(timeout=5)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
@dataclass
|
|
311
|
+
class FullEQPipeline:
|
|
312
|
+
tracker: ConversationTracker
|
|
313
|
+
intent: TrainedIntent
|
|
314
|
+
selector: Selector
|
|
315
|
+
|
|
316
|
+
@classmethod
|
|
317
|
+
def load(
|
|
318
|
+
cls,
|
|
319
|
+
affect_model: str | Path,
|
|
320
|
+
subtext_model: str | Path,
|
|
321
|
+
) -> "FullEQPipeline":
|
|
322
|
+
return cls(
|
|
323
|
+
tracker=ConversationTracker(
|
|
324
|
+
affect=TrainedAffect.load(affect_model),
|
|
325
|
+
subtext=TrainedSubtext.load(subtext_model),
|
|
326
|
+
),
|
|
327
|
+
intent=TrainedIntent.from_bundled(),
|
|
328
|
+
selector=Selector(),
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
def steer_messages(
|
|
332
|
+
self,
|
|
333
|
+
messages: list[dict],
|
|
334
|
+
*,
|
|
335
|
+
annotated: dict | None = None,
|
|
336
|
+
) -> tuple[list[dict], dict]:
|
|
337
|
+
tracked = self.tracker.infer(
|
|
338
|
+
messages,
|
|
339
|
+
turn_index=len(messages),
|
|
340
|
+
annotated=annotated or {},
|
|
341
|
+
)
|
|
342
|
+
raw_intent = self.intent.infer(messages)
|
|
343
|
+
intent = self.intent.reconcile_with_dialogue_evidence(
|
|
344
|
+
raw_intent,
|
|
345
|
+
messages,
|
|
346
|
+
tracked.subtext.signals,
|
|
347
|
+
)
|
|
348
|
+
selection = self.selector.select(tracked.state, intent)
|
|
349
|
+
steer = Steer.build(
|
|
350
|
+
selection.policy,
|
|
351
|
+
intent,
|
|
352
|
+
repair=tracked.repair,
|
|
353
|
+
interaction_quality=tracked.interaction_quality,
|
|
354
|
+
reference=tracked.reference,
|
|
355
|
+
)
|
|
356
|
+
instruction = steer.system_instruction()
|
|
357
|
+
|
|
358
|
+
steered = [{"role": "system", "content": instruction}, *messages]
|
|
359
|
+
metadata = {
|
|
360
|
+
"policy": selection.policy.name,
|
|
361
|
+
"factored_action": (
|
|
362
|
+
steer.action.to_dict()
|
|
363
|
+
if steer.action is not None
|
|
364
|
+
else None
|
|
365
|
+
),
|
|
366
|
+
"register": selection.policy.register.value,
|
|
367
|
+
"intent_kind": intent.kind,
|
|
368
|
+
"intent_confidence": intent.confidence,
|
|
369
|
+
"intent_kind_raw": raw_intent.kind,
|
|
370
|
+
"intent_belief_raw": (
|
|
371
|
+
raw_intent.belief.as_dict()
|
|
372
|
+
if raw_intent.belief is not None
|
|
373
|
+
else None
|
|
374
|
+
),
|
|
375
|
+
"intent_risk_decision_raw": (
|
|
376
|
+
{
|
|
377
|
+
"action": raw_intent.risk_decision.action,
|
|
378
|
+
"selected_intent": raw_intent.risk_decision.selected_intent,
|
|
379
|
+
"expected_loss": raw_intent.risk_decision.expected_loss,
|
|
380
|
+
"losses": raw_intent.risk_decision.losses_dict(),
|
|
381
|
+
"rationale": raw_intent.risk_decision.rationale,
|
|
382
|
+
}
|
|
383
|
+
if raw_intent.risk_decision is not None
|
|
384
|
+
else None
|
|
385
|
+
),
|
|
386
|
+
"intent_belief": (
|
|
387
|
+
intent.belief.as_dict()
|
|
388
|
+
if intent.belief is not None
|
|
389
|
+
else None
|
|
390
|
+
),
|
|
391
|
+
"intent_belief_entropy": (
|
|
392
|
+
intent.belief.normalized_entropy
|
|
393
|
+
if intent.belief is not None
|
|
394
|
+
else None
|
|
395
|
+
),
|
|
396
|
+
"intent_risk_decision": (
|
|
397
|
+
{
|
|
398
|
+
"action": intent.risk_decision.action,
|
|
399
|
+
"selected_intent": intent.risk_decision.selected_intent,
|
|
400
|
+
"expected_loss": intent.risk_decision.expected_loss,
|
|
401
|
+
"losses": intent.risk_decision.losses_dict(),
|
|
402
|
+
"rationale": intent.risk_decision.rationale,
|
|
403
|
+
}
|
|
404
|
+
if intent.risk_decision is not None
|
|
405
|
+
else None
|
|
406
|
+
),
|
|
407
|
+
"subtext": tracked.subtext.label,
|
|
408
|
+
"subtext_evidence_level": tracked.subtext.evidence_level,
|
|
409
|
+
"stance": tracked.stance.label,
|
|
410
|
+
"stance_evidence_level": tracked.stance.evidence_level,
|
|
411
|
+
"repair": {
|
|
412
|
+
"active": tracked.repair.active,
|
|
413
|
+
"kind": tracked.repair.kind,
|
|
414
|
+
"target_turn_index": tracked.repair.target_turn_index,
|
|
415
|
+
"target_excerpt": tracked.repair.target_excerpt,
|
|
416
|
+
"correction_excerpt": tracked.repair.correction_excerpt,
|
|
417
|
+
"replacement_excerpt": tracked.repair.replacement_excerpt,
|
|
418
|
+
"replacement_explicit": tracked.repair.replacement_explicit,
|
|
419
|
+
"repeated": tracked.repair.repeated,
|
|
420
|
+
"recent_repair_count": tracked.repair.recent_repair_count,
|
|
421
|
+
"confidence": tracked.repair.confidence,
|
|
422
|
+
"evidence_level": tracked.repair.evidence_level,
|
|
423
|
+
"evidence": list(tracked.repair.evidence),
|
|
424
|
+
},
|
|
425
|
+
"interaction_quality": {
|
|
426
|
+
"current": tracked.interaction_quality.current,
|
|
427
|
+
"delta": tracked.interaction_quality.delta,
|
|
428
|
+
"repeated_failure_count": tracked.interaction_quality.repeated_failure_count,
|
|
429
|
+
"unresolved_repair_count": tracked.interaction_quality.unresolved_repair_count,
|
|
430
|
+
"clarification_count": tracked.interaction_quality.clarification_count,
|
|
431
|
+
"evidence_level": tracked.interaction_quality.evidence_level,
|
|
432
|
+
"evidence": list(tracked.interaction_quality.evidence),
|
|
433
|
+
},
|
|
434
|
+
"dialogue_reference": {
|
|
435
|
+
"active": tracked.reference.active,
|
|
436
|
+
"form": tracked.reference.form,
|
|
437
|
+
"resolved": tracked.reference.resolved,
|
|
438
|
+
"target_turn_index": tracked.reference.target_turn_index,
|
|
439
|
+
"target_role": tracked.reference.target_role,
|
|
440
|
+
"target_excerpt": tracked.reference.target_excerpt,
|
|
441
|
+
"anchor_turn_index": tracked.reference.anchor_turn_index,
|
|
442
|
+
"anchor_excerpt": tracked.reference.anchor_excerpt,
|
|
443
|
+
"requires_clarification": tracked.reference.requires_clarification,
|
|
444
|
+
"confidence": tracked.reference.confidence,
|
|
445
|
+
"evidence_level": tracked.reference.evidence_level,
|
|
446
|
+
"evidence": list(tracked.reference.evidence),
|
|
447
|
+
"candidate_stack": [
|
|
448
|
+
{
|
|
449
|
+
"turn_index": candidate.turn_index,
|
|
450
|
+
"role": candidate.role,
|
|
451
|
+
"kind": candidate.kind,
|
|
452
|
+
"excerpt": candidate.excerpt,
|
|
453
|
+
"evidence": list(candidate.evidence),
|
|
454
|
+
}
|
|
455
|
+
for candidate in tracked.reference.candidate_stack
|
|
456
|
+
],
|
|
457
|
+
},
|
|
458
|
+
"selection_rationale": selection.rationale,
|
|
459
|
+
}
|
|
460
|
+
return steered, metadata
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def generate_pairs(
|
|
464
|
+
cases: Iterable[dict],
|
|
465
|
+
*,
|
|
466
|
+
model: CommandModel,
|
|
467
|
+
pipeline: FullEQPipeline,
|
|
468
|
+
seed: int = 0,
|
|
469
|
+
allow_annotations: bool = False,
|
|
470
|
+
auditor_regenerations: int = 0,
|
|
471
|
+
) -> list[dict]:
|
|
472
|
+
if auditor_regenerations not in {0, 1}:
|
|
473
|
+
raise ValueError("auditor_regenerations must be 0 or 1.")
|
|
474
|
+
|
|
475
|
+
rng = random.Random(seed)
|
|
476
|
+
output: list[dict] = []
|
|
477
|
+
seen: set[str] = set()
|
|
478
|
+
|
|
479
|
+
try:
|
|
480
|
+
for case in cases:
|
|
481
|
+
case_id = str(case["id"])
|
|
482
|
+
if case_id in seen:
|
|
483
|
+
raise ValueError(f"Duplicate experiment case id: {case_id}")
|
|
484
|
+
seen.add(case_id)
|
|
485
|
+
|
|
486
|
+
messages = list(case.get("transcript") or case.get("context") or [])
|
|
487
|
+
if not messages:
|
|
488
|
+
raise ValueError(f"Experiment case has no transcript/context: {case_id}")
|
|
489
|
+
|
|
490
|
+
annotated = case.get("annotated") if allow_annotations else None
|
|
491
|
+
eq_messages, eq_meta = pipeline.steer_messages(
|
|
492
|
+
messages,
|
|
493
|
+
annotated=annotated,
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
pair_seed = stable_case_seed(seed, case_id)
|
|
497
|
+
conditions = ["baseline", "eq"]
|
|
498
|
+
rng.shuffle(conditions)
|
|
499
|
+
responses: dict[str, str] = {}
|
|
500
|
+
|
|
501
|
+
for condition in conditions:
|
|
502
|
+
condition_messages = messages if condition == "baseline" else eq_messages
|
|
503
|
+
responses[condition] = model.generate(
|
|
504
|
+
condition_messages,
|
|
505
|
+
seed=pair_seed,
|
|
506
|
+
condition=condition,
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
initial_eq = responses["eq"]
|
|
510
|
+
initial_audit = audit_from_generation_metadata(
|
|
511
|
+
initial_eq,
|
|
512
|
+
messages,
|
|
513
|
+
eq_meta,
|
|
514
|
+
)
|
|
515
|
+
final_audit = initial_audit
|
|
516
|
+
regeneration_count = 0
|
|
517
|
+
|
|
518
|
+
if auditor_regenerations and initial_audit.should_regenerate:
|
|
519
|
+
instruction = regeneration_instruction(initial_audit)
|
|
520
|
+
if instruction:
|
|
521
|
+
retry_messages = [
|
|
522
|
+
eq_messages[0],
|
|
523
|
+
{"role": "system", "content": instruction},
|
|
524
|
+
*eq_messages[1:],
|
|
525
|
+
]
|
|
526
|
+
responses["eq"] = model.generate(
|
|
527
|
+
retry_messages,
|
|
528
|
+
seed=pair_seed,
|
|
529
|
+
condition="eq_audit_retry",
|
|
530
|
+
)
|
|
531
|
+
regeneration_count = 1
|
|
532
|
+
final_audit = audit_from_generation_metadata(
|
|
533
|
+
responses["eq"],
|
|
534
|
+
messages,
|
|
535
|
+
eq_meta,
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
record = {
|
|
539
|
+
"id": case_id,
|
|
540
|
+
"source": case.get("source"),
|
|
541
|
+
"context": messages,
|
|
542
|
+
"baseline": responses["baseline"],
|
|
543
|
+
"eq": responses["eq"],
|
|
544
|
+
"generation": {
|
|
545
|
+
"model_id": model.model_id,
|
|
546
|
+
"temperature": model.temperature,
|
|
547
|
+
"pair_seed": pair_seed,
|
|
548
|
+
"order": conditions,
|
|
549
|
+
"same_model_both_arms": True,
|
|
550
|
+
"annotations_used": bool(annotated),
|
|
551
|
+
"auditor_regenerations_allowed": auditor_regenerations,
|
|
552
|
+
"audit_regeneration_count": regeneration_count,
|
|
553
|
+
"response_audit_initial": initial_audit.to_dict(),
|
|
554
|
+
"response_audit": final_audit.to_dict(),
|
|
555
|
+
**eq_meta,
|
|
556
|
+
},
|
|
557
|
+
}
|
|
558
|
+
if regeneration_count:
|
|
559
|
+
record["eq_initial"] = initial_eq
|
|
560
|
+
|
|
561
|
+
output.append(record)
|
|
562
|
+
finally:
|
|
563
|
+
model.close()
|
|
564
|
+
|
|
565
|
+
return output
|
|
566
|
+
|
|
567
|
+
def experiment_manifest(
|
|
568
|
+
*,
|
|
569
|
+
model: CommandModel,
|
|
570
|
+
affect_model: str | Path,
|
|
571
|
+
subtext_model: str | Path,
|
|
572
|
+
cases_path: str | Path,
|
|
573
|
+
global_seed: int,
|
|
574
|
+
n_cases: int,
|
|
575
|
+
allow_annotations: bool,
|
|
576
|
+
git_commit: str | None = None,
|
|
577
|
+
auditor_regenerations: int = 0,
|
|
578
|
+
) -> dict:
|
|
579
|
+
return {
|
|
580
|
+
"design": "same-base-model paired baseline-vs-eq-layer",
|
|
581
|
+
"model_id": model.model_id,
|
|
582
|
+
"model_command_executable": model.command[0],
|
|
583
|
+
"model_command_argv_sha256": hashlib.sha256(
|
|
584
|
+
"\0".join(model.command).encode("utf-8")
|
|
585
|
+
).hexdigest(),
|
|
586
|
+
"temperature": model.temperature,
|
|
587
|
+
"global_seed": global_seed,
|
|
588
|
+
"n_cases": n_cases,
|
|
589
|
+
"annotations_used": allow_annotations,
|
|
590
|
+
"response_auditor_mode": (
|
|
591
|
+
"one-shot-regeneration"
|
|
592
|
+
if auditor_regenerations
|
|
593
|
+
else "passive-metadata-only"
|
|
594
|
+
),
|
|
595
|
+
"auditor_regenerations_allowed": auditor_regenerations,
|
|
596
|
+
"cases_path": str(cases_path),
|
|
597
|
+
"cases_sha256": file_sha256(cases_path),
|
|
598
|
+
"affect_model_sha256": file_sha256(affect_model),
|
|
599
|
+
"subtext_model_sha256": file_sha256(subtext_model),
|
|
600
|
+
"eq_layer_git_commit": git_commit,
|
|
601
|
+
"pairing_rule": "same model id, same sampling temperature, same per-case seed",
|
|
602
|
+
"arm_difference": (
|
|
603
|
+
"EQ arm prepends only the EQ-Layer system control instruction; "
|
|
604
|
+
"the original transcript is preserved identically in both arms."
|
|
605
|
+
),
|
|
606
|
+
}
|