eq-layer 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,606 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import json
5
+ import random
6
+ import shlex
7
+ import subprocess
8
+ from dataclasses import dataclass, field
9
+ from pathlib import Path
10
+ from typing import Iterable
11
+
12
+ from .auditor import audit_from_generation_metadata, regeneration_instruction
13
+ from .policies import Selector
14
+ from .steer import Steer
15
+ from .tracker import ConversationTracker
16
+ from .trained_affect import TrainedAffect
17
+ from .trained_intent import TrainedIntent
18
+ from .trained_subtext import TrainedSubtext
19
+
20
+
21
+
22
+ LEAKAGE_FIELDS = {
23
+ "expected_policy",
24
+ "expected_intent",
25
+ "expected_concepts",
26
+ "annotated",
27
+ }
28
+
29
+
30
+ def validate_experiment_cases(cases: list[dict], *, final: bool = False) -> dict:
31
+ errors: list[str] = []
32
+ seen: set[str] = set()
33
+ role_counts = {"user": 0, "assistant": 0, "system": 0, "other": 0}
34
+
35
+ for index, case in enumerate(cases, start=1):
36
+ case_id = str(case.get("id", "")).strip()
37
+ if not case_id:
38
+ errors.append(f"line {index}: missing id")
39
+ continue
40
+ if case_id in seen:
41
+ errors.append(f"{case_id}: duplicate id")
42
+ seen.add(case_id)
43
+
44
+ messages = case.get("transcript") or case.get("context")
45
+ if not isinstance(messages, list) or not messages:
46
+ errors.append(f"{case_id}: missing transcript/context")
47
+ continue
48
+ if not any(
49
+ isinstance(message, dict) and message.get("role") == "user"
50
+ for message in messages
51
+ ):
52
+ errors.append(f"{case_id}: transcript contains no user turn")
53
+
54
+ for message in messages:
55
+ if not isinstance(message, dict):
56
+ errors.append(f"{case_id}: non-object message")
57
+ continue
58
+ role = str(message.get("role", ""))
59
+ role_counts[role if role in role_counts else "other"] += 1
60
+ if not str(message.get("content", "")).strip():
61
+ errors.append(f"{case_id}: empty message content")
62
+
63
+ if final:
64
+ leaked = sorted(LEAKAGE_FIELDS.intersection(case))
65
+ if leaked:
66
+ errors.append(
67
+ f"{case_id}: final held-out case contains development/gold fields: "
68
+ + ", ".join(leaked)
69
+ )
70
+
71
+ if final and len(cases) < 120:
72
+ errors.append(
73
+ f"final run has {len(cases)} cases; protocol target is at least 120 "
74
+ "total cases to make ~90+ non-ties plausible"
75
+ )
76
+
77
+ return {
78
+ "n_cases": len(cases),
79
+ "unique_ids": len(seen),
80
+ "role_counts": role_counts,
81
+ "final_mode": final,
82
+ "errors": errors,
83
+ "ok": not errors,
84
+ }
85
+
86
+
87
+ def select_stratified_cases(cases: list[dict], limit: int) -> list[dict]:
88
+ """Select a deterministic, approximately balanced pilot across source strata.
89
+
90
+ Strata are read from case["source"]["stratum"]. Selection preserves the
91
+ within-stratum source order and interleaves strata round-robin so a simple
92
+ prefix cannot accidentally become single-domain.
93
+ """
94
+ if limit < 1:
95
+ raise ValueError("Stratified limit must be >= 1.")
96
+
97
+ groups: dict[str, list[dict]] = {}
98
+ for case in cases:
99
+ source = case.get("source")
100
+ stratum = source.get("stratum") if isinstance(source, dict) else None
101
+ if not stratum:
102
+ raise ValueError(
103
+ "Stratified selection requires source.stratum on every case."
104
+ )
105
+ groups.setdefault(str(stratum), []).append(case)
106
+
107
+ if not groups:
108
+ raise ValueError("No strata available for stratified selection.")
109
+
110
+ strata = sorted(groups)
111
+ target = min(limit, len(cases))
112
+ base = target // len(strata)
113
+ remainder = target % len(strata)
114
+ quotas = {
115
+ stratum: base + (1 if index < remainder else 0)
116
+ for index, stratum in enumerate(strata)
117
+ }
118
+
119
+ # If a stratum is too small, redistribute its unused quota deterministically.
120
+ deficit = 0
121
+ for stratum in strata:
122
+ available = len(groups[stratum])
123
+ if quotas[stratum] > available:
124
+ deficit += quotas[stratum] - available
125
+ quotas[stratum] = available
126
+
127
+ while deficit > 0:
128
+ progressed = False
129
+ for stratum in strata:
130
+ if quotas[stratum] < len(groups[stratum]):
131
+ quotas[stratum] += 1
132
+ deficit -= 1
133
+ progressed = True
134
+ if deficit == 0:
135
+ break
136
+ if not progressed:
137
+ break
138
+
139
+ selected: list[dict] = []
140
+ max_quota = max(quotas.values(), default=0)
141
+ for index in range(max_quota):
142
+ for stratum in strata:
143
+ if index < quotas[stratum]:
144
+ selected.append(groups[stratum][index])
145
+
146
+ return selected[:target]
147
+
148
+
149
+ def stratum_counts(cases: list[dict]) -> dict[str, int]:
150
+ counts: dict[str, int] = {}
151
+ for case in cases:
152
+ source = case.get("source")
153
+ stratum = source.get("stratum") if isinstance(source, dict) else None
154
+ key = str(stratum) if stratum else "unstratified"
155
+ counts[key] = counts.get(key, 0) + 1
156
+ return counts
157
+
158
+ def stable_case_seed(global_seed: int, case_id: str) -> int:
159
+ digest = hashlib.sha256(f"{global_seed}:{case_id}".encode("utf-8")).digest()
160
+ return int.from_bytes(digest[:4], "big", signed=False)
161
+
162
+
163
+ def file_sha256(path: str | Path) -> str:
164
+ digest = hashlib.sha256()
165
+ with open(path, "rb") as fh:
166
+ for chunk in iter(lambda: fh.read(1024 * 1024), b""):
167
+ digest.update(chunk)
168
+ return digest.hexdigest()
169
+
170
+
171
+ @dataclass
172
+ class CommandModel:
173
+ """Provider-neutral model adapter.
174
+
175
+ The configured command receives one JSON object on stdin and must write
176
+ either plain response text or a JSON object containing `text`/`response`
177
+ on stdout. The same command/model id is used for both experiment arms.
178
+ """
179
+
180
+ command: tuple[str, ...]
181
+ model_id: str
182
+ temperature: float = 0.0
183
+ timeout_seconds: int = 120
184
+ persistent: bool = False
185
+ _process: subprocess.Popen | None = field(default=None, init=False, repr=False)
186
+
187
+ @classmethod
188
+ def from_shell(
189
+ cls,
190
+ command: str,
191
+ *,
192
+ model_id: str,
193
+ temperature: float = 0.0,
194
+ timeout_seconds: int = 120,
195
+ persistent: bool = False,
196
+ ) -> "CommandModel":
197
+ parsed = tuple(shlex.split(command))
198
+ if not parsed:
199
+ raise ValueError("Model command cannot be empty.")
200
+ return cls(
201
+ command=parsed,
202
+ model_id=model_id,
203
+ temperature=temperature,
204
+ timeout_seconds=timeout_seconds,
205
+ persistent=persistent,
206
+ )
207
+
208
+ def generate(
209
+ self,
210
+ messages: list[dict],
211
+ *,
212
+ seed: int,
213
+ condition: str,
214
+ ) -> str:
215
+ payload = {
216
+ "model_id": self.model_id,
217
+ "seed": seed,
218
+ "temperature": self.temperature,
219
+ "messages": messages,
220
+ }
221
+ if self.persistent:
222
+ raw = self._persistent_generate(payload)
223
+ else:
224
+ completed = subprocess.run(
225
+ list(self.command),
226
+ input=json.dumps(payload, ensure_ascii=False),
227
+ text=True,
228
+ capture_output=True,
229
+ timeout=self.timeout_seconds,
230
+ check=False,
231
+ )
232
+ if completed.returncode != 0:
233
+ raise RuntimeError(
234
+ f"Model command failed ({completed.returncode}): "
235
+ f"{completed.stderr.strip()}"
236
+ )
237
+ raw = completed.stdout.strip()
238
+
239
+ return self._decode_response(raw)
240
+
241
+ def _persistent_generate(self, payload: dict) -> str:
242
+ if self._process is None:
243
+ self._process = subprocess.Popen(
244
+ list(self.command),
245
+ stdin=subprocess.PIPE,
246
+ stdout=subprocess.PIPE,
247
+ stderr=subprocess.PIPE,
248
+ text=True,
249
+ bufsize=1,
250
+ )
251
+
252
+ process = self._process
253
+ if process.stdin is None or process.stdout is None:
254
+ raise RuntimeError("Persistent model command pipes are unavailable.")
255
+ if process.poll() is not None:
256
+ stderr = process.stderr.read().strip() if process.stderr else ""
257
+ raise RuntimeError(
258
+ f"Persistent model command exited ({process.returncode}): {stderr}"
259
+ )
260
+
261
+ process.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n")
262
+ process.stdin.flush()
263
+ raw = process.stdout.readline().strip()
264
+ if not raw:
265
+ stderr = process.stderr.read().strip() if process.stderr else ""
266
+ raise RuntimeError(
267
+ "Persistent model command returned an empty response. "
268
+ f"stderr={stderr}"
269
+ )
270
+ return raw
271
+
272
+ @staticmethod
273
+ def _decode_response(raw: str) -> str:
274
+ if not raw:
275
+ raise RuntimeError("Model command returned an empty response.")
276
+
277
+ try:
278
+ decoded = json.loads(raw)
279
+ except json.JSONDecodeError:
280
+ return raw
281
+
282
+ if isinstance(decoded, dict):
283
+ for key in ("text", "response"):
284
+ value = decoded.get(key)
285
+ if isinstance(value, str) and value.strip():
286
+ return value.strip()
287
+
288
+ raise RuntimeError(
289
+ "Model command JSON must contain a non-empty 'text' or 'response'."
290
+ )
291
+
292
+ def close(self) -> None:
293
+ if self._process is None:
294
+ return
295
+ process = self._process
296
+ self._process = None
297
+ if process.stdin is not None:
298
+ process.stdin.close()
299
+ try:
300
+ process.wait(timeout=10)
301
+ except subprocess.TimeoutExpired:
302
+ process.terminate()
303
+ try:
304
+ process.wait(timeout=5)
305
+ except subprocess.TimeoutExpired:
306
+ process.kill()
307
+ process.wait(timeout=5)
308
+
309
+
310
+ @dataclass
311
+ class FullEQPipeline:
312
+ tracker: ConversationTracker
313
+ intent: TrainedIntent
314
+ selector: Selector
315
+
316
+ @classmethod
317
+ def load(
318
+ cls,
319
+ affect_model: str | Path,
320
+ subtext_model: str | Path,
321
+ ) -> "FullEQPipeline":
322
+ return cls(
323
+ tracker=ConversationTracker(
324
+ affect=TrainedAffect.load(affect_model),
325
+ subtext=TrainedSubtext.load(subtext_model),
326
+ ),
327
+ intent=TrainedIntent.from_bundled(),
328
+ selector=Selector(),
329
+ )
330
+
331
+ def steer_messages(
332
+ self,
333
+ messages: list[dict],
334
+ *,
335
+ annotated: dict | None = None,
336
+ ) -> tuple[list[dict], dict]:
337
+ tracked = self.tracker.infer(
338
+ messages,
339
+ turn_index=len(messages),
340
+ annotated=annotated or {},
341
+ )
342
+ raw_intent = self.intent.infer(messages)
343
+ intent = self.intent.reconcile_with_dialogue_evidence(
344
+ raw_intent,
345
+ messages,
346
+ tracked.subtext.signals,
347
+ )
348
+ selection = self.selector.select(tracked.state, intent)
349
+ steer = Steer.build(
350
+ selection.policy,
351
+ intent,
352
+ repair=tracked.repair,
353
+ interaction_quality=tracked.interaction_quality,
354
+ reference=tracked.reference,
355
+ )
356
+ instruction = steer.system_instruction()
357
+
358
+ steered = [{"role": "system", "content": instruction}, *messages]
359
+ metadata = {
360
+ "policy": selection.policy.name,
361
+ "factored_action": (
362
+ steer.action.to_dict()
363
+ if steer.action is not None
364
+ else None
365
+ ),
366
+ "register": selection.policy.register.value,
367
+ "intent_kind": intent.kind,
368
+ "intent_confidence": intent.confidence,
369
+ "intent_kind_raw": raw_intent.kind,
370
+ "intent_belief_raw": (
371
+ raw_intent.belief.as_dict()
372
+ if raw_intent.belief is not None
373
+ else None
374
+ ),
375
+ "intent_risk_decision_raw": (
376
+ {
377
+ "action": raw_intent.risk_decision.action,
378
+ "selected_intent": raw_intent.risk_decision.selected_intent,
379
+ "expected_loss": raw_intent.risk_decision.expected_loss,
380
+ "losses": raw_intent.risk_decision.losses_dict(),
381
+ "rationale": raw_intent.risk_decision.rationale,
382
+ }
383
+ if raw_intent.risk_decision is not None
384
+ else None
385
+ ),
386
+ "intent_belief": (
387
+ intent.belief.as_dict()
388
+ if intent.belief is not None
389
+ else None
390
+ ),
391
+ "intent_belief_entropy": (
392
+ intent.belief.normalized_entropy
393
+ if intent.belief is not None
394
+ else None
395
+ ),
396
+ "intent_risk_decision": (
397
+ {
398
+ "action": intent.risk_decision.action,
399
+ "selected_intent": intent.risk_decision.selected_intent,
400
+ "expected_loss": intent.risk_decision.expected_loss,
401
+ "losses": intent.risk_decision.losses_dict(),
402
+ "rationale": intent.risk_decision.rationale,
403
+ }
404
+ if intent.risk_decision is not None
405
+ else None
406
+ ),
407
+ "subtext": tracked.subtext.label,
408
+ "subtext_evidence_level": tracked.subtext.evidence_level,
409
+ "stance": tracked.stance.label,
410
+ "stance_evidence_level": tracked.stance.evidence_level,
411
+ "repair": {
412
+ "active": tracked.repair.active,
413
+ "kind": tracked.repair.kind,
414
+ "target_turn_index": tracked.repair.target_turn_index,
415
+ "target_excerpt": tracked.repair.target_excerpt,
416
+ "correction_excerpt": tracked.repair.correction_excerpt,
417
+ "replacement_excerpt": tracked.repair.replacement_excerpt,
418
+ "replacement_explicit": tracked.repair.replacement_explicit,
419
+ "repeated": tracked.repair.repeated,
420
+ "recent_repair_count": tracked.repair.recent_repair_count,
421
+ "confidence": tracked.repair.confidence,
422
+ "evidence_level": tracked.repair.evidence_level,
423
+ "evidence": list(tracked.repair.evidence),
424
+ },
425
+ "interaction_quality": {
426
+ "current": tracked.interaction_quality.current,
427
+ "delta": tracked.interaction_quality.delta,
428
+ "repeated_failure_count": tracked.interaction_quality.repeated_failure_count,
429
+ "unresolved_repair_count": tracked.interaction_quality.unresolved_repair_count,
430
+ "clarification_count": tracked.interaction_quality.clarification_count,
431
+ "evidence_level": tracked.interaction_quality.evidence_level,
432
+ "evidence": list(tracked.interaction_quality.evidence),
433
+ },
434
+ "dialogue_reference": {
435
+ "active": tracked.reference.active,
436
+ "form": tracked.reference.form,
437
+ "resolved": tracked.reference.resolved,
438
+ "target_turn_index": tracked.reference.target_turn_index,
439
+ "target_role": tracked.reference.target_role,
440
+ "target_excerpt": tracked.reference.target_excerpt,
441
+ "anchor_turn_index": tracked.reference.anchor_turn_index,
442
+ "anchor_excerpt": tracked.reference.anchor_excerpt,
443
+ "requires_clarification": tracked.reference.requires_clarification,
444
+ "confidence": tracked.reference.confidence,
445
+ "evidence_level": tracked.reference.evidence_level,
446
+ "evidence": list(tracked.reference.evidence),
447
+ "candidate_stack": [
448
+ {
449
+ "turn_index": candidate.turn_index,
450
+ "role": candidate.role,
451
+ "kind": candidate.kind,
452
+ "excerpt": candidate.excerpt,
453
+ "evidence": list(candidate.evidence),
454
+ }
455
+ for candidate in tracked.reference.candidate_stack
456
+ ],
457
+ },
458
+ "selection_rationale": selection.rationale,
459
+ }
460
+ return steered, metadata
461
+
462
+
463
+ def generate_pairs(
464
+ cases: Iterable[dict],
465
+ *,
466
+ model: CommandModel,
467
+ pipeline: FullEQPipeline,
468
+ seed: int = 0,
469
+ allow_annotations: bool = False,
470
+ auditor_regenerations: int = 0,
471
+ ) -> list[dict]:
472
+ if auditor_regenerations not in {0, 1}:
473
+ raise ValueError("auditor_regenerations must be 0 or 1.")
474
+
475
+ rng = random.Random(seed)
476
+ output: list[dict] = []
477
+ seen: set[str] = set()
478
+
479
+ try:
480
+ for case in cases:
481
+ case_id = str(case["id"])
482
+ if case_id in seen:
483
+ raise ValueError(f"Duplicate experiment case id: {case_id}")
484
+ seen.add(case_id)
485
+
486
+ messages = list(case.get("transcript") or case.get("context") or [])
487
+ if not messages:
488
+ raise ValueError(f"Experiment case has no transcript/context: {case_id}")
489
+
490
+ annotated = case.get("annotated") if allow_annotations else None
491
+ eq_messages, eq_meta = pipeline.steer_messages(
492
+ messages,
493
+ annotated=annotated,
494
+ )
495
+
496
+ pair_seed = stable_case_seed(seed, case_id)
497
+ conditions = ["baseline", "eq"]
498
+ rng.shuffle(conditions)
499
+ responses: dict[str, str] = {}
500
+
501
+ for condition in conditions:
502
+ condition_messages = messages if condition == "baseline" else eq_messages
503
+ responses[condition] = model.generate(
504
+ condition_messages,
505
+ seed=pair_seed,
506
+ condition=condition,
507
+ )
508
+
509
+ initial_eq = responses["eq"]
510
+ initial_audit = audit_from_generation_metadata(
511
+ initial_eq,
512
+ messages,
513
+ eq_meta,
514
+ )
515
+ final_audit = initial_audit
516
+ regeneration_count = 0
517
+
518
+ if auditor_regenerations and initial_audit.should_regenerate:
519
+ instruction = regeneration_instruction(initial_audit)
520
+ if instruction:
521
+ retry_messages = [
522
+ eq_messages[0],
523
+ {"role": "system", "content": instruction},
524
+ *eq_messages[1:],
525
+ ]
526
+ responses["eq"] = model.generate(
527
+ retry_messages,
528
+ seed=pair_seed,
529
+ condition="eq_audit_retry",
530
+ )
531
+ regeneration_count = 1
532
+ final_audit = audit_from_generation_metadata(
533
+ responses["eq"],
534
+ messages,
535
+ eq_meta,
536
+ )
537
+
538
+ record = {
539
+ "id": case_id,
540
+ "source": case.get("source"),
541
+ "context": messages,
542
+ "baseline": responses["baseline"],
543
+ "eq": responses["eq"],
544
+ "generation": {
545
+ "model_id": model.model_id,
546
+ "temperature": model.temperature,
547
+ "pair_seed": pair_seed,
548
+ "order": conditions,
549
+ "same_model_both_arms": True,
550
+ "annotations_used": bool(annotated),
551
+ "auditor_regenerations_allowed": auditor_regenerations,
552
+ "audit_regeneration_count": regeneration_count,
553
+ "response_audit_initial": initial_audit.to_dict(),
554
+ "response_audit": final_audit.to_dict(),
555
+ **eq_meta,
556
+ },
557
+ }
558
+ if regeneration_count:
559
+ record["eq_initial"] = initial_eq
560
+
561
+ output.append(record)
562
+ finally:
563
+ model.close()
564
+
565
+ return output
566
+
567
+ def experiment_manifest(
568
+ *,
569
+ model: CommandModel,
570
+ affect_model: str | Path,
571
+ subtext_model: str | Path,
572
+ cases_path: str | Path,
573
+ global_seed: int,
574
+ n_cases: int,
575
+ allow_annotations: bool,
576
+ git_commit: str | None = None,
577
+ auditor_regenerations: int = 0,
578
+ ) -> dict:
579
+ return {
580
+ "design": "same-base-model paired baseline-vs-eq-layer",
581
+ "model_id": model.model_id,
582
+ "model_command_executable": model.command[0],
583
+ "model_command_argv_sha256": hashlib.sha256(
584
+ "\0".join(model.command).encode("utf-8")
585
+ ).hexdigest(),
586
+ "temperature": model.temperature,
587
+ "global_seed": global_seed,
588
+ "n_cases": n_cases,
589
+ "annotations_used": allow_annotations,
590
+ "response_auditor_mode": (
591
+ "one-shot-regeneration"
592
+ if auditor_regenerations
593
+ else "passive-metadata-only"
594
+ ),
595
+ "auditor_regenerations_allowed": auditor_regenerations,
596
+ "cases_path": str(cases_path),
597
+ "cases_sha256": file_sha256(cases_path),
598
+ "affect_model_sha256": file_sha256(affect_model),
599
+ "subtext_model_sha256": file_sha256(subtext_model),
600
+ "eq_layer_git_commit": git_commit,
601
+ "pairing_rule": "same model id, same sampling temperature, same per-case seed",
602
+ "arm_difference": (
603
+ "EQ arm prepends only the EQ-Layer system control instruction; "
604
+ "the original transcript is preserved identically in both arms."
605
+ ),
606
+ }