eq-layer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eq_layer/__init__.py +68 -0
- eq_layer/ab_eval.py +302 -0
- eq_layer/actions.py +242 -0
- eq_layer/affect.py +227 -0
- eq_layer/auditor.py +321 -0
- eq_layer/cli.py +174 -0
- eq_layer/coarse_discourse.py +107 -0
- eq_layer/data/intent_train.jsonl +299 -0
- eq_layer/dbdc3.py +197 -0
- eq_layer/dialogbank_iso.py +272 -0
- eq_layer/dialogue_reference.py +393 -0
- eq_layer/intent.py +199 -0
- eq_layer/intent_belief.py +276 -0
- eq_layer/interaction.py +314 -0
- eq_layer/policies.py +295 -0
- eq_layer/preferences.py +125 -0
- eq_layer/response_experiment.py +606 -0
- eq_layer/steer.py +281 -0
- eq_layer/tracker.py +111 -0
- eq_layer/trained_affect.py +246 -0
- eq_layer/trained_breakdown.py +143 -0
- eq_layer/trained_disagreement.py +148 -0
- eq_layer/trained_intent.py +291 -0
- eq_layer/trained_subtext.py +436 -0
- eq_layer-0.1.0.dist-info/METADATA +527 -0
- eq_layer-0.1.0.dist-info/RECORD +29 -0
- eq_layer-0.1.0.dist-info/WHEEL +4 -0
- eq_layer-0.1.0.dist-info/entry_points.txt +2 -0
- eq_layer-0.1.0.dist-info/licenses/LICENSE +21 -0
eq_layer/affect.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Iterable, Protocol
|
|
6
|
+
|
|
7
|
+
from .policies import AffectState
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class AffectAdapter(Protocol):
|
|
11
|
+
name: str
|
|
12
|
+
|
|
13
|
+
def infer(
|
|
14
|
+
self,
|
|
15
|
+
messages: list[dict],
|
|
16
|
+
turn_index: int = 0,
|
|
17
|
+
annotated: dict | None = None,
|
|
18
|
+
) -> AffectState: ...
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# --- detectors -----------------------------------------------------------
|
|
22
|
+
# Keyword sets are a stand-in for a trained classifier. They are deliberately
|
|
23
|
+
# small and readable: when one of them is wrong you can see why without a
|
|
24
|
+
# gradient.
|
|
25
|
+
|
|
26
|
+
ESCALATION_MARKERS = (
|
|
27
|
+
"neden",
|
|
28
|
+
"sen de",
|
|
29
|
+
"saçmalama",
|
|
30
|
+
"yine",
|
|
31
|
+
"kaç kez",
|
|
32
|
+
"kaç defa",
|
|
33
|
+
"anlamıyorsun",
|
|
34
|
+
"sinir",
|
|
35
|
+
"rezalet",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
SOFTENERS = (
|
|
39
|
+
"galiba",
|
|
40
|
+
"aslında",
|
|
41
|
+
"belki",
|
|
42
|
+
"biraz",
|
|
43
|
+
"emin değilim",
|
|
44
|
+
"kısmen",
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
CORRECTION_MARKERS = (
|
|
48
|
+
"hayır, onu demedim",
|
|
49
|
+
"hayır onu demedim",
|
|
50
|
+
"öyle değil",
|
|
51
|
+
"yanlış anladın",
|
|
52
|
+
"yanlış anladınız",
|
|
53
|
+
"onu sormadım",
|
|
54
|
+
"ben onu demedim",
|
|
55
|
+
"demek istediğim",
|
|
56
|
+
"kastettiğim",
|
|
57
|
+
"that's not what i meant",
|
|
58
|
+
"you misunderstood",
|
|
59
|
+
"no, i meant",
|
|
60
|
+
"i didn't say that",
|
|
61
|
+
"i said",
|
|
62
|
+
"i meant",
|
|
63
|
+
"meant to say",
|
|
64
|
+
"that doesn't work",
|
|
65
|
+
"that does not work",
|
|
66
|
+
"that doesn’t work",
|
|
67
|
+
"that's wrong",
|
|
68
|
+
"that is wrong",
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
DISCLOSURE_MARKERS = (
|
|
72
|
+
"kimseye anlatma",
|
|
73
|
+
"kimseye söyleme",
|
|
74
|
+
"sır",
|
|
75
|
+
"gizli tut",
|
|
76
|
+
"benden başka kimse",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# A user opening consecutive turns with a wh-word is asking the same thing
|
|
80
|
+
# again. That is interrogation, not curiosity — and the register has to change.
|
|
81
|
+
QUESTION_OPENERS = ("ne", "neden", "niçin", "nasıl", "nedir", "hangi", "kaç", "mı", "mi")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass
|
|
85
|
+
class HeuristicAffect:
|
|
86
|
+
name: str = "heuristic"
|
|
87
|
+
|
|
88
|
+
# --- arousal ----------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
def _arousal(self, text: str) -> float:
|
|
91
|
+
if not text:
|
|
92
|
+
return 0.0
|
|
93
|
+
low = text.lower().strip()
|
|
94
|
+
hits = sum(1 for m in ESCALATION_MARKERS if m in low)
|
|
95
|
+
softens = sum(1 for s in SOFTENERS if s in low)
|
|
96
|
+
score = min(1.0, 0.25 + hits * 0.22 - softens * 0.12)
|
|
97
|
+
if "?" in low or "!" in low:
|
|
98
|
+
score = min(1.0, score + 0.1)
|
|
99
|
+
return round(score, 3)
|
|
100
|
+
|
|
101
|
+
def _has(self, text: str, markers: Iterable[str]) -> bool:
|
|
102
|
+
low = text.lower()
|
|
103
|
+
return any(m in low for m in markers)
|
|
104
|
+
|
|
105
|
+
# --- structural signals ----------------------------------------------
|
|
106
|
+
|
|
107
|
+
def _recent_user_turns(self, messages: list[dict], n: int = 4) -> list[str]:
|
|
108
|
+
turns = [m["content"] for m in messages if m.get("role") == "user"]
|
|
109
|
+
return turns[-n:]
|
|
110
|
+
|
|
111
|
+
def _escalating(self, user_turns: list[str]) -> bool:
|
|
112
|
+
"""Three or more consecutive turns opening with a wh-word."""
|
|
113
|
+
if len(user_turns) < 3:
|
|
114
|
+
return False
|
|
115
|
+
return all(
|
|
116
|
+
self._first_word(t) in QUESTION_OPENERS for t in user_turns[-3:] if t.strip()
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def _first_word(self, text: str) -> str:
|
|
120
|
+
"""First token with punctuation stripped. 'Neden?' must match 'neden',
|
|
121
|
+
otherwise a bare one-word question — the shape this is built to
|
|
122
|
+
detect — silently fails."""
|
|
123
|
+
token = text.lower().strip().split()[0]
|
|
124
|
+
return token.strip("?!.,;:")
|
|
125
|
+
|
|
126
|
+
def _demand_pattern(self, user_turns: list[str]) -> bool:
|
|
127
|
+
"""Short imperative demands, stacked. 'Anlat biçim!' repeated."""
|
|
128
|
+
recent = [t for t in user_turns[-3:] if t.strip()]
|
|
129
|
+
if len(recent) < 3:
|
|
130
|
+
return False
|
|
131
|
+
short = all(len(t.split()) <= 5 for t in recent)
|
|
132
|
+
punctuated = sum(1 for t in recent if t.strip().endswith(("?", "!"))) >= 2
|
|
133
|
+
return short and punctuated
|
|
134
|
+
|
|
135
|
+
def _high_arousal_run(self, user_turns: list[str]) -> bool:
|
|
136
|
+
return len(user_turns) >= 2 and all(self._arousal(t) >= 0.5 for t in user_turns[-2:])
|
|
137
|
+
|
|
138
|
+
# --- subtext ----------------------------------------------------------
|
|
139
|
+
|
|
140
|
+
def _subtext(
|
|
141
|
+
self,
|
|
142
|
+
user_turns: list[str],
|
|
143
|
+
annotated: dict,
|
|
144
|
+
has_prior_assistant: bool = False,
|
|
145
|
+
) -> str:
|
|
146
|
+
current = user_turns[-1] if user_turns else ""
|
|
147
|
+
low = current.lower()
|
|
148
|
+
|
|
149
|
+
if has_prior_assistant and self._has(current, CORRECTION_MARKERS):
|
|
150
|
+
return "correction"
|
|
151
|
+
if self._has(current, DISCLOSURE_MARKERS):
|
|
152
|
+
return "disclosure_request"
|
|
153
|
+
if annotated.get("subtext") in {"exhaustion", "resignation"}:
|
|
154
|
+
return annotated["subtext"]
|
|
155
|
+
if self._escalating(user_turns):
|
|
156
|
+
return "escalating"
|
|
157
|
+
if self._demand_pattern(user_turns):
|
|
158
|
+
return "demand"
|
|
159
|
+
|
|
160
|
+
stripped = current.rstrip()
|
|
161
|
+
if stripped.endswith("?") and not self._has(current, ESCALATION_MARKERS):
|
|
162
|
+
return "question"
|
|
163
|
+
if stripped.endswith(("!", "?")) or self._has(current, ESCALATION_MARKERS):
|
|
164
|
+
return "challenge"
|
|
165
|
+
return "statement"
|
|
166
|
+
|
|
167
|
+
def _stance(self, current: str, annotated: dict) -> str:
|
|
168
|
+
"""Whether the user is in the right. Not inferable from keywords —
|
|
169
|
+
this is a semantic judgement, so it defaults to unknown and expects an
|
|
170
|
+
annotation. Guessing here is how a layer starts confidently wrong."""
|
|
171
|
+
return annotated.get("stance", "unknown")
|
|
172
|
+
|
|
173
|
+
# --- entry point ------------------------------------------------------
|
|
174
|
+
|
|
175
|
+
def infer(
|
|
176
|
+
self,
|
|
177
|
+
messages: list[dict],
|
|
178
|
+
turn_index: int = 0,
|
|
179
|
+
annotated: dict | None = None,
|
|
180
|
+
) -> AffectState:
|
|
181
|
+
annotated = annotated or {}
|
|
182
|
+
user_turns = [m["content"] for m in messages if m.get("role") == "user"]
|
|
183
|
+
|
|
184
|
+
history = self._history(user_turns)
|
|
185
|
+
current = user_turns[-1] if user_turns else ""
|
|
186
|
+
previous = user_turns[-2] if len(user_turns) > 1 else ""
|
|
187
|
+
arousal = self._arousal(current)
|
|
188
|
+
|
|
189
|
+
return AffectState(
|
|
190
|
+
valence=round(0.5 - arousal, 3),
|
|
191
|
+
arousal=arousal,
|
|
192
|
+
escalation_delta=round(arousal - self._arousal(previous), 3) if previous else 0.0,
|
|
193
|
+
stance=self._stance(current, annotated),
|
|
194
|
+
subtext=self._subtext(
|
|
195
|
+
user_turns,
|
|
196
|
+
annotated,
|
|
197
|
+
has_prior_assistant=any(
|
|
198
|
+
m.get("role") == "assistant" for m in messages[:-1]
|
|
199
|
+
),
|
|
200
|
+
),
|
|
201
|
+
turn_index=turn_index,
|
|
202
|
+
history=history,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
def _history(self, user_turns: list[str]) -> tuple[AffectState, ...]:
|
|
206
|
+
"""Per-turn series carrying a real delta, so trend checks have
|
|
207
|
+
something to read. A history of zero-delta states cannot show
|
|
208
|
+
escalation no matter how angry the transcript gets."""
|
|
209
|
+
states = []
|
|
210
|
+
for i, text in enumerate(user_turns):
|
|
211
|
+
prev = self._arousal(user_turns[i - 1]) if i else 0.0
|
|
212
|
+
states.append(
|
|
213
|
+
AffectState(
|
|
214
|
+
valence=0.0,
|
|
215
|
+
arousal=self._arousal(text),
|
|
216
|
+
escalation_delta=round(self._arousal(text) - prev, 3),
|
|
217
|
+
stance="unknown",
|
|
218
|
+
subtext="",
|
|
219
|
+
turn_index=i,
|
|
220
|
+
)
|
|
221
|
+
)
|
|
222
|
+
return tuple(states)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def load_cases(path: str) -> list[dict]:
|
|
226
|
+
with open(path, encoding="utf-8") as fh:
|
|
227
|
+
return [json.loads(line) for line in fh if line.strip()]
|
eq_layer/auditor.py
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import asdict, dataclass
|
|
5
|
+
from difflib import SequenceMatcher
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
ABSOLUTE_PRIVACY_PROMISES = (
|
|
9
|
+
"kimseye söylemem",
|
|
10
|
+
"sadece bu sohbette kalır",
|
|
11
|
+
"aramızda kalacak",
|
|
12
|
+
"tamamen gizli kalacak",
|
|
13
|
+
"i guarantee confidentiality",
|
|
14
|
+
"this will stay between us",
|
|
15
|
+
"no one else will know",
|
|
16
|
+
"completely confidential",
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
STANCE_ADMISSIONS = (
|
|
20
|
+
"haklısın",
|
|
21
|
+
"sen haklısın",
|
|
22
|
+
"ben yanıldım",
|
|
23
|
+
"ben yanlış anladım",
|
|
24
|
+
"ben hatalıydım",
|
|
25
|
+
"you're right",
|
|
26
|
+
"you are right",
|
|
27
|
+
"i was wrong",
|
|
28
|
+
"my mistake",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class AuditIssue:
|
|
34
|
+
code: str
|
|
35
|
+
severity: str
|
|
36
|
+
message: str
|
|
37
|
+
evidence: str = ""
|
|
38
|
+
|
|
39
|
+
def to_dict(self) -> dict:
|
|
40
|
+
return asdict(self)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class AuditContext:
|
|
45
|
+
task_move: str
|
|
46
|
+
social_move: str
|
|
47
|
+
repair_move: str
|
|
48
|
+
verbosity: str
|
|
49
|
+
question_budget: int
|
|
50
|
+
no_guess: bool
|
|
51
|
+
stance: str = "unknown"
|
|
52
|
+
repair_active: bool = False
|
|
53
|
+
previous_assistant_responses: tuple[str, ...] = ()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class ResponseAudit:
|
|
58
|
+
passed: bool
|
|
59
|
+
should_regenerate: bool
|
|
60
|
+
hard_failures: int
|
|
61
|
+
soft_warnings: int
|
|
62
|
+
question_count: int
|
|
63
|
+
word_count: int
|
|
64
|
+
max_repeat_similarity: float
|
|
65
|
+
issues: tuple[AuditIssue, ...]
|
|
66
|
+
|
|
67
|
+
def to_dict(self) -> dict:
|
|
68
|
+
return {
|
|
69
|
+
"passed": self.passed,
|
|
70
|
+
"should_regenerate": self.should_regenerate,
|
|
71
|
+
"hard_failures": self.hard_failures,
|
|
72
|
+
"soft_warnings": self.soft_warnings,
|
|
73
|
+
"question_count": self.question_count,
|
|
74
|
+
"word_count": self.word_count,
|
|
75
|
+
"max_repeat_similarity": self.max_repeat_similarity,
|
|
76
|
+
"issues": [issue.to_dict() for issue in self.issues],
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _normalise(text: str) -> str:
|
|
81
|
+
return " ".join(text.lower().split())
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _sentence_starts(text: str) -> tuple[str, ...]:
|
|
85
|
+
parts = re.split(r"(?<=[.!?])\s+|\n+", _normalise(text))
|
|
86
|
+
return tuple(part.strip(" -—:;,.!?") for part in parts if part.strip())
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _max_repeat_similarity(
|
|
90
|
+
response: str,
|
|
91
|
+
previous_assistant_responses: tuple[str, ...],
|
|
92
|
+
) -> float:
|
|
93
|
+
current = _normalise(response)
|
|
94
|
+
if not current or not previous_assistant_responses:
|
|
95
|
+
return 0.0
|
|
96
|
+
|
|
97
|
+
scores = []
|
|
98
|
+
for previous in previous_assistant_responses[-3:]:
|
|
99
|
+
previous_norm = _normalise(previous)
|
|
100
|
+
if not previous_norm:
|
|
101
|
+
continue
|
|
102
|
+
scores.append(SequenceMatcher(None, current, previous_norm).ratio())
|
|
103
|
+
return round(max(scores, default=0.0), 4)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class ResponseAuditor:
|
|
107
|
+
"""Conservative structural auditor for generated responses.
|
|
108
|
+
|
|
109
|
+
This auditor checks inspectable control-contract violations. It does not
|
|
110
|
+
decide whether factual content is true, whether advice is correct, or
|
|
111
|
+
whether the user's stance is justified.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def audit(self, response: str, context: AuditContext) -> ResponseAudit:
|
|
115
|
+
issues: list[AuditIssue] = []
|
|
116
|
+
stripped = response.strip()
|
|
117
|
+
low = _normalise(stripped)
|
|
118
|
+
question_count = stripped.count("?")
|
|
119
|
+
word_count = len(stripped.split())
|
|
120
|
+
repeat_similarity = _max_repeat_similarity(
|
|
121
|
+
stripped,
|
|
122
|
+
context.previous_assistant_responses,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
if not stripped:
|
|
126
|
+
issues.append(
|
|
127
|
+
AuditIssue(
|
|
128
|
+
code="empty_response",
|
|
129
|
+
severity="hard",
|
|
130
|
+
message="The model returned an empty response.",
|
|
131
|
+
)
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
if question_count > context.question_budget:
|
|
135
|
+
issues.append(
|
|
136
|
+
AuditIssue(
|
|
137
|
+
code="question_budget_exceeded",
|
|
138
|
+
severity="hard",
|
|
139
|
+
message=(
|
|
140
|
+
f"Response asked {question_count} questions but the "
|
|
141
|
+
f"control action allowed {context.question_budget}."
|
|
142
|
+
),
|
|
143
|
+
evidence=f"question_count={question_count}",
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
clarification_required = (
|
|
148
|
+
context.task_move in {"clarify_goal", "clarify_correction"}
|
|
149
|
+
or context.repair_move in {"clarify_goal", "clarify_one"}
|
|
150
|
+
)
|
|
151
|
+
if clarification_required and question_count == 0:
|
|
152
|
+
issues.append(
|
|
153
|
+
AuditIssue(
|
|
154
|
+
code="missing_required_clarification",
|
|
155
|
+
severity="hard",
|
|
156
|
+
message="The control action required a targeted clarification question.",
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
for phrase in ABSOLUTE_PRIVACY_PROMISES:
|
|
161
|
+
if phrase in low:
|
|
162
|
+
issues.append(
|
|
163
|
+
AuditIssue(
|
|
164
|
+
code="absolute_privacy_promise",
|
|
165
|
+
severity="hard",
|
|
166
|
+
message="Response made an unsupported absolute privacy promise.",
|
|
167
|
+
evidence=phrase,
|
|
168
|
+
)
|
|
169
|
+
)
|
|
170
|
+
break
|
|
171
|
+
|
|
172
|
+
if context.stance == "unknown":
|
|
173
|
+
starts = _sentence_starts(stripped)
|
|
174
|
+
stance_hit = next(
|
|
175
|
+
(
|
|
176
|
+
phrase
|
|
177
|
+
for phrase in STANCE_ADMISSIONS
|
|
178
|
+
if any(
|
|
179
|
+
sentence == phrase
|
|
180
|
+
or sentence.startswith(phrase + " ")
|
|
181
|
+
or sentence.startswith(phrase + ",")
|
|
182
|
+
for sentence in starts
|
|
183
|
+
)
|
|
184
|
+
),
|
|
185
|
+
None,
|
|
186
|
+
)
|
|
187
|
+
if stance_hit is not None:
|
|
188
|
+
issues.append(
|
|
189
|
+
AuditIssue(
|
|
190
|
+
code="unsupported_stance_admission",
|
|
191
|
+
severity="hard",
|
|
192
|
+
message=(
|
|
193
|
+
"Response admitted that the user was right or the "
|
|
194
|
+
"assistant was wrong while factual stance was unknown."
|
|
195
|
+
),
|
|
196
|
+
evidence=stance_hit,
|
|
197
|
+
)
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
if (
|
|
201
|
+
context.verbosity == "low"
|
|
202
|
+
and word_count > 120
|
|
203
|
+
):
|
|
204
|
+
issues.append(
|
|
205
|
+
AuditIssue(
|
|
206
|
+
code="low_verbosity_overrun",
|
|
207
|
+
severity="soft",
|
|
208
|
+
message="Response substantially exceeded the low-verbosity target.",
|
|
209
|
+
evidence=f"word_count={word_count}",
|
|
210
|
+
)
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
if repeat_similarity >= 0.92 and word_count >= 8:
|
|
214
|
+
issues.append(
|
|
215
|
+
AuditIssue(
|
|
216
|
+
code="near_duplicate_response",
|
|
217
|
+
severity="soft",
|
|
218
|
+
message="Response is nearly identical to a recent assistant response.",
|
|
219
|
+
evidence=f"similarity={repeat_similarity:.4f}",
|
|
220
|
+
)
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
if (
|
|
224
|
+
context.repair_active
|
|
225
|
+
and context.repair_move in {"repair_targeted", "stop_restatement_and_repair"}
|
|
226
|
+
and repeat_similarity >= 0.80
|
|
227
|
+
):
|
|
228
|
+
issues.append(
|
|
229
|
+
AuditIssue(
|
|
230
|
+
code="repair_repeats_prior_answer",
|
|
231
|
+
severity="hard",
|
|
232
|
+
message=(
|
|
233
|
+
"Repair was active but the response substantially repeated "
|
|
234
|
+
"a recent assistant answer."
|
|
235
|
+
),
|
|
236
|
+
evidence=f"similarity={repeat_similarity:.4f}",
|
|
237
|
+
)
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
hard = sum(issue.severity == "hard" for issue in issues)
|
|
241
|
+
soft = sum(issue.severity == "soft" for issue in issues)
|
|
242
|
+
return ResponseAudit(
|
|
243
|
+
passed=hard == 0,
|
|
244
|
+
should_regenerate=hard > 0,
|
|
245
|
+
hard_failures=hard,
|
|
246
|
+
soft_warnings=soft,
|
|
247
|
+
question_count=question_count,
|
|
248
|
+
word_count=word_count,
|
|
249
|
+
max_repeat_similarity=repeat_similarity,
|
|
250
|
+
issues=tuple(issues),
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def audit_from_generation_metadata(
|
|
255
|
+
response: str,
|
|
256
|
+
messages: list[dict],
|
|
257
|
+
metadata: dict,
|
|
258
|
+
) -> ResponseAudit:
|
|
259
|
+
"""Audit an EQ-Layer response using generation metadata only."""
|
|
260
|
+
|
|
261
|
+
factored = metadata.get("factored_action") or {}
|
|
262
|
+
realization = factored.get("realization") or {}
|
|
263
|
+
repair = metadata.get("repair") or {}
|
|
264
|
+
previous = tuple(
|
|
265
|
+
str(message.get("content", ""))
|
|
266
|
+
for message in messages
|
|
267
|
+
if message.get("role") == "assistant"
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
context = AuditContext(
|
|
271
|
+
task_move=str(factored.get("task_move") or "respond_contextually"),
|
|
272
|
+
social_move=str(factored.get("social_move") or "neutral"),
|
|
273
|
+
repair_move=str(factored.get("repair_move") or "none"),
|
|
274
|
+
verbosity=str(realization.get("verbosity") or "normal"),
|
|
275
|
+
question_budget=int(realization.get("question_budget") or 0),
|
|
276
|
+
no_guess=bool(realization.get("no_guess", False)),
|
|
277
|
+
stance=str(metadata.get("stance") or "unknown"),
|
|
278
|
+
repair_active=bool(repair.get("active", False)),
|
|
279
|
+
previous_assistant_responses=previous,
|
|
280
|
+
)
|
|
281
|
+
return ResponseAuditor().audit(response, context)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
REGENERATION_GUIDANCE = {
|
|
285
|
+
"empty_response": "Return a non-empty response that follows the control action.",
|
|
286
|
+
"question_budget_exceeded": "Do not exceed the allowed number of questions.",
|
|
287
|
+
"missing_required_clarification": "Ask exactly one targeted clarification question before guessing.",
|
|
288
|
+
"absolute_privacy_promise": "Do not promise absolute privacy or confidentiality.",
|
|
289
|
+
"unsupported_stance_admission": (
|
|
290
|
+
"Do not claim that the user is right or that the assistant was wrong "
|
|
291
|
+
"when factual stance is unknown."
|
|
292
|
+
),
|
|
293
|
+
"repair_repeats_prior_answer": (
|
|
294
|
+
"Do not repeat the prior answer; repair the specific misunderstanding."
|
|
295
|
+
),
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def regeneration_instruction(audit: ResponseAudit) -> str:
|
|
300
|
+
"""Build a minimal one-shot repair instruction from hard audit failures."""
|
|
301
|
+
|
|
302
|
+
hard_codes = [
|
|
303
|
+
issue.code
|
|
304
|
+
for issue in audit.issues
|
|
305
|
+
if issue.severity == "hard"
|
|
306
|
+
]
|
|
307
|
+
guidance = [
|
|
308
|
+
REGENERATION_GUIDANCE[code]
|
|
309
|
+
for code in hard_codes
|
|
310
|
+
if code in REGENERATION_GUIDANCE
|
|
311
|
+
]
|
|
312
|
+
if not guidance:
|
|
313
|
+
return ""
|
|
314
|
+
|
|
315
|
+
bullets = "\n".join(f"- {item}" for item in dict.fromkeys(guidance))
|
|
316
|
+
return (
|
|
317
|
+
"The previous draft violated EQ-Layer structural control contracts. "
|
|
318
|
+
"Regenerate the answer once using the same user context and original "
|
|
319
|
+
"task. Correct only the listed violations; do not invent new facts.\n"
|
|
320
|
+
+ bullets
|
|
321
|
+
)
|