eq-layer 0.2.3__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eq_layer-0.2.5/.gitignore +23 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/PKG-INFO +36 -13
- {eq_layer-0.2.3 → eq_layer-0.2.5}/README.md +35 -12
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/__init__.py +7 -1
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/ab_eval.py +60 -21
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/affect.py +26 -10
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/cli.py +13 -5
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/intent.py +2 -1
- eq_layer-0.2.5/eq_layer/json_contract.py +41 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/model_state.py +0 -2
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/preferences.py +3 -10
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/response_experiment.py +34 -9
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/tracker.py +2 -1
- {eq_layer-0.2.3 → eq_layer-0.2.5}/pyproject.toml +1 -1
- eq_layer-0.2.3/.gitignore +0 -9
- {eq_layer-0.2.3 → eq_layer-0.2.5}/LICENSE +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/THIRD_PARTY_DATA.md +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/_deps.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/_version.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/actions.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/auditor.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/coarse_discourse.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/data/intent_train.jsonl +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/dbdc3.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/dialogbank_iso.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/dialogue_reference.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/intent_belief.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/interaction.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/policies.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/steer.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/textnorm.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/trained_affect.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/trained_breakdown.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/trained_disagreement.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/trained_intent.py +0 -0
- {eq_layer-0.2.3 → eq_layer-0.2.5}/eq_layer/trained_subtext.py +0 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
venv/
|
|
5
|
+
.env
|
|
6
|
+
.npmrc
|
|
7
|
+
.DS_Store
|
|
8
|
+
.idea/
|
|
9
|
+
.vscode/
|
|
10
|
+
|
|
11
|
+
# Python test/build outputs
|
|
12
|
+
.coverage
|
|
13
|
+
coverage.xml
|
|
14
|
+
htmlcov/
|
|
15
|
+
dist/
|
|
16
|
+
build/
|
|
17
|
+
*.egg-info/
|
|
18
|
+
.pytest_cache/
|
|
19
|
+
|
|
20
|
+
# Retired release trigger files (publishing is dispatched explicitly; see
|
|
21
|
+
# .github/workflows/create-release.yml and .github/workflows/publish-npm.yml)
|
|
22
|
+
.npm-release-request
|
|
23
|
+
.release-trigger
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: eq-layer
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Inspectable intent, affect, repair, uncertainty, and factored-action control for LLM responses
|
|
5
5
|
Project-URL: Homepage, https://github.com/Furox-Art/eq-layer
|
|
6
6
|
Project-URL: Repository, https://github.com/Furox-Art/eq-layer
|
|
@@ -51,7 +51,7 @@ So this repo does not retrain the model. It adds a layer:
|
|
|
51
51
|
1. **Affect state** — `{valence, arousal, escalation_delta, stance, subtext}` as an explicit intermediate representation, tracked across turns. `escalation_delta` is the point: snapshot classification cannot see a trend.
|
|
52
52
|
2. **Intent belief state** — the learned classifier preserves the full posterior over `action_request/status_check/explanation/question/statement/unknown` instead of collapsing uncertainty to one confidence number. A transparent one-step Bayes-risk router chooses whether to act or clarify from that distribution. This is POMDP-inspired uncertainty handling, not a full POMDP.
|
|
53
53
|
3. **Conversation health state** — `repair` identifies observable user→assistant misalignment and its target; `interaction_quality` tracks structural conversation failure separately from emotion using current score, delta, repeated failures, unresolved repair, and clarification load.
|
|
54
|
-
4. **Policy selection** — affect and intent jointly choose a response-policy family (`mirror`, `direct`, `repair`, `hold`, `boundary
|
|
54
|
+
4. **Policy selection** — affect and intent jointly choose a response-policy family (`mirror`, `validate_redirect`, `direct`, `ask`, `deflate`, `repair`, `hold`, `boundary`). Selection is a discrete decision, not generation.
|
|
55
55
|
5. **Factored action compilation** — the selected policy is split into orthogonal controls: `task_move`, `social_move`, `repair_move`, and `realization`. The task is driven primarily by user intent, so affective adaptation cannot silently replace what the user actually asked for.
|
|
56
56
|
6. **Steering** — the factored action, source policy, repair state, interaction quality, and canonical request are injected into the decode path so the control decision actually lands in the tokens.
|
|
57
57
|
7. **Structural response audit** — the generated response is checked against inspectable control contracts such as question budget, required clarification, unsupported privacy promises, unknown-stance admissions, repair repetition, and low-verbosity overruns. This is not a factual truth checker.
|
|
@@ -112,12 +112,19 @@ paths and source documentation.
|
|
|
112
112
|
|
|
113
113
|
## Quick start
|
|
114
114
|
|
|
115
|
-
Install the
|
|
115
|
+
Install the Python runtime:
|
|
116
116
|
|
|
117
117
|
```bash
|
|
118
118
|
pip install "eq-layer[ml]"
|
|
119
119
|
```
|
|
120
120
|
|
|
121
|
+
Install the npm skill installer and CLI bridge (Node >= 22.14):
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
npm install -g eq-layer
|
|
125
|
+
npx eq-layer --help
|
|
126
|
+
```
|
|
127
|
+
|
|
121
128
|
```python
|
|
122
129
|
from eq_layer import ConversationTracker, Selector, Steer, TrainedAffect, TrainedIntent, TrainedSubtext, compose_action
|
|
123
130
|
|
|
@@ -147,6 +154,18 @@ prompt = Steer.build(
|
|
|
147
154
|
).apply_to_prompt(messages[-1]["content"])
|
|
148
155
|
```
|
|
149
156
|
|
|
157
|
+
The two `.joblib` files under `artifacts/` are **not shipped** in the
|
|
158
|
+
repository, the PyPI wheel, or the npm tarball. Produce them with the training
|
|
159
|
+
commands below (`tools/train_emobank_affect.py` and
|
|
160
|
+
`tools/train_xdailydialog_subtext.py`) before running this block. Without those
|
|
161
|
+
artifacts, use the zero-dependency route, which reports
|
|
162
|
+
`"runtime_mode": "lightweight-local"` and needs no `.joblib` file:
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
printf '%s' '[{"role":"user","content":"devam et"}]' \
|
|
166
|
+
| python -m eq_layer.cli route --intent-mode heuristic --pretty
|
|
167
|
+
```
|
|
168
|
+
|
|
150
169
|
Run the seeded cases. Exits non-zero on any mismatch or any policy that no
|
|
151
170
|
case can reach:
|
|
152
171
|
|
|
@@ -562,16 +581,20 @@ bridge). Each side keeps its version in exactly one canonical location, and the
|
|
|
562
581
|
release workflows verify the GitHub release tag against those fields before
|
|
563
582
|
uploading.
|
|
564
583
|
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
584
|
+
`.github/workflows/create-release.yml` is the release entry point: a push to
|
|
585
|
+
`main` that changes `.release-request` creates the GitHub release when missing
|
|
586
|
+
and explicitly dispatches both publishers (`release.yml` for PyPI and
|
|
587
|
+
`publish-npm.yml` for npm). A release tag on its own does not publish npm;
|
|
588
|
+
GitHub suppresses the `release` event for releases created with `GITHUB_TOKEN`,
|
|
589
|
+
so the explicit dispatch is required.
|
|
590
|
+
|
|
591
|
+
npm publishing uses trusted publishing over GitHub OIDC by default and requests
|
|
592
|
+
a provenance attestation (`npm publish --access public --provenance`). The
|
|
593
|
+
presence of an `NPM_TOKEN` secret does **not** change that: token mode runs only
|
|
594
|
+
when a maintainer explicitly dispatches `publish-npm.yml` with
|
|
595
|
+
`use_token_fallback=true`. That fallback publishes with
|
|
596
|
+
`npm publish --access public --no-provenance`, because provenance is an
|
|
597
|
+
OIDC-only attestation. See `docs/PUBLISHING.md` for the full flow.
|
|
575
598
|
|
|
576
599
|
## Contributing
|
|
577
600
|
|
|
@@ -16,7 +16,7 @@ So this repo does not retrain the model. It adds a layer:
|
|
|
16
16
|
1. **Affect state** — `{valence, arousal, escalation_delta, stance, subtext}` as an explicit intermediate representation, tracked across turns. `escalation_delta` is the point: snapshot classification cannot see a trend.
|
|
17
17
|
2. **Intent belief state** — the learned classifier preserves the full posterior over `action_request/status_check/explanation/question/statement/unknown` instead of collapsing uncertainty to one confidence number. A transparent one-step Bayes-risk router chooses whether to act or clarify from that distribution. This is POMDP-inspired uncertainty handling, not a full POMDP.
|
|
18
18
|
3. **Conversation health state** — `repair` identifies observable user→assistant misalignment and its target; `interaction_quality` tracks structural conversation failure separately from emotion using current score, delta, repeated failures, unresolved repair, and clarification load.
|
|
19
|
-
4. **Policy selection** — affect and intent jointly choose a response-policy family (`mirror`, `direct`, `repair`, `hold`, `boundary
|
|
19
|
+
4. **Policy selection** — affect and intent jointly choose a response-policy family (`mirror`, `validate_redirect`, `direct`, `ask`, `deflate`, `repair`, `hold`, `boundary`). Selection is a discrete decision, not generation.
|
|
20
20
|
5. **Factored action compilation** — the selected policy is split into orthogonal controls: `task_move`, `social_move`, `repair_move`, and `realization`. The task is driven primarily by user intent, so affective adaptation cannot silently replace what the user actually asked for.
|
|
21
21
|
6. **Steering** — the factored action, source policy, repair state, interaction quality, and canonical request are injected into the decode path so the control decision actually lands in the tokens.
|
|
22
22
|
7. **Structural response audit** — the generated response is checked against inspectable control contracts such as question budget, required clarification, unsupported privacy promises, unknown-stance admissions, repair repetition, and low-verbosity overruns. This is not a factual truth checker.
|
|
@@ -77,12 +77,19 @@ paths and source documentation.
|
|
|
77
77
|
|
|
78
78
|
## Quick start
|
|
79
79
|
|
|
80
|
-
Install the
|
|
80
|
+
Install the Python runtime:
|
|
81
81
|
|
|
82
82
|
```bash
|
|
83
83
|
pip install "eq-layer[ml]"
|
|
84
84
|
```
|
|
85
85
|
|
|
86
|
+
Install the npm skill installer and CLI bridge (Node >= 22.14):
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
npm install -g eq-layer
|
|
90
|
+
npx eq-layer --help
|
|
91
|
+
```
|
|
92
|
+
|
|
86
93
|
```python
|
|
87
94
|
from eq_layer import ConversationTracker, Selector, Steer, TrainedAffect, TrainedIntent, TrainedSubtext, compose_action
|
|
88
95
|
|
|
@@ -112,6 +119,18 @@ prompt = Steer.build(
|
|
|
112
119
|
).apply_to_prompt(messages[-1]["content"])
|
|
113
120
|
```
|
|
114
121
|
|
|
122
|
+
The two `.joblib` files under `artifacts/` are **not shipped** in the
|
|
123
|
+
repository, the PyPI wheel, or the npm tarball. Produce them with the training
|
|
124
|
+
commands below (`tools/train_emobank_affect.py` and
|
|
125
|
+
`tools/train_xdailydialog_subtext.py`) before running this block. Without those
|
|
126
|
+
artifacts, use the zero-dependency route, which reports
|
|
127
|
+
`"runtime_mode": "lightweight-local"` and needs no `.joblib` file:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
printf '%s' '[{"role":"user","content":"devam et"}]' \
|
|
131
|
+
| python -m eq_layer.cli route --intent-mode heuristic --pretty
|
|
132
|
+
```
|
|
133
|
+
|
|
115
134
|
Run the seeded cases. Exits non-zero on any mismatch or any policy that no
|
|
116
135
|
case can reach:
|
|
117
136
|
|
|
@@ -527,16 +546,20 @@ bridge). Each side keeps its version in exactly one canonical location, and the
|
|
|
527
546
|
release workflows verify the GitHub release tag against those fields before
|
|
528
547
|
uploading.
|
|
529
548
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
549
|
+
`.github/workflows/create-release.yml` is the release entry point: a push to
|
|
550
|
+
`main` that changes `.release-request` creates the GitHub release when missing
|
|
551
|
+
and explicitly dispatches both publishers (`release.yml` for PyPI and
|
|
552
|
+
`publish-npm.yml` for npm). A release tag on its own does not publish npm;
|
|
553
|
+
GitHub suppresses the `release` event for releases created with `GITHUB_TOKEN`,
|
|
554
|
+
so the explicit dispatch is required.
|
|
555
|
+
|
|
556
|
+
npm publishing uses trusted publishing over GitHub OIDC by default and requests
|
|
557
|
+
a provenance attestation (`npm publish --access public --provenance`). The
|
|
558
|
+
presence of an `NPM_TOKEN` secret does **not** change that: token mode runs only
|
|
559
|
+
when a maintainer explicitly dispatches `publish-npm.yml` with
|
|
560
|
+
`use_token_fallback=true`. That fallback publishes with
|
|
561
|
+
`npm publish --access public --no-provenance`, because provenance is an
|
|
562
|
+
OIDC-only attestation. See `docs/PUBLISHING.md` for the full flow.
|
|
540
563
|
|
|
541
564
|
## Contributing
|
|
542
565
|
|
|
@@ -7,7 +7,9 @@ from .intent import HeuristicIntent, IntentAdapter, IntentState
|
|
|
7
7
|
from .intent_belief import IntentBelief, IntentRiskDecision, decide_intent_action, expected_loss
|
|
8
8
|
from .interaction import InteractionQualityState, RepairState, infer_interaction_quality, infer_repair_state
|
|
9
9
|
from .model_state import ModelStateError
|
|
10
|
-
from .
|
|
10
|
+
from . import policies as _policies
|
|
11
|
+
from .policies import REGISTRY, SUBTEXTS, AffectState, Policy, Register, Selector
|
|
12
|
+
|
|
11
13
|
from .preferences import SessionPreferences, infer_session_preferences
|
|
12
14
|
from .response_experiment import (
|
|
13
15
|
CommandModel,
|
|
@@ -22,6 +24,10 @@ from .trained_intent import TrainedIntent
|
|
|
22
24
|
from .trained_subtext import DialogueSignals, SubtextDecision, TrainedSubtext
|
|
23
25
|
from .tracker import AnnotationStanceResolver, ConversationTracker, StanceDecision, TrackingResult
|
|
24
26
|
|
|
27
|
+
# Snapshot re-export: code that must observe runtime overrides reads
|
|
28
|
+
# eq_layer.policies.POLICIES instead.
|
|
29
|
+
POLICIES = _policies.POLICIES
|
|
30
|
+
|
|
25
31
|
__all__ = [
|
|
26
32
|
"__version__",
|
|
27
33
|
"OptionalDependencyError",
|
|
@@ -6,6 +6,7 @@ import random
|
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
|
|
9
|
+
from .json_contract import loads as strict_json_loads
|
|
9
10
|
|
|
10
11
|
DIMENSIONS = (
|
|
11
12
|
"intent_fidelity",
|
|
@@ -17,8 +18,20 @@ DIMENSIONS = (
|
|
|
17
18
|
|
|
18
19
|
|
|
19
20
|
def load_jsonl(path: str | Path) -> list[dict]:
|
|
21
|
+
"""Read human-evaluation artifacts without ambiguous JSON interpretation."""
|
|
22
|
+
rows: list[dict] = []
|
|
20
23
|
with open(path, encoding="utf-8") as fh:
|
|
21
|
-
|
|
24
|
+
for lineno, line in enumerate(fh, start=1):
|
|
25
|
+
if not line.strip():
|
|
26
|
+
continue
|
|
27
|
+
try:
|
|
28
|
+
row = strict_json_loads(line)
|
|
29
|
+
except (ValueError, UnicodeError) as exc:
|
|
30
|
+
raise ValueError(f"{path}:{lineno}: invalid evaluation JSON: {exc}") from exc
|
|
31
|
+
if not isinstance(row, dict):
|
|
32
|
+
raise ValueError(f"{path}:{lineno}: evaluation row must be an object")
|
|
33
|
+
rows.append(row)
|
|
34
|
+
return rows
|
|
22
35
|
|
|
23
36
|
|
|
24
37
|
def write_jsonl(path: str | Path, rows: list[dict]) -> None:
|
|
@@ -47,15 +60,25 @@ def prepare_blinded(
|
|
|
47
60
|
|
|
48
61
|
seen: set[str] = set()
|
|
49
62
|
for row in pairs:
|
|
50
|
-
case_id =
|
|
63
|
+
case_id = row["id"]
|
|
64
|
+
if not isinstance(case_id, str) or not case_id.strip():
|
|
65
|
+
raise ValueError("A/B case id must be a non-empty string")
|
|
51
66
|
if case_id in seen:
|
|
52
67
|
raise ValueError(f"Duplicate A/B case id: {case_id}")
|
|
53
68
|
seen.add(case_id)
|
|
54
69
|
|
|
55
|
-
baseline =
|
|
56
|
-
eq_response =
|
|
57
|
-
if
|
|
58
|
-
|
|
70
|
+
baseline = row["baseline"]
|
|
71
|
+
eq_response = row["eq"]
|
|
72
|
+
if (
|
|
73
|
+
not isinstance(baseline, str)
|
|
74
|
+
or not isinstance(eq_response, str)
|
|
75
|
+
or not baseline.strip()
|
|
76
|
+
or not eq_response.strip()
|
|
77
|
+
):
|
|
78
|
+
raise ValueError(f"A/B responses must be non-empty strings: {case_id}")
|
|
79
|
+
source = row.get("source") or {}
|
|
80
|
+
if not isinstance(source, dict):
|
|
81
|
+
raise ValueError(f"A/B case source must be an object: {case_id}")
|
|
59
82
|
|
|
60
83
|
eq_is_a = bool(rng.getrandbits(1))
|
|
61
84
|
response_a = eq_response if eq_is_a else baseline
|
|
@@ -67,7 +90,7 @@ def prepare_blinded(
|
|
|
67
90
|
ballot.append(
|
|
68
91
|
{
|
|
69
92
|
"id": case_id,
|
|
70
|
-
"stratum":
|
|
93
|
+
"stratum": source.get("stratum"),
|
|
71
94
|
"context": row.get("context", []),
|
|
72
95
|
"response_A": response_a,
|
|
73
96
|
"response_B": response_b,
|
|
@@ -89,7 +112,9 @@ class PreferenceSummary:
|
|
|
89
112
|
exact_p_two_sided: float | None
|
|
90
113
|
|
|
91
114
|
|
|
92
|
-
def _wilson_interval(
|
|
115
|
+
def _wilson_interval(
|
|
116
|
+
successes: int, total: int, z: float = 1.959963984540054
|
|
117
|
+
) -> tuple[float, float]:
|
|
93
118
|
if total <= 0:
|
|
94
119
|
raise ValueError("Wilson interval requires total > 0.")
|
|
95
120
|
p = successes / total
|
|
@@ -141,6 +166,20 @@ def summarize(wins: int, losses: int, ties: int) -> PreferenceSummary:
|
|
|
141
166
|
)
|
|
142
167
|
|
|
143
168
|
|
|
169
|
+
def _validate_decode_arms(case_id: str, key: dict) -> dict[str, str]:
|
|
170
|
+
"""A blinded pair must decode to one EQ arm and one control arm."""
|
|
171
|
+
mapping = key.get(case_id)
|
|
172
|
+
if mapping is None:
|
|
173
|
+
raise ValueError(f"Missing decode key for case: {case_id}")
|
|
174
|
+
if (
|
|
175
|
+
not isinstance(mapping, dict)
|
|
176
|
+
or set(mapping) != {"A", "B"}
|
|
177
|
+
or {mapping["A"], mapping["B"]} != {"eq", "baseline"}
|
|
178
|
+
):
|
|
179
|
+
raise ValueError(f"Invalid or non-blinded decode key for case: {case_id}")
|
|
180
|
+
return mapping
|
|
181
|
+
|
|
182
|
+
|
|
144
183
|
def score_blinded(
|
|
145
184
|
rated_ballot: list[dict],
|
|
146
185
|
key: dict,
|
|
@@ -152,15 +191,17 @@ def score_blinded(
|
|
|
152
191
|
for dimension in DIMENSIONS
|
|
153
192
|
}
|
|
154
193
|
excluded: list[str] = []
|
|
194
|
+
seen_ids: set[str] = set()
|
|
155
195
|
|
|
156
196
|
for row in rated_ballot:
|
|
157
197
|
case_id = str(row["id"])
|
|
198
|
+
if case_id in seen_ids:
|
|
199
|
+
raise ValueError(f"Duplicate case id in rated ballot: {case_id}")
|
|
200
|
+
seen_ids.add(case_id)
|
|
158
201
|
if exclude_ids and case_id in exclude_ids:
|
|
159
202
|
excluded.append(case_id)
|
|
160
203
|
continue
|
|
161
|
-
|
|
162
|
-
raise ValueError(f"Missing decode key for case: {case_id}")
|
|
163
|
-
|
|
204
|
+
arm_mapping = _validate_decode_arms(case_id, key)
|
|
164
205
|
ratings = row.get("ratings") or {}
|
|
165
206
|
for dimension in DIMENSIONS:
|
|
166
207
|
vote = str(ratings.get(dimension, "")).strip().upper()
|
|
@@ -173,13 +214,12 @@ def score_blinded(
|
|
|
173
214
|
counts[dimension]["ties"] += 1
|
|
174
215
|
continue
|
|
175
216
|
|
|
176
|
-
winner =
|
|
217
|
+
winner = arm_mapping[vote]
|
|
218
|
+
# _validate_decode_arms already proves the target is eq or baseline.
|
|
177
219
|
if winner == "eq":
|
|
178
220
|
counts[dimension]["wins"] += 1
|
|
179
|
-
elif winner == "baseline":
|
|
180
|
-
counts[dimension]["losses"] += 1
|
|
181
221
|
else:
|
|
182
|
-
|
|
222
|
+
counts[dimension]["losses"] += 1
|
|
183
223
|
|
|
184
224
|
report = {
|
|
185
225
|
"design": "blind-pairwise-human-evaluation",
|
|
@@ -204,8 +244,7 @@ def score_blinded(
|
|
|
204
244
|
|
|
205
245
|
def _decoded_vote(row: dict, key: dict, dimension: str) -> str:
|
|
206
246
|
case_id = str(row["id"])
|
|
207
|
-
|
|
208
|
-
raise ValueError(f"Missing decode key for case: {case_id}")
|
|
247
|
+
arm_mapping = _validate_decode_arms(case_id, key)
|
|
209
248
|
ratings = row.get("ratings") or {}
|
|
210
249
|
vote = str(ratings.get(dimension, "")).strip().upper()
|
|
211
250
|
if vote == "TIE":
|
|
@@ -214,10 +253,8 @@ def _decoded_vote(row: dict, key: dict, dimension: str) -> str:
|
|
|
214
253
|
raise ValueError(
|
|
215
254
|
f"{case_id}/{dimension}: expected A, B, or tie; got {vote!r}"
|
|
216
255
|
)
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
raise ValueError(f"Invalid decode key target: {winner}")
|
|
220
|
-
return winner
|
|
256
|
+
# _validate_decode_arms has already rejected every other mapping target.
|
|
257
|
+
return arm_mapping[vote]
|
|
221
258
|
|
|
222
259
|
|
|
223
260
|
def _fleiss_kappa(case_votes: list[list[str]]) -> float | None:
|
|
@@ -275,6 +312,8 @@ def score_multiple_blinded(
|
|
|
275
312
|
rows_by_rater[rater_id] = mapping
|
|
276
313
|
|
|
277
314
|
assert case_ids is not None
|
|
315
|
+
if not case_ids:
|
|
316
|
+
raise ValueError("Multi-rater scoring requires at least one rated case.")
|
|
278
317
|
report = {
|
|
279
318
|
"design": "blind-pairwise-multi-rater-human-evaluation",
|
|
280
319
|
"n_raters": len(rated_ballots),
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
from collections.abc import Iterable
|
|
4
4
|
from dataclasses import dataclass
|
|
5
|
-
from typing import
|
|
5
|
+
from typing import Protocol
|
|
6
6
|
|
|
7
|
+
from .json_contract import loads as strict_json_loads
|
|
7
8
|
from .policies import AffectState
|
|
8
9
|
|
|
9
10
|
|
|
@@ -15,7 +16,8 @@ class AffectAdapter(Protocol):
|
|
|
15
16
|
messages: list[dict],
|
|
16
17
|
turn_index: int = 0,
|
|
17
18
|
annotated: dict | None = None,
|
|
18
|
-
) -> AffectState:
|
|
19
|
+
) -> AffectState:
|
|
20
|
+
pass # pragma: no cover - Protocol stub
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
# --- detectors -----------------------------------------------------------
|
|
@@ -93,7 +95,9 @@ class HeuristicAffect:
|
|
|
93
95
|
low = text.lower().strip()
|
|
94
96
|
hits = sum(1 for m in ESCALATION_MARKERS if m in low)
|
|
95
97
|
softens = sum(1 for s in SOFTENERS if s in low)
|
|
96
|
-
|
|
98
|
+
# Arousal is a [0, 1] scale: a softening-only turn must floor at 0, not
|
|
99
|
+
# go negative and hand the policy state an impossible value.
|
|
100
|
+
score = max(0.0, min(1.0, 0.25 + hits * 0.22 - softens * 0.12))
|
|
97
101
|
if "?" in low or "!" in low:
|
|
98
102
|
score = min(1.0, score + 0.1)
|
|
99
103
|
return round(score, 3)
|
|
@@ -110,11 +114,13 @@ class HeuristicAffect:
|
|
|
110
114
|
|
|
111
115
|
def _escalating(self, user_turns: list[str]) -> bool:
|
|
112
116
|
"""Three or more consecutive turns opening with a wh-word."""
|
|
113
|
-
|
|
117
|
+
# Blank turns are not wh-word turns. Counting them let three turns in
|
|
118
|
+
# which only one or two actually opened with a wh-word read as a
|
|
119
|
+
# three-turn interrogation run. Same filter as ``_demand_pattern``.
|
|
120
|
+
recent = [t for t in user_turns[-3:] if t.strip()]
|
|
121
|
+
if len(recent) < 3:
|
|
114
122
|
return False
|
|
115
|
-
return all(
|
|
116
|
-
self._first_word(t) in QUESTION_OPENERS for t in user_turns[-3:] if t.strip()
|
|
117
|
-
)
|
|
123
|
+
return all(self._first_word(t) in QUESTION_OPENERS for t in recent)
|
|
118
124
|
|
|
119
125
|
def _first_word(self, text: str) -> str:
|
|
120
126
|
"""First token with punctuation stripped. 'Neden?' must match 'neden',
|
|
@@ -144,7 +150,6 @@ class HeuristicAffect:
|
|
|
144
150
|
has_prior_assistant: bool = False,
|
|
145
151
|
) -> str:
|
|
146
152
|
current = user_turns[-1] if user_turns else ""
|
|
147
|
-
low = current.lower()
|
|
148
153
|
|
|
149
154
|
if has_prior_assistant and self._has(current, CORRECTION_MARKERS):
|
|
150
155
|
return "correction"
|
|
@@ -223,5 +228,16 @@ class HeuristicAffect:
|
|
|
223
228
|
|
|
224
229
|
|
|
225
230
|
def load_cases(path: str) -> list[dict]:
|
|
231
|
+
cases: list[dict] = []
|
|
226
232
|
with open(path, encoding="utf-8") as fh:
|
|
227
|
-
|
|
233
|
+
for lineno, line in enumerate(fh, start=1):
|
|
234
|
+
if not line.strip():
|
|
235
|
+
continue
|
|
236
|
+
try:
|
|
237
|
+
case = strict_json_loads(line)
|
|
238
|
+
except (ValueError, UnicodeError) as exc:
|
|
239
|
+
raise ValueError(f"{path}:{lineno}: invalid JSON case: {exc}") from exc
|
|
240
|
+
if not isinstance(case, dict):
|
|
241
|
+
raise ValueError(f"{path}:{lineno}: JSON case must be an object")
|
|
242
|
+
cases.append(case)
|
|
243
|
+
return cases
|
|
@@ -8,10 +8,11 @@ import sys
|
|
|
8
8
|
from dataclasses import asdict
|
|
9
9
|
|
|
10
10
|
from ._deps import ML_INSTALL_COMMAND, OptionalDependencyError, ml_extra_available
|
|
11
|
+
from .json_contract import loads as strict_json_loads
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
def _load_messages(raw: str) -> list[dict]:
|
|
14
|
-
payload =
|
|
15
|
+
payload = strict_json_loads(raw)
|
|
15
16
|
if isinstance(payload, dict):
|
|
16
17
|
payload = payload.get("messages")
|
|
17
18
|
if not isinstance(payload, list) or not payload:
|
|
@@ -20,12 +21,18 @@ def _load_messages(raw: str) -> list[dict]:
|
|
|
20
21
|
for index, item in enumerate(payload):
|
|
21
22
|
if not isinstance(item, dict):
|
|
22
23
|
raise ValueError(f"message {index} must be an object")
|
|
23
|
-
role =
|
|
24
|
-
content =
|
|
25
|
-
if role not in {"user", "assistant", "system"}:
|
|
24
|
+
role = item.get("role")
|
|
25
|
+
content = item.get("content")
|
|
26
|
+
if not isinstance(role, str) or role not in {"user", "assistant", "system"}:
|
|
26
27
|
raise ValueError(f"message {index} has unsupported role {role!r}")
|
|
28
|
+
if not isinstance(content, str):
|
|
29
|
+
raise ValueError(f"message {index} content must be a string")
|
|
27
30
|
if not content.strip():
|
|
28
31
|
raise ValueError(f"message {index} has empty content")
|
|
32
|
+
try:
|
|
33
|
+
content.encode("utf-8")
|
|
34
|
+
except UnicodeError as exc:
|
|
35
|
+
raise ValueError(f"message {index} content must be valid UTF-8") from exc
|
|
29
36
|
messages.append({"role": role, "content": content})
|
|
30
37
|
if not any(message["role"] == "user" for message in messages):
|
|
31
38
|
raise ValueError("Transcript must contain at least one user turn.")
|
|
@@ -179,8 +186,9 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
179
186
|
print(f"{parser.prog}: error: {exc}", file=sys.stderr)
|
|
180
187
|
return 2
|
|
181
188
|
except (ValueError, json.JSONDecodeError, ImportError) as exc:
|
|
189
|
+
# ArgumentParser.error terminates with SystemExit(2).
|
|
182
190
|
parser.error(str(exc))
|
|
183
|
-
|
|
191
|
+
return 2 # pragma: no cover - parser.error raises SystemExit
|
|
184
192
|
|
|
185
193
|
|
|
186
194
|
if __name__ == "__main__":
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Unambiguous finite UTF-8 JSON for externally supplied transcripts and cases."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
11
|
+
output: dict[str, Any] = {}
|
|
12
|
+
for key, value in pairs:
|
|
13
|
+
if key in output:
|
|
14
|
+
raise ValueError("duplicate JSON object key")
|
|
15
|
+
output[key] = value
|
|
16
|
+
return output
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _finite_float(value: str) -> float:
|
|
20
|
+
number = float(value)
|
|
21
|
+
if not math.isfinite(number):
|
|
22
|
+
raise ValueError("non-finite JSON number")
|
|
23
|
+
return number
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _reject_constant(value: str) -> None:
|
|
27
|
+
raise ValueError(f"non-standard JSON constant: {value}")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def loads(text: str) -> Any:
|
|
31
|
+
data = json.loads(
|
|
32
|
+
text,
|
|
33
|
+
object_pairs_hook=_unique_keys,
|
|
34
|
+
parse_float=_finite_float,
|
|
35
|
+
parse_constant=_reject_constant,
|
|
36
|
+
)
|
|
37
|
+
try:
|
|
38
|
+
json.dumps(data, ensure_ascii=False, allow_nan=False).encode("utf-8")
|
|
39
|
+
except UnicodeError as exc:
|
|
40
|
+
raise ValueError("JSON text must contain valid UTF-8") from exc
|
|
41
|
+
return data
|
|
@@ -21,8 +21,6 @@ from typing import Any
|
|
|
21
21
|
# estimator, so its concrete type is deliberately not pinned here.
|
|
22
22
|
FITTED = object()
|
|
23
23
|
|
|
24
|
-
_FLOAT_TYPES = (int, float)
|
|
25
|
-
|
|
26
24
|
|
|
27
25
|
class ModelStateError(ValueError):
|
|
28
26
|
"""A persisted model artifact failed schema validation at load time."""
|
|
@@ -5,7 +5,6 @@ from dataclasses import asdict, dataclass, replace
|
|
|
5
5
|
|
|
6
6
|
from .textnorm import normalise_light as _normalise
|
|
7
7
|
|
|
8
|
-
|
|
9
8
|
SESSION_SCOPE_PATTERNS = (
|
|
10
9
|
r"\bbundan sonra\b",
|
|
11
10
|
r"\bbu sohbet(?:te| boyunca)\b",
|
|
@@ -51,8 +50,7 @@ def _session_scoped(text: str) -> bool:
|
|
|
51
50
|
if any(re.search(pattern, low) for pattern in SESSION_SCOPE_PATTERNS):
|
|
52
51
|
return True
|
|
53
52
|
return bool(
|
|
54
|
-
STRONG_TURKISH_STYLE_PATTERN.search(low)
|
|
55
|
-
or STRONG_ENGLISH_STYLE_PATTERN.search(low)
|
|
53
|
+
STRONG_TURKISH_STYLE_PATTERN.search(low) or STRONG_ENGLISH_STYLE_PATTERN.search(low)
|
|
56
54
|
)
|
|
57
55
|
|
|
58
56
|
|
|
@@ -60,7 +58,7 @@ def _updates(text: str) -> dict:
|
|
|
60
58
|
low = _normalise(text)
|
|
61
59
|
updates: dict[str, object] = {}
|
|
62
60
|
|
|
63
|
-
if re.search(r"\b(kısa|öz|brief|concise)\b", low):
|
|
61
|
+
if re.search(r"\b(kısa|öz|brief|briefly|concise)\b", low):
|
|
64
62
|
updates["verbosity"] = "low"
|
|
65
63
|
if re.search(r"\b(detaylı|ayrıntılı|detailed)\b|\bin detail\b", low):
|
|
66
64
|
updates["verbosity"] = "high"
|
|
@@ -68,12 +66,7 @@ def _updates(text: str) -> dict:
|
|
|
68
66
|
if re.search(r"\b(net|direkt|doğrudan|direct)\b|\bstraight to the point\b", low):
|
|
69
67
|
updates["directness"] = "high"
|
|
70
68
|
|
|
71
|
-
if
|
|
72
|
-
"tahmin etme" in low
|
|
73
|
-
or "uydurma" in low
|
|
74
|
-
or "don't guess" in low
|
|
75
|
-
or "do not guess" in low
|
|
76
|
-
):
|
|
69
|
+
if "tahmin etme" in low or "uydurma" in low or "don't guess" in low or "do not guess" in low:
|
|
77
70
|
updates["no_guess"] = True
|
|
78
71
|
|
|
79
72
|
if (
|
|
@@ -9,7 +9,7 @@ import subprocess
|
|
|
9
9
|
import threading
|
|
10
10
|
from dataclasses import dataclass, field
|
|
11
11
|
from pathlib import Path
|
|
12
|
-
from typing import Iterable
|
|
12
|
+
from typing import Callable, Iterable
|
|
13
13
|
|
|
14
14
|
from .auditor import audit_from_generation_metadata, regeneration_instruction
|
|
15
15
|
from .policies import Selector
|
|
@@ -19,8 +19,6 @@ from .trained_affect import TrainedAffect
|
|
|
19
19
|
from .trained_intent import TrainedIntent
|
|
20
20
|
from .trained_subtext import TrainedSubtext
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
24
22
|
LEAKAGE_FIELDS = {
|
|
25
23
|
"expected_policy",
|
|
26
24
|
"expected_intent",
|
|
@@ -49,19 +47,23 @@ def _force_kill(process: subprocess.Popen) -> None:
|
|
|
49
47
|
try:
|
|
50
48
|
process.terminate()
|
|
51
49
|
except OSError:
|
|
50
|
+
# The child may already have been reaped by the OS.
|
|
52
51
|
pass
|
|
53
52
|
try:
|
|
54
53
|
process.wait(timeout=5)
|
|
55
54
|
return
|
|
56
55
|
except subprocess.TimeoutExpired:
|
|
56
|
+
# Ignored the graceful signal; escalate to kill below.
|
|
57
57
|
pass
|
|
58
58
|
try:
|
|
59
59
|
process.kill()
|
|
60
60
|
except OSError:
|
|
61
|
+
# The child exited between the timeout and the kill.
|
|
61
62
|
pass
|
|
62
63
|
try:
|
|
63
64
|
process.wait(timeout=5)
|
|
64
65
|
except subprocess.TimeoutExpired:
|
|
66
|
+
# Last-resort wait; nothing further can be done without a block.
|
|
65
67
|
pass
|
|
66
68
|
|
|
67
69
|
|
|
@@ -162,17 +164,16 @@ def select_stratified_cases(cases: list[dict], limit: int) -> list[dict]:
|
|
|
162
164
|
deficit += quotas[stratum] - available
|
|
163
165
|
quotas[stratum] = available
|
|
164
166
|
|
|
167
|
+
# target <= len(cases), and quota sums to target. Therefore when a
|
|
168
|
+
# deficit exists, at least one stratum has spare capacity; an exhausted
|
|
169
|
+
# full pass is mathematically impossible for valid inputs.
|
|
165
170
|
while deficit > 0:
|
|
166
|
-
progressed = False
|
|
167
171
|
for stratum in strata:
|
|
168
172
|
if quotas[stratum] < len(groups[stratum]):
|
|
169
173
|
quotas[stratum] += 1
|
|
170
174
|
deficit -= 1
|
|
171
|
-
progressed = True
|
|
172
175
|
if deficit == 0:
|
|
173
176
|
break
|
|
174
|
-
if not progressed:
|
|
175
|
-
break
|
|
176
177
|
|
|
177
178
|
selected: list[dict] = []
|
|
178
179
|
max_quota = max(quotas.values(), default=0)
|
|
@@ -198,6 +199,17 @@ def stable_case_seed(global_seed: int, case_id: str) -> int:
|
|
|
198
199
|
return int.from_bytes(digest[:4], "big", signed=False)
|
|
199
200
|
|
|
200
201
|
|
|
202
|
+
# The experiment's own clock. Reproducibility is a property of the artifact, so
|
|
203
|
+
# the wall clock must never leak into a generated record: the default is a
|
|
204
|
+
# fixed epoch and callers who need a real timestamp inject their own clock.
|
|
205
|
+
DEFAULT_EXPERIMENT_CLOCK_EPOCH = 0.0
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _fixed_experiment_clock() -> float:
|
|
209
|
+
"""Default experiment clock: a constant, so runs differ only by content."""
|
|
210
|
+
return DEFAULT_EXPERIMENT_CLOCK_EPOCH
|
|
211
|
+
|
|
212
|
+
|
|
201
213
|
def file_sha256(path: str | Path) -> str:
|
|
202
214
|
digest = hashlib.sha256()
|
|
203
215
|
with open(path, "rb") as fh:
|
|
@@ -349,7 +361,7 @@ class CommandModel:
|
|
|
349
361
|
|
|
350
362
|
@staticmethod
|
|
351
363
|
def _decode_response(raw: str) -> str:
|
|
352
|
-
if not raw:
|
|
364
|
+
if not raw.strip():
|
|
353
365
|
raise RuntimeError("Model command returned an empty response.")
|
|
354
366
|
|
|
355
367
|
try:
|
|
@@ -654,7 +666,18 @@ def generate_pairs(
|
|
|
654
666
|
auditor_regenerations: int = 0,
|
|
655
667
|
length_matched_arm: bool = False,
|
|
656
668
|
length_profiles: tuple[str, ...] | None = None,
|
|
669
|
+
rng: random.Random | None = None,
|
|
670
|
+
clock: Callable[[], float] | None = None,
|
|
657
671
|
) -> list[dict]:
|
|
672
|
+
"""Generate the same-model baseline/EQ pairs for one experiment run.
|
|
673
|
+
|
|
674
|
+
Generation is deterministic: the only randomness is the control-arm
|
|
675
|
+
shuffling, seeded from ``seed``, and it is quarantined behind the injectable
|
|
676
|
+
``rng`` so a caller can supply the same generator without re-deriving it
|
|
677
|
+
from the seed. Any timestamp is quarantined behind ``clock``, whose default
|
|
678
|
+
is a fixed epoch, so the same experiment plus the same fixture serialises
|
|
679
|
+
to identical bytes.
|
|
680
|
+
"""
|
|
658
681
|
if auditor_regenerations not in {0, 1}:
|
|
659
682
|
raise ValueError("auditor_regenerations must be 0 or 1.")
|
|
660
683
|
profiles = tuple(length_profiles or ())
|
|
@@ -665,7 +688,8 @@ def generate_pairs(
|
|
|
665
688
|
f"{sorted(LENGTH_CONTROL_PROFILES)}"
|
|
666
689
|
)
|
|
667
690
|
|
|
668
|
-
rng = random.Random(seed)
|
|
691
|
+
rng = rng if rng is not None else random.Random(seed)
|
|
692
|
+
clock = clock if clock is not None else _fixed_experiment_clock
|
|
669
693
|
output: list[dict] = []
|
|
670
694
|
seen: set[str] = set()
|
|
671
695
|
|
|
@@ -769,6 +793,7 @@ def generate_pairs(
|
|
|
769
793
|
"model_id": model.model_id,
|
|
770
794
|
"temperature": model.temperature,
|
|
771
795
|
"pair_seed": pair_seed,
|
|
796
|
+
"generated_at_epoch": round(float(clock()), 6),
|
|
772
797
|
"order": ["baseline", "eq", *controls],
|
|
773
798
|
"same_model_both_arms": True,
|
|
774
799
|
"annotations_used": bool(annotated),
|
eq_layer-0.2.3/.gitignore
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|