eq-layer 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {eq_layer-0.2.2 → eq_layer-0.2.3}/.gitignore +1 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/PKG-INFO +20 -1
- {eq_layer-0.2.2 → eq_layer-0.2.3}/README.md +19 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/__init__.py +4 -0
- eq_layer-0.2.3/eq_layer/_deps.py +62 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/cli.py +17 -6
- eq_layer-0.2.3/eq_layer/model_state.py +99 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/trained_affect.py +23 -7
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/trained_breakdown.py +21 -7
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/trained_disagreement.py +21 -7
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/trained_intent.py +87 -12
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/trained_subtext.py +25 -11
- {eq_layer-0.2.2 → eq_layer-0.2.3}/pyproject.toml +1 -1
- {eq_layer-0.2.2 → eq_layer-0.2.3}/LICENSE +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/THIRD_PARTY_DATA.md +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/_version.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/ab_eval.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/actions.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/affect.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/auditor.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/coarse_discourse.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/data/intent_train.jsonl +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/dbdc3.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/dialogbank_iso.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/dialogue_reference.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/intent.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/intent_belief.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/interaction.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/policies.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/preferences.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/response_experiment.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/steer.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/textnorm.py +0 -0
- {eq_layer-0.2.2 → eq_layer-0.2.3}/eq_layer/tracker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: eq-layer
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Inspectable intent, affect, repair, uncertainty, and factored-action control for LLM responses
|
|
5
5
|
Project-URL: Homepage, https://github.com/Furox-Art/eq-layer
|
|
6
6
|
Project-URL: Repository, https://github.com/Furox-Art/eq-layer
|
|
@@ -554,6 +554,25 @@ detection — were tested and retained as no-go results instead of being wired
|
|
|
554
554
|
into policy routing. The tracker exposes where labels are direct, derived,
|
|
555
555
|
structural, fallback, or externally verified.
|
|
556
556
|
|
|
557
|
+
## Publishing
|
|
558
|
+
|
|
559
|
+
Two artifacts, one semantic version: PyPI `eq-layer` (the Python runtime and
|
|
560
|
+
control layer) and npm `eq-layer` (the portable Agent Skill installer and CLI
|
|
561
|
+
bridge). Each side keeps its version in exactly one canonical location, and the
|
|
562
|
+
release workflows verify the GitHub release tag against those fields before
|
|
563
|
+
uploading.
|
|
564
|
+
|
|
565
|
+
npm publishing picks its credential mode at run time, based on repository
|
|
566
|
+
settings only:
|
|
567
|
+
|
|
568
|
+
- `NPM_TOKEN` configured: token mode, `npm publish --access public
|
|
569
|
+
--no-provenance`;
|
|
570
|
+
- no `NPM_TOKEN`: trusted publishing over GitHub OIDC, `npm publish
|
|
571
|
+
--access public --provenance`.
|
|
572
|
+
|
|
573
|
+
Provenance is an OIDC-only attestation, so it is only requested when no token is
|
|
574
|
+
configured. See `docs/PUBLISHING.md` for the full flow.
|
|
575
|
+
|
|
557
576
|
## Contributing
|
|
558
577
|
|
|
559
578
|
Cases are the main entry point. Open one for a failure mode you can describe concretely — especially the ones where two policies seem equally right.
|
|
@@ -519,6 +519,25 @@ detection — were tested and retained as no-go results instead of being wired
|
|
|
519
519
|
into policy routing. The tracker exposes where labels are direct, derived,
|
|
520
520
|
structural, fallback, or externally verified.
|
|
521
521
|
|
|
522
|
+
## Publishing
|
|
523
|
+
|
|
524
|
+
Two artifacts, one semantic version: PyPI `eq-layer` (the Python runtime and
|
|
525
|
+
control layer) and npm `eq-layer` (the portable Agent Skill installer and CLI
|
|
526
|
+
bridge). Each side keeps its version in exactly one canonical location, and the
|
|
527
|
+
release workflows verify the GitHub release tag against those fields before
|
|
528
|
+
uploading.
|
|
529
|
+
|
|
530
|
+
npm publishing picks its credential mode at run time, based on repository
|
|
531
|
+
settings only:
|
|
532
|
+
|
|
533
|
+
- `NPM_TOKEN` configured: token mode, `npm publish --access public
|
|
534
|
+
--no-provenance`;
|
|
535
|
+
- no `NPM_TOKEN`: trusted publishing over GitHub OIDC, `npm publish
|
|
536
|
+
--access public --provenance`.
|
|
537
|
+
|
|
538
|
+
Provenance is an OIDC-only attestation, so it is only requested when no token is
|
|
539
|
+
configured. See `docs/PUBLISHING.md` for the full flow.
|
|
540
|
+
|
|
522
541
|
## Contributing
|
|
523
542
|
|
|
524
543
|
Cases are the main entry point. Open one for a failure mode you can describe concretely — especially the ones where two policies seem equally right.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from ._deps import OptionalDependencyError
|
|
1
2
|
from ._version import __version__
|
|
2
3
|
from .affect import AffectAdapter, HeuristicAffect, load_cases
|
|
3
4
|
from .auditor import AuditContext, AuditIssue, ResponseAudit, ResponseAuditor, audit_from_generation_metadata
|
|
@@ -5,6 +6,7 @@ from .actions import FactoredAction, RealizationControls, compose_action
|
|
|
5
6
|
from .intent import HeuristicIntent, IntentAdapter, IntentState
|
|
6
7
|
from .intent_belief import IntentBelief, IntentRiskDecision, decide_intent_action, expected_loss
|
|
7
8
|
from .interaction import InteractionQualityState, RepairState, infer_interaction_quality, infer_repair_state
|
|
9
|
+
from .model_state import ModelStateError
|
|
8
10
|
from .policies import POLICIES, REGISTRY, SUBTEXTS, AffectState, Policy, Register, Selector
|
|
9
11
|
from .preferences import SessionPreferences, infer_session_preferences
|
|
10
12
|
from .response_experiment import (
|
|
@@ -22,6 +24,8 @@ from .tracker import AnnotationStanceResolver, ConversationTracker, StanceDecisi
|
|
|
22
24
|
|
|
23
25
|
__all__ = [
|
|
24
26
|
"__version__",
|
|
27
|
+
"OptionalDependencyError",
|
|
28
|
+
"ModelStateError",
|
|
25
29
|
"AffectAdapter",
|
|
26
30
|
"AffectState",
|
|
27
31
|
"audit_from_generation_metadata",
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Typed boundary for the optional ``ml`` extra.
|
|
2
|
+
|
|
3
|
+
The trained adapters import scikit-learn and their artifact loaders import
|
|
4
|
+
joblib; both ship in ``eq-layer[ml]``. Every optional import in the package
|
|
5
|
+
goes through this module so a missing or unusable extra surfaces as one typed,
|
|
6
|
+
actionable error that names the package and the install command, instead of a
|
|
7
|
+
bare ``ImportError`` traceback at request time (audit M-2/M-7).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
ML_EXTRA = "ml"
|
|
13
|
+
ML_PACKAGE = "scikit-learn"
|
|
14
|
+
JOBLIB_PACKAGE = "joblib"
|
|
15
|
+
ML_INSTALL_COMMAND = 'pip install "eq-layer[ml]"'
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class OptionalDependencyError(ImportError):
|
|
19
|
+
"""An optional dependency of the ``ml`` extra is missing or unusable.
|
|
20
|
+
|
|
21
|
+
Subclasses ``ImportError`` so existing ``except ImportError`` fallbacks
|
|
22
|
+
(the CLI's heuristic intent fallback under ``--intent-mode auto``) keep
|
|
23
|
+
working, while callers that want to map the failure to their own exit code
|
|
24
|
+
or message can catch the typed error instead of matching on message text.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
purpose: str,
|
|
30
|
+
*,
|
|
31
|
+
package: str = ML_PACKAGE,
|
|
32
|
+
cause: BaseException | None = None,
|
|
33
|
+
install_command: str = ML_INSTALL_COMMAND,
|
|
34
|
+
) -> None:
|
|
35
|
+
detail = f" ({cause})" if cause is not None else ""
|
|
36
|
+
super().__init__(
|
|
37
|
+
f"{purpose} requires the optional ML dependencies: "
|
|
38
|
+
f"{package} is unavailable{detail}. "
|
|
39
|
+
f"Install with: {install_command}"
|
|
40
|
+
)
|
|
41
|
+
self.purpose = purpose
|
|
42
|
+
self.package = package
|
|
43
|
+
self.install_command = install_command
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def missing_ml_dependency(purpose: str, cause: BaseException) -> OptionalDependencyError:
|
|
47
|
+
"""Build the typed error for a failed scikit-learn import."""
|
|
48
|
+
return OptionalDependencyError(purpose, package=ML_PACKAGE, cause=cause)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def missing_joblib(purpose: str, cause: BaseException) -> OptionalDependencyError:
|
|
52
|
+
"""Build the typed error for a failed joblib import."""
|
|
53
|
+
return OptionalDependencyError(purpose, package=JOBLIB_PACKAGE, cause=cause)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def ml_extra_available() -> bool:
|
|
57
|
+
"""Report whether the optional ML extra can be imported at all."""
|
|
58
|
+
try:
|
|
59
|
+
import sklearn # noqa: F401
|
|
60
|
+
except ImportError:
|
|
61
|
+
return False
|
|
62
|
+
return True
|
|
@@ -7,6 +7,8 @@ import json
|
|
|
7
7
|
import sys
|
|
8
8
|
from dataclasses import asdict
|
|
9
9
|
|
|
10
|
+
from ._deps import ML_INSTALL_COMMAND, OptionalDependencyError, ml_extra_available
|
|
11
|
+
|
|
10
12
|
|
|
11
13
|
def _load_messages(raw: str) -> list[dict]:
|
|
12
14
|
payload = json.loads(raw)
|
|
@@ -38,9 +40,11 @@ def _intent_adapter(mode: str):
|
|
|
38
40
|
return HeuristicIntent(), "heuristic"
|
|
39
41
|
try:
|
|
40
42
|
return TrainedIntent.from_bundled(), "trained-tfidf-logreg"
|
|
41
|
-
except
|
|
43
|
+
except OptionalDependencyError:
|
|
42
44
|
if mode == "learned":
|
|
43
45
|
raise
|
|
46
|
+
# A missing optional extra is not a routing failure: degrade to the
|
|
47
|
+
# zero-dependency heuristic path and say which backend answered.
|
|
44
48
|
return HeuristicIntent(), "heuristic-fallback-no-ml-extra"
|
|
45
49
|
|
|
46
50
|
|
|
@@ -127,16 +131,17 @@ def _route(args: argparse.Namespace) -> int:
|
|
|
127
131
|
def _doctor() -> int:
|
|
128
132
|
from . import __version__
|
|
129
133
|
|
|
134
|
+
ml_available = ml_extra_available()
|
|
130
135
|
payload = {
|
|
131
136
|
"package": "eq-layer",
|
|
132
137
|
"version": __version__,
|
|
133
138
|
"python": sys.version.split()[0],
|
|
134
|
-
"ml_extra_available":
|
|
139
|
+
"ml_extra_available": ml_available,
|
|
135
140
|
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
141
|
+
if not ml_available:
|
|
142
|
+
payload["ml_extra_hint"] = (
|
|
143
|
+
f"Optional ML dependencies are unavailable. Install with: {ML_INSTALL_COMMAND}"
|
|
144
|
+
)
|
|
140
145
|
print(json.dumps(payload, indent=2))
|
|
141
146
|
return 0
|
|
142
147
|
|
|
@@ -167,6 +172,12 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
167
172
|
args = parser.parse_args(argv)
|
|
168
173
|
try:
|
|
169
174
|
return int(args.handler(args))
|
|
175
|
+
except OptionalDependencyError as exc:
|
|
176
|
+
# A missing optional extra is a configuration problem, not a usage
|
|
177
|
+
# error: report it on stderr with the install command and exit 2
|
|
178
|
+
# without dumping the usage block on top of the actionable message.
|
|
179
|
+
print(f"{parser.prog}: error: {exc}", file=sys.stderr)
|
|
180
|
+
return 2
|
|
170
181
|
except (ValueError, json.JSONDecodeError, ImportError) as exc:
|
|
171
182
|
parser.error(str(exc))
|
|
172
183
|
return 2
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Schema validation for persisted trained-model state (audit H-2).
|
|
2
|
+
|
|
3
|
+
A trained adapter is restored from a joblib artifact. Before this validation
|
|
4
|
+
the loaders indexed the payload directly, so a malformed artifact surfaced as a
|
|
5
|
+
raw ``KeyError`` for a missing key, or a ``TypeError``/``ValueError`` raised
|
|
6
|
+
from inside the dataclass constructor for the wrong field type. Either way the
|
|
7
|
+
caller got an unhandled path with no hint about which field was at fault.
|
|
8
|
+
|
|
9
|
+
Every loader in ``eq_layer/trained_*.py`` now validates the payload schema
|
|
10
|
+
before constructing the adapter, and raises one typed error naming the offending
|
|
11
|
+
field.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
# A field that must be present and non-None but whose value is an opaque fitted
|
|
21
|
+
# estimator, so its concrete type is deliberately not pinned here.
|
|
22
|
+
FITTED = object()
|
|
23
|
+
|
|
24
|
+
_FLOAT_TYPES = (int, float)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ModelStateError(ValueError):
|
|
28
|
+
"""A persisted model artifact failed schema validation at load time."""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
model: str,
|
|
33
|
+
field: str,
|
|
34
|
+
problem: str,
|
|
35
|
+
path: str | Path | None = None,
|
|
36
|
+
) -> None:
|
|
37
|
+
location = f" in {path}" if path is not None else ""
|
|
38
|
+
super().__init__(
|
|
39
|
+
f"{model} artifact{location} has an invalid {field!r} field: {problem}"
|
|
40
|
+
)
|
|
41
|
+
self.model = model
|
|
42
|
+
self.field = field
|
|
43
|
+
self.problem = problem
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _type_names(types: tuple[type, ...]) -> str:
|
|
47
|
+
return " or ".join(entry.__name__ for entry in types)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _check(
|
|
51
|
+
model: str,
|
|
52
|
+
field: str,
|
|
53
|
+
value: Any,
|
|
54
|
+
accepted: Any,
|
|
55
|
+
path: str | Path | None,
|
|
56
|
+
) -> None:
|
|
57
|
+
if accepted is FITTED:
|
|
58
|
+
if value is None:
|
|
59
|
+
raise ModelStateError(model, field, "must not be None", path)
|
|
60
|
+
return
|
|
61
|
+
if not isinstance(value, accepted):
|
|
62
|
+
raise ModelStateError(
|
|
63
|
+
model,
|
|
64
|
+
field,
|
|
65
|
+
f"expected {_type_names(accepted)}, got {type(value).__name__}",
|
|
66
|
+
path,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def validate_state(
|
|
71
|
+
payload: Any,
|
|
72
|
+
*,
|
|
73
|
+
model: str,
|
|
74
|
+
path: str | Path | None = None,
|
|
75
|
+
required: Mapping[str, Any] | None = None,
|
|
76
|
+
optional: Mapping[str, Any] | None = None,
|
|
77
|
+
) -> None:
|
|
78
|
+
"""Validate a persisted model artifact before it is used.
|
|
79
|
+
|
|
80
|
+
``required``/``optional`` map a field name to an accepted type tuple, or to
|
|
81
|
+
``FITTED`` for an opaque fitted estimator. A missing required key, an empty
|
|
82
|
+
payload and a wrongly typed field each raise ``ModelStateError`` naming the
|
|
83
|
+
offending field.
|
|
84
|
+
"""
|
|
85
|
+
if not isinstance(payload, Mapping):
|
|
86
|
+
raise ModelStateError(
|
|
87
|
+
model,
|
|
88
|
+
"payload",
|
|
89
|
+
f"expected a mapping of fields, got {type(payload).__name__}",
|
|
90
|
+
path,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
for spec, must_exist in ((required or {}, True), (optional or {}, False)):
|
|
94
|
+
for field, accepted in spec.items():
|
|
95
|
+
if field not in payload:
|
|
96
|
+
if must_exist:
|
|
97
|
+
raise ModelStateError(model, field, "missing required key", path)
|
|
98
|
+
continue
|
|
99
|
+
_check(model, field, payload[field], accepted, path)
|
|
@@ -5,7 +5,9 @@ from dataclasses import dataclass, field
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from typing import Iterable
|
|
7
7
|
|
|
8
|
+
from ._deps import missing_joblib, missing_ml_dependency
|
|
8
9
|
from .affect import HeuristicAffect
|
|
10
|
+
from .model_state import FITTED, validate_state
|
|
9
11
|
from .policies import AffectState
|
|
10
12
|
|
|
11
13
|
EMOBANK_COMMIT = "248ce2a43e165a66d31aeaed83cff9641d6654e0"
|
|
@@ -23,10 +25,7 @@ def build_affect_pipeline():
|
|
|
23
25
|
from sklearn.linear_model import Ridge
|
|
24
26
|
from sklearn.pipeline import FeatureUnion, Pipeline
|
|
25
27
|
except ImportError as exc:
|
|
26
|
-
raise
|
|
27
|
-
"TrainedAffect requires the optional ML dependencies. "
|
|
28
|
-
'Install with: pip install "eq-layer[ml]"'
|
|
29
|
-
) from exc
|
|
28
|
+
raise missing_ml_dependency("TrainedAffect", exc) from exc
|
|
30
29
|
|
|
31
30
|
features = FeatureUnion(
|
|
32
31
|
[
|
|
@@ -205,7 +204,7 @@ class TrainedAffect:
|
|
|
205
204
|
try:
|
|
206
205
|
import joblib
|
|
207
206
|
except ImportError as exc:
|
|
208
|
-
raise
|
|
207
|
+
raise missing_joblib("Saving a trained affect model", exc) from exc
|
|
209
208
|
|
|
210
209
|
joblib.dump(
|
|
211
210
|
{
|
|
@@ -222,8 +221,25 @@ class TrainedAffect:
|
|
|
222
221
|
try:
|
|
223
222
|
import joblib
|
|
224
223
|
except ImportError as exc:
|
|
225
|
-
raise
|
|
226
|
-
|
|
224
|
+
raise missing_joblib("Loading a trained affect model", exc) from exc
|
|
225
|
+
try:
|
|
226
|
+
payload = joblib.load(path)
|
|
227
|
+
except ImportError as exc:
|
|
228
|
+
# Unpickling an artifact fitted on another machine needs the same
|
|
229
|
+
# optional extra, so a missing module must surface as a typed error
|
|
230
|
+
# rather than a raw traceback at request time.
|
|
231
|
+
raise missing_ml_dependency("Loading a trained affect model", exc) from exc
|
|
232
|
+
validate_state(
|
|
233
|
+
payload,
|
|
234
|
+
model="TrainedAffect",
|
|
235
|
+
path=path,
|
|
236
|
+
required={"model": FITTED},
|
|
237
|
+
optional={
|
|
238
|
+
"training_examples": (int,),
|
|
239
|
+
"training_source": (str,),
|
|
240
|
+
"training_license": (str,),
|
|
241
|
+
},
|
|
242
|
+
)
|
|
227
243
|
return cls(
|
|
228
244
|
model=payload["model"],
|
|
229
245
|
training_examples=int(payload.get("training_examples", 0)),
|
|
@@ -4,6 +4,9 @@ from dataclasses import dataclass
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Iterable
|
|
6
6
|
|
|
7
|
+
from ._deps import missing_joblib, missing_ml_dependency
|
|
8
|
+
from .model_state import FITTED, validate_state
|
|
9
|
+
|
|
7
10
|
|
|
8
11
|
def build_breakdown_pipeline():
|
|
9
12
|
try:
|
|
@@ -11,10 +14,7 @@ def build_breakdown_pipeline():
|
|
|
11
14
|
from sklearn.linear_model import SGDClassifier
|
|
12
15
|
from sklearn.pipeline import FeatureUnion, Pipeline
|
|
13
16
|
except ImportError as exc:
|
|
14
|
-
raise
|
|
15
|
-
"TrainedBreakdown requires the optional ML dependencies. "
|
|
16
|
-
'Install with: pip install "eq-layer[ml]"'
|
|
17
|
-
) from exc
|
|
17
|
+
raise missing_ml_dependency("TrainedBreakdown", exc) from exc
|
|
18
18
|
|
|
19
19
|
features = FeatureUnion(
|
|
20
20
|
[
|
|
@@ -115,7 +115,7 @@ class TrainedBreakdown:
|
|
|
115
115
|
try:
|
|
116
116
|
import joblib
|
|
117
117
|
except ImportError as exc:
|
|
118
|
-
raise
|
|
118
|
+
raise missing_joblib("Saving breakdown model", exc) from exc
|
|
119
119
|
|
|
120
120
|
joblib.dump(
|
|
121
121
|
{
|
|
@@ -132,9 +132,23 @@ class TrainedBreakdown:
|
|
|
132
132
|
try:
|
|
133
133
|
import joblib
|
|
134
134
|
except ImportError as exc:
|
|
135
|
-
raise
|
|
135
|
+
raise missing_joblib("Loading breakdown model", exc) from exc
|
|
136
136
|
|
|
137
|
-
|
|
137
|
+
try:
|
|
138
|
+
payload = joblib.load(path)
|
|
139
|
+
except ImportError as exc:
|
|
140
|
+
raise missing_ml_dependency("Loading breakdown model", exc) from exc
|
|
141
|
+
validate_state(
|
|
142
|
+
payload,
|
|
143
|
+
model="TrainedBreakdown",
|
|
144
|
+
path=path,
|
|
145
|
+
required={"model": FITTED},
|
|
146
|
+
optional={
|
|
147
|
+
"training_examples": (int,),
|
|
148
|
+
"positive_examples": (int,),
|
|
149
|
+
"threshold": (int, float),
|
|
150
|
+
},
|
|
151
|
+
)
|
|
138
152
|
return cls(
|
|
139
153
|
model=payload["model"],
|
|
140
154
|
training_examples=int(payload.get("training_examples", 0)),
|
|
@@ -4,6 +4,9 @@ from dataclasses import dataclass
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Iterable
|
|
6
6
|
|
|
7
|
+
from ._deps import missing_joblib, missing_ml_dependency
|
|
8
|
+
from .model_state import FITTED, validate_state
|
|
9
|
+
|
|
7
10
|
COARSE_DISCOURSE_ARCHIVE_SHA256 = (
|
|
8
11
|
"33cc25e906e677881c0031c1e134460b1e389fac707f2e52fe59b484dc6d0b61"
|
|
9
12
|
)
|
|
@@ -15,10 +18,7 @@ def build_disagreement_pipeline():
|
|
|
15
18
|
from sklearn.linear_model import SGDClassifier
|
|
16
19
|
from sklearn.pipeline import FeatureUnion, Pipeline
|
|
17
20
|
except ImportError as exc:
|
|
18
|
-
raise
|
|
19
|
-
"TrainedDisagreement requires the optional ML dependencies. "
|
|
20
|
-
'Install with: pip install "eq-layer[ml]"'
|
|
21
|
-
) from exc
|
|
21
|
+
raise missing_ml_dependency("TrainedDisagreement", exc) from exc
|
|
22
22
|
|
|
23
23
|
features = FeatureUnion(
|
|
24
24
|
[
|
|
@@ -120,7 +120,7 @@ class TrainedDisagreement:
|
|
|
120
120
|
try:
|
|
121
121
|
import joblib
|
|
122
122
|
except ImportError as exc:
|
|
123
|
-
raise
|
|
123
|
+
raise missing_joblib("Saving disagreement model", exc) from exc
|
|
124
124
|
|
|
125
125
|
joblib.dump(
|
|
126
126
|
{
|
|
@@ -137,9 +137,23 @@ class TrainedDisagreement:
|
|
|
137
137
|
try:
|
|
138
138
|
import joblib
|
|
139
139
|
except ImportError as exc:
|
|
140
|
-
raise
|
|
140
|
+
raise missing_joblib("Loading disagreement model", exc) from exc
|
|
141
141
|
|
|
142
|
-
|
|
142
|
+
try:
|
|
143
|
+
payload = joblib.load(path)
|
|
144
|
+
except ImportError as exc:
|
|
145
|
+
raise missing_ml_dependency("Loading disagreement model", exc) from exc
|
|
146
|
+
validate_state(
|
|
147
|
+
payload,
|
|
148
|
+
model="TrainedDisagreement",
|
|
149
|
+
path=path,
|
|
150
|
+
required={"model": FITTED},
|
|
151
|
+
optional={
|
|
152
|
+
"training_examples": (int,),
|
|
153
|
+
"positive_examples": (int,),
|
|
154
|
+
"threshold": (int, float),
|
|
155
|
+
},
|
|
156
|
+
)
|
|
143
157
|
return cls(
|
|
144
158
|
model=payload["model"],
|
|
145
159
|
training_examples=int(payload.get("training_examples", 0)),
|
|
@@ -2,12 +2,15 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
from dataclasses import dataclass, field, replace
|
|
5
|
+
from functools import lru_cache
|
|
5
6
|
from importlib import resources
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
from typing import Iterable
|
|
8
9
|
|
|
10
|
+
from ._deps import missing_joblib, missing_ml_dependency
|
|
9
11
|
from .intent import HeuristicIntent, IntentState
|
|
10
12
|
from .intent_belief import IntentBelief, decide_intent_action, fuse_dialogue_evidence
|
|
13
|
+
from .model_state import FITTED, validate_state
|
|
11
14
|
|
|
12
15
|
|
|
13
16
|
def build_pipeline():
|
|
@@ -17,10 +20,7 @@ def build_pipeline():
|
|
|
17
20
|
from sklearn.linear_model import LogisticRegression
|
|
18
21
|
from sklearn.pipeline import FeatureUnion, Pipeline
|
|
19
22
|
except ImportError as exc:
|
|
20
|
-
raise
|
|
21
|
-
"TrainedIntent requires the optional ML dependencies. "
|
|
22
|
-
'Install with: pip install "eq-layer[ml]"'
|
|
23
|
-
) from exc
|
|
23
|
+
raise missing_ml_dependency("TrainedIntent", exc) from exc
|
|
24
24
|
|
|
25
25
|
features = FeatureUnion(
|
|
26
26
|
[
|
|
@@ -64,6 +64,62 @@ def bundled_training_records() -> list[dict]:
|
|
|
64
64
|
return [json.loads(line) for line in fh if line.strip()]
|
|
65
65
|
|
|
66
66
|
|
|
67
|
+
def bundled_records_path() -> Path:
|
|
68
|
+
"""Locate the bundled corpus on the filesystem, when it has a path.
|
|
69
|
+
|
|
70
|
+
The cache key needs path+mtime so an edited corpus invalidates the memo.
|
|
71
|
+
Zipped/egg distributions expose no real path, in which case this raises and
|
|
72
|
+
the cache falls back to a stable key (a zipped corpus cannot change under
|
|
73
|
+
the running process anyway).
|
|
74
|
+
"""
|
|
75
|
+
here = Path(__file__).resolve()
|
|
76
|
+
for parent in here.parents:
|
|
77
|
+
for candidate in (parent / "eq_layer" / "data", parent / "data"):
|
|
78
|
+
artifact = candidate / "intent_train.jsonl"
|
|
79
|
+
if artifact.is_file():
|
|
80
|
+
return artifact
|
|
81
|
+
raise FileNotFoundError("Bundled intent training corpus has no filesystem path.")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _bundled_cache_key(
|
|
85
|
+
confidence_threshold: float,
|
|
86
|
+
margin_threshold: float,
|
|
87
|
+
) -> tuple[str, int, float, float]:
|
|
88
|
+
"""Cache key for the bundled fit: artifact path + mtime + thresholds."""
|
|
89
|
+
try:
|
|
90
|
+
artifact = bundled_records_path()
|
|
91
|
+
except FileNotFoundError:
|
|
92
|
+
return (":resources:", 0, float(confidence_threshold), float(margin_threshold))
|
|
93
|
+
return (
|
|
94
|
+
str(artifact),
|
|
95
|
+
artifact.stat().st_mtime_ns,
|
|
96
|
+
float(confidence_threshold),
|
|
97
|
+
float(margin_threshold),
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@lru_cache(maxsize=8)
|
|
102
|
+
def _fit_bundled(
|
|
103
|
+
cls: type["TrainedIntent"],
|
|
104
|
+
artifact: str,
|
|
105
|
+
mtime_ns: int,
|
|
106
|
+
confidence_threshold: float,
|
|
107
|
+
margin_threshold: float,
|
|
108
|
+
) -> "TrainedIntent":
|
|
109
|
+
"""Fit once per (artifact, mtime, thresholds); cached for the process."""
|
|
110
|
+
adapter = cls(
|
|
111
|
+
confidence_threshold=confidence_threshold,
|
|
112
|
+
margin_threshold=margin_threshold,
|
|
113
|
+
)
|
|
114
|
+
adapter.fit(bundled_training_records())
|
|
115
|
+
return adapter
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def clear_bundled_cache() -> None:
|
|
119
|
+
"""Drop the memoised bundled fits (used by tests and long-lived processes)."""
|
|
120
|
+
_fit_bundled.cache_clear()
|
|
121
|
+
|
|
122
|
+
|
|
67
123
|
@dataclass
|
|
68
124
|
class TrainedIntent:
|
|
69
125
|
"""Trainable intent adapter.
|
|
@@ -86,12 +142,18 @@ class TrainedIntent:
|
|
|
86
142
|
confidence_threshold: float = 0.40,
|
|
87
143
|
margin_threshold: float = 0.08,
|
|
88
144
|
) -> "TrainedIntent":
|
|
89
|
-
adapter
|
|
90
|
-
|
|
91
|
-
|
|
145
|
+
"""Return the bundled intent adapter, fitting it at most once.
|
|
146
|
+
|
|
147
|
+
The fit is deterministic but expensive (~seconds), so it is memoised
|
|
148
|
+
per process on (artifact path, artifact mtime, both thresholds). A
|
|
149
|
+
repeated call is O(1) and returns the *same* adapter instance, which
|
|
150
|
+
therefore has to be treated as read-only: callers must not call
|
|
151
|
+
``fit`` on it or they would mutate every other user of the cache.
|
|
152
|
+
"""
|
|
153
|
+
return _fit_bundled(
|
|
154
|
+
cls,
|
|
155
|
+
*_bundled_cache_key(confidence_threshold, margin_threshold),
|
|
92
156
|
)
|
|
93
|
-
adapter.fit(bundled_training_records())
|
|
94
|
-
return adapter
|
|
95
157
|
|
|
96
158
|
@classmethod
|
|
97
159
|
def from_jsonl(
|
|
@@ -265,7 +327,7 @@ class TrainedIntent:
|
|
|
265
327
|
try:
|
|
266
328
|
import joblib
|
|
267
329
|
except ImportError as exc:
|
|
268
|
-
raise
|
|
330
|
+
raise missing_joblib("Saving a trained intent model", exc) from exc
|
|
269
331
|
joblib.dump(
|
|
270
332
|
{
|
|
271
333
|
"model": self.model,
|
|
@@ -281,8 +343,21 @@ class TrainedIntent:
|
|
|
281
343
|
try:
|
|
282
344
|
import joblib
|
|
283
345
|
except ImportError as exc:
|
|
284
|
-
raise
|
|
285
|
-
|
|
346
|
+
raise missing_joblib("Loading a trained intent model", exc) from exc
|
|
347
|
+
try:
|
|
348
|
+
payload = joblib.load(path)
|
|
349
|
+
except ImportError as exc:
|
|
350
|
+
raise missing_ml_dependency("Loading a trained intent model", exc) from exc
|
|
351
|
+
validate_state(
|
|
352
|
+
payload,
|
|
353
|
+
model="TrainedIntent",
|
|
354
|
+
path=path,
|
|
355
|
+
required={"model": FITTED, "confidence_threshold": (int, float)},
|
|
356
|
+
optional={
|
|
357
|
+
"margin_threshold": (int, float),
|
|
358
|
+
"training_examples": (int,),
|
|
359
|
+
},
|
|
360
|
+
)
|
|
286
361
|
return cls(
|
|
287
362
|
confidence_threshold=float(payload["confidence_threshold"]),
|
|
288
363
|
margin_threshold=float(payload.get("margin_threshold", 0.08)),
|
|
@@ -4,7 +4,9 @@ from dataclasses import dataclass, field
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Iterable
|
|
6
6
|
|
|
7
|
+
from ._deps import missing_joblib, missing_ml_dependency
|
|
7
8
|
from .affect import CORRECTION_MARKERS, DISCLOSURE_MARKERS, HeuristicAffect
|
|
9
|
+
from .model_state import FITTED, validate_state
|
|
8
10
|
from .policies import AffectState
|
|
9
11
|
|
|
10
12
|
XDAILYDIALOG_COMMIT = "6e7ecf54c9f169215b4b8c18995c7aac74117127"
|
|
@@ -104,10 +106,7 @@ def build_feature_extractor():
|
|
|
104
106
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
105
107
|
from sklearn.pipeline import FeatureUnion
|
|
106
108
|
except ImportError as exc:
|
|
107
|
-
raise
|
|
108
|
-
"TrainedSubtext requires the optional ML dependencies. "
|
|
109
|
-
'Install with: pip install "eq-layer[ml]"'
|
|
110
|
-
) from exc
|
|
109
|
+
raise missing_ml_dependency("TrainedSubtext", exc) from exc
|
|
111
110
|
|
|
112
111
|
return FeatureUnion(
|
|
113
112
|
[
|
|
@@ -141,10 +140,7 @@ def build_classifier():
|
|
|
141
140
|
try:
|
|
142
141
|
from sklearn.linear_model import SGDClassifier
|
|
143
142
|
except ImportError as exc:
|
|
144
|
-
raise
|
|
145
|
-
"TrainedSubtext requires the optional ML dependencies. "
|
|
146
|
-
'Install with: pip install "eq-layer[ml]"'
|
|
147
|
-
) from exc
|
|
143
|
+
raise missing_ml_dependency("TrainedSubtext", exc) from exc
|
|
148
144
|
|
|
149
145
|
return SGDClassifier(
|
|
150
146
|
loss="log_loss",
|
|
@@ -375,7 +371,7 @@ class TrainedSubtext:
|
|
|
375
371
|
try:
|
|
376
372
|
import joblib
|
|
377
373
|
except ImportError as exc:
|
|
378
|
-
raise
|
|
374
|
+
raise missing_joblib("Saving a trained subtext model", exc) from exc
|
|
379
375
|
|
|
380
376
|
joblib.dump(
|
|
381
377
|
{
|
|
@@ -394,9 +390,27 @@ class TrainedSubtext:
|
|
|
394
390
|
try:
|
|
395
391
|
import joblib
|
|
396
392
|
except ImportError as exc:
|
|
397
|
-
raise
|
|
393
|
+
raise missing_joblib("Loading a trained subtext model", exc) from exc
|
|
398
394
|
|
|
399
|
-
|
|
395
|
+
try:
|
|
396
|
+
payload = joblib.load(path)
|
|
397
|
+
except ImportError as exc:
|
|
398
|
+
raise missing_ml_dependency("Loading a trained subtext model", exc) from exc
|
|
399
|
+
validate_state(
|
|
400
|
+
payload,
|
|
401
|
+
model="TrainedSubtext",
|
|
402
|
+
path=path,
|
|
403
|
+
required={
|
|
404
|
+
"features": FITTED,
|
|
405
|
+
"act_model": FITTED,
|
|
406
|
+
"emotion_model": FITTED,
|
|
407
|
+
},
|
|
408
|
+
optional={
|
|
409
|
+
"training_examples": (int,),
|
|
410
|
+
"act_confidence_threshold": (int, float),
|
|
411
|
+
"emotion_confidence_threshold": (int, float),
|
|
412
|
+
},
|
|
413
|
+
)
|
|
400
414
|
return cls(
|
|
401
415
|
features=payload["features"],
|
|
402
416
|
act_model=payload["act_model"],
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|