awdecide 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- awdecide/__init__.py +38 -0
- awdecide/__main__.py +7 -0
- awdecide/_doctor.py +190 -0
- awdecide/_selftest.py +284 -0
- awdecide/backends.py +238 -0
- awdecide/bridge.py +263 -0
- awdecide/cli.py +256 -0
- awdecide/contract.py +132 -0
- awdecide/door.py +357 -0
- awdecide/ledger.py +179 -0
- awdecide/loop.py +249 -0
- awdecide/mcp.py +204 -0
- awdecide/sources.py +315 -0
- awdecide-0.3.0.dist-info/METADATA +214 -0
- awdecide-0.3.0.dist-info/RECORD +19 -0
- awdecide-0.3.0.dist-info/WHEEL +5 -0
- awdecide-0.3.0.dist-info/entry_points.txt +2 -0
- awdecide-0.3.0.dist-info/licenses/LICENSE +15 -0
- awdecide-0.3.0.dist-info/top_level.txt +1 -0
awdecide/__init__.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""awdecide -- Aither World Decide.
|
|
2
|
+
|
|
3
|
+
One typed-decision contract -- choice / score / bool with a probability -- over
|
|
4
|
+
a ladder of backends you already run (rules, any local model as a callable, an
|
|
5
|
+
OpenAI-wire model's logprobs), fail-closed (decided=False is an answer), with a
|
|
6
|
+
Brier ledger that resolves every decision against its outcome and reports
|
|
7
|
+
whether the probabilities carry information beyond the base rate.
|
|
8
|
+
|
|
9
|
+
from awdecide import Question, Ladder, RulesBackend, Ledger
|
|
10
|
+
|
|
11
|
+
q = {"category": Question.choice(["billing", "technical", "sales"]),
|
|
12
|
+
"urgent": Question.bool(min_confidence=0.7)}
|
|
13
|
+
answers = Ladder([RulesBackend([(r"refund|invoice", "billing")])]).decide(state, q)
|
|
14
|
+
answers["category"].value, answers["category"].probability, answers["urgent"].decided
|
|
15
|
+
|
|
16
|
+
Stdlib only. Nothing here sends the state anywhere you did not name.
|
|
17
|
+
"""
|
|
18
|
+
from .backends import (
|
|
19
|
+
Backend,
|
|
20
|
+
CallableBackend,
|
|
21
|
+
Ladder,
|
|
22
|
+
LogprobBackend,
|
|
23
|
+
RulesBackend,
|
|
24
|
+
default_ladder,
|
|
25
|
+
parse_question_spec,
|
|
26
|
+
)
|
|
27
|
+
from .contract import Decision, Question, from_probabilities, normalize, undecided
|
|
28
|
+
from .door import DoorBackend, decision_from_door, door_request
|
|
29
|
+
from .ledger import Ledger
|
|
30
|
+
from .loop import ChatBackend, Loop
|
|
31
|
+
|
|
32
|
+
__version__ = "0.3.0"
|
|
33
|
+
__all__ = [
|
|
34
|
+
"Backend", "CallableBackend", "Ladder", "LogprobBackend", "RulesBackend",
|
|
35
|
+
"default_ladder", "parse_question_spec", "Decision", "Question",
|
|
36
|
+
"from_probabilities", "normalize", "undecided", "Ledger", "DoorBackend",
|
|
37
|
+
"decision_from_door", "door_request", "ChatBackend", "Loop", "__version__",
|
|
38
|
+
]
|
awdecide/__main__.py
ADDED
awdecide/_doctor.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Stack-aware `doctor` for awdecide.
|
|
2
|
+
|
|
3
|
+
GENERATED BY gen_aw_doctor.py -- DO NOT EDIT.
|
|
4
|
+
Regenerate it with the generator named above; a hand-edit here is reverted by
|
|
5
|
+
the next run and fails the parity gate.
|
|
6
|
+
|
|
7
|
+
Why a doctor exists at all: the aw* bricks are designed to COMPOSE, so the
|
|
8
|
+
interesting failures live BETWEEN them. "awdecide is installed" is not the useful
|
|
9
|
+
fact -- "awdecide is installed and the thing it pairs with is not" is. This reports
|
|
10
|
+
the whole stack, not just itself.
|
|
11
|
+
|
|
12
|
+
stdlib only, on purpose: a diagnostic that cannot run because a dependency is
|
|
13
|
+
missing is worthless precisely when you need it.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import importlib.util
|
|
18
|
+
import os
|
|
19
|
+
import shutil
|
|
20
|
+
import sys
|
|
21
|
+
|
|
22
|
+
#: Frozen from the Aither World registry at generation time. A shipped
|
|
23
|
+
#: package cannot read the registry, and a doctor that guessed at the family
|
|
24
|
+
#: would go stale in silence. Regenerate to update.
|
|
25
|
+
SELF = 'awdecide'
|
|
26
|
+
FAMILY = ['awask', 'awavatar', 'awbac', 'awbrain', 'awbrowse', 'awclassify', 'awdeck', 'awdelphi', 'awdit', 'awembed', 'awevolve', 'awfind', 'awflow', 'awfocus', 'awgit', 'awgraph', 'awgym', 'awiam', 'awkno', 'awm', 'awmail', 'awmine', 'awnboard', 'awnest', 'awnet', 'awnode', 'awpool', 'awpredict', 'awprism', 'awprove', 'awreason', 'awrecover', 'awrecurse', 'awrelay', 'awrena', 'awrepl', 'awreport', 'awresearch', 'awrise', 'awrouter', 'awrtifact', 'awrun', 'awscreen', 'awseal', 'awsettings', 'awshare', 'awsprite', 'awstorage', 'awswarm', 'awtax', 'awtoll', 'awtunnel', 'awvision', 'awvoice', 'awwall', 'gawbbonet']
|
|
27
|
+
PAIRS_WITH = ['awclassify', 'adk', 'awgraph', 'awpredict', 'awprove', 'awrepl', 'awtoll']
|
|
28
|
+
|
|
29
|
+
#: This brick's OWN config, read out of its source at generation time.
|
|
30
|
+
#: ENV_REQUIRED is `os.environ["X"]` -- absent, that is a KeyError the moment
|
|
31
|
+
#: the line runs. ENV_OPTIONAL is `os.getenv("X")`, which returns None and lets
|
|
32
|
+
#: the caller cope. Only this brick's namespace is listed: reporting the
|
|
33
|
+
#: platform-wide vars it also touches would be noise, and a doctor that floods
|
|
34
|
+
#: gets ignored.
|
|
35
|
+
ENV_REQUIRED = []
|
|
36
|
+
ENV_OPTIONAL = ['AWDECIDE_DB']
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _installed(mod: str) -> "str | None":
|
|
40
|
+
"""Version if importable, else None. Never raises -- a broken sibling must
|
|
41
|
+
not take the diagnostic down with it."""
|
|
42
|
+
try:
|
|
43
|
+
if importlib.util.find_spec(mod) is None:
|
|
44
|
+
return None
|
|
45
|
+
except (ImportError, ValueError):
|
|
46
|
+
return None
|
|
47
|
+
try:
|
|
48
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
49
|
+
try:
|
|
50
|
+
return version(mod)
|
|
51
|
+
except PackageNotFoundError:
|
|
52
|
+
return "installed"
|
|
53
|
+
except Exception:
|
|
54
|
+
return "installed"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def report(out=None) -> int:
|
|
58
|
+
"""Print the stack picture. 0 = this brick and its pairs are present."""
|
|
59
|
+
out = out or sys.stdout
|
|
60
|
+
print(f"{SELF} doctor", file=out)
|
|
61
|
+
|
|
62
|
+
mine = _installed(SELF)
|
|
63
|
+
print(f" self {SELF} {mine or 'NOT IMPORTABLE'}", file=out)
|
|
64
|
+
shim = shutil.which(SELF)
|
|
65
|
+
print(f" command {shim or 'not on PATH'}", file=out)
|
|
66
|
+
|
|
67
|
+
# The stack. Siblings this brick pairs with are called out separately,
|
|
68
|
+
# because a missing pair is a REASON, while a missing unrelated brick is
|
|
69
|
+
# just a fact about your machine.
|
|
70
|
+
missing_pairs, present = [], []
|
|
71
|
+
for name in FAMILY:
|
|
72
|
+
v = _installed(name)
|
|
73
|
+
if v:
|
|
74
|
+
present.append(name)
|
|
75
|
+
elif name in PAIRS_WITH:
|
|
76
|
+
missing_pairs.append(name)
|
|
77
|
+
print(f" stack {len(present)}/{len(FAMILY)} aw* packages installed",
|
|
78
|
+
file=out)
|
|
79
|
+
if present:
|
|
80
|
+
print(f" {' '.join(sorted(present))}", file=out)
|
|
81
|
+
|
|
82
|
+
missing_req = [v for v in ENV_REQUIRED if not os.environ.get(v)]
|
|
83
|
+
if ENV_REQUIRED or ENV_OPTIONAL:
|
|
84
|
+
have = sum(1 for v in ENV_REQUIRED + ENV_OPTIONAL if os.environ.get(v))
|
|
85
|
+
total = len(ENV_REQUIRED) + len(ENV_OPTIONAL)
|
|
86
|
+
print(f" config {have}/{total} of this brick's own vars set", file=out)
|
|
87
|
+
if missing_req:
|
|
88
|
+
# Not a preference. os.environ[...] raises the moment it runs.
|
|
89
|
+
print(f" MISSING REQUIRED: {' '.join(missing_req)}", file=out)
|
|
90
|
+
|
|
91
|
+
local = _local_checks()
|
|
92
|
+
for line in local:
|
|
93
|
+
print(f" {line}", file=out)
|
|
94
|
+
problems, unjudged = _local_verdict()
|
|
95
|
+
|
|
96
|
+
if mine is None:
|
|
97
|
+
print(f"\nverdict: {SELF} itself is not importable. Reinstall it before "
|
|
98
|
+
f"anything else here means much.", file=out)
|
|
99
|
+
return 1
|
|
100
|
+
if missing_req:
|
|
101
|
+
print(f"\nverdict: {SELF} is missing required config "
|
|
102
|
+
f"({', '.join(missing_req)}). Those are read with os.environ[...], "
|
|
103
|
+
f"so the code path that needs them raises rather than degrades.",
|
|
104
|
+
file=out)
|
|
105
|
+
return 1
|
|
106
|
+
# A measured NO outranks a missing optional pair: awrise printed
|
|
107
|
+
# "nothing wakes run-due" and still exited 0 because the missing-pairs
|
|
108
|
+
# branch returned first (measured 2026-09-20).
|
|
109
|
+
if problems:
|
|
110
|
+
for p in problems:
|
|
111
|
+
print(f"\nverdict: {p}", file=out)
|
|
112
|
+
return 1
|
|
113
|
+
if unjudged:
|
|
114
|
+
for u in unjudged:
|
|
115
|
+
print(f"\nverdict: UNJUDGED -- could not judge: {u}", file=out)
|
|
116
|
+
return 2
|
|
117
|
+
if missing_pairs:
|
|
118
|
+
print(f"\nverdict: {SELF} works, but pairs with "
|
|
119
|
+
f"{', '.join(sorted(missing_pairs))} which "
|
|
120
|
+
f"{'is' if len(missing_pairs) == 1 else 'are'} not installed. "
|
|
121
|
+
f"That is a capability you are missing, not an error.", file=out)
|
|
122
|
+
return 0
|
|
123
|
+
print(f"\nverdict: {SELF} and everything it pairs with are present.", file=out)
|
|
124
|
+
return 0
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _local_verdict() -> tuple:
|
|
128
|
+
"""The brick's OWN verdict, folded into the exit code.
|
|
129
|
+
|
|
130
|
+
Without this the local lines are DECORATION. Measured 2026-09-20: awrise
|
|
131
|
+
printed "hostclock NOT INSTALLED -- nothing wakes run-due" and "last tick
|
|
132
|
+
never", then exited 0 -- teaching an operator that a doctor exit code
|
|
133
|
+
carries no information. A brick opts in with `_doctor_local_verdict()`
|
|
134
|
+
returning (problems, unjudged); a brick without one is unaffected.
|
|
135
|
+
"""
|
|
136
|
+
try:
|
|
137
|
+
mod = importlib.import_module(f"{SELF}.doctor_local")
|
|
138
|
+
problems, unjudged = mod._doctor_local_verdict()
|
|
139
|
+
except Exception: # noqa: BLE001
|
|
140
|
+
return [], []
|
|
141
|
+
return list(problems or []), list(unjudged or [])
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _local_checks() -> "list[str]":
|
|
145
|
+
"""Per-brick checks, if this package defines them.
|
|
146
|
+
|
|
147
|
+
Kept as a HOOK rather than generated guesses: the generator knows the family
|
|
148
|
+
from the registry, but it does not know what awdecide needs at runtime, and a
|
|
149
|
+
doctor that invented config requirements would be confidently wrong. A
|
|
150
|
+
package supplies `_doctor_local()` returning display lines; absent, the
|
|
151
|
+
stack picture above still stands on its own.
|
|
152
|
+
"""
|
|
153
|
+
try:
|
|
154
|
+
mod = importlib.import_module(f"{SELF}.doctor_local")
|
|
155
|
+
except Exception:
|
|
156
|
+
return []
|
|
157
|
+
try:
|
|
158
|
+
return list(mod._doctor_local())
|
|
159
|
+
except Exception as exc: # noqa: BLE001
|
|
160
|
+
return [f"local checks raised {type(exc).__name__}: {exc}"]
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def main(argv: "list[str] | None" = None) -> int:
|
|
164
|
+
# --self-test delegates to a SIBLING module when one exists.
|
|
165
|
+
#
|
|
166
|
+
# This file is generated and a fresh run replaces it, so a self-test
|
|
167
|
+
# written HERE is deleted by the next regeneration. awdelphi learned that
|
|
168
|
+
# the expensive way: 125 lines exercising four real failure paths --
|
|
169
|
+
# convergence, roster anonymization, resume, gateway-down -- lived in this
|
|
170
|
+
# file and were destroyed by a routine regeneration, silently, leaving a
|
|
171
|
+
# --self-test flag that reported PASS while asserting nothing.
|
|
172
|
+
#
|
|
173
|
+
# So the seam is a separate module the generator never writes. A package
|
|
174
|
+
# with real machinery to prove puts it in _selftest.py; everything else
|
|
175
|
+
# keeps the honest answer below rather than a self-test that only ever
|
|
176
|
+
# passes.
|
|
177
|
+
argv = list(argv if argv is not None else __import__("sys").argv[1:])
|
|
178
|
+
if "--self-test" in argv:
|
|
179
|
+
try:
|
|
180
|
+
from . import _selftest as _st
|
|
181
|
+
except Exception:
|
|
182
|
+
print("no _selftest module: this doctor reports the stack, and has",
|
|
183
|
+
"no machinery of its own to prove")
|
|
184
|
+
return 0
|
|
185
|
+
return int(_st.run())
|
|
186
|
+
return report()
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
if __name__ == "__main__":
|
|
190
|
+
raise SystemExit(main())
|
awdecide/_selftest.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""`awdecide --self-test` — the contract must be able to FAIL.
|
|
2
|
+
|
|
3
|
+
Arms (each prints one line; the run exits 0 only when all pass):
|
|
4
|
+
1 rules rung answers a choice at p=1.0 and the answer is decided
|
|
5
|
+
2 an empty ladder answers decided=False with a reason -- fail-closed, never a guess
|
|
6
|
+
3 min_confidence turns a weak callable answer into decided=False and keeps the evidence
|
|
7
|
+
4 the logprob rung turns a real OpenAI-wire logprobs payload into a distribution
|
|
8
|
+
over the option labels only (served by an in-process HTTP server -- no network)
|
|
9
|
+
5 the ledger records, resolves, and reports: a calibrated set beats the base rate,
|
|
10
|
+
an overconfident set does not; an undecided answer is never recorded
|
|
11
|
+
6 the CLI grammar parses every primitive and rejects a malformed spec
|
|
12
|
+
7 the world-model door as a rung: choice/score/bool map onto choice/score/yesno,
|
|
13
|
+
the door's CALIBRATED probability survives the ladder unrenormalized,
|
|
14
|
+
source=none ABSTAINS (never the door's placeholder option), resolve() posts the
|
|
15
|
+
outcome back; and the bridge lands a door journal in the ledger idempotently,
|
|
16
|
+
skipping confidence-only legacy rows (fake in-process HTTP door -- no network)
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import random
|
|
22
|
+
import tempfile
|
|
23
|
+
import threading
|
|
24
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import List
|
|
27
|
+
|
|
28
|
+
from .backends import CallableBackend, Ladder, LogprobBackend, RulesBackend, parse_question_spec
|
|
29
|
+
from .contract import Question
|
|
30
|
+
from .ledger import Ledger
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _arm(results: List[str], name: str, ok: bool, detail: str = "") -> None:
|
|
34
|
+
results.append(f" arm {name}: {'OK' if ok else 'FAIL'}{(' -- ' + detail) if detail else ''}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class _FakeLLM(BaseHTTPRequestHandler):
|
|
38
|
+
"""Answers every chat completion with a logprobs payload over 'billing'/'sales'."""
|
|
39
|
+
|
|
40
|
+
def do_POST(self) -> None: # noqa: N802 - http.server API
|
|
41
|
+
n = int(self.headers.get("Content-Length", "0"))
|
|
42
|
+
self.rfile.read(n)
|
|
43
|
+
body = {"choices": [{"message": {"content": "billing"}, "logprobs": {"content": [{
|
|
44
|
+
"token": "billing", "logprob": -0.2231,
|
|
45
|
+
"top_logprobs": [
|
|
46
|
+
{"token": "billing", "logprob": -0.2231}, # 0.80
|
|
47
|
+
{"token": "sales", "logprob": -1.8971}, # 0.15
|
|
48
|
+
{"token": "The", "logprob": -2.9957}, # 0.05 -> not an option, dropped
|
|
49
|
+
]}]}}]}
|
|
50
|
+
data = json.dumps(body).encode("utf-8")
|
|
51
|
+
self.send_response(200)
|
|
52
|
+
self.send_header("Content-Type", "application/json")
|
|
53
|
+
self.send_header("Content-Length", str(len(data)))
|
|
54
|
+
self.end_headers()
|
|
55
|
+
self.wfile.write(data)
|
|
56
|
+
|
|
57
|
+
def log_message(self, *a: object) -> None: # silence
|
|
58
|
+
pass
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def run_self_test() -> int:
|
|
62
|
+
res: List[str] = []
|
|
63
|
+
rnd = random.Random(11)
|
|
64
|
+
|
|
65
|
+
# 1 rules
|
|
66
|
+
q = Question.choice(["billing", "technical", "sales"])
|
|
67
|
+
d = Ladder([RulesBackend([(r"refund|invoice", "billing")])]).decide_one(
|
|
68
|
+
"Customer asks about a refund on invoice 4411", q)
|
|
69
|
+
_arm(res, "1 rules", d.decided and d.value == "billing" and d.probability == 1.0
|
|
70
|
+
and d.backend == "rules", d.to_dict()["value"] or "undecided")
|
|
71
|
+
|
|
72
|
+
# 2 empty ladder -> fail-closed
|
|
73
|
+
d = Ladder([]).decide_one("anything", q)
|
|
74
|
+
_arm(res, "2 fail-closed", (not d.decided) and d.value is None and d.probability == 0.0
|
|
75
|
+
and any("no backends" in r for r in d.reasons), "; ".join(d.reasons))
|
|
76
|
+
|
|
77
|
+
# 3 min_confidence
|
|
78
|
+
weak = CallableBackend(lambda s, qq: {"billing": 0.4, "technical": 0.35, "sales": 0.25}, "weak")
|
|
79
|
+
q7 = Question.choice(["billing", "technical", "sales"], min_confidence=0.7)
|
|
80
|
+
d = Ladder([weak]).decide_one("x", q7)
|
|
81
|
+
_arm(res, "3 min_confidence", (not d.decided) and abs(d.probabilities["billing"] - 0.4) < 1e-9
|
|
82
|
+
and any("< min_confidence" in r for r in d.reasons))
|
|
83
|
+
|
|
84
|
+
# 4 logprob rung over a fake OpenAI-wire server
|
|
85
|
+
srv = HTTPServer(("127.0.0.1", 0), _FakeLLM)
|
|
86
|
+
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
87
|
+
t.start()
|
|
88
|
+
try:
|
|
89
|
+
lb = LogprobBackend(f"http://127.0.0.1:{srv.server_port}", "fake")
|
|
90
|
+
d = Ladder([lb]).decide_one("Customer asks about a refund", q)
|
|
91
|
+
ok = (d.decided and d.value == "billing" and abs(d.probability - 0.8 / 0.95) < 1e-3
|
|
92
|
+
and abs(sum(d.probabilities.values()) - 1.0) < 1e-9
|
|
93
|
+
and d.probabilities["technical"] == 0.0)
|
|
94
|
+
_arm(res, "4 logprob", ok, json.dumps(d.to_dict()["probabilities"]))
|
|
95
|
+
finally:
|
|
96
|
+
srv.shutdown()
|
|
97
|
+
srv.server_close()
|
|
98
|
+
|
|
99
|
+
# 5 ledger
|
|
100
|
+
with tempfile.TemporaryDirectory() as td:
|
|
101
|
+
led = Ledger(Path(td) / "awdecide.db")
|
|
102
|
+
led.record("k", "s", Ladder([]).decide_one("s", q)) # undecided -> not recorded
|
|
103
|
+
skipped_ok = led.pending() == 0
|
|
104
|
+
# calibrated: outcome drawn at the stated probability
|
|
105
|
+
for _ in range(300):
|
|
106
|
+
p = rnd.choice([0.55, 0.7, 0.85, 0.95])
|
|
107
|
+
dd = CallableBackend(lambda s, qq, _p=p: {"billing": _p, "sales": 1 - _p}, "cal")
|
|
108
|
+
ans = Ladder([dd]).decide_one("s", Question.choice(["billing", "sales"]))
|
|
109
|
+
did = led.record("k", "s", ans)
|
|
110
|
+
led.resolve(did, correct=rnd.random() < ans.probability)
|
|
111
|
+
rep = led.reliability()
|
|
112
|
+
cal_ok = rep["beats_base_rate"] and rep["resolved"] == 300
|
|
113
|
+
# overconfident, fresh ledger
|
|
114
|
+
led2 = Ledger(Path(td) / "over.db")
|
|
115
|
+
for _ in range(200):
|
|
116
|
+
dd = CallableBackend(lambda s, qq: {"billing": 0.95, "sales": 0.05}, "over")
|
|
117
|
+
ans = Ladder([dd]).decide_one("s", Question.choice(["billing", "sales"]))
|
|
118
|
+
led2.resolve(led2.record("k", "s", ans), correct=rnd.random() < 0.5)
|
|
119
|
+
rep2 = led2.reliability()
|
|
120
|
+
over_ok = not rep2["beats_base_rate"]
|
|
121
|
+
exported = led.export_platform_jsonl(Path(td) / "predictions.jsonl")
|
|
122
|
+
led.close()
|
|
123
|
+
led2.close()
|
|
124
|
+
_arm(res, "5 ledger", skipped_ok and cal_ok and over_ok and exported == 300,
|
|
125
|
+
f"cal brier={rep.get('brier')} clim={rep.get('climatology')} "
|
|
126
|
+
f"over brier={rep2.get('brier')} clim={rep2.get('climatology')}")
|
|
127
|
+
|
|
128
|
+
# 6 CLI grammar
|
|
129
|
+
try:
|
|
130
|
+
k1, q1 = parse_question_spec("category:choice=billing,technical@0.6")
|
|
131
|
+
k2, q2 = parse_question_spec("urgency:score=low,mid,high")
|
|
132
|
+
k3, q3 = parse_question_spec("urgent:bool")
|
|
133
|
+
try:
|
|
134
|
+
parse_question_spec("nope:maybe=1")
|
|
135
|
+
bad_rejected = False
|
|
136
|
+
except ValueError:
|
|
137
|
+
bad_rejected = True
|
|
138
|
+
_arm(res, "6 grammar", q1.kind == "choice" and q1.min_confidence == 0.6
|
|
139
|
+
and q2.kind == "score" and q2.options == ("low", "mid", "high")
|
|
140
|
+
and q3.kind == "bool" and q3.options == ("yes", "no") and bad_rejected)
|
|
141
|
+
except Exception as e: # pragma: no cover
|
|
142
|
+
_arm(res, "6 grammar", False, repr(e))
|
|
143
|
+
|
|
144
|
+
# 7 door rung + bridge, against a fake door
|
|
145
|
+
try:
|
|
146
|
+
_arm(res, "7 door", *_door_arm())
|
|
147
|
+
except Exception as e: # pragma: no cover
|
|
148
|
+
_arm(res, "7 door", False, repr(e))
|
|
149
|
+
|
|
150
|
+
print("awdecide --self-test")
|
|
151
|
+
print("\n".join(res))
|
|
152
|
+
ok = all(": OK" in r for r in res)
|
|
153
|
+
print("RESULT:", "OK" if ok else "FAIL")
|
|
154
|
+
return 0 if ok else 2
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class _FakeDoor(BaseHTTPRequestHandler):
|
|
158
|
+
"""A door that answers from a table keyed on the request kind, and records
|
|
159
|
+
outcomes. Shapes copied from decide.py Decider.decide / outcome."""
|
|
160
|
+
|
|
161
|
+
seen: List[dict] = []
|
|
162
|
+
outcomes: List[dict] = []
|
|
163
|
+
|
|
164
|
+
def do_POST(self) -> None: # noqa: N802 - http.server API
|
|
165
|
+
n = int(self.headers.get("Content-Length", "0"))
|
|
166
|
+
body = json.loads(self.rfile.read(n) or b"{}")
|
|
167
|
+
if self.path.endswith("/decide/outcome"):
|
|
168
|
+
_FakeDoor.outcomes.append({**body, "auth": self.headers.get("X-WM-Token")})
|
|
169
|
+
reply = {"ok": True, "reward": body.get("reward")}
|
|
170
|
+
else:
|
|
171
|
+
_FakeDoor.seen.append(body)
|
|
172
|
+
kind = body.get("kind")
|
|
173
|
+
if kind == "yesno":
|
|
174
|
+
reply = {"decision_id": "d-yes", "answer": "yes", "confidence": 0.6,
|
|
175
|
+
"source": "engine", "probabilities": {"yes": 0.7},
|
|
176
|
+
"probability": 0.7, "probability_source": "outcomes", "p_yes": 0.7,
|
|
177
|
+
"learned_from": 3}
|
|
178
|
+
elif kind == "score":
|
|
179
|
+
reply = {"decision_id": "d-score", "answer": "high", "confidence": 0.5,
|
|
180
|
+
"source": "neighbor", "probabilities": {"low": 0.2, "high": 0.65},
|
|
181
|
+
"probability": 0.65, "probability_source": "outcomes"}
|
|
182
|
+
elif body.get("state") == "cold":
|
|
183
|
+
reply = {"decision_id": "d-none", "answer": body["options"][0],
|
|
184
|
+
"confidence": 0.0, "source": "none", "probability": None,
|
|
185
|
+
"probability_source": "none"}
|
|
186
|
+
else:
|
|
187
|
+
# per-option calibrated P(right): does NOT sum to 1 on purpose
|
|
188
|
+
reply = {"decision_id": "d-choice", "answer": "orchestrator", "confidence": 0.8,
|
|
189
|
+
"source": "engine",
|
|
190
|
+
"probabilities": {"orchestrator": 0.9, "fast-local": 0.3},
|
|
191
|
+
"probability": 0.9, "probability_source": "outcomes",
|
|
192
|
+
"learned_from": 8}
|
|
193
|
+
data = json.dumps(reply).encode("utf-8")
|
|
194
|
+
self.send_response(200)
|
|
195
|
+
self.send_header("Content-Type", "application/json")
|
|
196
|
+
self.send_header("Content-Length", str(len(data)))
|
|
197
|
+
self.end_headers()
|
|
198
|
+
self.wfile.write(data)
|
|
199
|
+
|
|
200
|
+
def log_message(self, *a: object) -> None:
|
|
201
|
+
pass
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _door_arm() -> "tuple[bool, str]":
|
|
205
|
+
from . import bridge
|
|
206
|
+
from .door import DoorBackend
|
|
207
|
+
|
|
208
|
+
srv = HTTPServer(("127.0.0.1", 0), _FakeDoor)
|
|
209
|
+
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
210
|
+
t.start()
|
|
211
|
+
notes: List[str] = []
|
|
212
|
+
try:
|
|
213
|
+
door = DoorBackend(f"http://127.0.0.1:{srv.server_port}", token="t0k", fork="router")
|
|
214
|
+
lad = Ladder([door])
|
|
215
|
+
c = lad.decide_one("kind:code", Question.choice(["fast-local", "orchestrator"]))
|
|
216
|
+
s = lad.decide_one("x", Question.score(["low", "mid", "high"]))
|
|
217
|
+
b = lad.decide_one("x", Question.bool())
|
|
218
|
+
none = lad.decide_one("cold", Question.choice(["a", "b"]))
|
|
219
|
+
strict = lad.decide_one("kind:code", Question.choice(["fast-local", "orchestrator"],
|
|
220
|
+
min_confidence=0.95))
|
|
221
|
+
kinds = [r.get("kind") for r in _FakeDoor.seen[:3]]
|
|
222
|
+
ok_map = kinds == ["choice", "score", "yesno"] and \
|
|
223
|
+
_FakeDoor.seen[0]["domain"] == "decide.router"
|
|
224
|
+
ok_choice = c.decided and c.value == "orchestrator" and c.probability == 0.9 \
|
|
225
|
+
and c.backend == "door:engine" and c.id == "d-choice" \
|
|
226
|
+
and c.probabilities["fast-local"] == 0.3 # unrenormalized
|
|
227
|
+
ok_score = s.decided and s.value == "high" and s.probability == 0.65 \
|
|
228
|
+
and s.backend == "door:neighbor"
|
|
229
|
+
ok_bool = b.decided and b.value == "yes" and b.probability == 0.7 \
|
|
230
|
+
and abs(b.probabilities["no"] - 0.3) < 1e-9
|
|
231
|
+
ok_none = (not none.decided) and none.value is None and none.backend == "none" \
|
|
232
|
+
and any("door:none" in r for r in none.reasons)
|
|
233
|
+
ok_strict = (not strict.decided) and strict.probabilities["orchestrator"] == 0.9
|
|
234
|
+
notes.append(f"map={ok_map} choice={ok_choice} score={ok_score} bool={ok_bool} "
|
|
235
|
+
f"none={ok_none} strict={ok_strict}")
|
|
236
|
+
with tempfile.TemporaryDirectory() as td:
|
|
237
|
+
led = Ledger(Path(td) / "door.db")
|
|
238
|
+
did = led.record("route", "kind:code", c)
|
|
239
|
+
r = door.resolve(did, correct=True, ledger=led)
|
|
240
|
+
ok_resolve = did == "d-choice" and r["door"] == {"ok": True, "reward": 1.0} \
|
|
241
|
+
and abs(r["brier"] - 0.01) < 1e-9 and _FakeDoor.outcomes[-1]["auth"] == "t0k"
|
|
242
|
+
# bridge: a door journal with 2 proper pairs, 1 legacy, 1 none, 1 unresolved
|
|
243
|
+
ck = Path(td) / "ckpt"
|
|
244
|
+
ck.mkdir()
|
|
245
|
+
rows = [
|
|
246
|
+
{"kind": "decision", "decision_id": "j1", "domain": "decide.live.r", "state": "s",
|
|
247
|
+
"answer": "a", "confidence": 0.6, "probability": 0.8,
|
|
248
|
+
"probability_source": "outcomes", "question_kind": "choice",
|
|
249
|
+
"source": "engine", "ts": 1.0},
|
|
250
|
+
{"kind": "outcome", "decision_id": "j1", "reward": 1.0, "ts": 2.0},
|
|
251
|
+
{"kind": "decision", "decision_id": "j2", "domain": "decide.live.r", "state": "s",
|
|
252
|
+
"answer": "yes", "probability": 0.4, "question_kind": "yesno",
|
|
253
|
+
"source": "prior", "ts": 3.0},
|
|
254
|
+
{"kind": "outcome", "decision_id": "j2", "reward": -1.0, "ts": 4.0},
|
|
255
|
+
{"kind": "outcome", "decision_id": "j2", "reward": 1.0, "ts": 5.0}, # 2nd ignored
|
|
256
|
+
{"kind": "decision", "decision_id": "j3", "domain": "decide.live.r", "state": "s",
|
|
257
|
+
"answer": "a", "confidence": 0.9, "source": "engine", "ts": 6.0}, # legacy
|
|
258
|
+
{"kind": "outcome", "decision_id": "j3", "reward": 1.0, "ts": 7.0},
|
|
259
|
+
{"kind": "decision", "decision_id": "j4", "domain": "decide.live.r", "state": "s",
|
|
260
|
+
"answer": "a", "probability": None, "source": "none", "ts": 8.0},
|
|
261
|
+
{"kind": "outcome", "decision_id": "j4", "reward": 1.0, "ts": 9.0},
|
|
262
|
+
{"kind": "decision", "decision_id": "j5", "domain": "decide.live.r", "state": "s",
|
|
263
|
+
"answer": "a", "probability": 0.5, "source": "llm", "ts": 10.0}, # unresolved
|
|
264
|
+
{"kind": "outcome", "decision_id": None, "reward": 1.0, "ts": 11.0}, # teach row
|
|
265
|
+
]
|
|
266
|
+
with (ck / "decisions.jsonl").open("w", encoding="utf-8") as f:
|
|
267
|
+
for row in rows:
|
|
268
|
+
f.write(json.dumps(row) + "\n")
|
|
269
|
+
f.write("{torn\n")
|
|
270
|
+
r1 = bridge.ingest_journal(ck, led, label="t")
|
|
271
|
+
r2 = bridge.ingest_journal(ck, led, label="t")
|
|
272
|
+
rep = led.reliability()
|
|
273
|
+
ok_bridge = (r1["ingested"], r1["legacy_confidence_only"], r1["source_none"],
|
|
274
|
+
r1["matched"]) == (2, 1, 1, 4) \
|
|
275
|
+
and r2["ingested"] == 0 and r2["already_present"] == 2 \
|
|
276
|
+
and rep["resolved"] == 3 and rep["by_backend"]["door:prior"]["n"] == 1 \
|
|
277
|
+
and abs(rep["by_backend"]["door:prior"]["brier"] - 0.16) < 1e-9
|
|
278
|
+
led.close()
|
|
279
|
+
notes.append(f"resolve={ok_resolve} bridge={ok_bridge}")
|
|
280
|
+
return (ok_map and ok_choice and ok_score and ok_bool and ok_none and ok_strict
|
|
281
|
+
and ok_resolve and ok_bridge), " ".join(notes)
|
|
282
|
+
finally:
|
|
283
|
+
srv.shutdown()
|
|
284
|
+
srv.server_close()
|