awdecide 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- awdecide-0.3.0/LICENSE +15 -0
- awdecide-0.3.0/PKG-INFO +214 -0
- awdecide-0.3.0/README.md +198 -0
- awdecide-0.3.0/awdecide/__init__.py +38 -0
- awdecide-0.3.0/awdecide/__main__.py +7 -0
- awdecide-0.3.0/awdecide/_doctor.py +190 -0
- awdecide-0.3.0/awdecide/_selftest.py +284 -0
- awdecide-0.3.0/awdecide/backends.py +238 -0
- awdecide-0.3.0/awdecide/bridge.py +263 -0
- awdecide-0.3.0/awdecide/cli.py +256 -0
- awdecide-0.3.0/awdecide/contract.py +132 -0
- awdecide-0.3.0/awdecide/door.py +357 -0
- awdecide-0.3.0/awdecide/ledger.py +179 -0
- awdecide-0.3.0/awdecide/loop.py +249 -0
- awdecide-0.3.0/awdecide/mcp.py +204 -0
- awdecide-0.3.0/awdecide/sources.py +315 -0
- awdecide-0.3.0/awdecide.egg-info/PKG-INFO +214 -0
- awdecide-0.3.0/awdecide.egg-info/SOURCES.txt +27 -0
- awdecide-0.3.0/awdecide.egg-info/dependency_links.txt +1 -0
- awdecide-0.3.0/awdecide.egg-info/entry_points.txt +2 -0
- awdecide-0.3.0/awdecide.egg-info/requires.txt +3 -0
- awdecide-0.3.0/awdecide.egg-info/top_level.txt +1 -0
- awdecide-0.3.0/pyproject.toml +39 -0
- awdecide-0.3.0/setup.cfg +4 -0
- awdecide-0.3.0/tests/test_awdecide.py +64 -0
- awdecide-0.3.0/tests/test_bridge.py +138 -0
- awdecide-0.3.0/tests/test_door_backend.py +161 -0
- awdecide-0.3.0/tests/test_loop.py +220 -0
- awdecide-0.3.0/tests/test_sources.py +177 -0
awdecide-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Apache License 2.0
|
|
2
|
+
|
|
3
|
+
Copyright 2026 Aitherium
|
|
4
|
+
|
|
5
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
you may not use this file except in compliance with the License.
|
|
7
|
+
You may obtain a copy of the License at
|
|
8
|
+
|
|
9
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
|
|
11
|
+
Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
See the License for the specific language governing permissions and
|
|
15
|
+
limitations under the License.
|
awdecide-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: awdecide
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: One typed-decision contract -- choice / score / bool with a probability -- fail-closed, with a Brier ledger. Aither World Decide.
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://github.com/Aitherium/awdecide
|
|
7
|
+
Project-URL: Documentation, https://github.com/Aitherium/awdecide#readme
|
|
8
|
+
Project-URL: Repository, https://github.com/Aitherium/awdecide.git
|
|
9
|
+
Project-URL: Issues, https://github.com/Aitherium/awdecide/issues
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# awdecide — Aither World Decide
|
|
18
|
+
|
|
19
|
+
**One typed-decision contract — choice / score / bool with a probability — over the
|
|
20
|
+
backends you already run, fail-closed, with a Brier ledger that resolves every
|
|
21
|
+
decision against its outcome.**
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install -e AitherOS/packages/awdecide # monorepo; no public mirror yet
|
|
25
|
+
awdecide --self-test
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## The problem it exists for
|
|
29
|
+
|
|
30
|
+
Software makes the same bounded decision millions of times a day and asks a text
|
|
31
|
+
model each time, then parses prose. Hosted "decision models" fix the shape — a typed
|
|
32
|
+
question in, a typed answer with a probability out — but they send your program state
|
|
33
|
+
off-box on every call, emit a probability that is never resolved against what
|
|
34
|
+
happened, and cannot say what happens next if you act on it.
|
|
35
|
+
|
|
36
|
+
A decision is a function. The loop it sits in is the product. `awdecide` is the
|
|
37
|
+
function, written so the loop can be closed on your own machine:
|
|
38
|
+
|
|
39
|
+
- **the contract** — `choice` (one of N names), `score` (one of N *ordered* levels),
|
|
40
|
+
`bool`; every answer carries `probability`, `confidence`, `backend`, `decided`.
|
|
41
|
+
- **the ladder** — rungs asked in order, first answer above `min_confidence` wins:
|
|
42
|
+
`RulesBackend` (a matched rule answers at 1.0; no match abstains, never guesses),
|
|
43
|
+
`CallableBackend` (any `fn(state, question) -> {option: weight}` — a nanoGPT, an
|
|
44
|
+
sklearn model, a classify surface), `LogprobBackend` (an OpenAI-wire server with
|
|
45
|
+
`logprobs`: the mass over the option labels *is* the distribution; no prose parsed).
|
|
46
|
+
- **fail-closed** — `decided=False` is a first-class answer. `value` is `None`, the
|
|
47
|
+
reasons say which rung abstained and why, and the strongest sub-threshold evidence is
|
|
48
|
+
kept in `probabilities` so a reviewer can see what was *not* acted on.
|
|
49
|
+
- **the ledger** — SQLite. `record` a decided answer, `resolve` it against the outcome,
|
|
50
|
+
`reliability()` reports overall Brier, the climatology (base-rate) Brier it must beat,
|
|
51
|
+
and the per-bucket table (mean confidence vs observed frequency). Gate **1zdca**
|
|
52
|
+
(`check_decision_calibration.py --awdecide-db`) reads this file and goes red when the
|
|
53
|
+
probabilities stop carrying information.
|
|
54
|
+
|
|
55
|
+
## Use
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from awdecide import Question, Ladder, RulesBackend, LogprobBackend, Ledger
|
|
59
|
+
|
|
60
|
+
questions = {
|
|
61
|
+
"category": Question.choice(["billing", "technical", "sales"]),
|
|
62
|
+
"urgency": Question.score(["low", "medium", "high"]),
|
|
63
|
+
"escalate": Question.bool(min_confidence=0.7),
|
|
64
|
+
}
|
|
65
|
+
ladder = Ladder([
|
|
66
|
+
RulesBackend([(r"refund|invoice", "billing")]),
|
|
67
|
+
LogprobBackend("http://127.0.0.1:8150", "orchestrator"), # your own MicroScheduler
|
|
68
|
+
])
|
|
69
|
+
answers = ladder.decide(state_text, questions)
|
|
70
|
+
answers["category"].value # "billing"
|
|
71
|
+
answers["escalate"].decided # False if no rung cleared 0.7 -- act on that in code
|
|
72
|
+
|
|
73
|
+
led = Ledger() # ~/.aither/awdecide.db or $AWDECIDE_DB
|
|
74
|
+
did = led.record("escalate", state_text, answers["escalate"])
|
|
75
|
+
...
|
|
76
|
+
led.resolve(did, correct=True) # later, when you know
|
|
77
|
+
led.reliability() # brier, climatology, beats_base_rate, buckets
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
awdecide ask --state "Customer emailed twice about a failed refund" \
|
|
82
|
+
category:choice=billing,technical,sales urgent:bool@0.7 \
|
|
83
|
+
--rule 'refund|invoice=billing' --record
|
|
84
|
+
# exit 0 = every question decided · 3 = at least one decided=False (the JSON says why)
|
|
85
|
+
awdecide resolve <id> --correct
|
|
86
|
+
awdecide reliability
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## The loop: a decision you resolved is never paid for twice
|
|
90
|
+
|
|
91
|
+
`Ladder` answers a question. `Loop` remembers how the answer turned out. Ask, act,
|
|
92
|
+
resolve -- the next identical decision is answered from that evidence with no model
|
|
93
|
+
call, and an answer that was resolved wrong is never given again.
|
|
94
|
+
|
|
95
|
+
Add it to the agent harness you already use:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
claude mcp add awdecide -- uvx awdecide mcp # Claude Code
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
```toml
|
|
102
|
+
# Codex: ~/.codex/config.toml
|
|
103
|
+
[mcp_servers.awdecide]
|
|
104
|
+
command = "uvx"
|
|
105
|
+
args = ["awdecide", "mcp"]
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Then one paragraph in your `CLAUDE.md` / `AGENTS.md`:
|
|
109
|
+
|
|
110
|
+
> Before a bounded decision you make repeatedly here (which command, which branch, retry
|
|
111
|
+
> or stop), call `decide` with a STABLE `state` string. Act on the answer, then ALWAYS call
|
|
112
|
+
> `decide_outcome`. If it returns `decided=false`, decide yourself and `decide_teach` it.
|
|
113
|
+
|
|
114
|
+
The harness's own model is the brain on the first sighting; the ledger is the memory on
|
|
115
|
+
every one after. To give the loop its own brain, point it at any OpenAI-wire endpoint:
|
|
116
|
+
`AWDECIDE_LLM_URL`, `AWDECIDE_LLM_MODEL`, optional `AWDECIDE_LLM_KEY`.
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from awdecide import Loop, Ladder, Ledger, Question, ChatBackend
|
|
120
|
+
|
|
121
|
+
loop = Loop(Ladder([ChatBackend("http://127.0.0.1:11434", "qwen3:8b")]), Ledger())
|
|
122
|
+
d = loop.decide("test-runner", "lang:py,changed:tests",
|
|
123
|
+
Question.choice(["pytest -x", "pytest -n8", "tox"]))
|
|
124
|
+
loop.resolve(d.id, correct=run(d.value)) # d.backend is "evidence" next time
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Measure it: `awdecide bench` runs one deliberately imperfect brain (right 70% of the
|
|
128
|
+
time) over 40 situations seen 10 times each, two ways.
|
|
129
|
+
|
|
130
|
+
| | accuracy | model calls |
|
|
131
|
+
|---|---|---|
|
|
132
|
+
| call the brain every time | 70.8% | 400 |
|
|
133
|
+
| **behind the loop** | **97.0%** | **52** |
|
|
134
|
+
|
|
135
|
+
Same brain. More accurate because a wrong answer is resolved and retired; cheaper
|
|
136
|
+
because a right one is never asked for again. The run also reports Brier against the
|
|
137
|
+
base rate, so the probabilities are graded too. Change `--seed` and rerun it.
|
|
138
|
+
|
|
139
|
+
## The learning backend: the decision door
|
|
140
|
+
|
|
141
|
+
A rung may be a door that LEARNS. `DoorBackend` asks a world-model decision door
|
|
142
|
+
(engine -> neighbor -> neural -> llm -> prior), which answers with the probability its
|
|
143
|
+
own posted outcomes earned; `resolve()` sends the outcome back, so the next time that
|
|
144
|
+
state -- or one like it -- arrives, the answer comes from evidence instead of a model.
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from awdecide import DoorBackend, Ladder, Ledger, Question
|
|
148
|
+
|
|
149
|
+
state = "kind:code,len:short"
|
|
150
|
+
door = DoorBackend(url="http://127.0.0.1:8299/v1", token=TOKEN, fork="router")
|
|
151
|
+
d = Ladder([door]).decide_one(state, Question.choice(["fast-local", "reasoner"]))
|
|
152
|
+
d.backend # 'door:engine' -- which rung of the door answered
|
|
153
|
+
led = Ledger()
|
|
154
|
+
door.resolve(led.record("route", state, d), correct=True, ledger=led) # teaches both
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The door's own journals become ledger rows, idempotently by decision id:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
awdecide ingest-door /path/to/ckpt-dir # journaled pairs + prequential replay
|
|
161
|
+
awdecide ingest-door --generate # bench-generated pairs, labelled bench/
|
|
162
|
+
awdecide reliability # brier vs base rate, broken out by backend
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
A decision the door made with no probability, or with `source=none`, is not a claim and
|
|
166
|
+
never becomes a row.
|
|
167
|
+
|
|
168
|
+
## What it is not
|
|
169
|
+
|
|
170
|
+
It is not a model. It ships no weights and calls nothing you did not name; a
|
|
171
|
+
`LogprobBackend` with no URL is a refusal, not a default. It does not explain — a
|
|
172
|
+
probability is what you get, and the ledger is where you find out whether it meant
|
|
173
|
+
anything. Predicting what happens *after* the decision is `awpredict`'s contract;
|
|
174
|
+
classifying documents at the door is `awclassify`'s; proving a page did the right thing
|
|
175
|
+
is `awprove`'s.
|
|
176
|
+
|
|
177
|
+
## Self-test
|
|
178
|
+
|
|
179
|
+
`awdecide --self-test` runs seven arms and exits 0 only when all pass: rules answer,
|
|
180
|
+
an empty ladder fails closed, `min_confidence` demotes a weak answer and keeps the
|
|
181
|
+
evidence, the logprob rung turns a real logprobs payload (served in-process) into a
|
|
182
|
+
distribution over the option labels only, the ledger's calibrated set beats the base
|
|
183
|
+
rate and its overconfident set does not, the CLI grammar rejects a malformed spec, and the
|
|
184
|
+
door rung maps every primitive, keeps the door's calibrated probability unrenormalized,
|
|
185
|
+
abstains when the door has nothing, and ingests a door journal exactly once.
|
|
186
|
+
|
|
187
|
+
Apache-2.0. Python 3.10+. Standard library only.
|
|
188
|
+
## Sources and prediction
|
|
189
|
+
|
|
190
|
+
`awdecide.sources` adds where a decision's state comes from and a rung that tries to
|
|
191
|
+
predict the answer. `ReplState(session)` turns a live `awrepl` session into a STABLE
|
|
192
|
+
state descriptor -- variable names, types and bucketed sizes (`rows:list:1k-9k`), never
|
|
193
|
+
a repr and never a value, so the descriptor is safe to hash, log and resolve in the
|
|
194
|
+
ledger. `PredictBackend(env)` is a ladder rung that asks a value oracle what each option
|
|
195
|
+
is worth and ABSTAINS unless the top two are further apart than `margin`; the oracle is
|
|
196
|
+
anything exposing `value(state, option)`, including `AwpredictValueEnv`, which wraps an
|
|
197
|
+
`awpredict` engine's reward or value head. **The shipped default
|
|
198
|
+
(`default_predict_backend()`) is the self-updating last-outcome lookup, not the learned
|
|
199
|
+
model, and that is a measurement rather than a preference.**
|
|
200
|
+
A bench (`tool_outcome_predict_bench`) asked whether anything can predict
|
|
201
|
+
that the next run of a command shape will pass, well enough to skip the call, over
|
|
202
|
+
323,644 real tool outcomes (95.0% pass) with a temporal 80/20 split, scored on the
|
|
203
|
+
UNSEEN bucket only -- novel command shapes -- because a self-updating dictionary already
|
|
204
|
+
owns the seen rows and an aggregate is therefore structurally unable to move. On 37,668
|
|
205
|
+
UNSEEN rows the lookup, the online majority and "always run it" all score 0.9571, and
|
|
206
|
+
coarsening the key to the command family makes it worse (0.9291); on a 12,000-row window
|
|
207
|
+
with the learned arms in (1,785 UNSEEN) `awpredict`'s token-hash arm ties the base rate
|
|
208
|
+
exactly at 0.9434 and its whole-string arm loses to it at 0.9412 -- so the bench exits 1,
|
|
209
|
+
and exit 1 IS the finding: on a novel shape there is nothing in the history to learn
|
|
210
|
+
from, and every arm collapses onto the base rate. The number that decides the design is
|
|
211
|
+
not accuracy but skip-precision: 0.9434 on UNSEEN means roughly one in eighteen skipped
|
|
212
|
+
calls would really have failed, so the rung is wired to abstain by default and to answer
|
|
213
|
+
only where it has seen the option before. Re-run it when an engine with a real value head
|
|
214
|
+
is a candidate; the arm ships the day it beats the dictionary, not before.
|
awdecide-0.3.0/README.md
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# awdecide — Aither World Decide
|
|
2
|
+
|
|
3
|
+
**One typed-decision contract — choice / score / bool with a probability — over the
|
|
4
|
+
backends you already run, fail-closed, with a Brier ledger that resolves every
|
|
5
|
+
decision against its outcome.**
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install -e AitherOS/packages/awdecide # monorepo; no public mirror yet
|
|
9
|
+
awdecide --self-test
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## The problem it exists for
|
|
13
|
+
|
|
14
|
+
Software makes the same bounded decision millions of times a day and asks a text
|
|
15
|
+
model each time, then parses prose. Hosted "decision models" fix the shape — a typed
|
|
16
|
+
question in, a typed answer with a probability out — but they send your program state
|
|
17
|
+
off-box on every call, emit a probability that is never resolved against what
|
|
18
|
+
happened, and cannot say what happens next if you act on it.
|
|
19
|
+
|
|
20
|
+
A decision is a function. The loop it sits in is the product. `awdecide` is the
|
|
21
|
+
function, written so the loop can be closed on your own machine:
|
|
22
|
+
|
|
23
|
+
- **the contract** — `choice` (one of N names), `score` (one of N *ordered* levels),
|
|
24
|
+
`bool`; every answer carries `probability`, `confidence`, `backend`, `decided`.
|
|
25
|
+
- **the ladder** — rungs asked in order, first answer above `min_confidence` wins:
|
|
26
|
+
`RulesBackend` (a matched rule answers at 1.0; no match abstains, never guesses),
|
|
27
|
+
`CallableBackend` (any `fn(state, question) -> {option: weight}` — a nanoGPT, an
|
|
28
|
+
sklearn model, a classify surface), `LogprobBackend` (an OpenAI-wire server with
|
|
29
|
+
`logprobs`: the mass over the option labels *is* the distribution; no prose parsed).
|
|
30
|
+
- **fail-closed** — `decided=False` is a first-class answer. `value` is `None`, the
|
|
31
|
+
reasons say which rung abstained and why, and the strongest sub-threshold evidence is
|
|
32
|
+
kept in `probabilities` so a reviewer can see what was *not* acted on.
|
|
33
|
+
- **the ledger** — SQLite. `record` a decided answer, `resolve` it against the outcome,
|
|
34
|
+
`reliability()` reports overall Brier, the climatology (base-rate) Brier it must beat,
|
|
35
|
+
and the per-bucket table (mean confidence vs observed frequency). Gate **1zdca**
|
|
36
|
+
(`check_decision_calibration.py --awdecide-db`) reads this file and goes red when the
|
|
37
|
+
probabilities stop carrying information.
|
|
38
|
+
|
|
39
|
+
## Use
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from awdecide import Question, Ladder, RulesBackend, LogprobBackend, Ledger
|
|
43
|
+
|
|
44
|
+
questions = {
|
|
45
|
+
"category": Question.choice(["billing", "technical", "sales"]),
|
|
46
|
+
"urgency": Question.score(["low", "medium", "high"]),
|
|
47
|
+
"escalate": Question.bool(min_confidence=0.7),
|
|
48
|
+
}
|
|
49
|
+
ladder = Ladder([
|
|
50
|
+
RulesBackend([(r"refund|invoice", "billing")]),
|
|
51
|
+
LogprobBackend("http://127.0.0.1:8150", "orchestrator"), # your own MicroScheduler
|
|
52
|
+
])
|
|
53
|
+
answers = ladder.decide(state_text, questions)
|
|
54
|
+
answers["category"].value # "billing"
|
|
55
|
+
answers["escalate"].decided # False if no rung cleared 0.7 -- act on that in code
|
|
56
|
+
|
|
57
|
+
led = Ledger() # ~/.aither/awdecide.db or $AWDECIDE_DB
|
|
58
|
+
did = led.record("escalate", state_text, answers["escalate"])
|
|
59
|
+
...
|
|
60
|
+
led.resolve(did, correct=True) # later, when you know
|
|
61
|
+
led.reliability() # brier, climatology, beats_base_rate, buckets
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
awdecide ask --state "Customer emailed twice about a failed refund" \
|
|
66
|
+
category:choice=billing,technical,sales urgent:bool@0.7 \
|
|
67
|
+
--rule 'refund|invoice=billing' --record
|
|
68
|
+
# exit 0 = every question decided · 3 = at least one decided=False (the JSON says why)
|
|
69
|
+
awdecide resolve <id> --correct
|
|
70
|
+
awdecide reliability
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## The loop: a decision you resolved is never paid for twice
|
|
74
|
+
|
|
75
|
+
`Ladder` answers a question. `Loop` remembers how the answer turned out. Ask, act,
|
|
76
|
+
resolve -- the next identical decision is answered from that evidence with no model
|
|
77
|
+
call, and an answer that was resolved wrong is never given again.
|
|
78
|
+
|
|
79
|
+
Add it to the agent harness you already use:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
claude mcp add awdecide -- uvx awdecide mcp # Claude Code
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
```toml
|
|
86
|
+
# Codex: ~/.codex/config.toml
|
|
87
|
+
[mcp_servers.awdecide]
|
|
88
|
+
command = "uvx"
|
|
89
|
+
args = ["awdecide", "mcp"]
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Then one paragraph in your `CLAUDE.md` / `AGENTS.md`:
|
|
93
|
+
|
|
94
|
+
> Before a bounded decision you make repeatedly here (which command, which branch, retry
|
|
95
|
+
> or stop), call `decide` with a STABLE `state` string. Act on the answer, then ALWAYS call
|
|
96
|
+
> `decide_outcome`. If it returns `decided=false`, decide yourself and `decide_teach` it.
|
|
97
|
+
|
|
98
|
+
The harness's own model is the brain on the first sighting; the ledger is the memory on
|
|
99
|
+
every one after. To give the loop its own brain, point it at any OpenAI-wire endpoint:
|
|
100
|
+
`AWDECIDE_LLM_URL`, `AWDECIDE_LLM_MODEL`, optional `AWDECIDE_LLM_KEY`.
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from awdecide import Loop, Ladder, Ledger, Question, ChatBackend
|
|
104
|
+
|
|
105
|
+
loop = Loop(Ladder([ChatBackend("http://127.0.0.1:11434", "qwen3:8b")]), Ledger())
|
|
106
|
+
d = loop.decide("test-runner", "lang:py,changed:tests",
|
|
107
|
+
Question.choice(["pytest -x", "pytest -n8", "tox"]))
|
|
108
|
+
loop.resolve(d.id, correct=run(d.value)) # d.backend is "evidence" next time
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Measure it: `awdecide bench` runs one deliberately imperfect brain (right 70% of the
|
|
112
|
+
time) over 40 situations seen 10 times each, two ways.
|
|
113
|
+
|
|
114
|
+
| | accuracy | model calls |
|
|
115
|
+
|---|---|---|
|
|
116
|
+
| call the brain every time | 70.8% | 400 |
|
|
117
|
+
| **behind the loop** | **97.0%** | **52** |
|
|
118
|
+
|
|
119
|
+
Same brain. More accurate because a wrong answer is resolved and retired; cheaper
|
|
120
|
+
because a right one is never asked for again. The run also reports Brier against the
|
|
121
|
+
base rate, so the probabilities are graded too. Change `--seed` and rerun it.
|
|
122
|
+
|
|
123
|
+
## The learning backend: the decision door
|
|
124
|
+
|
|
125
|
+
A rung may be a door that LEARNS. `DoorBackend` asks a world-model decision door
|
|
126
|
+
(engine -> neighbor -> neural -> llm -> prior), which answers with the probability its
|
|
127
|
+
own posted outcomes earned; `resolve()` sends the outcome back, so the next time that
|
|
128
|
+
state -- or one like it -- arrives, the answer comes from evidence instead of a model.
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from awdecide import DoorBackend, Ladder, Ledger, Question
|
|
132
|
+
|
|
133
|
+
state = "kind:code,len:short"
|
|
134
|
+
door = DoorBackend(url="http://127.0.0.1:8299/v1", token=TOKEN, fork="router")
|
|
135
|
+
d = Ladder([door]).decide_one(state, Question.choice(["fast-local", "reasoner"]))
|
|
136
|
+
d.backend # 'door:engine' -- which rung of the door answered
|
|
137
|
+
led = Ledger()
|
|
138
|
+
door.resolve(led.record("route", state, d), correct=True, ledger=led) # teaches both
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
The door's own journals become ledger rows, idempotently by decision id:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
awdecide ingest-door /path/to/ckpt-dir # journaled pairs + prequential replay
|
|
145
|
+
awdecide ingest-door --generate # bench-generated pairs, labelled bench/
|
|
146
|
+
awdecide reliability # brier vs base rate, broken out by backend
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
A decision the door made with no probability, or with `source=none`, is not a claim and
|
|
150
|
+
never becomes a row.
|
|
151
|
+
|
|
152
|
+
## What it is not
|
|
153
|
+
|
|
154
|
+
It is not a model. It ships no weights and calls nothing you did not name; a
|
|
155
|
+
`LogprobBackend` with no URL is a refusal, not a default. It does not explain — a
|
|
156
|
+
probability is what you get, and the ledger is where you find out whether it meant
|
|
157
|
+
anything. Predicting what happens *after* the decision is `awpredict`'s contract;
|
|
158
|
+
classifying documents at the door is `awclassify`'s; proving a page did the right thing
|
|
159
|
+
is `awprove`'s.
|
|
160
|
+
|
|
161
|
+
## Self-test
|
|
162
|
+
|
|
163
|
+
`awdecide --self-test` runs seven arms and exits 0 only when all pass: rules answer,
|
|
164
|
+
an empty ladder fails closed, `min_confidence` demotes a weak answer and keeps the
|
|
165
|
+
evidence, the logprob rung turns a real logprobs payload (served in-process) into a
|
|
166
|
+
distribution over the option labels only, the ledger's calibrated set beats the base
|
|
167
|
+
rate and its overconfident set does not, the CLI grammar rejects a malformed spec, and the
|
|
168
|
+
door rung maps every primitive, keeps the door's calibrated probability unrenormalized,
|
|
169
|
+
abstains when the door has nothing, and ingests a door journal exactly once.
|
|
170
|
+
|
|
171
|
+
Apache-2.0. Python 3.10+. Standard library only.
|
|
172
|
+
## Sources and prediction
|
|
173
|
+
|
|
174
|
+
`awdecide.sources` adds where a decision's state comes from and a rung that tries to
|
|
175
|
+
predict the answer. `ReplState(session)` turns a live `awrepl` session into a STABLE
|
|
176
|
+
state descriptor -- variable names, types and bucketed sizes (`rows:list:1k-9k`), never
|
|
177
|
+
a repr and never a value, so the descriptor is safe to hash, log and resolve in the
|
|
178
|
+
ledger. `PredictBackend(env)` is a ladder rung that asks a value oracle what each option
|
|
179
|
+
is worth and ABSTAINS unless the top two are further apart than `margin`; the oracle is
|
|
180
|
+
anything exposing `value(state, option)`, including `AwpredictValueEnv`, which wraps an
|
|
181
|
+
`awpredict` engine's reward or value head. **The shipped default
|
|
182
|
+
(`default_predict_backend()`) is the self-updating last-outcome lookup, not the learned
|
|
183
|
+
model, and that is a measurement rather than a preference.**
|
|
184
|
+
A bench (`tool_outcome_predict_bench`) asked whether anything can predict
|
|
185
|
+
that the next run of a command shape will pass, well enough to skip the call, over
|
|
186
|
+
323,644 real tool outcomes (95.0% pass) with a temporal 80/20 split, scored on the
|
|
187
|
+
UNSEEN bucket only -- novel command shapes -- because a self-updating dictionary already
|
|
188
|
+
owns the seen rows and an aggregate is therefore structurally unable to move. On 37,668
|
|
189
|
+
UNSEEN rows the lookup, the online majority and "always run it" all score 0.9571, and
|
|
190
|
+
coarsening the key to the command family makes it worse (0.9291); on a 12,000-row window
|
|
191
|
+
with the learned arms in (1,785 UNSEEN) `awpredict`'s token-hash arm ties the base rate
|
|
192
|
+
exactly at 0.9434 and its whole-string arm loses to it at 0.9412 -- so the bench exits 1,
|
|
193
|
+
and exit 1 IS the finding: on a novel shape there is nothing in the history to learn
|
|
194
|
+
from, and every arm collapses onto the base rate. The number that decides the design is
|
|
195
|
+
not accuracy but skip-precision: 0.9434 on UNSEEN means roughly one in eighteen skipped
|
|
196
|
+
calls would really have failed, so the rung is wired to abstain by default and to answer
|
|
197
|
+
only where it has seen the option before. Re-run it when an engine with a real value head
|
|
198
|
+
is a candidate; the arm ships the day it beats the dictionary, not before.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""awdecide -- Aither World Decide.
|
|
2
|
+
|
|
3
|
+
One typed-decision contract -- choice / score / bool with a probability -- over
|
|
4
|
+
a ladder of backends you already run (rules, any local model as a callable, an
|
|
5
|
+
OpenAI-wire model's logprobs), fail-closed (decided=False is an answer), with a
|
|
6
|
+
Brier ledger that resolves every decision against its outcome and reports
|
|
7
|
+
whether the probabilities carry information beyond the base rate.
|
|
8
|
+
|
|
9
|
+
from awdecide import Question, Ladder, RulesBackend, Ledger
|
|
10
|
+
|
|
11
|
+
q = {"category": Question.choice(["billing", "technical", "sales"]),
|
|
12
|
+
"urgent": Question.bool(min_confidence=0.7)}
|
|
13
|
+
answers = Ladder([RulesBackend([(r"refund|invoice", "billing")])]).decide(state, q)
|
|
14
|
+
answers["category"].value, answers["category"].probability, answers["urgent"].decided
|
|
15
|
+
|
|
16
|
+
Stdlib only. Nothing here sends the state anywhere you did not name.
|
|
17
|
+
"""
|
|
18
|
+
from .backends import (
|
|
19
|
+
Backend,
|
|
20
|
+
CallableBackend,
|
|
21
|
+
Ladder,
|
|
22
|
+
LogprobBackend,
|
|
23
|
+
RulesBackend,
|
|
24
|
+
default_ladder,
|
|
25
|
+
parse_question_spec,
|
|
26
|
+
)
|
|
27
|
+
from .contract import Decision, Question, from_probabilities, normalize, undecided
|
|
28
|
+
from .door import DoorBackend, decision_from_door, door_request
|
|
29
|
+
from .ledger import Ledger
|
|
30
|
+
from .loop import ChatBackend, Loop
|
|
31
|
+
|
|
32
|
+
__version__ = "0.3.0"
|
|
33
|
+
__all__ = [
|
|
34
|
+
"Backend", "CallableBackend", "Ladder", "LogprobBackend", "RulesBackend",
|
|
35
|
+
"default_ladder", "parse_question_spec", "Decision", "Question",
|
|
36
|
+
"from_probabilities", "normalize", "undecided", "Ledger", "DoorBackend",
|
|
37
|
+
"decision_from_door", "door_request", "ChatBackend", "Loop", "__version__",
|
|
38
|
+
]
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Stack-aware `doctor` for awdecide.
|
|
2
|
+
|
|
3
|
+
GENERATED BY gen_aw_doctor.py -- DO NOT EDIT.
|
|
4
|
+
Regenerate it with the generator named above; a hand-edit here is reverted by
|
|
5
|
+
the next run and fails the parity gate.
|
|
6
|
+
|
|
7
|
+
Why a doctor exists at all: the aw* bricks are designed to COMPOSE, so the
|
|
8
|
+
interesting failures live BETWEEN them. "awdecide is installed" is not the useful
|
|
9
|
+
fact -- "awdecide is installed and the thing it pairs with is not" is. This reports
|
|
10
|
+
the whole stack, not just itself.
|
|
11
|
+
|
|
12
|
+
stdlib only, on purpose: a diagnostic that cannot run because a dependency is
|
|
13
|
+
missing is worthless precisely when you need it.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import importlib.util
|
|
18
|
+
import os
|
|
19
|
+
import shutil
|
|
20
|
+
import sys
|
|
21
|
+
|
|
22
|
+
#: Frozen from the Aither World registry at generation time. A shipped
|
|
23
|
+
#: package cannot read the registry, and a doctor that guessed at the family
|
|
24
|
+
#: would go stale in silence. Regenerate to update.
|
|
25
|
+
SELF = 'awdecide'
|
|
26
|
+
FAMILY = ['awask', 'awavatar', 'awbac', 'awbrain', 'awbrowse', 'awclassify', 'awdeck', 'awdelphi', 'awdit', 'awembed', 'awevolve', 'awfind', 'awflow', 'awfocus', 'awgit', 'awgraph', 'awgym', 'awiam', 'awkno', 'awm', 'awmail', 'awmine', 'awnboard', 'awnest', 'awnet', 'awnode', 'awpool', 'awpredict', 'awprism', 'awprove', 'awreason', 'awrecover', 'awrecurse', 'awrelay', 'awrena', 'awrepl', 'awreport', 'awresearch', 'awrise', 'awrouter', 'awrtifact', 'awrun', 'awscreen', 'awseal', 'awsettings', 'awshare', 'awsprite', 'awstorage', 'awswarm', 'awtax', 'awtoll', 'awtunnel', 'awvision', 'awvoice', 'awwall', 'gawbbonet']
|
|
27
|
+
PAIRS_WITH = ['awclassify', 'adk', 'awgraph', 'awpredict', 'awprove', 'awrepl', 'awtoll']
|
|
28
|
+
|
|
29
|
+
#: This brick's OWN config, read out of its source at generation time.
|
|
30
|
+
#: ENV_REQUIRED is `os.environ["X"]` -- absent, that is a KeyError the moment
|
|
31
|
+
#: the line runs. ENV_OPTIONAL is `os.getenv("X")`, which returns None and lets
|
|
32
|
+
#: the caller cope. Only this brick's namespace is listed: reporting the
|
|
33
|
+
#: platform-wide vars it also touches would be noise, and a doctor that floods
|
|
34
|
+
#: gets ignored.
|
|
35
|
+
ENV_REQUIRED = []
|
|
36
|
+
ENV_OPTIONAL = ['AWDECIDE_DB']
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _installed(mod: str) -> "str | None":
|
|
40
|
+
"""Version if importable, else None. Never raises -- a broken sibling must
|
|
41
|
+
not take the diagnostic down with it."""
|
|
42
|
+
try:
|
|
43
|
+
if importlib.util.find_spec(mod) is None:
|
|
44
|
+
return None
|
|
45
|
+
except (ImportError, ValueError):
|
|
46
|
+
return None
|
|
47
|
+
try:
|
|
48
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
49
|
+
try:
|
|
50
|
+
return version(mod)
|
|
51
|
+
except PackageNotFoundError:
|
|
52
|
+
return "installed"
|
|
53
|
+
except Exception:
|
|
54
|
+
return "installed"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def report(out=None) -> int:
|
|
58
|
+
"""Print the stack picture. 0 = this brick and its pairs are present."""
|
|
59
|
+
out = out or sys.stdout
|
|
60
|
+
print(f"{SELF} doctor", file=out)
|
|
61
|
+
|
|
62
|
+
mine = _installed(SELF)
|
|
63
|
+
print(f" self {SELF} {mine or 'NOT IMPORTABLE'}", file=out)
|
|
64
|
+
shim = shutil.which(SELF)
|
|
65
|
+
print(f" command {shim or 'not on PATH'}", file=out)
|
|
66
|
+
|
|
67
|
+
# The stack. Siblings this brick pairs with are called out separately,
|
|
68
|
+
# because a missing pair is a REASON, while a missing unrelated brick is
|
|
69
|
+
# just a fact about your machine.
|
|
70
|
+
missing_pairs, present = [], []
|
|
71
|
+
for name in FAMILY:
|
|
72
|
+
v = _installed(name)
|
|
73
|
+
if v:
|
|
74
|
+
present.append(name)
|
|
75
|
+
elif name in PAIRS_WITH:
|
|
76
|
+
missing_pairs.append(name)
|
|
77
|
+
print(f" stack {len(present)}/{len(FAMILY)} aw* packages installed",
|
|
78
|
+
file=out)
|
|
79
|
+
if present:
|
|
80
|
+
print(f" {' '.join(sorted(present))}", file=out)
|
|
81
|
+
|
|
82
|
+
missing_req = [v for v in ENV_REQUIRED if not os.environ.get(v)]
|
|
83
|
+
if ENV_REQUIRED or ENV_OPTIONAL:
|
|
84
|
+
have = sum(1 for v in ENV_REQUIRED + ENV_OPTIONAL if os.environ.get(v))
|
|
85
|
+
total = len(ENV_REQUIRED) + len(ENV_OPTIONAL)
|
|
86
|
+
print(f" config {have}/{total} of this brick's own vars set", file=out)
|
|
87
|
+
if missing_req:
|
|
88
|
+
# Not a preference. os.environ[...] raises the moment it runs.
|
|
89
|
+
print(f" MISSING REQUIRED: {' '.join(missing_req)}", file=out)
|
|
90
|
+
|
|
91
|
+
local = _local_checks()
|
|
92
|
+
for line in local:
|
|
93
|
+
print(f" {line}", file=out)
|
|
94
|
+
problems, unjudged = _local_verdict()
|
|
95
|
+
|
|
96
|
+
if mine is None:
|
|
97
|
+
print(f"\nverdict: {SELF} itself is not importable. Reinstall it before "
|
|
98
|
+
f"anything else here means much.", file=out)
|
|
99
|
+
return 1
|
|
100
|
+
if missing_req:
|
|
101
|
+
print(f"\nverdict: {SELF} is missing required config "
|
|
102
|
+
f"({', '.join(missing_req)}). Those are read with os.environ[...], "
|
|
103
|
+
f"so the code path that needs them raises rather than degrades.",
|
|
104
|
+
file=out)
|
|
105
|
+
return 1
|
|
106
|
+
# A measured NO outranks a missing optional pair: awrise printed
|
|
107
|
+
# "nothing wakes run-due" and still exited 0 because the missing-pairs
|
|
108
|
+
# branch returned first (measured 2026-09-20).
|
|
109
|
+
if problems:
|
|
110
|
+
for p in problems:
|
|
111
|
+
print(f"\nverdict: {p}", file=out)
|
|
112
|
+
return 1
|
|
113
|
+
if unjudged:
|
|
114
|
+
for u in unjudged:
|
|
115
|
+
print(f"\nverdict: UNJUDGED -- could not judge: {u}", file=out)
|
|
116
|
+
return 2
|
|
117
|
+
if missing_pairs:
|
|
118
|
+
print(f"\nverdict: {SELF} works, but pairs with "
|
|
119
|
+
f"{', '.join(sorted(missing_pairs))} which "
|
|
120
|
+
f"{'is' if len(missing_pairs) == 1 else 'are'} not installed. "
|
|
121
|
+
f"That is a capability you are missing, not an error.", file=out)
|
|
122
|
+
return 0
|
|
123
|
+
print(f"\nverdict: {SELF} and everything it pairs with are present.", file=out)
|
|
124
|
+
return 0
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _local_verdict() -> tuple:
|
|
128
|
+
"""The brick's OWN verdict, folded into the exit code.
|
|
129
|
+
|
|
130
|
+
Without this the local lines are DECORATION. Measured 2026-09-20: awrise
|
|
131
|
+
printed "hostclock NOT INSTALLED -- nothing wakes run-due" and "last tick
|
|
132
|
+
never", then exited 0 -- teaching an operator that a doctor exit code
|
|
133
|
+
carries no information. A brick opts in with `_doctor_local_verdict()`
|
|
134
|
+
returning (problems, unjudged); a brick without one is unaffected.
|
|
135
|
+
"""
|
|
136
|
+
try:
|
|
137
|
+
mod = importlib.import_module(f"{SELF}.doctor_local")
|
|
138
|
+
problems, unjudged = mod._doctor_local_verdict()
|
|
139
|
+
except Exception: # noqa: BLE001
|
|
140
|
+
return [], []
|
|
141
|
+
return list(problems or []), list(unjudged or [])
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _local_checks() -> "list[str]":
|
|
145
|
+
"""Per-brick checks, if this package defines them.
|
|
146
|
+
|
|
147
|
+
Kept as a HOOK rather than generated guesses: the generator knows the family
|
|
148
|
+
from the registry, but it does not know what awdecide needs at runtime, and a
|
|
149
|
+
doctor that invented config requirements would be confidently wrong. A
|
|
150
|
+
package supplies `_doctor_local()` returning display lines; absent, the
|
|
151
|
+
stack picture above still stands on its own.
|
|
152
|
+
"""
|
|
153
|
+
try:
|
|
154
|
+
mod = importlib.import_module(f"{SELF}.doctor_local")
|
|
155
|
+
except Exception:
|
|
156
|
+
return []
|
|
157
|
+
try:
|
|
158
|
+
return list(mod._doctor_local())
|
|
159
|
+
except Exception as exc: # noqa: BLE001
|
|
160
|
+
return [f"local checks raised {type(exc).__name__}: {exc}"]
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def main(argv: "list[str] | None" = None) -> int:
|
|
164
|
+
# --self-test delegates to a SIBLING module when one exists.
|
|
165
|
+
#
|
|
166
|
+
# This file is generated and a fresh run replaces it, so a self-test
|
|
167
|
+
# written HERE is deleted by the next regeneration. awdelphi learned that
|
|
168
|
+
# the expensive way: 125 lines exercising four real failure paths --
|
|
169
|
+
# convergence, roster anonymization, resume, gateway-down -- lived in this
|
|
170
|
+
# file and were destroyed by a routine regeneration, silently, leaving a
|
|
171
|
+
# --self-test flag that reported PASS while asserting nothing.
|
|
172
|
+
#
|
|
173
|
+
# So the seam is a separate module the generator never writes. A package
|
|
174
|
+
# with real machinery to prove puts it in _selftest.py; everything else
|
|
175
|
+
# keeps the honest answer below rather than a self-test that only ever
|
|
176
|
+
# passes.
|
|
177
|
+
argv = list(argv if argv is not None else __import__("sys").argv[1:])
|
|
178
|
+
if "--self-test" in argv:
|
|
179
|
+
try:
|
|
180
|
+
from . import _selftest as _st
|
|
181
|
+
except Exception:
|
|
182
|
+
print("no _selftest module: this doctor reports the stack, and has",
|
|
183
|
+
"no machinery of its own to prove")
|
|
184
|
+
return 0
|
|
185
|
+
return int(_st.run())
|
|
186
|
+
return report()
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
if __name__ == "__main__":
|
|
190
|
+
raise SystemExit(main())
|