decision-circuits 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- decision_circuits-0.1.0/.gitignore +6 -0
- decision_circuits-0.1.0/LICENSE +21 -0
- decision_circuits-0.1.0/PKG-INFO +144 -0
- decision_circuits-0.1.0/README.md +119 -0
- decision_circuits-0.1.0/pyproject.toml +49 -0
- decision_circuits-0.1.0/src/decision_circuits/__init__.py +66 -0
- decision_circuits-0.1.0/src/decision_circuits/dsl.py +507 -0
- decision_circuits-0.1.0/src/decision_circuits/gates.py +181 -0
- decision_circuits-0.1.0/src/decision_circuits/py.typed +0 -0
- decision_circuits-0.1.0/tests/test_dsl.py +133 -0
- decision_circuits-0.1.0/tests/test_gates.py +82 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 James Barney
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: decision-circuits
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deterministic decision gates over calibrated model answers: a small DSL that compiles to System One requests, evaluates AND/OR/NOT/threshold/argmax/majority/verify/order gates with explicit uncertainty, and renders the circuit.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Barneyjm/decision-circuits
|
|
6
|
+
Project-URL: Article, https://towardsdatascience.com/attaining-llm-certainty-with-ai-decision-circuits/
|
|
7
|
+
Author: James Barney
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: calibration,decision-circuits,gates,guardrails,llm,system-one,typesafe
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: pydantic>=2.0
|
|
22
|
+
Provides-Extra: http
|
|
23
|
+
Requires-Dist: httpx>=0.25; extra == 'http'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# decision-circuits
|
|
27
|
+
|
|
28
|
+
Deterministic decision gates over calibrated model answers.
|
|
29
|
+
|
|
30
|
+
A System One style model (TypeSafe's Jev, or an open model that speaks
|
|
31
|
+
the same contract) answers typed questions about a piece of state and
|
|
32
|
+
returns probabilities, not prose. A **decision circuit** is a set of
|
|
33
|
+
questions plus a set of **gates**: code that combines those
|
|
34
|
+
probabilities into decisions with thresholds, boolean logic, votes,
|
|
35
|
+
verification, and ordinal buckets, and that says out loud when it is
|
|
36
|
+
uncertain instead of guessing.
|
|
37
|
+
|
|
38
|
+
The model never sees the gates. That makes a circuit versionable,
|
|
39
|
+
testable offline, and auditable per item: every result carries its
|
|
40
|
+
probability, its outcome (`decided`, `abstain`, `escalate`, `default`),
|
|
41
|
+
and a trace of every input that fed it.
|
|
42
|
+
|
|
43
|
+
Background: [Attaining LLM Certainty with AI Decision Circuits](https://towardsdatascience.com/attaining-llm-certainty-with-ai-decision-circuits/).
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install decision-circuits
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Example
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import httpx
|
|
53
|
+
from decision_circuits import Circuit, Q, G, argmax, order
|
|
54
|
+
|
|
55
|
+
c = Circuit()
|
|
56
|
+
c.noul(
|
|
57
|
+
"pii",
|
|
58
|
+
"Does this text contain PII about a private individual?",
|
|
59
|
+
true="Email, phone, home address, ID, or card number",
|
|
60
|
+
false="No PII, or business-only details",
|
|
61
|
+
)
|
|
62
|
+
c.noul("business", "Are all identifying details about a business rather than a person?")
|
|
63
|
+
c.noul("angry", "Is the customer angry?")
|
|
64
|
+
c.choice("dept", "Which team should handle this?", {"billing": "Charges, refunds, invoices", "technical": "Bugs, outages, API", "other": "Anything else"})
|
|
65
|
+
c.score("urgency", "How urgent is this?", ["Low", "Medium", "High", "Critical"])
|
|
66
|
+
|
|
67
|
+
c.gate("redact", ((Q("pii") >= 0.7) & ~Q("business")) >= 0.6, on_uncertain="escalate")
|
|
68
|
+
c.gate("route", argmax("dept", min_confidence=0.35))
|
|
69
|
+
c.gate("tier", order("urgency", [1.0, 2.0, 2.6]))
|
|
70
|
+
c.gate("human", (Q("angry") | Q("urgency")[3]) >= 0.6, on_uncertain="escalate")
|
|
71
|
+
c.gate("bill_hot", (G("route")["billing"] & G("tier")[3]).at(0.5))
|
|
72
|
+
|
|
73
|
+
out = c.run(
|
|
74
|
+
httpx.Client(),
|
|
75
|
+
"Card charged twice, refund NOW, my card ends in 4412.",
|
|
76
|
+
url="https://api.typesafe.ai/v1/systemone",
|
|
77
|
+
headers={"Authorization": f"Bearer {TYPESAFE_API_KEY}"},
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
out["gates"]["redact"]
|
|
81
|
+
# {"value": True, "p": 0.88, "outcome": "decided", "trace": [...]}
|
|
82
|
+
out["gates"]["route"]
|
|
83
|
+
# {"value": "billing", "p": 0.81, "outcome": "decided", ...}
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
`run` posts the circuit as a normal System One request. If the server
|
|
87
|
+
evaluates gates itself, its results are used. If it returns answers
|
|
88
|
+
only (TypeSafe today), the gates are evaluated on the client from the
|
|
89
|
+
returned probabilities. `out["gates_evaluated_by"]` says which.
|
|
90
|
+
|
|
91
|
+
## The expression language
|
|
92
|
+
|
|
93
|
+
| form | meaning |
|
|
94
|
+
|---|---|
|
|
95
|
+
| `Q("noul_id")` | P(yes) for a yes/no question |
|
|
96
|
+
| `Q("choice_id")["option"]` | probability of one option |
|
|
97
|
+
| `Q("score_id")[level]` | probability of one score level |
|
|
98
|
+
| `G("gate_id")`, `G("gate_id")["value"]` | an earlier gate's probability, or one value of a categorical gate |
|
|
99
|
+
| `~e` | NOT: `1 - p` |
|
|
100
|
+
| `a & b & c` | AND: product (independence assumed and recorded in the trace) |
|
|
101
|
+
| `a \| b` | OR: `1 - prod(1 - p)` |
|
|
102
|
+
| `e >= tau`, `e.at(tau)` | threshold: boolean at `tau` with an uncertainty band around it |
|
|
103
|
+
| `argmax("choice_id", min_confidence=...)` | pick the top option; abstain below the confidence floor |
|
|
104
|
+
| `majority("c1", "c2", "c3")` | vote across paraphrased questions over the same options |
|
|
105
|
+
| `verify("choice_id", check=Q("supported"), tau=...)` | a negative checker: escalate when the check does not support the pick |
|
|
106
|
+
| `order("score_id", [c1, c2, ...])` | bucket the expected score by cutpoints |
|
|
107
|
+
|
|
108
|
+
An expression used as a gate without a threshold thresholds at 0.5.
|
|
109
|
+
`e >= tau` builds a node and does not compare; `bool(Q("x") >= 0.7)` is
|
|
110
|
+
always true, so use `.at(tau)` anywhere an operator would read as a
|
|
111
|
+
runtime comparison.
|
|
112
|
+
|
|
113
|
+
Every gate takes `on_uncertain="abstain" | "escalate" | "default"`
|
|
114
|
+
(with `default=value`) and `band=width`. A gate is uncertain when the
|
|
115
|
+
probability it acts on lies within `band` of its threshold, or when a
|
|
116
|
+
confidence floor fails. Uncertainty is surfaced, never silently
|
|
117
|
+
resolved.
|
|
118
|
+
|
|
119
|
+
## Without a server
|
|
120
|
+
|
|
121
|
+
`c.compile()` returns the flat `gates` map (the wire format).
|
|
122
|
+
`c.evaluate(answers)` runs the gates against answers you already have,
|
|
123
|
+
in the response format of any System One server. `evaluate_gates` is
|
|
124
|
+
the underlying function if you build the map by hand.
|
|
125
|
+
|
|
126
|
+
## Diagram
|
|
127
|
+
|
|
128
|
+
`c.to_mermaid()` renders the circuit as a Mermaid flowchart with three
|
|
129
|
+
columns, inputs, logic, and decisions; pass the results of a run to
|
|
130
|
+
color each node by outcome. Paste it into a README or render it with
|
|
131
|
+
`mmdc`.
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
print(c.to_mermaid(results=out["gates"], answers=out["answers"]))
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Status
|
|
138
|
+
|
|
139
|
+
Alpha. The wire format is TypeSafe's `POST /v1/systemone` request with
|
|
140
|
+
an added `gates` block; the reference server that evaluates gates is
|
|
141
|
+
[s1proto](https://github.com/Barneyjm/s1-proto). The gate semantics
|
|
142
|
+
(`and` as a product, `or` as noisy-or) assume independent questions;
|
|
143
|
+
the trace records that assumption on every result so a reviewer can
|
|
144
|
+
see it.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# decision-circuits
|
|
2
|
+
|
|
3
|
+
Deterministic decision gates over calibrated model answers.
|
|
4
|
+
|
|
5
|
+
A System One style model (TypeSafe's Jev, or an open model that speaks
|
|
6
|
+
the same contract) answers typed questions about a piece of state and
|
|
7
|
+
returns probabilities, not prose. A **decision circuit** is a set of
|
|
8
|
+
questions plus a set of **gates**: code that combines those
|
|
9
|
+
probabilities into decisions with thresholds, boolean logic, votes,
|
|
10
|
+
verification, and ordinal buckets, and that says out loud when it is
|
|
11
|
+
uncertain instead of guessing.
|
|
12
|
+
|
|
13
|
+
The model never sees the gates. That makes a circuit versionable,
|
|
14
|
+
testable offline, and auditable per item: every result carries its
|
|
15
|
+
probability, its outcome (`decided`, `abstain`, `escalate`, `default`),
|
|
16
|
+
and a trace of every input that fed it.
|
|
17
|
+
|
|
18
|
+
Background: [Attaining LLM Certainty with AI Decision Circuits](https://towardsdatascience.com/attaining-llm-certainty-with-ai-decision-circuits/).
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pip install decision-circuits
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Example
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
import httpx
|
|
28
|
+
from decision_circuits import Circuit, Q, G, argmax, order
|
|
29
|
+
|
|
30
|
+
c = Circuit()
|
|
31
|
+
c.noul(
|
|
32
|
+
"pii",
|
|
33
|
+
"Does this text contain PII about a private individual?",
|
|
34
|
+
true="Email, phone, home address, ID, or card number",
|
|
35
|
+
false="No PII, or business-only details",
|
|
36
|
+
)
|
|
37
|
+
c.noul("business", "Are all identifying details about a business rather than a person?")
|
|
38
|
+
c.noul("angry", "Is the customer angry?")
|
|
39
|
+
c.choice("dept", "Which team should handle this?", {"billing": "Charges, refunds, invoices", "technical": "Bugs, outages, API", "other": "Anything else"})
|
|
40
|
+
c.score("urgency", "How urgent is this?", ["Low", "Medium", "High", "Critical"])
|
|
41
|
+
|
|
42
|
+
c.gate("redact", ((Q("pii") >= 0.7) & ~Q("business")) >= 0.6, on_uncertain="escalate")
|
|
43
|
+
c.gate("route", argmax("dept", min_confidence=0.35))
|
|
44
|
+
c.gate("tier", order("urgency", [1.0, 2.0, 2.6]))
|
|
45
|
+
c.gate("human", (Q("angry") | Q("urgency")[3]) >= 0.6, on_uncertain="escalate")
|
|
46
|
+
c.gate("bill_hot", (G("route")["billing"] & G("tier")[3]).at(0.5))
|
|
47
|
+
|
|
48
|
+
out = c.run(
|
|
49
|
+
httpx.Client(),
|
|
50
|
+
"Card charged twice, refund NOW, my card ends in 4412.",
|
|
51
|
+
url="https://api.typesafe.ai/v1/systemone",
|
|
52
|
+
headers={"Authorization": f"Bearer {TYPESAFE_API_KEY}"},
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
out["gates"]["redact"]
|
|
56
|
+
# {"value": True, "p": 0.88, "outcome": "decided", "trace": [...]}
|
|
57
|
+
out["gates"]["route"]
|
|
58
|
+
# {"value": "billing", "p": 0.81, "outcome": "decided", ...}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
`run` posts the circuit as a normal System One request. If the server
|
|
62
|
+
evaluates gates itself, its results are used. If it returns answers
|
|
63
|
+
only (TypeSafe today), the gates are evaluated on the client from the
|
|
64
|
+
returned probabilities. `out["gates_evaluated_by"]` says which.
|
|
65
|
+
|
|
66
|
+
## The expression language
|
|
67
|
+
|
|
68
|
+
| form | meaning |
|
|
69
|
+
|---|---|
|
|
70
|
+
| `Q("noul_id")` | P(yes) for a yes/no question |
|
|
71
|
+
| `Q("choice_id")["option"]` | probability of one option |
|
|
72
|
+
| `Q("score_id")[level]` | probability of one score level |
|
|
73
|
+
| `G("gate_id")`, `G("gate_id")["value"]` | an earlier gate's probability, or one value of a categorical gate |
|
|
74
|
+
| `~e` | NOT: `1 - p` |
|
|
75
|
+
| `a & b & c` | AND: product (independence assumed and recorded in the trace) |
|
|
76
|
+
| `a \| b` | OR: `1 - prod(1 - p)` |
|
|
77
|
+
| `e >= tau`, `e.at(tau)` | threshold: boolean at `tau` with an uncertainty band around it |
|
|
78
|
+
| `argmax("choice_id", min_confidence=...)` | pick the top option; abstain below the confidence floor |
|
|
79
|
+
| `majority("c1", "c2", "c3")` | vote across paraphrased questions over the same options |
|
|
80
|
+
| `verify("choice_id", check=Q("supported"), tau=...)` | a negative checker: escalate when the check does not support the pick |
|
|
81
|
+
| `order("score_id", [c1, c2, ...])` | bucket the expected score by cutpoints |
|
|
82
|
+
|
|
83
|
+
An expression used as a gate without a threshold thresholds at 0.5.
|
|
84
|
+
`e >= tau` builds a node and does not compare; `bool(Q("x") >= 0.7)` is
|
|
85
|
+
always true, so use `.at(tau)` anywhere an operator would read as a
|
|
86
|
+
runtime comparison.
|
|
87
|
+
|
|
88
|
+
Every gate takes `on_uncertain="abstain" | "escalate" | "default"`
|
|
89
|
+
(with `default=value`) and `band=width`. A gate is uncertain when the
|
|
90
|
+
probability it acts on lies within `band` of its threshold, or when a
|
|
91
|
+
confidence floor fails. Uncertainty is surfaced, never silently
|
|
92
|
+
resolved.
|
|
93
|
+
|
|
94
|
+
## Without a server
|
|
95
|
+
|
|
96
|
+
`c.compile()` returns the flat `gates` map (the wire format).
|
|
97
|
+
`c.evaluate(answers)` runs the gates against answers you already have,
|
|
98
|
+
in the response format of any System One server. `evaluate_gates` is
|
|
99
|
+
the underlying function if you build the map by hand.
|
|
100
|
+
|
|
101
|
+
## Diagram
|
|
102
|
+
|
|
103
|
+
`c.to_mermaid()` renders the circuit as a Mermaid flowchart with three
|
|
104
|
+
columns, inputs, logic, and decisions; pass the results of a run to
|
|
105
|
+
color each node by outcome. Paste it into a README or render it with
|
|
106
|
+
`mmdc`.
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
print(c.to_mermaid(results=out["gates"], answers=out["answers"]))
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Status
|
|
113
|
+
|
|
114
|
+
Alpha. The wire format is TypeSafe's `POST /v1/systemone` request with
|
|
115
|
+
an added `gates` block; the reference server that evaluates gates is
|
|
116
|
+
[s1proto](https://github.com/Barneyjm/s1-proto). The gate semantics
|
|
117
|
+
(`and` as a product, `or` as noisy-or) assume independent questions;
|
|
118
|
+
the trace records that assumption on every result so a reviewer can
|
|
119
|
+
see it.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "decision-circuits"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Deterministic decision gates over calibrated model answers: a small DSL that compiles to System One requests, evaluates AND/OR/NOT/threshold/argmax/majority/verify/order gates with explicit uncertainty, and renders the circuit."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
authors = [{ name = "James Barney" }]
|
|
10
|
+
keywords = ["decision-circuits", "system-one", "typesafe", "calibration", "gates", "llm", "guardrails"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 3 - Alpha",
|
|
13
|
+
"Intended Audience :: Developers",
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Programming Language :: Python :: 3.10",
|
|
16
|
+
"Programming Language :: Python :: 3.11",
|
|
17
|
+
"Programming Language :: Python :: 3.12",
|
|
18
|
+
"Programming Language :: Python :: 3.13",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["pydantic>=2.0"]
|
|
23
|
+
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
http = ["httpx>=0.25"]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/Barneyjm/decision-circuits"
|
|
29
|
+
Article = "https://towardsdatascience.com/attaining-llm-certainty-with-ai-decision-circuits/"
|
|
30
|
+
|
|
31
|
+
[dependency-groups]
|
|
32
|
+
dev = ["pytest>=8", "ruff>=0.6", "httpx>=0.25"]
|
|
33
|
+
|
|
34
|
+
[build-system]
|
|
35
|
+
requires = ["hatchling"]
|
|
36
|
+
build-backend = "hatchling.build"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.targets.wheel]
|
|
39
|
+
packages = ["src/decision_circuits"]
|
|
40
|
+
|
|
41
|
+
[tool.hatch.build.targets.sdist]
|
|
42
|
+
include = ["src/decision_circuits", "tests", "README.md", "LICENSE"]
|
|
43
|
+
|
|
44
|
+
[tool.pytest.ini_options]
|
|
45
|
+
testpaths = ["tests"]
|
|
46
|
+
|
|
47
|
+
[tool.ruff]
|
|
48
|
+
line-length = 160
|
|
49
|
+
target-version = "py310"
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Decision circuits: deterministic gates over calibrated model answers.
|
|
2
|
+
|
|
3
|
+
Build a circuit with the DSL, send it to any System One style server
|
|
4
|
+
(TypeSafe's Jev, or an open model speaking the same contract), and get
|
|
5
|
+
back typed decisions with probabilities, traces, and explicit
|
|
6
|
+
uncertainty handling. Gates are evaluated by the server when it
|
|
7
|
+
supports them and here on the client otherwise, so the result is the
|
|
8
|
+
same either way.
|
|
9
|
+
|
|
10
|
+
from decision_circuits import Circuit, Q, argmax
|
|
11
|
+
|
|
12
|
+
c = Circuit()
|
|
13
|
+
c.noul("pii", "Does this text contain PII about a private individual?")
|
|
14
|
+
c.choice("dept", "Which team?", {"billing": None, "technical": None, "other": None})
|
|
15
|
+
c.gate("redact", Q("pii") >= 0.7, on_uncertain="escalate")
|
|
16
|
+
c.gate("route", argmax("dept", min_confidence=0.35))
|
|
17
|
+
|
|
18
|
+
out = c.run(httpx.Client(), state, url="https://api.typesafe.ai/v1/systemone",
|
|
19
|
+
headers={"Authorization": f"Bearer {key}"})
|
|
20
|
+
out["gates"]["redact"] # {"value": True, "p": 0.91, "outcome": "decided", "trace": [...]}
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from decision_circuits.dsl import (
|
|
24
|
+
And,
|
|
25
|
+
Categorical,
|
|
26
|
+
Circuit,
|
|
27
|
+
Expr,
|
|
28
|
+
G,
|
|
29
|
+
GateDef,
|
|
30
|
+
Not,
|
|
31
|
+
Or,
|
|
32
|
+
Q,
|
|
33
|
+
Threshold,
|
|
34
|
+
argmax,
|
|
35
|
+
majority,
|
|
36
|
+
order,
|
|
37
|
+
render_mermaid,
|
|
38
|
+
to_mermaid,
|
|
39
|
+
verify,
|
|
40
|
+
)
|
|
41
|
+
from decision_circuits.gates import Gate, GateResult, evaluate_gates
|
|
42
|
+
|
|
43
|
+
__version__ = "0.1.0"
|
|
44
|
+
|
|
45
|
+
__all__ = [
|
|
46
|
+
"And",
|
|
47
|
+
"Categorical",
|
|
48
|
+
"Circuit",
|
|
49
|
+
"Expr",
|
|
50
|
+
"G",
|
|
51
|
+
"Gate",
|
|
52
|
+
"GateDef",
|
|
53
|
+
"GateResult",
|
|
54
|
+
"Not",
|
|
55
|
+
"Or",
|
|
56
|
+
"Q",
|
|
57
|
+
"Threshold",
|
|
58
|
+
"__version__",
|
|
59
|
+
"argmax",
|
|
60
|
+
"evaluate_gates",
|
|
61
|
+
"majority",
|
|
62
|
+
"order",
|
|
63
|
+
"render_mermaid",
|
|
64
|
+
"to_mermaid",
|
|
65
|
+
"verify",
|
|
66
|
+
]
|
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
"""A small expression language for decision circuits.
|
|
2
|
+
|
|
3
|
+
Symbolic references combine with Python operators and compile to the
|
|
4
|
+
`gates` block the server evaluates. Same idiom as Django `Q` objects
|
|
5
|
+
or Polars expressions: build an expression, hand it to the request.
|
|
6
|
+
|
|
7
|
+
from decision_circuits import Circuit, Q, G, argmax, majority, verify, order
|
|
8
|
+
|
|
9
|
+
c = Circuit()
|
|
10
|
+
c.noul("pii", "Does this text contain PII about a private individual?",
|
|
11
|
+
true="Email, phone, home address, ID, card number", false="No PII or business-only")
|
|
12
|
+
c.noul("business", "Are all identifying details about a business, not a person?")
|
|
13
|
+
c.choice("dept", "Which team?", {"billing": "...", "technical": "...", "other": "..."})
|
|
14
|
+
c.score("urgency", "How urgent?", ["Low", "Medium", "High", "Critical"])
|
|
15
|
+
|
|
16
|
+
c.gate("redact", ((Q("pii") >= 0.7) & ~Q("business")) >= 0.6, on_uncertain="escalate")
|
|
17
|
+
c.gate("route", argmax("dept", min_confidence=0.35))
|
|
18
|
+
c.gate("tier", order("urgency", [1.0, 2.0, 2.6]))
|
|
19
|
+
c.gate("human", (Q("angry") | Q("urgency")[3]) >= 0.6, on_uncertain="escalate")
|
|
20
|
+
c.gate("bill_hot", (G("route")["billing"] & G("tier")[3]).at(0.5))
|
|
21
|
+
|
|
22
|
+
body = c.request(state) # JSON body for POST /v1/systemone
|
|
23
|
+
out = c.run(client, state) # or evaluate through a TestClient / httpx client
|
|
24
|
+
|
|
25
|
+
Semantics, in one paragraph. `Q("x")` is the probability behind a
|
|
26
|
+
question: a noul's P(yes), `Q("choice")["option"]` or `Q("score")[level]`
|
|
27
|
+
for one outcome (the `"choice:option"` string form also works), or
|
|
28
|
+
`G("name")` for an earlier gate, indexed the same way for categorical
|
|
29
|
+
gates. `~` flips it (a `not`
|
|
30
|
+
gate), `&` multiplies (an `and` gate, independence assumed and printed
|
|
31
|
+
in the trace), `|` is 1 - prod(1 - p) (an `or` gate). `>= tau` turns a
|
|
32
|
+
probability into a boolean at that threshold with an uncertainty band
|
|
33
|
+
around it; an expression used as a gate without `>=` thresholds at 0.5.
|
|
34
|
+
`argmax`, `majority`, `verify`, `order` are the categorical/ordinal
|
|
35
|
+
gates. Every gate carries `.on_uncertain("abstain" | "escalate" | "default", default=...)`
|
|
36
|
+
and `.band(width)`.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import itertools
|
|
42
|
+
from dataclasses import dataclass, field
|
|
43
|
+
from typing import Any
|
|
44
|
+
|
|
45
|
+
from decision_circuits.gates import Gate, evaluate_gates
|
|
46
|
+
|
|
47
|
+
# ---------------------------------------------------------------- expressions
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Expr:
|
|
51
|
+
"""Base for probability-valued expressions."""
|
|
52
|
+
|
|
53
|
+
def __and__(self, other: Expr) -> And:
|
|
54
|
+
return And([self, _as_expr(other)])
|
|
55
|
+
|
|
56
|
+
def __or__(self, other: Expr) -> Or:
|
|
57
|
+
return Or([self, _as_expr(other)])
|
|
58
|
+
|
|
59
|
+
def __invert__(self) -> Not:
|
|
60
|
+
return Not(self)
|
|
61
|
+
|
|
62
|
+
def __ge__(self, tau: float) -> Threshold:
|
|
63
|
+
"""`expr >= tau` builds a Threshold node; it does not compare.
|
|
64
|
+
`bool(Q("pii") >= 0.7)` is always True. Use `.at(tau)` where an
|
|
65
|
+
operator would read as a runtime comparison."""
|
|
66
|
+
return Threshold(self, float(tau))
|
|
67
|
+
|
|
68
|
+
def at(self, tau: float) -> Threshold:
|
|
69
|
+
"""Named form of `>=`: `Q("pii").at(0.7)`."""
|
|
70
|
+
return Threshold(self, float(tau))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _as_expr(x: Any) -> Expr:
|
|
74
|
+
if isinstance(x, Expr):
|
|
75
|
+
return x
|
|
76
|
+
if isinstance(x, str):
|
|
77
|
+
return Q(x)
|
|
78
|
+
raise TypeError(f"cannot use {x!r} in a circuit expression")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True)
|
|
82
|
+
class Q(Expr):
|
|
83
|
+
"""A question reference. A noul id is a probability on its own;
|
|
84
|
+
index a choice or score to get one option's probability:
|
|
85
|
+
`Q("dept")["billing"]`, `Q("urgency")[3]` (the `"dept:billing"`
|
|
86
|
+
string form is accepted too)."""
|
|
87
|
+
|
|
88
|
+
ref: str
|
|
89
|
+
|
|
90
|
+
def __getitem__(self, key: Any) -> Q:
|
|
91
|
+
if ":" in self.ref:
|
|
92
|
+
raise KeyError(f"{self.ref!r} already names an option")
|
|
93
|
+
return Q(f"{self.ref}:{key}")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class G(Expr):
|
|
98
|
+
"""A reference to an earlier gate. A boolean gate is a probability on
|
|
99
|
+
its own; index a categorical gate for one value: `G("route")["billing"]`."""
|
|
100
|
+
|
|
101
|
+
ref: str
|
|
102
|
+
|
|
103
|
+
def __getitem__(self, key: Any) -> G:
|
|
104
|
+
if ":" in self.ref:
|
|
105
|
+
raise KeyError(f"{self.ref!r} already names a value")
|
|
106
|
+
return G(f"{self.ref}:{key}")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@dataclass(frozen=True)
|
|
110
|
+
class Not(Expr):
|
|
111
|
+
inner: Expr
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclass(frozen=True)
|
|
115
|
+
class And(Expr):
|
|
116
|
+
parts: list[Expr]
|
|
117
|
+
|
|
118
|
+
def __and__(self, other: Expr) -> And: # flatten chains
|
|
119
|
+
return And([*self.parts, _as_expr(other)])
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class Or(Expr):
|
|
124
|
+
parts: list[Expr]
|
|
125
|
+
|
|
126
|
+
def __or__(self, other: Expr) -> Or:
|
|
127
|
+
return Or([*self.parts, _as_expr(other)])
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True)
|
|
131
|
+
class Threshold(Expr):
|
|
132
|
+
inner: Expr
|
|
133
|
+
tau: float
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# categorical / ordinal gate constructors ----------------------------------
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(frozen=True)
|
|
140
|
+
class Categorical:
|
|
141
|
+
op: str
|
|
142
|
+
input: str | None = None
|
|
143
|
+
inputs: list[str] | None = None
|
|
144
|
+
check: Expr | None = None
|
|
145
|
+
tau: float = 0.5
|
|
146
|
+
min_confidence: float = 0.0
|
|
147
|
+
cutpoints: list[float] | None = None
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def argmax(choice_id: str, min_confidence: float = 0.0) -> Categorical:
|
|
151
|
+
return Categorical("argmax", input=choice_id, min_confidence=min_confidence)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def majority(*choice_ids: str, min_confidence: float = 0.0) -> Categorical:
|
|
155
|
+
return Categorical("majority", inputs=list(choice_ids), min_confidence=min_confidence)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def verify(choice_id: str, check: Expr | str, tau: float = 0.6, min_confidence: float = 0.0) -> Categorical:
|
|
159
|
+
return Categorical("verify", input=choice_id, check=_as_expr(check), tau=tau, min_confidence=min_confidence)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def order(score_id: str, cutpoints: list[float]) -> Categorical:
|
|
163
|
+
return Categorical("order", input=score_id, cutpoints=list(cutpoints))
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
# ---------------------------------------------------------------- circuit
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@dataclass
|
|
170
|
+
class GateDef:
|
|
171
|
+
name: str
|
|
172
|
+
body: Expr | Categorical
|
|
173
|
+
on_uncertain_: str = "abstain"
|
|
174
|
+
default_: Any = None
|
|
175
|
+
band_: float = 0.1
|
|
176
|
+
|
|
177
|
+
def on_uncertain(self, policy: str, default: Any = None) -> GateDef:
|
|
178
|
+
self.on_uncertain_ = policy
|
|
179
|
+
self.default_ = default
|
|
180
|
+
return self
|
|
181
|
+
|
|
182
|
+
def band(self, width: float) -> GateDef:
|
|
183
|
+
self.band_ = width
|
|
184
|
+
return self
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass
|
|
188
|
+
class Circuit:
|
|
189
|
+
questions: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
190
|
+
gates: list[GateDef] = field(default_factory=list)
|
|
191
|
+
model: str = "s1-proto"
|
|
192
|
+
_server_gates: bool | None = field(default=None, repr=False, compare=False)
|
|
193
|
+
|
|
194
|
+
# questions ---------------------------------------------------------
|
|
195
|
+
def noul(self, qid: str, instructions: Any, true: str | None = None, false: str | None = None) -> Circuit:
|
|
196
|
+
q: dict[str, Any] = {"type": "noul", "instructions": instructions}
|
|
197
|
+
if true or false:
|
|
198
|
+
q["criteria"] = {"true": true, "false": false}
|
|
199
|
+
self.questions[qid] = q
|
|
200
|
+
return self
|
|
201
|
+
|
|
202
|
+
def choice(self, qid: str, instructions: Any, criteria: dict[str, Any]) -> Circuit:
|
|
203
|
+
self.questions[qid] = {"type": "choice", "instructions": instructions, "criteria": criteria}
|
|
204
|
+
return self
|
|
205
|
+
|
|
206
|
+
def score(self, qid: str, instructions: Any, levels: list[Any]) -> Circuit:
|
|
207
|
+
self.questions[qid] = {"type": "score", "instructions": instructions, "criteria": levels}
|
|
208
|
+
return self
|
|
209
|
+
|
|
210
|
+
# gates -------------------------------------------------------------
|
|
211
|
+
def gate(
|
|
212
|
+
self,
|
|
213
|
+
name: str,
|
|
214
|
+
body: Expr | Categorical | str,
|
|
215
|
+
*,
|
|
216
|
+
on_uncertain: str | None = None,
|
|
217
|
+
default: Any = None,
|
|
218
|
+
band: float | None = None,
|
|
219
|
+
) -> GateDef:
|
|
220
|
+
"""Add a gate. Policy can be given as keywords here or chained on
|
|
221
|
+
the returned GateDef (`.on_uncertain(...)`, `.band(...)`)."""
|
|
222
|
+
g = GateDef(name, _as_expr(body) if isinstance(body, str) else body)
|
|
223
|
+
if on_uncertain is not None:
|
|
224
|
+
g.on_uncertain(on_uncertain, default)
|
|
225
|
+
if band is not None:
|
|
226
|
+
g.band(band)
|
|
227
|
+
self.gates.append(g)
|
|
228
|
+
return g
|
|
229
|
+
|
|
230
|
+
# rendering ---------------------------------------------------------
|
|
231
|
+
def to_mermaid(self, results: dict[str, Any] | None = None, answers: dict[str, Any] | None = None, direction: str = "LR", plain: bool = False) -> str:
|
|
232
|
+
"""Schematic of the compiled circuit; see `render_mermaid`."""
|
|
233
|
+
return render_mermaid(self, results, answers, direction, plain)
|
|
234
|
+
|
|
235
|
+
# compile -----------------------------------------------------------
|
|
236
|
+
def compile(self) -> dict[str, dict[str, Any]]:
|
|
237
|
+
"""Lower the expression trees to the server's flat `gates` map.
|
|
238
|
+
Sub-expressions become auto-named helper gates (`_name_N`) so
|
|
239
|
+
every intermediate probability shows up in the trace."""
|
|
240
|
+
out: dict[str, dict[str, Any]] = {}
|
|
241
|
+
counter = itertools.count(1)
|
|
242
|
+
|
|
243
|
+
def helper(prefix: str, spec: dict[str, Any]) -> str:
|
|
244
|
+
name = f"_{prefix}_{next(counter)}"
|
|
245
|
+
out[name] = spec
|
|
246
|
+
return name
|
|
247
|
+
|
|
248
|
+
def ref_of(e: Expr, prefix: str) -> str:
|
|
249
|
+
"""A string reference the server accepts for this expression,
|
|
250
|
+
emitting helper gates where needed."""
|
|
251
|
+
if isinstance(e, Q | G):
|
|
252
|
+
return e.ref
|
|
253
|
+
if isinstance(e, Not):
|
|
254
|
+
return helper(prefix, {"op": "not", "input": ref_of(e.inner, prefix), "tau": 0.5, "band": 0.0})
|
|
255
|
+
if isinstance(e, And | Or):
|
|
256
|
+
return helper(prefix, {"op": "and" if isinstance(e, And) else "or", "inputs": [ref_of(p, prefix) for p in e.parts], "tau": 0.5, "band": 0.0})
|
|
257
|
+
if isinstance(e, Threshold):
|
|
258
|
+
return helper(prefix, {"op": "threshold", "input": ref_of(e.inner, prefix), "tau": e.tau, "band": 0.0})
|
|
259
|
+
raise TypeError(f"unsupported expression {e!r}")
|
|
260
|
+
|
|
261
|
+
for g in self.gates:
|
|
262
|
+
common = {"on_uncertain": g.on_uncertain_, "band": g.band_}
|
|
263
|
+
if g.on_uncertain_ == "default":
|
|
264
|
+
common["default"] = g.default_
|
|
265
|
+
b = g.body
|
|
266
|
+
if isinstance(b, Categorical):
|
|
267
|
+
spec: dict[str, Any] = {"op": b.op, "tau": b.tau, "min_confidence": b.min_confidence}
|
|
268
|
+
if b.input is not None:
|
|
269
|
+
spec["input"] = b.input
|
|
270
|
+
if b.inputs is not None:
|
|
271
|
+
spec["inputs"] = b.inputs
|
|
272
|
+
if b.check is not None:
|
|
273
|
+
spec["check"] = ref_of(b.check, g.name)
|
|
274
|
+
if b.cutpoints is not None:
|
|
275
|
+
spec["cutpoints"] = b.cutpoints
|
|
276
|
+
out[g.name] = {**spec, **common}
|
|
277
|
+
continue
|
|
278
|
+
# boolean expression: the top node becomes the named gate
|
|
279
|
+
tau = 0.5
|
|
280
|
+
e = b
|
|
281
|
+
if isinstance(e, Threshold):
|
|
282
|
+
tau, e = e.tau, e.inner
|
|
283
|
+
if isinstance(e, Q | G):
|
|
284
|
+
out[g.name] = {"op": "threshold", "input": e.ref, "tau": tau, **common}
|
|
285
|
+
elif isinstance(e, Not):
|
|
286
|
+
out[g.name] = {"op": "not", "input": ref_of(e.inner, g.name), "tau": tau, **common}
|
|
287
|
+
elif isinstance(e, And | Or):
|
|
288
|
+
out[g.name] = {"op": "and" if isinstance(e, And) else "or", "inputs": [ref_of(p, g.name) for p in e.parts], "tau": tau, **common}
|
|
289
|
+
else:
|
|
290
|
+
raise TypeError(f"unsupported gate body {b!r}")
|
|
291
|
+
return out
|
|
292
|
+
|
|
293
|
+
def request(self, state: Any) -> dict[str, Any]:
|
|
294
|
+
return {"state": state, "model": self.model, "questions": dict(self.questions), "gates": self.compile()}
|
|
295
|
+
|
|
296
|
+
def evaluate(self, answers: dict[str, Any]) -> dict[str, Any]:
|
|
297
|
+
"""Evaluate the compiled gates locally against answers already in hand."""
|
|
298
|
+
res = evaluate_gates({k: Gate.model_validate(v) for k, v in self.compile().items()}, answers)
|
|
299
|
+
return {k: v.model_dump() for k, v in res.items() if not k.startswith("_")}
|
|
300
|
+
|
|
301
|
+
def run(self, client: Any, state: Any, url: str = "/v1/systemone", headers: dict[str, str] | None = None, gates: str = "auto") -> dict[str, Any]:
|
|
302
|
+
"""POST through any client with `.post(url, json=..., headers=...)`
|
|
303
|
+
returning `.json()` (httpx, requests, FastAPI TestClient).
|
|
304
|
+
|
|
305
|
+
`gates="auto"` sends the compiled gates and lets the server
|
|
306
|
+
evaluate them if it can; a server that rejects the extra field
|
|
307
|
+
(TypeSafe today) or returns answers only gets a second request
|
|
308
|
+
without gates, and they are evaluated here. The outcome is cached
|
|
309
|
+
per Circuit so later calls make one request. `"server"` requires
|
|
310
|
+
server evaluation; `"client"` never sends gates."""
|
|
311
|
+
if gates not in ("auto", "server", "client"):
|
|
312
|
+
raise ValueError("gates must be 'auto', 'server', or 'client'")
|
|
313
|
+
body: dict[str, Any] | None = None
|
|
314
|
+
try_server = gates == "server" or (gates == "auto" and self._server_gates is not False)
|
|
315
|
+
if try_server:
|
|
316
|
+
r = client.post(url, json=self.request(state), headers=headers)
|
|
317
|
+
body = r.json()
|
|
318
|
+
ok = getattr(r, "status_code", 200) < 400 and "gates" in body
|
|
319
|
+
if ok:
|
|
320
|
+
self._server_gates = True
|
|
321
|
+
body["gates"] = {k: v for k, v in body["gates"].items() if not k.startswith("_")}
|
|
322
|
+
body["gates_evaluated_by"] = "server"
|
|
323
|
+
return body
|
|
324
|
+
if gates == "server":
|
|
325
|
+
raise RuntimeError(f"server did not evaluate gates: {str(body)[:200]}")
|
|
326
|
+
self._server_gates = False
|
|
327
|
+
req = {k: v for k, v in self.request(state).items() if k != "gates"}
|
|
328
|
+
r = client.post(url, json=req, headers=headers)
|
|
329
|
+
body = r.json()
|
|
330
|
+
if "answers" not in body:
|
|
331
|
+
raise RuntimeError(f"unexpected response: {str(body)[:200]}")
|
|
332
|
+
body["gates"] = self.evaluate(body["answers"])
|
|
333
|
+
body["gates_evaluated_by"] = "client"
|
|
334
|
+
return body
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# ---------------------------------------------------------------- rendering
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _short(s: str, n: int = 38) -> str:
|
|
341
|
+
s = " ".join(str(s).split())
|
|
342
|
+
return s if len(s) <= n else s[: n - 1] + "…"
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def render_mermaid(
|
|
346
|
+
circuit: Circuit, results: dict[str, Any] | None = None, answers: dict[str, Any] | None = None, direction: str = "LR", plain: bool = False
|
|
347
|
+
) -> str:
|
|
348
|
+
"""Render the compiled circuit as a schematic. `plain=True` omits the
|
|
349
|
+
init directive and inline HTML styling for stricter renderers (some
|
|
350
|
+
hosted Mermaid builds reject them); layout is the same.
|
|
351
|
+
|
|
352
|
+
Schematic: an input column of
|
|
353
|
+
questions, a logic column of gates, and a decisions column for the
|
|
354
|
+
gates nothing else consumes. Threshold and NOT helpers are folded
|
|
355
|
+
into edge labels rather than drawn as nodes. Only decisions are
|
|
356
|
+
coloured: green yes / grey no / amber abstained or escalated."""
|
|
357
|
+
compiled = circuit.compile()
|
|
358
|
+
|
|
359
|
+
# which gates feed other gates (their base name before ':')
|
|
360
|
+
consumers: dict[str, set[str]] = {g: set() for g in compiled}
|
|
361
|
+
for gid, spec in compiled.items():
|
|
362
|
+
refs = ([spec["input"]] if spec.get("input") else []) + list(spec.get("inputs", [])) + ([spec["check"]] if spec.get("check") else [])
|
|
363
|
+
for ref in refs:
|
|
364
|
+
base = ref.partition(":")[0]
|
|
365
|
+
if base in consumers:
|
|
366
|
+
consumers[base].add(gid)
|
|
367
|
+
|
|
368
|
+
# helpers that fold into an edge: threshold/not with one consumer
|
|
369
|
+
folded: dict[str, tuple[str, str]] = {} # helper -> (source ref, label)
|
|
370
|
+
for gid, spec in compiled.items():
|
|
371
|
+
if gid.startswith("_") and spec["op"] in ("threshold", "not") and len(consumers[gid]) == 1:
|
|
372
|
+
src_ref = spec["input"]
|
|
373
|
+
lab = "NOT" if spec["op"] == "not" else f"≥ {spec['tau']:g}"
|
|
374
|
+
# chain: a folded helper feeding a folded helper
|
|
375
|
+
while src_ref in folded:
|
|
376
|
+
inner_src, inner_lab = folded[src_ref]
|
|
377
|
+
lab = f"{inner_lab} · {lab}"
|
|
378
|
+
src_ref = inner_src
|
|
379
|
+
folded[gid] = (src_ref, lab)
|
|
380
|
+
|
|
381
|
+
def qnode(qid: str) -> str:
|
|
382
|
+
return f"q_{qid}"
|
|
383
|
+
|
|
384
|
+
def gnode(gid: str) -> str:
|
|
385
|
+
return "g_" + gid.replace("-", "_")
|
|
386
|
+
|
|
387
|
+
init = (
|
|
388
|
+
"%%{init: {'theme': 'base', 'flowchart': {'nodeSpacing': 40, 'rankSpacing': 120, 'curve': 'basis', 'useMaxWidth': false, 'htmlLabels': true}, "
|
|
389
|
+
"'themeVariables': {'fontFamily': 'IBM Plex Sans, system-ui, sans-serif', 'fontSize': '13px', 'lineColor': '#5F6B78'}}}%%"
|
|
390
|
+
)
|
|
391
|
+
L = [
|
|
392
|
+
*([] if plain else [init]),
|
|
393
|
+
f"flowchart {direction}",
|
|
394
|
+
" classDef q fill:#FFFFFF,stroke:#1264A3,stroke-width:1.5px,color:#1B1F24;",
|
|
395
|
+
" classDef logic fill:#F7F8F6,stroke:#5F6B78,stroke-width:1.5px,color:#1B1F24;",
|
|
396
|
+
" classDef yes fill:#DDF3E4,stroke:#2E7D4F,stroke-width:2.5px,color:#0F3D22;",
|
|
397
|
+
" classDef no fill:#EEF0F2,stroke:#98A2AD,stroke-width:2px,color:#3B4550;",
|
|
398
|
+
" classDef hold fill:#FFF1CC,stroke:#9A6B00,stroke-width:2.5px,color:#4A3300;",
|
|
399
|
+
" classDef col fill:none,stroke:#D9DEE3,stroke-dasharray:3 3,color:#5F6B78;",
|
|
400
|
+
]
|
|
401
|
+
|
|
402
|
+
# --- inputs
|
|
403
|
+
L.append(' subgraph IN["Inputs"]')
|
|
404
|
+
L.append(" direction TB")
|
|
405
|
+
for qid, q in circuit.questions.items():
|
|
406
|
+
kind = q["type"]
|
|
407
|
+
if answers and qid in answers:
|
|
408
|
+
a = answers[qid]
|
|
409
|
+
if kind == "noul":
|
|
410
|
+
val = f"yes {a['noul']:.0%}"
|
|
411
|
+
elif kind == "choice":
|
|
412
|
+
val = f"{a['choice']} {a['probabilities'][a['choice']]:.0%}"
|
|
413
|
+
else:
|
|
414
|
+
val = f"{a['score']:.1f} / {len(q['criteria']) - 1}"
|
|
415
|
+
label = f"<b>{qid}</b><br/>{val}"
|
|
416
|
+
else:
|
|
417
|
+
label = f"<b>{qid}</b><br/>{kind}" if plain else f"<b>{qid}</b><br/><span style='color:#5F6B78'>{kind}</span>"
|
|
418
|
+
L.append(f' {qnode(qid)}["{label}"]:::q')
|
|
419
|
+
L.append(" end")
|
|
420
|
+
L.append(" class IN col")
|
|
421
|
+
|
|
422
|
+
# --- logic and decisions
|
|
423
|
+
OP_LABEL = {"and": "AND", "or": "OR", "not": "NOT", "threshold": "≥", "argmax": "PICK", "majority": "VOTE", "verify": "VERIFY", "order": "BUCKET"}
|
|
424
|
+
SHAPES = {"and": ("{{", "}}"), "or": (">", "]"), "not": ("((", "))"), "threshold": ("{", "}")}
|
|
425
|
+
logic, decisions = [], []
|
|
426
|
+
for gid, spec in compiled.items():
|
|
427
|
+
if gid in folded:
|
|
428
|
+
continue
|
|
429
|
+
terminal = not gid.startswith("_") and not consumers[gid]
|
|
430
|
+
(decisions if terminal else logic).append(gid)
|
|
431
|
+
|
|
432
|
+
def edge_lines(gid: str, spec: dict[str, Any]) -> list[str]:
|
|
433
|
+
out = []
|
|
434
|
+
refs = ([spec["input"]] if spec.get("input") else []) + list(spec.get("inputs", [])) + ([spec["check"]] if spec.get("check") else [])
|
|
435
|
+
for ref in refs:
|
|
436
|
+
lab = "check" if spec.get("check") == ref else ""
|
|
437
|
+
base, _, opt = ref.partition(":")
|
|
438
|
+
if base in folded:
|
|
439
|
+
base, flab = folded[base]
|
|
440
|
+
base, _, opt = base.partition(":")
|
|
441
|
+
lab = flab + (" · " + lab if lab else "")
|
|
442
|
+
if opt:
|
|
443
|
+
q = circuit.questions.get(base)
|
|
444
|
+
shown = f"level {opt}" if (q and q["type"] == "score") else opt
|
|
445
|
+
lab = (f"{shown} " + lab).strip()
|
|
446
|
+
node = gnode(base) if base in compiled else qnode(base)
|
|
447
|
+
out.append(f" {node} --{'>' if not lab else f'>|{lab}|'} {gnode(gid)}")
|
|
448
|
+
return out
|
|
449
|
+
|
|
450
|
+
def node_line(gid: str, spec: dict[str, Any], terminal: bool) -> str:
|
|
451
|
+
op = spec["op"]
|
|
452
|
+
head = OP_LABEL[op]
|
|
453
|
+
if op == "order":
|
|
454
|
+
head += " " + " | ".join(f"{c:g}" for c in spec.get("cutpoints", []))
|
|
455
|
+
if op in ("argmax", "majority", "verify") and spec.get("min_confidence"):
|
|
456
|
+
head += f"<br/>conf ≥ {spec['min_confidence']:g}" if plain else f"<br/><span style='color:#5F6B78'>conf ≥ {spec['min_confidence']:g}</span>"
|
|
457
|
+
if op in ("and", "or") and not terminal and spec.get("tau", 0.5) != 0.5:
|
|
458
|
+
head += f" ≥ {spec['tau']:g}"
|
|
459
|
+
if terminal:
|
|
460
|
+
label = f"<b>{gid.replace('_', ' ').title()}</b><br/>{head}"
|
|
461
|
+
if op in ("and", "or", "threshold", "not"):
|
|
462
|
+
label += f" ≥ {spec.get('tau', 0.5):g}"
|
|
463
|
+
cls = "logic"
|
|
464
|
+
if results and gid in results:
|
|
465
|
+
r = results[gid]
|
|
466
|
+
val, pr, outc = r.get("value"), r.get("p"), r.get("outcome", "decided")
|
|
467
|
+
if outc != "decided":
|
|
468
|
+
verdict, cls = f"⚠ {outc.upper()}", "hold"
|
|
469
|
+
elif val is True:
|
|
470
|
+
verdict, cls = "✓ YES", "yes"
|
|
471
|
+
elif val is False:
|
|
472
|
+
verdict, cls = "✗ NO", "no"
|
|
473
|
+
else:
|
|
474
|
+
verdict, cls = f"✓ {val}", "yes"
|
|
475
|
+
label += f"<br/><b>{verdict}</b>" + (f" · {pr:.0%}" if pr is not None else "")
|
|
476
|
+
return f' {gnode(gid)}["{label}"]:::{cls}'
|
|
477
|
+
label = f"<b>{gid.replace('_', ' ')}</b><br/>{head}" if not gid.startswith("_") else head
|
|
478
|
+
if results and gid in results:
|
|
479
|
+
r = results[gid]
|
|
480
|
+
if r.get("outcome", "decided") != "decided":
|
|
481
|
+
label += f"<br/>⚠ {r['outcome']}"
|
|
482
|
+
elif op in ("argmax", "majority", "verify", "order"):
|
|
483
|
+
label += f"<br/>→ <b>{r.get('value')}</b>" + (f" · {r['p']:.0%}" if r.get("p") is not None else "")
|
|
484
|
+
elif r.get("p") is not None:
|
|
485
|
+
label += f"<br/>{r['p']:.0%}"
|
|
486
|
+
lo, hi = SHAPES.get(op, ("[", "]"))
|
|
487
|
+
return f' {gnode(gid)}{lo}"{label}"{hi}:::logic'
|
|
488
|
+
|
|
489
|
+
if logic:
|
|
490
|
+
L.append(' subgraph LOGIC["Logic"]')
|
|
491
|
+
L.append(" direction TB")
|
|
492
|
+
for gid in logic:
|
|
493
|
+
L.append(" " + node_line(gid, compiled[gid], False))
|
|
494
|
+
L.append(" end")
|
|
495
|
+
L.append(" class LOGIC col")
|
|
496
|
+
L.append(' subgraph OUT["Decisions"]')
|
|
497
|
+
L.append(" direction TB")
|
|
498
|
+
for gid in decisions:
|
|
499
|
+
L.append(" " + node_line(gid, compiled[gid], True))
|
|
500
|
+
L.append(" end")
|
|
501
|
+
L.append(" class OUT col")
|
|
502
|
+
for gid in logic + decisions:
|
|
503
|
+
L += edge_lines(gid, compiled[gid])
|
|
504
|
+
return "\n".join(L)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
to_mermaid = render_mermaid # module-level alias
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Decision circuits: deterministic gates over calibrated answers.
|
|
2
|
+
|
|
3
|
+
A circuit is the `questions` map of a normal request plus a `gates`
|
|
4
|
+
map. Each gate reads one or more answers (or earlier gates) and emits a
|
|
5
|
+
typed value with a probability and a trace. The model never sees the
|
|
6
|
+
gates; code evaluates them, so a circuit is versionable and testable
|
|
7
|
+
offline, and each item's path is auditable.
|
|
8
|
+
|
|
9
|
+
Gate ops (inputs may reference question ids or earlier gate ids):
|
|
10
|
+
|
|
11
|
+
threshold input: noul id, "choice_id:option", "score_id:level",
|
|
12
|
+
or "gate_id:value" for a categorical gate value: bool
|
|
13
|
+
p = P(input); passes if p >= tau
|
|
14
|
+
not input: noul or gate value: bool, p = 1 - p_in
|
|
15
|
+
and / or inputs: k nouls/gates value: bool
|
|
16
|
+
p = product / 1 - product(1-p) (independence assumption; reported as such)
|
|
17
|
+
majority inputs: k choice ids over the same option set value: option
|
|
18
|
+
votes by argmax; p = mean probability of the winner; margin reported
|
|
19
|
+
argmax input: choice id value: option or abstain
|
|
20
|
+
abstains when confidence < min_confidence
|
|
21
|
+
verify input: choice id, check: noul id value: option or escalate
|
|
22
|
+
the check asks "is <that answer> supported?"; escalates when
|
|
23
|
+
P(check) < tau or the choice is below min_confidence
|
|
24
|
+
order input: score id, cutpoints: [c1, c2, ...] value: bucket index (0..len)
|
|
25
|
+
bucket = number of cutpoints <= expected score
|
|
26
|
+
|
|
27
|
+
Every gate has `on_uncertain`: "abstain" | "escalate" | "default" (with
|
|
28
|
+
`default` value). A gate is uncertain when the probability it acts on
|
|
29
|
+
falls inside [tau - band, tau + band] (band defaults to 0.1), or when an
|
|
30
|
+
argmax/verify confidence check fails. Uncertainty is surfaced, never
|
|
31
|
+
silently resolved.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
from typing import Any, Literal
|
|
37
|
+
|
|
38
|
+
from pydantic import BaseModel, Field
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Gate(BaseModel):
|
|
42
|
+
op: Literal["threshold", "not", "and", "or", "majority", "argmax", "verify", "order"]
|
|
43
|
+
input: str | None = None
|
|
44
|
+
inputs: list[str] | None = None
|
|
45
|
+
check: str | None = None
|
|
46
|
+
tau: float = 0.5
|
|
47
|
+
band: float = 0.1
|
|
48
|
+
min_confidence: float = 0.0
|
|
49
|
+
cutpoints: list[float] | None = None
|
|
50
|
+
on_uncertain: Literal["abstain", "escalate", "default"] = "abstain"
|
|
51
|
+
default: Any = None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class GateResult(BaseModel):
|
|
55
|
+
value: Any
|
|
56
|
+
p: float | None = None
|
|
57
|
+
confidence: float | None = None
|
|
58
|
+
uncertain: bool = False
|
|
59
|
+
outcome: Literal["decided", "abstain", "escalate", "default"] = "decided"
|
|
60
|
+
trace: list[str] = Field(default_factory=list)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _noul_p(answers: dict[str, Any], results: dict[str, GateResult], ref: str) -> tuple[float, str, bool]:
|
|
64
|
+
"""(probability, trace, upstream_uncertain) behind a reference.
|
|
65
|
+
|
|
66
|
+
A reference is a noul id; 'choice_id:option'; 'score_id:level'; an
|
|
67
|
+
earlier boolean gate (contributes P(its condition), whatever its
|
|
68
|
+
value); or 'gate_id:value' for a categorical gate. If the upstream
|
|
69
|
+
gate was uncertain, the flag propagates so the consumer's
|
|
70
|
+
on_uncertain policy applies."""
|
|
71
|
+
if ref in results:
|
|
72
|
+
r = results[ref]
|
|
73
|
+
if r.p is None:
|
|
74
|
+
raise ValueError(f"gate {ref!r} carries no probability; use '{ref}:<value>'")
|
|
75
|
+
note = " (uncertain)" if r.uncertain else ""
|
|
76
|
+
return float(r.p), f"gate {ref} p={r.p:.2f}{note}", r.uncertain
|
|
77
|
+
if ":" in ref:
|
|
78
|
+
qid, opt = ref.split(":", 1)
|
|
79
|
+
if qid in results:
|
|
80
|
+
r = results[qid]
|
|
81
|
+
p = float(r.p if r.p is not None else 1.0)
|
|
82
|
+
match = str(r.value) == opt
|
|
83
|
+
pm = p if match else max(0.0, 1.0 - p)
|
|
84
|
+
note = " (uncertain)" if r.uncertain else ""
|
|
85
|
+
return pm, f"gate {qid}={r.value} -> P({opt})={pm:.2f}{note}", r.uncertain
|
|
86
|
+
a = answers[qid]
|
|
87
|
+
if a["type"] in ("choice", "score"):
|
|
88
|
+
p = float(a["probabilities"][opt])
|
|
89
|
+
return p, f"{qid}[{opt}] p={p:.2f}", False
|
|
90
|
+
raise ValueError(f"{qid!r} is not a choice or score")
|
|
91
|
+
a = answers[ref]
|
|
92
|
+
if a["type"] != "noul":
|
|
93
|
+
raise ValueError(f"{ref!r} is not a noul; use 'choice_id:option' for a choice option")
|
|
94
|
+
return float(a["noul"]), f"{ref} p={a['noul']:.2f}", False
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _settle(g: Gate, value: Any, p: float | None, uncertain: bool, trace: list[str], confidence: float | None = None) -> GateResult:
|
|
98
|
+
if not uncertain:
|
|
99
|
+
return GateResult(value=value, p=p, confidence=confidence, trace=trace)
|
|
100
|
+
if g.on_uncertain == "default":
|
|
101
|
+
trace.append(f"uncertain -> default {g.default!r}")
|
|
102
|
+
return GateResult(value=g.default, p=p, confidence=confidence, uncertain=True, outcome="default", trace=trace)
|
|
103
|
+
trace.append(f"uncertain -> {g.on_uncertain}")
|
|
104
|
+
return GateResult(value=None, p=p, confidence=confidence, uncertain=True, outcome=g.on_uncertain, trace=trace)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def evaluate_gates(gates: dict[str, Gate], answers: dict[str, Any]) -> dict[str, GateResult]:
|
|
108
|
+
"""Evaluate gates in dependency order (declaration order must respect
|
|
109
|
+
dependencies; a forward reference raises)."""
|
|
110
|
+
results: dict[str, GateResult] = {}
|
|
111
|
+
for gid, g in gates.items():
|
|
112
|
+
trace: list[str] = []
|
|
113
|
+
if g.op == "threshold":
|
|
114
|
+
p, t, unc = _noul_p(answers, results, g.input)
|
|
115
|
+
trace.append(t)
|
|
116
|
+
results[gid] = _settle(g, p >= g.tau, p, unc or abs(p - g.tau) < g.band, trace)
|
|
117
|
+
elif g.op == "not":
|
|
118
|
+
p, t, unc = _noul_p(answers, results, g.input)
|
|
119
|
+
trace += [t, f"not -> p={1 - p:.2f}"]
|
|
120
|
+
results[gid] = _settle(g, (1 - p) >= g.tau, 1 - p, unc or abs((1 - p) - g.tau) < g.band, trace)
|
|
121
|
+
elif g.op in ("and", "or"):
|
|
122
|
+
ps = []
|
|
123
|
+
unc = False
|
|
124
|
+
for ref in g.inputs or []:
|
|
125
|
+
p, t, u = _noul_p(answers, results, ref)
|
|
126
|
+
ps.append(p)
|
|
127
|
+
unc = unc or u
|
|
128
|
+
trace.append(t)
|
|
129
|
+
if g.op == "and":
|
|
130
|
+
p = 1.0
|
|
131
|
+
for x in ps:
|
|
132
|
+
p *= x
|
|
133
|
+
else:
|
|
134
|
+
q = 1.0
|
|
135
|
+
for x in ps:
|
|
136
|
+
q *= 1 - x
|
|
137
|
+
p = 1 - q
|
|
138
|
+
trace.append(f"{g.op} under independence -> p={p:.2f}")
|
|
139
|
+
results[gid] = _settle(g, p >= g.tau, p, unc or abs(p - g.tau) < g.band, trace)
|
|
140
|
+
elif g.op == "majority":
|
|
141
|
+
votes: dict[str, list[float]] = {}
|
|
142
|
+
for ref in g.inputs or []:
|
|
143
|
+
a = answers[ref]
|
|
144
|
+
if a["type"] != "choice":
|
|
145
|
+
raise ValueError(f"majority input {ref!r} must be a choice")
|
|
146
|
+
votes.setdefault(a["choice"], []).append(float(a["probabilities"][a["choice"]]))
|
|
147
|
+
trace.append(f"{ref} -> {a['choice']} ({a['probabilities'][a['choice']]:.2f})")
|
|
148
|
+
n = len(g.inputs or [])
|
|
149
|
+
winner, ps = max(votes.items(), key=lambda kv: (len(kv[1]), sum(kv[1])))
|
|
150
|
+
margin = len(ps) / n
|
|
151
|
+
p = sum(ps) / len(ps)
|
|
152
|
+
trace.append(f"majority {winner} {len(ps)}/{n}, mean p={p:.2f}")
|
|
153
|
+
results[gid] = _settle(g, winner, p, margin <= 0.5 or p < g.min_confidence, trace, confidence=margin)
|
|
154
|
+
elif g.op == "argmax":
|
|
155
|
+
a = answers[g.input]
|
|
156
|
+
if a["type"] != "choice":
|
|
157
|
+
raise ValueError(f"argmax input {g.input!r} must be a choice")
|
|
158
|
+
conf = float(a["confidence"])
|
|
159
|
+
p = float(a["probabilities"][a["choice"]])
|
|
160
|
+
trace.append(f"{g.input} -> {a['choice']} p={p:.2f} conf={conf:.2f} (min {g.min_confidence})")
|
|
161
|
+
results[gid] = _settle(g, a["choice"], p, conf < g.min_confidence, trace, confidence=conf)
|
|
162
|
+
elif g.op == "verify":
|
|
163
|
+
a = answers[g.input]
|
|
164
|
+
if a["type"] != "choice":
|
|
165
|
+
raise ValueError(f"verify input {g.input!r} must be a choice")
|
|
166
|
+
conf = float(a["confidence"])
|
|
167
|
+
p_check, t, u = _noul_p(answers, results, g.check)
|
|
168
|
+
trace += [f"{g.input} -> {a['choice']} conf={conf:.2f}", f"check {t} (tau {g.tau})"]
|
|
169
|
+
unc = u or conf < g.min_confidence or p_check < g.tau
|
|
170
|
+
results[gid] = _settle(g, a["choice"], p_check, unc, trace, confidence=conf)
|
|
171
|
+
elif g.op == "order":
|
|
172
|
+
a = answers[g.input]
|
|
173
|
+
if a["type"] != "score":
|
|
174
|
+
raise ValueError(f"order input {g.input!r} must be a score")
|
|
175
|
+
s = float(a["score"])
|
|
176
|
+
cuts = g.cutpoints or []
|
|
177
|
+
bucket = sum(1 for c in cuts if s >= c)
|
|
178
|
+
near = any(abs(s - c) < g.band for c in cuts)
|
|
179
|
+
trace.append(f"{g.input} score={s:.2f} cutpoints={cuts} -> bucket {bucket}" + (" (near a cutpoint)" if near else ""))
|
|
180
|
+
results[gid] = _settle(g, bucket, None, near, trace, confidence=float(a["confidence"]))
|
|
181
|
+
return results
|
|
File without changes
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from decision_circuits import Circuit, G, Q, argmax, majority, order, verify
|
|
4
|
+
|
|
5
|
+
ANSWERS = {
|
|
6
|
+
"pii": {"type": "noul", "noul": 0.92},
|
|
7
|
+
"business": {"type": "noul", "noul": 0.10},
|
|
8
|
+
"angry": {"type": "noul", "noul": 0.93},
|
|
9
|
+
"dept": {"type": "choice", "choice": "billing", "probabilities": {"billing": 0.7, "technical": 0.2, "other": 0.1}, "confidence": 0.55},
|
|
10
|
+
"dept2": {"type": "choice", "choice": "billing", "probabilities": {"billing": 0.6, "technical": 0.3, "other": 0.1}, "confidence": 0.4},
|
|
11
|
+
"supported": {"type": "noul", "noul": 0.85},
|
|
12
|
+
"urgency": {"type": "score", "score": 2.88, "legend": {}, "probabilities": {"0": 0.0, "1": 0.02, "2": 0.08, "3": 0.90}, "confidence": 0.73},
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_operators_compile_and_evaluate():
|
|
17
|
+
c = Circuit()
|
|
18
|
+
c.gate("redact", ((Q("pii") >= 0.7) & ~Q("business")) >= 0.6).on_uncertain("escalate")
|
|
19
|
+
c.gate("human", (Q("angry") | Q("urgency:3")) >= 0.6)
|
|
20
|
+
c.gate("route", argmax("dept", min_confidence=0.35))
|
|
21
|
+
c.gate("vote", majority("dept", "dept2"))
|
|
22
|
+
c.gate("checked", verify("dept", Q("supported"), tau=0.8)).on_uncertain("escalate")
|
|
23
|
+
c.gate("tier", order("urgency", [1.0, 2.0, 2.6]))
|
|
24
|
+
c.gate("bill_and_hot", (G("route:billing") & G("tier:3")) >= 0.5)
|
|
25
|
+
|
|
26
|
+
compiled = c.compile()
|
|
27
|
+
assert compiled["redact"]["op"] == "and" and compiled["redact"]["tau"] == 0.6 and compiled["redact"]["on_uncertain"] == "escalate"
|
|
28
|
+
# helpers exist for the threshold and the not
|
|
29
|
+
helper_ops = sorted(v["op"] for k, v in compiled.items() if k.startswith("_"))
|
|
30
|
+
assert helper_ops == ["not", "threshold"]
|
|
31
|
+
|
|
32
|
+
r = c.evaluate(ANSWERS)
|
|
33
|
+
assert set(r) == {"redact", "human", "route", "vote", "checked", "tier", "bill_and_hot"} # helpers hidden
|
|
34
|
+
assert r["redact"]["value"] is True and r["redact"]["p"] == pytest.approx(0.92 * 0.90, abs=1e-6)
|
|
35
|
+
assert r["human"]["value"] is True and r["human"]["p"] > 0.99
|
|
36
|
+
assert r["route"]["value"] == "billing"
|
|
37
|
+
assert r["vote"]["value"] == "billing"
|
|
38
|
+
assert r["checked"]["value"] == "billing" and r["checked"]["outcome"] == "decided"
|
|
39
|
+
assert r["tier"]["value"] == 3
|
|
40
|
+
assert r["bill_and_hot"]["value"] is True
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_chained_and_flattens():
|
|
44
|
+
c = Circuit()
|
|
45
|
+
c.gate("all3", (Q("pii") & Q("angry") & Q("supported")) >= 0.5)
|
|
46
|
+
spec = c.compile()["all3"]
|
|
47
|
+
assert spec["op"] == "and" and spec["inputs"] == ["pii", "angry", "supported"]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_bare_reference_thresholds_at_half():
|
|
51
|
+
c = Circuit()
|
|
52
|
+
c.gate("is_pii", Q("pii"))
|
|
53
|
+
assert c.compile()["is_pii"] == {"op": "threshold", "input": "pii", "tau": 0.5, "on_uncertain": "abstain", "band": 0.1}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_default_policy_carries_value():
|
|
57
|
+
c = Circuit()
|
|
58
|
+
c.gate("tier", order("urgency", [2.85])).on_uncertain("default", default=2).band(0.1)
|
|
59
|
+
r = c.evaluate(ANSWERS)["tier"]
|
|
60
|
+
assert r["outcome"] == "default" and r["value"] == 2
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class _FakeClient:
|
|
64
|
+
"""Looks like httpx/requests: .post(url, json=..., headers=...) -> .json()."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, answers, with_gates: bool):
|
|
67
|
+
self.answers = answers
|
|
68
|
+
self.with_gates = with_gates
|
|
69
|
+
self.calls = []
|
|
70
|
+
|
|
71
|
+
def post(self, url, json=None, headers=None):
|
|
72
|
+
self.calls.append((url, json, headers))
|
|
73
|
+
status = 200
|
|
74
|
+
body = {"answers": self.answers}
|
|
75
|
+
if "gates" in json:
|
|
76
|
+
if self.with_gates: # a server that evaluates gates itself (s1proto)
|
|
77
|
+
body["gates"] = {"_h_1": {"value": True}, "rush": {"value": True, "p": 0.9, "outcome": "decided"}}
|
|
78
|
+
else: # TypeSafe rejects unknown fields
|
|
79
|
+
status, body = 400, {"detail": {"error_type": "api_usage_error", "message": "Invalid request."}}
|
|
80
|
+
|
|
81
|
+
class R:
|
|
82
|
+
status_code = status
|
|
83
|
+
|
|
84
|
+
def json(self):
|
|
85
|
+
return body
|
|
86
|
+
|
|
87
|
+
return R()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _circuit():
|
|
91
|
+
c = Circuit()
|
|
92
|
+
c.noul("urgent", "Is this urgent?", true="Needs action today", false="Can wait")
|
|
93
|
+
c.choice("dept", "Which team?", {"billing": None, "technical": None, "other": None})
|
|
94
|
+
c.gate("rush", Q("urgent") >= 0.5)
|
|
95
|
+
c.gate("route", argmax("dept"))
|
|
96
|
+
return c
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_run_sends_gates_and_uses_server_evaluation_when_present():
|
|
100
|
+
client = _FakeClient({"urgent": {"type": "noul", "noul": 0.9}, "dept": ANSWERS["dept"]}, with_gates=True)
|
|
101
|
+
out = _circuit().run(client, "Card charged twice, please refund today.", headers={"Authorization": "Bearer k"})
|
|
102
|
+
url, body, headers = client.calls[0]
|
|
103
|
+
assert url == "/v1/systemone" and headers == {"Authorization": "Bearer k"}
|
|
104
|
+
assert set(body["questions"]) == {"urgent", "dept"} and set(body["gates"]) == {"rush", "route"}
|
|
105
|
+
assert set(out["gates"]) == {"rush"} # helper gates hidden; server's result kept as-is
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_run_evaluates_gates_locally_when_server_returns_answers_only():
|
|
109
|
+
client = _FakeClient({"urgent": {"type": "noul", "noul": 0.9}, "dept": ANSWERS["dept"]}, with_gates=False)
|
|
110
|
+
c = _circuit()
|
|
111
|
+
out = c.run(client, "Card charged twice.", url="https://api.typesafe.ai/v1/systemone", headers={"Authorization": "Bearer k"})
|
|
112
|
+
assert out["gates"]["rush"]["value"] is True and out["gates"]["route"]["value"] == "billing"
|
|
113
|
+
assert out["gates_evaluated_by"] == "client"
|
|
114
|
+
assert len(client.calls) == 2 and "gates" not in client.calls[1][1]
|
|
115
|
+
c.run(client, "Again.", url="https://api.typesafe.ai/v1/systemone")
|
|
116
|
+
assert len(client.calls) == 3 # remembered: one request the second time
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_indexing_and_keyword_policy_match_string_forms():
|
|
120
|
+
a = Circuit()
|
|
121
|
+
a.gate("hot", (Q("urgency:3") | G("route:billing")) >= 0.5).on_uncertain("escalate").band(0.05)
|
|
122
|
+
b = Circuit()
|
|
123
|
+
b.gate("hot", (Q("urgency")[3] | G("route")["billing"]).at(0.5), on_uncertain="escalate", band=0.05)
|
|
124
|
+
assert a.compile() == b.compile()
|
|
125
|
+
with pytest.raises(KeyError):
|
|
126
|
+
Q("urgency:3")["x"]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_to_mermaid_is_a_method():
|
|
130
|
+
c = Circuit()
|
|
131
|
+
c.noul("pii", "PII?")
|
|
132
|
+
c.gate("redact", Q("pii") >= 0.7)
|
|
133
|
+
assert "redact" in c.to_mermaid(plain=True)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from decision_circuits.gates import Gate, evaluate_gates
|
|
4
|
+
|
|
5
|
+
ANSWERS = {
|
|
6
|
+
"pii": {"type": "noul", "noul": 0.92},
|
|
7
|
+
"business": {"type": "noul", "noul": 0.10},
|
|
8
|
+
"dept": {"type": "choice", "choice": "billing", "probabilities": {"billing": 0.7, "technical": 0.2, "sales": 0.1}, "confidence": 0.55},
|
|
9
|
+
"dept2": {"type": "choice", "choice": "billing", "probabilities": {"billing": 0.6, "technical": 0.3, "sales": 0.1}, "confidence": 0.4},
|
|
10
|
+
"dept3": {"type": "choice", "choice": "technical", "probabilities": {"billing": 0.4, "technical": 0.5, "sales": 0.1}, "confidence": 0.3},
|
|
11
|
+
"supported": {"type": "noul", "noul": 0.85},
|
|
12
|
+
"urgency": {
|
|
13
|
+
"type": "score",
|
|
14
|
+
"score": 2.4,
|
|
15
|
+
"legend": {"0": "low", "1": "med", "2": "high", "3": "critical"},
|
|
16
|
+
"probabilities": {"0": 0.0, "1": 0.1, "2": 0.4, "3": 0.5},
|
|
17
|
+
"confidence": 0.6,
|
|
18
|
+
},
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def g(**kw):
|
|
23
|
+
return Gate.model_validate(kw)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_threshold_and_not_and_and():
|
|
27
|
+
r = evaluate_gates(
|
|
28
|
+
{
|
|
29
|
+
"has_pii": g(op="threshold", input="pii", tau=0.8),
|
|
30
|
+
"private": g(op="not", input="business", tau=0.8, band=0.05),
|
|
31
|
+
"redact": g(op="and", inputs=["has_pii", "private"], tau=0.7, on_uncertain="escalate"),
|
|
32
|
+
},
|
|
33
|
+
ANSWERS,
|
|
34
|
+
)
|
|
35
|
+
assert r["has_pii"].value is True and r["has_pii"].p == pytest.approx(0.92)
|
|
36
|
+
assert r["private"].value is True and r["private"].p == pytest.approx(0.90)
|
|
37
|
+
assert r["redact"].value is True and r["redact"].p == pytest.approx(0.92 * 0.90)
|
|
38
|
+
assert "independence" in " ".join(r["redact"].trace)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_uncertain_band_escalates():
|
|
42
|
+
r = evaluate_gates({"x": g(op="threshold", input="pii", tau=0.9, band=0.05, on_uncertain="escalate")}, ANSWERS)
|
|
43
|
+
assert r["x"].outcome == "escalate" and r["x"].value is None and r["x"].uncertain
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_uncertain_default():
|
|
47
|
+
r = evaluate_gates({"x": g(op="threshold", input="pii", tau=0.9, band=0.05, on_uncertain="default", default=False)}, ANSWERS)
|
|
48
|
+
assert r["x"].outcome == "default" and r["x"].value is False
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_argmax_confidence_gate():
|
|
52
|
+
ok = evaluate_gates({"route": g(op="argmax", input="dept", min_confidence=0.5)}, ANSWERS)["route"]
|
|
53
|
+
assert ok.value == "billing" and ok.outcome == "decided"
|
|
54
|
+
ab = evaluate_gates({"route": g(op="argmax", input="dept", min_confidence=0.6)}, ANSWERS)["route"]
|
|
55
|
+
assert ab.outcome == "abstain" and ab.value is None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_majority_over_paraphrases():
|
|
59
|
+
r = evaluate_gates({"vote": g(op="majority", inputs=["dept", "dept2", "dept3"])}, ANSWERS)["vote"]
|
|
60
|
+
assert r.value == "billing" and r.confidence == pytest.approx(2 / 3) and r.outcome == "decided"
|
|
61
|
+
tie = evaluate_gates({"vote": g(op="majority", inputs=["dept", "dept3"])}, ANSWERS)["vote"]
|
|
62
|
+
assert tie.outcome == "abstain"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_verify_with_negative_checker():
|
|
66
|
+
ok = evaluate_gates({"v": g(op="verify", input="dept", check="supported", tau=0.8, on_uncertain="escalate")}, ANSWERS)["v"]
|
|
67
|
+
assert ok.value == "billing" and ok.outcome == "decided"
|
|
68
|
+
weak = dict(ANSWERS, supported={"type": "noul", "noul": 0.4})
|
|
69
|
+
esc = evaluate_gates({"v": g(op="verify", input="dept", check="supported", tau=0.8, on_uncertain="escalate")}, weak)["v"]
|
|
70
|
+
assert esc.outcome == "escalate"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_order_buckets_and_cutpoint_band():
|
|
74
|
+
r = evaluate_gates({"tier": g(op="order", input="urgency", cutpoints=[1.0, 2.0, 2.8])}, ANSWERS)["tier"]
|
|
75
|
+
assert r.value == 2 and r.outcome == "decided"
|
|
76
|
+
near = evaluate_gates({"tier": g(op="order", input="urgency", cutpoints=[2.45], band=0.1, on_uncertain="escalate")}, ANSWERS)["tier"]
|
|
77
|
+
assert near.outcome == "escalate"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_forward_reference_is_an_error():
|
|
81
|
+
with pytest.raises(KeyError):
|
|
82
|
+
evaluate_gates({"a": g(op="and", inputs=["b"]), "b": g(op="threshold", input="pii")}, ANSWERS)
|