decision-jef 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- decision_jef-0.1.0/LICENSE +21 -0
- decision_jef-0.1.0/PKG-INFO +209 -0
- decision_jef-0.1.0/README.md +178 -0
- decision_jef-0.1.0/decision_jef/__init__.py +22 -0
- decision_jef-0.1.0/decision_jef/infer.py +130 -0
- decision_jef-0.1.0/decision_jef/model.py +161 -0
- decision_jef-0.1.0/decision_jef/pack.py +157 -0
- decision_jef-0.1.0/decision_jef/py.typed +0 -0
- decision_jef-0.1.0/decision_jef/wire.py +141 -0
- decision_jef-0.1.0/decision_jef.egg-info/PKG-INFO +209 -0
- decision_jef-0.1.0/decision_jef.egg-info/SOURCES.txt +14 -0
- decision_jef-0.1.0/decision_jef.egg-info/dependency_links.txt +1 -0
- decision_jef-0.1.0/decision_jef.egg-info/requires.txt +6 -0
- decision_jef-0.1.0/decision_jef.egg-info/top_level.txt +1 -0
- decision_jef-0.1.0/pyproject.toml +49 -0
- decision_jef-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Decision-Jef contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: decision-jef
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Typed decisions with calibrated probabilities, in one forward pass
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://huggingface.co/BarraHome/Decision-Jef-0.1
|
|
7
|
+
Project-URL: Model, https://huggingface.co/BarraHome/Decision-Jef-0.1
|
|
8
|
+
Project-URL: Issues, https://huggingface.co/BarraHome/Decision-Jef-0.1/discussions
|
|
9
|
+
Keywords: typed-decisions,calibration,classification,routing,scoring,encoder,multilingual,nlp
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: torch>=2.4
|
|
26
|
+
Requires-Dist: transformers>=4.48
|
|
27
|
+
Requires-Dist: huggingface-hub>=0.26
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
license: mit
|
|
34
|
+
base_model: jhu-clsp/mmBERT-base
|
|
35
|
+
base_model_relation: finetune
|
|
36
|
+
pipeline_tag: text-classification
|
|
37
|
+
inference: false
|
|
38
|
+
language:
|
|
39
|
+
- multilingual
|
|
40
|
+
tags:
|
|
41
|
+
- system-one
|
|
42
|
+
- calibrated-decisions
|
|
43
|
+
- typed-decisions
|
|
44
|
+
- classification
|
|
45
|
+
- routing
|
|
46
|
+
- scoring
|
|
47
|
+
- guardrails
|
|
48
|
+
- moderation
|
|
49
|
+
- multilingual
|
|
50
|
+
- commercial-use
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
# Decision-Jef-0.1
|
|
54
|
+
|
|
55
|
+
Answer several runtime-defined questions about one state, in a single forward
|
|
56
|
+
pass, with calibrated probabilities over exactly the options you supply.
|
|
57
|
+
|
|
58
|
+
307M parameters. **12.12 ms for four decisions in one forward pass.**
|
|
59
|
+
|
|
60
|
+
The answer space is built from the request, so a value you did not offer is not
|
|
61
|
+
representable — not merely unlikely. There is no classification head over a
|
|
62
|
+
fixed label set.
|
|
63
|
+
|
|
64
|
+
## Results
|
|
65
|
+
|
|
66
|
+
Typed-decisions test set, 2,000 decisions: 600 `choice`, 600 `noul`, 800
|
|
67
|
+
`score`. Same set and same split as the published competitors.
|
|
68
|
+
|
|
69
|
+
| model | global | choice | noul | score |
|
|
70
|
+
| --- | --- | --- | --- | --- |
|
|
71
|
+
| Decision-1.0-Lex | **78.15** | **74.00** | 84.67 | **76.38** |
|
|
72
|
+
| **Decision-Jef-0.1** | **77.00** | 73.67 | 84.33 | 74.00 |
|
|
73
|
+
| Laya Typed Decisions | 76.60 | 73.33 | **85.67** | 72.25 |
|
|
74
|
+
|
|
75
|
+
Ahead of Laya by 0.40 global and 1.75 on `score`. Behind Lex by 1.15 global,
|
|
76
|
+
with `choice` and `noul` within a third of a point and the gap concentrated in
|
|
77
|
+
`score`.
|
|
78
|
+
|
|
79
|
+
| | value |
|
|
80
|
+
| --- | --- |
|
|
81
|
+
| NLL, choice / noul / score | 0.7200 / 0.4043 / 0.7316 |
|
|
82
|
+
| ECE, 10 bins, calibrated | **0.060** |
|
|
83
|
+
| ECE, 10 bins, uncalibrated | 0.138 |
|
|
84
|
+
| mean confidence, calibrated | 0.827 |
|
|
85
|
+
|
|
86
|
+
Temperature scaling per (type, cardinality) bucket ships with the model and is
|
|
87
|
+
applied by default, taking ECE from 0.138 to **0.060**. It never changes an
|
|
88
|
+
argmax, so accuracy is 77.00 either way. Pass `calibrated=False` to `decide()`
|
|
89
|
+
for the raw distribution.
|
|
90
|
+
|
|
91
|
+
Question isolation is exact: adding a question moves another question's logits
|
|
92
|
+
by at most 3e-07.
|
|
93
|
+
|
|
94
|
+
## Latency
|
|
95
|
+
|
|
96
|
+
NVIDIA H100 NVL, bf16, median of 30 calls after warm-up. End to end: packing,
|
|
97
|
+
encoder and readout.
|
|
98
|
+
|
|
99
|
+
| questions in one call | median | p95 |
|
|
100
|
+
| --- | --- | --- |
|
|
101
|
+
| 1 | 11.73 ms | 14.09 ms |
|
|
102
|
+
| 2 | 11.88 ms | 12.17 ms |
|
|
103
|
+
| **4** | **12.12 ms** | 12.29 ms |
|
|
104
|
+
|
|
105
|
+
**Going from one decision to four costs 0.39 ms.** The state is encoded once and
|
|
106
|
+
the question branches are masked apart, so a request carrying four questions is
|
|
107
|
+
not four requests. Throughput at batch 64 and 1,024 tokens is 2.66 ms per
|
|
108
|
+
decision.
|
|
109
|
+
|
|
110
|
+
## Usage
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
pip install decision-jef
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The weights are published separately from the package. Authenticate with
|
|
117
|
+
`hf auth login` if the model repository is not yet public, or point
|
|
118
|
+
`from_pretrained` at a local directory holding `model.pt`.
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from decision_jef import Decider, Question
|
|
122
|
+
|
|
123
|
+
d = Decider.from_pretrained("BarraHome/Decision-Jef-0.1")
|
|
124
|
+
|
|
125
|
+
state = """from: user@acme.com
|
|
126
|
+
subject: Duplicate charge on invoice #4411
|
|
127
|
+
body: We were billed twice for March. Please refund the duplicate today
|
|
128
|
+
or we will cancel our plan."""
|
|
129
|
+
|
|
130
|
+
answers = d.decide(state, {
|
|
131
|
+
"department": Question("choice", "Which department should handle this?", {
|
|
132
|
+
"billing": "invoices, payments, refunds",
|
|
133
|
+
"technical": "bugs, outages, system errors",
|
|
134
|
+
"sales": "pricing, new contracts",
|
|
135
|
+
"other": "everything else",
|
|
136
|
+
}),
|
|
137
|
+
"urgency": Question("score", "How urgent is this request?",
|
|
138
|
+
["not urgent", "soon", "critical or blocking"]),
|
|
139
|
+
"churn_risk": Question("noul", "Does the user threaten to leave?", {
|
|
140
|
+
"false": "The user makes no threat to stop using the service.",
|
|
141
|
+
"true": "The user threatens to cancel, churn or leave.",
|
|
142
|
+
}),
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
for qid, a in answers.items():
|
|
146
|
+
print(qid, a.choice or a.p("true"), a.confidence, a.probabilities)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
```
|
|
150
|
+
department billing 0.94 {'billing': 0.9383, 'technical': 0.0147, 'sales': 0.0301, 'other': 0.017}
|
|
151
|
+
urgency 2 0.58 {'0': 0.156, '1': 0.2662, '2': 0.5778} score=1.42
|
|
152
|
+
churn_risk 0.87 0.87 {'false': 0.1292, 'true': 0.8708}
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
`d.to_wire(answers)` returns the same content as a JSON-ready response body.
|
|
156
|
+
|
|
157
|
+
### Give every yes/no outcome a description
|
|
158
|
+
|
|
159
|
+
This is a requirement, not a style note. The model scores the option text, so
|
|
160
|
+
a bare yes/no gives it nothing to compare. On the example above:
|
|
161
|
+
|
|
162
|
+
| question | p(true) with bare yes/no | p(true) with descriptions |
|
|
163
|
+
| --- | --- | --- |
|
|
164
|
+
| threatens to leave | 0.096 — wrong | **0.850** — right |
|
|
165
|
+
| requests a refund | 0.309 — wrong | **0.912** — right |
|
|
166
|
+
|
|
167
|
+
## The three question types
|
|
168
|
+
|
|
169
|
+
| type | `criteria` | answer |
|
|
170
|
+
| --- | --- | --- |
|
|
171
|
+
| `choice` | ordered map of key to description, up to 255 | `choice`, `probabilities` |
|
|
172
|
+
| `noul` | optional map of `false` and `true` to a description — supply it | `noul` probability |
|
|
173
|
+
| `score` | ordered array of 2 to 10 level descriptions | probability-weighted `score`, `legend` |
|
|
174
|
+
|
|
175
|
+
Option order is part of the question. The same options in a different order
|
|
176
|
+
are a different request, and the model is sensitive to it — include a
|
|
177
|
+
permutation check in any evaluation.
|
|
178
|
+
|
|
179
|
+
## How it works
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
<bos> state ... <eos> shared, encoded once
|
|
183
|
+
[Q] instructions ... [OPT] opt0 [OPT] opt1 ... [DEC] question 1
|
|
184
|
+
[Q] instructions ... [OPT] opt0 [OPT] opt1 ... [DEC] question 2
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
The query is read at `[DEC]`, after the whole option list, so the decision sees
|
|
188
|
+
every option. Keys come from each `[OPT]` in the same pass, so the options are
|
|
189
|
+
read together rather than scored in isolation. Each question attends to the
|
|
190
|
+
state and to itself only; the state attends to neither.
|
|
191
|
+
|
|
192
|
+
## Limitations
|
|
193
|
+
|
|
194
|
+
- **ECE is 0.060 only with the shipped temperatures applied**, 0.138 without.
|
|
195
|
+
They are on by default; do not disable them unless you are recalibrating.
|
|
196
|
+
- **`score` is the weakest type** at 74.00, 2.38 behind Lex.
|
|
197
|
+
- Trained and measured on **English** typed decisions. The backbone is
|
|
198
|
+
multilingual and the tokenizer covers 256k tokens, but no non-English
|
|
199
|
+
benchmark has been run — treat multilingual use as untested.
|
|
200
|
+
- The `guardrails` and `moderation` tags reflect coverage of toxicity and
|
|
201
|
+
hate-speech decisions. **Neither capability has been benchmarked.**
|
|
202
|
+
- Sensitive to option order, as above.
|
|
203
|
+
- Long states are truncated to the window with the questions reserved first.
|
|
204
|
+
|
|
205
|
+
## License and provenance
|
|
206
|
+
|
|
207
|
+
MIT, following the `jhu-clsp/mmBERT-base` backbone. No weights, gradients or
|
|
208
|
+
private data from any third-party decision service are used or claimed; the
|
|
209
|
+
design follows publicly documented API behaviour and public benchmark splits.
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
---
|
|
2
|
+
license: mit
|
|
3
|
+
base_model: jhu-clsp/mmBERT-base
|
|
4
|
+
base_model_relation: finetune
|
|
5
|
+
pipeline_tag: text-classification
|
|
6
|
+
inference: false
|
|
7
|
+
language:
|
|
8
|
+
- multilingual
|
|
9
|
+
tags:
|
|
10
|
+
- system-one
|
|
11
|
+
- calibrated-decisions
|
|
12
|
+
- typed-decisions
|
|
13
|
+
- classification
|
|
14
|
+
- routing
|
|
15
|
+
- scoring
|
|
16
|
+
- guardrails
|
|
17
|
+
- moderation
|
|
18
|
+
- multilingual
|
|
19
|
+
- commercial-use
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
# Decision-Jef-0.1
|
|
23
|
+
|
|
24
|
+
Answer several runtime-defined questions about one state, in a single forward
|
|
25
|
+
pass, with calibrated probabilities over exactly the options you supply.
|
|
26
|
+
|
|
27
|
+
307M parameters. **12.12 ms for four decisions in one forward pass.**
|
|
28
|
+
|
|
29
|
+
The answer space is built from the request, so a value you did not offer is not
|
|
30
|
+
representable — not merely unlikely. There is no classification head over a
|
|
31
|
+
fixed label set.
|
|
32
|
+
|
|
33
|
+
## Results
|
|
34
|
+
|
|
35
|
+
Typed-decisions test set, 2,000 decisions: 600 `choice`, 600 `noul`, 800
|
|
36
|
+
`score`. Same set and same split as the published competitors.
|
|
37
|
+
|
|
38
|
+
| model | global | choice | noul | score |
|
|
39
|
+
| --- | --- | --- | --- | --- |
|
|
40
|
+
| Decision-1.0-Lex | **78.15** | **74.00** | 84.67 | **76.38** |
|
|
41
|
+
| **Decision-Jef-0.1** | **77.00** | 73.67 | 84.33 | 74.00 |
|
|
42
|
+
| Laya Typed Decisions | 76.60 | 73.33 | **85.67** | 72.25 |
|
|
43
|
+
|
|
44
|
+
Ahead of Laya by 0.40 global and 1.75 on `score`. Behind Lex by 1.15 global,
|
|
45
|
+
with `choice` and `noul` within a third of a point and the gap concentrated in
|
|
46
|
+
`score`.
|
|
47
|
+
|
|
48
|
+
| | value |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
| NLL, choice / noul / score | 0.7200 / 0.4043 / 0.7316 |
|
|
51
|
+
| ECE, 10 bins, calibrated | **0.060** |
|
|
52
|
+
| ECE, 10 bins, uncalibrated | 0.138 |
|
|
53
|
+
| mean confidence, calibrated | 0.827 |
|
|
54
|
+
|
|
55
|
+
Temperature scaling per (type, cardinality) bucket ships with the model and is
|
|
56
|
+
applied by default, taking ECE from 0.138 to **0.060**. It never changes an
|
|
57
|
+
argmax, so accuracy is 77.00 either way. Pass `calibrated=False` to `decide()`
|
|
58
|
+
for the raw distribution.
|
|
59
|
+
|
|
60
|
+
Question isolation is exact: adding a question moves another question's logits
|
|
61
|
+
by at most 3e-07.
|
|
62
|
+
|
|
63
|
+
## Latency
|
|
64
|
+
|
|
65
|
+
NVIDIA H100 NVL, bf16, median of 30 calls after warm-up. End to end: packing,
|
|
66
|
+
encoder and readout.
|
|
67
|
+
|
|
68
|
+
| questions in one call | median | p95 |
|
|
69
|
+
| --- | --- | --- |
|
|
70
|
+
| 1 | 11.73 ms | 14.09 ms |
|
|
71
|
+
| 2 | 11.88 ms | 12.17 ms |
|
|
72
|
+
| **4** | **12.12 ms** | 12.29 ms |
|
|
73
|
+
|
|
74
|
+
**Going from one decision to four costs 0.39 ms.** The state is encoded once and
|
|
75
|
+
the question branches are masked apart, so a request carrying four questions is
|
|
76
|
+
not four requests. Throughput at batch 64 and 1,024 tokens is 2.66 ms per
|
|
77
|
+
decision.
|
|
78
|
+
|
|
79
|
+
## Usage
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install decision-jef
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The weights are published separately from the package. Authenticate with
|
|
86
|
+
`hf auth login` if the model repository is not yet public, or point
|
|
87
|
+
`from_pretrained` at a local directory holding `model.pt`.
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from decision_jef import Decider, Question
|
|
91
|
+
|
|
92
|
+
d = Decider.from_pretrained("BarraHome/Decision-Jef-0.1")
|
|
93
|
+
|
|
94
|
+
state = """from: user@acme.com
|
|
95
|
+
subject: Duplicate charge on invoice #4411
|
|
96
|
+
body: We were billed twice for March. Please refund the duplicate today
|
|
97
|
+
or we will cancel our plan."""
|
|
98
|
+
|
|
99
|
+
answers = d.decide(state, {
|
|
100
|
+
"department": Question("choice", "Which department should handle this?", {
|
|
101
|
+
"billing": "invoices, payments, refunds",
|
|
102
|
+
"technical": "bugs, outages, system errors",
|
|
103
|
+
"sales": "pricing, new contracts",
|
|
104
|
+
"other": "everything else",
|
|
105
|
+
}),
|
|
106
|
+
"urgency": Question("score", "How urgent is this request?",
|
|
107
|
+
["not urgent", "soon", "critical or blocking"]),
|
|
108
|
+
"churn_risk": Question("noul", "Does the user threaten to leave?", {
|
|
109
|
+
"false": "The user makes no threat to stop using the service.",
|
|
110
|
+
"true": "The user threatens to cancel, churn or leave.",
|
|
111
|
+
}),
|
|
112
|
+
})
|
|
113
|
+
|
|
114
|
+
for qid, a in answers.items():
|
|
115
|
+
print(qid, a.choice or a.p("true"), a.confidence, a.probabilities)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
department billing 0.94 {'billing': 0.9383, 'technical': 0.0147, 'sales': 0.0301, 'other': 0.017}
|
|
120
|
+
urgency 2 0.58 {'0': 0.156, '1': 0.2662, '2': 0.5778} score=1.42
|
|
121
|
+
churn_risk 0.87 0.87 {'false': 0.1292, 'true': 0.8708}
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`d.to_wire(answers)` returns the same content as a JSON-ready response body.
|
|
125
|
+
|
|
126
|
+
### Give every yes/no outcome a description
|
|
127
|
+
|
|
128
|
+
This is a requirement, not a style note. The model scores the option text, so
|
|
129
|
+
a bare yes/no gives it nothing to compare. On the example above:
|
|
130
|
+
|
|
131
|
+
| question | p(true) with bare yes/no | p(true) with descriptions |
|
|
132
|
+
| --- | --- | --- |
|
|
133
|
+
| threatens to leave | 0.096 — wrong | **0.850** — right |
|
|
134
|
+
| requests a refund | 0.309 — wrong | **0.912** — right |
|
|
135
|
+
|
|
136
|
+
## The three question types
|
|
137
|
+
|
|
138
|
+
| type | `criteria` | answer |
|
|
139
|
+
| --- | --- | --- |
|
|
140
|
+
| `choice` | ordered map of key to description, up to 255 | `choice`, `probabilities` |
|
|
141
|
+
| `noul` | optional map of `false` and `true` to a description — supply it | `noul` probability |
|
|
142
|
+
| `score` | ordered array of 2 to 10 level descriptions | probability-weighted `score`, `legend` |
|
|
143
|
+
|
|
144
|
+
Option order is part of the question. The same options in a different order
|
|
145
|
+
are a different request, and the model is sensitive to it — include a
|
|
146
|
+
permutation check in any evaluation.
|
|
147
|
+
|
|
148
|
+
## How it works
|
|
149
|
+
|
|
150
|
+
```
|
|
151
|
+
<bos> state ... <eos> shared, encoded once
|
|
152
|
+
[Q] instructions ... [OPT] opt0 [OPT] opt1 ... [DEC] question 1
|
|
153
|
+
[Q] instructions ... [OPT] opt0 [OPT] opt1 ... [DEC] question 2
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
The query is read at `[DEC]`, after the whole option list, so the decision sees
|
|
157
|
+
every option. Keys come from each `[OPT]` in the same pass, so the options are
|
|
158
|
+
read together rather than scored in isolation. Each question attends to the
|
|
159
|
+
state and to itself only; the state attends to neither.
|
|
160
|
+
|
|
161
|
+
## Limitations
|
|
162
|
+
|
|
163
|
+
- **ECE is 0.060 only with the shipped temperatures applied**, 0.138 without.
|
|
164
|
+
They are on by default; do not disable them unless you are recalibrating.
|
|
165
|
+
- **`score` is the weakest type** at 74.00, 2.38 behind Lex.
|
|
166
|
+
- Trained and measured on **English** typed decisions. The backbone is
|
|
167
|
+
multilingual and the tokenizer covers 256k tokens, but no non-English
|
|
168
|
+
benchmark has been run — treat multilingual use as untested.
|
|
169
|
+
- The `guardrails` and `moderation` tags reflect coverage of toxicity and
|
|
170
|
+
hate-speech decisions. **Neither capability has been benchmarked.**
|
|
171
|
+
- Sensitive to option order, as above.
|
|
172
|
+
- Long states are truncated to the window with the questions reserved first.
|
|
173
|
+
|
|
174
|
+
## License and provenance
|
|
175
|
+
|
|
176
|
+
MIT, following the `jhu-clsp/mmBERT-base` backbone. No weights, gradients or
|
|
177
|
+
private data from any third-party decision service are used or claimed; the
|
|
178
|
+
design follows publicly documented API behaviour and public benchmark splits.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Decision-Jef: typed decisions with calibrated probabilities.
|
|
2
|
+
|
|
3
|
+
from decision_jef import Decider, Question
|
|
4
|
+
|
|
5
|
+
d = Decider.from_pretrained("BarraHome/Decision-Jef-0.1")
|
|
6
|
+
out = d.decide(
|
|
7
|
+
state="Payouts have failed three times. The bank says everything is fine.",
|
|
8
|
+
questions={
|
|
9
|
+
"queue": Question("choice", "Which team should handle this?",
|
|
10
|
+
{"payments": "Payout failures",
|
|
11
|
+
"account": "Login and account access",
|
|
12
|
+
"other": "Something else"}),
|
|
13
|
+
"escalate": Question("noul", "Does this need urgent human attention?"),
|
|
14
|
+
"mood": Question("score", "How frustrated is the customer?",
|
|
15
|
+
["Calm", "Frustrated", "Very angry"]),
|
|
16
|
+
},
|
|
17
|
+
)
|
|
18
|
+
"""
|
|
19
|
+
from decision_jef.infer import Decider
|
|
20
|
+
from decision_jef.wire import Answer, Question, Request
|
|
21
|
+
|
|
22
|
+
__all__ = ["Decider", "Question", "Request", "Answer"]
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Inference API.
|
|
2
|
+
|
|
3
|
+
The answer space is built from the request, so the model cannot return a value
|
|
4
|
+
you did not offer. Probabilities are temperature-scaled per (type, cardinality)
|
|
5
|
+
bucket, with the temperatures fitted on a held-out calibration split.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from typing import Dict, Optional, Sequence
|
|
13
|
+
|
|
14
|
+
import torch
|
|
15
|
+
|
|
16
|
+
from decision_jef.model import DecisionJef
|
|
17
|
+
from decision_jef.pack import pack
|
|
18
|
+
from decision_jef.wire import Answer, Question, Request
|
|
19
|
+
|
|
20
|
+
# Two decimal places is what a reported probability is worth here, so a
|
|
21
|
+
# floor well below that costs no information and avoids claiming certainty.
|
|
22
|
+
PROB_FLOOR = 1e-4
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Decider:
|
|
26
|
+
def __init__(self, model: DecisionJef, tokenizer, temperatures=None,
|
|
27
|
+
max_len: int = 1024):
|
|
28
|
+
self.model = model.eval()
|
|
29
|
+
self.tok = tokenizer
|
|
30
|
+
self.temperatures = temperatures or {}
|
|
31
|
+
self.max_len = max_len
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def from_pretrained(cls, repo_or_path: str, device: str = "auto",
|
|
35
|
+
max_len: int = 1024) -> "Decider":
|
|
36
|
+
from huggingface_hub import hf_hub_download
|
|
37
|
+
from transformers import AutoTokenizer
|
|
38
|
+
|
|
39
|
+
def grab(name):
|
|
40
|
+
if os.path.isdir(repo_or_path):
|
|
41
|
+
p = os.path.join(repo_or_path, name)
|
|
42
|
+
return p if os.path.exists(p) else None
|
|
43
|
+
try:
|
|
44
|
+
return hf_hub_download(repo_or_path, name)
|
|
45
|
+
except Exception:
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
weights = grab("model.pt")
|
|
49
|
+
if weights is None:
|
|
50
|
+
raise FileNotFoundError(
|
|
51
|
+
f"No model.pt in {repo_or_path!r}.\n"
|
|
52
|
+
"The weights are published separately from this package. If the "
|
|
53
|
+
"repository is private or gated, authenticate first with "
|
|
54
|
+
"`hf auth login`, or pass a local directory containing model.pt.\n"
|
|
55
|
+
"See https://huggingface.co/BarraHome/Decision-Jef-0.1")
|
|
56
|
+
if device == "auto":
|
|
57
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
58
|
+
dev = torch.device(device)
|
|
59
|
+
ck = torch.load(weights, map_location=dev, weights_only=False)
|
|
60
|
+
cfg = dict(ck["config"])
|
|
61
|
+
# Read any unrecorded geometry flag off the weights rather than
|
|
62
|
+
# trusting a default that may have changed.
|
|
63
|
+
if "noul_head" not in cfg:
|
|
64
|
+
cfg["noul_head"] = any(k.startswith("noul_head.") for k in ck["model"])
|
|
65
|
+
model = DecisionJef(**cfg).to(dev)
|
|
66
|
+
model.load_state_dict(ck["model"])
|
|
67
|
+
tok = AutoTokenizer.from_pretrained(cfg["model_name"])
|
|
68
|
+
temps = ck.get("temperatures") or {}
|
|
69
|
+
tf = grab("temperatures.json")
|
|
70
|
+
if not temps and tf:
|
|
71
|
+
with open(tf) as fh:
|
|
72
|
+
temps = json.load(fh)
|
|
73
|
+
return cls(model, tok, temps, max_len)
|
|
74
|
+
|
|
75
|
+
def _temp(self, kind: str, n: int) -> float:
|
|
76
|
+
from decision_jef.model import temperature_bucket
|
|
77
|
+
b = temperature_bucket(kind, n)
|
|
78
|
+
return float(self.temperatures.get(b, self.temperatures.get("_global", 1.0)))
|
|
79
|
+
|
|
80
|
+
@torch.no_grad()
|
|
81
|
+
def decide(self, state: str, questions: Dict[str, Question],
|
|
82
|
+
calibrated: bool = True) -> Dict[str, Answer]:
|
|
83
|
+
"""One forward pass for every question. Questions cannot see each
|
|
84
|
+
other, so adding one never changes another's answer."""
|
|
85
|
+
req = Request(state, questions)
|
|
86
|
+
dev = next(self.model.parameters()).device
|
|
87
|
+
batch = pack([req], self.tok, self.max_len).to(dev)
|
|
88
|
+
logits = self.model(batch).float()
|
|
89
|
+
out: Dict[str, Answer] = {}
|
|
90
|
+
counts = batch.opt_mask.sum(-1).tolist()
|
|
91
|
+
for i, (qid, kind, keys) in enumerate(
|
|
92
|
+
zip(batch.qids, batch.kinds, batch.keys)):
|
|
93
|
+
row = logits[i]
|
|
94
|
+
if calibrated:
|
|
95
|
+
row = row / self._temp(kind, int(counts[i]))
|
|
96
|
+
p = torch.softmax(row[: len(keys)], -1).cpu().tolist()
|
|
97
|
+
# Never report certainty. Sharpening temperatures (around 0.34
|
|
98
|
+
# here) can drive a confident logit gap to a rounded 1.0, and a
|
|
99
|
+
# probability of exactly 1 claims the answer cannot be wrong.
|
|
100
|
+
# Clamp and renormalise so the distribution still sums to 1.
|
|
101
|
+
p = [min(max(v, PROB_FLOOR), 1.0 - PROB_FLOOR) for v in p]
|
|
102
|
+
total = sum(p)
|
|
103
|
+
p = [v / total for v in p]
|
|
104
|
+
d = {k: round(float(v), 4) for k, v in zip(keys, p)}
|
|
105
|
+
best = max(d, key=d.get)
|
|
106
|
+
out[qid] = Answer(
|
|
107
|
+
kind=kind, probabilities=d,
|
|
108
|
+
choice=best if kind != "noul" else None,
|
|
109
|
+
score=round(sum(j * v for j, v in enumerate(p)), 4)
|
|
110
|
+
if kind == "score" else None,
|
|
111
|
+
legend={str(j): t for j, t in enumerate(questions[qid].descriptions)}
|
|
112
|
+
if kind == "score" else None,
|
|
113
|
+
confidence=round(max(p), 4))
|
|
114
|
+
return out
|
|
115
|
+
|
|
116
|
+
def to_wire(self, answers: Dict[str, Answer]) -> dict:
|
|
117
|
+
"""The response body, ready to serialise."""
|
|
118
|
+
body = {"model": "Decision-Jef-0.1", "answers": {}}
|
|
119
|
+
for qid, a in answers.items():
|
|
120
|
+
d = {"type": a.kind, "probabilities": a.probabilities,
|
|
121
|
+
"confidence": a.confidence}
|
|
122
|
+
if a.kind == "noul":
|
|
123
|
+
d["noul"] = a.probabilities.get("true", 0.0)
|
|
124
|
+
else:
|
|
125
|
+
d["choice"] = a.choice
|
|
126
|
+
if a.kind == "score":
|
|
127
|
+
d["score"] = a.score
|
|
128
|
+
d["legend"] = a.legend
|
|
129
|
+
body["answers"][qid] = d
|
|
130
|
+
return body
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Decision-Jef.
|
|
2
|
+
|
|
3
|
+
h = encoder(state + question blocks)
|
|
4
|
+
query = q_proj(h[DEC]) one per question
|
|
5
|
+
keys = o_proj(h[OPT_i]) one per option, in order
|
|
6
|
+
logits = <query, key_i> * scale
|
|
7
|
+
|
|
8
|
+
The query is read after the whole option list, so the decision sees every
|
|
9
|
+
option. The keys come from the same pass, so the options are read together
|
|
10
|
+
rather than scored in isolation.
|
|
11
|
+
|
|
12
|
+
The answer space is exactly the options supplied at call time. There is no
|
|
13
|
+
classification head over a fixed label set, so a value you did not offer is
|
|
14
|
+
not representable.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from typing import Dict, Optional
|
|
20
|
+
|
|
21
|
+
import torch
|
|
22
|
+
import torch.nn as nn
|
|
23
|
+
import torch.nn.functional as F
|
|
24
|
+
from transformers import AutoConfig, AutoModel
|
|
25
|
+
from transformers.models.modernbert.modeling_modernbert import (
|
|
26
|
+
create_bidirectional_mask,
|
|
27
|
+
create_bidirectional_sliding_window_mask,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
from decision_jef.pack import Packed, segment_attention_mask
|
|
31
|
+
|
|
32
|
+
KINDS = ("choice", "noul", "score")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class DecisionJef(nn.Module):
|
|
36
|
+
def __init__(self, model_name: str = "jhu-clsp/mmBERT-base",
|
|
37
|
+
proj_dim: int = 256, dropout: float = 0.0,
|
|
38
|
+
trainable_layers: int = -1, isolate_questions: bool = True,
|
|
39
|
+
noul_head: bool = True):
|
|
40
|
+
super().__init__()
|
|
41
|
+
cfg = AutoConfig.from_pretrained(model_name)
|
|
42
|
+
cfg.attention_dropout = dropout
|
|
43
|
+
self.encoder = AutoModel.from_pretrained(model_name, config=cfg)
|
|
44
|
+
self.config = self.encoder.config
|
|
45
|
+
d = self.config.hidden_size
|
|
46
|
+
self.isolate_questions = isolate_questions
|
|
47
|
+
|
|
48
|
+
# Separate projections for the question and the option side.
|
|
49
|
+
self.q_proj = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, proj_dim))
|
|
50
|
+
self.o_proj = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, proj_dim))
|
|
51
|
+
# Learned temperature on normalised vectors.
|
|
52
|
+
self.logit_scale = nn.Parameter(torch.tensor(2.996))
|
|
53
|
+
# A yes/no question's two options are restatements of its
|
|
54
|
+
# instructions, so its two logits are read straight off [DEC].
|
|
55
|
+
# The outcome space is still exactly two.
|
|
56
|
+
self.noul_head = nn.Sequential(
|
|
57
|
+
nn.LayerNorm(d), nn.Linear(d, d), nn.GELU(), nn.Linear(d, 2)
|
|
58
|
+
) if noul_head else None
|
|
59
|
+
self.model_name = model_name
|
|
60
|
+
self.proj_dim = proj_dim
|
|
61
|
+
self.use_noul_head = noul_head
|
|
62
|
+
self.set_trainable(trainable_layers)
|
|
63
|
+
|
|
64
|
+
def set_trainable(self, n: int):
|
|
65
|
+
"""n < 0 unfreezes the whole encoder; n >= 0 keeps the top n layers."""
|
|
66
|
+
if n < 0:
|
|
67
|
+
for p in self.encoder.parameters():
|
|
68
|
+
p.requires_grad_(True)
|
|
69
|
+
return
|
|
70
|
+
for p in self.encoder.parameters():
|
|
71
|
+
p.requires_grad_(False)
|
|
72
|
+
for layer in list(self.encoder.layers)[len(self.encoder.layers) - n:]:
|
|
73
|
+
for p in layer.parameters():
|
|
74
|
+
p.requires_grad_(True)
|
|
75
|
+
if hasattr(self.encoder, "final_norm"):
|
|
76
|
+
for p in self.encoder.final_norm.parameters():
|
|
77
|
+
p.requires_grad_(True)
|
|
78
|
+
|
|
79
|
+
def _masks(self, batch: Packed, hidden) -> Dict[str, torch.Tensor]:
|
|
80
|
+
"""The encoder's own masks, restricted to the per-question segments."""
|
|
81
|
+
kw = dict(config=self.config, inputs_embeds=hidden,
|
|
82
|
+
attention_mask=batch.attention_mask)
|
|
83
|
+
masks = {"full_attention": create_bidirectional_mask(**kw),
|
|
84
|
+
"sliding_attention": create_bidirectional_sliding_window_mask(**kw)}
|
|
85
|
+
if not self.isolate_questions:
|
|
86
|
+
return masks
|
|
87
|
+
|
|
88
|
+
allowed = segment_attention_mask(batch.segment_ids, batch.attention_mask)
|
|
89
|
+
out = {}
|
|
90
|
+
for name, m in masks.items():
|
|
91
|
+
if m is None:
|
|
92
|
+
# The constructors return None when a batch has no padding,
|
|
93
|
+
# so build the mask here.
|
|
94
|
+
base = allowed.clone()
|
|
95
|
+
if name == "sliding_attention":
|
|
96
|
+
base = base & self._window(batch)
|
|
97
|
+
out[name] = base
|
|
98
|
+
elif m.dtype == torch.bool:
|
|
99
|
+
out[name] = m & allowed
|
|
100
|
+
else:
|
|
101
|
+
# Finite large negative, not -inf: a fully masked row with
|
|
102
|
+
# -inf yields NaN.
|
|
103
|
+
out[name] = m.masked_fill(~allowed, -1e4)
|
|
104
|
+
return out
|
|
105
|
+
|
|
106
|
+
def _window(self, batch: Packed) -> torch.Tensor:
|
|
107
|
+
"""The local-attention band the sliding layers use, [1, 1, L, L] bool."""
|
|
108
|
+
L = batch.input_ids.shape[1]
|
|
109
|
+
half = self.config.local_attention // 2
|
|
110
|
+
idx = torch.arange(L, device=batch.input_ids.device)
|
|
111
|
+
band = (idx.unsqueeze(1) - idx.unsqueeze(0)).abs() <= half
|
|
112
|
+
return band.unsqueeze(0).unsqueeze(0)
|
|
113
|
+
|
|
114
|
+
def forward(self, batch: Packed, restrict: bool = True) -> torch.Tensor:
|
|
115
|
+
hidden = self.encoder.embeddings(input_ids=batch.input_ids)
|
|
116
|
+
h = self.encoder(input_ids=batch.input_ids,
|
|
117
|
+
attention_mask=self._masks(batch, hidden)).last_hidden_state
|
|
118
|
+
|
|
119
|
+
flat = batch.batch_idx
|
|
120
|
+
q = h[flat, batch.dec_pos] # [Nq, D]
|
|
121
|
+
qv = F.normalize(self.q_proj(q), dim=-1)
|
|
122
|
+
|
|
123
|
+
Nq, mo = batch.opt_pos.shape
|
|
124
|
+
rows = flat.unsqueeze(1).expand(Nq, mo)
|
|
125
|
+
o = h[rows.reshape(-1), batch.opt_pos.reshape(-1)] # [Nq*mo, D]
|
|
126
|
+
ov = F.normalize(self.o_proj(o), dim=-1).view(Nq, mo, -1)
|
|
127
|
+
|
|
128
|
+
logits = (qv.unsqueeze(1) * ov).sum(-1) * self.logit_scale.exp()
|
|
129
|
+
|
|
130
|
+
if self.noul_head is not None and batch.kinds:
|
|
131
|
+
is_noul = torch.tensor([k == "noul" for k in batch.kinds],
|
|
132
|
+
device=logits.device)
|
|
133
|
+
if bool(is_noul.any()):
|
|
134
|
+
direct = self.noul_head(q) # [Nq, 2]
|
|
135
|
+
pad = logits.shape[1] - 2
|
|
136
|
+
if pad > 0:
|
|
137
|
+
direct = torch.cat(
|
|
138
|
+
[direct, direct.new_full((direct.shape[0], pad), -1e4)], 1)
|
|
139
|
+
logits = torch.where(is_noul.unsqueeze(1), direct[:, : logits.shape[1]],
|
|
140
|
+
logits)
|
|
141
|
+
|
|
142
|
+
if restrict:
|
|
143
|
+
logits = logits.masked_fill(~batch.opt_mask, -1e4)
|
|
144
|
+
return logits
|
|
145
|
+
|
|
146
|
+
def trainable_parameters(self) -> int:
|
|
147
|
+
return sum(p.numel() for p in self.parameters() if p.requires_grad)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def temperature_bucket(kind: str, n_options: int) -> str:
|
|
151
|
+
"""Which temperature applies to a question of this type and size."""
|
|
152
|
+
if kind == "noul":
|
|
153
|
+
return "noul"
|
|
154
|
+
if kind == "score":
|
|
155
|
+
return f"score:{n_options}"
|
|
156
|
+
|
|
157
|
+
if n_options <= 3:
|
|
158
|
+
return "choice:2-3"
|
|
159
|
+
if n_options <= 6:
|
|
160
|
+
return "choice:4-6"
|
|
161
|
+
return "choice:7+"
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Turn a request into one input sequence.
|
|
2
|
+
|
|
3
|
+
Layout:
|
|
4
|
+
|
|
5
|
+
<bos> state ... <eos> segment 0, shared
|
|
6
|
+
[Q] instructions ... [OPT] opt0 [OPT] opt1 ... [DEC] segment 1
|
|
7
|
+
[Q] instructions ... [OPT] opt0 [OPT] opt1 ... [DEC] segment 2
|
|
8
|
+
|
|
9
|
+
The options of a question sit inline, in the order given. The [DEC] marker
|
|
10
|
+
closes each question's block. Every question attends to the state and to
|
|
11
|
+
itself only, and the state attends to neither, so adding a question never
|
|
12
|
+
changes another question's answer.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import List, Optional, Sequence, Tuple
|
|
19
|
+
|
|
20
|
+
import torch
|
|
21
|
+
|
|
22
|
+
from decision_jef.wire import Question, Request
|
|
23
|
+
|
|
24
|
+
OPT_MARKER = "<unused0>"
|
|
25
|
+
DEC_MARKER = "<unused1>"
|
|
26
|
+
Q_MARKER = "<unused2>"
|
|
27
|
+
|
|
28
|
+
# Reserve room for the questions first, then give the state whatever is left,
|
|
29
|
+
# so a large schema is never silently truncated away.
|
|
30
|
+
MIN_STATE_TOKENS = 16
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Packed:
|
|
35
|
+
input_ids: torch.Tensor # [B, L]
|
|
36
|
+
attention_mask: torch.Tensor # [B, L] 1 = real token
|
|
37
|
+
segment_ids: torch.Tensor # [B, L] 0 = state, i+1 = question i
|
|
38
|
+
dec_pos: torch.Tensor # [Nq] index of each question's [DEC]
|
|
39
|
+
opt_pos: torch.Tensor # [Nq, max_opts] index of each [OPT]
|
|
40
|
+
opt_mask: torch.Tensor # [Nq, max_opts] True = real option
|
|
41
|
+
batch_idx: torch.Tensor # [Nq] which row of the batch
|
|
42
|
+
kinds: List[str] # [Nq]
|
|
43
|
+
qids: List[str] # [Nq]
|
|
44
|
+
keys: List[List[str]] # [Nq][n_opts] answer keys, in order
|
|
45
|
+
dropped: int = 0 # questions that did not fit
|
|
46
|
+
|
|
47
|
+
def to(self, device) -> "Packed":
|
|
48
|
+
d = {k: (v.to(device) if torch.is_tensor(v) else v)
|
|
49
|
+
for k, v in self.__dict__.items()}
|
|
50
|
+
return Packed(**d)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _question_block(tok, q: Question, kind_prefix: bool = True) -> Tuple[List[int], List[int], int]:
|
|
54
|
+
"""Token ids for one question, plus the offsets of its [OPT] markers and
|
|
55
|
+
its [DEC] marker, relative to the block start."""
|
|
56
|
+
opt_id = tok.convert_tokens_to_ids(OPT_MARKER)
|
|
57
|
+
dec_id = tok.convert_tokens_to_ids(DEC_MARKER)
|
|
58
|
+
q_id = tok.convert_tokens_to_ids(Q_MARKER)
|
|
59
|
+
|
|
60
|
+
enc = lambda s: tok(s, add_special_tokens=False)["input_ids"]
|
|
61
|
+
# The question type is named in the text rather than carried by a separate
|
|
62
|
+
# type embedding.
|
|
63
|
+
head = f"{q.kind} question: {q.instructions}" if kind_prefix else q.instructions
|
|
64
|
+
ids = [q_id] + enc(head)
|
|
65
|
+
opts = []
|
|
66
|
+
for i, desc in enumerate(q.descriptions):
|
|
67
|
+
# Ordinal rank is written into the level text, so the levels carry
|
|
68
|
+
# their order.
|
|
69
|
+
text = f"level {i}: {desc}" if q.kind == "score" else desc
|
|
70
|
+
opts.append(len(ids))
|
|
71
|
+
ids.append(opt_id)
|
|
72
|
+
ids.extend(enc(text)[:48]) # option text cap
|
|
73
|
+
dec = len(ids)
|
|
74
|
+
ids.append(dec_id)
|
|
75
|
+
return ids, opts, dec
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def pack(requests: Sequence[Request], tok, max_len: int = 1024,
|
|
79
|
+
kinds: Optional[Sequence[str]] = None) -> Packed:
|
|
80
|
+
bos = tok.bos_token_id if tok.bos_token_id is not None else tok.cls_token_id
|
|
81
|
+
eos = tok.eos_token_id if tok.eos_token_id is not None else tok.sep_token_id
|
|
82
|
+
pad = tok.pad_token_id
|
|
83
|
+
enc = lambda s: tok(s, add_special_tokens=False)["input_ids"]
|
|
84
|
+
|
|
85
|
+
rows, seg_rows, per_q, dropped = [], [], [], 0
|
|
86
|
+
for b, req in enumerate(requests):
|
|
87
|
+
blocks = []
|
|
88
|
+
for qid, q in req.questions.items():
|
|
89
|
+
if len(q.keys) < 2:
|
|
90
|
+
dropped += 1
|
|
91
|
+
continue
|
|
92
|
+
blocks.append((qid, q) + _question_block(tok, q))
|
|
93
|
+
budget = max_len - 2 - sum(len(ids) for *_, ids, _, _ in
|
|
94
|
+
[(0, 0, b[2], b[3], b[4]) for b in blocks])
|
|
95
|
+
state = enc(req.state)[:max(MIN_STATE_TOKENS, budget)]
|
|
96
|
+
|
|
97
|
+
ids = [bos] + state + [eos]
|
|
98
|
+
seg = [0] * len(ids)
|
|
99
|
+
for qi, (qid, q, blk, opts, dec) in enumerate(blocks):
|
|
100
|
+
if len(ids) + len(blk) > max_len:
|
|
101
|
+
dropped += 1
|
|
102
|
+
continue
|
|
103
|
+
base = len(ids)
|
|
104
|
+
ids.extend(blk)
|
|
105
|
+
seg.extend([qi + 1] * len(blk))
|
|
106
|
+
per_q.append(dict(batch=b, qid=qid, kind=q.kind, keys=q.keys,
|
|
107
|
+
dec=base + dec, opts=[base + o for o in opts]))
|
|
108
|
+
rows.append(ids)
|
|
109
|
+
seg_rows.append(seg)
|
|
110
|
+
|
|
111
|
+
B, L = len(rows), max(len(r) for r in rows)
|
|
112
|
+
input_ids = torch.full((B, L), pad, dtype=torch.long)
|
|
113
|
+
attn = torch.zeros((B, L), dtype=torch.long)
|
|
114
|
+
segs = torch.zeros((B, L), dtype=torch.long)
|
|
115
|
+
for i, (r, s) in enumerate(zip(rows, seg_rows)):
|
|
116
|
+
input_ids[i, : len(r)] = torch.tensor(r)
|
|
117
|
+
attn[i, : len(r)] = 1
|
|
118
|
+
segs[i, : len(s)] = torch.tensor(s)
|
|
119
|
+
|
|
120
|
+
Nq = len(per_q)
|
|
121
|
+
mo = max((len(p["opts"]) for p in per_q), default=1)
|
|
122
|
+
dec_pos = torch.zeros(Nq, dtype=torch.long)
|
|
123
|
+
opt_pos = torch.zeros((Nq, mo), dtype=torch.long)
|
|
124
|
+
opt_mask = torch.zeros((Nq, mo), dtype=torch.bool)
|
|
125
|
+
batch_idx = torch.zeros(Nq, dtype=torch.long)
|
|
126
|
+
for i, p in enumerate(per_q):
|
|
127
|
+
dec_pos[i] = p["dec"]
|
|
128
|
+
batch_idx[i] = p["batch"]
|
|
129
|
+
for j, o in enumerate(p["opts"]):
|
|
130
|
+
opt_pos[i, j] = o
|
|
131
|
+
opt_mask[i, j] = True
|
|
132
|
+
|
|
133
|
+
return Packed(input_ids=input_ids, attention_mask=attn, segment_ids=segs,
|
|
134
|
+
dec_pos=dec_pos, opt_pos=opt_pos, opt_mask=opt_mask,
|
|
135
|
+
batch_idx=batch_idx,
|
|
136
|
+
kinds=[p["kind"] for p in per_q],
|
|
137
|
+
qids=[p["qid"] for p in per_q],
|
|
138
|
+
keys=[p["keys"] for p in per_q],
|
|
139
|
+
dropped=dropped)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def segment_attention_mask(segment_ids: torch.Tensor,
|
|
143
|
+
attention_mask: torch.Tensor) -> torch.Tensor:
|
|
144
|
+
"""[B, 1, L, L] boolean: True where attention is allowed.
|
|
145
|
+
|
|
146
|
+
A position may attend to the state (segment 0) and to its own segment.
|
|
147
|
+
The state may not attend to any question, which is what makes the state
|
|
148
|
+
encoding identical no matter which questions accompany it.
|
|
149
|
+
"""
|
|
150
|
+
sq = segment_ids.unsqueeze(2) # querying position
|
|
151
|
+
sk = segment_ids.unsqueeze(1) # key position
|
|
152
|
+
allowed = ((sk == 0) | (sq == sk)) & (sq != 0) | ((sq == 0) & (sk == 0))
|
|
153
|
+
allowed = allowed & attention_mask.bool().unsqueeze(1)
|
|
154
|
+
L = segment_ids.shape[-1]
|
|
155
|
+
eye = torch.arange(L, device=segment_ids.device)
|
|
156
|
+
allowed[:, eye, eye] = True # never mask a row entirely
|
|
157
|
+
return allowed.unsqueeze(1)
|
|
File without changes
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""The request and response types.
|
|
2
|
+
|
|
3
|
+
choice criteria is an ordered map of option key -> description, up to 255.
|
|
4
|
+
noul yes/no. The answer is one probability.
|
|
5
|
+
score criteria is an ordered array of 2 to 10 level descriptions. The
|
|
6
|
+
answer adds a legend and a probability-weighted score.
|
|
7
|
+
|
|
8
|
+
Every container preserves insertion order: option order is part of the
|
|
9
|
+
question, and nothing here sorts keys.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Dict, List, Optional, Sequence, Union
|
|
17
|
+
|
|
18
|
+
KINDS = ("choice", "noul", "score")
|
|
19
|
+
MAX_CHOICE_OPTIONS = 255 # 2**8 - 1
|
|
20
|
+
MAX_SCORE_LEVELS = 10
|
|
21
|
+
MIN_SCORE_LEVELS = 2
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Question:
|
|
26
|
+
kind: str
|
|
27
|
+
instructions: str
|
|
28
|
+
# choice: ordered {key: description}. score: ordered [description, ...].
|
|
29
|
+
# noul: empty.
|
|
30
|
+
criteria: Union[Dict[str, str], List[str], None] = None
|
|
31
|
+
|
|
32
|
+
def __post_init__(self):
|
|
33
|
+
if self.kind not in KINDS:
|
|
34
|
+
raise ValueError(f"unknown kind {self.kind!r}; expected one of {KINDS}")
|
|
35
|
+
if self.kind == "choice":
|
|
36
|
+
if not isinstance(self.criteria, dict) or len(self.criteria) < 2:
|
|
37
|
+
raise ValueError("choice needs an ordered map of at least 2 options")
|
|
38
|
+
if len(self.criteria) > MAX_CHOICE_OPTIONS:
|
|
39
|
+
raise ValueError(f"choice accepts at most {MAX_CHOICE_OPTIONS} options")
|
|
40
|
+
elif self.kind == "score":
|
|
41
|
+
if not isinstance(self.criteria, (list, tuple)):
|
|
42
|
+
raise ValueError("score needs an ordered array of level descriptions")
|
|
43
|
+
if not MIN_SCORE_LEVELS <= len(self.criteria) <= MAX_SCORE_LEVELS:
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"score needs {MIN_SCORE_LEVELS} to {MAX_SCORE_LEVELS} levels")
|
|
46
|
+
elif self.criteria is not None:
|
|
47
|
+
# A noul question may optionally carry descriptions for false and
|
|
48
|
+
# true; anything else is rejected.
|
|
49
|
+
if not isinstance(self.criteria, dict) or \
|
|
50
|
+
{k.lower() for k in self.criteria} != {"false", "true"}:
|
|
51
|
+
raise ValueError("noul criteria must be exactly false and true")
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def keys(self) -> List[str]:
|
|
55
|
+
"""The answer keys, in order. Score levels are indexed from 0 as strings,
|
|
56
|
+
which is how the API returns them in `probabilities` and `legend`."""
|
|
57
|
+
if self.kind == "choice":
|
|
58
|
+
return list(self.criteria)
|
|
59
|
+
if self.kind == "score":
|
|
60
|
+
return [str(i) for i in range(len(self.criteria))]
|
|
61
|
+
return ["false", "true"] # noul reports p(true) as `noul`
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def descriptions(self) -> List[str]:
|
|
65
|
+
"""Option text in the same order as `keys`. A noul question has no
|
|
66
|
+
option text at all -- its two outcomes are implicit."""
|
|
67
|
+
if self.kind == "choice":
|
|
68
|
+
return list(self.criteria.values())
|
|
69
|
+
if self.kind == "score":
|
|
70
|
+
return list(self.criteria)
|
|
71
|
+
if self.criteria:
|
|
72
|
+
low = {k.lower(): v for k, v in self.criteria.items()}
|
|
73
|
+
return [low["false"], low["true"]]
|
|
74
|
+
return ["no", "yes"]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class Request:
|
|
79
|
+
state: str
|
|
80
|
+
questions: Dict[str, Question] # ordered
|
|
81
|
+
model: str = "jev-latest"
|
|
82
|
+
|
|
83
|
+
def to_json(self) -> dict:
|
|
84
|
+
out = {"model": self.model, "state": self.state, "questions": {}}
|
|
85
|
+
for qid, q in self.questions.items():
|
|
86
|
+
d = {"type": q.kind, "instructions": q.instructions}
|
|
87
|
+
if q.kind == "choice":
|
|
88
|
+
d["criteria"] = dict(q.criteria)
|
|
89
|
+
elif q.kind == "score":
|
|
90
|
+
d["criteria"] = list(q.criteria)
|
|
91
|
+
out["questions"][qid] = d
|
|
92
|
+
return out
|
|
93
|
+
|
|
94
|
+
@classmethod
|
|
95
|
+
def from_json(cls, d: dict) -> "Request":
|
|
96
|
+
qs = {}
|
|
97
|
+
for qid, q in d["questions"].items():
|
|
98
|
+
qs[qid] = Question(q["type"], q["instructions"], q.get("criteria"))
|
|
99
|
+
return cls(state=d["state"], questions=qs, model=d.get("model", "jev-latest"))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class Answer:
|
|
104
|
+
kind: str
|
|
105
|
+
probabilities: Dict[str, float] # ordered, sums to 1
|
|
106
|
+
choice: Optional[str] = None
|
|
107
|
+
score: Optional[float] = None
|
|
108
|
+
legend: Optional[Dict[str, str]] = None
|
|
109
|
+
confidence: Optional[float] = None
|
|
110
|
+
|
|
111
|
+
@classmethod
|
|
112
|
+
def from_json(cls, d: dict) -> "Answer":
|
|
113
|
+
kind = d["type"]
|
|
114
|
+
if kind == "noul":
|
|
115
|
+
# The API returns a single `noul` probability, not a distribution.
|
|
116
|
+
# Expand it so every kind reads the same downstream.
|
|
117
|
+
p = float(d["noul"])
|
|
118
|
+
probs = {"false": 1.0 - p, "true": p}
|
|
119
|
+
else:
|
|
120
|
+
probs = {k: float(v) for k, v in d["probabilities"].items()}
|
|
121
|
+
return cls(kind=kind, probabilities=probs, choice=d.get("choice"),
|
|
122
|
+
score=d.get("score"), legend=d.get("legend"),
|
|
123
|
+
confidence=d.get("confidence"))
|
|
124
|
+
|
|
125
|
+
def p(self, key: str) -> float:
|
|
126
|
+
return self.probabilities.get(key, 0.0)
|
|
127
|
+
|
|
128
|
+
def log_odds(self, a: str, b: str, floor: float = 5e-3) -> float:
|
|
129
|
+
"""Log odds of one key against another.
|
|
130
|
+
|
|
131
|
+
The API rounds probabilities to two decimals, so a reported 0.00 is a
|
|
132
|
+
reporting floor and could be anything under 0.005. Clamping to that
|
|
133
|
+
floor keeps the ratio finite and states the assumption in one place.
|
|
134
|
+
"""
|
|
135
|
+
import math
|
|
136
|
+
return math.log(max(self.p(a), floor) / max(self.p(b), floor))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def answers_from_response(d: dict) -> Dict[str, Answer]:
|
|
140
|
+
return {qid: Answer.from_json(a) for qid, a in d["answers"].items()}
|
|
141
|
+
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: decision-jef
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Typed decisions with calibrated probabilities, in one forward pass
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://huggingface.co/BarraHome/Decision-Jef-0.1
|
|
7
|
+
Project-URL: Model, https://huggingface.co/BarraHome/Decision-Jef-0.1
|
|
8
|
+
Project-URL: Issues, https://huggingface.co/BarraHome/Decision-Jef-0.1/discussions
|
|
9
|
+
Keywords: typed-decisions,calibration,classification,routing,scoring,encoder,multilingual,nlp
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: torch>=2.4
|
|
26
|
+
Requires-Dist: transformers>=4.48
|
|
27
|
+
Requires-Dist: huggingface-hub>=0.26
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
license: mit
|
|
34
|
+
base_model: jhu-clsp/mmBERT-base
|
|
35
|
+
base_model_relation: finetune
|
|
36
|
+
pipeline_tag: text-classification
|
|
37
|
+
inference: false
|
|
38
|
+
language:
|
|
39
|
+
- multilingual
|
|
40
|
+
tags:
|
|
41
|
+
- system-one
|
|
42
|
+
- calibrated-decisions
|
|
43
|
+
- typed-decisions
|
|
44
|
+
- classification
|
|
45
|
+
- routing
|
|
46
|
+
- scoring
|
|
47
|
+
- guardrails
|
|
48
|
+
- moderation
|
|
49
|
+
- multilingual
|
|
50
|
+
- commercial-use
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
# Decision-Jef-0.1
|
|
54
|
+
|
|
55
|
+
Answer several runtime-defined questions about one state, in a single forward
|
|
56
|
+
pass, with calibrated probabilities over exactly the options you supply.
|
|
57
|
+
|
|
58
|
+
307M parameters. **12.12 ms for four decisions in one forward pass.**
|
|
59
|
+
|
|
60
|
+
The answer space is built from the request, so a value you did not offer is not
|
|
61
|
+
representable — not merely unlikely. There is no classification head over a
|
|
62
|
+
fixed label set.
|
|
63
|
+
|
|
64
|
+
## Results
|
|
65
|
+
|
|
66
|
+
Typed-decisions test set, 2,000 decisions: 600 `choice`, 600 `noul`, 800
|
|
67
|
+
`score`. Same set and same split as the published competitors.
|
|
68
|
+
|
|
69
|
+
| model | global | choice | noul | score |
|
|
70
|
+
| --- | --- | --- | --- | --- |
|
|
71
|
+
| Decision-1.0-Lex | **78.15** | **74.00** | 84.67 | **76.38** |
|
|
72
|
+
| **Decision-Jef-0.1** | **77.00** | 73.67 | 84.33 | 74.00 |
|
|
73
|
+
| Laya Typed Decisions | 76.60 | 73.33 | **85.67** | 72.25 |
|
|
74
|
+
|
|
75
|
+
Ahead of Laya by 0.40 global and 1.75 on `score`. Behind Lex by 1.15 global,
|
|
76
|
+
with `choice` and `noul` within a third of a point and the gap concentrated in
|
|
77
|
+
`score`.
|
|
78
|
+
|
|
79
|
+
| | value |
|
|
80
|
+
| --- | --- |
|
|
81
|
+
| NLL, choice / noul / score | 0.7200 / 0.4043 / 0.7316 |
|
|
82
|
+
| ECE, 10 bins, calibrated | **0.060** |
|
|
83
|
+
| ECE, 10 bins, uncalibrated | 0.138 |
|
|
84
|
+
| mean confidence, calibrated | 0.827 |
|
|
85
|
+
|
|
86
|
+
Temperature scaling per (type, cardinality) bucket ships with the model and is
|
|
87
|
+
applied by default, taking ECE from 0.138 to **0.060**. It never changes an
|
|
88
|
+
argmax, so accuracy is 77.00 either way. Pass `calibrated=False` to `decide()`
|
|
89
|
+
for the raw distribution.
|
|
90
|
+
|
|
91
|
+
Question isolation is exact: adding a question moves another question's logits
|
|
92
|
+
by at most 3e-07.
|
|
93
|
+
|
|
94
|
+
## Latency
|
|
95
|
+
|
|
96
|
+
NVIDIA H100 NVL, bf16, median of 30 calls after warm-up. End to end: packing,
|
|
97
|
+
encoder and readout.
|
|
98
|
+
|
|
99
|
+
| questions in one call | median | p95 |
|
|
100
|
+
| --- | --- | --- |
|
|
101
|
+
| 1 | 11.73 ms | 14.09 ms |
|
|
102
|
+
| 2 | 11.88 ms | 12.17 ms |
|
|
103
|
+
| **4** | **12.12 ms** | 12.29 ms |
|
|
104
|
+
|
|
105
|
+
**Going from one decision to four costs 0.39 ms.** The state is encoded once and
|
|
106
|
+
the question branches are masked apart, so a request carrying four questions is
|
|
107
|
+
not four requests. Throughput at batch 64 and 1,024 tokens is 2.66 ms per
|
|
108
|
+
decision.
|
|
109
|
+
|
|
110
|
+
## Usage
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
pip install decision-jef
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The weights are published separately from the package. Authenticate with
|
|
117
|
+
`hf auth login` if the model repository is not yet public, or point
|
|
118
|
+
`from_pretrained` at a local directory holding `model.pt`.
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from decision_jef import Decider, Question
|
|
122
|
+
|
|
123
|
+
d = Decider.from_pretrained("BarraHome/Decision-Jef-0.1")
|
|
124
|
+
|
|
125
|
+
state = """from: user@acme.com
|
|
126
|
+
subject: Duplicate charge on invoice #4411
|
|
127
|
+
body: We were billed twice for March. Please refund the duplicate today
|
|
128
|
+
or we will cancel our plan."""
|
|
129
|
+
|
|
130
|
+
answers = d.decide(state, {
|
|
131
|
+
"department": Question("choice", "Which department should handle this?", {
|
|
132
|
+
"billing": "invoices, payments, refunds",
|
|
133
|
+
"technical": "bugs, outages, system errors",
|
|
134
|
+
"sales": "pricing, new contracts",
|
|
135
|
+
"other": "everything else",
|
|
136
|
+
}),
|
|
137
|
+
"urgency": Question("score", "How urgent is this request?",
|
|
138
|
+
["not urgent", "soon", "critical or blocking"]),
|
|
139
|
+
"churn_risk": Question("noul", "Does the user threaten to leave?", {
|
|
140
|
+
"false": "The user makes no threat to stop using the service.",
|
|
141
|
+
"true": "The user threatens to cancel, churn or leave.",
|
|
142
|
+
}),
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
for qid, a in answers.items():
|
|
146
|
+
print(qid, a.choice or a.p("true"), a.confidence, a.probabilities)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
```
|
|
150
|
+
department billing 0.94 {'billing': 0.9383, 'technical': 0.0147, 'sales': 0.0301, 'other': 0.017}
|
|
151
|
+
urgency 2 0.58 {'0': 0.156, '1': 0.2662, '2': 0.5778} score=1.42
|
|
152
|
+
churn_risk 0.87 0.87 {'false': 0.1292, 'true': 0.8708}
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
`d.to_wire(answers)` returns the same content as a JSON-ready response body.
|
|
156
|
+
|
|
157
|
+
### Give every yes/no outcome a description
|
|
158
|
+
|
|
159
|
+
This is a requirement, not a style note. The model scores the option text, so
|
|
160
|
+
a bare yes/no gives it nothing to compare. On the example above:
|
|
161
|
+
|
|
162
|
+
| question | p(true) with bare yes/no | p(true) with descriptions |
|
|
163
|
+
| --- | --- | --- |
|
|
164
|
+
| threatens to leave | 0.096 — wrong | **0.850** — right |
|
|
165
|
+
| requests a refund | 0.309 — wrong | **0.912** — right |
|
|
166
|
+
|
|
167
|
+
## The three question types
|
|
168
|
+
|
|
169
|
+
| type | `criteria` | answer |
|
|
170
|
+
| --- | --- | --- |
|
|
171
|
+
| `choice` | ordered map of key to description, up to 255 | `choice`, `probabilities` |
|
|
172
|
+
| `noul` | optional map of `false` and `true` to a description — supply it | `noul` probability |
|
|
173
|
+
| `score` | ordered array of 2 to 10 level descriptions | probability-weighted `score`, `legend` |
|
|
174
|
+
|
|
175
|
+
Option order is part of the question. The same options in a different order
|
|
176
|
+
are a different request, and the model is sensitive to it — include a
|
|
177
|
+
permutation check in any evaluation.
|
|
178
|
+
|
|
179
|
+
## How it works
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
<bos> state ... <eos> shared, encoded once
|
|
183
|
+
[Q] instructions ... [OPT] opt0 [OPT] opt1 ... [DEC] question 1
|
|
184
|
+
[Q] instructions ... [OPT] opt0 [OPT] opt1 ... [DEC] question 2
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
The query is read at `[DEC]`, after the whole option list, so the decision sees
|
|
188
|
+
every option. Keys come from each `[OPT]` in the same pass, so the options are
|
|
189
|
+
read together rather than scored in isolation. Each question attends to the
|
|
190
|
+
state and to itself only; the state attends to neither.
|
|
191
|
+
|
|
192
|
+
## Limitations
|
|
193
|
+
|
|
194
|
+
- **ECE is 0.060 only with the shipped temperatures applied**, 0.138 without.
|
|
195
|
+
They are on by default; do not disable them unless you are recalibrating.
|
|
196
|
+
- **`score` is the weakest type** at 74.00, 2.38 behind Lex.
|
|
197
|
+
- Trained and measured on **English** typed decisions. The backbone is
|
|
198
|
+
multilingual and the tokenizer covers 256k tokens, but no non-English
|
|
199
|
+
benchmark has been run — treat multilingual use as untested.
|
|
200
|
+
- The `guardrails` and `moderation` tags reflect coverage of toxicity and
|
|
201
|
+
hate-speech decisions. **Neither capability has been benchmarked.**
|
|
202
|
+
- Sensitive to option order, as above.
|
|
203
|
+
- Long states are truncated to the window with the questions reserved first.
|
|
204
|
+
|
|
205
|
+
## License and provenance
|
|
206
|
+
|
|
207
|
+
MIT, following the `jhu-clsp/mmBERT-base` backbone. No weights, gradients or
|
|
208
|
+
private data from any third-party decision service are used or claimed; the
|
|
209
|
+
design follows publicly documented API behaviour and public benchmark splits.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
decision_jef/__init__.py
|
|
5
|
+
decision_jef/infer.py
|
|
6
|
+
decision_jef/model.py
|
|
7
|
+
decision_jef/pack.py
|
|
8
|
+
decision_jef/py.typed
|
|
9
|
+
decision_jef/wire.py
|
|
10
|
+
decision_jef.egg-info/PKG-INFO
|
|
11
|
+
decision_jef.egg-info/SOURCES.txt
|
|
12
|
+
decision_jef.egg-info/dependency_links.txt
|
|
13
|
+
decision_jef.egg-info/requires.txt
|
|
14
|
+
decision_jef.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
decision_jef
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "decision-jef"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Typed decisions with calibrated probabilities, in one forward pass"
|
|
9
|
+
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
keywords = [
|
|
14
|
+
"typed-decisions", "calibration", "classification", "routing",
|
|
15
|
+
"scoring", "encoder", "multilingual", "nlp",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 4 - Beta",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Intended Audience :: Science/Research",
|
|
21
|
+
"Operating System :: OS Independent",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Programming Language :: Python :: 3.13",
|
|
27
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
28
|
+
"Topic :: Text Processing :: Linguistic",
|
|
29
|
+
"Typing :: Typed",
|
|
30
|
+
]
|
|
31
|
+
dependencies = [
|
|
32
|
+
"torch>=2.4",
|
|
33
|
+
"transformers>=4.48",
|
|
34
|
+
"huggingface-hub>=0.26",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
dev = ["pytest>=8"]
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://huggingface.co/BarraHome/Decision-Jef-0.1"
|
|
42
|
+
Model = "https://huggingface.co/BarraHome/Decision-Jef-0.1"
|
|
43
|
+
Issues = "https://huggingface.co/BarraHome/Decision-Jef-0.1/discussions"
|
|
44
|
+
|
|
45
|
+
[tool.setuptools]
|
|
46
|
+
packages = ["decision_jef"]
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.package-data]
|
|
49
|
+
decision_jef = ["py.typed"]
|