decision-gem 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- decision_gem/__init__.py +4 -0
- decision_gem/__main__.py +34 -0
- decision_gem/core.py +474 -0
- decision_gem-0.1.0.dist-info/METADATA +366 -0
- decision_gem-0.1.0.dist-info/RECORD +9 -0
- decision_gem-0.1.0.dist-info/WHEEL +5 -0
- decision_gem-0.1.0.dist-info/entry_points.txt +2 -0
- decision_gem-0.1.0.dist-info/licenses/LICENSE +21 -0
- decision_gem-0.1.0.dist-info/top_level.txt +1 -0
decision_gem/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
"""Jev-style decision model on a local Gemma (MLX). See decision_gem.core."""
|
|
2
|
+
from .core import MODEL, ORDERS, DecisionGem, answer, fit_calibration, letter_ids, options_of, orders_of, question_text, split_open
|
|
3
|
+
|
|
4
|
+
__all__ = ["MODEL", "ORDERS", "DecisionGem", "answer", "fit_calibration", "letter_ids", "options_of", "orders_of", "question_text", "split_open"]
|
decision_gem/__main__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Answer one Jev request ({"state", "questions"}) or one prompt ({"prompt", "state" optional, "max_tokens"}) and
|
|
2
|
+
print the response.
|
|
3
|
+
|
|
4
|
+
python -m decision_gem request.json
|
|
5
|
+
cat request.json | python -m decision_gem - --calibration cal.json
|
|
6
|
+
"""
|
|
7
|
+
import argparse
|
|
8
|
+
import json
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from .core import MODEL, ORDERS, DecisionGem
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main():
|
|
15
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
16
|
+
ap.add_argument("request", help="the request, JSON file or - for stdin")
|
|
17
|
+
ap.add_argument("--model", default=MODEL, help="HF repo id or local model folder")
|
|
18
|
+
ap.add_argument("--calibration", help="JSON {question id: [a, b] (noul) or T (choice / score)}")
|
|
19
|
+
ap.add_argument("--orders", choices=ORDERS, default="reverse",
|
|
20
|
+
help="option orders: in order and reversed (2 passes) or every rotation (n passes)")
|
|
21
|
+
ap.add_argument("--online", action="store_true", help="allow downloading the model")
|
|
22
|
+
ap.add_argument("--raw", action="store_true",
|
|
23
|
+
help="add the uncalibrated option log-probabilities to every answer (input of fit_calibration)")
|
|
24
|
+
args = ap.parse_args()
|
|
25
|
+
request = json.load(sys.stdin if args.request == "-" else open(args.request, encoding="utf-8"))
|
|
26
|
+
calibration = json.load(open(args.calibration, encoding="utf-8")) if args.calibration else None
|
|
27
|
+
gem = DecisionGem(args.model, calibration, offline=not args.online, orders=args.orders)
|
|
28
|
+
if args.raw:
|
|
29
|
+
request = {**request, "raw": True}
|
|
30
|
+
print(json.dumps(gem(request), ensure_ascii=False, indent=1))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
if __name__ == "__main__":
|
|
34
|
+
main()
|
decision_gem/core.py
ADDED
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
"""DecisionGem: a local LLM (Gemma, MLX) read as a decision model, with the request / response contract of TypeSafe's
|
|
2
|
+
Jev (https://developers.cloudflare.com/ai/models/typesafe/jev/). Runs fully offline.
|
|
3
|
+
|
|
4
|
+
Request: {"state": text or JSON object, "questions": {id: question}}; a question is
|
|
5
|
+
- noul: {"type": "noul", "instructions": ..., "criteria": {"true": ..., "false": ...}} (criteria optional)
|
|
6
|
+
- choice: {"type": "choice", "instructions": ..., "criteria": {option: description} or [option, ...]} (2..26)
|
|
7
|
+
- score: {"type": "score", "instructions": ..., "criteria": {value: description} or [description, ...]} (2..10
|
|
8
|
+
levels, in order; list levels are worth 1..N)
|
|
9
|
+
Response: {"model", "answers": {id: answer}, "usage": {"input_tokens", "output_tokens": 0}}; an answer is
|
|
10
|
+
- noul: {"type", "noul": P(true)}
|
|
11
|
+
- choice: {"type", "choice": option, "confidence", "probabilities": {option: p}}
|
|
12
|
+
- score: {"type", "score": sum of p * value, "confidence", "legend": {value: description}, "probabilities"}
|
|
13
|
+
plus, beyond Jev, "orders_agree": whether every option order (below) picks the same answer.
|
|
14
|
+
|
|
15
|
+
How: nothing is generated. The state goes first in the prompt and is read once into a KV cache; every question is
|
|
16
|
+
appended to a copy of that cache, its options listed under the letters A, B, C …, and the logits of those letters
|
|
17
|
+
at the first answer position are read. Every question is read in several option orders and the log-probabilities
|
|
18
|
+
of each option are averaged, so a preference for a letter or a position cancels out. orders="reverse" (default): twice,
|
|
19
|
+
in order and reversed — 2 passes, but with 3+ options the middle stays in the middle, so a pull to or from the middle
|
|
20
|
+
letters is not cancelled. orders="cyclic": every rotation, each option once under every letter — n passes.
|
|
21
|
+
confidence is the probability of the answer. Calibration (Platt scaling on hand-labelled calls) per question id:
|
|
22
|
+
noul p = sigmoid(a * logit + b) from [a, b]; choice / score probabilities = softmax(logprob / T) from T.
|
|
23
|
+
|
|
24
|
+
Beyond Jev, generate() runs the model as an ordinary chat model: a prompt (after a state, or a kept state's key, or
|
|
25
|
+
alone) and the generated answer; a request with "prompt" instead of "questions" does the same.
|
|
26
|
+
|
|
27
|
+
Everything but DecisionGem itself is pure Python; the model needs mlx and mlx-lm (Apple silicon).
|
|
28
|
+
"""
|
|
29
|
+
import copy
|
|
30
|
+
import json
|
|
31
|
+
import math
|
|
32
|
+
import os
|
|
33
|
+
import re
|
|
34
|
+
import string
|
|
35
|
+
|
|
36
|
+
MODEL = "mlx-community/gemma-4-26b-a4b-it-4bit"
|
|
37
|
+
LETTERS = string.ascii_uppercase
|
|
38
|
+
_QUESTION = "\x00question\x00" # where the question goes in the rendered prompt
|
|
39
|
+
STATE_INTRO = "Read the state below, then answer the question after it.\n\n<state>\n"
|
|
40
|
+
STATE_CLOSE = "\n</state>\n\n" # what follows the state text; an open state leaves it out of the cache
|
|
41
|
+
KINDS = ("noul", "choice", "score")
|
|
42
|
+
ORDERS = ("reverse", "cyclic")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def options_of(question):
|
|
46
|
+
"""[(key, description)] of a question's options, in order: noul true / false, choice options, score levels."""
|
|
47
|
+
kind, criteria = question.get("type"), question.get("criteria")
|
|
48
|
+
if kind not in KINDS:
|
|
49
|
+
raise ValueError(f"question type must be one of {', '.join(KINDS)}, got {kind!r}")
|
|
50
|
+
if kind == "noul":
|
|
51
|
+
criteria = criteria or {}
|
|
52
|
+
return [("true", criteria.get("true", "yes")), ("false", criteria.get("false", "no"))]
|
|
53
|
+
if isinstance(criteria, dict):
|
|
54
|
+
items = [(str(k), str(v)) for k, v in criteria.items()]
|
|
55
|
+
elif isinstance(criteria, list):
|
|
56
|
+
items = [(str(x), str(x)) for x in criteria] if kind == "choice" else \
|
|
57
|
+
[(str(i + 1), str(x)) for i, x in enumerate(criteria)]
|
|
58
|
+
else:
|
|
59
|
+
raise ValueError(f"{kind} question needs criteria")
|
|
60
|
+
low, high = (2, len(LETTERS)) if kind == "choice" else (2, 10)
|
|
61
|
+
if not low <= len(items) <= high:
|
|
62
|
+
raise ValueError(f"{kind} question needs {low}..{high} options, got {len(items)}")
|
|
63
|
+
return items
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def question_text(question, items):
|
|
67
|
+
"""The question as appended after the state, its options under the letters A, B, C …"""
|
|
68
|
+
lines = [f"Question: {question.get('instructions', '').strip()}"]
|
|
69
|
+
if question["type"] == "score":
|
|
70
|
+
lines.append("The options are levels of an ordered scale.")
|
|
71
|
+
for letter, (key, desc) in zip(LETTERS, items):
|
|
72
|
+
label = key if desc == key else f"{key} ({desc})" if question["type"] != "noul" else f"{key}: {desc}"
|
|
73
|
+
lines.append(f"{letter}: {label}")
|
|
74
|
+
lines.append(f"Answer with one letter: {', '.join(LETTERS[:len(items)])}.")
|
|
75
|
+
return "\n".join(lines)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def orders_of(n, how="reverse"):
|
|
79
|
+
"""The option orders a question of n options is read in, each a list of option indexes by letter."""
|
|
80
|
+
if how == "reverse":
|
|
81
|
+
return [list(range(n)), list(range(n))[::-1]]
|
|
82
|
+
if how == "cyclic":
|
|
83
|
+
return [[(s + i) % n for i in range(n)] for s in range(n)]
|
|
84
|
+
raise ValueError(f"orders must be one of {', '.join(ORDERS)}, got {how!r}")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def split_open(prompt):
|
|
88
|
+
"""(head, close) of a rendered prompt: head ends right after the state text, close is what follows it."""
|
|
89
|
+
at = prompt.find(STATE_CLOSE)
|
|
90
|
+
if at < 0:
|
|
91
|
+
raise ValueError("the prompt has no closing </state>")
|
|
92
|
+
return prompt[:at], STATE_CLOSE
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def float32_matmuls(mx):
|
|
96
|
+
"""Whether this process's MLX multiplies float32 matrices on the GPU in float32: the GPU's product against the
|
|
97
|
+
CPU's. On GPUs with neural accelerators (M5) MLX multiplies them in TF32 (relative error ~5e-4, float32 ~1e-6)
|
|
98
|
+
from 128 rows up (dense: from 2), unless MLX_ENABLE_TF32=0 was set before the process's first matmul."""
|
|
99
|
+
a = mx.random.normal((256, 256), key=mx.random.key(0))
|
|
100
|
+
gpu, cpu = a @ a.T, mx.matmul(a, a.T, stream=mx.cpu)
|
|
101
|
+
return (mx.abs(gpu - cpu).max() / mx.abs(cpu).max()).item() < 1e-5
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def letter_ids(tok):
|
|
105
|
+
"""Token ids of the letters A..Z as the first token of the answer. Each letter is appended to the tail of the
|
|
106
|
+
chat prompt (what follows the question) and must add exactly one token there. A tokenizer that splits a letter or
|
|
107
|
+
merges it with the tail cannot be read this way: ValueError, before any question runs."""
|
|
108
|
+
if not getattr(tok, "chat_template", None):
|
|
109
|
+
raise ValueError("the model has no chat template: decision-gem needs an instruction-tuned (chat) model")
|
|
110
|
+
prompt = tok.apply_chat_template([{"role": "user", "content": _QUESTION}], add_generation_prompt=True,
|
|
111
|
+
tokenize=False, enable_thinking=False)
|
|
112
|
+
tail = prompt.split(_QUESTION)[1]
|
|
113
|
+
base = tok.encode(tail, add_special_tokens=False)
|
|
114
|
+
ids = []
|
|
115
|
+
for x in LETTERS:
|
|
116
|
+
full = tok.encode(tail + x, add_special_tokens=False)
|
|
117
|
+
if full[:len(base)] != base or len(full) != len(base) + 1:
|
|
118
|
+
raise ValueError(f"letter {x!r} after the chat prompt is not one token in this tokenizer "
|
|
119
|
+
f"({len(full) - len(base)} tokens): the answer cannot be read from letter logits")
|
|
120
|
+
ids.append(full[-1])
|
|
121
|
+
if len(set(ids)) != len(ids):
|
|
122
|
+
raise ValueError("two letters share a token in this tokenizer")
|
|
123
|
+
return ids
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def softmax(xs):
|
|
127
|
+
top = max(xs)
|
|
128
|
+
es = [math.exp(x - top) for x in xs]
|
|
129
|
+
return [e / sum(es) for e in es]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def answer(question, items, logprobs, agree, cal=None, raw=False):
|
|
133
|
+
"""The Jev answer from the averaged log-probabilities of the options; cal is the question's calibration.
|
|
134
|
+
raw: add "logprobs" {option: log-probability before calibration}, the input of fit_calibration()."""
|
|
135
|
+
kind = question["type"]
|
|
136
|
+
extra = {"logprobs": {k: round(x, 6) for (k, _), x in zip(items, logprobs)}} if raw else {}
|
|
137
|
+
if kind == "noul":
|
|
138
|
+
a, b = cal or (1.0, 0.0)
|
|
139
|
+
p = 1 / (1 + math.exp(-(a * (logprobs[0] - logprobs[1]) + b)))
|
|
140
|
+
return {"type": kind, "noul": round(p, 4), "orders_agree": agree, **extra}
|
|
141
|
+
probs = softmax([x / (cal or 1.0) for x in logprobs])
|
|
142
|
+
best = max(range(len(items)), key=lambda i: probs[i])
|
|
143
|
+
out = {"type": kind}
|
|
144
|
+
if kind == "choice":
|
|
145
|
+
out["choice"] = items[best][0]
|
|
146
|
+
else:
|
|
147
|
+
values = [float(k) if re.fullmatch(r"-?\d+(\.\d+)?", k) else i + 1 for i, (k, _) in enumerate(items)]
|
|
148
|
+
out["score"] = round(sum(p * v for p, v in zip(probs, values)), 4)
|
|
149
|
+
out["legend"] = {k: d for k, d in items}
|
|
150
|
+
out.update(confidence=round(probs[best], 4),
|
|
151
|
+
probabilities={k: round(p, 4) for (k, _), p in zip(items, probs)}, orders_agree=agree, **extra)
|
|
152
|
+
return out
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _fit_platt(xs, ys):
|
|
156
|
+
"""[a, b] of p = sigmoid(a * x + b) by Newton's method on Platt's smoothed targets, which keep a and b finite
|
|
157
|
+
on separable labels."""
|
|
158
|
+
pos = sum(ys)
|
|
159
|
+
neg = len(ys) - pos
|
|
160
|
+
hi, lo = (pos + 1) / (pos + 2), 1 / (neg + 2)
|
|
161
|
+
ts = [hi if y else lo for y in ys]
|
|
162
|
+
|
|
163
|
+
def loss(a, b):
|
|
164
|
+
out = 0.0
|
|
165
|
+
for x, t in zip(xs, ts):
|
|
166
|
+
z = a * x + b
|
|
167
|
+
out += max(z, 0) + math.log1p(math.exp(-abs(z))) - t * z # -t·log p - (1-t)·log(1-p)
|
|
168
|
+
return out
|
|
169
|
+
|
|
170
|
+
a, b = 0.0, math.log((pos + 1) / (neg + 1))
|
|
171
|
+
for _ in range(100):
|
|
172
|
+
ga = gb = haa = hab = hbb = 0.0
|
|
173
|
+
for x, t in zip(xs, ts):
|
|
174
|
+
p = 1 / (1 + math.exp(-(a * x + b))) if a * x + b > -700 else 0.0
|
|
175
|
+
d, w = p - t, max(p * (1 - p), 1e-12)
|
|
176
|
+
ga, gb, haa, hab, hbb = ga + d * x, gb + d, haa + w * x * x, hab + w * x, hbb + w
|
|
177
|
+
det = haa * hbb - hab * hab + 1e-12
|
|
178
|
+
da, db = (hbb * ga - hab * gb) / det, (haa * gb - hab * ga) / det
|
|
179
|
+
step, before = 1.0, loss(a, b)
|
|
180
|
+
while step > 1e-10 and loss(a - step * da, b - step * db) > before + 1e-12:
|
|
181
|
+
step /= 2
|
|
182
|
+
a, b = a - step * da, b - step * db
|
|
183
|
+
if abs(step * da) < 1e-9 and abs(step * db) < 1e-9:
|
|
184
|
+
break
|
|
185
|
+
return [round(a, 4), round(b, 4)]
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _fit_temperature(rows):
|
|
189
|
+
"""T of softmax(logprob / T) with the least log-loss: the loss is convex in 1/T, so its derivative is bisected.
|
|
190
|
+
rows: [(logprobs, index of the label)]."""
|
|
191
|
+
def slope(beta): # d loss / d beta = sum of E[logprob] under softmax(beta·logprob) - logprob of the label
|
|
192
|
+
out = 0.0
|
|
193
|
+
for lp, y in rows:
|
|
194
|
+
ws = [beta * x for x in lp]
|
|
195
|
+
top = max(ws)
|
|
196
|
+
es = [math.exp(w - top) for w in ws]
|
|
197
|
+
out += sum(e * x for e, x in zip(es, lp)) / sum(es) - lp[y]
|
|
198
|
+
return out
|
|
199
|
+
lo, hi = math.log(1e-3), math.log(100.0) # T from 0.01 to 1000
|
|
200
|
+
if slope(math.exp(hi)) < 0:
|
|
201
|
+
return round(1 / math.exp(hi), 4)
|
|
202
|
+
if slope(math.exp(lo)) > 0:
|
|
203
|
+
return round(1 / math.exp(lo), 4)
|
|
204
|
+
for _ in range(100):
|
|
205
|
+
mid = (lo + hi) / 2
|
|
206
|
+
lo, hi = (mid, hi) if slope(math.exp(mid)) < 0 else (lo, mid)
|
|
207
|
+
return round(1 / math.exp((lo + hi) / 2), 4)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def fit_calibration(answers, labels):
|
|
211
|
+
"""The calibration {question id: [a, b] (noul) or T (choice / score)} fitted on labelled answers.
|
|
212
|
+
answers: the "answers" of responses decided with raw=True, one per state; labels: a {question id: label} per
|
|
213
|
+
state, in the same order. A label is True / False for noul, the option for choice, the level (value) for score;
|
|
214
|
+
a question with no label in a state is skipped there. Pass the result as DecisionGem(calibration=...) or in the
|
|
215
|
+
--calibration file."""
|
|
216
|
+
if len(answers) != len(labels):
|
|
217
|
+
raise ValueError(f"answers and labels must have the same length, got {len(answers)} and {len(labels)}")
|
|
218
|
+
rows = {}
|
|
219
|
+
for got, want in zip(answers, labels):
|
|
220
|
+
for qid, label in want.items():
|
|
221
|
+
a = got.get(qid)
|
|
222
|
+
if a is None:
|
|
223
|
+
continue
|
|
224
|
+
if "logprobs" not in a:
|
|
225
|
+
raise ValueError(f"answer {qid!r} has no logprobs: decide with raw=True")
|
|
226
|
+
rows.setdefault(qid, (a["type"], []))[1].append((a["logprobs"], label))
|
|
227
|
+
out = {}
|
|
228
|
+
for qid, (kind, pairs) in rows.items():
|
|
229
|
+
if kind == "noul":
|
|
230
|
+
xs = [lp["true"] - lp["false"] for lp, _ in pairs]
|
|
231
|
+
ys = []
|
|
232
|
+
for _, label in pairs:
|
|
233
|
+
if label not in (True, False, "true", "false"):
|
|
234
|
+
raise ValueError(f"noul label of {qid!r} must be True / False, got {label!r}")
|
|
235
|
+
ys.append(label in (True, "true"))
|
|
236
|
+
if all(ys) or not any(ys):
|
|
237
|
+
raise ValueError(f"noul {qid!r} needs both labels, true and false")
|
|
238
|
+
out[qid] = _fit_platt(xs, ys)
|
|
239
|
+
else:
|
|
240
|
+
fit = []
|
|
241
|
+
for lp, label in pairs:
|
|
242
|
+
keys = list(lp)
|
|
243
|
+
if str(label) not in keys:
|
|
244
|
+
raise ValueError(f"label {label!r} of {qid!r} is not one of {keys}")
|
|
245
|
+
fit.append(([lp[k] for k in keys], keys.index(str(label))))
|
|
246
|
+
out[qid] = _fit_temperature(fit)
|
|
247
|
+
return out
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
class DecisionGem:
|
|
251
|
+
def __init__(self, repo=MODEL, calibration=None, offline=True, cache_limit=0, orders="reverse", float32=True):
|
|
252
|
+
"""repo: a Hugging Face repo id or a local model folder. offline: load only what is in the local HF cache.
|
|
253
|
+
orders: the option orders every question is read in, "reverse" or "cyclic" (see the module doc).
|
|
254
|
+
cache_limit: bytes of freed buffers MLX may keep for reuse (mx.set_cache_limit), None to leave MLX's own.
|
|
255
|
+
MLX's limit is the process's: unload() puts back the one there was.
|
|
256
|
+
float32: compute in float32 (the bf16 norms and quantization scales cast up; +1.5 GiB, KV cache 2x, reads
|
|
257
|
+
about 1.3x slower). In bf16 a read's rounding depends on how many tokens it holds and flips the MoE router's
|
|
258
|
+
top-8 on near-ties, so one text read whole, in chunks or with </state> cut elsewhere gave noul 0.14..0.65
|
|
259
|
+
(1590 tokens, 10.19); in float32 0.568..0.574. extend() equals one read only in float32.
|
|
260
|
+
float32 also needs float32 matmuls: on an M5 MLX multiplies in TF32 by default, and the same 1590 tokens
|
|
261
|
+
read whole or in chunks gave 0.546 vs 0.347 (10.19). So the gem sets MLX_ENABLE_TF32=0; MLX reads it at the
|
|
262
|
+
process's first matmul, and a RuntimeError says so if one ran before with TF32 on. Cost on the M5 Max: reads
|
|
263
|
+
2x slower than in TF32 (1594 tokens 1.6 s vs 0.8 s, a question 0.30 s vs 0.18 s)."""
|
|
264
|
+
import mlx.core as mx
|
|
265
|
+
from mlx_lm import load
|
|
266
|
+
orders_of(2, orders) # a bad value fails before the model loads
|
|
267
|
+
self.orders = orders
|
|
268
|
+
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
|
|
269
|
+
if float32:
|
|
270
|
+
os.environ.setdefault("MLX_ENABLE_TF32", "0")
|
|
271
|
+
if not float32_matmuls(mx):
|
|
272
|
+
raise RuntimeError("MLX multiplies float32 in TF32 in this process: set MLX_ENABLE_TF32=0 before the "
|
|
273
|
+
"first MLX matmul (MLX reads it once), or pass float32=False")
|
|
274
|
+
path = repo
|
|
275
|
+
if offline and not os.path.isdir(repo):
|
|
276
|
+
from huggingface_hub import snapshot_download
|
|
277
|
+
path = snapshot_download(repo, local_files_only=True)
|
|
278
|
+
self.mx, self.repo, self.float32 = mx, repo, float32
|
|
279
|
+
# MLX keeps freed buffers for reuse; the cache copies per question piled 13 GiB of them on the 13 GiB model
|
|
280
|
+
# (peak footprint 27.8 GiB on the 5 longest calls, 13.10). With no buffer cache: 17.2 GiB, same time.
|
|
281
|
+
self._cache_limit_before = None if cache_limit is None else mx.set_cache_limit(cache_limit)
|
|
282
|
+
self.model, self.tok = load(path)
|
|
283
|
+
if float32:
|
|
284
|
+
from mlx.utils import tree_map
|
|
285
|
+
self.model.update(tree_map(lambda p: p.astype(mx.float32) if p.dtype == mx.bfloat16 else p,
|
|
286
|
+
self.model.parameters()))
|
|
287
|
+
mx.eval(self.model.parameters())
|
|
288
|
+
self.letter_ids = letter_ids(self.tok)
|
|
289
|
+
self.calibration = calibration or {}
|
|
290
|
+
self.states = {} # key → a state read by prefill() / load(): {"cache", "tokens", "tail", "close"}
|
|
291
|
+
|
|
292
|
+
@property
|
|
293
|
+
def dtype(self):
|
|
294
|
+
"""The compute dtype, saved with a state: load() takes a state only into a gem of the same dtype."""
|
|
295
|
+
return "float32" if self.float32 else "bfloat16"
|
|
296
|
+
|
|
297
|
+
def unload(self):
|
|
298
|
+
"""Free the model's memory without ending the process (one big model at a time); decide() fails after."""
|
|
299
|
+
import gc
|
|
300
|
+
if self.model is None:
|
|
301
|
+
return
|
|
302
|
+
self.model = self.tok = None
|
|
303
|
+
self.states = {}
|
|
304
|
+
gc.collect()
|
|
305
|
+
self.mx.clear_cache()
|
|
306
|
+
if self._cache_limit_before is not None:
|
|
307
|
+
self.mx.set_cache_limit(self._cache_limit_before)
|
|
308
|
+
self._cache_limit_before = None
|
|
309
|
+
|
|
310
|
+
def __enter__(self):
|
|
311
|
+
return self
|
|
312
|
+
|
|
313
|
+
def __exit__(self, *_):
|
|
314
|
+
self.unload()
|
|
315
|
+
|
|
316
|
+
def _encode(self, text):
|
|
317
|
+
return self.tok.encode(text, add_special_tokens=False)
|
|
318
|
+
|
|
319
|
+
def _read(self, state, open=False):
|
|
320
|
+
"""The state read into a fresh KV cache: (cache, its tokens, the prompt's tail after the question, close).
|
|
321
|
+
open: the cache stops after the state text and close is what decide() puts before each question; a closed
|
|
322
|
+
state has the closing tag in its cache and close is ''."""
|
|
323
|
+
from mlx_lm.models.cache import make_prompt_cache
|
|
324
|
+
if self.model is None:
|
|
325
|
+
raise RuntimeError("the model is unloaded")
|
|
326
|
+
if not isinstance(state, str):
|
|
327
|
+
state = json.dumps(state, ensure_ascii=False, indent=1)
|
|
328
|
+
content = f"{STATE_INTRO}{state.strip()}{STATE_CLOSE}{_QUESTION}"
|
|
329
|
+
prompt = self.tok.apply_chat_template([{"role": "user", "content": content}], add_generation_prompt=True,
|
|
330
|
+
tokenize=False, enable_thinking=False)
|
|
331
|
+
head, tail = prompt.split(_QUESTION)
|
|
332
|
+
close = ""
|
|
333
|
+
if open:
|
|
334
|
+
head, close = split_open(head)
|
|
335
|
+
prefix = self._encode(head)
|
|
336
|
+
cache = make_prompt_cache(self.model)
|
|
337
|
+
self.mx.eval(self.model(self.mx.array(prefix)[None], cache=cache))
|
|
338
|
+
return cache, len(prefix), tail, close
|
|
339
|
+
|
|
340
|
+
def prefill(self, key, state, open=False):
|
|
341
|
+
"""Read a state once and keep it under `key`, for decide(key=...) later; the number of its tokens.
|
|
342
|
+
open: stop after the state text, so extend(key, text) can go on; decide() adds the closing tag per question.
|
|
343
|
+
Memory: about 0.43 MiB per token in float32 (691 MiB for 1590 tokens, 10.19; 0.2 MiB in bf16) until
|
|
344
|
+
forget(key) or unload()."""
|
|
345
|
+
cache, tokens, tail, close = self._read(state, open)
|
|
346
|
+
self.states[key] = {"cache": cache, "tokens": tokens, "tail": tail, "close": close}
|
|
347
|
+
return tokens
|
|
348
|
+
|
|
349
|
+
def extend(self, key, text):
|
|
350
|
+
"""Read more text into an open state's cache, for good; the number of tokens added. ValueError on a closed one.
|
|
351
|
+
Seam rule: extend = one read only if the text starts at a token boundary. The leading space or newline goes in
|
|
352
|
+
the new text, a word is never split, and the text never ends in whitespace (ValueError): "hello. " + "Operator"
|
|
353
|
+
tokenizes differently from "hello. Operator". prefill() strips the state, so it never ends in whitespace."""
|
|
354
|
+
if self.model is None:
|
|
355
|
+
raise RuntimeError("the model is unloaded")
|
|
356
|
+
kept = self.states[key]
|
|
357
|
+
if not kept["close"]:
|
|
358
|
+
raise ValueError(f"state {key!r} is closed: prefill it with open=True")
|
|
359
|
+
if text != text.rstrip():
|
|
360
|
+
raise ValueError("extend text must not end in whitespace: put it at the start of the next text")
|
|
361
|
+
ids = self._encode(text)
|
|
362
|
+
if ids:
|
|
363
|
+
self.mx.eval(self.model(self.mx.array(ids)[None], cache=kept["cache"]))
|
|
364
|
+
kept["tokens"] += len(ids)
|
|
365
|
+
return len(ids)
|
|
366
|
+
|
|
367
|
+
def forget(self, key):
|
|
368
|
+
"""Drop a kept state; its memory goes back to the system."""
|
|
369
|
+
del self.states[key]
|
|
370
|
+
|
|
371
|
+
def save(self, key, path):
|
|
372
|
+
"""Write a kept state to a file (.safetensors); load() reads it back in about 0.02 s instead of a prefill.
|
|
373
|
+
The file is the call's text in other form: keep it where the text may be."""
|
|
374
|
+
from mlx_lm.models.cache import save_prompt_cache
|
|
375
|
+
kept = self.states[key]
|
|
376
|
+
meta = {"tokens": str(kept["tokens"]), "tail": kept["tail"], "close": kept["close"], "repo": self.repo,
|
|
377
|
+
"dtype": self.dtype}
|
|
378
|
+
save_prompt_cache(path, kept["cache"], meta)
|
|
379
|
+
|
|
380
|
+
def load(self, key, path):
|
|
381
|
+
"""Keep the state saved in `path` under `key`; it must have been saved with the same model and dtype."""
|
|
382
|
+
from mlx_lm.models.cache import load_prompt_cache
|
|
383
|
+
cache, meta = load_prompt_cache(path, return_metadata=True)
|
|
384
|
+
if meta.get("repo") != self.repo:
|
|
385
|
+
raise ValueError(f"{path} was saved with {meta.get('repo')}, not {self.repo}")
|
|
386
|
+
saved = meta.get("dtype", "bfloat16") # no field: saved before float32 became the default
|
|
387
|
+
if saved != self.dtype:
|
|
388
|
+
raise ValueError(f"{path} was saved in {saved}, not {self.dtype}: a mixed-dtype cache reads wrong")
|
|
389
|
+
self.states[key] = {"cache": cache, "tokens": int(meta["tokens"]), "tail": meta["tail"],
|
|
390
|
+
"close": meta.get("close", "")}
|
|
391
|
+
return self.states[key]["tokens"]
|
|
392
|
+
|
|
393
|
+
def decide(self, state=None, questions=None, *, key=None, orders=None, raw=False):
|
|
394
|
+
"""The Jev response for one request (see the module doc): on `state`, read now and dropped after, or on the
|
|
395
|
+
state kept under `key` by prefill() / load() — exactly one of the two. orders: this request's, else the gem's.
|
|
396
|
+
raw: every answer also carries "logprobs", the option log-probabilities before calibration (fit_calibration)."""
|
|
397
|
+
if (state is None) == (key is None):
|
|
398
|
+
raise ValueError("give either state or key")
|
|
399
|
+
if orders is None:
|
|
400
|
+
orders = self.orders
|
|
401
|
+
orders_of(2, orders)
|
|
402
|
+
options = {qid: options_of(q) for qid, q in (questions or {}).items()} # a bad question fails before the model
|
|
403
|
+
if key is None:
|
|
404
|
+
cache, used, tail, close = self._read(state)
|
|
405
|
+
else:
|
|
406
|
+
if self.model is None:
|
|
407
|
+
raise RuntimeError("the model is unloaded")
|
|
408
|
+
kept = self.states[key]
|
|
409
|
+
cache, used, tail, close = kept["cache"], kept["tokens"], kept["tail"], kept["close"]
|
|
410
|
+
answers = {}
|
|
411
|
+
for qid, question in (questions or {}).items():
|
|
412
|
+
items = options[qid]
|
|
413
|
+
n = len(items)
|
|
414
|
+
logprobs, picks = [0.0] * n, []
|
|
415
|
+
read = orders_of(n, orders)
|
|
416
|
+
for order in read:
|
|
417
|
+
suffix = self._encode(close + question_text(question, [items[i] for i in order]) + tail)
|
|
418
|
+
used += len(suffix)
|
|
419
|
+
logits = self.model(self.mx.array(suffix)[None], cache=copy.deepcopy(cache))[0, -1]
|
|
420
|
+
z = [logits[t].item() for t in self.letter_ids[:n]]
|
|
421
|
+
lse = max(z) + math.log(sum(math.exp(x - max(z)) for x in z))
|
|
422
|
+
for pos, i in enumerate(order):
|
|
423
|
+
logprobs[i] += (z[pos] - lse) / len(read)
|
|
424
|
+
picks.append(order[max(range(n), key=lambda pos: z[pos])])
|
|
425
|
+
answers[qid] = answer(question, items, logprobs, len(set(picks)) == 1, self.calibration.get(qid), raw)
|
|
426
|
+
return {"model": f"decision-gem/{self.repo.rstrip('/').split('/')[-1]}", "answers": answers,
|
|
427
|
+
"usage": {"input_tokens": used, "output_tokens": 0}}
|
|
428
|
+
|
|
429
|
+
def generate(self, prompt, state=None, *, key=None, max_tokens=512, temperature=0.0):
|
|
430
|
+
"""The model as an ordinary chat model: its answer to `prompt`, generated. With a `state` (read now) or the
|
|
431
|
+
state kept under `key` (prefill / load — not read again; the answer is generated on a copy of its cache, as
|
|
432
|
+
decide() asks), the prompt follows the state in the same user turn, where a question would; with neither, the
|
|
433
|
+
prompt is the whole user turn. temperature 0: greedy, the same prompt gives the same text.
|
|
434
|
+
{"model", "text", "finish_reason": "stop" | "length", "usage": {"input_tokens", "output_tokens"}}."""
|
|
435
|
+
from mlx_lm import stream_generate
|
|
436
|
+
from mlx_lm.models.cache import make_prompt_cache
|
|
437
|
+
from mlx_lm.sample_utils import make_sampler
|
|
438
|
+
if not isinstance(prompt, str) or not prompt.strip():
|
|
439
|
+
raise ValueError("prompt must be a non-empty string")
|
|
440
|
+
if state is not None and key is not None:
|
|
441
|
+
raise ValueError("give state or key, not both")
|
|
442
|
+
if self.model is None:
|
|
443
|
+
raise RuntimeError("the model is unloaded")
|
|
444
|
+
if key is not None:
|
|
445
|
+
kept = self.states[key]
|
|
446
|
+
cache, used = copy.deepcopy(kept["cache"]), kept["tokens"]
|
|
447
|
+
tokens = self._encode(kept["close"] + prompt.strip() + kept["tail"]) # an open state closes here
|
|
448
|
+
elif state is not None:
|
|
449
|
+
cache, used, tail, _ = self._read(state)
|
|
450
|
+
tokens = self._encode(prompt.strip() + tail)
|
|
451
|
+
else:
|
|
452
|
+
text = self.tok.apply_chat_template([{"role": "user", "content": prompt.strip()}],
|
|
453
|
+
add_generation_prompt=True, tokenize=False, enable_thinking=False)
|
|
454
|
+
cache, used, tokens = make_prompt_cache(self.model), 0, self._encode(text)
|
|
455
|
+
parts, last = [], None
|
|
456
|
+
for last in stream_generate(self.model, self.tok, self.mx.array(tokens), max_tokens=max_tokens,
|
|
457
|
+
sampler=make_sampler(temp=temperature), prompt_cache=cache):
|
|
458
|
+
parts.append(last.text)
|
|
459
|
+
return {"model": f"decision-gem/{self.repo.rstrip('/').split('/')[-1]}", "text": "".join(parts).strip(),
|
|
460
|
+
"finish_reason": last.finish_reason if last else "length",
|
|
461
|
+
"usage": {"input_tokens": used + len(tokens), "output_tokens": last.generation_tokens if last else 0}}
|
|
462
|
+
|
|
463
|
+
def __call__(self, request):
|
|
464
|
+
"""decide() on a whole Jev request {"state", "questions"}, or {"key", "questions"} on a kept state ("raw": true
|
|
465
|
+
adds the uncalibrated "logprobs");
|
|
466
|
+
generate() on {"prompt", "state" or "key" (optional), "max_tokens", "temperature"}."""
|
|
467
|
+
if "prompt" in request:
|
|
468
|
+
if "questions" in request:
|
|
469
|
+
raise ValueError("a request has questions or a prompt, not both")
|
|
470
|
+
return self.generate(request["prompt"], request.get("state"), key=request.get("key"),
|
|
471
|
+
max_tokens=request.get("max_tokens", 512),
|
|
472
|
+
temperature=request.get("temperature", 0.0))
|
|
473
|
+
return self.decide(request.get("state"), request["questions"], key=request.get("key"),
|
|
474
|
+
raw=request.get("raw", False))
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: decision-gem
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Jev-style decision model (noul / choice / score) on a local Gemma, MLX; prompts too
|
|
5
|
+
Author: tumi
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tumikosha/decision-gem
|
|
8
|
+
Project-URL: Changelog, https://github.com/tumikosha/decision-gem/blob/main/CHANGELOG.md
|
|
9
|
+
Keywords: llm,mlx,gemma,classification,decision-model,jev,apple-silicon
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: MacOS
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: mlx>=0.32
|
|
20
|
+
Requires-Dist: mlx-lm>=0.32
|
|
21
|
+
Requires-Dist: huggingface_hub>=0.30
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: pytest; extra == "test"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# decision-gem
|
|
27
|
+
|
|
28
|
+
A decision model in the style of [TypeSafe Jev](https://developers.cloudflare.com/ai/models/typesafe/jev/) on a local
|
|
29
|
+
Gemma (MLX). It answers three kinds of questions about a text (the "state"): yes / no, a choice among options, a score
|
|
30
|
+
on a scale. No text is generated: the answer is read from the probabilities of the options' letters. Runs offline.
|
|
31
|
+
The same model also answers ordinary prompts by generating (`generate`, see "Prompts").
|
|
32
|
+
|
|
33
|
+
Russian version: [README.md](https://github.com/tumikosha/decision-gem/blob/main/README.md).
|
|
34
|
+
|
|
35
|
+
## What it does
|
|
36
|
+
|
|
37
|
+
Request: `{"state": text or JSON object, "questions": {id: question}}`.
|
|
38
|
+
|
|
39
|
+
| Type | Question | Answer |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| `noul` | `instructions`, `criteria` `{"true": …, "false": …}` (optional) | `noul` = P(true) |
|
|
42
|
+
| `choice` | `instructions`, `criteria` `{option: description}` or `[option, …]`, 2..26 | `choice`, `confidence`, `probabilities` |
|
|
43
|
+
| `score` | `instructions`, `criteria` `{value: description}` or `[description, …]` (levels 1..N), 2..10 | `score` = Σ p·value, `confidence`, `legend`, `probabilities` |
|
|
44
|
+
|
|
45
|
+
Beyond Jev, every answer has `orders_agree`: whether the choice is the same in every option order (below).
|
|
46
|
+
`false` means an unsure answer, worth checking. The whole response: `{"model", "answers": {id: answer},
|
|
47
|
+
"usage": {"input_tokens", "output_tokens": 0}}`.
|
|
48
|
+
|
|
49
|
+
How it works:
|
|
50
|
+
1. The state is read once into a KV cache.
|
|
51
|
+
2. Each question is appended to a copy of the cache, its options under the letters A, B, C …
|
|
52
|
+
3. The logits of those letters at the first answer position are read.
|
|
53
|
+
4. Each question is read in several option orders. Each option's log-probabilities are averaged, so a
|
|
54
|
+
preference for a letter or a position cancels out.
|
|
55
|
+
|
|
56
|
+
### Option orders
|
|
57
|
+
|
|
58
|
+
`DecisionGem(orders=...)`, per request `decide(..., orders=...)`, on the CLI `--orders`:
|
|
59
|
+
|
|
60
|
+
| `orders` | Passes per question | What cancels |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| `"reverse"` (default) | 2: in order and reversed | with 2 options, everything; with 3+ the middle stays in the middle (of 3 options, B is B both times), so a pull to or from the middle letters stays |
|
|
63
|
+
| `"cyclic"` | n: every rotation | each option once under every letter: position cancels fully |
|
|
64
|
+
|
|
65
|
+
For noul (2 options) both are the same two passes.
|
|
66
|
+
|
|
67
|
+
Under `cyclic`, `orders_agree` means all n passes agree, stricter than two; its `false` rate is not comparable
|
|
68
|
+
across modes. Calibration `T` was fitted on `reverse`; fit it again for `cyclic`.
|
|
69
|
+
|
|
70
|
+
### Calibration
|
|
71
|
+
|
|
72
|
+
Raw LLM probabilities are overconfident: `noul` 1.0 does not mean "yes" 100% of the time. Calibration remaps them so
|
|
73
|
+
that 0.8 means "yes" in about 80% of such answers. Each question has its own bias, so the coefficients are kept per
|
|
74
|
+
question id: `[a, b]` for noul (`p = sigmoid(a·logit + b)`, `logit` being the log-probability of true minus that of
|
|
75
|
+
false), `T` for choice / score (`softmax(logprob / T)`, `T > 1` softens).
|
|
76
|
+
|
|
77
|
+
```json
|
|
78
|
+
{"spam": [0.35, -0.8], "kind": 2.1}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Pass it as `DecisionGem(calibration=...)` or `--calibration cal.json`. To fit it:
|
|
82
|
+
|
|
83
|
+
1. Label 100–300 of your own texts by hand: the answer to every question.
|
|
84
|
+
2. Get answers with the raw log-probabilities: `decide(..., raw=True)`, `{"raw": true}` in a request or `--raw` in the
|
|
85
|
+
CLI. Every answer gets `logprobs` {option: log-probability before calibration}.
|
|
86
|
+
3. `fit_calibration(answers, labels)` returns the calibration.
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from decision_gem import DecisionGem, fit_calibration
|
|
90
|
+
|
|
91
|
+
with DecisionGem() as gem:
|
|
92
|
+
answers = [gem.decide(text, qs, raw=True)["answers"] for text in texts]
|
|
93
|
+
cal = fit_calibration(answers, labels) # labels: [{"spam": True, "kind": "phishing"}, ...] in the same order
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
A label is `True` / `False` for noul, the option for choice, the level for score; a question with no label in a text is
|
|
97
|
+
skipped. noul is fitted by Platt scaling (logistic regression on `logit`), `T` by the least log-loss. Both noul labels
|
|
98
|
+
and some model mistakes must be in the set: where the model is always right, `T` goes to 0.01 and `a` grows.
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
## Differences from Jev
|
|
102
|
+
|
|
103
|
+
The request and the response are Jev's (noul / choice / score). Everything under the contract differs. Facts about Jev
|
|
104
|
+
come from the Cloudflare and TypeSafe docs (`docs.typesafe.ai/models.md`), November 2026; they do not disclose how Jev
|
|
105
|
+
works inside.
|
|
106
|
+
|
|
107
|
+
| | Jev (TypeSafe) | decision-gem |
|
|
108
|
+
|---|---|---|
|
|
109
|
+
| Where it runs | a cloud API (Cloudflare Workers AI, TypeSafe `POST /v1/systemone`), $0.042 per 1M input tokens | locally on Apple silicon (MLX), offline, data stays on the machine |
|
|
110
|
+
| Model | its own, trained to return calibrated decisions (RLCD), architecture not disclosed | any instruct LLM in mlx-lm format, see "Other models" |
|
|
111
|
+
| How it computes | not disclosed | the logits of the option letters on a copy of the state's KV cache, averaged over option orders |
|
|
112
|
+
| Calibration | built in by training | fitted on your own labels: `fit_calibration`, see "Calibration" |
|
|
113
|
+
| Question types | noul, choice, score, Box (from Jev 1.14) | noul, choice, score |
|
|
114
|
+
| Images (`assets`) | yes (beta) | no: text or JSON |
|
|
115
|
+
| Limits | 64k tokens per request, 32k per question (262k with a header) | the model's context and memory; choice 2..26 options, score 2..10 levels |
|
|
116
|
+
|
|
117
|
+
Only in decision-gem:
|
|
118
|
+
|
|
119
|
+
- `orders_agree` in every answer and `orders="cyclic"` (see "Option orders");
|
|
120
|
+
- `raw=True`: the raw option `logprobs`, and `fit_calibration` on them;
|
|
121
|
+
- keyed states: `prefill`, `decide(key=...)`, `forget`, `save` / `load`;
|
|
122
|
+
- open states and `extend`: text is read into the cache as it grows;
|
|
123
|
+
- `generate`: the same model answers ordinary prompts;
|
|
124
|
+
- score `criteria` also as `{value: description}`, choice `criteria` also as a list.
|
|
125
|
+
|
|
126
|
+
## Keyed states
|
|
127
|
+
|
|
128
|
+
`decide(state, questions)` reads the state, answers and drops its cache. To ask one email many questions at different
|
|
129
|
+
times, or to switch between emails, read the state once and keep it under a key:
|
|
130
|
+
|
|
131
|
+
| Method | What it does |
|
|
132
|
+
|---|---|
|
|
133
|
+
| `prefill(key, state)` | reads the state into a KV cache and keeps it under `key`; returns the number of tokens |
|
|
134
|
+
| `decide(key=key, questions=qs)` | questions on a kept state; exactly one of `state` / `key`, else `ValueError` |
|
|
135
|
+
| `gem({"key": key, "questions": qs})` | the same as a Jev request |
|
|
136
|
+
| `forget(key)` | frees the state's memory |
|
|
137
|
+
| `save(key, path)` / `load(key, path)` | the state to a `.safetensors` file and back; `load` takes only a file saved with the same model |
|
|
138
|
+
|
|
139
|
+
Each question runs on its own copy of the cache, so the kept state does not change: questions are independent and the
|
|
140
|
+
model does not remember earlier questions (it is not a chat). An answer by key is byte-identical to the answer of
|
|
141
|
+
`decide(state, ...)` — checked on 3 calls in reverse order and after `save` + `load`. Copying the cache is almost
|
|
142
|
+
free: MLX arrays are immutable, the data is shared until a question is appended to the copy.
|
|
143
|
+
|
|
144
|
+
A state file is the email's text in another form: keep it only where the text itself may be kept.
|
|
145
|
+
|
|
146
|
+
### Open states
|
|
147
|
+
|
|
148
|
+
`prefill(key, state, open=True)` stops the cache right after the state text; the closing `</state>` is not read yet.
|
|
149
|
+
`extend(key, text)` reads more text into that cache for good (it returns the tokens added; `ValueError` on a closed
|
|
150
|
+
state), so a growing thread can be kept open and new emails read into it. `decide` on an open state puts `</state>` and the question after the
|
|
151
|
+
cache on every copy and never writes the kept cache. `save` / `load` keep the open flag. `split_open(prompt)` is the
|
|
152
|
+
pure helper that cuts a rendered prompt before the closing tag.
|
|
153
|
+
|
|
154
|
+
Seam rule: `extend` equals one read only when the new text starts at a token boundary. Put the leading space or newline
|
|
155
|
+
in the new text, never split a word, and never end a chunk in whitespace (`extend` raises `ValueError`): `"hello. "` +
|
|
156
|
+
`"Operator"` tokenizes differently from `"hello. Operator"`. `prefill` strips the state, so it never ends in whitespace.
|
|
157
|
+
|
|
158
|
+
Precision: the gem computes in float32 by default (`float32=True`), see "The bf16 bug" below. `extend` equals one
|
|
159
|
+
read only in float32.
|
|
160
|
+
|
|
161
|
+
## Prompts
|
|
162
|
+
|
|
163
|
+
`generate(prompt, state=None, *, key=None, max_tokens=512, temperature=0.0)` runs the model as an ordinary chat model:
|
|
164
|
+
the answer to the prompt is generated (`mlx_lm.stream_generate`) by the model already loaded.
|
|
165
|
+
|
|
166
|
+
| Call | What the model sees |
|
|
167
|
+
|---|---|
|
|
168
|
+
| `generate(prompt)` | the prompt only |
|
|
169
|
+
| `generate(prompt, state)` | the state, then the prompt where a question would go |
|
|
170
|
+
| `generate(prompt, key=key)` | the kept state (`prefill` / `load`), not read again; generated on a copy of its cache |
|
|
171
|
+
| `gem({"prompt": …, "state" or "key", "max_tokens", "temperature"})` | the same as a request; the CLI takes this JSON too |
|
|
172
|
+
|
|
173
|
+
Response: `{"model", "text", "finish_reason": "stop" | "length", "usage": {"input_tokens", "output_tokens"}}`.
|
|
174
|
+
`temperature` 0 is greedy: the same prompt gives the same text. Generating leaves a kept state as it was:
|
|
175
|
+
`decide(key=…)` before and after `generate(key=…)` gives byte-identical answers (the `-m models` test); the answer on a
|
|
176
|
+
key equals the answer on the same `state`.
|
|
177
|
+
|
|
178
|
+
Limits: the answer is not constrained to a JSON schema (mlx-lm has no structured output) — check JSON and ask again
|
|
179
|
+
on an error; do not run beside Ollama with the same Gemma (memory).
|
|
180
|
+
|
|
181
|
+
## The bf16 bug: noisy answers on long states (found 2026-10-19)
|
|
182
|
+
|
|
183
|
+
**Symptom.** The same text gave a different answer depending on how it was read: whole, in chunks with `extend`,
|
|
184
|
+
or with `</state>` in another place. Up to ~460 tokens the answers agreed, beyond that they drifted:
|
|
185
|
+
|
|
186
|
+
| State tokens | Whole | In chunks (`extend`) |
|
|
187
|
+
|---|---|---|
|
|
188
|
+
| ≤ 462 | same | same |
|
|
189
|
+
| 692 | 0.75 | 0.37 |
|
|
190
|
+
| ~1400 | 0.45 | 0.22 |
|
|
191
|
+
|
|
192
|
+
Open vs closed state on one long text: 0.45 vs 0.61. One 1590-token text read in different ways gave `noul` from
|
|
193
|
+
0.14 to 0.65: at a 0.5 threshold the yes / no answer depended on the way of reading, not on the text.
|
|
194
|
+
|
|
195
|
+
**Cause.** bf16 compute flips the expert choice of the MoE router. bf16 rounding depends on how many tokens one
|
|
196
|
+
read holds. Each layer's router makes a hard top-8 expert pick (`argpartition`). When the 8th and 9th expert scores
|
|
197
|
+
are nearly equal (at layer 8 the gap was 2.8e-5), rounding flips the pick, and the difference grows through the
|
|
198
|
+
later layers. The norms and the quantization scales and biases were bf16; the weights themselves are 4-bit.
|
|
199
|
+
|
|
200
|
+
**Fix.** `DecisionGem(..., float32=True)` by default. After loading, every bf16 parameter
|
|
201
|
+
(quantization scales and biases, norms, embeddings, router) is cast to float32; the packed 4/8-bit weights stay as
|
|
202
|
+
they are. In float32 the spread is 0.568..0.574 and `extend` matches one read to within 0.006 (not bit-exact).
|
|
203
|
+
`float32=False` keeps the old behaviour.
|
|
204
|
+
|
|
205
|
+
Cost: +1.5 GiB for the model (13.2 → 14.7 GiB), twice the KV cache (≈0.43 MiB per token), reads about 1.3x slower,
|
|
206
|
+
a short +1.5 GiB at load. The "Resources" numbers were measured in bf16.
|
|
207
|
+
|
|
208
|
+
**Saved states.** `save` records the compute dtype (`dtype`), `load` raises `ValueError` when it
|
|
209
|
+
differs from the gem's. A file without the field counts as bf16 (saved before the fix). Without the check a bf16
|
|
210
|
+
state loaded silently into a float32 gem and gave a mixed cache with the same noise.
|
|
211
|
+
|
|
212
|
+
**What it touched.**
|
|
213
|
+
- Everything computed before 2026-10-19 (before version 0.1.0) ran in bf16.
|
|
214
|
+
- Operator / client roles on 145 calls (state = first 20 lines): in float32 no call changed its operator, accuracy
|
|
215
|
+
is the same; 163 s vs 127 s. On short states the noise does not show.
|
|
216
|
+
- Answers on long calls (the checklist, the whole call in the state) may have depended on the noise. Calibrations
|
|
217
|
+
fitted on bf16 answers need refitting.
|
|
218
|
+
|
|
219
|
+
**TF32 on M5 (found 2026-10-19).** Cause: on GPUs with neural accelerators (M5) MLX multiplies float32 matrices in
|
|
220
|
+
TF32 (relative error ~5e-4 instead of ~1e-6) from 128 rows up (dense: from 2), with another kernel below that. So the
|
|
221
|
+
same tokens get different rounding depending on how many one read holds, and the router's near-ties flip as in bf16.
|
|
222
|
+
Symptom on an M5 Max: 0.347 in chunks vs 0.546 whole. `DecisionGem(float32=True)` sets `MLX_ENABLE_TF32=0` and
|
|
223
|
+
checks a float32 matmul against the CPU (`float32_matmuls`); MLX reads the variable once, at the process's first
|
|
224
|
+
matmul, so if one already ran with TF32 on the gem raises `RuntimeError`. With it: 0.5739 in chunks vs 0.5741 whole.
|
|
225
|
+
Cost: reads 2x slower than in TF32 (1594 tokens 1.6 s vs 0.8 s, a question 0.30 s vs 0.18 s).
|
|
226
|
+
|
|
227
|
+
**Check.** `tests/test_open_model.py` (with the model): a long state in chunks equals one read, open equals closed,
|
|
228
|
+
tolerance 0.02. In bf16 these two tests fail (0.2306 and 0.1544). Model-free tests in `tests/test_core.py` cover the
|
|
229
|
+
cast and `dtype` in `save` / `load`.
|
|
230
|
+
|
|
231
|
+
## Where it was used
|
|
232
|
+
|
|
233
|
+
Speech analytics of call-centre calls, model `mlx-community/gemma-4-26b-a4b-it-4bit`.
|
|
234
|
+
|
|
235
|
+
- Operator / client roles: the right operator in 99 of the 100 calls of the roles reference.
|
|
236
|
+
- Checklist item statuses, 199 calls, 2250 items: 7.1 s per call, 9 of 9 manual rulings matched, the option orders
|
|
237
|
+
agree on 2094 of 2250.
|
|
238
|
+
- The full checklist with no analysis prompt: scenario, outcome, call flags, then the items; 199 calls, 15.5 s per
|
|
239
|
+
call, 9 of 9 manual rulings.
|
|
240
|
+
|
|
241
|
+
## Resources
|
|
242
|
+
|
|
243
|
+
Measured on the longest call (3051 tokens of state) and on 199 calls, 2026-10-13, an Apple silicon Mac with 36 GB:
|
|
244
|
+
|
|
245
|
+
| What | Time / memory |
|
|
246
|
+
|---|---|
|
|
247
|
+
| model load | about 6 s, 13.2 GiB |
|
|
248
|
+
| reading a state (`prefill` or the start of `decide`) | 4.2 s for 3051 tokens (0.8 s for 882) |
|
|
249
|
+
| a state's KV cache | ≈0.2 MiB per token: 656 MiB for 3051 tokens; 3 calls of 882–2430 tokens — 1.11 GiB |
|
|
250
|
+
| cache copy per question | 0.1 ms |
|
|
251
|
+
| one question (two passes, options in order and reversed) | 221 ms |
|
|
252
|
+
| generation (`generate`) | 86–97 tokens/s; a call's summary (653 state tokens, 71 answer tokens) in 0.7 s (16.10) |
|
|
253
|
+
| `save` / `load` of a state | 0.34 s / 0.02 s, file 656 MiB |
|
|
254
|
+
| peak memory on long calls | about 17 GiB (27.8 GiB with the MLX buffer cache) |
|
|
255
|
+
| a call in the runs above | 2–30 s, depends on the number of questions |
|
|
256
|
+
|
|
257
|
+
On a 36 GB Mac do not run it next to another large model. The GPU gets about 27 GiB: besides the model, about 15
|
|
258
|
+
kept long calls fit in memory; more through `save` / `load`.
|
|
259
|
+
|
|
260
|
+
## Memory
|
|
261
|
+
|
|
262
|
+
- `cache_limit` (default 0) — bytes of freed buffers MLX keeps for reuse (`mx.set_cache_limit`); `None` — leave the
|
|
263
|
+
MLX limit as it is. With the buffers the peak was 27.8 GiB instead of 17. The limit is process-wide: `unload()`
|
|
264
|
+
puts back the previous one (the MLX default is 34.2 GiB on this Mac).
|
|
265
|
+
- `gem.unload()` frees the model and all kept states (13.2 GiB → 0) without ending the process; `decide()` after it
|
|
266
|
+
raises `RuntimeError`. Or `with DecisionGem() as gem: ...` — unloaded on exit.
|
|
267
|
+
- `forget(key)` — one state's memory (3 calls: 1.11 GiB, after `forget` of one 0.59 GiB).
|
|
268
|
+
|
|
269
|
+
## Other models
|
|
270
|
+
|
|
271
|
+
The model is set by `DecisionGem(repo=...)` or `--model`: an HF repo id or a local folder in mlx-lm format.
|
|
272
|
+
|
|
273
|
+
```python
|
|
274
|
+
gem = DecisionGem("mlx-community/Qwen3-14B-4bit")
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
```bash
|
|
278
|
+
decision-gem request.json --model mlx-community/Qwen3-14B-4bit
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
What a model needs:
|
|
282
|
+
|
|
283
|
+
- mlx-lm format (`mlx_lm.load` loads it), Apple silicon;
|
|
284
|
+
- instruction-tuned, with a chat template;
|
|
285
|
+
- every letter A..Z after the answer template is exactly one token. Checked at load (`letter_ids`): otherwise
|
|
286
|
+
`ValueError` before the first question;
|
|
287
|
+
- the first answer token is a letter, not the start of reasoning. The template gets `enable_thinking=False`. A model
|
|
288
|
+
that always reasons is read wrong, and the check does not catch it.
|
|
289
|
+
|
|
290
|
+
**Small models.** Models up to ~8B parameters, with no fine-tuning and no calibration, answer poorly. In JevBench
|
|
291
|
+
v1.6.0 (Benchmark Heaven, 2026-10-05) raw Qwen3 with a logit readout scored a Capability of 34.6 (4B Instruct),
|
|
292
|
+
30.1 (8B), 8.3 (1.7B) and 8.2 (0.6B); frozen Gemma-4-12B-it and Gemma-4-31B-it scored 70.9 and 77.6, Jev 1.13.0 76.5.
|
|
293
|
+
This covers `gemma-4-e2b` and `gemma-4-e4b` in "Model comparison" too: the tests on them check the package, not the
|
|
294
|
+
quality of the answers. Take a small model only with a calibration fitted on your own labels and its accuracy checked
|
|
295
|
+
on them.
|
|
296
|
+
|
|
297
|
+
Calibration (`[a, b]`, `T`) is per model. The numbers in this README were measured on `gemma-4-26b-a4b-it-4bit`.
|
|
298
|
+
|
|
299
|
+
### Model comparison
|
|
300
|
+
|
|
301
|
+
Measured on 2026-11-13, an Apple silicon Mac with 36 GB, `float32=True`, `orders="reverse"`. Set: 3 short calls,
|
|
302
|
+
6 questions each — 3 noul, 2 choice, 1 score, 18 in all. Tests: `DECISION_GEM_TEST_MODEL=<repo> pytest -m models`.
|
|
303
|
+
|
|
304
|
+
| Model | `-m models` tests | Load | 18 questions |
|
|
305
|
+
|---|---|---|---|
|
|
306
|
+
| `mlx-community/gemma-4-e2b-it-4bit` | 6 of 6 | 1.4 s | 1.7 s |
|
|
307
|
+
| `mlx-community/gemma-4-e4b-it-4bit` | 6 of 6 | 1.8 s | 3.1 s |
|
|
308
|
+
| `mlx-community/Qwen3-14B-4bit` | 6 of 6 | 3.1 s | 9.2 s |
|
|
309
|
+
| `mlx-community/gemma-4-26b-a4b-it-4bit` (default) | 6 of 6 | 4.6 s | 6.7 s |
|
|
310
|
+
|
|
311
|
+
Gemma 26B is an MoE (~4B active parameters), so it answers faster than the dense Qwen3-14B.
|
|
312
|
+
|
|
313
|
+
## Install and run
|
|
314
|
+
|
|
315
|
+
Apple silicon only (MLX). The model must already be in the HF cache, or run with `--online` / `offline=False`.
|
|
316
|
+
|
|
317
|
+
```bash
|
|
318
|
+
pip install decision-gem # or from source: pip install -e ".[test]"
|
|
319
|
+
python -m decision_gem request.json # or: decision-gem request.json; - reads stdin
|
|
320
|
+
python -m decision_gem request.json --calibration cal.json --model <repo or folder>
|
|
321
|
+
pytest # tests without the model; pytest -m models — on the model
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
```python
|
|
325
|
+
from decision_gem import DecisionGem
|
|
326
|
+
|
|
327
|
+
email = {"from": '"Secure Bank" <security@secure-bank-verify.top>', # the state: text or a JSON object
|
|
328
|
+
"subject": "Your account has been limited!",
|
|
329
|
+
"body": "Dear customer, we noticed unusual activity. Verify your identity within 24 hours: "
|
|
330
|
+
"http://secure-bank-verify.top/login or your account will be permanently suspended."}
|
|
331
|
+
qs = {"spam": {"type": "noul", "instructions": "Is this email spam or phishing?"},
|
|
332
|
+
"kind": {"type": "choice", "instructions": "What kind of email is it?",
|
|
333
|
+
"criteria": {"phishing": "tries to steal logins or payment data", "ads": "unsolicited advertising",
|
|
334
|
+
"scam": "promises money or prizes", "normal": "an ordinary email"}},
|
|
335
|
+
"urgency": {"type": "score", "instructions": "How hard does the email push the reader to act right away?",
|
|
336
|
+
"criteria": ["no pressure", "some pressure", "strong pressure"]}}
|
|
337
|
+
|
|
338
|
+
gem = DecisionGem() # mlx-community/gemma-4-26b-a4b-it-4bit from the local HF cache
|
|
339
|
+
gem({"state": email, "questions": qs})
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
The answer (Gemma 26B, 474 input tokens; `legend` and `probabilities` cut):
|
|
343
|
+
|
|
344
|
+
```json
|
|
345
|
+
{"spam": {"type": "noul", "noul": 1.0, "orders_agree": true},
|
|
346
|
+
"kind": {"type": "choice", "choice": "phishing", "confidence": 1.0, "orders_agree": true},
|
|
347
|
+
"urgency": {"type": "score", "score": 3.0, "confidence": 1.0, "orders_agree": true}}
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
On an ordinary email from a colleague about meeting notes the same questions give `spam` 0.0, `kind` `normal`,
|
|
351
|
+
`urgency` 1.01.
|
|
352
|
+
|
|
353
|
+
Several states at once (see "Keyed states"):
|
|
354
|
+
|
|
355
|
+
```python
|
|
356
|
+
with DecisionGem() as gem:
|
|
357
|
+
gem.prefill("mail-1", email1) # number of tokens; ≈0.2 MiB of memory per token until forget / unload
|
|
358
|
+
gem.prefill("mail-2", email2)
|
|
359
|
+
gem.decide(key="mail-2", questions=qs) # the same answer as decide(email2, qs), without reading again
|
|
360
|
+
gem({"key": "mail-1", "questions": qs}) # a Jev request with key instead of state
|
|
361
|
+
gem.save("mail-1", "mail-1.safetensors") # to disk; load() reads it back in ~0.02 s
|
|
362
|
+
gem.forget("mail-1")
|
|
363
|
+
gem.load("mail-1", "mail-1.safetensors") # only a file saved with the same model
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
The full contract is also in the docstring of `decision_gem/core.py`.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
decision_gem/__init__.py,sha256=8C-Fx2rj63CL4FSQmyh1H0w308m-kM4vk7thhf3oza8,357
|
|
2
|
+
decision_gem/__main__.py,sha256=hStFmQH2-YQBReuhsUA_Ur3wePV1nSlFsnsvl5t0tXA,1635
|
|
3
|
+
decision_gem/core.py,sha256=lzpF96nbfXx7aWvpy_4VsVIzhJqq69gvz06UeJrITPY,26398
|
|
4
|
+
decision_gem-0.1.0.dist-info/licenses/LICENSE,sha256=vQmaftqGH8Wv0ndMxkhrA_1B_OBEEnyXSW8iB8nvF68,1061
|
|
5
|
+
decision_gem-0.1.0.dist-info/METADATA,sha256=Q18Rpj51_VyL_K4TJ9ZLlbJxXwJJN_XL8UciNptokxM,20675
|
|
6
|
+
decision_gem-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
7
|
+
decision_gem-0.1.0.dist-info/entry_points.txt,sha256=d2iFugdn1_zg28kjjcwhbVbqLY_lKvHmOmkgyTWiyuA,60
|
|
8
|
+
decision_gem-0.1.0.dist-info/top_level.txt,sha256=GP1jgYVnDwRvloZmzCzMV85DjMaGZByhDJZsCV-8Lqk,13
|
|
9
|
+
decision_gem-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 tumi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
decision_gem
|