decideit 7.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentguard/agentguard.py +467 -0
- decideit-7.0.1.dist-info/METADATA +275 -0
- decideit-7.0.1.dist-info/RECORD +60 -0
- decideit-7.0.1.dist-info/WHEEL +5 -0
- decideit-7.0.1.dist-info/entry_points.txt +5 -0
- decideit-7.0.1.dist-info/licenses/LICENSE +21 -0
- decideit-7.0.1.dist-info/top_level.txt +3 -0
- rsijev/__init__.py +21 -0
- rsijev/adaptive.py +715 -0
- rsijev/arch.py +1146 -0
- rsijev/calibrate.py +385 -0
- rsijev/contract.py +130 -0
- rsijev/data.py +243 -0
- rsijev/encode.py +487 -0
- rsijev/evaluate.py +594 -0
- rsijev/exit_policy.py +791 -0
- rsijev/fit.py +1051 -0
- rsijev/frozen_tok.py +65 -0
- rsijev/image_text.py +28 -0
- rsijev/metrics.py +170 -0
- rsijev/mlx/__init__.py +19 -0
- rsijev/mlx/collate.py +79 -0
- rsijev/mlx/convert.py +334 -0
- rsijev/mlx/heads.py +224 -0
- rsijev/mlx/model.py +326 -0
- rsijev/mlx/quant.py +155 -0
- rsijev/mlx/serving.py +492 -0
- rsijev/mlx/text.py +630 -0
- rsijev/mlx/vision.py +327 -0
- rsijev/rl2.py +999 -0
- rsijev/targets.py +123 -0
- rsijev/targets_suite.py +791 -0
- rsijev/train.py +218 -0
- rsijev/vision.py +285 -0
- rsijev/vision_fit.py +122 -0
- serve/__init__.py +1 -0
- serve/accel.py +69 -0
- serve/app.py +342 -0
- serve/autostart.py +226 -0
- serve/batcher.py +317 -0
- serve/bench.py +180 -0
- serve/cli.py +71 -0
- serve/daemon.py +113 -0
- serve/decide_cli.py +222 -0
- serve/decider.py +108 -0
- serve/demo.py +155 -0
- serve/demo_images.py +228 -0
- serve/effort.py +156 -0
- serve/examples.json +623 -0
- serve/images.py +197 -0
- serve/infer.py +973 -0
- serve/kernels.py +100 -0
- serve/padded.py +417 -0
- serve/plan.py +248 -0
- serve/release.py +574 -0
- serve/runtime.py +249 -0
- serve/serve_tune.py +107 -0
- serve/server.py +456 -0
- serve/ui.html +1296 -0
- serve/wire.py +217 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: decideit
|
|
3
|
+
Version: 7.0.1
|
|
4
|
+
Summary: RSI-Jev typed-decision models: a Jev-compatible server, a Python API and the training code
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/Shanghua-Gao/RSI-Jev
|
|
7
|
+
Project-URL: Weights, https://huggingface.co/shgao
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: torch>=2.13; sys_platform != "darwin" or platform_machine != "arm64"
|
|
12
|
+
Requires-Dist: transformers<5.18,>=5.17; sys_platform != "darwin" or platform_machine != "arm64"
|
|
13
|
+
Requires-Dist: mlx>=0.32; sys_platform == "darwin" and platform_machine == "arm64"
|
|
14
|
+
Requires-Dist: tokenizers>=0.22; sys_platform == "darwin" and platform_machine == "arm64"
|
|
15
|
+
Requires-Dist: numpy; sys_platform == "darwin" and platform_machine == "arm64"
|
|
16
|
+
Requires-Dist: pillow; sys_platform == "darwin" and platform_machine == "arm64"
|
|
17
|
+
Requires-Dist: safetensors
|
|
18
|
+
Requires-Dist: huggingface_hub
|
|
19
|
+
Requires-Dist: fastapi
|
|
20
|
+
Requires-Dist: uvicorn
|
|
21
|
+
Provides-Extra: fast
|
|
22
|
+
Requires-Dist: flash-linear-attention<0.6,>=0.5.2; extra == "fast"
|
|
23
|
+
Requires-Dist: fla-core<0.6,>=0.5.2; extra == "fast"
|
|
24
|
+
Provides-Extra: vision
|
|
25
|
+
Requires-Dist: pillow; extra == "vision"
|
|
26
|
+
Requires-Dist: torchvision; extra == "vision"
|
|
27
|
+
Provides-Extra: http
|
|
28
|
+
Requires-Dist: orjson>=3.10; extra == "http"
|
|
29
|
+
Requires-Dist: uvloop>=0.21; sys_platform != "win32" and extra == "http"
|
|
30
|
+
Requires-Dist: httptools>=0.6; extra == "http"
|
|
31
|
+
Provides-Extra: test
|
|
32
|
+
Requires-Dist: pytest; extra == "test"
|
|
33
|
+
Requires-Dist: httpx; extra == "test"
|
|
34
|
+
Provides-Extra: torch
|
|
35
|
+
Requires-Dist: torch>=2.13; extra == "torch"
|
|
36
|
+
Requires-Dist: transformers<5.18,>=5.17; extra == "torch"
|
|
37
|
+
Provides-Extra: mlx
|
|
38
|
+
Requires-Dist: mlx>=0.32; extra == "mlx"
|
|
39
|
+
Requires-Dist: numpy; extra == "mlx"
|
|
40
|
+
Requires-Dist: pillow; extra == "mlx"
|
|
41
|
+
Dynamic: license-file
|
|
42
|
+
|
|
43
|
+
<h1 align="center">RSI-Jev</h1>
|
|
44
|
+
|
|
45
|
+
<p align="center">
|
|
46
|
+
<b>A recursively self-improving research loop that builds
|
|
47
|
+
<a href="https://docs.typesafe.ai/api">Jev</a>-style <i>System One</i> decision models.</b>
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
<p align="center">
|
|
51
|
+
<b><a href="https://shanghua-gao.github.io/RSI-Jev/">🌐 shanghua-gao.github.io/RSI-Jev</a></b>
|
|
52
|
+
</p>
|
|
53
|
+
|
|
54
|
+
<p align="center">
|
|
55
|
+
<a href="https://huggingface.co/shgao">🤗 Models</a> ·
|
|
56
|
+
<a href="docs/inference.md">Docs</a> ·
|
|
57
|
+
<a href="EXPLORE.md">Every experiment</a> ·
|
|
58
|
+
<a href="versions/v7.0-vl-3b.md">Release notes</a>
|
|
59
|
+
</p>
|
|
60
|
+
|
|
61
|
+
<p align="center">
|
|
62
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/code-MIT-black" alt="MIT"></a>
|
|
63
|
+
<a href="https://huggingface.co/shgao/rsi-jev-v7.0-vl-4b"><img src="https://img.shields.io/badge/weights-Apache--2.0-black" alt="Apache-2.0 weights"></a>
|
|
64
|
+
<a href="serve/README.md"><img src="https://img.shields.io/badge/API-Jev%20compatible-black" alt="Jev-compatible API"></a>
|
|
65
|
+
</p>
|
|
66
|
+
|
|
67
|
+
AI agents propose the hypotheses, register their predictions before spending GPU time, run the
|
|
68
|
+
experiments, and retire their own champions when the evidence says to. Every release, and every
|
|
69
|
+
experiment that failed on the way, is published with its numbers. The loop is the next version of
|
|
70
|
+
[AutoScientists](https://github.com/mims-harvard/AutoScientists).
|
|
71
|
+
|
|
72
|
+
- **9 releases in 41 days**, from v1.0 to v7.0-VL, each trained, evaluated and documented by the loop.
|
|
73
|
+
- **801 experiments**, every one written up, failures included: [EXPLORE.md](EXPLORE.md).
|
|
74
|
+
- **v6.1-VL 27B has the highest score on the Decision Index 0.3 public set: 65.64**, with `--effort auto`. The official score is pending.
|
|
75
|
+
- **v7.0-VL 4B scores 60.63 on the public Decision Index 0.3**, up from 50.98 for v6.1-VL 4B.
|
|
76
|
+
- **v7.0-VL 3B**, the 4B cut at layer 20, scores 60.01 at two thirds of its latency.
|
|
77
|
+
|
|
78
|
+
<p align="center">
|
|
79
|
+
<picture>
|
|
80
|
+
<source media="(prefers-color-scheme: dark)" srcset="assets/board-v6.1-27b-dark.svg">
|
|
81
|
+
<img src="assets/board-v6.1-27b-light.svg" width="760" alt="Decision Index 0.3 public set: RSI-Jev v6.1-VL 27B 65.64, Torchcast Decision 27B 65.10, Perplexity Decider v1.1 (27B) 62.25, Cloudflare clef 61.71, Jev 57.96, RSI-Jev v6.1-VL 4B 50.98">
|
|
82
|
+
</picture>
|
|
83
|
+
</p>
|
|
84
|
+
|
|
85
|
+
## What it builds
|
|
86
|
+
|
|
87
|
+
Models that answer a yes/no, pick-one-of-*k* or rate-on-a-rubric question about a document, a chat
|
|
88
|
+
or an image. One forward pass returns a calibrated probability for every option. Nothing is
|
|
89
|
+
generated, so there are no reasoning tokens to spend.
|
|
90
|
+
|
|
91
|
+
What v7.0-VL can spend is depth. It has answer heads at layers 16, 20 and 32, and `effort` picks
|
|
92
|
+
how many layers a request uses:
|
|
93
|
+
|
|
94
|
+
| effort | layers | median / mean latency | use it for |
|
|
95
|
+
|---|---|---|---|
|
|
96
|
+
| `low` | 16 | 24 / 65 ms | intent, routing, retrieval |
|
|
97
|
+
| `medium` | 20 | 28 / 78 ms | classification and judgement |
|
|
98
|
+
| `high` | 32 | 42 / 117 ms | knowledge, multi-step reasoning, code |
|
|
99
|
+
| `auto` | 16, 20 or 32 | 41 / 85 ms | mixed traffic: stops at the first layer that is confident enough |
|
|
100
|
+
|
|
101
|
+
v7.0-VL 3B is the same model cut at layer 20: `low` stops at 16, `medium` and `high` at 20.
|
|
102
|
+
v6.1-VL 27B has the same levels at layers 48, 56 and 64.
|
|
103
|
+
|
|
104
|
+
Fast when it's easy, deep when it's hard. Which tasks need which depth:
|
|
105
|
+
[docs/effort-depth.md](docs/effort-depth.md).
|
|
106
|
+
|
|
107
|
+
<p align="center">
|
|
108
|
+
<picture><source media="(prefers-color-scheme: dark)" srcset="site/assets/v4/capsules-dark.webp"><img src="site/assets/v4/capsules-light.webp" width="49%" alt="A tray of gel capsules, one leaking: defective, 68% sure"></picture>
|
|
109
|
+
<picture><source media="(prefers-color-scheme: dark)" srcset="site/assets/v4/keep-file-dark.webp"><img src="site/assets/v4/keep-file-light.webp" width="49%" alt="A delete-file dialog; the user said keep the file: Cancel, 99% sure"></picture>
|
|
110
|
+
<br><sub>Capsules photo from VisA (Zou et al. 2022, CC BY 4.0, resized).</sub>
|
|
111
|
+
</p>
|
|
112
|
+
|
|
113
|
+
## Quickstart
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
pip install "rsi-jev[fast,vision] @ git+https://github.com/Shanghua-Gao/RSI-Jev"
|
|
117
|
+
rsi-jev serve v7.0-vl-4b --effort auto --port 8000
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
`rsi-jev serve v7.0-vl-3b` serves the 3B (6.8 GB). On a 96 GB GPU, `rsi-jev serve v6.1-vl-27b --effort auto` serves the 27B.
|
|
121
|
+
|
|
122
|
+
On an Apple Silicon Mac (the 3B runs in about 2 GB) or on Linux, nothing else needs to be installed first:
|
|
123
|
+
`curl -fsSL https://github.com/Shanghua-Gao/RSI-Jev/releases/latest/download/install.sh | sh`, then
|
|
124
|
+
`git diff | decide "Does this change touch authentication?"`. Building on it: [decideit](docs/decideit.md);
|
|
125
|
+
on a Mac, see also [the Mac guide](docs/mac.md).
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
curl localhost:8000/v1/systemone -H 'Content-Type: application/json' -d '{
|
|
129
|
+
"model": "jev-latest",
|
|
130
|
+
"state": "I was charged twice. Please refund.",
|
|
131
|
+
"questions": {"refund": {"type": "noul", "instructions": "Does the user request a refund?"}}
|
|
132
|
+
}'
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Or in Python, with no server:
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
from rsijev import Decider
|
|
139
|
+
|
|
140
|
+
d = Decider("v7.0-vl-4b")
|
|
141
|
+
d.decide("Customer photo: <image>\nThe customer says it arrived damaged.",
|
|
142
|
+
{"damaged": {"type": "noul", "instructions": "Does the photo show visible damage?"}},
|
|
143
|
+
images=["photo.jpg"])
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
The server speaks Jev's API, so an existing Jev client works once its base URL points here. Every
|
|
147
|
+
response reports which layer answered and how confident it was. Options and limits:
|
|
148
|
+
[`serve/README.md`](serve/README.md); setups and what each costs: [`docs/inference.md`](docs/inference.md).
|
|
149
|
+
|
|
150
|
+
## Releases
|
|
151
|
+
|
|
152
|
+
| Model | Input | Decision Index 0.3 | Decision Index 0.2.1 | Held-out set | |
|
|
153
|
+
|---|---|---|---|---|---|
|
|
154
|
+
| **v7.0-VL-4B** | text, images | 60.63 | 59.63 | **0.811** | [🤗](https://huggingface.co/shgao/rsi-jev-v7.0-vl-4b) |
|
|
155
|
+
| v7.0-VL-3B | text, images | 60.01 | 58.84 | 0.797 | [🤗](https://huggingface.co/shgao/rsi-jev-v7.0-vl-3b) |
|
|
156
|
+
| v6.1-VL-27B, `auto` | text, images | **65.64** | **64.92** | 0.808 | [🤗](https://huggingface.co/shgao/rsi-jev-v6.1-vl-27b) |
|
|
157
|
+
| v6.1-VL-4B | text, images | 50.98 | 50.74 | 0.729 | [🤗](https://huggingface.co/shgao/rsi-jev-v6.1-vl-4b) |
|
|
158
|
+
| v6.0-VL-4B | text, images | 46.23 | 46.24 | 0.698 | [🤗](https://huggingface.co/shgao/rsi-jev-v6.0-vl-4b) |
|
|
159
|
+
| v5.0-VL-3B | text, images | – | 38.38 | 0.689 | [🤗](https://huggingface.co/shgao/rsi-jev-v5.0-vl-3b) |
|
|
160
|
+
| v4.0-VL-2B | text, images | – | – | 0.653 | [🤗](https://huggingface.co/shgao/rsi-jev-v4.0-vl-qwen3.5-2b) |
|
|
161
|
+
| v3.0-2B | text | – | – | 0.649 | [🤗](https://huggingface.co/shgao/rsi-jev-v3.0-qwen3.5-2b) |
|
|
162
|
+
| v2.1-2B | text | – | – | 0.633 | [🤗](https://huggingface.co/shgao/rsi-jev-v2.1-qwen3.5-2b) |
|
|
163
|
+
| v2.0-2B | text | – | – | – | [🤗](https://huggingface.co/shgao/rsi-jev-v2.0-qwen3.5-2b) |
|
|
164
|
+
| v1.0-2B | text | – | – | 0.604 | [🤗](https://huggingface.co/shgao/rsi-jev-v1.0-qwen3.5-2b) |
|
|
165
|
+
| v1.0-0.8B | text | – | – | – | [🤗](https://huggingface.co/shgao/rsi-jev-v1.0-qwen3.5-0.8b) |
|
|
166
|
+
|
|
167
|
+
Decision Index is the public benchmark, full runs of the default configuration (the 27B with
|
|
168
|
+
`--effort auto`, which we recommend): the 0.3 public
|
|
169
|
+
index (public set; v6.1-VL 4B scored with the pre-release 0.3 kit) and 0.2.1. The held-out set
|
|
170
|
+
is our zero-shot check: no release trained on it. The 4B and smaller are fine-tuned from Qwen3.5
|
|
171
|
+
Base; v6.0-VL and v6.1-VL 4B run the whole Qwen3.5-4B-Base and can answer at layer 16, 20 or 32
|
|
172
|
+
of it. v7.0-VL 4B starts from the instruction-tuned Qwen3.5-4B, repeats the options and reads
|
|
173
|
+
them with attention that runs both ways over the prompt; it answers at layer 16, 20 or 32 the same
|
|
174
|
+
way. v7.0-VL 3B is v7.0-VL 4B cut at layer 20, with no retraining. v6.1-VL 27B is fine-tuned from Qwen3.8-27B and answers at layer 48, 56 or 64. Every
|
|
175
|
+
number, caveat and per-benchmark score is in the [release records](versions/).
|
|
176
|
+
|
|
177
|
+
- **2026-11-06 · v7.0-VL 3B** is v7.0-VL 4B cut at layer 20, no retraining: the 4B's answers at `medium`, 3.30B parameters. Decision Index 0.3 60.01.
|
|
178
|
+
- **2026-11-05 · v7.0-VL 4B** reads the options in a new way (option echo, prefix-bidirectional attention) and is trained once on 2.76M examples. Decision Index 0.3 50.98 → 60.63.
|
|
179
|
+
- **2026-10-08 · v6.1-VL 27B** brings the same method to Qwen3.8-27B. Decision Index 0.3 public set 65.64 with `auto`.
|
|
180
|
+
- **2026-10-07 · v6.1-VL 4B** averages v6.0-VL with a second fine-tune of the same base. Decision Index 0.3 46.23 → 50.98.
|
|
181
|
+
- **2026-10-06 · v6.0-VL 4B** chooses its depth per question. Decision Index 38.38 → 46.24.
|
|
182
|
+
- **2026-10-02 · v5.0-VL 3B** cuts the model to the first 20 of 32 layers and says "unknown" when a question has no answer.
|
|
183
|
+
- **2026-10-01 · v4.0-VL** reads images, with text held at v3.0's level.
|
|
184
|
+
- **2026-09-28 · v3.0** is the first release where reinforcement learning helps.
|
|
185
|
+
|
|
186
|
+
## How it got here
|
|
187
|
+
|
|
188
|
+
<p align="center">
|
|
189
|
+
<img src="assets/loop-social.gif" width="680" alt="The champion line climbs from v1.0 (0.622) to v2.0 (0.709), v2.1 (0.736) and v3.0 (0.756) on the 15-benchmark suite; grey dots are the experiments that did not clear it">
|
|
190
|
+
<br><sub>The loop's first three releases: each grey dot is an experiment that did not beat the champion.</sub>
|
|
191
|
+
</p>
|
|
192
|
+
|
|
193
|
+
The experiments that made a release are in [docs/history.md](docs/history.md); all the others,
|
|
194
|
+
with why each failed, are in [EXPLORE.md](EXPLORE.md); the reinforcement-learning arms are in
|
|
195
|
+
[docs/rl.md](docs/rl.md).
|
|
196
|
+
|
|
197
|
+
<details>
|
|
198
|
+
<summary><b>Each experiment that moved a release, from v1.0 to v7.0-VL</b></summary>
|
|
199
|
+
|
|
200
|
+
| where it went | result | what it established |
|
|
201
|
+
|---|---|---|
|
|
202
|
+
| **v1.0** | typed-decisions **0.662** | the bar to clear, and it had never seen a benchmark's train split |
|
|
203
|
+
| train on the benchmark's own train split | typed-decisions 0.7794 | worth +0.12 — and it costs general knowledge, so it was **rejected** |
|
|
204
|
+
| the same + 15% general-knowledge replay | typed-decisions 0.796 | the replay pays that cost back. The first keeper |
|
|
205
|
+
| the other benchmarks' train splits too, 3k per source | suite 0.7049 | the lever is not specific to one benchmark |
|
|
206
|
+
| a per-question confidence head, forbidden to sharpen score questions | suite ECE 0.0736 | confidence that means something, and it never changes which option wins. Shipped |
|
|
207
|
+
| **v2.0** | suite **0.7056**, ECE **0.0546** | capability was v1.0's lever; **what it trains on, and calibrating it afterwards, is a second one** — and free at inference. But the gain was on the benchmarks it had trained for |
|
|
208
|
+
| the replay weighted toward science, half of it recast to ten options | MMLU-Pro 0.331 | more and better-shaped data barely moves general knowledge: +0.008 |
|
|
209
|
+
| **freeze** the tower's lower third instead | MMLU-Pro 0.339 | holding those layers still does recover some of it — and breaks three of the decision benchmarks, because they do need to adapt |
|
|
210
|
+
| the bottom 8 layers at **one tenth** the learning rate | MMLU-Pro 0.359 | it was never a data problem. Fitting decisions through the whole tower at one rate was overwriting what the knowledge sat in — frozen layers cannot adapt, slow ones can |
|
|
211
|
+
| **v2.1** | suite **0.7291**, MMLU-Pro **0.383** | and it travels: ahead of v1.0 on all three benchmarks it never trained on, where v2.0 was not |
|
|
212
|
+
| wider coverage (data-scale-2): +84k questions, emotion/SST-5/ANLI and more tasksource and Open-Jev train items | suite +0.011 / +0.013 on two seeds | coverage is still the lever that moves the suite |
|
|
213
|
+
| a two-layer MLP combining the option and cross-attention features | suite 0.7589 on the new 15-benchmark suite | the readout had a little left in it: +0.002 |
|
|
214
|
+
| a listwise reranking reward (NDCG@5 over 16 candidates) on top, against a KL anchor | hippo R@1 0.192 → 0.308, suite −0.003 | **the first RL stage kept**: a reward over the *order* of candidates carries what per-item labels cannot |
|
|
215
|
+
| **v3.0** | suite **0.756**, ECE **0.066** | **the first release where RL works**: +60% reranking R@1 over its supervised parent, the rest held level |
|
|
216
|
+
| image training on the train splits of twelve public datasets, with text replay | held-out image top-1 0.809 against 0.796 for a text-only control | the model reads images through the base model's frozen vision tower |
|
|
217
|
+
| generated reasoning questions with soft targets where the model is at chance | held-out ECE after calibration, BBH excluded, 0.096 → 0.075 | the image rounds had made it overconfident; this repairs most of it |
|
|
218
|
+
| an RL stage charging a confident mistake 4x a timid correct answer | held-out ECE before calibration 0.200 → 0.082 | against a matched supervised stage: better accuracy and calibration, no detectable ranking difference (one seed) |
|
|
219
|
+
| **v4.0-VL** | image top-1 **0.803** held out, suite **0.756**, ECE **0.043** | **reads images**, with text held at v3.0's level |
|
|
220
|
+
| the same recipe on Qwen3.5-4B, read at layer 16, 20 or 24 of its 32 | suite 0.744 · 0.760 · 0.760 | a System One model doesn't need the deep layers: quality flattens by layer 20 |
|
|
221
|
+
| images on the layer-20 model, plus questions whose right answer is "unknown" | KoBBQ unknown-when-ambiguous 0.679 → 0.891 | images cost the model its "unknown" answer until the data asked for it |
|
|
222
|
+
| pool each option over its own tokens, not the separator that follows the previous one | CLINC150 0.383 → 0.753 | a readout bug found by an outside benchmark; the same weights read correctly once each option is pooled over its own tokens |
|
|
223
|
+
| retrain only the decision head for that readout, tower frozen, 600 steps | final ECE after calibration 0.080 → 0.042, Decision Index 38.38 | the head had learned the old pooling; a short head-only stage recovers the calibration |
|
|
224
|
+
| **v5.0-VL** | MMLU-Pro **0.429**, Decision Index **38.38**, KoBBQ unknown **0.932** | **3B: the first 20 of 32 layers** |
|
|
225
|
+
| early-exit heads that read a detached copy of their layer, so no gradient from them reaches the trunk | MMLU-Pro 0.443 at layer 32 | retuning heads on a trunk trained with attached exits had not recovered depth: the trunk had lost it |
|
|
226
|
+
| 10,000 steps on a cleaned corpus instead of 40,000 on the old one | held-out +0.026, MMLU-Pro +0.043 | longer training overfits the suite; shorter and cleaner wins held out |
|
|
227
|
+
| one temperature per exit and an exit policy fitted on held-out proxies of the test mix | every release bar passes at 20.9 layers on average | policies tuned on in-distribution rows stop too early on hard questions |
|
|
228
|
+
| **v6.0-VL** | Decision Index **46.24**, MMLU-Pro **0.440** | **4B: answers at layer 16, 20 or 32, whichever is confident first** |
|
|
229
|
+
| v6.0-VL averaged, weight 0.5 each, with a second fine-tune of the same base trained on other data | suite 0.793 and held-out 0.728 at layer 32, against 0.770 and 0.695 | the average beats both members; the gain is in the tower, the heads add nothing |
|
|
230
|
+
| one temperature per exit and both thresholds chosen again for the average | `auto` at 0.85 / 0.50 matches layer 32 at 19.5 layers | averaging shrinks the logits: at v6.0-VL's temperatures it was underconfident; calibration ends worse than v6.0-VL's (final ECE 0.048 vs 0.036) |
|
|
231
|
+
| **v6.1-VL** | Decision Index 0.3 **50.98**, held-out **0.729** | **4B: v6.0-VL averaged with a second fine-tune** |
|
|
232
|
+
| two LoRA fine-tunes of Qwen3.8-27B, averaged weight 0.5 each, chosen from 9 full Decision Index 0.3 reads | Decision Index 0.3 65.59, against 64.83 and 65.30 for the two members | averaging carries over to 27B; uneven weights and a third ingredient read 65.37 to 65.58 |
|
|
233
|
+
| short inputs right-padded to a multiple of 64 tokens and replayed as CUDA graphs | short one-question requests in about half the time | at one request a time the 27B is bound by kernel launches, not compute |
|
|
234
|
+
| **v6.1-VL 27B** | Decision Index 0.3 public set **65.64** with `auto`, held-out **0.808** | **27B: the same method on Qwen3.8-27B, answers at layer 48, 56 or 64** |
|
|
235
|
+
| repeat the option list once and read the second copy; let the prompt and the options attend both ways | our 4B pick set +3.5 and +2.6, +3.8 over v6.1-VL together with an option-order consistency loss | what the readout sees mattered more than ~150 later loss and schedule variants |
|
|
236
|
+
| start from the instruction-tuned model, keep the base model's reasoning traces as a training signal | held level at 4B, +1.78 at 2B | reasoning stays a training-time teacher; nothing is generated at inference |
|
|
237
|
+
| data rounds 7 to 12: each round's weakest families get training data built from public train splits | Decision Index 0.3 53.06 → 60.62 | targeted data moves its targets and dilutes the rest; rounds are judged per benchmark |
|
|
238
|
+
| **v7.0-VL** | Decision Index 0.3 **60.63**, held-out **0.811** | **4B: option echo and prefix-bidirectional attention, instruction-tuned start, one end-to-end run** |
|
|
239
|
+
| **v7.0-VL 3B** | Decision Index 0.3 **60.01**, held-out **0.797**, median latency 19.6 ms against 29.4 ms | **3B: v7.0-VL 4B cut at layer 20, no retraining** |
|
|
240
|
+
|
|
241
|
+
</details>
|
|
242
|
+
|
|
243
|
+
## Documentation
|
|
244
|
+
|
|
245
|
+
| | |
|
|
246
|
+
|---|---|
|
|
247
|
+
| [Inference guide](docs/inference.md) | setups, measured latency, speed-ups |
|
|
248
|
+
| [Mac (MLX)](docs/mac.md) | the 8-bit and 4-bit Apple-silicon builds and what each costs |
|
|
249
|
+
| [Effort and depth](docs/effort-depth.md) | which tasks and questions need the deep layers |
|
|
250
|
+
| [HTTP API](serve/README.md) | the Jev-compatible server |
|
|
251
|
+
| [Benchmarks](BENCHMARKS.md) | what each number means and how to reproduce it |
|
|
252
|
+
| [Release records](versions/) | how each release was built, what it scores, its limitations |
|
|
253
|
+
| [Reinforcement learning](docs/rl.md) | where RL beat supervised training, and where it didn't |
|
|
254
|
+
| [Speed](docs/speed.md) | how inference was made faster |
|
|
255
|
+
| [Code](rsijev/README.md) | training and the files the loop may rewrite |
|
|
256
|
+
|
|
257
|
+
## Contributing
|
|
258
|
+
|
|
259
|
+
A case where the model is confidently wrong, or a variant you tried that failed, is the most useful
|
|
260
|
+
thing you can send: [report a wrong answer](https://github.com/Shanghua-Gao/RSI-Jev/issues/new?template=wrong-answer.yml)
|
|
261
|
+
· [report an experiment](https://github.com/Shanghua-Gao/RSI-Jev/issues/new?template=experiment.yml).
|
|
262
|
+
Each becomes a registered prediction tested in the next version. See
|
|
263
|
+
[`CONTRIBUTING.md`](CONTRIBUTING.md). To collaborate or support the work with compute,
|
|
264
|
+
contact [Shanghua Gao](https://shgao.site).
|
|
265
|
+
|
|
266
|
+
## Acknowledgements
|
|
267
|
+
|
|
268
|
+
Thanks to **HP** and **NVIDIA** for providing the **HP ZGX Nano AI Station**, powered by the
|
|
269
|
+
**NVIDIA GB10 Grace Blackwell Superchip**.
|
|
270
|
+
|
|
271
|
+
## License
|
|
272
|
+
|
|
273
|
+
Code: MIT. Weights: Apache-2.0, following the base model; some image training sources are
|
|
274
|
+
non-commercial, listed on each model card. Not affiliated with TypeSafe AI. To cite, use
|
|
275
|
+
[`CITATION.cff`](CITATION.cff).
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
agentguard/agentguard.py,sha256=kokw2sZq24G2_UypVDHklRRUCH5Wbm8z9fMX-7KIOhI,21697
|
|
2
|
+
decideit-7.0.1.dist-info/licenses/LICENSE,sha256=EQOzpvYbfhf58xZP9kwabyQI6_9Z-kPIKyZ9PKdFrn4,1069
|
|
3
|
+
rsijev/__init__.py,sha256=C4s6KTw_WJzY_1FnTSLEy9lwjWoB5vKAYHvHLlGBYrU,1003
|
|
4
|
+
rsijev/adaptive.py,sha256=eYS3u6xuJHTS5ZTwr2tksDVdCKDpgHVUoUDlrVe52BI,33710
|
|
5
|
+
rsijev/arch.py,sha256=nOgJSpcGk8TR9vEJQCw2oIHz2EXPB3JZOnLYwzJfZM8,61917
|
|
6
|
+
rsijev/calibrate.py,sha256=4iqkDMNGmTNxUY79dcsvkWxl3f6N6y7mm4IB4YYRLcE,18069
|
|
7
|
+
rsijev/contract.py,sha256=1BL53ZjyT9w_Cqv6O8aySBzojJn36EmBQKTB0jUuodc,5162
|
|
8
|
+
rsijev/data.py,sha256=vp6Hx2uPHB4-jxuR8uY3az7tE0l8IqNFfX06hj8hQqE,11797
|
|
9
|
+
rsijev/encode.py,sha256=sT1PIuveLZs-gsGMxFNtQyHez7FTQvub9QjGOD7hUtM,24281
|
|
10
|
+
rsijev/evaluate.py,sha256=Tw3JgsmPxqNSVUc5tIvfjr5ACJzr9PGRYzPU1t0dwMs,27810
|
|
11
|
+
rsijev/exit_policy.py,sha256=YqA-oRipUdrvpdgOkk4byUwArh7q7d5qnROSrZ-pdTg,38934
|
|
12
|
+
rsijev/fit.py,sha256=MjS72ssUvmTcnk17AMuRXYKZBQEyWkGZBeby-WH71QE,57398
|
|
13
|
+
rsijev/frozen_tok.py,sha256=R2m696zSvAmwn27HjJuQ8fDViYBLpIWmxVuIt5_kZjs,2736
|
|
14
|
+
rsijev/image_text.py,sha256=JB7t6WsNg7i7jVtbWEPMw2zu_1I81mBf2SzBcZapmVE,1355
|
|
15
|
+
rsijev/metrics.py,sha256=56C5cnoryWg8Ys0N4ozyMvcJhLCoqQLUumVioT7Z9Sk,6579
|
|
16
|
+
rsijev/rl2.py,sha256=WWbjCQ2xfy77pbxgpvy4qsq9DhRY4_2PynK973jKS9U,52253
|
|
17
|
+
rsijev/targets.py,sha256=ZkA1zfZvC-o21Q_oAqr0Af7oFGvT80JRzayr0ISjNi0,5115
|
|
18
|
+
rsijev/targets_suite.py,sha256=MGFym3l-Fszh7S_CbxM9Vx9YOcd-08M2SrJozwN34A4,42289
|
|
19
|
+
rsijev/train.py,sha256=A_a6BMDv2BYkFc1ip6tPEOpSXKMSUHtxhJJ42FEd22c,9975
|
|
20
|
+
rsijev/vision.py,sha256=x_vj6U51GnNGFcQPXQzaIoMykTr6WFiOtH4M6bKNtiA,15333
|
|
21
|
+
rsijev/vision_fit.py,sha256=V7-lcmHp1XiqMCw7qOF-TBNq9hbG11qH9OiV5iGSrVQ,6609
|
|
22
|
+
rsijev/mlx/__init__.py,sha256=Z-5EvZ3Qsm6O8jHAALYFG8PYSo2Ii5kXoyMq4x-wgOU,1160
|
|
23
|
+
rsijev/mlx/collate.py,sha256=mgxsKQoano34qB7I_wL66fm5aY7tSeU2VGJAWtWnHNE,3183
|
|
24
|
+
rsijev/mlx/convert.py,sha256=pu8L84a3jEp97ZsMRpJBQkHcIpM0OP_yUWZs6kMMimc,17445
|
|
25
|
+
rsijev/mlx/heads.py,sha256=j0kUnaCda2anmvkqOdGfbb7BjeuMmx2FYmyu3WMvDcI,10209
|
|
26
|
+
rsijev/mlx/model.py,sha256=IWrgDctxTQNK058wiyKqPMPcuiwoWaxRc744E2fDN5c,16320
|
|
27
|
+
rsijev/mlx/quant.py,sha256=IQCURFUWD-DV6jV7qTIiLTQU_Cb0wtfeEErI2FHhz7M,7297
|
|
28
|
+
rsijev/mlx/serving.py,sha256=koNwg--rmpQS6-leoTOm7JGiX6KNcSGpz6QakoMfCKI,23987
|
|
29
|
+
rsijev/mlx/text.py,sha256=AhyeWxZshvdaubAVXl4nTXAMre7wC8VYItVQWJIpajw,30396
|
|
30
|
+
rsijev/mlx/vision.py,sha256=FyoFNq57wDZDzvF9hmgOf5jYBQ-ZhH_jXVCdd0vz7SA,16057
|
|
31
|
+
serve/__init__.py,sha256=MkGgPISXMwTV_k4Ffv3nIpA6mFuUvN4f54ioIHUpkCo,67
|
|
32
|
+
serve/accel.py,sha256=dbGn4I8gtAEvwWj3VfWF9P0P6l9LxiSZW7G3stiMbBs,2993
|
|
33
|
+
serve/app.py,sha256=xpkR1503dr-UljKlgaRJ_LI8pP8s4kATBRl3_SYNRT8,15965
|
|
34
|
+
serve/autostart.py,sha256=H8E-iu92o0haArczmo1puJFvz5OKYz35bOIF639LLqs,7855
|
|
35
|
+
serve/batcher.py,sha256=iRLa-Oiw7chq6-Ra6ESojZ9E_tk-bQETv-wFoQmWRwI,14751
|
|
36
|
+
serve/bench.py,sha256=xEahsqczy2dg78530MyOw1Ktrlv6d4KP9N4bRUDkebw,8283
|
|
37
|
+
serve/cli.py,sha256=PCvTAqCOeZ9K16Mv8hbKwub3HPRxFO1LkSc8WAhm_p0,2621
|
|
38
|
+
serve/daemon.py,sha256=tUxXi_v5PNwUUQe-NOMvqafxcoLZM_kliV025ebX9V8,4111
|
|
39
|
+
serve/decide_cli.py,sha256=RPyApGd3XpkSZWCzswWNQQRNqPPYk95ofZQU-9ZD6XM,8867
|
|
40
|
+
serve/decider.py,sha256=yQA3JHEcdXMoMt0jrKFTRsxkhvnu0uCBaxSo0amRIfw,5441
|
|
41
|
+
serve/demo.py,sha256=Q6C8ksLlG_VheBnrWAbqnK8rVN6MropFJzdaV2GQEc8,7207
|
|
42
|
+
serve/demo_images.py,sha256=ELMcUI0aNN4OiXWY19lKuZUYHxKzbSdUSfpNqIk8zZw,12209
|
|
43
|
+
serve/effort.py,sha256=fqKucC2-Q4VL3eD0HLFk9uXaKkZEOjemDZgdO0mqzvw,7574
|
|
44
|
+
serve/examples.json,sha256=4bMLn1qhYIh5EMv3_RrwoOjDOqaIgqOZSGxEWnlQ3Nk,113904
|
|
45
|
+
serve/images.py,sha256=BUREyY8oV16WFAxS7egHqwruvOKgPuKOMJ8jQ4IgGWk,9458
|
|
46
|
+
serve/infer.py,sha256=ZwGSiiqTPx3V5FM2yAFPUV_Bt9_IJ8VrnsC3BxlDYjY,50740
|
|
47
|
+
serve/kernels.py,sha256=ghBaH4Nc6ltMIaIdiHTvwYDWF6ZAJj9KYh6Zjvn1yEc,4017
|
|
48
|
+
serve/padded.py,sha256=wcvq8J2UW6E4N0xnd_THAo777-1R4xwDAqcR8lt2fB0,21588
|
|
49
|
+
serve/plan.py,sha256=q7kPdGz6FtKR7a7yo7-RxNOqQ43ySuaJDfWYeuzV2JU,12332
|
|
50
|
+
serve/release.py,sha256=l8-NlIx98wCjvxFoISWotSCXcb2-3Xf_Xrq96_DQ_4w,29648
|
|
51
|
+
serve/runtime.py,sha256=ycUDpeEZI4Yq7chEK1jYTXJgFdBbhri-EwzmKlbDOkE,11028
|
|
52
|
+
serve/serve_tune.py,sha256=59aPXfAfYg1EVOhzCK1HUaiUeAnZWLV-GviryQoyaQc,5592
|
|
53
|
+
serve/server.py,sha256=7P3vdY9C6HOaRYQAFIEvt9eJXc-Yr8WLKlr31u5wtiI,25058
|
|
54
|
+
serve/ui.html,sha256=CEe8fsqE2cn8VE10kAyhrKizBO4AoQZKGmcG8u14hcI,60973
|
|
55
|
+
serve/wire.py,sha256=GLCO-DX9TZTirXxVd9NZbd_QSEtVw_kQyStBkf-_jXc,10424
|
|
56
|
+
decideit-7.0.1.dist-info/METADATA,sha256=KeW3a-Hzssep5nVqVO3w8nWwMLkaNi6Ny055biAVxN4,20180
|
|
57
|
+
decideit-7.0.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
58
|
+
decideit-7.0.1.dist-info/entry_points.txt,sha256=DOPDWjgoZB-lReKw4hHGQUo4oGvCls_ORR_W0YTz7k0,147
|
|
59
|
+
decideit-7.0.1.dist-info/top_level.txt,sha256=aJzPLsHgp6OzliBarNts_hDSjnt7e46niuMOvrn75_E,24
|
|
60
|
+
decideit-7.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shanghua Gao
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
rsijev/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Typed-decision models: a fine-tuned tower with a trained option scorer.
|
|
2
|
+
|
|
3
|
+
The file split is a guard rail, not a style. An automated experiment may rewrite
|
|
4
|
+
the EDITABLE modules and may not touch the PROTECTED ones, so a change to the
|
|
5
|
+
model, the data or the training recipe can never quietly become a change to how
|
|
6
|
+
it is scored. `rsijev/README.md` has the table.
|
|
7
|
+
|
|
8
|
+
PROTECTED contract.py encode.py evaluate.py metrics.py targets.py
|
|
9
|
+
EDITABLE arch.py data.py train.py (fit.py: bookkeeping is not)
|
|
10
|
+
"""
|
|
11
|
+
__all__ = ["contract", "metrics", "arch", "data", "train"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def __getattr__(name):
|
|
15
|
+
# `from rsijev import Decider` is the one-line way in. It lives with the server
|
|
16
|
+
# (serve/decider.py) because it IS the server's code path; imported lazily so
|
|
17
|
+
# that training and evaluation never pull in the HTTP stack.
|
|
18
|
+
if name == "Decider":
|
|
19
|
+
from serve.decider import Decider
|
|
20
|
+
return Decider
|
|
21
|
+
raise AttributeError(f"module 'rsijev' has no attribute {name!r}")
|