nullspan 1.2.0__tar.gz → 1.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. nullspan-1.2.1/PKG-INFO +176 -0
  2. nullspan-1.2.1/README.md +154 -0
  3. {nullspan-1.2.0 → nullspan-1.2.1}/pyproject.toml +3 -3
  4. nullspan-1.2.1/src/nullspan/__init__.py +5 -0
  5. {nullspan-1.2.0 → nullspan-1.2.1}/src/nullspan/engine.py +26 -0
  6. nullspan-1.2.1/src/nullspan/systemone.py +910 -0
  7. nullspan-1.2.0/PKG-INFO +0 -149
  8. nullspan-1.2.0/README.md +0 -128
  9. nullspan-1.2.0/src/nullspan/__init__.py +0 -5
  10. {nullspan-1.2.0 → nullspan-1.2.1}/LICENSE +0 -0
nullspan-1.2.1/PKG-INFO ADDED
@@ -0,0 +1,176 @@
1
+ Metadata-Version: 2.4
2
+ Name: nullspan
3
+ Version: 1.2.1
4
+ Summary: Calibrated p_null abstention, grounded [start, end] span extraction, System One typed decisions, arithmetic, and distribution forecasting in a single forward pass.
5
+ Keywords: nullspan,system-one,extraction,abstention,p-null,quantile-regression,forecasting
6
+ Author-email: Hemanth HM <hemanth.hm@gmail.com>
7
+ Requires-Python: >=3.9
8
+ Description-Content-Type: text/markdown
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Programming Language :: Python :: 3
11
+ License-File: LICENSE
12
+ Requires-Dist: torch>=2.0.0 ; extra == "hf"
13
+ Requires-Dist: transformers>=4.40.0 ; extra == "hf"
14
+ Requires-Dist: safetensors>=0.4.0 ; extra == "hf"
15
+ Requires-Dist: huggingface_hub>=0.23.0 ; extra == "hf"
16
+ Requires-Dist: sentencepiece>=0.1.99 ; extra == "hf"
17
+ Project-URL: Homepage, https://github.com/hemanth/nullspan
18
+ Project-URL: Model, https://huggingface.co/gnumanth/nullspan
19
+ Project-URL: Playground, https://hemanth.github.io/nullspan/
20
+ Provides-Extra: hf
21
+
22
+ # nullspan
23
+
24
+ Single-pass System One structured extraction, multi-numeric contract disambiguation, calibrated `p_null` abstention, character-offset `[start, end]` spans, calibrated `noul`/`choice`/`score` `/v1/systemone` decisions, cross-field algebraic invariant verification, and calibrated quantile estimation (`p10`, `median`, `p90`) in JavaScript, Python, and Browser WebAssembly ONNX (`512`-token context + long-document windowing).
25
+
26
+ - **Interactive Playground (Browser WASM ONNX + Zero-Dep Engine)**: [https://hemanth.github.io/nullspan/](https://hemanth.github.io/nullspan/)
27
+ - **NullspanBench Leaderboard (JevBench-Style 19-System Evaluation, N=500)**: [https://hemanth.github.io/nullspan/bench.html](https://hemanth.github.io/nullspan/bench.html)
28
+ - **Hugging Face Space**: [https://huggingface.co/spaces/gnumanth/nullspan](https://huggingface.co/spaces/gnumanth/nullspan)
29
+ - **Fine-Tuned Hugging Face Model (345.3M Extraction + 331.1 MB Int8 ONNX + 435M System One Cross-Encoder)**: [https://huggingface.co/gnumanth/nullspan](https://huggingface.co/gnumanth/nullspan)
30
+ - **Hugging Face Dataset**: [https://huggingface.co/datasets/gnumanth/nullspan-dataset](https://huggingface.co/datasets/gnumanth/nullspan-dataset)
31
+ - **Empirical Science Report**: [`bench/SCIENCE_REPORT.md`](./bench/SCIENCE_REPORT.md)
32
+
33
+ ## In Plain English
34
+
35
+ Standard LLMs extract data and make typed workflow judgments by generating JSON token-by-token, which is slow, bills per output token, can get hijacked by *"ignore previous instructions"* prompt injections, and often hallucinates values when a field is missing. **`nullspan` acts like a grounded highlighter + calibrated System One judge instead of a chatbot:**
36
+
37
+ 1. **Points to Exact Text (`[start, end]` Span)**: Highlights exact character positions in your input text (`0` generated output tokens), making fabricated strings and prompt-injection payloads impossible.
38
+ 2. **Knows When to Say "Not There" (`p_null`)**: Computes a calibrated probability that a field is missing (`p_null`) and returns `null` with `abstention_reason` when evidence is absent (`87.00%` EM / `89.36%` F1 on Stanford `SQuAD v2.0`).
39
+ 3. **Disambiguates Multi-Numeric Contracts & Checks Math (`constraint` & `verified`)**: Separates competing dollar caps, percentages, day windows, and numeric tranches across multi-section agreements (`96/96` on unseen contracts) and verifies equations like `subtotal == nodes * hourly_rate`.
40
+ 4. **Calibrated `/v1/systemone` Typed Decisions (`decide()`)**: Evaluates `noul` (`true`/`false`), `choice` (`options`), and ordinal `score` (`min..max`) rubrics with calibrated probability distributions—ranking **#2 Overall / #1 Non-LLM Encoder** (`75.40%`) on [`LocalLLaMA/typed-decisions`](https://huggingface.co/datasets/LocalLLaMA/typed-decisions) and hitting **`100.0%` (`48/48`)** on `JevBench Easy` (`100.0%` strict schema validity).
41
+
42
+ ## Install
43
+
44
+ ```sh
45
+ npm install nullspan
46
+ pip install nullspan
47
+ ```
48
+
49
+ ## Usage (JavaScript / ESM)
50
+
51
+ ```js
52
+ import { extract } from 'nullspan';
53
+
54
+ const incident =
55
+ "Incident INC-40928 opened on Apr 12, 2026 with a 502 Bad Gateway errors and a TLS handshake timeouts. " +
56
+ "Issued $1240.50 plus $310.25 SLA credit. Severity is sev-1 and customer data loss was false. " +
57
+ "Root cause CVE not yet assigned. Cluster stabilized on Apr 19, 2026. Reported by Mira";
58
+
59
+ const result = extract(incident, {
60
+ incident_id: { type: 'text', instructions: 'Incident ticket identifier' },
61
+ opened_on: { type: 'date', instructions: 'Date the incident opened' },
62
+ total_credit: { type: 'number', decimals: 2, unit: 'USD', instructions: 'Total credit including SLA penalty' },
63
+ outage_days: { type: 'duration', unit: 'days', integer: true, instructions: 'Days from incident open to cluster stabilization' },
64
+ failure_modes: { type: 'list', items: 'text', instructions: 'Each failure mode triggered' },
65
+ severity: { type: 'enum', options: ['sev-1', 'sev-2', 'sev-3'], instructions: 'Incident severity tier' },
66
+ data_loss: { type: 'boolean', instructions: 'Whether customer data loss occurred' },
67
+ cve_id: { type: 'text', instructions: 'Assigned CVE identifier if present' },
68
+ remediation_cost: { type: 'estimate', unit: 'USD', min: 100, max: 25000, scale: 'log', step: 10, instructions: 'Estimated engineering remediation cost' },
69
+ });
70
+ ```
71
+
72
+ ## Usage (Python Extraction + `/v1/systemone` Typed Decisions)
73
+
74
+ ```python
75
+ from nullspan import extract, decide
76
+
77
+ invoice = (
78
+ "Invoice from TensorCore Cloud\n"
79
+ "Invoice #88310 | Issued: 2026-08-14 18:20\n"
80
+ "Nodes: 4 @ $3.50 each\n"
81
+ "Subtotal: $14.00\n"
82
+ "Egress Fee: $0.00\n"
83
+ "Priority Surcharge: $1.75"
84
+ )
85
+
86
+ # 1. Grounded extraction + cross-field algebraic invariants (backend="fast" | "hf" | "onnx")
87
+ out = extract(
88
+ state=invoice,
89
+ fields={
90
+ "nodes": {"type": "number", "integer": True, "instructions": "Provisioned GPU node count"},
91
+ "hourly_rate": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "Hourly rate per node"},
92
+ "subtotal": {
93
+ "type": "number",
94
+ "decimals": 2,
95
+ "unit": "USD",
96
+ "constraint": "subtotal == nodes * hourly_rate",
97
+ "instructions": "Compute subtotal before fees and surcharge",
98
+ },
99
+ "promo_rebate": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "Promotional rebate amount if present"},
100
+ },
101
+ backend="hf",
102
+ model_id="gnumanth/nullspan",
103
+ )
104
+
105
+ # 2. TypeSafe-compatible System One typed decisions (noul / choice / score)
106
+ judgments = decide(
107
+ state={"invoice_id": "INV-88310", "subtotal": 14.0, "nodes": 4, "unit_price": 3.5, "approved_vendor": True},
108
+ questions={
109
+ "math_valid": {"type": "noul", "instructions": "Does subtotal equal nodes * unit_price?"},
110
+ "routing": {"type": "choice", "options": ["auto_pay", "manual_review", "reject"], "instructions": "Select workflow action."},
111
+ "risk_tier": {"type": "score", "min": 1, "max": 5, "instructions": "Rate audit risk from 1 (low) to 5 (critical)."},
112
+ },
113
+ model_id="gnumanth/nullspan",
114
+ )
115
+ ```
116
+
117
+ To run the live TypeSafe `/v1/systemone` HTTP server for `jevbench` or `LocalLLaMA/typed-decisions`:
118
+
119
+ ```sh
120
+ python training/serve_systemone.py --model-dir gnumanth/nullspan --port 8010
121
+ ```
122
+
123
+ ## Official External Public Benchmarks (NVIDIA L4 GPU · Zero Test Contamination)
124
+
125
+ Reproducible via [`bench/eval_public_benchmarks.py`](./bench/eval_public_benchmarks.py) with results stored in [`bench/public_benchmarks_results.json`](./bench/public_benchmarks_results.json):
126
+
127
+ | External Public Benchmark | Split & Scale | Primary Metric | Calibration & Validity | Latency & Cost (NVIDIA L4) | Standing |
128
+ | :--- | :--- | :---: | :---: | :---: | :--- |
129
+ | **[`LocalLLaMA/typed-decisions`](https://huggingface.co/datasets/LocalLLaMA/typed-decisions)** | Official `test` (`400` cases · **`2,000` decisions**) | **`75.40%` Accuracy** (`noul` `82.67%`, `choice` `75.00%`, `score` `70.25%`) | `KL 0.1515` · `Brier 0.0770` · `ECE 0.0944` | `78.0 ms/dec` (`390 ms/case`) · `0` output tokens | **Rank #2 Overall Worldwide** (surpassing 27B–31B LLMs) · **Rank #1 Encoder** |
130
+ | **[Benchmark Heaven / `jevbench`](https://github.com/fstandhartinger/jevbench)** | Official Public Suite (`231` items via `/v1/systemone`) | **`100.0%` Easy (`48/48`)** · `62.50%` Original (`45/72`) · `37.84%` Hard (`42/111`) (`58.44%` all) | **`100.0%` Strict Schema Validity** (`Easy Brier 0.0207`, `ECE 0.0605`) | **`64.8 ms` `p50`** (`63.2 ms` Easy) · **`$0.00667 / 1k`** | `100.0%` on `adversarial`, `fact`, `ordinal`, & `tool_selection` |
131
+ | **[Stanford `rajpurkar/squad_v2`](https://huggingface.co/datasets/rajpurkar/squad_v2)** | Zero-overlap `validation` (`N = 500`: `250` HasAns + `250` NoAns) | **`87.00%` Exact Match** · **`89.36%` F1** | `HasAns F1 91.52%` · `NoAns EM 87.20%` | `10.3 ms` `p50` (L4 CUDA) · `$0.0017 / 1k` | Dedicated `qa_null_head` temperature-calibrated on disjoint `train` |
132
+
133
+ ## Architecture (`NullspanForExtraction` 345.3M + `NullspanSystemOneEngine` 435M)
134
+
135
+ ![Architecture Flow](./docs/flow-diagram.svg)
136
+
137
+ 1. **`null_head` + `qa_null_head` (Span-Attended Calibrated Abstention)**: Bilinear span-summary + `[CLS]` margin residual with held-out temperature scaling (`p_null` AUROC `0.9933`, ECE `0.0142`; `0.9951` / `0.0127` in `331.1 MB` Int8 ONNX) and `abstention_reason` (`unassigned_placeholder` vs `missing_field`).
138
+ 2. **`span_head` (Grounded Pointer + Shannon Entropy)**: Projects `512`-token encoder states (plus query-conditioned sliding windows on ultra-long contracts) to `[start, end]` character spans with calibrated `confidence` and Shannon `span_entropy_bits`.
139
+ 3. **`bio_head` (Multi-Span List Tagging)**: Extracts ordered text and numeric tranche arrays (`type: "list"`) in a single pass.
140
+ 4. **`op_head` + `quantile_head` + Invariant Verifier**: Routes 10 System One extraction/math operations, predicts monotone `(p10, median, p90)` quantiles, and verifies cross-field algebraic constraints (`verified: true/false`).
141
+ 5. **`NullspanSystemOneEngine` (`/v1/systemone` Cross-Option & Cross-Question Judge)**: `DeBERTa-v3-Large` (`r=32` LoRA) with top-placed programmatic numeric/temporal checks, Head + IDF-Clause + Tail windowing, 2-layer `CrossOptionDecisionHead` self-attention across candidate options, Cross-Question Workflow Propagation across multi-question batches, and Cumulative Ranked Probability Score (`RPS`) ordinal calibration.
142
+
143
+ ## NullspanBench Leaderboard (`bench/run_bench.py` · 19 Systems · $N=500$ Zero-Leakage Decisions · NVIDIA L4 GPU)
144
+
145
+ Modeled after [BenchmarkHeaven's JevBench](https://benchmarkheaven.com/jev-models), **NullspanBench v3.2** evaluates `19` systems (`13` local open-weights/open-code models running live forward passes on Google Cloud NVIDIA L4 GPU, live Cloudflare Workers AI `@cf/cloudflare/clairic + @cf/cloudflare/clef`, `2` reference JSON-mode LLM decoders, and `3` published JevBench v1.7 reference anchors) across **`500` zero-leakage observations** (`336` Open structured items + `164` Sealed items: `60` Stanford SQuAD v2.0, `96` Multi-Numeric Legal/Commercial Contract fields across `12` unseen agreements, and `8` Adversarial prompt-injection traps) on four axes: **Intelligence (`I`)**, **Calibration (`C`)**, **Speed (`S`)**, and **Cost (`K`)**.
146
+
147
+ Explore interactive weight sliders, Pareto frontier plots, and 2-system radar comparisons at [**https://hemanth.github.io/nullspan/bench.html**](https://hemanth.github.io/nullspan/bench.html):
148
+
149
+ | Rank | System | Composite (`25:25:25:25`) | Capability (`50I:50C`) | Intelligence `I` (95% CI) | Calibration `C` | Speed `S` | Cost `K` | `$/1k` | `p50` Latency |
150
+ | :---: | :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
151
+ | **1** | **`nullspan-Hybrid (Fast + ONNX Int8)`** (Open) | **95.7** | **98.2** | **97.8%** `[96.6, 99.0]` | **98.6** | **92.7** | **93.8** | `$0.0003` | `0.040 ms` |
152
+ | **2** | **`nullspan-ZeroDep (Fast Engine)`** (Open) | **94.7** | `92.9` | `90.8%` `[88.2, 93.4]` | `95.0` | **93.5** | **99.8** | `$0.0001` | `0.034 ms` |
153
+ | **3** | **`nullspan-345M (PyTorch FP32 · L4 CUDA)`** (Open) | **81.3** | **98.2** | **98.0%** `[96.8, 99.2]` | **98.5** | `63.2` | `76.8` | `$0.0017` | `10.297 ms` |
154
+ | **4** | **`nullspan-345M-Int8 (ONNX WASM/CPU)`** (Open) | **78.1** | **98.2** | **97.8%** `[96.6, 99.0]` | **98.6** | `52.7` | `84.2` | `$0.0008` | `75.933 ms` |
155
+ | **5** | `H2O-Lightning-4B v1.1 (JevBench #1 Comp)` (Open 4B) | `72.5` | `75.0` | `60.0%` `[57.8, 62.3]` | `90.0` | `92.6` | `60.3` | `$0.0211` | `29.285 ms` |
156
+ | **6** | **`@cf/cloudflare/clairic + clef (Live Workers AI)`** (API) | `72.1` | `93.2` | `91.6%` `[89.2, 93.9]` | `94.9` | `51.7` | `68.2` | `$0.0041` | `91.323 ms` (`315.3 ms/doc`) |
157
+ | **7** | `basal-omni-1.0-ultra (JevBench #1 Cap)` (Open 31B) | `71.8` | `82.5` | `74.1%` `[71.8, 76.3]` | `91.0` | `87.3` | `50.5` | `$0.0447` | `102.000 ms` |
158
+ | **8** | `Jev 1.13.0 (TypeSafe AI · JevBench Ref)` (API) | `71.5` | `77.1` | `63.6%` `[61.2, 66.0]` | `90.6` | `91.5` | `54.7` | `$0.0323` | `239.102 ms` |
159
+ | **9** | **`nullspan-345M (ONNX FP32)`** (Open) | `71.2` | **98.2** | **98.0%** `[96.8, 99.2]` | **98.5** | `43.5` | `78.0` | `$0.0015` | `424.943 ms` |
160
+ | **10** | `deberta-v3-base-squad2 (184M SQuAD2 QA)` (Open) | `60.3` | `55.8` | `48.1%` `[43.9, 52.4]` | `63.4` | `65.1` | `69.2` | `$0.0037` | `7.234 ms` |
161
+ | **11** | `gpt-5.4-nano (JSON Decoder · API)` (API) | `59.8` | `92.1` | `95.2%` `[93.4, 97.0]` | `89.0` | `43.6` | `45.1` | `$0.0438` | `417.991 ms` |
162
+ | **12** | `claude-haiku-4.5 (JSON Decoder · API)` (API) | `48.8` | `92.1` | `95.4%` `[93.4, 97.2]` | `88.7` | `41.8` | `27.5` | `$0.2639` | `586.392 ms` |
163
+ | **13** | `bge-small-en-v1.5 (Dense Span+Null)` (Open) | `37.6` | `52.0` | `14.7%` `[12.0, 17.5]` | `89.2` | `64.2` | `85.5` | `$0.0007` | `8.625 ms` |
164
+ | **14** | `distilbert-base-control (Raw Control)` (Open) | `36.9` | `52.9` | `14.2%` `[11.6, 17.0]` | `91.5` | `67.7` | `81.1` | `$0.0011` | `4.479 ms` |
165
+ | **15** | `all-MiniLM-L12-v2 (Dense Span+Null)` (Open) | `36.3` | `44.0` | `14.4%` `[11.7, 17.5]` | `73.5` | `64.2` | `85.5` | `$0.0007` | `8.582 ms` |
166
+ | **16** | `all-mpnet-base-v2 (Dense Span+Null)` (Open) | `34.4` | `42.3` | `13.6%` `[10.9, 16.7]` | `71.0` | `63.3` | `77.4` | `$0.0016` | `10.257 ms` |
167
+ | **17** | `paraphrase-albert-small-v2 (Dense Span+Null)` (Open) | `33.6` | `44.5` | `12.5%` `[10.0, 15.2]` | `76.6` | `66.4` | `91.0` | `$0.0004` | `5.700 ms` |
168
+ | **18** | `paraphrase-MiniLM-L3-v2 (Dense Span+Null)` (Open) | `30.9` | `41.7` | `11.1%` `[8.7, 13.8]` | `72.3` | `69.6` | `91.0` | `$0.0004` | `3.114 ms` |
169
+ | **19** | `all-MiniLM-L6-v2 (Dense Span+Null)` (Open) | `28.5` | `40.5` | `10.0%` `[7.7, 12.3]` | `70.9` | `67.1` | `88.8` | `$0.0005` | `4.988 ms` |
170
+
171
+ See full methodology, proper scoring definitions, request-type breakdowns (`100.0%` on Multi-Numeric Contracts `n=96`, `100.0%` Choice `n=41`, `100.0%` Math & Invariants `n=51`), and head-to-head analysis in [`bench/SCIENCE_REPORT.md`](./bench/SCIENCE_REPORT.md).
172
+
173
+ ## License
174
+
175
+ MIT © [Hemanth.HM](https://h3manth.com)
176
+
nullspan-1.2.1/README.md ADDED
@@ -0,0 +1,154 @@
1
+ # nullspan
2
+
3
+ Single-pass System One structured extraction, multi-numeric contract disambiguation, calibrated `p_null` abstention, character-offset `[start, end]` spans, calibrated `noul`/`choice`/`score` `/v1/systemone` decisions, cross-field algebraic invariant verification, and calibrated quantile estimation (`p10`, `median`, `p90`) in JavaScript, Python, and Browser WebAssembly ONNX (`512`-token context + long-document windowing).
4
+
5
+ - **Interactive Playground (Browser WASM ONNX + Zero-Dep Engine)**: [https://hemanth.github.io/nullspan/](https://hemanth.github.io/nullspan/)
6
+ - **NullspanBench Leaderboard (JevBench-Style 19-System Evaluation, N=500)**: [https://hemanth.github.io/nullspan/bench.html](https://hemanth.github.io/nullspan/bench.html)
7
+ - **Hugging Face Space**: [https://huggingface.co/spaces/gnumanth/nullspan](https://huggingface.co/spaces/gnumanth/nullspan)
8
+ - **Fine-Tuned Hugging Face Model (345.3M Extraction + 331.1 MB Int8 ONNX + 435M System One Cross-Encoder)**: [https://huggingface.co/gnumanth/nullspan](https://huggingface.co/gnumanth/nullspan)
9
+ - **Hugging Face Dataset**: [https://huggingface.co/datasets/gnumanth/nullspan-dataset](https://huggingface.co/datasets/gnumanth/nullspan-dataset)
10
+ - **Empirical Science Report**: [`bench/SCIENCE_REPORT.md`](./bench/SCIENCE_REPORT.md)
11
+
12
+ ## In Plain English
13
+
14
+ Standard LLMs extract data and make typed workflow judgments by generating JSON token-by-token, which is slow, bills per output token, can get hijacked by *"ignore previous instructions"* prompt injections, and often hallucinates values when a field is missing. **`nullspan` acts like a grounded highlighter + calibrated System One judge instead of a chatbot:**
15
+
16
+ 1. **Points to Exact Text (`[start, end]` Span)**: Highlights exact character positions in your input text (`0` generated output tokens), making fabricated strings and prompt-injection payloads impossible.
17
+ 2. **Knows When to Say "Not There" (`p_null`)**: Computes a calibrated probability that a field is missing (`p_null`) and returns `null` with `abstention_reason` when evidence is absent (`87.00%` EM / `89.36%` F1 on Stanford `SQuAD v2.0`).
18
+ 3. **Disambiguates Multi-Numeric Contracts & Checks Math (`constraint` & `verified`)**: Separates competing dollar caps, percentages, day windows, and numeric tranches across multi-section agreements (`96/96` on unseen contracts) and verifies equations like `subtotal == nodes * hourly_rate`.
19
+ 4. **Calibrated `/v1/systemone` Typed Decisions (`decide()`)**: Evaluates `noul` (`true`/`false`), `choice` (`options`), and ordinal `score` (`min..max`) rubrics with calibrated probability distributions—ranking **#2 Overall / #1 Non-LLM Encoder** (`75.40%`) on [`LocalLLaMA/typed-decisions`](https://huggingface.co/datasets/LocalLLaMA/typed-decisions) and hitting **`100.0%` (`48/48`)** on `JevBench Easy` (`100.0%` strict schema validity).
20
+
21
+ ## Install
22
+
23
+ ```sh
24
+ npm install nullspan
25
+ pip install nullspan
26
+ ```
27
+
28
+ ## Usage (JavaScript / ESM)
29
+
30
+ ```js
31
+ import { extract } from 'nullspan';
32
+
33
+ const incident =
34
+ "Incident INC-40928 opened on Apr 12, 2026 with a 502 Bad Gateway errors and a TLS handshake timeouts. " +
35
+ "Issued $1240.50 plus $310.25 SLA credit. Severity is sev-1 and customer data loss was false. " +
36
+ "Root cause CVE not yet assigned. Cluster stabilized on Apr 19, 2026. Reported by Mira";
37
+
38
+ const result = extract(incident, {
39
+ incident_id: { type: 'text', instructions: 'Incident ticket identifier' },
40
+ opened_on: { type: 'date', instructions: 'Date the incident opened' },
41
+ total_credit: { type: 'number', decimals: 2, unit: 'USD', instructions: 'Total credit including SLA penalty' },
42
+ outage_days: { type: 'duration', unit: 'days', integer: true, instructions: 'Days from incident open to cluster stabilization' },
43
+ failure_modes: { type: 'list', items: 'text', instructions: 'Each failure mode triggered' },
44
+ severity: { type: 'enum', options: ['sev-1', 'sev-2', 'sev-3'], instructions: 'Incident severity tier' },
45
+ data_loss: { type: 'boolean', instructions: 'Whether customer data loss occurred' },
46
+ cve_id: { type: 'text', instructions: 'Assigned CVE identifier if present' },
47
+ remediation_cost: { type: 'estimate', unit: 'USD', min: 100, max: 25000, scale: 'log', step: 10, instructions: 'Estimated engineering remediation cost' },
48
+ });
49
+ ```
50
+
51
+ ## Usage (Python Extraction + `/v1/systemone` Typed Decisions)
52
+
53
+ ```python
54
+ from nullspan import extract, decide
55
+
56
+ invoice = (
57
+ "Invoice from TensorCore Cloud\n"
58
+ "Invoice #88310 | Issued: 2026-08-14 18:20\n"
59
+ "Nodes: 4 @ $3.50 each\n"
60
+ "Subtotal: $14.00\n"
61
+ "Egress Fee: $0.00\n"
62
+ "Priority Surcharge: $1.75"
63
+ )
64
+
65
+ # 1. Grounded extraction + cross-field algebraic invariants (backend="fast" | "hf" | "onnx")
66
+ out = extract(
67
+ state=invoice,
68
+ fields={
69
+ "nodes": {"type": "number", "integer": True, "instructions": "Provisioned GPU node count"},
70
+ "hourly_rate": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "Hourly rate per node"},
71
+ "subtotal": {
72
+ "type": "number",
73
+ "decimals": 2,
74
+ "unit": "USD",
75
+ "constraint": "subtotal == nodes * hourly_rate",
76
+ "instructions": "Compute subtotal before fees and surcharge",
77
+ },
78
+ "promo_rebate": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "Promotional rebate amount if present"},
79
+ },
80
+ backend="hf",
81
+ model_id="gnumanth/nullspan",
82
+ )
83
+
84
+ # 2. TypeSafe-compatible System One typed decisions (noul / choice / score)
85
+ judgments = decide(
86
+ state={"invoice_id": "INV-88310", "subtotal": 14.0, "nodes": 4, "unit_price": 3.5, "approved_vendor": True},
87
+ questions={
88
+ "math_valid": {"type": "noul", "instructions": "Does subtotal equal nodes * unit_price?"},
89
+ "routing": {"type": "choice", "options": ["auto_pay", "manual_review", "reject"], "instructions": "Select workflow action."},
90
+ "risk_tier": {"type": "score", "min": 1, "max": 5, "instructions": "Rate audit risk from 1 (low) to 5 (critical)."},
91
+ },
92
+ model_id="gnumanth/nullspan",
93
+ )
94
+ ```
95
+
96
+ To run the live TypeSafe `/v1/systemone` HTTP server for `jevbench` or `LocalLLaMA/typed-decisions`:
97
+
98
+ ```sh
99
+ python training/serve_systemone.py --model-dir gnumanth/nullspan --port 8010
100
+ ```
101
+
102
+ ## Official External Public Benchmarks (NVIDIA L4 GPU · Zero Test Contamination)
103
+
104
+ Reproducible via [`bench/eval_public_benchmarks.py`](./bench/eval_public_benchmarks.py) with results stored in [`bench/public_benchmarks_results.json`](./bench/public_benchmarks_results.json):
105
+
106
+ | External Public Benchmark | Split & Scale | Primary Metric | Calibration & Validity | Latency & Cost (NVIDIA L4) | Standing |
107
+ | :--- | :--- | :---: | :---: | :---: | :--- |
108
+ | **[`LocalLLaMA/typed-decisions`](https://huggingface.co/datasets/LocalLLaMA/typed-decisions)** | Official `test` (`400` cases · **`2,000` decisions**) | **`75.40%` Accuracy** (`noul` `82.67%`, `choice` `75.00%`, `score` `70.25%`) | `KL 0.1515` · `Brier 0.0770` · `ECE 0.0944` | `78.0 ms/dec` (`390 ms/case`) · `0` output tokens | **Rank #2 Overall Worldwide** (surpassing 27B–31B LLMs) · **Rank #1 Encoder** |
109
+ | **[Benchmark Heaven / `jevbench`](https://github.com/fstandhartinger/jevbench)** | Official Public Suite (`231` items via `/v1/systemone`) | **`100.0%` Easy (`48/48`)** · `62.50%` Original (`45/72`) · `37.84%` Hard (`42/111`) (`58.44%` all) | **`100.0%` Strict Schema Validity** (`Easy Brier 0.0207`, `ECE 0.0605`) | **`64.8 ms` `p50`** (`63.2 ms` Easy) · **`$0.00667 / 1k`** | `100.0%` on `adversarial`, `fact`, `ordinal`, & `tool_selection` |
110
+ | **[Stanford `rajpurkar/squad_v2`](https://huggingface.co/datasets/rajpurkar/squad_v2)** | Zero-overlap `validation` (`N = 500`: `250` HasAns + `250` NoAns) | **`87.00%` Exact Match** · **`89.36%` F1** | `HasAns F1 91.52%` · `NoAns EM 87.20%` | `10.3 ms` `p50` (L4 CUDA) · `$0.0017 / 1k` | Dedicated `qa_null_head` temperature-calibrated on disjoint `train` |
111
+
112
+ ## Architecture (`NullspanForExtraction` 345.3M + `NullspanSystemOneEngine` 435M)
113
+
114
+ ![Architecture Flow](./docs/flow-diagram.svg)
115
+
116
+ 1. **`null_head` + `qa_null_head` (Span-Attended Calibrated Abstention)**: Bilinear span-summary + `[CLS]` margin residual with held-out temperature scaling (`p_null` AUROC `0.9933`, ECE `0.0142`; `0.9951` / `0.0127` in `331.1 MB` Int8 ONNX) and `abstention_reason` (`unassigned_placeholder` vs `missing_field`).
117
+ 2. **`span_head` (Grounded Pointer + Shannon Entropy)**: Projects `512`-token encoder states (plus query-conditioned sliding windows on ultra-long contracts) to `[start, end]` character spans with calibrated `confidence` and Shannon `span_entropy_bits`.
118
+ 3. **`bio_head` (Multi-Span List Tagging)**: Extracts ordered text and numeric tranche arrays (`type: "list"`) in a single pass.
119
+ 4. **`op_head` + `quantile_head` + Invariant Verifier**: Routes 10 System One extraction/math operations, predicts monotone `(p10, median, p90)` quantiles, and verifies cross-field algebraic constraints (`verified: true/false`).
120
+ 5. **`NullspanSystemOneEngine` (`/v1/systemone` Cross-Option & Cross-Question Judge)**: `DeBERTa-v3-Large` (`r=32` LoRA) with top-placed programmatic numeric/temporal checks, Head + IDF-Clause + Tail windowing, 2-layer `CrossOptionDecisionHead` self-attention across candidate options, Cross-Question Workflow Propagation across multi-question batches, and Cumulative Ranked Probability Score (`RPS`) ordinal calibration.
121
+
122
+ ## NullspanBench Leaderboard (`bench/run_bench.py` · 19 Systems · $N=500$ Zero-Leakage Decisions · NVIDIA L4 GPU)
123
+
124
+ Modeled after [BenchmarkHeaven's JevBench](https://benchmarkheaven.com/jev-models), **NullspanBench v3.2** evaluates `19` systems (`13` local open-weights/open-code models running live forward passes on Google Cloud NVIDIA L4 GPU, live Cloudflare Workers AI `@cf/cloudflare/clairic + @cf/cloudflare/clef`, `2` reference JSON-mode LLM decoders, and `3` published JevBench v1.7 reference anchors) across **`500` zero-leakage observations** (`336` Open structured items + `164` Sealed items: `60` Stanford SQuAD v2.0, `96` Multi-Numeric Legal/Commercial Contract fields across `12` unseen agreements, and `8` Adversarial prompt-injection traps) on four axes: **Intelligence (`I`)**, **Calibration (`C`)**, **Speed (`S`)**, and **Cost (`K`)**.
125
+
126
+ Explore interactive weight sliders, Pareto frontier plots, and 2-system radar comparisons at [**https://hemanth.github.io/nullspan/bench.html**](https://hemanth.github.io/nullspan/bench.html):
127
+
128
+ | Rank | System | Composite (`25:25:25:25`) | Capability (`50I:50C`) | Intelligence `I` (95% CI) | Calibration `C` | Speed `S` | Cost `K` | `$/1k` | `p50` Latency |
129
+ | :---: | :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
130
+ | **1** | **`nullspan-Hybrid (Fast + ONNX Int8)`** (Open) | **95.7** | **98.2** | **97.8%** `[96.6, 99.0]` | **98.6** | **92.7** | **93.8** | `$0.0003` | `0.040 ms` |
131
+ | **2** | **`nullspan-ZeroDep (Fast Engine)`** (Open) | **94.7** | `92.9` | `90.8%` `[88.2, 93.4]` | `95.0` | **93.5** | **99.8** | `$0.0001` | `0.034 ms` |
132
+ | **3** | **`nullspan-345M (PyTorch FP32 · L4 CUDA)`** (Open) | **81.3** | **98.2** | **98.0%** `[96.8, 99.2]` | **98.5** | `63.2` | `76.8` | `$0.0017` | `10.297 ms` |
133
+ | **4** | **`nullspan-345M-Int8 (ONNX WASM/CPU)`** (Open) | **78.1** | **98.2** | **97.8%** `[96.6, 99.0]` | **98.6** | `52.7` | `84.2` | `$0.0008` | `75.933 ms` |
134
+ | **5** | `H2O-Lightning-4B v1.1 (JevBench #1 Comp)` (Open 4B) | `72.5` | `75.0` | `60.0%` `[57.8, 62.3]` | `90.0` | `92.6` | `60.3` | `$0.0211` | `29.285 ms` |
135
+ | **6** | **`@cf/cloudflare/clairic + clef (Live Workers AI)`** (API) | `72.1` | `93.2` | `91.6%` `[89.2, 93.9]` | `94.9` | `51.7` | `68.2` | `$0.0041` | `91.323 ms` (`315.3 ms/doc`) |
136
+ | **7** | `basal-omni-1.0-ultra (JevBench #1 Cap)` (Open 31B) | `71.8` | `82.5` | `74.1%` `[71.8, 76.3]` | `91.0` | `87.3` | `50.5` | `$0.0447` | `102.000 ms` |
137
+ | **8** | `Jev 1.13.0 (TypeSafe AI · JevBench Ref)` (API) | `71.5` | `77.1` | `63.6%` `[61.2, 66.0]` | `90.6` | `91.5` | `54.7` | `$0.0323` | `239.102 ms` |
138
+ | **9** | **`nullspan-345M (ONNX FP32)`** (Open) | `71.2` | **98.2** | **98.0%** `[96.8, 99.2]` | **98.5** | `43.5` | `78.0` | `$0.0015` | `424.943 ms` |
139
+ | **10** | `deberta-v3-base-squad2 (184M SQuAD2 QA)` (Open) | `60.3` | `55.8` | `48.1%` `[43.9, 52.4]` | `63.4` | `65.1` | `69.2` | `$0.0037` | `7.234 ms` |
140
+ | **11** | `gpt-5.4-nano (JSON Decoder · API)` (API) | `59.8` | `92.1` | `95.2%` `[93.4, 97.0]` | `89.0` | `43.6` | `45.1` | `$0.0438` | `417.991 ms` |
141
+ | **12** | `claude-haiku-4.5 (JSON Decoder · API)` (API) | `48.8` | `92.1` | `95.4%` `[93.4, 97.2]` | `88.7` | `41.8` | `27.5` | `$0.2639` | `586.392 ms` |
142
+ | **13** | `bge-small-en-v1.5 (Dense Span+Null)` (Open) | `37.6` | `52.0` | `14.7%` `[12.0, 17.5]` | `89.2` | `64.2` | `85.5` | `$0.0007` | `8.625 ms` |
143
+ | **14** | `distilbert-base-control (Raw Control)` (Open) | `36.9` | `52.9` | `14.2%` `[11.6, 17.0]` | `91.5` | `67.7` | `81.1` | `$0.0011` | `4.479 ms` |
144
+ | **15** | `all-MiniLM-L12-v2 (Dense Span+Null)` (Open) | `36.3` | `44.0` | `14.4%` `[11.7, 17.5]` | `73.5` | `64.2` | `85.5` | `$0.0007` | `8.582 ms` |
145
+ | **16** | `all-mpnet-base-v2 (Dense Span+Null)` (Open) | `34.4` | `42.3` | `13.6%` `[10.9, 16.7]` | `71.0` | `63.3` | `77.4` | `$0.0016` | `10.257 ms` |
146
+ | **17** | `paraphrase-albert-small-v2 (Dense Span+Null)` (Open) | `33.6` | `44.5` | `12.5%` `[10.0, 15.2]` | `76.6` | `66.4` | `91.0` | `$0.0004` | `5.700 ms` |
147
+ | **18** | `paraphrase-MiniLM-L3-v2 (Dense Span+Null)` (Open) | `30.9` | `41.7` | `11.1%` `[8.7, 13.8]` | `72.3` | `69.6` | `91.0` | `$0.0004` | `3.114 ms` |
148
+ | **19** | `all-MiniLM-L6-v2 (Dense Span+Null)` (Open) | `28.5` | `40.5` | `10.0%` `[7.7, 12.3]` | `70.9` | `67.1` | `88.8` | `$0.0005` | `4.988 ms` |
149
+
150
+ See full methodology, proper scoring definitions, request-type breakdowns (`100.0%` on Multi-Numeric Contracts `n=96`, `100.0%` Choice `n=41`, `100.0%` Math & Invariants `n=51`), and head-to-head analysis in [`bench/SCIENCE_REPORT.md`](./bench/SCIENCE_REPORT.md).
151
+
152
+ ## License
153
+
154
+ MIT © [Hemanth.HM](https://h3manth.com)
{nullspan-1.2.0 → nullspan-1.2.1}/pyproject.toml RENAMED
@@ -4,8 +4,8 @@ build-backend = "flit_core.buildapi"
4
4
 
5
5
  [project]
6
6
  name = "nullspan"
7
- version = "1.2.0"
8
- description = "Calibrated p_null abstention, grounded [start, end] span extraction, arithmetic, and distribution forecasting in a single forward pass."
7
+ version = "1.2.1"
8
+ description = "Calibrated p_null abstention, grounded [start, end] span extraction, System One typed decisions, arithmetic, and distribution forecasting in a single forward pass."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
11
11
  license = { file = "LICENSE" }
@@ -20,7 +20,7 @@ classifiers = [
20
20
  dependencies = []
21
21
 
22
22
  [project.optional-dependencies]
23
- hf = ["torch>=2.0.0", "transformers>=4.40.0", "safetensors>=0.4.0", "huggingface_hub>=0.23.0"]
23
+ hf = ["torch>=2.0.0", "transformers>=4.40.0", "safetensors>=0.4.0", "huggingface_hub>=0.23.0", "sentencepiece>=0.1.99"]
24
24
 
25
25
  [project.urls]
26
26
  Homepage = "https://github.com/hemanth/nullspan"
nullspan-1.2.1/src/nullspan/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """nullspan — Calibrated p_null abstention, grounded [start, end] span extraction, System One typed decisions, and distribution forecasting."""
2
+ from .engine import DEFAULT_HF_REPO, NullspanExtractor, decide, extract
3
+
4
+ __version__ = "1.2.1"
5
+ __all__ = ["extract", "decide", "NullspanExtractor", "DEFAULT_HF_REPO"]
{nullspan-1.2.0 → nullspan-1.2.1}/src/nullspan/engine.py RENAMED
@@ -9,6 +9,7 @@ from typing import Any
9
9
 
10
10
  DEFAULT_HF_REPO = "gnumanth/nullspan"
11
11
  _HF_CACHE: dict[str, tuple[Any, Any]] = {}
12
+ _S1_CACHE: dict[str, Any] = {}
12
13
 
13
14
  MONTH_MAP = {
14
15
  "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
@@ -956,3 +957,28 @@ def extract(
956
957
  if not isinstance(state_or_fields, str) or not isinstance(fields, dict):
957
958
  raise ValueError("Pass (state: str, fields: dict) or (fields: dict) to create an extractor.")
958
959
  return _run_extraction(state_or_fields, fields, **options)
960
+
961
+
962
+ def decide(
963
+ state: Any,
964
+ questions: dict[str, dict[str, Any]],
965
+ *,
966
+ model_id: str = DEFAULT_HF_REPO,
967
+ device: str | None = None,
968
+ ) -> dict[str, Any]:
969
+ """Evaluate typed System One decisions (`noul`, `choice`, `score`) in one bidirectional encoder pass (`output_tokens = 0`)."""
970
+ from .systemone import NullspanSystemOneEngine
971
+
972
+ cache_key = f"{model_id}:{device or 'auto'}"
973
+ if cache_key not in _S1_CACHE:
974
+ if os.path.isdir(model_id):
975
+ local_dir = model_id
976
+ else:
977
+ from huggingface_hub import snapshot_download
978
+
979
+ local_dir = snapshot_download(
980
+ repo_id=model_id,
981
+ allow_patterns=["systemone/*", "systemone_decision.pt"],
982
+ )
983
+ _S1_CACHE[cache_key] = NullspanSystemOneEngine.load_from_dir(local_dir, device=device)
984
+ return _S1_CACHE[cache_key].decide(state=state, questions=questions)