anyjev 0.0.1__tar.gz

anyjev-0.0.1/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

anyjev-0.0.1/PKG-INFO

@@ -0,0 +1,167 @@
+Metadata-Version: 2.4
+Name: anyjev
+Version: 0.0.1
+Summary: Turn any LLM into a Jev-style decision model: typed decisions with calibrated probabilities, training-free by default. Not affiliated with TypeSafe AI.
+Author: Jiamu (Morris) Zhang
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/MorrisZJ/AnyJev
+Project-URL: Repository, https://github.com/MorrisZJ/AnyJev
+Project-URL: Changelog, https://github.com/MorrisZJ/AnyJev/blob/main/CHANGELOG.md
+Keywords: llm,decision,calibration,jev,system-one
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Provides-Extra: hf
+Requires-Dist: torch>=2.3; extra == "hf"
+Requires-Dist: transformers>=4.45; extra == "hf"
+Provides-Extra: bench
+Requires-Dist: datasets>=2.20; extra == "bench"
+Requires-Dist: scikit-learn>=1.4; extra == "bench"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Dynamic: license-file
+
+# AnyJev
+
+**English** | [简体中文](README.zh-CN.md)
+
+**Turn any LLM into a Jev-style decision model. Typed decisions, real probabilities, no training.**
+
+Give it a state and typed questions, get back a decision and a probability per question in one prefill, no generation. Works with the model you already run.
+
+What makes it different from reading logits with `max_tokens=1`:
+
+- **Training-free debiasing on by default (L0).** Cyclic-shift marginalization removes option-order bias; a label-free prior estimate removes the model's label bias. Zero labels.
+- **Post-hoc calibration when you have labels (L1).** Temperature scaling per (model, question), stored as a small artifact.
+- **Every result says which one you got.** `decision.level` is `raw`, `L0`, or `L1`. Downstream code can refuse to act on the wrong one.
+- **A benchmark that reports calibration.** Accuracy, Brier, ECE, order-flip rate, coverage at 5% risk, from one command.
+
+> Not affiliated with, endorsed by, or derived from TypeSafe AI or Jev. All comparisons are measured and reproducible from `bench/results/`.
+
+## Install
+
+```bash
+pip install -e ".[hf]"          # library + transformers backend
+pip install -e ".[hf,bench,dev]" # plus datasets and pytest
+```
+
+## Ten lines
+
+```python
+from anyjev import Decider, Question
+from anyjev.backends.hf import HFBackend
+
+d = Decider(HFBackend("Qwen/Qwen3-8B"))
+
+route = Question.choice("Which handler should process this request?",
+                        ["billing", "technical", "sales", "other"], name="route")
+safe  = Question.noul("Is the proposed tool call destructive or irreversible?", name="safe")
+done  = Question.score("How complete is the task on a 0 to 1 scale?", bins=5, name="done")
+
+state = {"conversation": [...], "proposed_tool_call": {...}}
+r = d.decide(state, [route, safe, done])
+
+r["route"].argmax          # "billing"
+r["route"].distribution    # {"billing": 0.81, "technical": 0.07, ...}
+r["safe"].p_true           # 0.12
+r["done"].value            # 0.35
+r.level                    # "L0"  (debiased, not calibrated)
+```
+
+With labels:
+
+```python
+art = d.calibrate(safe, calib_states, calib_labels)   # ~100 to 500 examples -> L1 artifact
+r = d.decide(state, [safe], level="L1")
+```
+
+## Why L0 is not optional
+
+A `noul` question, "Is this email spam?". Raw readout on the email gives P(Yes) = 0.62. The same prompt with the email replaced by `N/A` gives P(Yes) = 0.70: the model leans Yes regardless of content. Divide by that prior and renormalize, and the answer is P(Yes) = 0.41. The judgment flips. Position bias does the same thing to `choice` questions when you reorder the options. Both are fixable without training, and AnyJev fixes them by default.
+
+## Levels
+
+Full contract in [docs/levels.md](docs/levels.md).
+
+| Level | Needs | Does | Does not |
+|---|---|---|---|
+| `raw` | nothing | restricted softmax over label tokens (what the clones do) | anything about bias or calibration |
+| `L0` | nothing | removes position bias and label-prior bias | make the model's uncertainty calibrated |
+| `L1` | 100 to 500 labels per question | temperature scaling on top of L0 | survive distribution shift beyond the calibration set |
+
+## Benchmark
+
+```bash
+python -m bench.run --model Qwen/Qwen3-8B --tasks newsgroups,injection,banking20 --n 300 --calib 200
+```
+
+Results land in `bench/results/<date>/` as Markdown and JSON with hardware and library versions. `python -m bench.table bench/results/<date>` regenerates the table below; nothing is typed in by hand.
+
+### Reorder the options and one in five answers changes
+
+Three open models, three tasks, 300 test items each. `flip` is the fraction of items whose answer changes when the option list is reversed (`choice`) or the phrasing is swapped between "Yes or No" and "No or Yes" (`noul`). `raw` is what every logit-reading clone does. `L0` is AnyJev's default, zero labels. `L1` adds one temperature fit on 200 labels.
+
+| model | task | K | raw flip | L0 flip | raw acc | L0 acc | raw ECE | L1 ECE |
+|---|---|---|---|---|---|---|---|---|
+| Qwen3-8B | banking20 | 20 | 0.227 | 0.077 | 0.750 | 0.807 | 0.235 | 0.100 |
+| Qwen3-8B | newsgroups | 20 | 0.237 | 0.173 | 0.640 | 0.660 | 0.331 | 0.157 |
+| Qwen3-8B | injection | 2 | 0.060 | 0.000 | 0.693 | 0.710 | 0.287 | 0.162 |
+| Qwen2.5-7B-Instruct | banking20 | 20 | 0.197 | 0.067 | 0.723 | 0.767 | 0.236 | 0.072 |
+| Qwen2.5-7B-Instruct | newsgroups | 20 | 0.233 | 0.123 | 0.660 | 0.707 | 0.276 | 0.082 |
+| Qwen2.5-7B-Instruct | injection | 2 | 0.053 | 0.000 | 0.737 | 0.813 | 0.189 | 0.037 |
+| Qwen3-30B-A3B-Instruct-2507 | banking20 | 20 | 0.143 | 0.097 | 0.733 | 0.767 | 0.246 | 0.079 |
+| Qwen3-30B-A3B-Instruct-2507 | newsgroups | 20 | 0.133 | 0.100 | 0.730 | 0.740 | 0.249 | 0.086 |
+| Qwen3-30B-A3B-Instruct-2507 | injection | 2 | 0.093 | 0.000 | 0.723 | 0.767 | 0.253 | 0.080 |
+
+Full table with every ablation row (permutation only, each prior alone, Brier, coverage at 5% risk): [docs/results_bench.md](docs/results_bench.md). One H100, bf16, transformers 4.55.4; regenerate with `python -m bench.table bench/results_batchprior_v0/2026-09-20`.
+
+### On Laya's own benchmark, zero-shot
+
+| system | acc | soft_acc | ece | brier_mean | score_mae |
+|---|---|---|---|---|---|
+| laya-multilingual (zero-shot), measured here | 0.340 | 0.325 | 0.287 | 0.269 | 0.688 |
+| laya (zero-shot), measured here | 0.359 | 0.331 | 0.177 | 0.227 | 0.694 |
+| Qwen2.5-7B-Instruct + raw logits (clone baseline) | 0.621 | 0.514 | 0.287 | 0.209 | 0.437 |
+| Qwen3-8B + raw logits (clone baseline) | 0.626 | 0.520 | 0.328 | 0.210 | 0.621 |
+| Qwen2.5-7B-Instruct + AnyJev L0, zero-shot | 0.628 | 0.506 | 0.200 | 0.176 | 0.451 |
+| Qwen2.5-7B-Instruct + AnyJev L1, temperature from 200 train cases | 0.632 | 0.452 | 0.047 | 0.149 | 0.443 |
+| Qwen3-8B + AnyJev L0, zero-shot | 0.640 | 0.523 | 0.273 | 0.196 | 0.617 |
+| Qwen3-8B + AnyJev L1, temperature from 200 train cases | 0.646 | 0.457 | 0.056 | 0.143 | 0.474 |
+| Qwen3-32B + raw logits (clone baseline) | 0.684 | 0.556 | 0.206 | 0.144 | 0.488 |
+| Qwen3-32B + AnyJev L0, zero-shot | 0.700 | 0.548 | 0.133 | 0.128 | 0.456 |
+| Qwen3-32B + AnyJev L1, temperature from 200 train cases | 0.701 | 0.502 | 0.034 | 0.120 | 0.412 |
+| Jev 1.13.0 (published by TypeSafe / Laya; not rerun) | 0.727 | 0.580 | 0.144 | 0.148 | 0.391 |
+| laya-typed-decisions (fine-tuned on this set's train split), measured here | 0.768 | 0.471 | 0.215 | 0.118 | 0.243 |
+
+Laya's headline is 0.766 on this set, above Jev's 0.727. All rows except Jev were measured here on the same 2,000 decisions; the Laya fine-tuned checkpoint reproduces its published number. Read it two ways. On argmax accuracy, Laya fine-tuned on the train split wins, and a 32B open model with zero training is 2.7 points behind Jev. On the probabilities, which is what a System One model is for, the order flips: the fine-tuned Laya's soft accuracy (0.471) is below every Qwen row, and its ECE (0.215) is six times AnyJev L1's (0.034). Laya's zero-shot checkpoints, the ones you would use on a question they were not trained for, score 0.34 to 0.36 against a 0.32 random baseline. Full table with per-workflow and per-type breakdown: [docs/results_typed.md](docs/results_typed.md); regenerate with `python -m bench.run_typed --model <model>` and `python -m bench.providers.laya`.
+
+### Inside NanoJev's maze harness
+
+| engine | goal test | goal ood | attempts | collisions | edge acc | majority | edge Brier | edge questions |
+|---|---|---|---|---|---|---|---|---|
+| Qwen3-0.6B + AnyJev L0 (batch prior) | 10/11 | 3/4 | 15616 | 6236 | 0.490 | 0.618 | 0.271 | 23156 |
+| Qwen3-0.6B + AnyJev L0 (content_free prior) | 10/11 | 4/4 | 16278 | 6563 | 0.490 | 0.614 | 0.274 | 23284 |
+| Qwen3-0.6B + AnyJev L0 (none prior) | 10/11 | 3/4 | 21851 | 9136 | 0.403 | 0.597 | 0.460 | 28040 |
+| Qwen3-0.6B + AnyJev raw | 11/11 | 4/4 | 5825 | 2616 | 0.537 | 0.539 | 0.363 | 10944 |
+| Qwen3-8B + AnyJev L0 (batch prior) | 11/11 | 3/4 | 17841 | 7171 | 0.555 | 0.600 | 0.344 | 25124 |
+| Qwen3-0.6B native A/B readout (NanoJev's 'Untuned Qwen' protocol) | 10/11 | 3/4 | 20555 | 8496 | 0.419 | 0.607 | 0.305 | 27660 |
+
+NanoJev's README compares its trained 0.6B model against "Untuned Qwen3-0.6B", which reads A/B logits for four Boolean questions per maze cell. We ran NanoJev's own frozen exploration code on its 15 test and out-of-distribution mazes and only swapped the engine. Two things are true at once. The untuned model's result depends heavily on how you read it: the same Qwen3-0.6B goes from 13/15 mazes and 20,555 attempts under NanoJev's A/B readout to 15/15 and 5,825 attempts under AnyJev's raw Yes/No readout. And no LLM readout, not even Qwen3-8B, answers "is one step north clear?" better than always saying the majority label (edge accuracy 0.42 to 0.56 against a 0.54 to 0.62 majority). The maze differences come from how each readout's average probability interacts with the controller's p >= 0.5 probe rule, not from map reading. We report it because it is the comparison NanoJev invites; we do not headline it. Full table: [docs/results_maze.md](docs/results_maze.md).
+
+## Status and roadmap
+
+v0.0.1. The library, both backends, and all three benches are real and measured; the tables above are regenerated from committed JSON. Actively developed: the plan with dates is in [ROADMAP.md](ROADMAP.md) (next up: PyPI release, span readout for more than 26 options, conformal abstention, latency column, a live demo, Llama and Gemma rows; then a Jev-compatible server, more backends, and multimodal state). Backends and bench providers are one file each; see [CONTRIBUTING.md](CONTRIBUTING.md). What landed: [CHANGELOG.md](CHANGELOG.md). Who we build on: [CREDITS.md](CREDITS.md).
+
+## License
+
+Apache-2.0. Datasets keep their own licenses, see `THIRD_PARTY.md`.

anyjev-0.0.1/README.md

@@ -0,0 +1,134 @@
+# AnyJev
+
+**English** | [简体中文](README.zh-CN.md)
+
+**Turn any LLM into a Jev-style decision model. Typed decisions, real probabilities, no training.**
+
+Give it a state and typed questions, get back a decision and a probability per question in one prefill, no generation. Works with the model you already run.
+
+What makes it different from reading logits with `max_tokens=1`:
+
+- **Training-free debiasing on by default (L0).** Cyclic-shift marginalization removes option-order bias; a label-free prior estimate removes the model's label bias. Zero labels.
+- **Post-hoc calibration when you have labels (L1).** Temperature scaling per (model, question), stored as a small artifact.
+- **Every result says which one you got.** `decision.level` is `raw`, `L0`, or `L1`. Downstream code can refuse to act on the wrong one.
+- **A benchmark that reports calibration.** Accuracy, Brier, ECE, order-flip rate, coverage at 5% risk, from one command.
+
+> Not affiliated with, endorsed by, or derived from TypeSafe AI or Jev. All comparisons are measured and reproducible from `bench/results/`.
+
+## Install
+
+```bash
+pip install -e ".[hf]"          # library + transformers backend
+pip install -e ".[hf,bench,dev]" # plus datasets and pytest
+```
+
+## Ten lines
+
+```python
+from anyjev import Decider, Question
+from anyjev.backends.hf import HFBackend
+
+d = Decider(HFBackend("Qwen/Qwen3-8B"))
+
+route = Question.choice("Which handler should process this request?",
+                        ["billing", "technical", "sales", "other"], name="route")
+safe  = Question.noul("Is the proposed tool call destructive or irreversible?", name="safe")
+done  = Question.score("How complete is the task on a 0 to 1 scale?", bins=5, name="done")
+
+state = {"conversation": [...], "proposed_tool_call": {...}}
+r = d.decide(state, [route, safe, done])
+
+r["route"].argmax          # "billing"
+r["route"].distribution    # {"billing": 0.81, "technical": 0.07, ...}
+r["safe"].p_true           # 0.12
+r["done"].value            # 0.35
+r.level                    # "L0"  (debiased, not calibrated)
+```
+
+With labels:
+
+```python
+art = d.calibrate(safe, calib_states, calib_labels)   # ~100 to 500 examples -> L1 artifact
+r = d.decide(state, [safe], level="L1")
+```
+
+## Why L0 is not optional
+
+A `noul` question, "Is this email spam?". Raw readout on the email gives P(Yes) = 0.62. The same prompt with the email replaced by `N/A` gives P(Yes) = 0.70: the model leans Yes regardless of content. Divide by that prior and renormalize, and the answer is P(Yes) = 0.41. The judgment flips. Position bias does the same thing to `choice` questions when you reorder the options. Both are fixable without training, and AnyJev fixes them by default.
+
+## Levels
+
+Full contract in [docs/levels.md](docs/levels.md).
+
+| Level | Needs | Does | Does not |
+|---|---|---|---|
+| `raw` | nothing | restricted softmax over label tokens (what the clones do) | anything about bias or calibration |
+| `L0` | nothing | removes position bias and label-prior bias | make the model's uncertainty calibrated |
+| `L1` | 100 to 500 labels per question | temperature scaling on top of L0 | survive distribution shift beyond the calibration set |
+
+## Benchmark
+
+```bash
+python -m bench.run --model Qwen/Qwen3-8B --tasks newsgroups,injection,banking20 --n 300 --calib 200
+```
+
+Results land in `bench/results/<date>/` as Markdown and JSON with hardware and library versions. `python -m bench.table bench/results/<date>` regenerates the table below; nothing is typed in by hand.
+
+### Reorder the options and one in five answers changes
+
+Three open models, three tasks, 300 test items each. `flip` is the fraction of items whose answer changes when the option list is reversed (`choice`) or the phrasing is swapped between "Yes or No" and "No or Yes" (`noul`). `raw` is what every logit-reading clone does. `L0` is AnyJev's default, zero labels. `L1` adds one temperature fit on 200 labels.
+
+| model | task | K | raw flip | L0 flip | raw acc | L0 acc | raw ECE | L1 ECE |
+|---|---|---|---|---|---|---|---|---|
+| Qwen3-8B | banking20 | 20 | 0.227 | 0.077 | 0.750 | 0.807 | 0.235 | 0.100 |
+| Qwen3-8B | newsgroups | 20 | 0.237 | 0.173 | 0.640 | 0.660 | 0.331 | 0.157 |
+| Qwen3-8B | injection | 2 | 0.060 | 0.000 | 0.693 | 0.710 | 0.287 | 0.162 |
+| Qwen2.5-7B-Instruct | banking20 | 20 | 0.197 | 0.067 | 0.723 | 0.767 | 0.236 | 0.072 |
+| Qwen2.5-7B-Instruct | newsgroups | 20 | 0.233 | 0.123 | 0.660 | 0.707 | 0.276 | 0.082 |
+| Qwen2.5-7B-Instruct | injection | 2 | 0.053 | 0.000 | 0.737 | 0.813 | 0.189 | 0.037 |
+| Qwen3-30B-A3B-Instruct-2507 | banking20 | 20 | 0.143 | 0.097 | 0.733 | 0.767 | 0.246 | 0.079 |
+| Qwen3-30B-A3B-Instruct-2507 | newsgroups | 20 | 0.133 | 0.100 | 0.730 | 0.740 | 0.249 | 0.086 |
+| Qwen3-30B-A3B-Instruct-2507 | injection | 2 | 0.093 | 0.000 | 0.723 | 0.767 | 0.253 | 0.080 |
+
+Full table with every ablation row (permutation only, each prior alone, Brier, coverage at 5% risk): [docs/results_bench.md](docs/results_bench.md). One H100, bf16, transformers 4.55.4; regenerate with `python -m bench.table bench/results_batchprior_v0/2026-09-20`.
+
+### On Laya's own benchmark, zero-shot
+
+| system | acc | soft_acc | ece | brier_mean | score_mae |
+|---|---|---|---|---|---|
+| laya-multilingual (zero-shot), measured here | 0.340 | 0.325 | 0.287 | 0.269 | 0.688 |
+| laya (zero-shot), measured here | 0.359 | 0.331 | 0.177 | 0.227 | 0.694 |
+| Qwen2.5-7B-Instruct + raw logits (clone baseline) | 0.621 | 0.514 | 0.287 | 0.209 | 0.437 |
+| Qwen3-8B + raw logits (clone baseline) | 0.626 | 0.520 | 0.328 | 0.210 | 0.621 |
+| Qwen2.5-7B-Instruct + AnyJev L0, zero-shot | 0.628 | 0.506 | 0.200 | 0.176 | 0.451 |
+| Qwen2.5-7B-Instruct + AnyJev L1, temperature from 200 train cases | 0.632 | 0.452 | 0.047 | 0.149 | 0.443 |
+| Qwen3-8B + AnyJev L0, zero-shot | 0.640 | 0.523 | 0.273 | 0.196 | 0.617 |
+| Qwen3-8B + AnyJev L1, temperature from 200 train cases | 0.646 | 0.457 | 0.056 | 0.143 | 0.474 |
+| Qwen3-32B + raw logits (clone baseline) | 0.684 | 0.556 | 0.206 | 0.144 | 0.488 |
+| Qwen3-32B + AnyJev L0, zero-shot | 0.700 | 0.548 | 0.133 | 0.128 | 0.456 |
+| Qwen3-32B + AnyJev L1, temperature from 200 train cases | 0.701 | 0.502 | 0.034 | 0.120 | 0.412 |
+| Jev 1.13.0 (published by TypeSafe / Laya; not rerun) | 0.727 | 0.580 | 0.144 | 0.148 | 0.391 |
+| laya-typed-decisions (fine-tuned on this set's train split), measured here | 0.768 | 0.471 | 0.215 | 0.118 | 0.243 |
+
+Laya's headline is 0.766 on this set, above Jev's 0.727. All rows except Jev were measured here on the same 2,000 decisions; the Laya fine-tuned checkpoint reproduces its published number. Read it two ways. On argmax accuracy, Laya fine-tuned on the train split wins, and a 32B open model with zero training is 2.7 points behind Jev. On the probabilities, which is what a System One model is for, the order flips: the fine-tuned Laya's soft accuracy (0.471) is below every Qwen row, and its ECE (0.215) is six times AnyJev L1's (0.034). Laya's zero-shot checkpoints, the ones you would use on a question they were not trained for, score 0.34 to 0.36 against a 0.32 random baseline. Full table with per-workflow and per-type breakdown: [docs/results_typed.md](docs/results_typed.md); regenerate with `python -m bench.run_typed --model <model>` and `python -m bench.providers.laya`.
+
+### Inside NanoJev's maze harness
+
+| engine | goal test | goal ood | attempts | collisions | edge acc | majority | edge Brier | edge questions |
+|---|---|---|---|---|---|---|---|---|
+| Qwen3-0.6B + AnyJev L0 (batch prior) | 10/11 | 3/4 | 15616 | 6236 | 0.490 | 0.618 | 0.271 | 23156 |
+| Qwen3-0.6B + AnyJev L0 (content_free prior) | 10/11 | 4/4 | 16278 | 6563 | 0.490 | 0.614 | 0.274 | 23284 |
+| Qwen3-0.6B + AnyJev L0 (none prior) | 10/11 | 3/4 | 21851 | 9136 | 0.403 | 0.597 | 0.460 | 28040 |
+| Qwen3-0.6B + AnyJev raw | 11/11 | 4/4 | 5825 | 2616 | 0.537 | 0.539 | 0.363 | 10944 |
+| Qwen3-8B + AnyJev L0 (batch prior) | 11/11 | 3/4 | 17841 | 7171 | 0.555 | 0.600 | 0.344 | 25124 |
+| Qwen3-0.6B native A/B readout (NanoJev's 'Untuned Qwen' protocol) | 10/11 | 3/4 | 20555 | 8496 | 0.419 | 0.607 | 0.305 | 27660 |
+
+NanoJev's README compares its trained 0.6B model against "Untuned Qwen3-0.6B", which reads A/B logits for four Boolean questions per maze cell. We ran NanoJev's own frozen exploration code on its 15 test and out-of-distribution mazes and only swapped the engine. Two things are true at once. The untuned model's result depends heavily on how you read it: the same Qwen3-0.6B goes from 13/15 mazes and 20,555 attempts under NanoJev's A/B readout to 15/15 and 5,825 attempts under AnyJev's raw Yes/No readout. And no LLM readout, not even Qwen3-8B, answers "is one step north clear?" better than always saying the majority label (edge accuracy 0.42 to 0.56 against a 0.54 to 0.62 majority). The maze differences come from how each readout's average probability interacts with the controller's p >= 0.5 probe rule, not from map reading. We report it because it is the comparison NanoJev invites; we do not headline it. Full table: [docs/results_maze.md](docs/results_maze.md).
+
+## Status and roadmap
+
+v0.0.1. The library, both backends, and all three benches are real and measured; the tables above are regenerated from committed JSON. Actively developed: the plan with dates is in [ROADMAP.md](ROADMAP.md) (next up: PyPI release, span readout for more than 26 options, conformal abstention, latency column, a live demo, Llama and Gemma rows; then a Jev-compatible server, more backends, and multimodal state). Backends and bench providers are one file each; see [CONTRIBUTING.md](CONTRIBUTING.md). What landed: [CHANGELOG.md](CHANGELOG.md). Who we build on: [CREDITS.md](CREDITS.md).
+
+## License
+
+Apache-2.0. Datasets keep their own licenses, see `THIRD_PARTY.md`.

anyjev-0.0.1/anyjev/__init__.py

@@ -0,0 +1,14 @@
+"""AnyJev: turn any causal LLM into a Jev-style decision model.
+
+Typed decisions (choice / score / noul) with probabilities, read from the
+model's next-token logits in one prefill, no generation. Training-free
+debiasing (L0) is on by default; post-hoc calibration (L1) when you have labels.
+
+Not affiliated with, endorsed by, or derived from TypeSafe AI or Jev.
+"""
+from anyjev.decider import Decider
+from anyjev.question import Question
+from anyjev.result import Decision, DecisionSet
+
+__all__ = ["Question", "Decision", "DecisionSet", "Decider"]
+__version__ = "0.0.1"

anyjev-0.0.1/anyjev/backends/__init__.py

@@ -0,0 +1,3 @@
+from anyjev.backends.base import Backend
+
+__all__ = ["Backend"]

anyjev-0.0.1/anyjev/backends/base.py

@@ -0,0 +1,21 @@
+"""Backend protocol. A backend does exactly one thing: given rendered prompts,
+return the next-token log-probabilities of the requested token ids.
+Everything else (debiasing, calibration, abstention) lives above it."""
+from __future__ import annotations
+
+from typing import Any, List, Protocol, Sequence
+
+import numpy as np
+
+
+class Backend(Protocol):
+    tokenizer: Any   # must provide .encode(text, add_special_tokens=False) and optionally .chat_template
+    name: str
+
+    def next_token_logprobs(self, prompts: Sequence[str],
+                            token_ids: Sequence[Sequence[int]]) -> List[np.ndarray]:
+        """For prompt i, return log p(token | prompt_i) for each id in token_ids[i],
+        taken from the full-vocabulary log-softmax at the last position.
+        Decision mode never samples; a backend that cannot expose restricted
+        next-token logits is not a supported backend."""
+        ...

anyjev-0.0.1/anyjev/backends/fake.py

@@ -0,0 +1,94 @@
+"""A synthetic backend with known, injectable biases. Used by the unit tests
+and by the docs to show what L0 removes. It parses the prompts that
+anyjev.readout builds, so it exercises the real prompt path."""
+from __future__ import annotations
+
+import re
+from typing import Callable, Dict, List, Optional, Sequence
+
+import numpy as np
+
+from anyjev.calibrate.contextual import DEFAULT_PROBES
+
+_OPT_LINE = re.compile(r"^([A-Z]|\d+)\. (.*)$")
+_NOUL_LINE = re.compile(r"^Answer (Yes|No) or (Yes|No)\.$")
+
+
+class FakeTokenizer:
+    chat_template = None
+
+    def __init__(self):
+        self._vocab: Dict[str, int] = {}
+
+    def encode(self, text: str, add_special_tokens: bool = False) -> List[int]:
+        # every label we care about is one token; anything else is "long"
+        if re.fullmatch(r" ?([A-Z]|\d|Yes|No)", text):
+            key = text.strip()
+            if key not in self._vocab:
+                self._vocab[key] = 1000 + len(self._vocab)
+            return [self._vocab[key]]
+        return [1, 2]
+
+    def id_to_label(self, tid: int) -> str:
+        for k, v in self._vocab.items():
+            if v == tid:
+                return k
+        raise KeyError(tid)
+
+
+class FakeBackend:
+    """logit(position j, option o) = content(state, o) + position_bias[j] + label_prior[label_j]
+
+    content is 0 on content-free probes, so the prior is exactly the bias term.
+    """
+
+    def __init__(self, content: Callable[[str, str], float],
+                 position_bias: Optional[Sequence[float]] = None,
+                 label_prior: Optional[Dict[str, float]] = None,
+                 temperature: float = 1.0):
+        self.name = "fake"
+        self.tokenizer = FakeTokenizer()
+        self.content = content
+        self.position_bias = list(position_bias or [])
+        self.label_prior = dict(label_prior or {})
+        self.temperature = temperature
+        self.calls = 0
+        self.prompts_seen = 0
+
+    def _parse(self, prompt: str):
+        state = prompt.split("State:\n", 1)[1].split("\n\nQuestion:", 1)[0]
+        if state == "(empty)":
+            state = ""
+        labels, options = [], []
+        for line in prompt.splitlines():
+            m = _OPT_LINE.match(line)
+            if m:
+                labels.append(m.group(1))
+                options.append(m.group(2))
+            m = _NOUL_LINE.match(line)
+            if m:
+                labels = [m.group(1), m.group(2)]
+                options = list(labels)
+        return state, labels, options
+
+    def next_token_logprobs(self, prompts: Sequence[str],
+                            token_ids: Sequence[Sequence[int]]) -> List[np.ndarray]:
+        self.calls += 1
+        self.prompts_seen += len(prompts)
+        out = []
+        for prompt, ids in zip(prompts, token_ids):
+            state, labels, options = self._parse(prompt)
+            is_probe = state in DEFAULT_PROBES
+            logits = np.zeros(len(ids))
+            by_label = {self.tokenizer.id_to_label(t): k for k, t in enumerate(ids)}
+            for j, (lab, opt) in enumerate(zip(labels, options)):
+                z = 0.0 if is_probe else self.content(state, opt)
+                if j < len(self.position_bias):
+                    z += self.position_bias[j]
+                z += self.label_prior.get(lab, 0.0)
+                logits[by_label[lab]] = z / self.temperature
+            # full-vocab log-softmax: pretend a bit of mass lives elsewhere
+            z = np.concatenate([logits, [-5.0]])
+            lp = z - np.log(np.exp(z - z.max()).sum()) - z.max()
+            out.append(lp[:-1])
+        return out