tokenfold 0.5.1__tar.gz → 0.5.2__tar.gz

{tokenfold-0.5.1 → tokenfold-0.5.2}/Cargo.lock

@@ -847,9 +847,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.45"
+version = "0.23.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
+checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
 dependencies = [
  "log",
  "once_cell",
@@ -1109,7 +1109,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "serde",
  "serde_json",
@@ -1118,7 +1118,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1128,7 +1128,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "clap",
  "serde",
@@ -1143,7 +1143,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "proptest",
  "regex",
@@ -1158,25 +1158,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.5.1"
+version = "0.5.2"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "clap",
  "ctrlc",
@@ -1190,7 +1190,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1199,7 +1199,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "tokenfold-core",
 ]

{tokenfold-0.5.1 → tokenfold-0.5.2}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.5.1"
+version = "0.5.2"
 edition = "2024"
 rust-version = "1.98"
 license = "Apache-2.0"

{tokenfold-0.5.1 → tokenfold-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenfold
-Version: 0.5.1
+Version: 0.5.2
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -361,7 +361,53 @@ Tokenfold solves a different problem first: repeated keys, values, schemas, rows
 
 **When structure ends, rank what matters.**
 
-Tokenfold Select is an Apache-2.0 LoRA adapter on
+**Select v2 is the default scorer backend in the optional `tokenfold-select`
+0.5.2 package: local FP32 ONNX inference, original text, whole-group ranking.**
+Provision the separate pinned model artifact, generate explicit runtime approval,
+and invoke `tokenfold --experimental select`. Core remains model-free and
+deterministic; installing or upgrading Tokenfold does not silently load a model
+or enable lossy selection. See the [Select v2 setup and limits](packages/tokenfold-select/README.md).
+The separate model bundle also carries Safetensors weights and a custom reference
+loader; ONNX remains the CLI worker default. V2 is an experimental preview:
+[Both formats are published under Apache-2.0](https://huggingface.co/snchimata/tokenfold-select-v2).
+Synthetic CPU parity is verified, but task-quality and cold/warm performance
+qualification are still open. The [example notebook](examples/quickstart.ipynb#10-tokenfold-select-v2-optional-local-ranking)
+keeps model execution off until you explicitly enable it.
+
+<details>
+<summary><strong>Python: try Tokenfold Select v2 (ONNX)</strong></summary>
+
+Install `tokenfold-select[onnx]==0.5.2` and `huggingface_hub==1.33.0`, then download
+only the ONNX profile at the pinned release commit:
+
+```python
+from pathlib import Path
+from huggingface_hub import snapshot_download
+from tokenfold_select import OnnxScorer
+
+root = Path(snapshot_download(
+    "snchimata/tokenfold-select-v2",
+    revision="78736d9afea819322f212d6d53eedcf68ac7a39b",
+    allow_patterns=["model.onnx", "model.onnx.data", "onnx-manifest.json",
+                    "tokenizer/*", "runtime/*", "LICENSE", "NOTICE.md"],
+)).resolve()
+scorer = OnnxScorer(root, development=True)
+response = scorer.score({
+    "schema_version": 1, "model_revision": scorer.revision,
+    "query": "Which delivery city?",
+    "groups": [{"id": "delivery", "text": "Delivery city: Paris"}],
+})
+print(response)  # scores only; your controller owns protection and allocation
+```
+
+For Safetensors, use the model card's custom loader rather than a stock
+Transformers classification class. Neither format silently changes Core behavior.
+
+</details>
+
+### Tokenfold Select v1 (legacy model)
+
+The separately published Select v1 is an Apache-2.0 LoRA adapter on
 `ibm-granite/granite-embedding-reranker-english-r2`. It scores candidate spans
 against a query; your allocator applies the token budget and force-keeps
 required content. It is a separately distributed offering. Core remains
@@ -374,7 +420,7 @@ model-free and deterministic and neither loads nor invokes Select.
 | Output | Compressed payload + exact receipt | Ranking logits |
 
 <details>
-<summary><strong>Python: load the model and score spans</strong></summary>
+<summary><strong>Legacy v1 Python: load the LoRA adapter and score spans</strong></summary>
 
 ```python
 from pathlib import Path
@@ -415,7 +461,7 @@ def score(query: str, spans: list[str]) -> list[float]:
 
 </details>
 
-### Tokenfold Select benchmarks
+### Tokenfold Select v1 benchmarks (not v2 evidence)
 
 [Tokenfold Select][tokenfold-select] is Tokenfold's query-aware model for choosing
 what fills a tight context window after structural compression ends.

{tokenfold-0.5.1 → tokenfold-0.5.2}/README.md

@@ -339,7 +339,53 @@ Tokenfold solves a different problem first: repeated keys, values, schemas, rows
 
 **When structure ends, rank what matters.**
 
-Tokenfold Select is an Apache-2.0 LoRA adapter on
+**Select v2 is the default scorer backend in the optional `tokenfold-select`
+0.5.2 package: local FP32 ONNX inference, original text, whole-group ranking.**
+Provision the separate pinned model artifact, generate explicit runtime approval,
+and invoke `tokenfold --experimental select`. Core remains model-free and
+deterministic; installing or upgrading Tokenfold does not silently load a model
+or enable lossy selection. See the [Select v2 setup and limits](packages/tokenfold-select/README.md).
+The separate model bundle also carries Safetensors weights and a custom reference
+loader; ONNX remains the CLI worker default. V2 is an experimental preview:
+[Both formats are published under Apache-2.0](https://huggingface.co/snchimata/tokenfold-select-v2).
+Synthetic CPU parity is verified, but task-quality and cold/warm performance
+qualification are still open. The [example notebook](examples/quickstart.ipynb#10-tokenfold-select-v2-optional-local-ranking)
+keeps model execution off until you explicitly enable it.
+
+<details>
+<summary><strong>Python: try Tokenfold Select v2 (ONNX)</strong></summary>
+
+Install `tokenfold-select[onnx]==0.5.2` and `huggingface_hub==1.33.0`, then download
+only the ONNX profile at the pinned release commit:
+
+```python
+from pathlib import Path
+from huggingface_hub import snapshot_download
+from tokenfold_select import OnnxScorer
+
+root = Path(snapshot_download(
+    "snchimata/tokenfold-select-v2",
+    revision="78736d9afea819322f212d6d53eedcf68ac7a39b",
+    allow_patterns=["model.onnx", "model.onnx.data", "onnx-manifest.json",
+                    "tokenizer/*", "runtime/*", "LICENSE", "NOTICE.md"],
+)).resolve()
+scorer = OnnxScorer(root, development=True)
+response = scorer.score({
+    "schema_version": 1, "model_revision": scorer.revision,
+    "query": "Which delivery city?",
+    "groups": [{"id": "delivery", "text": "Delivery city: Paris"}],
+})
+print(response)  # scores only; your controller owns protection and allocation
+```
+
+For Safetensors, use the model card's custom loader rather than a stock
+Transformers classification class. Neither format silently changes Core behavior.
+
+</details>
+
+### Tokenfold Select v1 (legacy model)
+
+The separately published Select v1 is an Apache-2.0 LoRA adapter on
 `ibm-granite/granite-embedding-reranker-english-r2`. It scores candidate spans
 against a query; your allocator applies the token budget and force-keeps
 required content. It is a separately distributed offering. Core remains
@@ -352,7 +398,7 @@ model-free and deterministic and neither loads nor invokes Select.
 | Output | Compressed payload + exact receipt | Ranking logits |
 
 <details>
-<summary><strong>Python: load the model and score spans</strong></summary>
+<summary><strong>Legacy v1 Python: load the LoRA adapter and score spans</strong></summary>
 
 ```python
 from pathlib import Path
@@ -393,7 +439,7 @@ def score(query: str, spans: list[str]) -> list[float]:
 
 </details>
 
-### Tokenfold Select benchmarks
+### Tokenfold Select v1 benchmarks (not v2 evidence)
 
 [Tokenfold Select][tokenfold-select] is Tokenfold's query-aware model for choosing
 what fills a tight context window after structural compression ends.

{tokenfold-0.5.1 → tokenfold-0.5.2}/crates/tokenfold-py/Cargo.toml

@@ -16,7 +16,7 @@ name = "tokenfold_py"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-tokenfold-core = { version = "0.5.1", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.5.2", path = "../tokenfold-core" }
 serde_json = { workspace = true }
 pyo3 = { version = "0.29.0", features = ["abi3-py39"] }