tokenfold 0.5.4__tar.gz → 0.6.0__tar.gz

{tokenfold-0.5.4 → tokenfold-0.6.0}/Cargo.lock

@@ -242,6 +242,15 @@ version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
 
+[[package]]
+name = "convert_case"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.17"
@@ -261,6 +270,12 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "ctor"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
+
 [[package]]
 name = "ctrlc"
 version = "3.5.2"
@@ -406,6 +421,94 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "futures"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"
+
+[[package]]
+name = "futures-task"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
+
+[[package]]
+name = "futures-util"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "slab",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -518,6 +621,16 @@ version = "0.2.186"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
+[[package]]
+name = "libloading"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -536,6 +649,64 @@ version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
 
+[[package]]
+name = "napi"
+version = "3.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "907ff7366fb7182e28edd0f08ef8ac5d6b13b8d65a3c0b3a10bf3ac092437d6b"
+dependencies = [
+ "bitflags",
+ "ctor",
+ "futures",
+ "libc",
+ "napi-build",
+ "napi-sys",
+ "nohash-hasher",
+ "rustc-hash 2.1.3",
+]
+
+[[package]]
+name = "napi-build"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b899b545d3aa6dca985939059f258c5488d34e4ecf39c274e20009748f4b846d"
+
+[[package]]
+name = "napi-derive"
+version = "3.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d32eb00f92f84cf462fc328f49082b29c2a7d2eebeb1d447d19c1f9fdd9f89ee"
+dependencies = [
+ "convert_case",
+ "ctor",
+ "napi-derive-backend",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
+[[package]]
+name = "napi-derive-backend"
+version = "7.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1fe28c64d8fc12e69f9b817d0febdbe7914555760f1e353106281b8ff99cc90"
+dependencies = [
+ "convert_case",
+ "proc-macro2",
+ "quote",
+ "semver",
+ "syn 3.0.4",
+]
+
+[[package]]
+name = "napi-sys"
+version = "3.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d153beb7d5c973f4b9395be8a83054a28968b61188f1dd1a2cc2321af22daea2"
+dependencies = [
+ "libloading",
+]
+
 [[package]]
 name = "nix"
 version = "0.31.3"
@@ -548,6 +719,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nohash-hasher"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -590,6 +767,12 @@ version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
 [[package]]
 name = "pkcs8"
 version = "0.10.2"
@@ -823,6 +1006,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
+[[package]]
+name = "rustc-hash"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.1"
@@ -977,6 +1166,12 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
+[[package]]
+name = "slab"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+
 [[package]]
 name = "spki"
 version = "0.7.3"
@@ -1092,7 +1287,7 @@ dependencies = [
  "fancy-regex",
  "lazy_static",
  "regex",
- "rustc-hash",
+ "rustc-hash 1.1.0",
 ]
 
 [[package]]
@@ -1109,7 +1304,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "serde",
  "serde_json",
@@ -1118,7 +1313,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1128,7 +1323,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "clap",
  "serde",
@@ -1143,7 +1338,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "proptest",
  "regex",
@@ -1158,25 +1353,38 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.5.4"
+version = "0.6.0"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "tokenfold-core",
 ]
 
+[[package]]
+name = "tokenfold-node"
+version = "0.6.0"
+dependencies = [
+ "napi",
+ "napi-build",
+ "napi-derive",
+ "serde",
+ "serde_json",
+ "tokenfold-adapters",
+ "tokenfold-core",
+]
+
 [[package]]
 name = "tokenfold-output"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "clap",
  "ctrlc",
@@ -1190,16 +1398,17 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "pyo3",
  "serde_json",
+ "tokenfold-adapters",
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.5.4"
+version = "0.6.0"
 dependencies = [
  "tokenfold-core",
 ]
@@ -1275,6 +1484,12 @@ version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"

{tokenfold-0.5.4 → tokenfold-0.6.0}/Cargo.toml

@@ -1,9 +1,9 @@
 [workspace]
 resolver = "2"
-members = ["crates/tokenfold-core", "crates/tokenfold-py"]
+members = ["crates/tokenfold-core", "crates/tokenfold-py", "crates/tokenfold-adapters"]
 
 [workspace.package]
-version = "0.5.4"
+version = "0.6.0"
 edition = "2024"
 rust-version = "1.98"
 license = "Apache-2.0"
@@ -11,7 +11,7 @@ repository = "https://github.com/snchimata/tokenfold"
 
 [workspace.dependencies]
 serde = { version = "1", features = ["derive"] }
-serde_json = { version = "1", features = ["preserve_order"] }
+serde_json = { version = "1", features = ["preserve_order", "float_roundtrip"] }
 thiserror = "1"
 regex = "1"
 clap = { version = "4", features = ["derive"] }

tokenfold-0.6.0/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.4
+Name: tokenfold
+Version: 0.6.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+Requires-Dist: tokenfold-select-runtime==0.6.0 ; extra == 'select'
+Provides-Extra: select
+License-File: LICENSE
+Summary: Token-compression toolkit for LLM payloads (Python binding over the tokenfold-core Rust crate)
+Keywords: llm,tokens,compression,tokenizer,openai,json,prompt-engineering,cli,token-optimization
+License-Expression: Apache-2.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+# Tokenfold Python SDK
+
+Offline native request fitting. Python 3.12 or newer is required by the replacement
+SDK. Experimental Select uses a separately approved, explicitly configured runtime;
+base import and Core fitting never load models or download assets.
+
+```python
+from tokenfold import fit
+
+result = fit("Keep this text unchanged.", target_tokens=32)
+text = result.require_met()
+```
+
+Advanced compression, decoding, retrieval and historical compression report readers
+are available under `tokenfold.lowlevel`. Fit receipts are a separate schema.
+
+`fit` never sends a request. Text is opaque and unchanged; `Data(value)` renders
+compact JSON. `Data(value, representation="folded")` explicitly opts into the
+reversible Tokenfold codec: the consumer must understand that syntax (or decode
+with `tokenfold.lowlevel.decode`). Typed JSON normalizes safe integral floats and
+signed zero in Rust; default JSON preserves insertion order, while explicit folded
+JSON may normalize per-object key order. Opaque text is byte-preserved.
+
+Provider `Request.openai(...)` / `Request.anthropic(...)` payloads are full request
+objects, not bare message lists. The default encoding counter cannot establish
+provider framing and returns unavailable. Supply a complete-request counter, or
+explicitly choose the historical `ProviderCounter.openai(model="gpt-4o-2024-08-06")`
+local **estimate**. Estimates never become `met`: strict `require_met()` rejects
+them; per-fit `budget_mode="best_effort"` permits `require_fit()` only when the
+finite estimate fits. This offline recipe is not proof of billing or model availability.
+
+A client owns independent fitting sessions and supports `with`, `async with` and
+`await client.afit(...)`. Custom counters are synchronous and must support concurrent,
+deterministic calls; async fitting runs them off-loop. Blocking application callbacks
+cannot be forcibly preempted, so no bounded whole-call/close deadline is promised
+across them. Cancellation waits for native inference cleanup before raising.
+
+Selection is experimental and unqualified. It requires **both** an approved runtime
+(`SelectRuntime(config_path, experimental=True)`) configured on `Tokenfold` and
+`allow_selection=True` on that individual fit. Group annotations alone are not
+consent. Required groups and the query are retained; callers must bundle semantic
+dependencies themselves. `warmup()` only loads that configured runtime; import and
+already-fitting Core requests never start it. Provisioning the optional runtime and
+acquiring its pinned local model are separate explicit setup actions.
+
+Install with `pip install tokenfold==0.6.0`. The exact
+`tokenfold[select]` extra adds only `tokenfold-select-runtime==0.6.0`, not ML packages.
+Prebuilt targets: Linux GNU x64/arm64, macOS x64/arm64, Windows x64, gated by installed
+artifact CI before release. See the [migration guide](https://github.com/snchimata/tokenfold/blob/main/docs/sdk-migration.md)
+for provisioning, grouping recipes and queue/cold-start limits. The pinned tokenizer
+may refuse long valid inputs with AccountingError (and its own panic diagnostic);
+there is no heuristic/zero fallback.
+

tokenfold-0.6.0/README.md

@@ -0,0 +1,52 @@
+# Tokenfold Python SDK
+
+Offline native request fitting. Python 3.12 or newer is required by the replacement
+SDK. Experimental Select uses a separately approved, explicitly configured runtime;
+base import and Core fitting never load models or download assets.
+
+```python
+from tokenfold import fit
+
+result = fit("Keep this text unchanged.", target_tokens=32)
+text = result.require_met()
+```
+
+Advanced compression, decoding, retrieval and historical compression report readers
+are available under `tokenfold.lowlevel`. Fit receipts are a separate schema.
+
+`fit` never sends a request. Text is opaque and unchanged; `Data(value)` renders
+compact JSON. `Data(value, representation="folded")` explicitly opts into the
+reversible Tokenfold codec: the consumer must understand that syntax (or decode
+with `tokenfold.lowlevel.decode`). Typed JSON normalizes safe integral floats and
+signed zero in Rust; default JSON preserves insertion order, while explicit folded
+JSON may normalize per-object key order. Opaque text is byte-preserved.
+
+Provider `Request.openai(...)` / `Request.anthropic(...)` payloads are full request
+objects, not bare message lists. The default encoding counter cannot establish
+provider framing and returns unavailable. Supply a complete-request counter, or
+explicitly choose the historical `ProviderCounter.openai(model="gpt-4o-2024-08-06")`
+local **estimate**. Estimates never become `met`: strict `require_met()` rejects
+them; per-fit `budget_mode="best_effort"` permits `require_fit()` only when the
+finite estimate fits. This offline recipe is not proof of billing or model availability.
+
+A client owns independent fitting sessions and supports `with`, `async with` and
+`await client.afit(...)`. Custom counters are synchronous and must support concurrent,
+deterministic calls; async fitting runs them off-loop. Blocking application callbacks
+cannot be forcibly preempted, so no bounded whole-call/close deadline is promised
+across them. Cancellation waits for native inference cleanup before raising.
+
+Selection is experimental and unqualified. It requires **both** an approved runtime
+(`SelectRuntime(config_path, experimental=True)`) configured on `Tokenfold` and
+`allow_selection=True` on that individual fit. Group annotations alone are not
+consent. Required groups and the query are retained; callers must bundle semantic
+dependencies themselves. `warmup()` only loads that configured runtime; import and
+already-fitting Core requests never start it. Provisioning the optional runtime and
+acquiring its pinned local model are separate explicit setup actions.
+
+Install with `pip install tokenfold==0.6.0`. The exact
+`tokenfold[select]` extra adds only `tokenfold-select-runtime==0.6.0`, not ML packages.
+Prebuilt targets: Linux GNU x64/arm64, macOS x64/arm64, Windows x64, gated by installed
+artifact CI before release. See the [migration guide](https://github.com/snchimata/tokenfold/blob/main/docs/sdk-migration.md)
+for provisioning, grouping recipes and queue/cold-start limits. The pinned tokenizer
+may refuse long valid inputs with AccountingError (and its own panic diagnostic);
+there is no heuristic/zero fallback.

tokenfold-0.6.0/crates/tokenfold-adapters/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "tokenfold-adapters"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Explicit, model-free provider observation adapters for Tokenfold."
+readme = "README.md"
+
+[dependencies]
+tokenfold-core = { version = "0.6.0", path = "../tokenfold-core" }
+serde = { version = "1", features = ["derive"] }
+serde_json = { version = "1", features = ["preserve_order"] }

tokenfold-0.6.0/crates/tokenfold-adapters/README.md

@@ -0,0 +1,37 @@
+# tokenfold-adapters
+
+Explicit provider-wire adapters for Tokenfold. No provider SDK or model runtime is required.
+
+- `compress_for` and `verify_shape_parity`: OpenAI/Anthropic wire-shape checks.
+- `observation`: opt-in OpenAI tool-result JSON compression with envelope validation,
+  protected content, thresholds and no-growth fallback.
+- `session`: in-process commitments or opt-in locked, bounded fingerprint snapshots
+  for restart-safe observation idempotence; not durable history compaction.
+- `dedup`: conservative duplicate encoding with public-decoder byte round-trip checks.
+- `manifest::build_task_manifest`: caller-declared task/revision and required state;
+  missing provenance, conflicting facts or exceeded limits reject the manifest.
+- `tools::select_tools_for_query`: deterministic lexical ranking, preserved JSON values
+  and key order, forced/companion tools and a caller-owned discovery tool.
+- `select`: experimental text/ID mapping, approved direct-child scorer invocation with
+  deadline/cancellation, strict score validation, whole-group selection and prompt recount.
+  No model runtime is bundled and auxiliary model files remain caller-owned.
+- `semantic`: experimental claim-backed literal extraction, **not** a verified generative
+  summarizer. Its synchronous strategy interface cannot enforce a hard inference deadline.
+
+Policies are disabled by default. Callers own task association, authorized catalogs,
+discovery execution, prompt insertion and recovery. No host hooks are installed.
+Framework names denote compatible wire shapes, not independently qualified SDK integrations.
+JSON value preservation does not promise preservation of original JSON whitespace.
+
+```rust
+use tokenfold_adapters::{compress_for, AdapterFormat};
+use tokenfold_core::CompressionPolicy;
+
+let policy = CompressionPolicy::builder().build()?;
+let result = compress_for(AdapterFormat::OpenAiChat,
+    br#"{"messages":[{"role":"user","content":"Hello"}]}"#, &policy)?;
+# Ok::<(), tokenfold_core::TokenFoldError>(())
+```
+
+Experimental helpers have not established live task-quality or cache non-inferiority.
+Do not silently substitute them for default presets.