datoon 1.9.1__tar.gz → 3.0.0__tar.gz

{datoon-1.9.1 → datoon-3.0.0}/.github/workflows/package-installer-smoke.yml

@@ -15,10 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
 
       - name: Setup Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.12"
 

{datoon-1.9.1 → datoon-3.0.0}/.github/workflows/pages.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
 
       - name: Setup Pages
         id: setup_pages

{datoon-1.9.1 → datoon-3.0.0}/.github/workflows/pre-commit.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
 
       - name: Setup uv
         uses: astral-sh/setup-uv@v7

{datoon-1.9.1 → datoon-3.0.0}/.github/workflows/publish-mcp.yml

@@ -26,7 +26,7 @@ jobs:
       || github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           ref: main
           fetch-depth: 0

{datoon-1.9.1 → datoon-3.0.0}/.github/workflows/publish.yml

@@ -24,7 +24,7 @@ jobs:
     outputs:
       version: ${{ steps.ver.outputs.version }}
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           ref: main
           fetch-depth: 0

{datoon-1.9.1 → datoon-3.0.0}/.github/workflows/release.yml

@@ -15,17 +15,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           fetch-depth: 0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           node-version: "22"
 
       - name: Setup Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.12"
 

datoon-3.0.0/.github/workflows/tests.yml

@@ -0,0 +1,24 @@
+name: Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7.0.1
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Run tests
+        run: uv run --with ".[dev]" pytest

{datoon-1.9.1 → datoon-3.0.0}/.github/workflows/validate-artifacts.yml

@@ -18,10 +18,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
 
       - name: Setup Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.12"
 

{datoon-1.9.1 → datoon-3.0.0}/.github/workflows/workflow-lint.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
         with:
           persist-credentials: false
 
@@ -35,7 +35,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
         with:
           persist-credentials: false
 

{datoon-1.9.1 → datoon-3.0.0}/.gitignore

@@ -40,6 +40,7 @@ benchmarks/results/**/*.json
 # Local Codex/Claude artifacts
 .agents/skills/
 .claude/
+.omc/
 skills-lock.json
 
 # Generated by setuptools-scm during local builds/tests

{datoon-1.9.1 → datoon-3.0.0}/CHANGELOG.md

@@ -1,3 +1,42 @@
+# [3.0.0](https://github.com/andrii-su/datoon/compare/v2.0.0...v3.0.0) (2026-11-11)
+
+
+* feat!: selectable token counters (auto, chars, tiktoken, claude, hf, callable) ([cab2bfb](https://github.com/andrii-su/datoon/commit/cab2bfb9f1106f2a4cf40122588249acb0371f94))
+
+
+### Bug Fixes
+
+* **mcp:** accept only local token counters as a tool argument ([1f2e54d](https://github.com/andrii-su/datoon/commit/1f2e54d4f83ea266590933f0762019d2d37f7270))
+
+
+### BREAKING CHANGES
+
+* `ConversionConfig.token_encoding` is replaced by
+`tokenizer`; the CLI flag `--encoding` and the MCP `encoding` argument are
+replaced by `--tokenizer` / `tokenizer` (default from `DATOON_TOKENIZER`,
+else `auto`). `ConversionReport` gains a `tokenizer` field. Every counter
+returns at least 1, including for empty text.
+
+# [2.0.0](https://github.com/andrii-su/datoon/compare/v1.9.1...v2.0.0) (2026-11-11)
+
+
+* feat!: encode TOON in pure Python, drop the Node.js CLI ([1bffef7](https://github.com/andrii-su/datoon/commit/1bffef7c068cd3f2a73df80a9348f0af28dd3040))
+* feat!: encode TOON in pure Python, drop the Node.js CLI ([#69](https://github.com/andrii-su/datoon/issues/69)) ([7e28513](https://github.com/andrii-su/datoon/commit/7e28513aa1fb0cf0be315d8e09586eedb0ce6800))
+
+
+### BREAKING CHANGES
+
+* `toon-format` is a required dependency; the
+`ConversionConfig.toon_cli_timeout` field, the `--timeout` CLI flag and
+the `timeout` argument of the `convert_json` / `convert_text` MCP tools
+are removed. The skip reason for a failed conversion is now
+"TOON encoding failed; falling back to JSON."
+* `toon-format` is a required dependency; the
+`ConversionConfig.toon_cli_timeout` field, the `--timeout` CLI flag and
+the `timeout` argument of the `convert_json` / `convert_text` MCP tools
+are removed. The skip reason for a failed conversion is now
+"TOON encoding failed; falling back to JSON."
+
 ## [1.9.1](https://github.com/andrii-su/datoon/compare/v1.9.0...v1.9.1) (2026-07-07)
 
 

{datoon-1.9.1 → datoon-3.0.0}/CLAUDE.md

@@ -30,8 +30,9 @@ datoon/
 │
 ├── src/datoon/                       # Python package source
 │   ├── analyzer.py                   # Payload structure suitability checks
-│   ├── converter.py                  # JSON normalization, TOON CLI call, policy gating
+│   ├── converter.py                  # JSON normalization, TOON encoder call, policy gating
 │   ├── models.py                     # Config/report/outcome dataclasses
+│   ├── tokenizers.py                 # Token counter specs and estimates
 │   ├── cli.py                        # CLI entrypoint
 │   └── mcp_server.py                 # MCP server tools
 │
@@ -68,6 +69,7 @@ Edit these files directly:
 | Conversion policy | `src/datoon/converter.py` |
 | Payload analysis | `src/datoon/analyzer.py` |
 | Public data contracts | `src/datoon/models.py` |
+| Token counters | `src/datoon/tokenizers.py` |
 | CLI behavior | `src/datoon/cli.py` |
 | MCP behavior | `src/datoon/mcp_server.py` |
 | Skill behavior | `skills/datoon/SKILL.md` |
@@ -105,11 +107,12 @@ It runs on pull requests and pushes to `main`. The old auto-sync-on-main flow wa
 
 - Never change semantic values or key names during conversion.
 - Invalid JSON must fail loudly.
-- Missing `npx` must not break auto mode; fallback to JSON unless `force=True`.
+- An encoder failure must not break auto mode; fallback to JSON unless `force=True`.
 - `--force` is for controlled experiments and should bypass gating, not hide conversion errors.
 - Auto mode skips before conversion when analysis rejects the payload, and skips after conversion when estimated savings are below `min_savings_ratio`.
 - Keep skip reasons user-facing, specific, and stable enough for CLI/MCP troubleshooting.
-- Token counts are estimates. Be precise about whether a number is payload-token estimate, visible output estimate, or real model usage.
+- Token counts are estimates, whichever tokenizer produced them (`claude:` counts included). Be precise about whether a number is payload-token estimate, visible output estimate, or real model usage, and name the tokenizer it came from.
+- Only the `auto` tokenizer falls back silently; an explicit spec with a missing package or credentials must raise `DatoonError`, in auto and force mode alike.
 - Benchmarks in README must come from committed scripts/artifacts.
 - Keep generated caches out of commits: `.venv/`, `.pytest_cache/`, `.ruff_cache/`, `__pycache__/`, local benchmark result JSON under `benchmarks/results/`.
 
@@ -122,7 +125,7 @@ python scripts/validate_plugin_metadata.py
 uvx pre-commit run --all-files
 ```
 
-Integration tests require Node.js and `npx`.
+Conversion is pure Python (`toon-format`); no Node.js is needed for tests.
 
 ## Benchmark Commands
 
@@ -168,7 +171,7 @@ Config writes must:
 - remove only `datoon`-owned entries on uninstall;
 - keep dry-run side-effect free.
 
-Security and privacy docs should state that the installer does not read or upload user payloads, but selected targets may call external tooling: Claude Code through `claude plugin ...`, Python package resolution through `uvx`, and TOON conversion through `npx --yes @toon-format/cli@2` when conversion actually runs.
+Security and privacy docs should state that the installer does not read or upload user payloads, but selected targets may call external tooling: Claude Code through `claude plugin ...`, Python package resolution through `uvx`; TOON conversion runs in-process through the `toon-format` package and calls no external tooling.
 
 ## Known Gaps
 

{datoon-1.9.1 → datoon-3.0.0}/CONTRIBUTING.md

@@ -15,7 +15,7 @@ python scripts/validate_skill_sync.py
 python scripts/validate_plugin_metadata.py
 ```
 
-Integration tests need Node.js and `npx`, because conversion uses the official TOON CLI.
+Conversion is pure Python (the `toon-format` package); no Node.js is required for tests.
 
 ## What To Edit
 
@@ -67,12 +67,6 @@ Run the full suite:
 uv run pytest
 ```
 
-Run unit tests only:
-
-```bash
-uv run pytest -m "not integration"
-```
-
 Run pre-commit locally:
 
 ```bash
@@ -112,7 +106,7 @@ Use Conventional Commit subjects:
 
 ```text
 feat: add MCP analyze_json tool
-fix: preserve JSON fallback on missing npx
+fix: preserve JSON fallback on encoder failure
 docs: clarify Claude Code plugin install
 test: cover non-uniform arrays
 ```

{datoon-1.9.1 → datoon-3.0.0}/INSTALL.md

@@ -16,7 +16,6 @@ The setup script:
 
 - checks for Python 3.12+;
 - installs `uv` if it is missing;
-- warns if Node.js is missing;
 - runs `uv sync --extra dev`;
 - installs the editable `datoon` CLI globally through `uv tool install`.
 
@@ -43,9 +42,29 @@ Optional extras:
 
 ```bash
 pip install "datoon[tokens]"  # tiktoken token estimates
+pip install "datoon[claude]"  # Claude token counts via the Anthropic API
+pip install "datoon[hf]"      # Hugging Face tokenizers
 pip install "datoon[mcp]"     # MCP server dependencies
 ```
 
+### Choosing a tokenizer
+
+Select the token counter with `--tokenizer <spec>` (CLI), the `tokenizer` argument (MCP tools), or `DATOON_TOKENIZER`; the default is `auto`. The MCP tools accept only local counters (`auto`, `chars`, `tiktoken`) per call; `claude:` and `hf:` reach the network, so only the server operator can enable them through `DATOON_TOKENIZER`.
+
+| Spec | Counts with | Needs |
+|---|---|---|
+| `auto` | `tiktoken:o200k_base` when tiktoken is installed, else `chars:3` | — |
+| `chars` / `chars:<N>` | `ceil(len(text) / N)`, default `N = 3` | — |
+| `tiktoken` / `tiktoken:<encoding>` | tiktoken, default `o200k_base` | `datoon[tokens]` |
+| `claude:<model-id>` | Anthropic token counting API | `datoon[claude]`, Anthropic credentials (e.g. `ANTHROPIC_API_KEY`) |
+| `hf:<repo-id>` | Hugging Face `tokenizers` tokenizer | `datoon[hf]` |
+
+Local counters suit runtime gating; `claude:` and `hf:` suit benchmarks and evals where counts for the target model matter. `claude:` makes network calls, and Anthropic documents its counts as estimates. `hf:` downloads the tokenizer from the Hugging Face Hub on first use.
+
+```bash
+datoon input.json --tokenizer claude:claude-opus-5-5 --report-stdout
+```
+
 ## CLI
 
 Read from stdin:
@@ -69,21 +88,16 @@ datoon input.json --force --report-stdout
 
 ## TOON Dependency
 
-`datoon` calls the official TOON CLI through `npx`.
-
-Required for conversion:
+`datoon` converts to TOON in pure Python using the official [`toon-format`](https://pypi.org/project/toon-format/) package, which is installed automatically as a dependency. No Node.js is needed.
 
-- Node.js
-- `npx` in `PATH`
-
-Without Node.js, structure analysis still works, but conversion falls back to JSON in auto mode. Forced conversion fails loudly.
+If the encoder fails, auto mode falls back to JSON. Forced conversion fails loudly.
 
 In auto mode, `datoon` skips conversion and returns compact JSON when:
 
 - the payload has no uniform object array with at least the configured row count;
 - the payload exceeds the configured maximum nesting depth;
 - estimated TOON savings are below `--min-savings`;
-- Node.js or `npx` is unavailable, or the TOON CLI fails.
+- the TOON encoder fails.
 
 Invalid JSON never falls back silently. It raises an error.
 
@@ -232,12 +246,7 @@ echo '{"users":[{"id":1,"name":"Ada"},{"id":2,"name":"Lin"},{"id":3,"name":"Grac
   | datoon --report-stdout
 ```
 
-If the reason mentions the TOON dependency, install Node.js and confirm `npx` is available:
-
-```bash
-node --version
-npx --version
-```
+If the reason says TOON encoding failed, rerun with `--force` to see the underlying encoder error.
 
 If the reason mentions structure or savings, the skip is expected auto-mode behavior. Use `--force` only for controlled experiments where a failed TOON conversion should fail the command.
 
@@ -264,8 +273,6 @@ python scripts/validate_skill_sync.py
 PYTHONPATH=src python benchmarks/run.py --dry-run
 ```
 
-Full integration tests require Node.js and `npx`.
-
 ## Uninstall
 
 For the editable CLI installed by `setup.sh`:

{datoon-1.9.1 → datoon-3.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datoon
-Version: 1.9.1
+Version: 3.0.0
 Summary: Smart structured-data-to-TOON gateway with pragmatic auto-gating for LLM prompts.
 Author-email: Andrii Suruhov <andrii.suruhov@gmail.com>
 License: MIT License
@@ -37,12 +37,17 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: toon-format>=1.2.0
 Provides-Extra: dev
 Requires-Dist: pytest>=9.0.3; extra == "dev"
 Requires-Dist: pre-commit>=4.6.0; extra == "dev"
 Requires-Dist: pyyaml>=6.0; extra == "dev"
 Provides-Extra: tokens
 Requires-Dist: tiktoken>=0.13.0; extra == "tokens"
+Provides-Extra: claude
+Requires-Dist: anthropic>=2.0.0; extra == "claude"
+Provides-Extra: hf
+Requires-Dist: tokenizers>=0.23.0; extra == "hf"
 Provides-Extra: mcp
 Requires-Dist: mcp>=1.28.0; extra == "mcp"
 Provides-Extra: yaml
@@ -55,7 +60,7 @@ Requires-Dist: fastavro>=1.9; extra == "columnar"
 Provides-Extra: numbers
 Requires-Dist: numbers-parser>=4.4; extra == "numbers"
 Provides-Extra: all
-Requires-Dist: datoon[columnar,excel,mcp,numbers,tokens,yaml]; extra == "all"
+Requires-Dist: datoon[claude,columnar,excel,hf,mcp,numbers,tokens,yaml]; extra == "all"
 Dynamic: license-file
 
 <p align="center">
@@ -169,7 +174,7 @@ Same result. Zero JSON serialization in your code.
 {"decision":"skip","reason":"No uniform object arrays found with at least 3 rows."}
 ```
 
-No Node.js call. No silent corruption.
+No silent corruption.
 
 </td>
 </tr>
@@ -211,6 +216,10 @@ pip install "datoon[numbers]"
 # with tiktoken-based token counting
 pip install "datoon[tokens]"
 
+# with Claude token counting (Anthropic API) / Hugging Face tokenizers
+pip install "datoon[claude]"
+pip install "datoon[hf]"
+
 # with MCP server
 pip install "datoon[mcp]"
 
@@ -218,7 +227,7 @@ pip install "datoon[mcp]"
 pip install "datoon[all]"
 ```
 
-Requires Python 3.12+. TOON conversion requires Node.js with `npx` in PATH — analysis and format reading work without it.
+Requires Python 3.12+. TOON conversion is pure Python (official [`toon-format`](https://pypi.org/project/toon-format/) encoder, installed automatically) — no Node.js needed.
 
 For Claude Code plugin, Codex, and MCP config → [**INSTALL.md**](./INSTALL.md).
 
@@ -252,11 +261,26 @@ For Claude Code plugin, Codex, and MCP config → [**INSTALL.md**](./INSTALL.md)
 1. **Detect format** — from `--format` flag, file extension, or default to JSON for stdin
 1. **Read + normalize** — parse source into list of row dicts; serialize to compact JSON
 1. **Analyze structure** — uniform object arrays? acceptable depth? minimum rows?
-1. **Gate early** — non-candidates skip before any CLI call; no Node.js overhead
-1. **Convert + estimate** — TOON CLI runs, token savings calculated
+1. **Gate early** — non-candidates skip before any TOON encoding
+1. **Convert + estimate** — TOON encoder runs in-process, token savings calculated
 1. **Gate savings** — below threshold → return JSON; above → return TOON with report
 
-Every path returns a `ConversionReport` with `decision`, `reason`, and token estimates. Pipelines never get silent surprises.
+Every path returns a `ConversionReport` with `decision`, `reason`, token estimates, and the `tokenizer` that produced them. Pipelines never get silent surprises.
+
+## Choosing a tokenizer
+
+Token estimates drive the savings gate, so pick the counter that fits the job. Pass one spec with `--tokenizer`, the `tokenizer` MCP argument, `ConversionConfig(tokenizer=...)`, or the `DATOON_TOKENIZER` environment variable; in Python any callable `str -> int` works too. The MCP tools accept only local counters (`auto`, `chars`, `tiktoken`) per call; `claude:` and `hf:` reach the network, so only the server operator can enable them through `DATOON_TOKENIZER`.
+
+| Spec | Counts with | Needs |
+|---|---|---|
+| `auto` (default) | `tiktoken:o200k_base` when tiktoken is installed, else `chars:3` | — |
+| `chars` / `chars:<N>` | `ceil(len(text) / N)`, default `N = 3` | — |
+| `tiktoken` / `tiktoken:<encoding>` | tiktoken, default `o200k_base` | `datoon[tokens]` |
+| `claude:<model-id>` | Anthropic token counting API for that model | `datoon[claude]`, Anthropic credentials (e.g. `ANTHROPIC_API_KEY`) |
+| `hf:<repo-id>` | Hugging Face `tokenizers` tokenizer from the Hub | `datoon[hf]` |
+| a callable (Python) | your own function, reported as `custom` | — |
+
+Use the local counters (`auto`, `chars`, `tiktoken`) to gate payloads at runtime: they are fast and run locally (tiktoken fetches its encoding file once on first use). Use `claude:` or `hf:` in benchmarks and evals where counts for the target model matter. `claude:` makes network calls, and Anthropic documents its counts as estimates. Only `auto` falls back silently: an explicit spec with a missing package fails with an error that names the extra to install, and `claude:` without Anthropic credentials fails too. Every report records the resolved name in `tokenizer`.
 
 ______________________________________________________________________
 
@@ -344,7 +368,7 @@ outcome = convert_json_for_llm(json_text, ConversionConfig())
 send_to_model(outcome.payload_text)
 ```
 
-**Structure-only analysis (no Node.js required):**
+**Structure-only analysis:**
 
 ```python
 from datoon.analyzer import analyze_payload
@@ -366,7 +390,7 @@ ______________________________________________________________________
 |---|---|
 | `convert_json` | Full JSON conversion with policy gating |
 | `convert_text` | Converts CSV, YAML, XML, or JSONL text with policy gating |
-| `analyze_json` | Structure analysis only — no Node.js needed |
+| `analyze_json` | Structure analysis only — no conversion |
 
 **Claude Desktop / Cursor / Windsurf config:**
 
@@ -419,7 +443,7 @@ ______________________________________________________________________
 | `--min-savings` | `0.15` | Minimum relative token savings required |
 | `--max-depth` | `6` | Maximum nesting depth for auto-conversion |
 | `--min-uniform-rows` | `3` | Minimum rows in uniform object arrays |
-| `--timeout` | `30` | Seconds before TOON CLI call is aborted |
+| `--tokenizer <spec>` | `$DATOON_TOKENIZER`, else `auto` | Token counter for estimates — see [Choosing a tokenizer](#choosing-a-tokenizer) |
 | `--report <path>` | — | Write JSON conversion report to file |
 | `--report-stdout` | — | Print JSON conversion report to stderr |
 | `-o <path>` | stdout | Output file path |
@@ -514,8 +538,7 @@ uvx pre-commit install
 **Tests:**
 
 ```bash
-pytest -m "not integration"   # unit only (102 tests)
-pytest                        # with integration (requires Node.js + npx)
+pytest
 ```
 
 **Skill sync + plugin metadata:**

{datoon-1.9.1 → datoon-3.0.0}/README.md

@@ -109,7 +109,7 @@ Same result. Zero JSON serialization in your code.
 {"decision":"skip","reason":"No uniform object arrays found with at least 3 rows."}
 ```
 
-No Node.js call. No silent corruption.
+No silent corruption.
 
 </td>
 </tr>
@@ -151,6 +151,10 @@ pip install "datoon[numbers]"
 # with tiktoken-based token counting
 pip install "datoon[tokens]"
 
+# with Claude token counting (Anthropic API) / Hugging Face tokenizers
+pip install "datoon[claude]"
+pip install "datoon[hf]"
+
 # with MCP server
 pip install "datoon[mcp]"
 
@@ -158,7 +162,7 @@ pip install "datoon[mcp]"
 pip install "datoon[all]"
 ```
 
-Requires Python 3.12+. TOON conversion requires Node.js with `npx` in PATH — analysis and format reading work without it.
+Requires Python 3.12+. TOON conversion is pure Python (official [`toon-format`](https://pypi.org/project/toon-format/) encoder, installed automatically) — no Node.js needed.
 
 For Claude Code plugin, Codex, and MCP config → [**INSTALL.md**](./INSTALL.md).
 
@@ -192,11 +196,26 @@ For Claude Code plugin, Codex, and MCP config → [**INSTALL.md**](./INSTALL.md)
 1. **Detect format** — from `--format` flag, file extension, or default to JSON for stdin
 1. **Read + normalize** — parse source into list of row dicts; serialize to compact JSON
 1. **Analyze structure** — uniform object arrays? acceptable depth? minimum rows?
-1. **Gate early** — non-candidates skip before any CLI call; no Node.js overhead
-1. **Convert + estimate** — TOON CLI runs, token savings calculated
+1. **Gate early** — non-candidates skip before any TOON encoding
+1. **Convert + estimate** — TOON encoder runs in-process, token savings calculated
 1. **Gate savings** — below threshold → return JSON; above → return TOON with report
 
-Every path returns a `ConversionReport` with `decision`, `reason`, and token estimates. Pipelines never get silent surprises.
+Every path returns a `ConversionReport` with `decision`, `reason`, token estimates, and the `tokenizer` that produced them. Pipelines never get silent surprises.
+
+## Choosing a tokenizer
+
+Token estimates drive the savings gate, so pick the counter that fits the job. Pass one spec with `--tokenizer`, the `tokenizer` MCP argument, `ConversionConfig(tokenizer=...)`, or the `DATOON_TOKENIZER` environment variable; in Python any callable `str -> int` works too. The MCP tools accept only local counters (`auto`, `chars`, `tiktoken`) per call; `claude:` and `hf:` reach the network, so only the server operator can enable them through `DATOON_TOKENIZER`.
+
+| Spec | Counts with | Needs |
+|---|---|---|
+| `auto` (default) | `tiktoken:o200k_base` when tiktoken is installed, else `chars:3` | — |
+| `chars` / `chars:<N>` | `ceil(len(text) / N)`, default `N = 3` | — |
+| `tiktoken` / `tiktoken:<encoding>` | tiktoken, default `o200k_base` | `datoon[tokens]` |
+| `claude:<model-id>` | Anthropic token counting API for that model | `datoon[claude]`, Anthropic credentials (e.g. `ANTHROPIC_API_KEY`) |
+| `hf:<repo-id>` | Hugging Face `tokenizers` tokenizer from the Hub | `datoon[hf]` |
+| a callable (Python) | your own function, reported as `custom` | — |
+
+Use the local counters (`auto`, `chars`, `tiktoken`) to gate payloads at runtime: they are fast and run locally (tiktoken fetches its encoding file once on first use). Use `claude:` or `hf:` in benchmarks and evals where counts for the target model matter. `claude:` makes network calls, and Anthropic documents its counts as estimates. Only `auto` falls back silently: an explicit spec with a missing package fails with an error that names the extra to install, and `claude:` without Anthropic credentials fails too. Every report records the resolved name in `tokenizer`.
 
 ______________________________________________________________________
 
@@ -284,7 +303,7 @@ outcome = convert_json_for_llm(json_text, ConversionConfig())
 send_to_model(outcome.payload_text)
 ```
 
-**Structure-only analysis (no Node.js required):**
+**Structure-only analysis:**
 
 ```python
 from datoon.analyzer import analyze_payload
@@ -306,7 +325,7 @@ ______________________________________________________________________
 |---|---|
 | `convert_json` | Full JSON conversion with policy gating |
 | `convert_text` | Converts CSV, YAML, XML, or JSONL text with policy gating |
-| `analyze_json` | Structure analysis only — no Node.js needed |
+| `analyze_json` | Structure analysis only — no conversion |
 
 **Claude Desktop / Cursor / Windsurf config:**
 
@@ -359,7 +378,7 @@ ______________________________________________________________________
 | `--min-savings` | `0.15` | Minimum relative token savings required |
 | `--max-depth` | `6` | Maximum nesting depth for auto-conversion |
 | `--min-uniform-rows` | `3` | Minimum rows in uniform object arrays |
-| `--timeout` | `30` | Seconds before TOON CLI call is aborted |
+| `--tokenizer <spec>` | `$DATOON_TOKENIZER`, else `auto` | Token counter for estimates — see [Choosing a tokenizer](#choosing-a-tokenizer) |
 | `--report <path>` | — | Write JSON conversion report to file |
 | `--report-stdout` | — | Print JSON conversion report to stderr |
 | `-o <path>` | stdout | Output file path |
@@ -454,8 +473,7 @@ uvx pre-commit install
 **Tests:**
 
 ```bash
-pytest -m "not integration"   # unit only (102 tests)
-pytest                        # with integration (requires Node.js + npx)
+pytest
 ```
 
 **Skill sync + plugin metadata:**

{datoon-1.9.1/plugins/datoon/skills/datoon → datoon-3.0.0}/SKILL.md

@@ -50,7 +50,7 @@ Skip conversion when any condition fails:
 
 - Never change semantic values or key names.
 - Fail loudly for invalid JSON.
-- If TOON CLI dependency is unavailable, keep JSON and report fallback reason.
+- If TOON encoding fails, keep JSON and report fallback reason.
 - Use `--force` only for controlled experiments.
 
 ## Boundaries

{datoon-1.9.1 → datoon-3.0.0}/benchmarks/agent_skill_eval/generate_payloads.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """Generate deterministic JSON payloads for agent skill evaluation."""
 
 from __future__ import annotations
@@ -67,12 +66,10 @@ def expected_for(payload: dict[str, object]) -> dict[str, object]:
         region_totals[region] = region_totals.get(region, 0) + revenue
         category_totals[category] = category_totals.get(category, 0) + revenue
 
-    top_region = sorted(region_totals.items(), key=lambda item: (-item[1], item[0]))[0][
+    top_region = min(region_totals.items(), key=lambda item: (-item[1], item[0]))[0][0]
+    top_category = min(category_totals.items(), key=lambda item: (-item[1], item[0]))[
         0
-    ]
-    top_category = sorted(
-        category_totals.items(), key=lambda item: (-item[1], item[0])
-    )[0][0]
+    ][0]
 
     return {
         "scenario": payload["scenario"],