datoon 1.8.0__tar.gz → 1.9.0__tar.gz

{datoon-1.8.0 → datoon-1.9.0}/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [1.9.0](https://github.com/andrii-su/datoon/compare/v1.8.0...v1.9.0) (2026-07-07)
+
+
+### Features
+
+* **tokens:** make token encoding configurable, default to o200k_base ([#46](https://github.com/andrii-su/datoon/issues/46)) ([8483eb7](https://github.com/andrii-su/datoon/commit/8483eb7aca27c97df2827d552e0086cf9bfbe157))
+
 # [1.8.0](https://github.com/andrii-su/datoon/compare/v1.7.4...v1.8.0) (2026-07-07)
 
 

{datoon-1.8.0 → datoon-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datoon
-Version: 1.8.0
+Version: 1.9.0
 Summary: Smart structured-data-to-TOON gateway with pragmatic auto-gating for LLM prompts.
 Author-email: Andrii Suruhov <andrii.suruhov@gmail.com>
 License: MIT License
@@ -451,12 +451,12 @@ Auto mode avoids low-benefit and high-risk payloads (`orders-nested`, `mixed-non
 
 | Dataset | JSON | TOON (forced) | Raw Saved | Auto | Auto Tokens | Auto Saved |
 |---|---:|---:|---:|---|---:|---:|
-| users-small | 56 | 31 | 44.6% | convert | 31 | 44.6% |
-| events-medium | 198 | 111 | 43.9% | convert | 111 | 43.9% |
-| orders-nested | 93 | 91 | 2.2% | skip | 93 | 0.0% |
-| mixed-non-uniform | 35 | 37 | -5.7% | skip | 35 | 0.0% |
-| metrics-wide | 133 | 63 | 52.6% | convert | 63 | 52.6% |
-| **Average** | **103** | **67** | **27.5%** | **3/5 convert** | **67** | **28.2%** |
+| users-small | 54 | 40 | 25.9% | convert | 40 | 25.9% |
+| events-medium | 219 | 162 | 26.0% | convert | 162 | 26.0% |
+| orders-nested | 106 | 116 | -9.4% | skip | 106 | 0.0% |
+| mixed-non-uniform | 35 | 47 | -34.3% | skip | 35 | 0.0% |
+| metrics-wide | 142 | 103 | 27.5% | convert | 103 | 27.5% |
+| **Average** | **111** | **94** | **7.1%** | **3/5 convert** | **89** | **15.9%** |
 
 *Forced conversion succeeded for 5/5 payloads.*
 

{datoon-1.8.0 → datoon-1.9.0}/README.md

@@ -391,12 +391,12 @@ Auto mode avoids low-benefit and high-risk payloads (`orders-nested`, `mixed-non
 
 | Dataset | JSON | TOON (forced) | Raw Saved | Auto | Auto Tokens | Auto Saved |
 |---|---:|---:|---:|---|---:|---:|
-| users-small | 56 | 31 | 44.6% | convert | 31 | 44.6% |
-| events-medium | 198 | 111 | 43.9% | convert | 111 | 43.9% |
-| orders-nested | 93 | 91 | 2.2% | skip | 93 | 0.0% |
-| mixed-non-uniform | 35 | 37 | -5.7% | skip | 35 | 0.0% |
-| metrics-wide | 133 | 63 | 52.6% | convert | 63 | 52.6% |
-| **Average** | **103** | **67** | **27.5%** | **3/5 convert** | **67** | **28.2%** |
+| users-small | 54 | 40 | 25.9% | convert | 40 | 25.9% |
+| events-medium | 219 | 162 | 26.0% | convert | 162 | 26.0% |
+| orders-nested | 106 | 116 | -9.4% | skip | 106 | 0.0% |
+| mixed-non-uniform | 35 | 47 | -34.3% | skip | 35 | 0.0% |
+| metrics-wide | 142 | 103 | 27.5% | convert | 103 | 27.5% |
+| **Average** | **111** | **94** | **7.1%** | **3/5 convert** | **89** | **15.9%** |
 
 *Forced conversion succeeded for 5/5 payloads.*
 

{datoon-1.8.0 → datoon-1.9.0}/src/datoon/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.8.0'
-__version_tuple__ = version_tuple = (1, 8, 0)
+__version__ = version = '1.9.0'
+__version_tuple__ = version_tuple = (1, 9, 0)
 
-__commit_id__ = commit_id = 'gacb329fb8'
+__commit_id__ = commit_id = 'g412011b90'

{datoon-1.8.0 → datoon-1.9.0}/src/datoon/cli.py

@@ -5,12 +5,12 @@ from __future__ import annotations
 import argparse
 import json
 import sys
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Sequence
 
 from datoon import __version__
 from datoon.converter import DatoonError, convert_json_for_llm
-from datoon.models import ConversionConfig
+from datoon.models import DEFAULT_TOKEN_ENCODING, ConversionConfig
 from datoon.readers import ALL_FORMATS, BINARY_FORMATS, detect_format, read_tabular
 
 
@@ -87,6 +87,14 @@ def build_parser() -> argparse.ArgumentParser:
         default=0,
         help="Table index within a Numbers sheet (default: 0).",
     )
+    parser.add_argument(
+        "--encoding",
+        default=DEFAULT_TOKEN_ENCODING,
+        help=(
+            "tiktoken encoding for token estimates "
+            f"(default: {DEFAULT_TOKEN_ENCODING})."
+        ),
+    )
     parser.add_argument(
         "--report",
         help="Optional path to write conversion report as JSON.",
@@ -171,6 +179,7 @@ def main(argv: Sequence[str] | None = None) -> int:
             min_uniform_rows=args.min_uniform_rows,
             force=args.force,
             toon_cli_timeout=args.timeout,
+            token_encoding=args.encoding,
         )
 
         fmt = _resolve_format(args.input, args.format)

{datoon-1.8.0 → datoon-1.9.0}/src/datoon/converter.py

@@ -9,7 +9,9 @@ from functools import lru_cache
 from typing import Any
 
 from datoon.analyzer import analyze_payload
+from datoon.errors import DatoonError
 from datoon.models import (
+    DEFAULT_TOKEN_ENCODING,
     ConversionConfig,
     ConversionOutcome,
     ConversionReport,
@@ -18,9 +20,7 @@ from datoon.models import (
 
 TOON_CLI_PACKAGE = "@toon-format/cli@2"
 
-
-class DatoonError(RuntimeError):
-    """Raised when datoon cannot process the payload safely."""
+__all__ = ["DatoonError", "convert_json_for_llm", "estimate_tokens"]
 
 
 def _reject_non_finite(constant: str) -> Any:
@@ -43,20 +43,20 @@ def _normalize_json(raw_text: str) -> tuple[Any, str]:
     return parsed, normalized
 
 
-@lru_cache(maxsize=1)
-def _load_token_encoder() -> Any | None:
-    """Load a token encoder once, returning None when optional dependency is absent."""
+@lru_cache(maxsize=8)
+def _load_token_encoder(encoding: str) -> Any | None:
+    """Load a token encoder once per name, None when the optional dep is absent."""
     try:
         import tiktoken  # type: ignore
     except ImportError:
         return None
 
-    return tiktoken.get_encoding("cl100k_base")
+    return tiktoken.get_encoding(encoding)
 
 
-def estimate_tokens(text: str) -> int:
+def estimate_tokens(text: str, encoding: str = DEFAULT_TOKEN_ENCODING) -> int:
     """Estimate token count with tiktoken when installed, else use a char heuristic."""
-    encoder = _load_token_encoder()
+    encoder = _load_token_encoder(encoding)
     if encoder is not None:
         return len(encoder.encode(text))
 
@@ -119,7 +119,7 @@ def convert_json_for_llm(raw_text: str, config: ConversionConfig) -> ConversionO
     """Convert JSON to TOON when structure and savings meet configured policy."""
     parsed, normalized_json = _normalize_json(raw_text)
     analysis = analyze_payload(parsed, config)
-    input_tokens = estimate_tokens(normalized_json)
+    input_tokens = estimate_tokens(normalized_json, config.token_encoding)
 
     if not analysis.is_candidate and not config.force:
         return _build_skip_outcome(
@@ -144,7 +144,7 @@ def convert_json_for_llm(raw_text: str, config: ConversionConfig) -> ConversionO
             analysis=analysis,
         )
 
-    output_tokens = estimate_tokens(toon_text)
+    output_tokens = estimate_tokens(toon_text, config.token_encoding)
     savings_ratio = (input_tokens - output_tokens) / input_tokens
 
     if savings_ratio < config.min_savings_ratio and not config.force:

datoon-1.9.0/src/datoon/errors.py

@@ -0,0 +1,20 @@
+"""Canonical exception types for datoon.
+
+Kept in a leaf module so any layer — readers, converter, CLI, MCP — can raise
+the datoon error taxonomy without importing a higher-level module.
+
+Convention:
+
+* ``DatoonError`` — the user's *payload* is malformed or unsuitable (invalid
+  JSON/XML/YAML, non-object rows, a refused DTD). These are the errors surfaced
+  to end users of the CLI/MCP.
+* ``ValueError`` — a *usage* error in how datoon was called (unknown format
+  name, text passed where a file path is required). Programmer-facing.
+* ``ImportError`` — an optional extra needed for a format is not installed.
+"""
+
+from __future__ import annotations
+
+
+class DatoonError(RuntimeError):
+    """Raised when datoon cannot process the payload safely."""

{datoon-1.8.0 → datoon-1.9.0}/src/datoon/mcp_server.py

@@ -15,7 +15,7 @@ except ImportError as exc:
 from datoon import __version__
 from datoon.analyzer import analyze_payload
 from datoon.converter import DatoonError, convert_json_for_llm
-from datoon.models import ConversionConfig
+from datoon.models import DEFAULT_TOKEN_ENCODING, ConversionConfig
 from datoon.readers import TEXT_FORMATS, read_tabular
 
 mcp = FastMCP(
@@ -40,6 +40,7 @@ def convert_json(
     max_depth: int = 6,
     min_uniform_rows: int = 3,
     timeout: int = 30,
+    encoding: str = DEFAULT_TOKEN_ENCODING,
 ) -> dict[str, Any]:
     """Convert a JSON payload to TOON when structure and savings meet policy.
 
@@ -53,6 +54,7 @@ def convert_json(
             min_uniform_rows=min_uniform_rows,
             force=force,
             toon_cli_timeout=timeout,
+            token_encoding=encoding,
         )
         outcome = convert_json_for_llm(json_text, config)
         return {
@@ -105,6 +107,7 @@ def convert_text(
     max_depth: int = 6,
     min_uniform_rows: int = 3,
     timeout: int = 30,
+    encoding: str = DEFAULT_TOKEN_ENCODING,
 ) -> dict[str, Any]:
     """Convert structured text (csv, yaml, xml, jsonl) to TOON with policy gating.
 
@@ -127,6 +130,7 @@ def convert_text(
             min_uniform_rows=min_uniform_rows,
             force=force,
             toon_cli_timeout=timeout,
+            token_encoding=encoding,
         )
         outcome = convert_json_for_llm(json_text, config)
         return {

{datoon-1.8.0 → datoon-1.9.0}/src/datoon/models.py

@@ -7,6 +7,10 @@ from typing import Literal
 
 Decision = Literal["convert", "skip"]
 
+# Default tiktoken encoding for token estimates. o200k_base matches GPT-4o and
+# the o-series; it is closer to modern targets than the older cl100k_base.
+DEFAULT_TOKEN_ENCODING = "o200k_base"
+
 
 @dataclass(frozen=True, slots=True)
 class ConversionConfig:
@@ -17,6 +21,7 @@ class ConversionConfig:
     min_uniform_rows: int = 3
     force: bool = False
     toon_cli_timeout: int = 30
+    token_encoding: str = DEFAULT_TOKEN_ENCODING
 
     def __post_init__(self) -> None:
         """Validate boundary values to avoid ambiguous runtime decisions."""
@@ -28,6 +33,8 @@ class ConversionConfig:
             raise ValueError("min_uniform_rows must be at least 2.")
         if self.toon_cli_timeout < 1:
             raise ValueError("toon_cli_timeout must be at least 1.")
+        if not self.token_encoding:
+            raise ValueError("token_encoding must be a non-empty encoding name.")
 
 
 @dataclass(frozen=True, slots=True)

{datoon-1.8.0 → datoon-1.9.0}/src/datoon/readers/jsonl.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import json
 from typing import Any
 
-from datoon.converter import DatoonError
+from datoon.errors import DatoonError
 
 
 def read_jsonl(text: str) -> list[dict[str, Any]]:

{datoon-1.8.0 → datoon-1.9.0}/src/datoon/readers/xml.py

@@ -13,6 +13,7 @@ import re
 import xml.etree.ElementTree as ET
 from typing import Any
 
+from datoon.errors import DatoonError
 from datoon.readers._coerce import coerce_scalar as _coerce
 
 _DOCTYPE_RE = re.compile(r"<!DOCTYPE", re.IGNORECASE)
@@ -21,18 +22,18 @@ _DOCTYPE_RE = re.compile(r"<!DOCTYPE", re.IGNORECASE)
 def read_xml(text: str) -> list[dict[str, Any]]:
     """Parse XML into a list of row dicts from the dominant repeated child element."""
     if _DOCTYPE_RE.search(text):
-        raise ValueError(
+        raise DatoonError(
             "XML with a DOCTYPE/DTD is not allowed (entity-expansion risk)."
         )
 
     try:
         root = ET.fromstring(text)
     except ET.ParseError as exc:
-        raise ValueError(f"Invalid XML: {exc}") from exc
+        raise DatoonError(f"Invalid XML: {exc}") from exc
 
     children = list(root)
     if not children:
-        raise ValueError("XML root element has no children.")
+        raise DatoonError("XML root element has no children.")
 
     tag_counts: dict[str, int] = {}
     for child in children:

{datoon-1.8.0 → datoon-1.9.0}/src/datoon/readers/yaml.py

@@ -4,6 +4,12 @@ from __future__ import annotations
 
 from typing import Any
 
+from datoon.errors import DatoonError
+
+_SHAPE_ERROR = (
+    "YAML must be a list of objects, or a mapping with exactly one list value."
+)
+
 
 def read_yaml(text: str) -> list[dict[str, Any]]:
     """Parse YAML text into a list of row dicts."""
@@ -19,12 +25,16 @@ def read_yaml(text: str) -> list[dict[str, Any]]:
 
 
 def _normalize(data: Any) -> list[dict[str, Any]]:
+    rows: list[Any] | None = None
     if isinstance(data, list):
-        return data
-    if isinstance(data, dict):
+        rows = data
+    elif isinstance(data, dict):
         lists = [v for v in data.values() if isinstance(v, list) and v]
         if len(lists) == 1:
-            return lists[0]
-    raise ValueError(
-        "YAML must be a list of objects, or a mapping with exactly one list value."
-    )
+            rows = lists[0]
+
+    if rows is None:
+        raise DatoonError(_SHAPE_ERROR)
+    if not all(isinstance(item, dict) for item in rows):
+        raise DatoonError(_SHAPE_ERROR)
+    return rows

{datoon-1.8.0 → datoon-1.9.0}/src/datoon.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datoon
-Version: 1.8.0
+Version: 1.9.0
 Summary: Smart structured-data-to-TOON gateway with pragmatic auto-gating for LLM prompts.
 Author-email: Andrii Suruhov <andrii.suruhov@gmail.com>
 License: MIT License
@@ -451,12 +451,12 @@ Auto mode avoids low-benefit and high-risk payloads (`orders-nested`, `mixed-non
 
 | Dataset | JSON | TOON (forced) | Raw Saved | Auto | Auto Tokens | Auto Saved |
 |---|---:|---:|---:|---|---:|---:|
-| users-small | 56 | 31 | 44.6% | convert | 31 | 44.6% |
-| events-medium | 198 | 111 | 43.9% | convert | 111 | 43.9% |
-| orders-nested | 93 | 91 | 2.2% | skip | 93 | 0.0% |
-| mixed-non-uniform | 35 | 37 | -5.7% | skip | 35 | 0.0% |
-| metrics-wide | 133 | 63 | 52.6% | convert | 63 | 52.6% |
-| **Average** | **103** | **67** | **27.5%** | **3/5 convert** | **67** | **28.2%** |
+| users-small | 54 | 40 | 25.9% | convert | 40 | 25.9% |
+| events-medium | 219 | 162 | 26.0% | convert | 162 | 26.0% |
+| orders-nested | 106 | 116 | -9.4% | skip | 106 | 0.0% |
+| mixed-non-uniform | 35 | 47 | -34.3% | skip | 35 | 0.0% |
+| metrics-wide | 142 | 103 | 27.5% | convert | 103 | 27.5% |
+| **Average** | **111** | **94** | **7.1%** | **3/5 convert** | **89** | **15.9%** |
 
 *Forced conversion succeeded for 5/5 payloads.*
 

{datoon-1.8.0 → datoon-1.9.0}/src/datoon.egg-info/SOURCES.txt

@@ -91,6 +91,7 @@ src/datoon/_version.py
 src/datoon/analyzer.py
 src/datoon/cli.py
 src/datoon/converter.py
+src/datoon/errors.py
 src/datoon/mcp_server.py
 src/datoon/models.py
 src/datoon/py.typed
@@ -118,8 +119,10 @@ tests/integration/test_toon_cli_integration.py
 tests/unit/test_analyzer.py
 tests/unit/test_cli.py
 tests/unit/test_converter_policy.py
+tests/unit/test_errors.py
 tests/unit/test_installer.py
 tests/unit/test_mcp_server.py
 tests/unit/test_models.py
 tests/unit/test_readers.py
-tests/unit/test_tabular_rows.py
+tests/unit/test_tabular_rows.py
+tests/unit/test_tokens.py

{datoon-1.8.0 → datoon-1.9.0}/src/datoon.egg-info/scm_file_list.json

@@ -39,6 +39,7 @@
     "src/datoon/models.py",
     "src/datoon/converter.py",
     "src/datoon/cli.py",
+    "src/datoon/errors.py",
     "src/datoon/readers/xml.py",
     "src/datoon/readers/_coerce.py",
     "src/datoon/readers/__init__.py",
@@ -63,7 +64,9 @@
     "tests/unit/test_mcp_server.py",
     "tests/unit/test_tabular_rows.py",
     "tests/unit/test_analyzer.py",
+    "tests/unit/test_tokens.py",
     "tests/unit/test_installer.py",
+    "tests/unit/test_errors.py",
     "tests/unit/test_cli.py",
     ".claude-plugin/plugin.json",
     ".claude-plugin/marketplace.json",

{datoon-1.8.0 → datoon-1.9.0}/src/datoon.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "1.8.0",
+  "tag": "1.9.0",
   "distance": 0,
-  "node": "gacb329fb8ab013fc074f180d839e72bc2381dad3",
+  "node": "g412011b90239e0e07c401c668d255671348628c2",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-07-07"

{datoon-1.8.0 → datoon-1.9.0}/tests/unit/test_cli.py

@@ -53,7 +53,9 @@ def test_main_converts_json_from_stdin(
         lambda _, **kw: "rows[3]{id,v}:\n  1,a\n  2,b\n  3,c\n",
     )
     token_seq = iter([100, 40])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([])
     assert rc == 0
@@ -88,7 +90,9 @@ def test_main_converts_csv_via_flag(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(csv_file), "--format", "csv"])
     assert rc == 0
@@ -107,7 +111,9 @@ def test_main_auto_detects_csv_from_extension(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(csv_file)])
     assert rc == 0
@@ -134,7 +140,9 @@ def test_main_converts_jsonl_from_stdin(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main(["--format", "jsonl"])
     assert rc == 0
@@ -153,7 +161,9 @@ def test_main_auto_detects_jsonl_from_extension(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(f)])
     assert rc == 0
@@ -182,7 +192,9 @@ def test_main_converts_xml_via_flag(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(f), "--format", "xml"])
     assert rc == 0
@@ -223,7 +235,9 @@ def test_main_csv_report_stdout(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(csv_file), "--report-stdout"])
     assert rc == 0
@@ -244,7 +258,9 @@ def test_main_csv_skips_when_savings_insufficient(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 98])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(csv_file), "--report-stdout"])
     assert rc == 0
@@ -315,3 +331,23 @@ def test_main_forwards_sheet_and_table_for_binary(
     )
     assert rc == 0
     assert captured == {"sheet": 2, "table": 1}
+
+
+def test_main_forwards_encoding_to_estimate(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    payload = '{"rows":[{"id":1,"v":"a"},{"id":2,"v":"b"},{"id":3,"v":"c"}]}'
+    monkeypatch.setattr("sys.stdin", __import__("io").StringIO(payload))
+    monkeypatch.setattr("datoon.converter._run_toon_cli", lambda _, **kw: "x\n")
+
+    seen: list[str] = []
+
+    def fake_estimate(text: str, encoding: str = "o200k_base") -> int:
+        seen.append(encoding)
+        return 50 if len(seen) == 1 else 10
+
+    monkeypatch.setattr("datoon.converter.estimate_tokens", fake_estimate)
+
+    rc = main(["--force", "--encoding", "cl100k_base"])
+    assert rc == 0
+    assert seen and all(enc == "cl100k_base" for enc in seen)

{datoon-1.8.0 → datoon-1.9.0}/tests/unit/test_converter_policy.py

@@ -107,7 +107,7 @@ def test_policy_converts_when_savings_above_threshold(
 
     token_sequence = iter([100, 60])  # input, output
     monkeypatch.setattr(
-        "datoon.converter.estimate_tokens", lambda _: next(token_sequence)
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_sequence)
     )
 
     outcome = convert_json_for_llm(payload, ConversionConfig(min_savings_ratio=0.15))
@@ -127,7 +127,7 @@ def test_policy_skips_when_savings_below_threshold(
 
     token_sequence = iter([100, 95])  # input, output
     monkeypatch.setattr(
-        "datoon.converter.estimate_tokens", lambda _: next(token_sequence)
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_sequence)
     )
 
     outcome = convert_json_for_llm(payload, ConversionConfig(min_savings_ratio=0.10))

datoon-1.9.0/tests/unit/test_errors.py

@@ -0,0 +1,39 @@
+"""Tests for the datoon.errors module and its use across layers."""
+
+from __future__ import annotations
+
+import inspect
+
+import pytest
+
+
+def test_datoon_error_lives_in_errors_module() -> None:
+    from datoon.errors import DatoonError
+
+    assert issubclass(DatoonError, RuntimeError)
+
+
+def test_converter_reexports_the_same_error_object() -> None:
+    # Backward compatibility: `from datoon.converter import DatoonError` must
+    # keep working and refer to the single canonical class.
+    from datoon.converter import DatoonError as FromConverter
+    from datoon.errors import DatoonError as Canonical
+
+    assert FromConverter is Canonical
+
+
+def test_jsonl_reader_does_not_depend_on_converter() -> None:
+    # The low-level reader must not import the high-level conversion service
+    # just for an exception type (layering inversion).
+    import datoon.readers.jsonl as jsonl
+
+    source = inspect.getsource(jsonl)
+    assert "datoon.converter" not in source
+
+
+def test_jsonl_raises_canonical_error() -> None:
+    from datoon.errors import DatoonError
+    from datoon.readers.jsonl import read_jsonl
+
+    with pytest.raises(DatoonError, match="line 2"):
+        read_jsonl('{"a":1}\nbad json\n')

{datoon-1.8.0 → datoon-1.9.0}/tests/unit/test_mcp_server.py

@@ -92,7 +92,9 @@ def test_convert_json_converts_when_savings_sufficient(
         lambda _, **kw: "rows[3]{id,v}:\n  1,a\n  2,b\n  3,c\n",
     )
     token_seq = iter([100, 60])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     result = convert_json(payload, min_savings=0.15)
     assert "error" not in result
@@ -122,7 +124,9 @@ def test_convert_text_csv_converts(monkeypatch: pytest.MonkeyPatch) -> None:
         **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n  2,Lin,analyst\n  3,Grace,viewer\n",
     )
     token_seq = iter([100, 40])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     result = convert_text(csv_text, fmt="csv")
     assert "error" not in result
@@ -141,7 +145,9 @@ def test_convert_text_jsonl_converts(monkeypatch: pytest.MonkeyPatch) -> None:
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 40])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     result = convert_text(jsonl_text, fmt="jsonl")
     assert "error" not in result
@@ -162,7 +168,9 @@ def test_convert_text_xml_converts(monkeypatch: pytest.MonkeyPatch) -> None:
         lambda _, **kw: "rows[3]{id,name}:\n  1,Ada\n",
     )
     token_seq = iter([100, 40])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     result = convert_text(xml_text, fmt="xml")
     assert "error" not in result

{datoon-1.8.0 → datoon-1.9.0}/tests/unit/test_readers.py

@@ -8,6 +8,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
+from datoon.errors import DatoonError
 from datoon.readers import (
     ALL_FORMATS,
     BINARY_FORMATS,
@@ -215,6 +216,39 @@ def test_read_yaml_missing_dep(monkeypatch: pytest.MonkeyPatch) -> None:
         yaml_reader.read_yaml("- id: 1")
 
 
+def test_read_yaml_rejects_list_of_scalars(monkeypatch: pytest.MonkeyPatch) -> None:
+    yaml_mock = MagicMock()
+    yaml_mock.safe_load.return_value = [1, 2, 3]
+    monkeypatch.setitem(sys.modules, "yaml", yaml_mock)
+
+    from datoon.readers.yaml import read_yaml
+
+    with pytest.raises(DatoonError, match="list of objects"):
+        read_yaml("- 1\n- 2\n")
+
+
+def test_read_yaml_rejects_mixed_list(monkeypatch: pytest.MonkeyPatch) -> None:
+    yaml_mock = MagicMock()
+    yaml_mock.safe_load.return_value = [{"id": 1}, "oops", {"id": 3}]
+    monkeypatch.setitem(sys.modules, "yaml", yaml_mock)
+
+    from datoon.readers.yaml import read_yaml
+
+    with pytest.raises(DatoonError, match="list of objects"):
+        read_yaml("- id: 1\n- oops\n")
+
+
+def test_read_yaml_rejects_bad_shape(monkeypatch: pytest.MonkeyPatch) -> None:
+    yaml_mock = MagicMock()
+    yaml_mock.safe_load.return_value = 42
+    monkeypatch.setitem(sys.modules, "yaml", yaml_mock)
+
+    from datoon.readers.yaml import read_yaml
+
+    with pytest.raises(DatoonError, match="list of objects"):
+        read_yaml("42\n")
+
+
 # ---------------------------------------------------------------------------
 # XML reader
 # ---------------------------------------------------------------------------
@@ -258,7 +292,7 @@ def test_read_xml_rejects_dtd_billion_laughs() -> None:
         '<!ENTITY lol2 "&lol;&lol;&lol;">]>'
         "<rows><r>&lol2;</r></rows>"
     )
-    with pytest.raises(ValueError, match="DOCTYPE"):
+    with pytest.raises(DatoonError, match="DOCTYPE"):
         read_xml(payload)
 
 
@@ -267,17 +301,17 @@ def test_read_xml_rejects_external_entity_dtd() -> None:
         '<!DOCTYPE foo [<!ENTITY xxe SYSTEM "file:///etc/passwd">]>'
         "<rows><r>&xxe;</r></rows>"
     )
-    with pytest.raises(ValueError, match="DOCTYPE"):
+    with pytest.raises(DatoonError, match="DOCTYPE"):
         read_xml(payload)
 
 
 def test_read_xml_invalid_raises() -> None:
-    with pytest.raises(ValueError, match="Invalid XML"):
+    with pytest.raises(DatoonError, match="Invalid XML"):
         read_xml("<not closed>")
 
 
 def test_read_xml_empty_root_raises() -> None:
-    with pytest.raises(ValueError, match="no children"):
+    with pytest.raises(DatoonError, match="no children"):
         read_xml("<root/>")
 
 

datoon-1.9.0/tests/unit/test_tokens.py

@@ -0,0 +1,43 @@
+"""Tests for configurable token-encoding estimation."""
+
+from __future__ import annotations
+
+import pytest
+
+from datoon.converter import estimate_tokens
+from datoon.models import DEFAULT_TOKEN_ENCODING, ConversionConfig
+
+
+def test_default_encoding_is_o200k() -> None:
+    assert DEFAULT_TOKEN_ENCODING == "o200k_base"
+    assert ConversionConfig().token_encoding == "o200k_base"
+
+
+def test_config_rejects_empty_encoding() -> None:
+    with pytest.raises(ValueError, match="token_encoding"):
+        ConversionConfig(token_encoding="")
+
+
+def test_estimate_tokens_respects_encoding() -> None:
+    tiktoken = pytest.importorskip("tiktoken")
+    text = '{"rows":[{"id":1,"name":"Ada","role":"admin"}]}'
+
+    for name in ("o200k_base", "cl100k_base"):
+        expected = len(tiktoken.get_encoding(name).encode(text))
+        assert estimate_tokens(text, encoding=name) == expected
+
+
+def test_estimate_tokens_falls_back_to_heuristic_without_tiktoken(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import sys
+
+    import datoon.converter as conv
+
+    conv._load_token_encoder.cache_clear()
+    monkeypatch.setitem(sys.modules, "tiktoken", None)  # type: ignore[arg-type]
+    try:
+        # 30 chars -> ceil(30 / 3) == 10 by the char heuristic.
+        assert estimate_tokens("x" * 30, encoding="o200k_base") == 10
+    finally:
+        conv._load_token_encoder.cache_clear()