datoon 1.7.4__tar.gz → 1.9.0__tar.gz

{datoon-1.7.4 → datoon-1.9.0}/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [1.9.0](https://github.com/andrii-su/datoon/compare/v1.8.0...v1.9.0) (2026-07-07)
+
+
+### Features
+
+* **tokens:** make token encoding configurable, default to o200k_base ([#46](https://github.com/andrii-su/datoon/issues/46)) ([8483eb7](https://github.com/andrii-su/datoon/commit/8483eb7aca27c97df2827d552e0086cf9bfbe157))
+
+# [1.8.0](https://github.com/andrii-su/datoon/compare/v1.7.4...v1.8.0) (2026-07-07)
+
+
+### Features
+
+* **cli:** add --sheet and --table for excel/numbers input ([#39](https://github.com/andrii-su/datoon/issues/39)) ([4799712](https://github.com/andrii-su/datoon/commit/4799712b003ca4fae25cd555aec83b2b1cece136))
+
 ## [1.7.4](https://github.com/andrii-su/datoon/compare/v1.7.3...v1.7.4) (2026-07-07)
 
 

{datoon-1.7.4 → datoon-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datoon
-Version: 1.7.4
+Version: 1.9.0
 Summary: Smart structured-data-to-TOON gateway with pragmatic auto-gating for LLM prompts.
 Author-email: Andrii Suruhov <andrii.suruhov@gmail.com>
 License: MIT License
@@ -451,12 +451,12 @@ Auto mode avoids low-benefit and high-risk payloads (`orders-nested`, `mixed-non
 
 | Dataset | JSON | TOON (forced) | Raw Saved | Auto | Auto Tokens | Auto Saved |
 |---|---:|---:|---:|---|---:|---:|
-| users-small | 56 | 31 | 44.6% | convert | 31 | 44.6% |
-| events-medium | 198 | 111 | 43.9% | convert | 111 | 43.9% |
-| orders-nested | 93 | 91 | 2.2% | skip | 93 | 0.0% |
-| mixed-non-uniform | 35 | 37 | -5.7% | skip | 35 | 0.0% |
-| metrics-wide | 133 | 63 | 52.6% | convert | 63 | 52.6% |
-| **Average** | **103** | **67** | **27.5%** | **3/5 convert** | **67** | **28.2%** |
+| users-small | 54 | 40 | 25.9% | convert | 40 | 25.9% |
+| events-medium | 219 | 162 | 26.0% | convert | 162 | 26.0% |
+| orders-nested | 106 | 116 | -9.4% | skip | 106 | 0.0% |
+| mixed-non-uniform | 35 | 47 | -34.3% | skip | 35 | 0.0% |
+| metrics-wide | 142 | 103 | 27.5% | convert | 103 | 27.5% |
+| **Average** | **111** | **94** | **7.1%** | **3/5 convert** | **89** | **15.9%** |
 
 *Forced conversion succeeded for 5/5 payloads.*
 

{datoon-1.7.4 → datoon-1.9.0}/README.md

@@ -391,12 +391,12 @@ Auto mode avoids low-benefit and high-risk payloads (`orders-nested`, `mixed-non
 
 | Dataset | JSON | TOON (forced) | Raw Saved | Auto | Auto Tokens | Auto Saved |
 |---|---:|---:|---:|---|---:|---:|
-| users-small | 56 | 31 | 44.6% | convert | 31 | 44.6% |
-| events-medium | 198 | 111 | 43.9% | convert | 111 | 43.9% |
-| orders-nested | 93 | 91 | 2.2% | skip | 93 | 0.0% |
-| mixed-non-uniform | 35 | 37 | -5.7% | skip | 35 | 0.0% |
-| metrics-wide | 133 | 63 | 52.6% | convert | 63 | 52.6% |
-| **Average** | **103** | **67** | **27.5%** | **3/5 convert** | **67** | **28.2%** |
+| users-small | 54 | 40 | 25.9% | convert | 40 | 25.9% |
+| events-medium | 219 | 162 | 26.0% | convert | 162 | 26.0% |
+| orders-nested | 106 | 116 | -9.4% | skip | 106 | 0.0% |
+| mixed-non-uniform | 35 | 47 | -34.3% | skip | 35 | 0.0% |
+| metrics-wide | 142 | 103 | 27.5% | convert | 103 | 27.5% |
+| **Average** | **111** | **94** | **7.1%** | **3/5 convert** | **89** | **15.9%** |
 
 *Forced conversion succeeded for 5/5 payloads.*
 

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.7.4'
-__version_tuple__ = version_tuple = (1, 7, 4)
+__version__ = version = '1.9.0'
+__version_tuple__ = version_tuple = (1, 9, 0)
 
-__commit_id__ = commit_id = 'g9c1017702'
+__commit_id__ = commit_id = 'g412011b90'

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/cli.py

@@ -5,12 +5,12 @@ from __future__ import annotations
 import argparse
 import json
 import sys
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Sequence
 
 from datoon import __version__
 from datoon.converter import DatoonError, convert_json_for_llm
-from datoon.models import ConversionConfig
+from datoon.models import DEFAULT_TOKEN_ENCODING, ConversionConfig
 from datoon.readers import ALL_FORMATS, BINARY_FORMATS, detect_format, read_tabular
 
 
@@ -75,6 +75,26 @@ def build_parser() -> argparse.ArgumentParser:
         default=30,
         help="Seconds to wait for the TOON CLI before aborting (default: 30).",
     )
+    parser.add_argument(
+        "--sheet",
+        type=int,
+        default=0,
+        help="Worksheet index for excel/numbers input (default: 0).",
+    )
+    parser.add_argument(
+        "--table",
+        type=int,
+        default=0,
+        help="Table index within a Numbers sheet (default: 0).",
+    )
+    parser.add_argument(
+        "--encoding",
+        default=DEFAULT_TOKEN_ENCODING,
+        help=(
+            "tiktoken encoding for token estimates "
+            f"(default: {DEFAULT_TOKEN_ENCODING})."
+        ),
+    )
     parser.add_argument(
         "--report",
         help="Optional path to write conversion report as JSON.",
@@ -159,6 +179,7 @@ def main(argv: Sequence[str] | None = None) -> int:
             min_uniform_rows=args.min_uniform_rows,
             force=args.force,
             toon_cli_timeout=args.timeout,
+            token_encoding=args.encoding,
         )
 
         fmt = _resolve_format(args.input, args.format)
@@ -172,7 +193,9 @@ def main(argv: Sequence[str] | None = None) -> int:
                     f"datoon error: --format {fmt} requires a file path, not stdin.\n"
                 )
                 return 1
-            rows = read_tabular(fmt, path=Path(args.input))
+            rows = read_tabular(
+                fmt, path=Path(args.input), sheet=args.sheet, table=args.table
+            )
             json_text = json.dumps(rows, ensure_ascii=False, separators=(",", ":"))
             outcome = convert_json_for_llm(json_text, config)
         else:

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/converter.py

@@ -9,7 +9,9 @@ from functools import lru_cache
 from typing import Any
 
 from datoon.analyzer import analyze_payload
+from datoon.errors import DatoonError
 from datoon.models import (
+    DEFAULT_TOKEN_ENCODING,
     ConversionConfig,
     ConversionOutcome,
     ConversionReport,
@@ -18,9 +20,7 @@ from datoon.models import (
 
 TOON_CLI_PACKAGE = "@toon-format/cli@2"
 
-
-class DatoonError(RuntimeError):
-    """Raised when datoon cannot process the payload safely."""
+__all__ = ["DatoonError", "convert_json_for_llm", "estimate_tokens"]
 
 
 def _reject_non_finite(constant: str) -> Any:
@@ -43,20 +43,20 @@ def _normalize_json(raw_text: str) -> tuple[Any, str]:
     return parsed, normalized
 
 
-@lru_cache(maxsize=1)
-def _load_token_encoder() -> Any | None:
-    """Load a token encoder once, returning None when optional dependency is absent."""
+@lru_cache(maxsize=8)
+def _load_token_encoder(encoding: str) -> Any | None:
+    """Load a token encoder once per name, None when the optional dep is absent."""
     try:
         import tiktoken  # type: ignore
     except ImportError:
         return None
 
-    return tiktoken.get_encoding("cl100k_base")
+    return tiktoken.get_encoding(encoding)
 
 
-def estimate_tokens(text: str) -> int:
+def estimate_tokens(text: str, encoding: str = DEFAULT_TOKEN_ENCODING) -> int:
     """Estimate token count with tiktoken when installed, else use a char heuristic."""
-    encoder = _load_token_encoder()
+    encoder = _load_token_encoder(encoding)
     if encoder is not None:
         return len(encoder.encode(text))
 
@@ -119,7 +119,7 @@ def convert_json_for_llm(raw_text: str, config: ConversionConfig) -> ConversionO
     """Convert JSON to TOON when structure and savings meet configured policy."""
     parsed, normalized_json = _normalize_json(raw_text)
     analysis = analyze_payload(parsed, config)
-    input_tokens = estimate_tokens(normalized_json)
+    input_tokens = estimate_tokens(normalized_json, config.token_encoding)
 
     if not analysis.is_candidate and not config.force:
         return _build_skip_outcome(
@@ -144,7 +144,7 @@ def convert_json_for_llm(raw_text: str, config: ConversionConfig) -> ConversionO
             analysis=analysis,
         )
 
-    output_tokens = estimate_tokens(toon_text)
+    output_tokens = estimate_tokens(toon_text, config.token_encoding)
     savings_ratio = (input_tokens - output_tokens) / input_tokens
 
     if savings_ratio < config.min_savings_ratio and not config.force:

datoon-1.9.0/src/datoon/errors.py

@@ -0,0 +1,20 @@
+"""Canonical exception types for datoon.
+
+Kept in a leaf module so any layer — readers, converter, CLI, MCP — can raise
+the datoon error taxonomy without importing a higher-level module.
+
+Convention:
+
+* ``DatoonError`` — the user's *payload* is malformed or unsuitable (invalid
+  JSON/XML/YAML, non-object rows, a refused DTD). These are the errors surfaced
+  to end users of the CLI/MCP.
+* ``ValueError`` — a *usage* error in how datoon was called (unknown format
+  name, text passed where a file path is required). Programmer-facing.
+* ``ImportError`` — an optional extra needed for a format is not installed.
+"""
+
+from __future__ import annotations
+
+
+class DatoonError(RuntimeError):
+    """Raised when datoon cannot process the payload safely."""

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/mcp_server.py

@@ -15,7 +15,7 @@ except ImportError as exc:
 from datoon import __version__
 from datoon.analyzer import analyze_payload
 from datoon.converter import DatoonError, convert_json_for_llm
-from datoon.models import ConversionConfig
+from datoon.models import DEFAULT_TOKEN_ENCODING, ConversionConfig
 from datoon.readers import TEXT_FORMATS, read_tabular
 
 mcp = FastMCP(
@@ -40,6 +40,7 @@ def convert_json(
     max_depth: int = 6,
     min_uniform_rows: int = 3,
     timeout: int = 30,
+    encoding: str = DEFAULT_TOKEN_ENCODING,
 ) -> dict[str, Any]:
     """Convert a JSON payload to TOON when structure and savings meet policy.
 
@@ -53,6 +54,7 @@ def convert_json(
             min_uniform_rows=min_uniform_rows,
             force=force,
             toon_cli_timeout=timeout,
+            token_encoding=encoding,
         )
         outcome = convert_json_for_llm(json_text, config)
         return {
@@ -105,6 +107,7 @@ def convert_text(
     max_depth: int = 6,
     min_uniform_rows: int = 3,
     timeout: int = 30,
+    encoding: str = DEFAULT_TOKEN_ENCODING,
 ) -> dict[str, Any]:
     """Convert structured text (csv, yaml, xml, jsonl) to TOON with policy gating.
 
@@ -127,6 +130,7 @@ def convert_text(
             min_uniform_rows=min_uniform_rows,
             force=force,
             toon_cli_timeout=timeout,
+            token_encoding=encoding,
         )
         outcome = convert_json_for_llm(json_text, config)
         return {

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/models.py

@@ -7,6 +7,10 @@ from typing import Literal
 
 Decision = Literal["convert", "skip"]
 
+# Default tiktoken encoding for token estimates. o200k_base matches GPT-4o and
+# the o-series; it is closer to modern targets than the older cl100k_base.
+DEFAULT_TOKEN_ENCODING = "o200k_base"
+
 
 @dataclass(frozen=True, slots=True)
 class ConversionConfig:
@@ -17,6 +21,7 @@ class ConversionConfig:
     min_uniform_rows: int = 3
     force: bool = False
     toon_cli_timeout: int = 30
+    token_encoding: str = DEFAULT_TOKEN_ENCODING
 
     def __post_init__(self) -> None:
         """Validate boundary values to avoid ambiguous runtime decisions."""
@@ -28,6 +33,8 @@ class ConversionConfig:
             raise ValueError("min_uniform_rows must be at least 2.")
         if self.toon_cli_timeout < 1:
             raise ValueError("toon_cli_timeout must be at least 1.")
+        if not self.token_encoding:
+            raise ValueError("token_encoding must be a non-empty encoding name.")
 
 
 @dataclass(frozen=True, slots=True)

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/readers/__init__.py

@@ -37,16 +37,20 @@ def read_tabular(
     *,
     text: str | None = None,
     path: Path | None = None,
+    sheet: int = 0,
+    table: int = 0,
 ) -> list[dict[str, Any]]:
     """Read structured data and return rows as a list of dicts.
 
     Text formats (csv, jsonl, yaml, xml) require ``text``.
     Binary formats (excel, parquet, orc, avro, numbers) require ``path``.
+    ``sheet`` selects the worksheet for excel/numbers; ``table`` selects the
+    table within a Numbers sheet. Both are ignored by columnar formats.
     """
     if fmt in BINARY_FORMATS:
         if path is None:
             raise ValueError(f"Format '{fmt}' requires a file path, not text input.")
-        return _read_binary(fmt, path)
+        return _read_binary(fmt, path, sheet=sheet, table=table)
     if fmt in TEXT_FORMATS:
         if text is None:
             raise ValueError(f"Format '{fmt}' requires text input.")
@@ -74,11 +78,13 @@ def _read_text(fmt: str, text: str) -> list[dict[str, Any]]:
     raise ValueError(f"Unknown text format: {fmt}")
 
 
-def _read_binary(fmt: str, path: Path) -> list[dict[str, Any]]:
+def _read_binary(
+    fmt: str, path: Path, *, sheet: int = 0, table: int = 0
+) -> list[dict[str, Any]]:
     if fmt == "excel":
         from datoon.readers.excel import read_excel
 
-        return read_excel(path)
+        return read_excel(path, sheet=sheet)
     if fmt == "parquet":
         from datoon.readers.columnar import read_parquet
 
@@ -94,5 +100,5 @@ def _read_binary(fmt: str, path: Path) -> list[dict[str, Any]]:
     if fmt == "numbers":
         from datoon.readers.numbers import read_numbers
 
-        return read_numbers(path)
+        return read_numbers(path, sheet=sheet, table=table)
     raise ValueError(f"Unknown binary format: {fmt}")

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/readers/jsonl.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import json
 from typing import Any
 
-from datoon.converter import DatoonError
+from datoon.errors import DatoonError
 
 
 def read_jsonl(text: str) -> list[dict[str, Any]]:

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/readers/xml.py

@@ -13,6 +13,7 @@ import re
 import xml.etree.ElementTree as ET
 from typing import Any
 
+from datoon.errors import DatoonError
 from datoon.readers._coerce import coerce_scalar as _coerce
 
 _DOCTYPE_RE = re.compile(r"<!DOCTYPE", re.IGNORECASE)
@@ -21,18 +22,18 @@ _DOCTYPE_RE = re.compile(r"<!DOCTYPE", re.IGNORECASE)
 def read_xml(text: str) -> list[dict[str, Any]]:
     """Parse XML into a list of row dicts from the dominant repeated child element."""
     if _DOCTYPE_RE.search(text):
-        raise ValueError(
+        raise DatoonError(
             "XML with a DOCTYPE/DTD is not allowed (entity-expansion risk)."
         )
 
     try:
         root = ET.fromstring(text)
     except ET.ParseError as exc:
-        raise ValueError(f"Invalid XML: {exc}") from exc
+        raise DatoonError(f"Invalid XML: {exc}") from exc
 
     children = list(root)
     if not children:
-        raise ValueError("XML root element has no children.")
+        raise DatoonError("XML root element has no children.")
 
     tag_counts: dict[str, int] = {}
     for child in children:

{datoon-1.7.4 → datoon-1.9.0}/src/datoon/readers/yaml.py

@@ -4,6 +4,12 @@ from __future__ import annotations
 
 from typing import Any
 
+from datoon.errors import DatoonError
+
+_SHAPE_ERROR = (
+    "YAML must be a list of objects, or a mapping with exactly one list value."
+)
+
 
 def read_yaml(text: str) -> list[dict[str, Any]]:
     """Parse YAML text into a list of row dicts."""
@@ -19,12 +25,16 @@ def read_yaml(text: str) -> list[dict[str, Any]]:
 
 
 def _normalize(data: Any) -> list[dict[str, Any]]:
+    rows: list[Any] | None = None
     if isinstance(data, list):
-        return data
-    if isinstance(data, dict):
+        rows = data
+    elif isinstance(data, dict):
         lists = [v for v in data.values() if isinstance(v, list) and v]
         if len(lists) == 1:
-            return lists[0]
-    raise ValueError(
-        "YAML must be a list of objects, or a mapping with exactly one list value."
-    )
+            rows = lists[0]
+
+    if rows is None:
+        raise DatoonError(_SHAPE_ERROR)
+    if not all(isinstance(item, dict) for item in rows):
+        raise DatoonError(_SHAPE_ERROR)
+    return rows

{datoon-1.7.4 → datoon-1.9.0}/src/datoon.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datoon
-Version: 1.7.4
+Version: 1.9.0
 Summary: Smart structured-data-to-TOON gateway with pragmatic auto-gating for LLM prompts.
 Author-email: Andrii Suruhov <andrii.suruhov@gmail.com>
 License: MIT License
@@ -451,12 +451,12 @@ Auto mode avoids low-benefit and high-risk payloads (`orders-nested`, `mixed-non
 
 | Dataset | JSON | TOON (forced) | Raw Saved | Auto | Auto Tokens | Auto Saved |
 |---|---:|---:|---:|---|---:|---:|
-| users-small | 56 | 31 | 44.6% | convert | 31 | 44.6% |
-| events-medium | 198 | 111 | 43.9% | convert | 111 | 43.9% |
-| orders-nested | 93 | 91 | 2.2% | skip | 93 | 0.0% |
-| mixed-non-uniform | 35 | 37 | -5.7% | skip | 35 | 0.0% |
-| metrics-wide | 133 | 63 | 52.6% | convert | 63 | 52.6% |
-| **Average** | **103** | **67** | **27.5%** | **3/5 convert** | **67** | **28.2%** |
+| users-small | 54 | 40 | 25.9% | convert | 40 | 25.9% |
+| events-medium | 219 | 162 | 26.0% | convert | 162 | 26.0% |
+| orders-nested | 106 | 116 | -9.4% | skip | 106 | 0.0% |
+| mixed-non-uniform | 35 | 47 | -34.3% | skip | 35 | 0.0% |
+| metrics-wide | 142 | 103 | 27.5% | convert | 103 | 27.5% |
+| **Average** | **111** | **94** | **7.1%** | **3/5 convert** | **89** | **15.9%** |
 
 *Forced conversion succeeded for 5/5 payloads.*
 

{datoon-1.7.4 → datoon-1.9.0}/src/datoon.egg-info/SOURCES.txt

@@ -91,6 +91,7 @@ src/datoon/_version.py
 src/datoon/analyzer.py
 src/datoon/cli.py
 src/datoon/converter.py
+src/datoon/errors.py
 src/datoon/mcp_server.py
 src/datoon/models.py
 src/datoon/py.typed
@@ -118,8 +119,10 @@ tests/integration/test_toon_cli_integration.py
 tests/unit/test_analyzer.py
 tests/unit/test_cli.py
 tests/unit/test_converter_policy.py
+tests/unit/test_errors.py
 tests/unit/test_installer.py
 tests/unit/test_mcp_server.py
 tests/unit/test_models.py
 tests/unit/test_readers.py
-tests/unit/test_tabular_rows.py
+tests/unit/test_tabular_rows.py
+tests/unit/test_tokens.py

{datoon-1.7.4 → datoon-1.9.0}/src/datoon.egg-info/scm_file_list.json

@@ -39,6 +39,7 @@
     "src/datoon/models.py",
     "src/datoon/converter.py",
     "src/datoon/cli.py",
+    "src/datoon/errors.py",
     "src/datoon/readers/xml.py",
     "src/datoon/readers/_coerce.py",
     "src/datoon/readers/__init__.py",
@@ -63,7 +64,9 @@
     "tests/unit/test_mcp_server.py",
     "tests/unit/test_tabular_rows.py",
     "tests/unit/test_analyzer.py",
+    "tests/unit/test_tokens.py",
     "tests/unit/test_installer.py",
+    "tests/unit/test_errors.py",
     "tests/unit/test_cli.py",
     ".claude-plugin/plugin.json",
     ".claude-plugin/marketplace.json",

{datoon-1.7.4 → datoon-1.9.0}/src/datoon.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "1.7.4",
+  "tag": "1.9.0",
   "distance": 0,
-  "node": "g9c10177027b097628e6bb075bb54501482175b78",
+  "node": "g412011b90239e0e07c401c668d255671348628c2",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-07-07"

{datoon-1.7.4 → datoon-1.9.0}/tests/unit/test_cli.py

@@ -53,7 +53,9 @@ def test_main_converts_json_from_stdin(
         lambda _, **kw: "rows[3]{id,v}:\n  1,a\n  2,b\n  3,c\n",
     )
     token_seq = iter([100, 40])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([])
     assert rc == 0
@@ -88,7 +90,9 @@ def test_main_converts_csv_via_flag(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(csv_file), "--format", "csv"])
     assert rc == 0
@@ -107,7 +111,9 @@ def test_main_auto_detects_csv_from_extension(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(csv_file)])
     assert rc == 0
@@ -134,7 +140,9 @@ def test_main_converts_jsonl_from_stdin(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main(["--format", "jsonl"])
     assert rc == 0
@@ -153,7 +161,9 @@ def test_main_auto_detects_jsonl_from_extension(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(f)])
     assert rc == 0
@@ -182,7 +192,9 @@ def test_main_converts_xml_via_flag(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(f), "--format", "xml"])
     assert rc == 0
@@ -223,7 +235,9 @@ def test_main_csv_report_stdout(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 30])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(csv_file), "--report-stdout"])
     assert rc == 0
@@ -244,7 +258,9 @@ def test_main_csv_skips_when_savings_insufficient(
         lambda _, **kw: "rows[3]{id,name,role}:\n  1,Ada,admin\n",
     )
     token_seq = iter([100, 98])
-    monkeypatch.setattr("datoon.converter.estimate_tokens", lambda _: next(token_seq))
+    monkeypatch.setattr(
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_seq)
+    )
 
     rc = main([str(csv_file), "--report-stdout"])
     assert rc == 0
@@ -280,3 +296,58 @@ def test_mcp_subcommand_missing_extra_errors(
     rc = main(["mcp"])
     assert rc == 1
     assert "mcp" in capsys.readouterr().err
+
+
+# ---------------------------------------------------------------------------
+# --sheet / --table passthrough for spreadsheet formats
+# ---------------------------------------------------------------------------
+
+
+def test_main_forwards_sheet_and_table_for_binary(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    captured: dict[str, int] = {}
+
+    def fake_read_tabular(
+        fmt: str,
+        *,
+        text: str | None = None,
+        path: Path | None = None,
+        sheet: int = 0,
+        table: int = 0,
+    ) -> list[dict[str, object]]:
+        captured["sheet"] = sheet
+        captured["table"] = table
+        return [{"id": 1, "v": "a"}]
+
+    monkeypatch.setattr("datoon.cli.read_tabular", fake_read_tabular)
+    monkeypatch.setattr("datoon.converter._run_toon_cli", lambda _, **kw: "x\n")
+    xlsx = tmp_path / "data.xlsx"
+    xlsx.write_bytes(b"")
+
+    rc = main(
+        ["--format", "excel", "--sheet", "2", "--table", "1", "--force", str(xlsx)]
+    )
+    assert rc == 0
+    assert captured == {"sheet": 2, "table": 1}
+
+
+def test_main_forwards_encoding_to_estimate(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    payload = '{"rows":[{"id":1,"v":"a"},{"id":2,"v":"b"},{"id":3,"v":"c"}]}'
+    monkeypatch.setattr("sys.stdin", __import__("io").StringIO(payload))
+    monkeypatch.setattr("datoon.converter._run_toon_cli", lambda _, **kw: "x\n")
+
+    seen: list[str] = []
+
+    def fake_estimate(text: str, encoding: str = "o200k_base") -> int:
+        seen.append(encoding)
+        return 50 if len(seen) == 1 else 10
+
+    monkeypatch.setattr("datoon.converter.estimate_tokens", fake_estimate)
+
+    rc = main(["--force", "--encoding", "cl100k_base"])
+    assert rc == 0
+    assert seen and all(enc == "cl100k_base" for enc in seen)

{datoon-1.7.4 → datoon-1.9.0}/tests/unit/test_converter_policy.py

@@ -107,7 +107,7 @@ def test_policy_converts_when_savings_above_threshold(
 
     token_sequence = iter([100, 60])  # input, output
     monkeypatch.setattr(
-        "datoon.converter.estimate_tokens", lambda _: next(token_sequence)
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_sequence)
     )
 
     outcome = convert_json_for_llm(payload, ConversionConfig(min_savings_ratio=0.15))
@@ -127,7 +127,7 @@ def test_policy_skips_when_savings_below_threshold(
 
     token_sequence = iter([100, 95])  # input, output
     monkeypatch.setattr(
-        "datoon.converter.estimate_tokens", lambda _: next(token_sequence)
+        "datoon.converter.estimate_tokens", lambda *_a, **_k: next(token_sequence)
     )
 
     outcome = convert_json_for_llm(payload, ConversionConfig(min_savings_ratio=0.10))