datoon 1.7.0__py3-none-any.whl

datoon/__init__.py

@@ -0,0 +1,31 @@
+"""Core package for datoon JSON-to-TOON conversion tooling."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from datoon.converter import DatoonError, convert_json_for_llm
+from datoon.models import (
+    ConversionConfig,
+    ConversionOutcome,
+    ConversionReport,
+    PayloadAnalysis,
+)
+from datoon.readers import detect_format, read_tabular
+
+__all__ = [
+    "__version__",
+    "convert_json_for_llm",
+    "read_tabular",
+    "detect_format",
+    "ConversionConfig",
+    "ConversionOutcome",
+    "ConversionReport",
+    "PayloadAnalysis",
+    "DatoonError",
+]
+
+try:
+    __version__ = version("datoon")
+except PackageNotFoundError:
+    __version__ = "0.0.0"

datoon/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '1.7.0'
+__version_tuple__ = version_tuple = (1, 7, 0)
+
+__commit_id__ = commit_id = None

datoon/analyzer.py

@@ -0,0 +1,89 @@
+"""Payload analysis helpers used to decide if TOON conversion is beneficial."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+from datoon.models import ConversionConfig, PayloadAnalysis
+
+
+def _max_depth(value: Any, depth: int = 1) -> int:
+    """Compute the maximum nested depth for JSON-like payloads."""
+    if isinstance(value, dict):
+        if not value:
+            return depth
+        return max(_max_depth(child, depth + 1) for child in value.values())
+
+    if isinstance(value, list):
+        if not value:
+            return depth
+        return max(_max_depth(child, depth + 1) for child in value)
+
+    return depth
+
+
+def _iter_arrays(value: Any) -> Iterator[list[Any]]:
+    """Yield every list node found within a nested JSON-like structure."""
+    if isinstance(value, list):
+        yield value
+        for child in value:
+            yield from _iter_arrays(child)
+        return
+
+    if isinstance(value, dict):
+        for child in value.values():
+            yield from _iter_arrays(child)
+
+
+def _is_uniform_object_array(items: list[Any], min_rows: int) -> bool:
+    """Check whether list entries are object rows with identical key sets."""
+    if len(items) < min_rows:
+        return False
+    if not all(isinstance(item, dict) for item in items):
+        return False
+
+    first_keys = list(items[0].keys())
+    if not first_keys:
+        return False
+
+    return all(list(item.keys()) == first_keys for item in items)
+
+
+def analyze_payload(data: Any, config: ConversionConfig) -> PayloadAnalysis:
+    """Evaluate whether payload structure is a practical TOON candidate."""
+    max_depth = _max_depth(data)
+    uniform_array_count = sum(
+        1
+        for items in _iter_arrays(data)
+        if _is_uniform_object_array(items, config.min_uniform_rows)
+    )
+
+    if uniform_array_count == 0:
+        return PayloadAnalysis(
+            is_candidate=False,
+            reason=(
+                "No uniform object arrays found with at least "
+                f"{config.min_uniform_rows} rows."
+            ),
+            max_depth=max_depth,
+            uniform_array_count=0,
+        )
+
+    if max_depth > config.max_depth:
+        return PayloadAnalysis(
+            is_candidate=False,
+            reason=f"Payload depth {max_depth} exceeds max_depth {config.max_depth}.",
+            max_depth=max_depth,
+            uniform_array_count=uniform_array_count,
+        )
+
+    return PayloadAnalysis(
+        is_candidate=True,
+        reason=(
+            f"Detected {uniform_array_count} uniform object array(s) "
+            f"within depth {max_depth}."
+        ),
+        max_depth=max_depth,
+        uniform_array_count=uniform_array_count,
+    )

datoon/cli.py

@@ -0,0 +1,197 @@
+"""Command-line interface for datoon JSON-to-TOON conversion."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Sequence
+
+from datoon import __version__
+from datoon.converter import DatoonError, convert_json_for_llm
+from datoon.models import ConversionConfig
+from datoon.readers import ALL_FORMATS, BINARY_FORMATS, detect_format, read_tabular
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Create CLI parser with conversion policy and reporting options."""
+    parser = argparse.ArgumentParser(
+        prog="datoon",
+        description=(
+            "Convert structured data (JSON, CSV, YAML, XML, JSONL, Parquet, "
+            "Avro, ORC, Excel, Numbers) to TOON only when conversion is beneficial."
+        ),
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+    parser.add_argument(
+        "input",
+        nargs="?",
+        help="Path to input file. If omitted, reads from stdin (JSON/CSV/YAML/XML/JSONL).",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Path to write resulting payload. If omitted, writes to stdout.",
+    )
+    parser.add_argument(
+        "--format",
+        choices=sorted(ALL_FORMATS | {"json"}),
+        help=(
+            "Input format. Auto-detected from file extension when omitted; "
+            "defaults to 'json' for stdin."
+        ),
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Force TOON conversion and bypass gating thresholds.",
+    )
+    parser.add_argument(
+        "--min-savings",
+        type=float,
+        default=0.15,
+        help="Minimum relative token savings required for conversion (default: 0.15).",
+    )
+    parser.add_argument(
+        "--max-depth",
+        type=int,
+        default=6,
+        help="Maximum nested depth allowed for automatic conversion (default: 6).",
+    )
+    parser.add_argument(
+        "--min-uniform-rows",
+        type=int,
+        default=3,
+        help="Minimum row count for uniform object arrays (default: 3).",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=30,
+        help="Seconds to wait for the TOON CLI before aborting (default: 30).",
+    )
+    parser.add_argument(
+        "--report",
+        help="Optional path to write conversion report as JSON.",
+    )
+    parser.add_argument(
+        "--report-stdout",
+        action="store_true",
+        help="Print conversion report JSON to stderr.",
+    )
+    return parser
+
+
+def _read_input(path: str | None) -> str:
+    """Read input payload from file path or stdin."""
+    if path is None:
+        return sys.stdin.read()
+    return Path(path).read_text(encoding="utf-8")
+
+
+def _write_text(path: str | None, text: str) -> None:
+    """Write output payload to file path or stdout."""
+    if path is None:
+        sys.stdout.write(text)
+        return
+    Path(path).write_text(text, encoding="utf-8")
+
+
+def _emit_report(
+    *, report_path: str | None, report_stdout: bool, payload: dict[str, object]
+) -> None:
+    """Emit conversion report to selected destinations."""
+    report_json = json.dumps(payload, ensure_ascii=False, indent=2)
+    if report_path:
+        Path(report_path).write_text(f"{report_json}\n", encoding="utf-8")
+    if report_stdout:
+        sys.stderr.write(f"{report_json}\n")
+
+
+def _resolve_format(input_path: str | None, format_override: str | None) -> str:
+    """Determine input format from explicit flag, file extension, or default."""
+    if format_override:
+        return format_override
+    if input_path is not None:
+        detected = detect_format(input_path)
+        if detected is not None:
+            return detected
+    return "json"
+
+
+def _run_mcp_server() -> int:
+    """Launch the bundled MCP server via the `datoon mcp` subcommand.
+
+    Kept as a thin wrapper so the official MCP Registry can launch datoon
+    through its package command (`uvx --from datoon[mcp] datoon mcp`) without a
+    separate distribution. The standalone `datoon-mcp` script remains available.
+    """
+    try:
+        from datoon.mcp_server import main as mcp_main
+    except ImportError:
+        sys.stderr.write(
+            "datoon error: MCP support requires the 'mcp' extra: "
+            "pip install 'datoon[mcp]'\n"
+        )
+        return 1
+    mcp_main()
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """CLI entrypoint that executes conversion and returns process exit code."""
+    raw_argv = list(sys.argv[1:] if argv is None else argv)
+    if raw_argv and raw_argv[0] == "mcp":
+        return _run_mcp_server()
+
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        config = ConversionConfig(
+            min_savings_ratio=args.min_savings,
+            max_depth=args.max_depth,
+            min_uniform_rows=args.min_uniform_rows,
+            force=args.force,
+            toon_cli_timeout=args.timeout,
+        )
+
+        fmt = _resolve_format(args.input, args.format)
+
+        if fmt == "json":
+            raw_input = _read_input(args.input)
+            outcome = convert_json_for_llm(raw_input, config)
+        elif fmt in BINARY_FORMATS:
+            if args.input is None:
+                sys.stderr.write(
+                    f"datoon error: --format {fmt} requires a file path, not stdin.\n"
+                )
+                return 1
+            rows = read_tabular(fmt, path=Path(args.input))
+            json_text = json.dumps(rows, ensure_ascii=False, separators=(",", ":"))
+            outcome = convert_json_for_llm(json_text, config)
+        else:
+            raw_input = _read_input(args.input)
+            rows = read_tabular(fmt, text=raw_input)
+            json_text = json.dumps(rows, ensure_ascii=False, separators=(",", ":"))
+            outcome = convert_json_for_llm(json_text, config)
+
+        _write_text(args.output, outcome.payload_text)
+        _emit_report(
+            report_path=args.report,
+            report_stdout=args.report_stdout,
+            payload=outcome.report.as_dict(),
+        )
+        return 0
+    except (OSError, ValueError, DatoonError) as exc:
+        sys.stderr.write(f"datoon error: {exc}\n")
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

datoon/converter.py

@@ -0,0 +1,165 @@
+"""Conversion service that transforms JSON payloads into TOON when beneficial."""
+
+from __future__ import annotations
+
+import json
+import math
+import subprocess
+from functools import lru_cache
+from typing import Any
+
+from datoon.analyzer import analyze_payload
+from datoon.models import (
+    ConversionConfig,
+    ConversionOutcome,
+    ConversionReport,
+    PayloadAnalysis,
+)
+
+TOON_CLI_PACKAGE = "@toon-format/cli@2"
+
+
+class DatoonError(RuntimeError):
+    """Raised when datoon cannot process the payload safely."""
+
+
+def _normalize_json(raw_text: str) -> tuple[Any, str]:
+    """Parse and re-serialize JSON into a deterministic compact representation."""
+    try:
+        parsed = json.loads(raw_text)
+    except json.JSONDecodeError as exc:
+        raise DatoonError(f"Invalid JSON input: {exc.msg} (pos={exc.pos}).") from exc
+
+    normalized = json.dumps(parsed, ensure_ascii=False, separators=(",", ":"))
+    return parsed, normalized
+
+
+@lru_cache(maxsize=1)
+def _load_token_encoder() -> Any | None:
+    """Load a token encoder once, returning None when optional dependency is absent."""
+    try:
+        import tiktoken  # type: ignore
+    except ImportError:
+        return None
+
+    return tiktoken.get_encoding("cl100k_base")
+
+
+def estimate_tokens(text: str) -> int:
+    """Estimate token count with tiktoken when installed, else use a char heuristic."""
+    encoder = _load_token_encoder()
+    if encoder is not None:
+        return len(encoder.encode(text))
+
+    return max(1, math.ceil(len(text) / 3))
+
+
+def _run_toon_cli(normalized_json: str, timeout: int = 30) -> str:
+    """Convert normalized JSON to TOON via the official TOON CLI."""
+    command = ["npx", "--yes", TOON_CLI_PACKAGE]
+
+    try:
+        result = subprocess.run(
+            command,
+            input=normalized_json,
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=timeout,
+        )
+    except FileNotFoundError as exc:
+        raise DatoonError(
+            "`npx` is not available in PATH. Install Node.js before TOON conversion."
+        ) from exc
+    except subprocess.TimeoutExpired as exc:
+        raise DatoonError(f"TOON CLI timed out after {timeout}s.") from exc
+
+    if result.returncode != 0:
+        stderr = result.stderr.strip() or "unknown TOON CLI error"
+        raise DatoonError(f"TOON CLI failed: {stderr}")
+
+    toon_text = result.stdout.strip()
+    if not toon_text:
+        raise DatoonError("TOON CLI returned empty output.")
+
+    return f"{toon_text}\n"
+
+
+def _build_skip_outcome(
+    normalized_json: str,
+    *,
+    reason: str,
+    config: ConversionConfig,
+    input_tokens: int,
+    analysis: PayloadAnalysis,
+) -> ConversionOutcome:
+    """Build a standard skip outcome to keep reporting consistent."""
+    report = ConversionReport(
+        decision="skip",
+        reason=reason,
+        was_forced=config.force,
+        input_token_estimate=input_tokens,
+        output_token_estimate=input_tokens,
+        savings_ratio=0.0,
+        analysis=analysis,
+    )
+    return ConversionOutcome(payload_text=normalized_json, report=report)
+
+
+def convert_json_for_llm(raw_text: str, config: ConversionConfig) -> ConversionOutcome:
+    """Convert JSON to TOON when structure and savings meet configured policy."""
+    parsed, normalized_json = _normalize_json(raw_text)
+    analysis = analyze_payload(parsed, config)
+    input_tokens = estimate_tokens(normalized_json)
+
+    if not analysis.is_candidate and not config.force:
+        return _build_skip_outcome(
+            normalized_json,
+            reason=analysis.reason,
+            config=config,
+            input_tokens=input_tokens,
+            analysis=analysis,
+        )
+
+    try:
+        toon_text = _run_toon_cli(normalized_json, timeout=config.toon_cli_timeout)
+    except DatoonError:
+        if config.force:
+            raise
+
+        return _build_skip_outcome(
+            normalized_json,
+            reason="TOON conversion dependency unavailable; falling back to JSON.",
+            config=config,
+            input_tokens=input_tokens,
+            analysis=analysis,
+        )
+
+    output_tokens = estimate_tokens(toon_text)
+    savings_ratio = (input_tokens - output_tokens) / input_tokens
+
+    if savings_ratio < config.min_savings_ratio and not config.force:
+        return _build_skip_outcome(
+            normalized_json,
+            reason=(
+                f"Estimated savings {savings_ratio:.2%} below threshold "
+                f"{config.min_savings_ratio:.2%}."
+            ),
+            config=config,
+            input_tokens=input_tokens,
+            analysis=analysis,
+        )
+
+    report = ConversionReport(
+        decision="convert",
+        reason=(
+            f"Estimated savings {savings_ratio:.2%} "
+            f"(threshold {config.min_savings_ratio:.2%})."
+        ),
+        was_forced=config.force,
+        input_token_estimate=input_tokens,
+        output_token_estimate=output_tokens,
+        savings_ratio=savings_ratio,
+        analysis=analysis,
+    )
+    return ConversionOutcome(payload_text=toon_text, report=report)

datoon/mcp_server.py

@@ -0,0 +1,140 @@
+"""MCP server exposing datoon conversion and analysis as tools."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+try:
+    from mcp.server.fastmcp import FastMCP
+except ImportError as exc:
+    raise ImportError(
+        "MCP support requires the 'mcp' extra: pip install datoon[mcp]"
+    ) from exc
+
+from datoon.analyzer import analyze_payload
+from datoon.converter import DatoonError, convert_json_for_llm
+from datoon.models import ConversionConfig
+from datoon.readers import TEXT_FORMATS, read_tabular
+
+mcp = FastMCP(
+    "datoon",
+    instructions=(
+        "Smart JSON-to-TOON gateway. "
+        "Use convert_json to compress structured payloads before sending to a model. "
+        "Use analyze_json to check if a payload is a good TOON candidate without converting."
+    ),
+)
+
+
+@mcp.tool()
+def convert_json(
+    json_text: str,
+    force: bool = False,
+    min_savings: float = 0.15,
+    max_depth: int = 6,
+    min_uniform_rows: int = 3,
+    timeout: int = 30,
+) -> dict[str, Any]:
+    """Convert a JSON payload to TOON when structure and savings meet policy.
+
+    Returns the final payload text (TOON or original JSON) and a report
+    explaining the conversion decision.
+    """
+    try:
+        config = ConversionConfig(
+            min_savings_ratio=min_savings,
+            max_depth=max_depth,
+            min_uniform_rows=min_uniform_rows,
+            force=force,
+            toon_cli_timeout=timeout,
+        )
+        outcome = convert_json_for_llm(json_text, config)
+        return {
+            "payload": outcome.payload_text,
+            "report": outcome.report.as_dict(),
+        }
+    except DatoonError as exc:
+        return {"error": str(exc)}
+    except ValueError as exc:
+        return {"error": f"Invalid config: {exc}"}
+
+
+@mcp.tool()
+def analyze_json(
+    json_text: str,
+    max_depth: int = 6,
+    min_uniform_rows: int = 3,
+) -> dict[str, Any]:
+    """Analyze a JSON payload and report whether it is a TOON candidate.
+
+    Does not invoke the TOON CLI — safe to call without Node.js installed.
+    """
+    try:
+        parsed = json.loads(json_text)
+    except json.JSONDecodeError as exc:
+        return {"error": f"Invalid JSON: {exc.msg} (pos={exc.pos})"}
+
+    try:
+        config = ConversionConfig(
+            max_depth=max_depth, min_uniform_rows=min_uniform_rows
+        )
+    except ValueError as exc:
+        return {"error": f"Invalid config: {exc}"}
+
+    analysis = analyze_payload(parsed, config)
+    return {
+        "is_candidate": analysis.is_candidate,
+        "reason": analysis.reason,
+        "max_depth": analysis.max_depth,
+        "uniform_array_count": analysis.uniform_array_count,
+    }
+
+
+@mcp.tool()
+def convert_text(
+    text: str,
+    fmt: str = "csv",
+    force: bool = False,
+    min_savings: float = 0.15,
+    max_depth: int = 6,
+    min_uniform_rows: int = 3,
+    timeout: int = 30,
+) -> dict[str, Any]:
+    """Convert structured text (csv, yaml, xml, jsonl) to TOON with policy gating.
+
+    Supported formats: csv, yaml, xml, jsonl.
+    For JSON input use convert_json. Binary formats (parquet, excel, etc.) require the CLI.
+    """
+    if fmt not in TEXT_FORMATS:
+        return {
+            "error": (
+                f"Unsupported format '{fmt}' for text input. "
+                f"Supported: {sorted(TEXT_FORMATS)}."
+            )
+        }
+    try:
+        rows = read_tabular(fmt, text=text)
+        json_text = json.dumps(rows, ensure_ascii=False, separators=(",", ":"))
+        config = ConversionConfig(
+            min_savings_ratio=min_savings,
+            max_depth=max_depth,
+            min_uniform_rows=min_uniform_rows,
+            force=force,
+            toon_cli_timeout=timeout,
+        )
+        outcome = convert_json_for_llm(json_text, config)
+        return {
+            "payload": outcome.payload_text,
+            "report": outcome.report.as_dict(),
+        }
+    except (DatoonError, ValueError, ImportError) as exc:
+        return {"error": str(exc)}
+
+
+def main() -> None:
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()