datoon 1.7.0__py3-none-any.whl

datoon/models.py

@@ -0,0 +1,65 @@
+"""Domain models for conversion configuration and results."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Literal
+
+Decision = Literal["convert", "skip"]
+
+
+@dataclass(frozen=True, slots=True)
+class ConversionConfig:
+    """Runtime configuration that governs auto-conversion behavior."""
+
+    min_savings_ratio: float = 0.15
+    max_depth: int = 6
+    min_uniform_rows: int = 3
+    force: bool = False
+    toon_cli_timeout: int = 30
+
+    def __post_init__(self) -> None:
+        """Validate boundary values to avoid ambiguous runtime decisions."""
+        if not 0 <= self.min_savings_ratio <= 1:
+            raise ValueError("min_savings_ratio must be between 0 and 1.")
+        if self.max_depth < 1:
+            raise ValueError("max_depth must be at least 1.")
+        if self.min_uniform_rows < 2:
+            raise ValueError("min_uniform_rows must be at least 2.")
+        if self.toon_cli_timeout < 1:
+            raise ValueError("toon_cli_timeout must be at least 1.")
+
+
+@dataclass(frozen=True, slots=True)
+class PayloadAnalysis:
+    """Heuristic analysis of whether a JSON payload is TOON-friendly."""
+
+    is_candidate: bool
+    reason: str
+    max_depth: int
+    uniform_array_count: int
+
+
+@dataclass(frozen=True, slots=True)
+class ConversionReport:
+    """Metadata report for a conversion decision and savings estimate."""
+
+    decision: Decision
+    reason: str
+    was_forced: bool
+    input_token_estimate: int
+    output_token_estimate: int
+    savings_ratio: float
+    analysis: PayloadAnalysis
+
+    def as_dict(self) -> dict[str, object]:
+        """Return a JSON-serializable representation of the report."""
+        return asdict(self)
+
+
+@dataclass(frozen=True, slots=True)
+class ConversionOutcome:
+    """Container holding final payload text and its conversion report."""
+
+    payload_text: str
+    report: ConversionReport

datoon/readers/__init__.py

@@ -0,0 +1,98 @@
+"""Format readers that normalize structured data to list-of-dicts for TOON conversion."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+BINARY_FORMATS: frozenset[str] = frozenset(
+    {"excel", "parquet", "orc", "avro", "numbers"}
+)
+TEXT_FORMATS: frozenset[str] = frozenset({"csv", "jsonl", "yaml", "xml"})
+ALL_FORMATS: frozenset[str] = BINARY_FORMATS | TEXT_FORMATS
+
+_EXTENSION_MAP: dict[str, str] = {
+    ".csv": "csv",
+    ".jsonl": "jsonl",
+    ".ndjson": "jsonl",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".xml": "xml",
+    ".xlsx": "excel",
+    ".xls": "excel",
+    ".parquet": "parquet",
+    ".avro": "avro",
+    ".orc": "orc",
+    ".numbers": "numbers",
+}
+
+
+def detect_format(path: str | Path) -> str | None:
+    """Return the format name for a path by extension, or None if unknown."""
+    return _EXTENSION_MAP.get(Path(path).suffix.lower())
+
+
+def read_tabular(
+    fmt: str,
+    *,
+    text: str | None = None,
+    path: Path | None = None,
+) -> list[dict[str, Any]]:
+    """Read structured data and return rows as a list of dicts.
+
+    Text formats (csv, jsonl, yaml, xml) require ``text``.
+    Binary formats (excel, parquet, orc, avro, numbers) require ``path``.
+    """
+    if fmt in BINARY_FORMATS:
+        if path is None:
+            raise ValueError(f"Format '{fmt}' requires a file path, not text input.")
+        return _read_binary(fmt, path)
+    if fmt in TEXT_FORMATS:
+        if text is None:
+            raise ValueError(f"Format '{fmt}' requires text input.")
+        return _read_text(fmt, text)
+    raise ValueError(f"Unknown format '{fmt}'. Supported: {sorted(ALL_FORMATS)}.")
+
+
+def _read_text(fmt: str, text: str) -> list[dict[str, Any]]:
+    if fmt == "csv":
+        from datoon.readers.csv import read_csv
+
+        return read_csv(text)
+    if fmt == "jsonl":
+        from datoon.readers.jsonl import read_jsonl
+
+        return read_jsonl(text)
+    if fmt == "yaml":
+        from datoon.readers.yaml import read_yaml
+
+        return read_yaml(text)
+    if fmt == "xml":
+        from datoon.readers.xml import read_xml
+
+        return read_xml(text)
+    raise ValueError(f"Unknown text format: {fmt}")
+
+
+def _read_binary(fmt: str, path: Path) -> list[dict[str, Any]]:
+    if fmt == "excel":
+        from datoon.readers.excel import read_excel
+
+        return read_excel(path)
+    if fmt == "parquet":
+        from datoon.readers.columnar import read_parquet
+
+        return read_parquet(path)
+    if fmt == "orc":
+        from datoon.readers.columnar import read_orc
+
+        return read_orc(path)
+    if fmt == "avro":
+        from datoon.readers.columnar import read_avro
+
+        return read_avro(path)
+    if fmt == "numbers":
+        from datoon.readers.numbers import read_numbers
+
+        return read_numbers(path)
+    raise ValueError(f"Unknown binary format: {fmt}")

datoon/readers/columnar.py

@@ -0,0 +1,42 @@
+"""Columnar format readers: Parquet, ORC, Avro (requires datoon[columnar])."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+_INSTALL_HINT = "pip install 'datoon[columnar]'"
+
+
+def read_parquet(path: Path) -> list[dict[str, Any]]:
+    """Read a Parquet file into a list of row dicts."""
+    try:
+        import pyarrow.parquet as pq  # type: ignore
+    except ImportError as exc:
+        raise ImportError(
+            f"Parquet support requires the 'columnar' extra: {_INSTALL_HINT}"
+        ) from exc
+    return pq.read_table(path).to_pylist()
+
+
+def read_orc(path: Path) -> list[dict[str, Any]]:
+    """Read an ORC file into a list of row dicts."""
+    try:
+        import pyarrow.orc as orc  # type: ignore
+    except ImportError as exc:
+        raise ImportError(
+            f"ORC support requires the 'columnar' extra: {_INSTALL_HINT}"
+        ) from exc
+    return orc.read_table(path).to_pylist()
+
+
+def read_avro(path: Path) -> list[dict[str, Any]]:
+    """Read an Avro file into a list of row dicts."""
+    try:
+        import fastavro  # type: ignore
+    except ImportError as exc:
+        raise ImportError(
+            f"Avro support requires the 'columnar' extra: {_INSTALL_HINT}"
+        ) from exc
+    with open(path, "rb") as f:
+        return list(fastavro.reader(f))

datoon/readers/csv.py

@@ -0,0 +1,35 @@
+"""CSV reader with basic type coercion."""
+
+from __future__ import annotations
+
+import csv
+import io
+from typing import Any
+
+
+def _coerce(value: str) -> int | float | bool | str | None:
+    stripped = value.strip()
+    if stripped == "":
+        return None
+    low = stripped.lower()
+    if low == "true":
+        return True
+    if low == "false":
+        return False
+    try:
+        return int(stripped)
+    except ValueError:
+        pass
+    try:
+        return float(stripped)
+    except ValueError:
+        pass
+    return stripped
+
+
+def read_csv(text: str, *, coerce_types: bool = True) -> list[dict[str, Any]]:
+    """Parse CSV text into a list of row dicts."""
+    reader = csv.DictReader(io.StringIO(text))
+    if coerce_types:
+        return [{k: _coerce(str(v)) for k, v in row.items()} for row in reader]
+    return [dict(row) for row in reader]

datoon/readers/excel.py

@@ -0,0 +1,31 @@
+"""Excel reader (requires openpyxl extra: pip install 'datoon[excel]')."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+def read_excel(path: Path, *, sheet: int = 0) -> list[dict[str, Any]]:
+    """Read the first sheet of an Excel file into a list of row dicts."""
+    try:
+        import openpyxl  # type: ignore
+    except ImportError as exc:
+        raise ImportError(
+            "Excel support requires the 'excel' extra: pip install 'datoon[excel]'"
+        ) from exc
+
+    wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
+    ws = wb.worksheets[sheet]
+    rows = list(ws.iter_rows(values_only=True))
+    wb.close()
+
+    if not rows:
+        return []
+
+    headers = [str(h) if h is not None else f"col{i}" for i, h in enumerate(rows[0])]
+    return [
+        {h: v for h, v in zip(headers, row)}
+        for row in rows[1:]
+        if any(v is not None for v in row)
+    ]

datoon/readers/jsonl.py

@@ -0,0 +1,27 @@
+"""JSONL (newline-delimited JSON) reader."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from datoon.converter import DatoonError
+
+
+def read_jsonl(text: str) -> list[dict[str, Any]]:
+    """Parse JSONL text into a list of row dicts."""
+    rows: list[dict[str, Any]] = []
+    for i, line in enumerate(text.strip().splitlines(), start=1):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise DatoonError(
+                f"Invalid JSONL at line {i}: {exc.msg} (pos={exc.pos})."
+            ) from exc
+        if not isinstance(obj, dict):
+            raise DatoonError(f"JSONL line {i} is not a JSON object.")
+        rows.append(obj)
+    return rows

datoon/readers/numbers.py

@@ -0,0 +1,30 @@
+"""Apple Numbers reader (requires numbers-parser extra: pip install 'datoon[numbers]')."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+def read_numbers(path: Path, *, sheet: int = 0, table: int = 0) -> list[dict[str, Any]]:
+    """Read an Apple Numbers file into a list of row dicts."""
+    try:
+        from numbers_parser import Document  # type: ignore
+    except ImportError as exc:
+        raise ImportError(
+            "Numbers support requires the 'numbers' extra: pip install 'datoon[numbers]'"
+        ) from exc
+
+    doc = Document(str(path))
+    tbl = doc.sheets[sheet].tables[table]
+    rows = list(tbl.rows(values_only=True))
+
+    if not rows:
+        return []
+
+    headers = [str(h) if h is not None else f"col{i}" for i, h in enumerate(rows[0])]
+    return [
+        {h: v for h, v in zip(headers, row)}
+        for row in rows[1:]
+        if any(v is not None for v in row)
+    ]

datoon/readers/xml.py

@@ -0,0 +1,58 @@
+"""XML reader using stdlib xml.etree.ElementTree."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from typing import Any
+
+
+def read_xml(text: str) -> list[dict[str, Any]]:
+    """Parse XML into a list of row dicts from the dominant repeated child element."""
+    try:
+        root = ET.fromstring(text)
+    except ET.ParseError as exc:
+        raise ValueError(f"Invalid XML: {exc}") from exc
+
+    children = list(root)
+    if not children:
+        raise ValueError("XML root element has no children.")
+
+    tag_counts: dict[str, int] = {}
+    for child in children:
+        tag_counts[child.tag] = tag_counts.get(child.tag, 0) + 1
+    dominant_tag = max(tag_counts, key=lambda t: tag_counts[t])
+    items = [c for c in children if c.tag == dominant_tag]
+
+    return [_element_to_dict(item) for item in items]
+
+
+def _element_to_dict(element: ET.Element) -> dict[str, Any]:
+    result: dict[str, Any] = {k: _coerce(v) for k, v in element.attrib.items()}
+    for child in element:
+        if list(child):
+            result[child.tag] = _element_to_dict(child)
+        else:
+            result[child.tag] = _coerce(child.text or "")
+    if not list(element) and element.text and element.text.strip():
+        result["_text"] = element.text.strip()
+    return result
+
+
+def _coerce(value: str) -> int | float | bool | str | None:
+    stripped = value.strip()
+    if not stripped:
+        return None
+    low = stripped.lower()
+    if low == "true":
+        return True
+    if low == "false":
+        return False
+    try:
+        return int(stripped)
+    except ValueError:
+        pass
+    try:
+        return float(stripped)
+    except ValueError:
+        pass
+    return stripped

datoon/readers/yaml.py

@@ -0,0 +1,30 @@
+"""YAML reader (requires pyyaml extra: pip install 'datoon[yaml]')."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def read_yaml(text: str) -> list[dict[str, Any]]:
+    """Parse YAML text into a list of row dicts."""
+    try:
+        import yaml  # type: ignore
+    except ImportError as exc:
+        raise ImportError(
+            "YAML support requires the 'yaml' extra: pip install 'datoon[yaml]'"
+        ) from exc
+
+    data = yaml.safe_load(text)
+    return _normalize(data)
+
+
+def _normalize(data: Any) -> list[dict[str, Any]]:
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        lists = [v for v in data.values() if isinstance(v, list) and v]
+        if len(lists) == 1:
+            return lists[0]
+    raise ValueError(
+        "YAML must be a list of objects, or a mapping with exactly one list value."
+    )