flow-toon-format 0.9.0b2__py3-none-any.whl

toon_format/normalize.py

@@ -0,0 +1,237 @@
+# Copyright (c) 2025 TOON Format Organization
+# SPDX-License-Identifier: MIT
+"""Value normalization for TOON encoding.
+
+Converts Python-specific types to JSON-compatible values before encoding:
+- datetime/date → ISO 8601 strings
+- Decimal → float
+- tuple/set/frozenset → sorted lists
+- Infinity/NaN → null
+- Functions/callables → null
+- Negative zero → zero
+"""
+
+import math
+import sys
+from collections.abc import Mapping
+from datetime import date, datetime
+from decimal import Decimal
+from typing import Any
+
+# TypeGuard was added in Python 3.10, use typing_extensions for older versions
+if sys.version_info >= (3, 10):
+    from typing import TypeGuard
+else:
+    from typing_extensions import TypeGuard
+
+from .logging_config import get_logger
+from .types import JsonArray, JsonObject, JsonPrimitive, JsonValue
+
+# Module logger
+logger = get_logger(__name__)
+
+_MAX_SAFE_INTEGER = 2**53 - 1
+
+
+def normalize_value(value: Any) -> JsonValue:
+    """Normalize Python value to JSON-compatible type.
+
+    Converts Python-specific types to JSON-compatible equivalents:
+    - datetime objects → ISO 8601 strings
+    - sets → sorted lists
+    - Large integers (>2^53-1) → strings (for JS compatibility)
+    - Non-finite floats (inf, -inf, NaN) → null
+    - Negative zero → positive zero
+    - Mapping types → dicts with string keys
+    - Unsupported types → null
+
+    Args:
+        value: Python value to normalize.
+
+    Returns:
+        JsonValue: Normalized value (None, bool, int, float, str, list, or dict).
+
+    Examples:
+        >>> normalize_value(datetime(2024, 1, 1))
+        '2024-01-01T00:00:00'
+
+        >>> normalize_value({1, 2, 3})
+        [1, 2, 3]
+
+        >>> normalize_value(float('inf'))
+        None
+
+        >>> normalize_value(2**60)  # Large integer
+        '1152921504606846976'
+
+    Note:
+        - Recursive: normalizes nested structures
+        - Sets are sorted for deterministic output
+        - Heterogeneous sets sorted by repr() if natural sorting fails
+    """
+    if value is None:
+        return None
+
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value
+
+    if isinstance(value, int):
+        # Python integers have arbitrary precision and are encoded directly
+        # Note: JavaScript BigInt types are converted to strings during normalization
+        # (per spec Section 3), but Python ints don't need this conversion
+        return value
+
+    if isinstance(value, float):
+        # Handle non-finite first
+        if not math.isfinite(value) or value != value:  # includes inf, -inf, NaN
+            logger.debug(f"Converting non-finite float to null: {value}")
+            return None
+        if value == 0.0 and math.copysign(1.0, value) == -1.0:
+            logger.debug("Converting negative zero to positive zero")
+            return 0
+        return value
+
+    # Handle Decimal
+    if isinstance(value, Decimal):
+        if not value.is_finite():
+            logger.debug(f"Converting non-finite Decimal to null: {value}")
+            return None
+        return float(value)
+
+    if isinstance(value, datetime):
+        try:
+            result = value.isoformat()
+            logger.debug(f"Converting datetime to ISO string: {value}")
+            return result
+        except Exception as e:
+            raise ValueError(f"Failed to convert datetime to ISO format: {e}") from e
+
+    if isinstance(value, date):
+        try:
+            result = value.isoformat()
+            logger.debug(f"Converting date to ISO string: {value}")
+            return result
+        except Exception as e:
+            raise ValueError(f"Failed to convert date to ISO format: {e}") from e
+
+    if isinstance(value, list):
+        if not value:
+            return []
+        return [normalize_value(item) for item in value]
+
+    if isinstance(value, tuple):
+        logger.debug(f"Converting tuple to list: {len(value)} items")
+        return [normalize_value(item) for item in value]
+
+    if isinstance(value, (set, frozenset)):
+        logger.debug(f"Converting {type(value).__name__} to sorted list: {len(value)} items")
+        try:
+            return [normalize_value(item) for item in sorted(value)]
+        except TypeError:
+            # Fall back to stable conversion for heterogeneous sets/frozensets
+            logger.debug(
+                f"{type(value).__name__} contains heterogeneous types, using repr() for sorting"
+            )
+            return [normalize_value(item) for item in sorted(value, key=lambda x: repr(x))]
+
+    # Handle generic mapping types (Map-like) and dicts
+    if isinstance(value, Mapping):
+        logger.debug(f"Converting {type(value).__name__} to dict: {len(value)} items")
+        try:
+            return {str(k): normalize_value(v) for k, v in value.items()}
+        except Exception as e:
+            raise ValueError(
+                f"Failed to convert mapping to dict: {e}. "
+                "Check that all keys can be converted to strings."
+            ) from e
+
+    # Handle callables -> null
+    if callable(value):
+        logger.debug(f"Converting callable {type(value).__name__} to null")
+        return None
+
+    # Fallback for other types
+    logger.warning(
+        f"Unsupported type {type(value).__name__}, converting to null. Value: {str(value)[:50]}"
+    )
+    return None
+
+
+def is_json_primitive(value: Any) -> TypeGuard[JsonPrimitive]:
+    """Check if value is a JSON primitive type.
+
+    Args:
+        value: Value to check.
+
+    Returns:
+        TypeGuard[JsonPrimitive]: True if value is None, str, int, float, or bool.
+    """
+    return value is None or isinstance(value, (str, int, float, bool))
+
+
+def is_json_array(value: Any) -> TypeGuard[JsonArray]:
+    """Check if value is a JSON array (Python list).
+
+    Args:
+        value: Value to check.
+
+    Returns:
+        TypeGuard[JsonArray]: True if value is a list.
+    """
+    return isinstance(value, list)
+
+
+def is_json_object(value: Any) -> TypeGuard[JsonObject]:
+    """Check if value is a JSON object (Python dict).
+
+    Args:
+        value: Value to check.
+
+    Returns:
+        TypeGuard[JsonObject]: True if value is a dict.
+    """
+    return isinstance(value, dict)
+
+
+def is_array_of_primitives(value: JsonArray) -> bool:
+    """Check if array contains only primitive values.
+
+    Args:
+        value: List to check.
+
+    Returns:
+        bool: True if all items are primitives. Empty arrays return True.
+    """
+    if not value:
+        return True
+    return all(is_json_primitive(item) for item in value)
+
+
+def is_array_of_arrays(value: JsonArray) -> bool:
+    """Check if array contains only arrays.
+
+    Args:
+        value: List to check.
+
+    Returns:
+        bool: True if all items are lists. Empty arrays return True.
+    """
+    if not value:
+        return True
+    return all(is_json_array(item) for item in value)
+
+
+def is_array_of_objects(value: JsonArray) -> bool:
+    """Check if array contains only objects.
+
+    Args:
+        value: List to check.
+
+    Returns:
+        bool: True if all items are dicts. Empty arrays return True.
+    """
+    if not value:
+        return True
+    return all(is_json_object(item) for item in value)

toon_format/primitives.py

@@ -0,0 +1,171 @@
+# Copyright (c) 2025 TOON Format Organization
+# SPDX-License-Identifier: MIT
+"""Primitive value encoding utilities.
+
+Handles encoding of primitive values (strings, numbers, booleans, null) and
+array headers. Implements quoting rules, escape sequences, and header formatting
+for inline and tabular array formats.
+"""
+
+import re
+from typing import List, Literal, Optional, Union
+
+from ._string_utils import escape_string
+from ._validation import is_safe_unquoted, is_valid_unquoted_key
+from .constants import (
+    CLOSE_BRACE,
+    CLOSE_BRACKET,
+    COLON,
+    COMMA,
+    CONTROL_CHARS_REGEX,
+    DOUBLE_QUOTE,
+    FALSE_LITERAL,
+    NULL_LITERAL,
+    NUMERIC_REGEX,
+    OCTAL_REGEX,
+    OPEN_BRACE,
+    OPEN_BRACKET,
+    STRUCTURAL_CHARS_REGEX,
+    TRUE_LITERAL,
+    VALID_KEY_REGEX,
+)
+from .logging_config import get_logger
+from .types import Delimiter, JsonPrimitive
+
+# Precompiled patterns for performance
+_STRUCTURAL_CHARS_PATTERN = re.compile(STRUCTURAL_CHARS_REGEX)
+_CONTROL_CHARS_PATTERN = re.compile(CONTROL_CHARS_REGEX)
+_NUMERIC_PATTERN = re.compile(NUMERIC_REGEX, re.IGNORECASE)
+_OCTAL_PATTERN = re.compile(OCTAL_REGEX)
+_VALID_KEY_PATTERN = re.compile(VALID_KEY_REGEX, re.IGNORECASE)
+
+
+logger = get_logger(__name__)
+
+
+def encode_primitive(value: JsonPrimitive, delimiter: str = COMMA) -> str:
+    """Encode a primitive value.
+
+    Args:
+        value: Primitive value
+        delimiter: Current delimiter being used
+
+    Returns:
+        Encoded string
+    """
+    if value is None:
+        return NULL_LITERAL
+    if isinstance(value, bool):
+        return TRUE_LITERAL if value else FALSE_LITERAL
+    if isinstance(value, (int, float)):
+        # Format numbers in decimal form without scientific notation
+        # Per spec Section 2: numbers must be rendered without exponent notation
+        if isinstance(value, int):
+            return str(value)
+        # For floats, use Python's default conversion first
+        formatted = str(value)
+        # Check if Python used scientific notation
+        if "e" in formatted or "E" in formatted:
+            # Convert to fixed-point decimal notation
+            # Use format with enough precision, then strip trailing zeros
+            from decimal import Decimal
+
+            # Convert through Decimal to get exact decimal representation
+            dec = Decimal(str(value))
+            formatted = format(dec, "f")
+        return formatted
+    if isinstance(value, str):
+        return encode_string_literal(value, delimiter)
+    return str(value)
+
+
+# Note: escape_string and is_safe_unquoted are now imported from _string_utils and _validation
+
+
+def encode_string_literal(value: str, delimiter: str = COMMA) -> str:
+    """Encode a string, quoting only if necessary.
+
+    Args:
+        value: String value
+        delimiter: Current delimiter being used
+
+    Returns:
+        Encoded string
+    """
+    if is_safe_unquoted(value, delimiter):
+        return value
+    return f"{DOUBLE_QUOTE}{escape_string(value)}{DOUBLE_QUOTE}"
+
+
+def encode_key(key: str) -> str:
+    """Encode an object key.
+
+    Args:
+        key: Key string
+
+    Returns:
+        Encoded key
+    """
+    # Keys matching /^[A-Z_][\w.]*$/i don't require quotes
+    if is_valid_unquoted_key(key):
+        return key
+    return f"{DOUBLE_QUOTE}{escape_string(key)}{DOUBLE_QUOTE}"
+
+
+def join_encoded_values(values: List[str], delimiter: Delimiter) -> str:
+    """Join encoded primitive values with a delimiter.
+
+    Args:
+        values: List of encoded values
+        delimiter: Delimiter to use
+
+    Returns:
+        Joined string
+    """
+    return delimiter.join(values)
+
+
+def format_header(
+    key: Optional[str],
+    length: int,
+    fields: Optional[List[str]],
+    delimiter: Delimiter,
+    length_marker: Union[str, Literal[False], None],
+) -> str:
+    """Format array/table header.
+
+    Args:
+        key: Optional key name
+        length: Array length
+        fields: Optional field names for tabular format
+        delimiter: Delimiter character
+        length_marker: Optional length marker prefix
+
+    Returns:
+        Formatted header string
+    """
+    # Build length marker
+    marker_prefix = length_marker if length_marker else ""
+
+    # Build fields if provided
+    fields_str = ""
+    if fields:
+        # Encode each field name as a key (may need quoting per Section 7.3)
+        encoded_fields = [encode_key(field) for field in fields]
+        fields_str = f"{OPEN_BRACE}{delimiter.join(encoded_fields)}{CLOSE_BRACE}"
+
+    # Build length string with delimiter when needed
+    # Rules per TOON spec: delimiter is optional in bracket [N<delim?>]
+    # - Only include delimiter if it's NOT comma (comma is the default)
+    # - This applies to both tabular and primitive arrays
+    if delimiter != COMMA:
+        # Non-comma delimiter: show delimiter in bracket
+        length_str = f"{OPEN_BRACKET}{marker_prefix}{length}{delimiter}{CLOSE_BRACKET}"
+    else:
+        # Comma delimiter (default): just [length]
+        length_str = f"{OPEN_BRACKET}{marker_prefix}{length}{CLOSE_BRACKET}"
+
+    # Combine parts
+    if key:
+        return f"{encode_key(key)}{length_str}{fields_str}{COLON}"
+    return f"{length_str}{fields_str}{COLON}"

toon_format/types.py

@@ -0,0 +1,64 @@
+# Copyright (c) 2025 TOON Format Organization
+# SPDX-License-Identifier: MIT
+"""Type definitions for TOON format.
+
+Defines type aliases and TypedDict classes for JSON values, encoding/decoding
+options, and internal types used throughout the package.
+"""
+
+from typing import Any, Dict, List, Literal, TypedDict, Union
+
+# JSON-compatible types
+JsonPrimitive = Union[str, int, float, bool, None]
+JsonObject = Dict[str, Any]
+JsonArray = List[Any]
+JsonValue = Union[JsonPrimitive, JsonArray, JsonObject]
+
+# Delimiter type
+Delimiter = str
+DelimiterKey = Literal["comma", "tab", "pipe"]
+
+
+class EncodeOptions(TypedDict, total=False):
+    """Options for TOON encoding.
+
+    Attributes:
+        indent: Number of spaces per indentation level (default: 2)
+        delimiter: Delimiter character for arrays (default: comma)
+        lengthMarker: Optional marker to prefix array lengths (default: False)
+    """
+
+    indent: int
+    delimiter: Delimiter
+    lengthMarker: Union[Literal["#"], Literal[False]]
+
+
+class ResolvedEncodeOptions:
+    """Resolved encoding options with defaults applied."""
+
+    def __init__(
+        self,
+        indent: int = 2,
+        delimiter: str = ",",
+        length_marker: Union[Literal["#"], Literal[False]] = False,
+    ) -> None:
+        self.indent = indent
+        self.delimiter = delimiter
+        self.lengthMarker: Union[str, Literal[False]] = length_marker
+
+
+class DecodeOptions:
+    """Options for TOON decoding.
+
+    Attributes:
+        indent: Number of spaces per indentation level (default: 2)
+        strict: Enable strict validation (default: True)
+    """
+
+    def __init__(self, indent: int = 2, strict: bool = True) -> None:
+        self.indent = indent
+        self.strict = strict
+
+
+# Depth type for tracking indentation level
+Depth = int

toon_format/utils.py

@@ -0,0 +1,187 @@
+# Copyright (c) 2025 TOON Format Organization
+# SPDX-License-Identifier: MIT
+"""Token analysis utilities for TOON format.
+
+This module provides utilities for counting tokens and comparing
+token efficiency between JSON and TOON formats. Useful for:
+- Estimating API costs (tokens are the primary cost driver)
+- Optimizing prompt sizes for LLM context windows
+- Benchmarking TOON's token efficiency
+
+Functions:
+    count_tokens: Count tokens in a text string
+    estimate_savings: Compare JSON vs TOON token counts
+    compare_formats: Generate formatted comparison table
+
+Requirements:
+    tiktoken: Install with `uv add tiktoken` or `uv add toon_format[benchmark]`
+
+Example:
+    >>> import toon_format
+    >>> data = {"name": "Alice", "age": 30}
+    >>> result = toon_format.estimate_savings(data)
+    >>> print(f"TOON saves {result['savings_percent']:.1f}% tokens")
+"""
+
+import functools
+import json
+from typing import Any, Dict
+
+# Import encode from parent package (defined in __init__.py before this module is imported)
+# __init__.py defines encode() before importing utils, so this is safe
+from . import encode
+
+__all__ = ["count_tokens", "estimate_savings", "compare_formats"]
+
+
+_TIKTOKEN_MISSING_MSG = (
+    "tiktoken is required for token counting. "
+    "Install with: uv add tiktoken or uv add toon_format[benchmark]"
+)
+
+
+def _require_tiktoken():
+    try:
+        import tiktoken  # type: ignore[import-not-found]
+    except ImportError as exc:  # pragma: no cover - exercised via count_tokens
+        raise RuntimeError(_TIKTOKEN_MISSING_MSG) from exc
+    return tiktoken
+
+
+@functools.lru_cache(maxsize=1)
+def _get_tokenizer():
+    """Get cached tiktoken tokenizer for o200k_base encoding.
+
+    Returns:
+        tiktoken.Encoding: The o200k_base tokenizer (gpt5/gpt5-mini).
+
+    Raises:
+        RuntimeError: If tiktoken is not installed.
+    """
+    tiktoken = _require_tiktoken()
+    return tiktoken.get_encoding("o200k_base")
+
+
+def count_tokens(text: str, encoding: str = "o200k_base") -> int:
+    """Count tokens in a text string using tiktoken.
+
+    Args:
+        text: The string to tokenize.
+        encoding: Tokenizer encoding name (default: 'o200k_base' for gpt5/gpt5-mini).
+                  Other options include 'cl100k_base' (GPT-3.5), 'p50k_base' (older models).
+
+    Returns:
+        int: The number of tokens in the text.
+
+    Example:
+        >>> import toon_format
+        >>> text = "Hello, world!"
+        >>> toon_format.count_tokens(text)
+        4
+
+    Note:
+        Requires tiktoken to be installed: uv add tiktoken or uv add toon_format[benchmark]
+    """
+    if encoding == "o200k_base":
+        enc = _get_tokenizer()
+    else:
+        tiktoken = _require_tiktoken()
+        enc = tiktoken.get_encoding(encoding)
+
+    return len(enc.encode(text))
+
+
+def estimate_savings(data: Any, encoding: str = "o200k_base") -> Dict[str, Any]:
+    """Compare token counts between JSON and TOON formats.
+
+    Args:
+        data: Python dict or list to compare.
+        encoding: Tokenizer encoding name (default: 'o200k_base').
+
+    Returns:
+        dict: Dictionary containing:
+            - json_tokens (int): Token count for JSON format
+            - toon_tokens (int): Token count for TOON format
+            - savings (int): Absolute token savings (json_tokens - toon_tokens)
+            - savings_percent (float): Percentage savings
+
+    Example:
+        >>> import toon_format
+        >>> data = {"employees": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]}
+        >>> result = toon_format.estimate_savings(data)
+        >>> print(f"Savings: {result['savings_percent']:.1f}%")
+        Savings: 42.3%
+
+    Note:
+        Significant savings are typically achieved with structured data,
+        especially arrays of uniform objects (tabular data).
+    """
+    # Encode as JSON
+    json_str = json.dumps(data, indent=2, ensure_ascii=False)
+    json_tokens = count_tokens(json_str, encoding)
+
+    # Encode as TOON
+    toon_str = encode(data)
+    toon_tokens = count_tokens(toon_str, encoding)
+
+    # Calculate savings
+    savings = max(0, json_tokens - toon_tokens)
+    savings_percent = (savings / json_tokens * 100.0) if json_tokens > 0 else 0.0
+
+    return {
+        "json_tokens": json_tokens,
+        "toon_tokens": toon_tokens,
+        "savings": savings,
+        "savings_percent": savings_percent,
+    }
+
+
+def compare_formats(data: Any, encoding: str = "o200k_base") -> str:
+    """Generate a formatted comparison table showing JSON vs TOON metrics.
+
+    Args:
+        data: Python dict or list to compare.
+        encoding: Tokenizer encoding name (default: 'o200k_base').
+
+    Returns:
+        str: Formatted table as multi-line string showing token counts,
+             character sizes, and savings percentage.
+
+    Example:
+        >>> import toon_format
+        >>> data = {"users": [{"id": 1, "name": "Alice"}]}
+        >>> print(toon_format.compare_formats(data))
+        Format Comparison
+        ────────────────────────────────────────────────
+        Format      Tokens    Size (chars)
+        JSON         1,234         5,678
+        TOON           789         3,456
+        ────────────────────────────────────────────────
+        Savings: 445 tokens (36.1%)
+
+    Note:
+        This is useful for quick visual comparison during development.
+    """
+    # Get token metrics
+    metrics = estimate_savings(data, encoding)
+
+    # Encode both formats to get character counts
+    json_str = json.dumps(data, indent=2, ensure_ascii=False)
+    toon_str = encode(data)
+
+    json_chars = len(json_str)
+    toon_chars = len(toon_str)
+
+    # Build formatted table
+    separator = "─" * 48
+    lines = [
+        "Format Comparison",
+        separator,
+        "Format      Tokens    Size (chars)",
+        f"JSON      {metrics['json_tokens']:>7,}    {json_chars:>11,}",
+        f"TOON      {metrics['toon_tokens']:>7,}    {toon_chars:>11,}",
+        separator,
+        f"Savings: {metrics['savings']:,} tokens ({metrics['savings_percent']:.1f}%)",
+    ]
+
+    return "\n".join(lines)