emergent-translator 1.1.0__py3-none-any.whl

emergent_translator/format_handlers.py

@@ -0,0 +1,901 @@
+#!/usr/bin/env python3
+"""
+Format Handlers for Emergent Language Batch Encoder
+
+Converts CSV, JSONL, YAML, TOML, XML, MessagePack, Protobuf, and Parquet
+to/from List[Dict] for use with BatchEncoder.encode_batch() / decode_batch().
+
+Text formats:   CSV, JSONL, YAML, TOML, XML
+Binary formats: MessagePack, Protobuf, Parquet
+
+Usage:
+    from format_handlers import csv_to_dicts, dicts_to_csv
+    from format_handlers import detect_format, get_handler, is_binary_format
+"""
+
+import csv
+import io
+import json
+import os
+import xml.etree.ElementTree as ET
+from typing import Any, Callable, Dict, List, Tuple, Union
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Type Inference
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def infer_type(value: str) -> Any:
+    """Infer Python type from a string value.
+
+    Recognizes int, float, bool (true/false), None/null/empty, and falls back
+    to str.
+    """
+    if value == "":
+        return None
+
+    lower = value.lower()
+    if lower in ("null", "none"):
+        return None
+    if lower == "true":
+        return True
+    if lower == "false":
+        return False
+
+    # Try int first (must not have decimal point)
+    try:
+        if "." not in value and "e" not in lower:
+            return int(value)
+    except ValueError:
+        pass
+
+    # Try float
+    try:
+        return float(value)
+    except ValueError:
+        pass
+
+    return value
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CSV
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def csv_to_dicts(text: str, infer_types: bool = True) -> List[Dict]:
+    """Parse CSV text into a list of dicts.
+
+    Args:
+        text: CSV string with a header row.
+        infer_types: If True, convert values to int/float/bool/None where
+                     possible. If False, keep everything as strings.
+
+    Returns:
+        List of dicts, one per CSV row.
+    """
+    reader = csv.DictReader(io.StringIO(text))
+    rows = []
+    for row in reader:
+        if infer_types:
+            row = {k: infer_type(v) for k, v in row.items()}
+        rows.append(dict(row))
+    return rows
+
+
+def dicts_to_csv(dicts: List[Dict]) -> str:
+    """Serialize a list of dicts to CSV text.
+
+    Nested values (dict/list) are JSON-stringified. None becomes empty string.
+    Field order is the union of all keys, preserving first-seen order.
+    """
+    if not dicts:
+        return ""
+
+    # Collect all keys preserving insertion order
+    fieldnames = []
+    seen = set()
+    for d in dicts:
+        for k in d:
+            if k not in seen:
+                fieldnames.append(k)
+                seen.add(k)
+
+    output = io.StringIO()
+    writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore")
+    writer.writeheader()
+    for d in dicts:
+        row = {}
+        for k in fieldnames:
+            v = d.get(k)
+            if v is None:
+                row[k] = ""
+            elif isinstance(v, bool):
+                row[k] = str(v).lower()
+            elif isinstance(v, (dict, list)):
+                row[k] = json.dumps(v, separators=(",", ":"))
+            else:
+                row[k] = v
+        writer.writerow(row)
+    return output.getvalue()
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# JSONL / NDJSON
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def jsonl_to_dicts(text: str) -> List[Dict]:
+    """Parse newline-delimited JSON (JSONL/NDJSON) into a list of dicts.
+
+    Each non-empty line is parsed as a separate JSON object.
+    """
+    result = []
+    for lineno, line in enumerate(text.splitlines(), 1):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON on line {lineno}: {e}") from e
+        if not isinstance(obj, dict):
+            raise ValueError(
+                f"Expected dict on line {lineno}, got {type(obj).__name__}"
+            )
+        result.append(obj)
+    return result
+
+
+def dicts_to_jsonl(dicts: List[Dict]) -> str:
+    """Serialize a list of dicts to JSONL (one JSON object per line)."""
+    if not dicts:
+        return ""
+    return "\n".join(json.dumps(d, separators=(",", ":")) for d in dicts) + "\n"
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# YAML
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def _import_yaml():
+    """Import pyyaml with a helpful error message."""
+    try:
+        import yaml
+        return yaml
+    except ImportError:
+        raise ImportError(
+            "PyYAML is required for YAML support. "
+            "Install it with: pip install pyyaml  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def yaml_to_dicts(text: str) -> List[Dict]:
+    """Parse YAML text into a list of dicts.
+
+    Handles:
+    - A YAML list of dicts (most common)
+    - A single YAML dict (wrapped in a list)
+    - Multi-document YAML (--- separators)
+
+    Raises ValueError for scalars, empty docs, and non-dict list items.
+    """
+    yaml = _import_yaml()
+
+    # Try multi-document first
+    docs = list(yaml.safe_load_all(text))
+
+    # Filter out None docs (empty documents between ---)
+    docs = [d for d in docs if d is not None]
+
+    if not docs:
+        raise ValueError("YAML document is empty")
+
+    # Single document
+    if len(docs) == 1:
+        data = docs[0]
+        if isinstance(data, list):
+            for item in data:
+                if not isinstance(item, dict):
+                    raise ValueError(
+                        f"Expected list of dicts, got list containing {type(item).__name__}"
+                    )
+            return data
+        if isinstance(data, dict):
+            return [data]
+        raise ValueError(
+            f"Expected dict or list of dicts, got {type(data).__name__}"
+        )
+
+    # Multiple documents — each should be a dict
+    result = []
+    for doc in docs:
+        if isinstance(doc, dict):
+            result.append(doc)
+        elif isinstance(doc, list):
+            for item in doc:
+                if not isinstance(item, dict):
+                    raise ValueError(
+                        f"Expected dicts in multi-doc YAML, got {type(item).__name__}"
+                    )
+                result.append(item)
+        else:
+            raise ValueError(
+                f"Expected dict in multi-doc YAML, got {type(doc).__name__}"
+            )
+    return result
+
+
+def dicts_to_yaml(dicts: List[Dict]) -> str:
+    """Serialize a list of dicts to YAML text."""
+    yaml = _import_yaml()
+    return yaml.dump(dicts, default_flow_style=False, sort_keys=False)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# TOML
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def _import_tomllib():
+    """Import tomllib (3.11+) or tomli backport for reading TOML."""
+    try:
+        import tomllib
+        return tomllib
+    except ImportError:
+        pass
+    try:
+        import tomli as tomllib
+        return tomllib
+    except ImportError:
+        raise ImportError(
+            "TOML reading requires Python 3.11+ or the 'tomli' package. "
+            "Install it with: pip install tomli  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def _import_tomli_w():
+    """Import tomli_w for writing TOML."""
+    try:
+        import tomli_w
+        return tomli_w
+    except ImportError:
+        raise ImportError(
+            "TOML writing requires the 'tomli_w' package. "
+            "Install it with: pip install tomli_w  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def toml_to_dicts(text: str) -> List[Dict]:
+    """Parse TOML text into a list of dicts.
+
+    If the TOML has an ``items``, ``records``, ``entries``, or ``data`` key
+    whose value is a list of tables, that list is returned directly.
+    Otherwise the root table is returned wrapped in a list.
+    """
+    tomllib = _import_tomllib()
+    data = tomllib.loads(text)
+
+    # Look for a conventional list-of-tables key
+    for key in ("items", "records", "entries", "data"):
+        if key in data and isinstance(data[key], list):
+            return data[key]
+
+    return [data]
+
+
+def dicts_to_toml(dicts: List[Dict]) -> str:
+    """Serialize a list of dicts to TOML text.
+
+    A single dict is written as a flat TOML document.  Multiple dicts are
+    written as an ``[[items]]`` array of tables.  None values are omitted
+    (TOML has no null type).
+    """
+    tomli_w = _import_tomli_w()
+    cleaned = [_toml_clean(d) for d in dicts]
+    if len(cleaned) == 1:
+        return tomli_w.dumps(cleaned[0])
+    return tomli_w.dumps({"items": cleaned})
+
+
+def _toml_clean(obj: Any) -> Any:
+    """Remove None values from nested structures for TOML compatibility."""
+    if isinstance(obj, dict):
+        return {k: _toml_clean(v) for k, v in obj.items() if v is not None}
+    if isinstance(obj, list):
+        return [_toml_clean(item) for item in obj]
+    return obj
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# XML
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def xml_to_dicts(text: str) -> List[Dict]:
+    """Parse XML text into a list of dicts.
+
+    Expected structure:
+        <root>
+          <item attr="val">
+            <key>value</key>
+            ...
+          </item>
+          ...
+        </root>
+
+    Conventions:
+    - XML attributes become ``@attr`` keys.
+    - Text content of an element with children/attributes becomes ``#text``.
+    - Repeated sibling tags become lists.
+    - Leaf text is type-inferred (int/float/bool/None).
+    """
+    root = ET.fromstring(text)
+    result = []
+    for child in root:
+        result.append(_xml_element_to_dict(child))
+    return result
+
+
+def _xml_element_to_dict(elem: ET.Element) -> Dict:
+    """Recursively convert an XML element to a dict."""
+    d = {}
+
+    # Attributes → @attr keys
+    for attr_name, attr_val in elem.attrib.items():
+        d["@" + attr_name] = infer_type(attr_val)
+
+    # Group children by tag to detect repeated siblings
+    children_by_tag = {}
+    for child in elem:
+        children_by_tag.setdefault(child.tag, []).append(child)
+
+    for tag, children in children_by_tag.items():
+        if len(children) == 1:
+            child = children[0]
+            if len(child) == 0 and not child.attrib:
+                # Leaf element
+                d[tag] = infer_type(child.text or "")
+            else:
+                d[tag] = _xml_element_to_dict(child)
+        else:
+            # Repeated tag → list
+            items = []
+            for child in children:
+                if len(child) == 0 and not child.attrib:
+                    items.append(infer_type(child.text or ""))
+                else:
+                    items.append(_xml_element_to_dict(child))
+            d[tag] = items
+
+    # Text content
+    text = (elem.text or "").strip()
+    if text:
+        if d:
+            d["#text"] = text
+        else:
+            d["#text"] = infer_type(text)
+            return d
+
+    return d
+
+
+def dicts_to_xml(dicts: List[Dict], root_tag: str = "root",
+                 item_tag: str = "item") -> str:
+    """Serialize a list of dicts to XML text.
+
+    Args:
+        dicts: Data to serialize.
+        root_tag: Name of the root element.
+        item_tag: Name of each item element.
+
+    Returns:
+        XML string with declaration.
+    """
+    root = ET.Element(root_tag)
+    for d in dicts:
+        item = ET.SubElement(root, item_tag)
+        _dict_to_xml_element(d, item)
+
+    # Pretty-print (ET.indent is 3.9+)
+    try:
+        ET.indent(root, space="  ")
+    except AttributeError:
+        pass
+
+    return ET.tostring(root, encoding="unicode", xml_declaration=True)
+
+
+def _dict_to_xml_element(d: Dict, parent: ET.Element) -> None:
+    """Recursively convert a dict to XML sub-elements under *parent*."""
+    for key, value in d.items():
+        if key.startswith("@"):
+            parent.set(key[1:], _xml_value_str(value))
+        elif key == "#text":
+            parent.text = _xml_value_str(value)
+        elif isinstance(value, dict):
+            child = ET.SubElement(parent, key)
+            _dict_to_xml_element(value, child)
+        elif isinstance(value, list):
+            for item in value:
+                child = ET.SubElement(parent, key)
+                if isinstance(item, dict):
+                    _dict_to_xml_element(item, child)
+                else:
+                    child.text = _xml_value_str(item)
+        else:
+            child = ET.SubElement(parent, key)
+            child.text = _xml_value_str(value)
+
+
+def _xml_value_str(value: Any) -> str:
+    """Convert a Python value to its XML text representation."""
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return str(value).lower()
+    return str(value)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# MessagePack (requires: pip install msgpack)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def _import_msgpack():
+    """Import msgpack with a helpful error message."""
+    try:
+        import msgpack
+        return msgpack
+    except ImportError:
+        raise ImportError(
+            "msgpack is required for MessagePack support. "
+            "Install it with: pip install msgpack  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def msgpack_to_dicts(data: bytes) -> List[Dict]:
+    """Deserialize MessagePack bytes into a list of dicts.
+
+    Accepts a packed list of dicts or a single packed dict (wrapped in a list).
+    """
+    msgpack = _import_msgpack()
+    unpacked = msgpack.unpackb(data, raw=False)
+    if isinstance(unpacked, list):
+        return unpacked
+    if isinstance(unpacked, dict):
+        return [unpacked]
+    raise ValueError(
+        f"Expected list or dict in MessagePack data, got {type(unpacked).__name__}"
+    )
+
+
+def dicts_to_msgpack(dicts: List[Dict]) -> bytes:
+    """Serialize a list of dicts to MessagePack bytes."""
+    msgpack = _import_msgpack()
+    return msgpack.packb(dicts, use_bin_type=True)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Protobuf (requires: pip install protobuf)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def _import_protobuf():
+    """Import google.protobuf struct utilities with a helpful error message."""
+    try:
+        from google.protobuf import struct_pb2, json_format
+        return struct_pb2, json_format
+    except ImportError:
+        raise ImportError(
+            "protobuf is required for Protobuf support. "
+            "Install it with: pip install protobuf  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def protobuf_to_dicts(data: bytes) -> List[Dict]:
+    """Deserialize Protobuf bytes (google.protobuf.Struct wrapper) to dicts.
+
+    Uses ``google.protobuf.struct_pb2.Struct`` as a schema-less container.
+    Note: all numeric values are stored as doubles in Protobuf Struct, so
+    integers will round-trip as floats (e.g. 42 -> 42.0).
+    """
+    struct_pb2, json_format = _import_protobuf()
+    wrapper = struct_pb2.Struct()
+    wrapper.ParseFromString(data)
+    raw = json_format.MessageToDict(wrapper)
+    items = raw.get("items", [raw])
+    return items
+
+
+def dicts_to_protobuf(dicts: List[Dict]) -> bytes:
+    """Serialize a list of dicts to Protobuf bytes using Struct wrapper.
+
+    Wraps the list under an ``items`` key in a ``google.protobuf.Struct``.
+    """
+    struct_pb2, json_format = _import_protobuf()
+    wrapper = struct_pb2.Struct()
+    json_format.ParseDict({"items": dicts}, wrapper)
+    return wrapper.SerializeToString()
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Parquet (requires: pip install pyarrow)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def _import_pyarrow():
+    """Import pyarrow with a helpful error message."""
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+        return pa, pq
+    except ImportError:
+        raise ImportError(
+            "pyarrow is required for Parquet support. "
+            "Install it with: pip install pyarrow  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def parquet_to_dicts(data: bytes) -> List[Dict]:
+    """Deserialize Parquet bytes into a list of dicts.
+
+    Best suited for flat/tabular data.  Nested structures depend on Arrow's
+    schema inference.
+    """
+    pa, pq = _import_pyarrow()
+    buf = pa.BufferReader(data)
+    table = pq.read_table(buf)
+    return table.to_pylist()
+
+
+def dicts_to_parquet(dicts: List[Dict]) -> bytes:
+    """Serialize a list of dicts to Parquet bytes.
+
+    Best suited for flat/tabular data with uniform schemas.
+    """
+    pa, pq = _import_pyarrow()
+    table = pa.Table.from_pylist(dicts)
+    sink = pa.BufferOutputStream()
+    pq.write_table(table, sink)
+    return sink.getvalue().to_pybytes()
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Arrow IPC / Feather (requires: pip install pyarrow — shared with Parquet)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def arrow_to_dicts(data: bytes) -> List[Dict]:
+    """Deserialize Arrow IPC (Feather v2) bytes into a list of dicts."""
+    pa, _ = _import_pyarrow()
+    reader = pa.ipc.open_file(pa.BufferReader(data))
+    table = reader.read_all()
+    return table.to_pylist()
+
+
+def dicts_to_arrow(dicts: List[Dict]) -> bytes:
+    """Serialize a list of dicts to Arrow IPC (Feather v2) bytes."""
+    pa, _ = _import_pyarrow()
+    table = pa.Table.from_pylist(dicts)
+    sink = pa.BufferOutputStream()
+    writer = pa.ipc.new_file(sink, table.schema)
+    writer.write_table(table)
+    writer.close()
+    return sink.getvalue().to_pybytes()
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# BSON (requires: pip install pymongo)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def _import_bson():
+    """Import bson (from pymongo) with a helpful error message."""
+    try:
+        import bson
+        return bson
+    except ImportError:
+        raise ImportError(
+            "pymongo is required for BSON support. "
+            "Install it with: pip install pymongo  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def bson_to_dicts(data: bytes) -> List[Dict]:
+    """Deserialize BSON bytes into a list of dicts.
+
+    Expects a sequence of concatenated BSON documents (standard MongoDB
+    wire format).  ObjectId values are converted to strings.
+    """
+    bson_mod = _import_bson()
+    result = []
+    offset = 0
+    while offset < len(data):
+        # Each BSON doc starts with a 4-byte little-endian length
+        if offset + 4 > len(data):
+            break
+        import struct as _struct
+        doc_len = _struct.unpack_from("<i", data, offset)[0]
+        doc_bytes = data[offset:offset + doc_len]
+        doc = bson_mod.decode(doc_bytes)
+        # Convert ObjectId to string for JSON compatibility
+        if "_id" in doc:
+            doc["_id"] = str(doc["_id"])
+        result.append(doc)
+        offset += doc_len
+    return result
+
+
+def dicts_to_bson(dicts: List[Dict]) -> bytes:
+    """Serialize a list of dicts to concatenated BSON bytes."""
+    bson_mod = _import_bson()
+    parts = []
+    for d in dicts:
+        parts.append(bson_mod.encode(d))
+    return b"".join(parts)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CBOR (requires: pip install cbor2)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def _import_cbor2():
+    """Import cbor2 with a helpful error message."""
+    try:
+        import cbor2
+        return cbor2
+    except ImportError:
+        raise ImportError(
+            "cbor2 is required for CBOR support. "
+            "Install it with: pip install cbor2  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def cbor_to_dicts(data: bytes) -> List[Dict]:
+    """Deserialize CBOR bytes into a list of dicts.
+
+    Accepts a CBOR-encoded list of maps or a single map (wrapped in a list).
+    """
+    cbor2 = _import_cbor2()
+    decoded = cbor2.loads(data)
+    if isinstance(decoded, list):
+        return decoded
+    if isinstance(decoded, dict):
+        return [decoded]
+    raise ValueError(
+        f"Expected list or dict in CBOR data, got {type(decoded).__name__}"
+    )
+
+
+def dicts_to_cbor(dicts: List[Dict]) -> bytes:
+    """Serialize a list of dicts to CBOR bytes."""
+    cbor2 = _import_cbor2()
+    return cbor2.dumps(dicts)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# INI (stdlib configparser)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def ini_to_dicts(text: str) -> List[Dict]:
+    """Parse INI text into a list of dicts, one per section.
+
+    Each section becomes a dict with all its key-value pairs plus a
+    ``_section`` key holding the section name.  Values are type-inferred.
+    The ``[DEFAULT]`` section values are merged into every section (standard
+    configparser behaviour).
+    """
+    import configparser
+    parser = configparser.ConfigParser()
+    parser.read_string(text)
+    result = []
+    for section in parser.sections():
+        d = {"_section": section}
+        for key, val in parser.items(section):
+            d[key] = infer_type(val)
+        result.append(d)
+    return result
+
+
+def dicts_to_ini(dicts: List[Dict]) -> str:
+    """Serialize a list of dicts to INI text.
+
+    Each dict becomes a section.  The section name is taken from a
+    ``_section`` key (defaulting to ``section_N``).  None values are
+    written as empty strings.
+    """
+    import configparser
+    parser = configparser.ConfigParser()
+    for i, d in enumerate(dicts):
+        section = str(d.get("_section", f"section_{i}"))
+        parser.add_section(section)
+        for key, val in d.items():
+            if key == "_section":
+                continue
+            if val is None:
+                parser.set(section, key, "")
+            elif isinstance(val, bool):
+                parser.set(section, key, str(val).lower())
+            elif isinstance(val, (dict, list)):
+                parser.set(section, key, json.dumps(val, separators=(",", ":")))
+            else:
+                parser.set(section, key, str(val))
+    output = io.StringIO()
+    parser.write(output)
+    return output.getvalue()
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Excel / XLSX (requires: pip install openpyxl)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def _import_openpyxl():
+    """Import openpyxl with a helpful error message."""
+    try:
+        import openpyxl
+        return openpyxl
+    except ImportError:
+        raise ImportError(
+            "openpyxl is required for Excel/XLSX support. "
+            "Install it with: pip install openpyxl  "
+            "or: pip install emergent-translator[formats]"
+        )
+
+
+def xlsx_to_dicts(data: bytes) -> List[Dict]:
+    """Deserialize Excel XLSX bytes into a list of dicts.
+
+    Reads the active sheet.  The first row is treated as headers.
+    Values are type-inferred from their cell values (Excel already
+    distinguishes numbers, strings, booleans, and None).
+    """
+    openpyxl = _import_openpyxl()
+    wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True)
+    ws = wb.active
+    rows = list(ws.iter_rows(values_only=True))
+    wb.close()
+    if not rows:
+        return []
+    headers = [str(h) if h is not None else f"col_{i}" for i, h in enumerate(rows[0])]
+    result = []
+    for row in rows[1:]:
+        d = {}
+        for key, val in zip(headers, row):
+            d[key] = val
+        result.append(d)
+    return result
+
+
+def dicts_to_xlsx(dicts: List[Dict]) -> bytes:
+    """Serialize a list of dicts to Excel XLSX bytes.
+
+    Nested values (dict/list) are JSON-stringified.
+    """
+    openpyxl = _import_openpyxl()
+    wb = openpyxl.Workbook()
+    ws = wb.active
+
+    if not dicts:
+        buf = io.BytesIO()
+        wb.save(buf)
+        return buf.getvalue()
+
+    # Collect headers
+    fieldnames = []
+    seen = set()
+    for d in dicts:
+        for k in d:
+            if k not in seen:
+                fieldnames.append(k)
+                seen.add(k)
+
+    ws.append(fieldnames)
+    for d in dicts:
+        row = []
+        for k in fieldnames:
+            v = d.get(k)
+            if isinstance(v, (dict, list)):
+                row.append(json.dumps(v, separators=(",", ":")))
+            else:
+                row.append(v)
+        ws.append(row)
+
+    buf = io.BytesIO()
+    wb.save(buf)
+    return buf.getvalue()
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Format Registry
+# ═══════════════════════════════════════════════════════════════════════════════
+
+# Maps format name → (parse_fn, serialize_fn)
+_HANDLERS: Dict[str, Tuple[Callable, Callable]] = {
+    "csv": (csv_to_dicts, dicts_to_csv),
+    "jsonl": (jsonl_to_dicts, dicts_to_jsonl),
+    "ndjson": (jsonl_to_dicts, dicts_to_jsonl),
+    "yaml": (yaml_to_dicts, dicts_to_yaml),
+    "yml": (yaml_to_dicts, dicts_to_yaml),
+    "toml": (toml_to_dicts, dicts_to_toml),
+    "ini": (ini_to_dicts, dicts_to_ini),
+    "xml": (xml_to_dicts, dicts_to_xml),
+    "msgpack": (msgpack_to_dicts, dicts_to_msgpack),
+    "protobuf": (protobuf_to_dicts, dicts_to_protobuf),
+    "parquet": (parquet_to_dicts, dicts_to_parquet),
+    "arrow": (arrow_to_dicts, dicts_to_arrow),
+    "feather": (arrow_to_dicts, dicts_to_arrow),
+    "bson": (bson_to_dicts, dicts_to_bson),
+    "cbor": (cbor_to_dicts, dicts_to_cbor),
+    "xlsx": (xlsx_to_dicts, dicts_to_xlsx),
+    "excel": (xlsx_to_dicts, dicts_to_xlsx),
+}
+
+_EXTENSION_MAP: Dict[str, str] = {
+    ".csv": "csv",
+    ".jsonl": "jsonl",
+    ".ndjson": "jsonl",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".toml": "toml",
+    ".ini": "ini",
+    ".cfg": "ini",
+    ".xml": "xml",
+    ".json": "json",
+    ".msgpack": "msgpack",
+    ".mpk": "msgpack",
+    ".pb": "protobuf",
+    ".parquet": "parquet",
+    ".arrow": "arrow",
+    ".feather": "arrow",
+    ".ipc": "arrow",
+    ".bson": "bson",
+    ".cbor": "cbor",
+    ".xlsx": "xlsx",
+    ".xls": "xlsx",
+}
+
+_BINARY_FORMATS = frozenset({
+    "msgpack", "protobuf", "parquet", "arrow", "feather", "bson", "cbor",
+    "xlsx", "excel",
+})
+
+
+def is_binary_format(name: str) -> bool:
+    """Return True if the named format uses binary (bytes) I/O."""
+    return name.lower() in _BINARY_FORMATS
+
+
+def detect_format(filepath: str) -> str:
+    """Detect format from file extension.
+
+    Returns a format name string (e.g. 'csv', 'yaml', 'json', 'msgpack').
+    Raises ValueError for unknown extensions.
+    """
+    ext = os.path.splitext(filepath)[1].lower()
+    fmt = _EXTENSION_MAP.get(ext)
+    if fmt is None:
+        raise ValueError(f"Unknown file extension: {ext!r}")
+    return fmt
+
+
+def get_handler(name: str) -> Tuple[Callable, Callable]:
+    """Get (parse_fn, serialize_fn) for a format name.
+
+    Args:
+        name: Format name ('csv', 'jsonl', 'yaml', 'toml', 'xml',
+              'msgpack', 'protobuf', 'parquet').
+
+    Returns:
+        Tuple of (parse_fn, serialize_fn).
+
+    Raises:
+        ValueError: If format name is not recognized.
+    """
+    name = name.lower()
+    handler = _HANDLERS.get(name)
+    if handler is None:
+        raise ValueError(
+            f"Unknown format: {name!r}. "
+            f"Supported formats: {', '.join(sorted(set(_HANDLERS.keys())))}"
+        )
+    return handler