etlplus 0.10.4__py3-none-any.whl → 0.12.2__py3-none-any.whl

etlplus/file/README.md

@@ -0,0 +1,105 @@
+# etlplus.file subpackage
+
+Documentation for the `etlplus.file` subpackage: unified file format support and helpers for reading
+and writing data files.
+
+- Provides a consistent interface for reading and writing files in various formats
+- Supports all formats defined in `FileFormat` (see below)
+- Includes helpers for inferring file format and compression from filenames, extensions, or MIME
+  types
+- Exposes a `File` class with instance methods for reading and writing data
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [etlplus.file subpackage](#etlplusfile-subpackage)
+  - [Supported File Formats](#supported-file-formats)
+  - [Inferring File Format and Compression](#inferring-file-format-and-compression)
+  - [Reading and Writing Files](#reading-and-writing-files)
+    - [Reading a File](#reading-a-file)
+    - [Writing a File](#writing-a-file)
+  - [File Instance Methods](#file-instance-methods)
+  - [Example: Reading and Writing](#example-reading-and-writing)
+  - [See Also](#see-also)
+
+## Supported File Formats
+
+The following formats are defined in `FileFormat` and supported for reading and writing:
+
+| Format    | Description                                 |
+|-----------|---------------------------------------------|
+| avro      | Apache Avro binary serialization            |
+| csv       | Comma-separated values text files           |
+| feather   | Apache Arrow Feather columnar format        |
+| gz        | Gzip-compressed files (see Compression)     |
+| json      | Standard JSON files                         |
+| ndjson    | Newline-delimited JSON (JSON Lines)         |
+| orc       | Apache ORC columnar format                  |
+| parquet   | Apache Parquet columnar format              |
+| tsv       | Tab-separated values text files             |
+| txt       | Plain text files                            |
+| xls       | Microsoft Excel (legacy .xls)               |
+| xlsx      | Microsoft Excel (modern .xlsx)              |
+| zip       | ZIP-compressed files (see Compression)      |
+| xml       | XML files                                   |
+| yaml      | YAML files                                  |
+
+Compression formats (gz, zip) are also supported as wrappers for other formats.
+
+## Inferring File Format and Compression
+
+Use `infer_file_format_and_compression(value, filename=None)` to infer the file format and
+compression from a filename, extension, or MIME type. Returns a tuple `(file_format,
+compression_format)`.
+
+## Reading and Writing Files
+
+The main entry point for file operations is the `File` class. To read or write files:
+
+### Reading a File
+
+```python
+from etlplus.file import File
+
+f = File("data/sample.csv")
+data = f.read()
+```
+
+- The `read()` method automatically detects the format and compression.
+- Returns parsed data (e.g., list of dicts for tabular formats).
+
+### Writing a File
+
+```python
+from etlplus.file import File
+
+f = File("output.json")
+f.write(data)
+```
+
+- The `write()` method serializes and writes data in the appropriate format.
+- Supports all formats listed above.
+
+## File Instance Methods
+
+- `read()`: Reads and parses the file, returning structured data.
+- `write(data)`: Writes structured data to the file in the detected format.
+
+## Example: Reading and Writing
+
+```python
+from etlplus.file import File
+
+# Read CSV
+csv_file = File("data.csv")
+rows = csv_file.read()
+
+# Write JSON
+json_file = File("output.json")
+json_file.write(rows)
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- File format enums in [enums.py](enums.py)
+- Compression format enums in [enums.py](enums.py)

etlplus/file/__init__.py

@@ -0,0 +1,25 @@
+"""
+:mod:`etlplus.file` package.
+
+Public file IO helpers.
+"""
+
+from __future__ import annotations
+
+from .core import File
+from .enums import CompressionFormat
+from .enums import FileFormat
+from .enums import infer_file_format_and_compression
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Class
+    'File',
+    # Enums
+    'CompressionFormat',
+    'FileFormat',
+    # Functions
+    'infer_file_format_and_compression',
+]

etlplus/file/avro.py

@@ -0,0 +1,198 @@
+"""
+:mod:`etlplus.file.avro` module.
+
+Helpers for reading/writing Avro files.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+_FASTAVRO_CACHE: dict[str, Any] = {}
+
+
+_PRIMITIVE_TYPES: tuple[type, ...] = (
+    bool,
+    int,
+    float,
+    str,
+    bytes,
+    bytearray,
+)
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _get_fastavro() -> Any:
+    """
+    Return the fastavro module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+    """
+    mod = _FASTAVRO_CACHE.get('mod')
+    if mod is not None:  # pragma: no cover - tiny branch
+        return mod
+    try:
+        _fastavro = __import__('fastavro')  # type: ignore[assignment]
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            'AVRO support requires optional dependency "fastavro".\n'
+            'Install with: pip install fastavro',
+        ) from e
+    _FASTAVRO_CACHE['mod'] = _fastavro
+
+    return _fastavro
+
+
+def _normalize_records(data: JSONData) -> JSONList:
+    """
+    Normalize JSON payloads into a list of dictionaries.
+
+    Raises TypeError when payloads contain non-dict items.
+    """
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError('AVRO payloads must contain only objects (dicts)')
+        return cast(JSONList, data)
+    return [cast(JSONDict, data)]
+
+
+def _infer_value_type(value: object) -> str | list[str]:
+    """
+    Infer the Avro type for a primitive value.
+
+    Raises TypeError for unsupported types.
+    """
+    if value is None:
+        return 'null'
+    if isinstance(value, bool):
+        return 'boolean'
+    if isinstance(value, int):
+        return 'long'
+    if isinstance(value, float):
+        return 'double'
+    if isinstance(value, str):
+        return 'string'
+    if isinstance(value, (bytes, bytearray)):
+        return 'bytes'
+    raise TypeError('AVRO payloads must contain only primitive values')
+
+
+def _merge_types(types: list[str]) -> str | list[str]:
+    """Return a stable Avro type union for a list of types."""
+    unique = list(dict.fromkeys(types))
+    if len(unique) == 1:
+        return unique[0]
+    ordered = ['null'] + sorted(t for t in unique if t != 'null')
+    return ordered
+
+
+def _infer_schema(records: JSONList) -> dict[str, Any]:
+    """
+    Infer a basic Avro schema from record payloads.
+
+    Only primitive field values are supported; complex values raise TypeError.
+    """
+    field_names = sorted({key for record in records for key in record})
+    fields: list[dict[str, Any]] = []
+    for name in field_names:
+        types: list[str] = []
+        for record in records:
+            value = record.get(name)
+            if value is None:
+                types.append('null')
+                continue
+            if isinstance(value, dict | list):
+                raise TypeError(
+                    'AVRO payloads must contain only primitive values',
+                )
+            if not isinstance(value, _PRIMITIVE_TYPES):
+                raise TypeError(
+                    'AVRO payloads must contain only primitive values',
+                )
+            types.append(cast(str, _infer_value_type(value)))
+        fields.append({'name': name, 'type': _merge_types(types)})
+
+    return {
+        'name': 'etlplus_record',
+        'type': 'record',
+        'fields': fields,
+    }
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read AVRO content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the AVRO file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the AVRO file.
+    """
+    fastavro = _get_fastavro()
+    with path.open('rb') as handle:
+        reader = fastavro.reader(handle)
+        return [cast(JSONDict, record) for record in reader]
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to AVRO at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the AVRO file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+    """
+    records = _normalize_records(data)
+    if not records:
+        return 0
+
+    fastavro = _get_fastavro()
+    schema = _infer_schema(records)
+    parsed_schema = fastavro.parse_schema(schema)
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('wb') as handle:
+        fastavro.writer(handle, parsed_schema, records)
+
+    return len(records)

etlplus/file/core.py

@@ -0,0 +1,287 @@
+"""
+:mod:`etlplus.file.core` module.
+
+Shared helpers for reading and writing structured and semi-structured data
+files.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..types import JSONData
+from . import avro
+from . import csv
+from . import feather
+from . import gz
+from . import json
+from . import ndjson
+from . import orc
+from . import parquet
+from . import tsv
+from . import txt
+from . import xls
+from . import xlsx
+from . import xml
+from . import yaml
+from . import zip as zip_
+from .enums import FileFormat
+from .enums import infer_file_format_and_compression
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = ['File']
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+@dataclass(slots=True)
+class File:
+    """
+    Convenience wrapper around structured file IO.
+
+    This class encapsulates the one-off helpers in this module as convenient
+    instance methods while retaining the original function API for
+    backward compatibility (those functions delegate to this class).
+
+    Attributes
+    ----------
+    path : Path
+        Path to the file on disk.
+    file_format : FileFormat | None, optional
+        Explicit format. If omitted, the format is inferred from the file
+        extension (``.csv``, ``.json``, etc.).
+
+    Parameters
+    ----------
+    path : StrPath
+        Path to the file on disk.
+    file_format : FileFormat | str | None, optional
+        Explicit format. If omitted, the format is inferred from the file
+        extension (``.csv``, ``.json``, etc.).
+    """
+
+    # -- Attributes -- #
+
+    path: Path
+    file_format: FileFormat | None = None
+
+    # -- Magic Methods (Object Lifecycle) -- #
+
+    def __post_init__(self) -> None:
+        """
+        Auto-detect and set the file format on initialization.
+
+        If no explicit ``file_format`` is provided, attempt to infer it from
+        the file path's extension and update :attr:`file_format`. If the
+        extension is unknown, the attribute is left as ``None`` and will be
+        validated later by :meth:`_ensure_format`.
+        """
+        self.path = Path(self.path)
+        self.file_format = self._coerce_format(self.file_format)
+        if self.file_format is None:
+            self.file_format = self._maybe_guess_format()
+
+    # -- Internal Instance Methods -- #
+
+    def _assert_exists(self) -> None:
+        """
+        Raise FileNotFoundError if :attr:`path` does not exist.
+
+        This centralizes existence checks across multiple read methods.
+        """
+        if not self.path.exists():
+            raise FileNotFoundError(f'File not found: {self.path}')
+
+    def _coerce_format(
+        self,
+        file_format: FileFormat | str | None,
+    ) -> FileFormat | None:
+        """
+        Normalize the file format input.
+
+        Parameters
+        ----------
+        file_format : FileFormat | str | None
+            File format specifier. Strings are coerced into
+            :class:`FileFormat`.
+
+        Returns
+        -------
+        FileFormat | None
+            A normalized file format, or ``None`` when unspecified.
+        """
+        if file_format is None or isinstance(file_format, FileFormat):
+            return file_format
+        return FileFormat.coerce(file_format)
+
+    def _ensure_format(self) -> FileFormat:
+        """
+        Resolve the active format, guessing from extension if needed.
+
+        Returns
+        -------
+        FileFormat
+            The resolved file format.
+        """
+        return (
+            self.file_format
+            if self.file_format is not None
+            else self._guess_format()
+        )
+
+    def _guess_format(self) -> FileFormat:
+        """
+        Infer the file format from the filename extension.
+
+        Returns
+        -------
+        FileFormat
+            The inferred file format based on the file extension.
+
+        Raises
+        ------
+        ValueError
+            If the extension is unknown or unsupported.
+        """
+        fmt, compression = infer_file_format_and_compression(self.path)
+        if fmt is not None:
+            return fmt
+        if compression is not None:
+            raise ValueError(
+                'Cannot infer file format from compressed file '
+                f'{self.path!r} with compression {compression.value!r}',
+            )
+        raise ValueError(
+            f'Cannot infer file format from extension {self.path.suffix!r}',
+        )
+
+    def _maybe_guess_format(self) -> FileFormat | None:
+        """
+        Try to infer the format, returning ``None`` if it cannot be inferred.
+
+        Returns
+        -------
+        FileFormat | None
+            The inferred format, or ``None`` if inference fails.
+        """
+        try:
+            return self._guess_format()
+        except ValueError:
+            # Leave as None; _ensure_format() will raise on use if needed.
+            return None
+
+    # -- Instance Methods -- #
+
+    def read(self) -> JSONData:
+        """
+        Read structured data from :attr:`path` using :attr:`file_format`.
+
+        Returns
+        -------
+        JSONData
+            The structured data read from the file.
+
+        Raises
+        ------
+        ValueError
+            If the resolved file format is unsupported.
+        """
+        self._assert_exists()
+        fmt = self._ensure_format()
+        match fmt:
+            case FileFormat.AVRO:
+                return avro.read(self.path)
+            case FileFormat.CSV:
+                return csv.read(self.path)
+            case FileFormat.FEATHER:
+                return feather.read(self.path)
+            case FileFormat.GZ:
+                return gz.read(self.path)
+            case FileFormat.JSON:
+                return json.read(self.path)
+            case FileFormat.NDJSON:
+                return ndjson.read(self.path)
+            case FileFormat.ORC:
+                return orc.read(self.path)
+            case FileFormat.PARQUET:
+                return parquet.read(self.path)
+            case FileFormat.TSV:
+                return tsv.read(self.path)
+            case FileFormat.TXT:
+                return txt.read(self.path)
+            case FileFormat.XLS:
+                return xls.read(self.path)
+            case FileFormat.XLSX:
+                return xlsx.read(self.path)
+            case FileFormat.XML:
+                return xml.read(self.path)
+            case FileFormat.YAML:
+                return yaml.read(self.path)
+            case FileFormat.ZIP:
+                return zip_.read(self.path)
+        raise ValueError(f'Unsupported format: {fmt}')
+
+    def write(
+        self,
+        data: JSONData,
+        *,
+        root_tag: str = xml.DEFAULT_XML_ROOT,
+    ) -> int:
+        """
+        Write ``data`` to :attr:`path` using :attr:`file_format`.
+
+        Parameters
+        ----------
+        data : JSONData
+            Data to write to the file.
+        root_tag : str, optional
+            Root tag name to use when writing XML files. Defaults to
+            ``'root'``.
+
+        Returns
+        -------
+        int
+            The number of records written.
+
+        Raises
+        ------
+        ValueError
+            If the resolved file format is unsupported.
+        """
+        fmt = self._ensure_format()
+        match fmt:
+            case FileFormat.AVRO:
+                return avro.write(self.path, data)
+            case FileFormat.CSV:
+                return csv.write(self.path, data)
+            case FileFormat.FEATHER:
+                return feather.write(self.path, data)
+            case FileFormat.GZ:
+                return gz.write(self.path, data)
+            case FileFormat.JSON:
+                return json.write(self.path, data)
+            case FileFormat.NDJSON:
+                return ndjson.write(self.path, data)
+            case FileFormat.ORC:
+                return orc.write(self.path, data)
+            case FileFormat.PARQUET:
+                return parquet.write(self.path, data)
+            case FileFormat.TSV:
+                return tsv.write(self.path, data)
+            case FileFormat.TXT:
+                return txt.write(self.path, data)
+            case FileFormat.XLS:
+                return xls.write(self.path, data)
+            case FileFormat.XLSX:
+                return xlsx.write(self.path, data)
+            case FileFormat.XML:
+                return xml.write(self.path, data, root_tag=root_tag)
+            case FileFormat.YAML:
+                return yaml.write(self.path, data)
+            case FileFormat.ZIP:
+                return zip_.write(self.path, data)
+        raise ValueError(f'Unsupported format: {fmt}')

etlplus/file/csv.py

@@ -0,0 +1,91 @@
+"""
+:mod:`etlplus.file.csv` module.
+
+Helpers for reading/writing CSV files.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read CSV content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the CSV file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the CSV file.
+    """
+    with path.open('r', encoding='utf-8', newline='') as handle:
+        reader: csv.DictReader[str] = csv.DictReader(handle)
+        rows: JSONList = []
+        for row in reader:
+            if not any(row.values()):
+                continue
+            rows.append(cast(JSONDict, dict(row)))
+    return rows
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to CSV at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the CSV file on disk.
+    data : JSONData
+        Data to write as CSV. Should be a list of dictionaries or a
+        single dictionary.
+
+    Returns
+    -------
+    int
+        The number of rows written to the CSV file.
+    """
+    rows: list[JSONDict]
+    if isinstance(data, list):
+        rows = [row for row in data if isinstance(row, dict)]
+    else:
+        rows = [data]
+
+    if not rows:
+        return 0
+
+    fieldnames = sorted({key for row in rows for key in row})
+    with path.open('w', encoding='utf-8', newline='') as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames)
+        writer.writeheader()
+        for row in rows:
+            writer.writerow({field: row.get(field) for field in fieldnames})
+
+    return len(rows)