etlplus 0.10.4__py3-none-any.whl → 0.11.11__py3-none-any.whl

etlplus/file/core.py

@@ -0,0 +1,287 @@
+"""
+:mod:`etlplus.file.core` module.
+
+Shared helpers for reading and writing structured and semi-structured data
+files.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..types import JSONData
+from . import avro
+from . import csv
+from . import feather
+from . import gz
+from . import json
+from . import ndjson
+from . import orc
+from . import parquet
+from . import tsv
+from . import txt
+from . import xls
+from . import xlsx
+from . import xml
+from . import yaml
+from . import zip
+from .enums import FileFormat
+from .enums import infer_file_format_and_compression
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = ['File']
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+@dataclass(slots=True)
+class File:
+    """
+    Convenience wrapper around structured file IO.
+
+    This class encapsulates the one-off helpers in this module as convenient
+    instance methods while retaining the original function API for
+    backward compatibility (those functions delegate to this class).
+
+    Attributes
+    ----------
+    path : Path
+        Path to the file on disk.
+    file_format : FileFormat | None, optional
+        Explicit format. If omitted, the format is inferred from the file
+        extension (``.csv``, ``.json``, etc.).
+
+    Parameters
+    ----------
+    path : StrPath
+        Path to the file on disk.
+    file_format : FileFormat | str | None, optional
+        Explicit format. If omitted, the format is inferred from the file
+        extension (``.csv``, ``.json``, etc.).
+    """
+
+    # -- Attributes -- #
+
+    path: Path
+    file_format: FileFormat | None = None
+
+    # -- Magic Methods (Object Lifecycle) -- #
+
+    def __post_init__(self) -> None:
+        """
+        Auto-detect and set the file format on initialization.
+
+        If no explicit ``file_format`` is provided, attempt to infer it from
+        the file path's extension and update :attr:`file_format`. If the
+        extension is unknown, the attribute is left as ``None`` and will be
+        validated later by :meth:`_ensure_format`.
+        """
+        self.path = Path(self.path)
+        self.file_format = self._coerce_format(self.file_format)
+        if self.file_format is None:
+            self.file_format = self._maybe_guess_format()
+
+    # -- Internal Instance Methods -- #
+
+    def _assert_exists(self) -> None:
+        """
+        Raise FileNotFoundError if :attr:`path` does not exist.
+
+        This centralizes existence checks across multiple read methods.
+        """
+        if not self.path.exists():
+            raise FileNotFoundError(f'File not found: {self.path}')
+
+    def _coerce_format(
+        self,
+        file_format: FileFormat | str | None,
+    ) -> FileFormat | None:
+        """
+        Normalize the file format input.
+
+        Parameters
+        ----------
+        file_format : FileFormat | str | None
+            File format specifier. Strings are coerced into
+            :class:`FileFormat`.
+
+        Returns
+        -------
+        FileFormat | None
+            A normalized file format, or ``None`` when unspecified.
+        """
+        if file_format is None or isinstance(file_format, FileFormat):
+            return file_format
+        return FileFormat.coerce(file_format)
+
+    def _ensure_format(self) -> FileFormat:
+        """
+        Resolve the active format, guessing from extension if needed.
+
+        Returns
+        -------
+        FileFormat
+            The resolved file format.
+        """
+        return (
+            self.file_format
+            if self.file_format is not None
+            else self._guess_format()
+        )
+
+    def _guess_format(self) -> FileFormat:
+        """
+        Infer the file format from the filename extension.
+
+        Returns
+        -------
+        FileFormat
+            The inferred file format based on the file extension.
+
+        Raises
+        ------
+        ValueError
+            If the extension is unknown or unsupported.
+        """
+        fmt, compression = infer_file_format_and_compression(self.path)
+        if fmt is not None:
+            return fmt
+        if compression is not None:
+            raise ValueError(
+                'Cannot infer file format from compressed file '
+                f'{self.path!r} with compression {compression.value!r}',
+            )
+        raise ValueError(
+            f'Cannot infer file format from extension {self.path.suffix!r}',
+        )
+
+    def _maybe_guess_format(self) -> FileFormat | None:
+        """
+        Try to infer the format, returning ``None`` if it cannot be inferred.
+
+        Returns
+        -------
+        FileFormat | None
+            The inferred format, or ``None`` if inference fails.
+        """
+        try:
+            return self._guess_format()
+        except ValueError:
+            # Leave as None; _ensure_format() will raise on use if needed.
+            return None
+
+    # -- Instance Methods -- #
+
+    def read(self) -> JSONData:
+        """
+        Read structured data from :attr:`path` using :attr:`file_format`.
+
+        Returns
+        -------
+        JSONData
+            The structured data read from the file.
+
+        Raises
+        ------
+        ValueError
+            If the resolved file format is unsupported.
+        """
+        self._assert_exists()
+        fmt = self._ensure_format()
+        match fmt:
+            case FileFormat.AVRO:
+                return avro.read(self.path)
+            case FileFormat.CSV:
+                return csv.read(self.path)
+            case FileFormat.FEATHER:
+                return feather.read(self.path)
+            case FileFormat.GZ:
+                return gz.read(self.path)
+            case FileFormat.JSON:
+                return json.read(self.path)
+            case FileFormat.NDJSON:
+                return ndjson.read(self.path)
+            case FileFormat.ORC:
+                return orc.read(self.path)
+            case FileFormat.PARQUET:
+                return parquet.read(self.path)
+            case FileFormat.TSV:
+                return tsv.read(self.path)
+            case FileFormat.TXT:
+                return txt.read(self.path)
+            case FileFormat.XLS:
+                return xls.read(self.path)
+            case FileFormat.XLSX:
+                return xlsx.read(self.path)
+            case FileFormat.XML:
+                return xml.read(self.path)
+            case FileFormat.YAML:
+                return yaml.read(self.path)
+            case FileFormat.ZIP:
+                return zip.read(self.path)
+        raise ValueError(f'Unsupported format: {fmt}')
+
+    def write(
+        self,
+        data: JSONData,
+        *,
+        root_tag: str = xml.DEFAULT_XML_ROOT,
+    ) -> int:
+        """
+        Write ``data`` to :attr:`path` using :attr:`file_format`.
+
+        Parameters
+        ----------
+        data : JSONData
+            Data to write to the file.
+        root_tag : str, optional
+            Root tag name to use when writing XML files. Defaults to
+            ``'root'``.
+
+        Returns
+        -------
+        int
+            The number of records written.
+
+        Raises
+        ------
+        ValueError
+            If the resolved file format is unsupported.
+        """
+        fmt = self._ensure_format()
+        match fmt:
+            case FileFormat.AVRO:
+                return avro.write(self.path, data)
+            case FileFormat.CSV:
+                return csv.write(self.path, data)
+            case FileFormat.FEATHER:
+                return feather.write(self.path, data)
+            case FileFormat.GZ:
+                return gz.write(self.path, data)
+            case FileFormat.JSON:
+                return json.write(self.path, data)
+            case FileFormat.NDJSON:
+                return ndjson.write(self.path, data)
+            case FileFormat.ORC:
+                return orc.write(self.path, data)
+            case FileFormat.PARQUET:
+                return parquet.write(self.path, data)
+            case FileFormat.TSV:
+                return tsv.write(self.path, data)
+            case FileFormat.TXT:
+                return txt.write(self.path, data)
+            case FileFormat.XLS:
+                return xls.write(self.path, data)
+            case FileFormat.XLSX:
+                return xlsx.write(self.path, data)
+            case FileFormat.XML:
+                return xml.write(self.path, data, root_tag=root_tag)
+            case FileFormat.YAML:
+                return yaml.write(self.path, data)
+            case FileFormat.ZIP:
+                return zip.write(self.path, data)
+        raise ValueError(f'Unsupported format: {fmt}')

etlplus/file/csv.py

@@ -0,0 +1,82 @@
+"""
+:mod:`etlplus.file.csv` module.
+
+CSV read/write helpers.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Load CSV content as a list of dictionaries.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the CSV file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the CSV file.
+    """
+    with path.open('r', encoding='utf-8', newline='') as handle:
+        reader: csv.DictReader[str] = csv.DictReader(handle)
+        rows: JSONList = []
+        for row in reader:
+            if not any(row.values()):
+                continue
+            rows.append(cast(JSONDict, dict(row)))
+    return rows
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write CSV rows to ``path`` and return the number of rows.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the CSV file on disk.
+    data : JSONData
+        Data to write as CSV. Should be a list of dictionaries or a
+        single dictionary.
+
+    Returns
+    -------
+    int
+        The number of rows written to the CSV file.
+    """
+    rows: list[JSONDict]
+    if isinstance(data, list):
+        rows = [row for row in data if isinstance(row, dict)]
+    else:
+        rows = [data]
+
+    if not rows:
+        return 0
+
+    fieldnames = sorted({key for row in rows for key in row})
+    with path.open('w', encoding='utf-8', newline='') as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames)
+        writer.writeheader()
+        for row in rows:
+            writer.writerow({field: row.get(field) for field in fieldnames})
+
+    return len(rows)

etlplus/file/enums.py

@@ -0,0 +1,238 @@
+"""
+:mod:`etlplus.file.enums` module.
+
+File-specific enums and helpers.
+"""
+
+from __future__ import annotations
+
+from pathlib import PurePath
+
+from ..enums import CoercibleStrEnum
+from ..types import StrStrMap
+
+# SECTION: EXPORTS ========================================================= #
+
+__all__ = [
+    'CompressionFormat',
+    'FileFormat',
+    'infer_file_format_and_compression',
+]
+
+
+# SECTION: ENUMS ============================================================ #
+
+
+class CompressionFormat(CoercibleStrEnum):
+    """Supported compression formats."""
+
+    # -- Constants -- #
+
+    GZ = 'gz'
+    ZIP = 'zip'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            # File extensions
+            '.gz': 'gz',
+            '.gzip': 'gz',
+            '.zip': 'zip',
+            # MIME types
+            'application/gzip': 'gz',
+            'application/x-gzip': 'gz',
+            'application/zip': 'zip',
+            'application/x-zip-compressed': 'zip',
+        }
+
+
+class FileFormat(CoercibleStrEnum):
+    """Supported file formats for extraction."""
+
+    # -- Constants -- #
+
+    AVRO = 'avro'
+    CSV = 'csv'
+    FEATHER = 'feather'
+    GZ = 'gz'
+    JSON = 'json'
+    NDJSON = 'ndjson'
+    ORC = 'orc'
+    PARQUET = 'parquet'
+    TSV = 'tsv'
+    TXT = 'txt'
+    XLS = 'xls'
+    XLSX = 'xlsx'
+    ZIP = 'zip'
+    XML = 'xml'
+    YAML = 'yaml'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            # Common shorthand
+            'parq': 'parquet',
+            'yml': 'yaml',
+            # File extensions
+            '.avro': 'avro',
+            '.csv': 'csv',
+            '.feather': 'feather',
+            '.gz': 'gz',
+            '.json': 'json',
+            '.jsonl': 'ndjson',
+            '.ndjson': 'ndjson',
+            '.orc': 'orc',
+            '.parquet': 'parquet',
+            '.pq': 'parquet',
+            '.tsv': 'tsv',
+            '.txt': 'txt',
+            '.xls': 'xls',
+            '.xlsx': 'xlsx',
+            '.zip': 'zip',
+            '.xml': 'xml',
+            '.yaml': 'yaml',
+            '.yml': 'yaml',
+            # MIME types
+            'application/avro': 'avro',
+            'application/csv': 'csv',
+            'application/feather': 'feather',
+            'application/gzip': 'gz',
+            'application/json': 'json',
+            'application/jsonlines': 'ndjson',
+            'application/ndjson': 'ndjson',
+            'application/orc': 'orc',
+            'application/parquet': 'parquet',
+            'application/vnd.apache.avro': 'avro',
+            'application/vnd.apache.parquet': 'parquet',
+            'application/vnd.apache.arrow.file': 'feather',
+            'application/vnd.apache.orc': 'orc',
+            'application/vnd.ms-excel': 'xls',
+            (
+                'application/vnd.openxmlformats-'
+                'officedocument.spreadsheetml.sheet'
+            ): 'xlsx',
+            'application/x-avro': 'avro',
+            'application/x-csv': 'csv',
+            'application/x-feather': 'feather',
+            'application/x-orc': 'orc',
+            'application/x-ndjson': 'ndjson',
+            'application/x-parquet': 'parquet',
+            'application/x-yaml': 'yaml',
+            'application/xml': 'xml',
+            'application/zip': 'zip',
+            'text/csv': 'csv',
+            'text/plain': 'txt',
+            'text/tab-separated-values': 'tsv',
+            'text/tsv': 'tsv',
+            'text/xml': 'xml',
+            'text/yaml': 'yaml',
+        }
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+# Compression formats that are also file formats.
+_COMPRESSION_FILE_FORMATS: set[FileFormat] = {
+    FileFormat.GZ,
+    FileFormat.ZIP,
+}
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+# TODO: Convert to a method on FileFormat or CompressionFormat?
+def infer_file_format_and_compression(
+    value: object,
+    filename: object | None = None,
+) -> tuple[FileFormat | None, CompressionFormat | None]:
+    """
+    Infer data format and compression from a filename, extension, or MIME type.
+
+    Parameters
+    ----------
+    value : object
+        A filename, extension, MIME type, or existing enum member.
+    filename : object | None, optional
+        A filename to consult for extension-based inference (e.g. when
+        ``value`` is ``application/octet-stream``).
+
+    Returns
+    -------
+    tuple[FileFormat | None, CompressionFormat | None]
+        The inferred data format and compression, if any.
+    """
+    if isinstance(value, FileFormat):
+        if value in _COMPRESSION_FILE_FORMATS:
+            return None, CompressionFormat.coerce(value.value)
+        return value, None
+    if isinstance(value, CompressionFormat):
+        return None, value
+
+    text = str(value).strip()
+    if not text:
+        return None, None
+
+    normalized = text.casefold()
+    mime = normalized.split(';', 1)[0].strip()
+
+    is_octet_stream = mime == 'application/octet-stream'
+    compression = CompressionFormat.try_coerce(mime)
+    fmt = None if is_octet_stream else FileFormat.try_coerce(mime)
+
+    is_mime = mime.startswith(
+        (
+            'application/',
+            'text/',
+            'audio/',
+            'image/',
+            'video/',
+            'multipart/',
+        ),
+    )
+    suffix_source: object | None = filename if filename is not None else text
+    if is_mime and filename is None:
+        suffix_source = None
+
+    suffixes = (
+        PurePath(str(suffix_source)).suffixes
+        if suffix_source is not None
+        else []
+    )
+    if suffixes:
+        normalized_suffixes = [suffix.casefold() for suffix in suffixes]
+        compression = (
+            CompressionFormat.try_coerce(normalized_suffixes[-1])
+            or compression
+        )
+        if compression is not None:
+            normalized_suffixes = normalized_suffixes[:-1]
+        if normalized_suffixes:
+            fmt = FileFormat.try_coerce(normalized_suffixes[-1]) or fmt
+
+    if fmt in _COMPRESSION_FILE_FORMATS:
+        compression = compression or CompressionFormat.coerce(fmt.value)
+        fmt = None
+
+    return fmt, compression

etlplus/file/feather.py

@@ -0,0 +1,59 @@
+"""
+:mod:`etlplus.file.feather` module.
+
+Stub helpers for FEATHER read/write.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..types import JSONData
+
+# SECTION: EXPORTS ========================================================== #
+
+
+def read(path: Path) -> JSONData:
+    """
+    Read FEATHER content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the FEATHER file on disk.
+
+    Returns
+    -------
+    JSONData
+        Parsed payload.
+
+    Raises
+    ------
+    NotImplementedError
+        FEATHER :func:`read` is not implemented yet.
+    """
+    raise NotImplementedError('FEATHER read is not implemented yet')
+
+
+def write(path: Path, data: JSONData) -> int:
+    """
+    Write ``data`` to FEATHER at ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the FEATHER file on disk.
+    data : JSONData
+        Data to write.
+
+    Returns
+    -------
+    int
+        Number of records written.
+
+    Raises
+    ------
+    NotImplementedError
+        FEATHER :func:`write` is not implemented yet.
+    """
+    raise NotImplementedError('FEATHER write is not implemented yet')