etlplus 0.9.1__py3-none-any.whl → 0.9.2__py3-none-any.whl

etlplus/file/README.md

@@ -0,0 +1,105 @@
+# `etlplus.file` Subpackage
+
+Documentation for the `etlplus.file` subpackage: unified file format support and helpers for reading
+and writing data files.
+
+- Provides a consistent interface for reading and writing files in various formats
+- Supports all formats defined in `FileFormat` (see below)
+- Includes helpers for inferring file format and compression from filenames, extensions, or MIME
+  types
+- Exposes a `File` class with instance methods for reading and writing data
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [`etlplus.file` Subpackage](#etlplusfile-subpackage)
+  - [Supported File Formats](#supported-file-formats)
+  - [Inferring File Format and Compression](#inferring-file-format-and-compression)
+  - [Reading and Writing Files](#reading-and-writing-files)
+    - [Reading a File](#reading-a-file)
+    - [Writing a File](#writing-a-file)
+  - [File Instance Methods](#file-instance-methods)
+  - [Example: Reading and Writing](#example-reading-and-writing)
+  - [See Also](#see-also)
+
+## Supported File Formats
+
+The following formats are defined in `FileFormat` and supported for reading and writing:
+
+| Format    | Description                                 |
+|-----------|---------------------------------------------|
+| avro      | Apache Avro binary serialization            |
+| csv       | Comma-separated values text files           |
+| feather   | Apache Arrow Feather columnar format        |
+| gz        | Gzip-compressed files (see Compression)     |
+| json      | Standard JSON files                         |
+| ndjson    | Newline-delimited JSON (JSON Lines)         |
+| orc       | Apache ORC columnar format                  |
+| parquet   | Apache Parquet columnar format              |
+| tsv       | Tab-separated values text files             |
+| txt       | Plain text files                            |
+| xls       | Microsoft Excel (legacy .xls)               |
+| xlsx      | Microsoft Excel (modern .xlsx)              |
+| zip       | ZIP-compressed files (see Compression)      |
+| xml       | XML files                                   |
+| yaml      | YAML files                                  |
+
+Compression formats (gz, zip) are also supported as wrappers for other formats.
+
+## Inferring File Format and Compression
+
+Use `infer_file_format_and_compression(value, filename=None)` to infer the file format and
+compression from a filename, extension, or MIME type. Returns a tuple `(file_format,
+compression_format)`.
+
+## Reading and Writing Files
+
+The main entry point for file operations is the `File` class. To read or write files:
+
+### Reading a File
+
+```python
+from etlplus.file import File
+
+f = File("data/sample.csv")
+data = f.read()
+```
+
+- The `read()` method automatically detects the format and compression.
+- Returns parsed data (e.g., list of dicts for tabular formats).
+
+### Writing a File
+
+```python
+from etlplus.file import File
+
+f = File("output.json")
+f.write(data)
+```
+
+- The `write()` method serializes and writes data in the appropriate format.
+- Supports all formats listed above.
+
+## File Instance Methods
+
+- `read()`: Reads and parses the file, returning structured data.
+- `write(data)`: Writes structured data to the file in the detected format.
+
+## Example: Reading and Writing
+
+```python
+from etlplus.file import File
+
+# Read CSV
+csv_file = File("data.csv")
+rows = csv_file.read()
+
+# Write JSON
+json_file = File("output.json")
+json_file.write(rows)
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- File format enums in [enums.py](enums.py)
+- Compression format enums in [enums.py](enums.py)

etlplus/file/__init__.py

@@ -0,0 +1,25 @@
+"""
+:mod:`etlplus.file` package.
+
+Public file IO helpers.
+"""
+
+from __future__ import annotations
+
+from .core import File
+from .enums import CompressionFormat
+from .enums import FileFormat
+from .enums import infer_file_format_and_compression
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Class
+    'File',
+    # Enums
+    'CompressionFormat',
+    'FileFormat',
+    # Functions
+    'infer_file_format_and_compression',
+]

etlplus/file/_imports.py

@@ -0,0 +1,141 @@
+"""
+:mod:`etlplus.file._imports` module.
+
+Shared helpers for optional dependency imports.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import Any
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+_MODULE_CACHE: dict[str, Any] = {}
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _error_message(
+    module_name: str,
+    format_name: str,
+) -> str:
+    """
+    Build an import error message for an optional dependency.
+
+    Parameters
+    ----------
+    module_name : str
+        Module name to look up.
+    format_name : str
+        Human-readable format name for templated messages.
+
+    Returns
+    -------
+    str
+        Formatted error message.
+    """
+    return (
+        f'{format_name} support requires '
+        f'optional dependency "{module_name}".\n'
+        f'Install with: pip install {module_name}'
+    )
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def get_optional_module(
+    module_name: str,
+    *,
+    error_message: str,
+) -> Any:
+    """
+    Return an optional dependency module, caching on first import.
+
+    Parameters
+    ----------
+    module_name : str
+        Name of the module to import.
+    error_message : str
+        Error message to surface when the module is missing.
+
+    Returns
+    -------
+    Any
+        The imported module.
+
+    Raises
+    ------
+    ImportError
+        If the optional dependency is missing.
+    """
+    cached = _MODULE_CACHE.get(module_name)
+    if cached is not None:  # pragma: no cover - tiny branch
+        return cached
+    try:
+        module = import_module(module_name)
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(error_message) from e
+    _MODULE_CACHE[module_name] = module
+    return module
+
+
+def get_fastavro() -> Any:
+    """
+    Return the fastavro module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+
+    Notes
+    -----
+    Prefer :func:`get_optional_module` for new call sites.
+    """
+    return get_optional_module(
+        'fastavro',
+        error_message=_error_message('fastavro', format_name='AVRO'),
+    )
+
+
+def get_pandas(
+    format_name: str,
+) -> Any:
+    """
+    Return the pandas module, importing it on first use.
+
+    Parameters
+    ----------
+    format_name : str
+        Human-readable format name for error messages.
+
+    Returns
+    -------
+    Any
+        The pandas module.
+
+    Notes
+    -----
+    Prefer :func:`get_optional_module` for new call sites.
+    """
+    return get_optional_module(
+        'pandas',
+        error_message=_error_message('pandas', format_name=format_name),
+    )
+
+
+def get_yaml() -> Any:
+    """
+    Return the PyYAML module, importing it on first use.
+
+    Raises an informative ImportError if the optional dependency is missing.
+
+    Notes
+    -----
+    Prefer :func:`get_optional_module` for new call sites.
+    """
+    return get_optional_module(
+        'yaml',
+        error_message=_error_message('PyYAML', format_name='YAML'),
+    )

etlplus/file/_io.py

@@ -0,0 +1,160 @@
+"""
+:mod:`etlplus.file._io` module.
+
+Shared helpers for record normalization and delimited text formats.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def coerce_record_payload(
+    payload: Any,
+    *,
+    format_name: str,
+) -> JSONData:
+    """
+    Validate that ``payload`` is an object or list of objects.
+
+    Parameters
+    ----------
+    payload : Any
+        Parsed payload to validate.
+    format_name : str
+        Human-readable format name for error messages.
+
+    Returns
+    -------
+    JSONData
+        ``payload`` when it is a dict or a list of dicts.
+
+    Raises
+    ------
+    TypeError
+        If the payload is not a dict or list of dicts.
+    """
+    if isinstance(payload, dict):
+        return cast(JSONDict, payload)
+    if isinstance(payload, list):
+        if all(isinstance(item, dict) for item in payload):
+            return cast(JSONList, payload)
+        raise TypeError(
+            f'{format_name} array must contain only objects (dicts)',
+        )
+    raise TypeError(
+        f'{format_name} root must be an object or an array of objects',
+    )
+
+
+def normalize_records(
+    data: JSONData,
+    format_name: str,
+) -> JSONList:
+    """
+    Normalize payloads into a list of dictionaries.
+
+    Parameters
+    ----------
+    data : JSONData
+        Input payload to normalize.
+    format_name : str
+        Human-readable format name for error messages.
+
+    Returns
+    -------
+    JSONList
+        Normalized list of dictionaries.
+
+    Raises
+    ------
+    TypeError
+        If a list payload contains non-dict items.
+    """
+    if isinstance(data, list):
+        if not all(isinstance(item, dict) for item in data):
+            raise TypeError(
+                f'{format_name} payloads must contain only objects (dicts)',
+            )
+        return cast(JSONList, data)
+    return [cast(JSONDict, data)]
+
+
+def read_delimited(path: Path, *, delimiter: str) -> JSONList:
+    """
+    Read delimited content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the delimited file on disk.
+    delimiter : str
+        Delimiter character for parsing.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the delimited file.
+    """
+    with path.open('r', encoding='utf-8', newline='') as handle:
+        reader: csv.DictReader[str] = csv.DictReader(
+            handle,
+            delimiter=delimiter,
+        )
+        rows: JSONList = []
+        for row in reader:
+            if not any(row.values()):
+                continue
+            rows.append(cast(JSONDict, dict(row)))
+    return rows
+
+
+def write_delimited(path: Path, data: JSONData, *, delimiter: str) -> int:
+    """
+    Write ``data`` to a delimited file and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the delimited file on disk.
+    data : JSONData
+        Data to write as delimited rows.
+    delimiter : str
+        Delimiter character for writing.
+
+    Returns
+    -------
+    int
+        The number of rows written.
+    """
+    rows: list[JSONDict]
+    if isinstance(data, list):
+        rows = [row for row in data if isinstance(row, dict)]
+    else:
+        rows = [data]
+
+    if not rows:
+        return 0
+
+    fieldnames = sorted({key for row in rows for key in row})
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('w', encoding='utf-8', newline='') as handle:
+        writer = csv.DictWriter(
+            handle,
+            fieldnames=fieldnames,
+            delimiter=delimiter,
+        )
+        writer.writeheader()
+        for row in rows:
+            writer.writerow({field: row.get(field) for field in fieldnames})
+
+    return len(rows)

etlplus/file/accdb.py

@@ -0,0 +1,78 @@
+"""
+:mod:`etlplus.file.accdb` module.
+
+Helpers for reading/writing newer Microsoft Access database (ACCDB) files.
+
+Notes
+-----
+- An ACCDB file is a proprietary database file format used by Microsoft Access
+    2007 and later.
+- Common cases:
+    - Storing relational data for small to medium-sized applications.
+    - Desktop database applications.
+    - Data management for non-enterprise solutions.
+- Rule of thumb:
+    - If the file follows the ACCDB specification, use this module for reading
+        and writing.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..types import JSONData
+from ..types import JSONList
+from . import stub
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read ACCDB content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the ACCDB file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the ACCDB file.
+    """
+    return stub.read(path, format_name='ACCDB')
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to ACCDB at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the ACCDB file on disk.
+    data : JSONData
+        Data to write as ACCDB. Should be a list of dictionaries or a single
+        dictionary.
+
+    Returns
+    -------
+    int
+        The number of rows written to the ACCDB file.
+    """
+    return stub.write(path, data, format_name='ACCDB')

etlplus/file/arrow.py

@@ -0,0 +1,78 @@
+"""
+:mod:`etlplus.file.arrow` module.
+
+Helpers for reading/writing Apache Arrow (ARROW) files.
+
+Notes
+-----
+- An ARROW file is a binary file format designed for efficient
+    columnar data storage and processing.
+- Common cases:
+    - High-performance data analytics.
+    - Interoperability between different data processing systems.
+    - In-memory data representation for fast computations.
+- Rule of thumb:
+    - If the file follows the Apache Arrow specification, use this module for
+        reading and writing.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..types import JSONData
+from ..types import JSONList
+from . import stub
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read ARROW content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the Apache Arrow file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the Apache Arrow file.
+    """
+    return stub.read(path, format_name='ARROW')
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to ARROW at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the ARROW file on disk.
+    data : JSONData
+        Data to write as ARROW. Should be a list of dictionaries or a
+        single dictionary.
+
+    Returns
+    -------
+    int
+        The number of rows written to the ARROW file.
+    """
+    return stub.write(path, data, format_name='ARROW')