etlplus 0.9.0__py3-none-any.whl → 0.11.5__py3-none-any.whl

etlplus/cli/commands.py

@@ -36,7 +36,7 @@ from typing import cast
 import typer
 
 from .. import __version__
-from ..enums import FileFormat
+from ..file import FileFormat
 from . import handlers
 from .constants import CLI_DESCRIPTION
 from .constants import CLI_EPILOG
@@ -443,9 +443,9 @@ def extract_cmd(
         Source (JSON payload, file/folder path, URL/URI, or - for STDIN)
         from which to extract data. Default is ``-``.
     source_format : SourceFormatOption, optional
-        Source data format. Overrides the inferred format (``csv``, ``json``,
-        ``parquet``, ``xml``) based on filename extension or STDIN content.
-        Default is ``None``.
+        Data source format. Overrides the inferred format (``csv``, ``json``,
+        etc.) based on filename extension or STDIN content. Default is
+        ``None``.
     source_type : SourceTypeOption, optional
         Data source type. Overrides the inferred type (``api``, ``database``,
         ``file``, ``folder``) based on URI/URL schema. Default is ``None``.
@@ -523,15 +523,15 @@ def load_cmd(
     ctx : typer.Context
         The Typer context.
     source_format : SourceFormatOption, optional
-        Source data format. Overrides the inferred format (``csv``, ``json``,
-        ``parquet``, ``xml``) based on STDIN content. Default is ``None``.
+        Data source format. Overrides the inferred format (``csv``, ``json``,
+        etc.) based on filename extension or STDIN content. Default is
+        ``None``.
     target : TargetArg, optional
         Target (file/folder path, URL/URI, or - for STDOUT) into which to load
         data. Default is ``-``.
     target_format : TargetFormatOption, optional
-        Format of the target data. Overrides the inferred format (``csv``,
-        ``json``, ``parquet``, ``xml``) based on filename extension. Default is
-        ``None``.
+        Target data format. Overrides the inferred format (``csv``, ``json``,
+        etc.) based on filename extension. Default is ``None``.
     target_type : TargetTypeOption, optional
         Data target type. Overrides the inferred type (``api``, ``database``,
         ``file``, ``folder``) based on URI/URL schema. Default is ``None``.
@@ -760,9 +760,9 @@ def transform_cmd(
         Source (JSON payload, file/folder path, URL/URI, or - for STDIN) from
         which to extract data. Default is ``-``.
     source_format : SourceFormatOption, optional
-        Source data format. Overrides the inferred format (``csv``, ``json``,
-        ``parquet``, ``xml``) based on filename extension or STDIN content.
-        Default is ``None``.
+        Data source format. Overrides the inferred format (``csv``, ``json``,
+        etc.) based on filename extension or STDIN content. Default is
+        ``None``.
     source_type : SourceTypeOption, optional
         Data source type. Overrides the inferred type (``api``, ``database``,
         ``file``, ``folder``) based on URI/URL schema. Default is ``None``.
@@ -770,9 +770,8 @@ def transform_cmd(
         Target (file/folder path, URL/URI, or - for STDOUT) into which to load
         data. Default is ``-``.
     target_format : TargetFormatOption, optional
-        Format of the target data. Overrides the inferred format (``csv``,
-        ``json``, ``parquet``, ``xml``) based on filename extension. Default is
-        ``None``.
+        Target data format. Overrides the inferred format (``csv``, ``json``,
+        etc.) based on filename extension. Default is ``None``.
     target_type : TargetTypeOption, optional
         Data target type. Overrides the inferred type (``api``, ``database``,
         ``file``, ``folder``) based on URI/URL schema. Default is ``None``.
@@ -876,11 +875,12 @@ def validate_cmd(
     source : SourceArg
         Data source to validate (path, JSON payload, or - for STDIN).
     source_format : SourceFormatOption, optional
-        Format of the source. Overrides filename-based inference when provided.
-        Default is ``None``.
-    source_type : SourceTypeOption, optional
-        Override the inferred source type (file, database, api). Default is
+        Data source format. Overrides the inferred format (``csv``, ``json``,
+        etc.) based on filename extension or STDIN content. Default is
         ``None``.
+    source_type : SourceTypeOption, optional
+        Data source type. Overrides the inferred type (``api``, ``database``,
+        ``file``, ``folder``) based on URI/URL schema. Default is ``None``.
     output : OutputOption, optional
         Output file for validated output (- for STDOUT). Default is ``None``.
 

etlplus/cli/constants.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 from typing import Final
 
 from ..enums import DataConnectorType
-from ..enums import FileFormat
+from ..file import FileFormat
 
 # SECTION: EXPORTS ========================================================== #
 

etlplus/cli/io.py

@@ -15,8 +15,8 @@ from pathlib import Path
 from typing import Any
 from typing import cast
 
-from ..enums import FileFormat
 from ..file import File
+from ..file import FileFormat
 from ..types import JSONData
 from ..utils import print_json
 
@@ -331,6 +331,6 @@ def write_json_output(
     """
     if not output_path or output_path == '-':
         return False
-    File(Path(output_path), FileFormat.JSON).write_json(data)
+    File(Path(output_path), FileFormat.JSON).write(data)
     print(f'{success_message} {output_path}')
     return True

etlplus/config/pipeline.py

@@ -24,8 +24,8 @@ from typing import Any
 from typing import Self
 
 from ..api import ApiConfig
-from ..enums import FileFormat
 from ..file import File
+from ..file import FileFormat
 from ..types import StrAnyMap
 from ..utils import coerce_dict
 from ..utils import maybe_mapping
@@ -246,7 +246,7 @@ class PipelineConfig:
         TypeError
             If the YAML root is not a mapping/object.
         """
-        raw = File(Path(path), FileFormat.YAML).read_yaml()
+        raw = File(Path(path), FileFormat.YAML).read()
         if not isinstance(raw, dict):
             raise TypeError('Pipeline YAML must have a mapping/object root')
 

etlplus/database/ddl.py

@@ -203,7 +203,7 @@ def load_table_spec(
         raise ValueError('Spec must be .json, .yml, or .yaml')
 
     try:
-        spec = File.read_file(spec_path)
+        spec = File.from_path(spec_path).read()
     except ImportError as e:
         if suffix in {'.yml', '.yaml'}:
             raise RuntimeError(

etlplus/enums.py

@@ -19,16 +19,13 @@ from .types import StrStrMap
 
 
 __all__ = [
+    # Enums
     'AggregateName',
     'CoercibleStrEnum',
     'DataConnectorType',
-    'FileFormat',
     'HttpMethod',
     'OperatorName',
     'PipelineStep',
-    'coerce_data_connector_type',
-    'coerce_file_format',
-    'coerce_http_method',
 ]
 
 
@@ -203,38 +200,6 @@ class DataConnectorType(CoercibleStrEnum):
         }
 
 
-class FileFormat(CoercibleStrEnum):
-    """Supported file formats for extraction."""
-
-    # -- Constants -- #
-
-    CSV = 'csv'
-    JSON = 'json'
-    XML = 'xml'
-    YAML = 'yaml'
-
-    # -- Class Methods -- #
-
-    @classmethod
-    def aliases(cls) -> StrStrMap:
-        """
-        Return a mapping of common aliases for each enum member.
-
-        Returns
-        -------
-        StrStrMap
-            A mapping of alias names to their corresponding enum member names.
-        """
-        return {
-            # Common shorthand
-            'yml': 'yaml',
-            # MIME types
-            'text/csv': 'csv',
-            'application/json': 'json',
-            'application/xml': 'xml',
-        }
-
-
 class HttpMethod(CoercibleStrEnum):
     """Supported HTTP verbs that accept JSON payloads."""
 
@@ -260,8 +225,8 @@ class HttpMethod(CoercibleStrEnum):
         Notes
         -----
         - RFCs do not strictly forbid bodies on some other methods (e.g.,
-          ``DELETE``), but many servers/clients do not expect them. We mark
-          ``POST``, ``PUT``, and ``PATCH`` as True.
+            ``DELETE``), but many servers/clients do not expect them. We mark
+            ``POST``, ``PUT``, and ``PATCH`` as True.
         """
         return self in {HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH}
 
@@ -373,42 +338,3 @@ _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
     PipelineStep.SORT: 3,
     PipelineStep.AGGREGATE: 4,
 }
-
-
-# SECTION: FUNCTIONS ======================================================== #
-
-
-def coerce_data_connector_type(
-    connector: DataConnectorType | str,
-) -> DataConnectorType:
-    """
-    Normalize textual data connector values to :class:`DataConnectorType`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`DataConnectorType.coerce` going forward.
-    """
-    return DataConnectorType.coerce(connector)
-
-
-def coerce_file_format(
-    file_format: FileFormat | str,
-) -> FileFormat:
-    """
-    Normalize textual file format values to :class:`FileFormat`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`FileFormat.coerce` going forward.
-    """
-    return FileFormat.coerce(file_format)
-
-
-def coerce_http_method(
-    http_method: HttpMethod | str,
-) -> HttpMethod:
-    """
-    Normalize textual HTTP method values to :class:`HttpMethod`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`HttpMethod.coerce` going forward.
-    """
-    return HttpMethod.coerce(http_method)

etlplus/extract.py

@@ -13,11 +13,9 @@ from typing import cast
 import requests  # type: ignore[import]
 
 from .enums import DataConnectorType
-from .enums import FileFormat
 from .enums import HttpMethod
-from .enums import coerce_data_connector_type
-from .enums import coerce_file_format
 from .file import File
+from .file import FileFormat
 from .types import JSONData
 from .types import JSONDict
 from .types import JSONList
@@ -55,7 +53,7 @@ def extract_from_file(
     # If no explicit format is provided, let File infer from extension.
     if file_format is None:
         return File(path, None).read()
-    fmt = coerce_file_format(file_format)
+    fmt = FileFormat.coerce(file_format)
 
     # Let file module perform existence and format validation.
     return File(path, fmt).read()
@@ -202,7 +200,7 @@ def extract(
     ValueError
         If `source_type` is not one of the supported values.
     """
-    match coerce_data_connector_type(source_type):
+    match DataConnectorType.coerce(source_type):
         case DataConnectorType.FILE:
             # Prefer explicit format if provided, else infer from filename.
             return extract_from_file(source, file_format)
@@ -213,6 +211,6 @@ def extract(
             # ``file_format`` is ignored for APIs.
             return extract_from_api(str(source), **kwargs)
         case _:
-            # ``coerce_data_connector_type`` covers invalid entries, but keep
-            # explicit guard for defensive programming.
+            # :meth:`coerce` already raises for invalid connector types, but
+            # keep explicit guard for defensive programming.
             raise ValueError(f'Invalid source type: {source_type}')

etlplus/file/__init__.py

@@ -0,0 +1,25 @@
+"""
+:mod:`etlplus.file` package.
+
+Public file IO helpers.
+"""
+
+from __future__ import annotations
+
+from .core import File
+from .enums import CompressionFormat
+from .enums import FileFormat
+from .enums import infer_file_format_and_compression
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Class
+    'File',
+    # Enums
+    'CompressionFormat',
+    'FileFormat',
+    # Functions
+    'infer_file_format_and_compression',
+]

etlplus/file/core.py

@@ -0,0 +1,287 @@
+"""
+:mod:`etlplus.file.core` module.
+
+Shared helpers for reading and writing structured and semi-structured data
+files.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..types import JSONData
+from ..types import StrPath
+from . import csv
+from . import json
+from . import xml
+from . import yaml
+from .enums import FileFormat
+from .enums import infer_file_format_and_compression
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = ['File']
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+@dataclass(slots=True)
+class File:
+    """
+    Convenience wrapper around structured file IO.
+
+    This class encapsulates the one-off helpers in this module as convenient
+    instance methods while retaining the original function API for
+    backward compatibility (those functions delegate to this class).
+
+    Attributes
+    ----------
+    path : Path
+        Path to the file on disk.
+    file_format : FileFormat | None, optional
+        Explicit format. If omitted, the format is inferred from the file
+        extension (``.csv``, ``.json``, or ``.xml``).
+    """
+
+    # -- Attributes -- #
+
+    path: Path
+    file_format: FileFormat | None = None
+
+    # -- Magic Methods (Object Lifecycle) -- #
+
+    def __post_init__(self) -> None:
+        """
+        Auto-detect and set the file format on initialization.
+
+        If no explicit ``file_format`` is provided, attempt to infer it from
+        the file path's extension and update :attr:`file_format`. If the
+        extension is unknown, the attribute is left as ``None`` and will be
+        validated later by :meth:`_ensure_format`.
+        """
+        # Normalize incoming path (allow str in constructor) to Path.
+        if isinstance(self.path, str):
+            self.path = Path(self.path)
+
+        if self.file_format is None:
+            try:
+                self.file_format = self._guess_format()
+            except ValueError:
+                # Leave as None; _ensure_format() will raise on use if needed.
+                pass
+
+    # -- Internal Instance Methods -- #
+
+    def _assert_exists(self) -> None:
+        """
+        Raise FileNotFoundError if :attr:`path` does not exist.
+
+        This centralizes existence checks across multiple read methods.
+        """
+        if not self.path.exists():
+            raise FileNotFoundError(f'File not found: {self.path}')
+
+    def _ensure_format(self) -> FileFormat:
+        """
+        Resolve the active format, guessing from extension if needed.
+
+        Returns
+        -------
+        FileFormat
+            The resolved file format.
+        """
+        return (
+            self.file_format
+            if self.file_format is not None
+            else self._guess_format()
+        )
+
+    def _guess_format(self) -> FileFormat:
+        """
+        Infer the file format from the filename extension.
+
+        Returns
+        -------
+        FileFormat
+            The inferred file format based on the file extension.
+
+        Raises
+        ------
+        ValueError
+            If the extension is unknown or unsupported.
+        """
+        fmt, compression = infer_file_format_and_compression(self.path)
+        if fmt is not None:
+            return fmt
+        if compression is not None:
+            raise ValueError(
+                'Cannot infer file format from compressed file '
+                f'{self.path!r} with compression {compression.value!r}',
+            )
+        raise ValueError(
+            f'Cannot infer file format from extension {self.path.suffix!r}',
+        )
+
+    # -- Instance Methods (Generic API) -- #
+
+    def read(self) -> JSONData:
+        """
+        Read structured data from :attr:`path` using :attr:`file_format`.
+
+        Returns
+        -------
+        JSONData
+            The structured data read from the file.
+
+        Raises
+        ------
+        ValueError
+            If the resolved file format is unsupported.
+        """
+        self._assert_exists()
+        fmt = self._ensure_format()
+        match fmt:
+            case FileFormat.CSV:
+                return csv.read(self.path)
+            case FileFormat.JSON:
+                return json.read(self.path)
+            case FileFormat.XML:
+                return xml.read(self.path)
+            case FileFormat.YAML:
+                return yaml.read(self.path)
+        raise ValueError(f'Unsupported format: {fmt}')
+
+    def write(
+        self,
+        data: JSONData,
+        *,
+        root_tag: str = xml.DEFAULT_XML_ROOT,
+    ) -> int:
+        """
+        Write ``data`` to :attr:`path` using :attr:`file_format`.
+
+        Parameters
+        ----------
+        data : JSONData
+            Data to write to the file.
+        root_tag : str, optional
+            Root tag name to use when writing XML files. Defaults to
+            ``'root'``.
+
+        Returns
+        -------
+        int
+            The number of records written.
+
+        Raises
+        ------
+        ValueError
+            If the resolved file format is unsupported.
+        """
+        fmt = self._ensure_format()
+        match fmt:
+            case FileFormat.CSV:
+                return csv.write(self.path, data)
+            case FileFormat.JSON:
+                return json.write(self.path, data)
+            case FileFormat.XML:
+                return xml.write(self.path, data, root_tag=root_tag)
+            case FileFormat.YAML:
+                return yaml.write(self.path, data)
+        raise ValueError(f'Unsupported format: {fmt}')
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def from_path(
+        cls,
+        path: StrPath,
+        *,
+        file_format: FileFormat | str | None = None,
+    ) -> File:
+        """
+        Create a :class:`File` from any path-like and optional format.
+
+        Parameters
+        ----------
+        path : StrPath
+            Path to the file on disk.
+        file_format : FileFormat | str | None, optional
+            Explicit format. If omitted, the format is inferred from the file
+            extension (``.csv``, ``.json``, or ``.xml``).
+
+        Returns
+        -------
+        File
+            The constructed :class:`File` instance.
+        """
+        resolved = Path(path)
+        ff: FileFormat | None
+        if isinstance(file_format, str):
+            ff = FileFormat.coerce(file_format)
+        else:
+            ff = file_format
+
+        return cls(resolved, ff)
+
+    @classmethod
+    def read_file(
+        cls,
+        path: StrPath,
+        file_format: FileFormat | str | None = None,
+    ) -> JSONData:
+        """
+        Read structured data.
+
+        Parameters
+        ----------
+        path : StrPath
+            Path to the file on disk.
+        file_format : FileFormat | str | None, optional
+            Explicit format. If omitted, the format is inferred from the file
+            extension (``.csv``, ``.json``, or ``.xml``).
+
+        Returns
+        -------
+        JSONData
+            The structured data read from the file.
+        """
+        return cls.from_path(path, file_format=file_format).read()
+
+    @classmethod
+    def write_file(
+        cls,
+        path: StrPath,
+        data: JSONData,
+        file_format: FileFormat | str | None = None,
+        *,
+        root_tag: str = xml.DEFAULT_XML_ROOT,
+    ) -> int:
+        """
+        Write structured data and count written records.
+
+        Parameters
+        ----------
+        path : StrPath
+            Path to the file on disk.
+        data : JSONData
+            Data to write to the file.
+        file_format : FileFormat | str | None, optional
+            Explicit format. If omitted, the format is inferred from the file
+            extension (``.csv``, ``.json``, or ``.xml``).
+        root_tag : str, optional
+            Root tag name to use when writing XML files. Defaults to
+            ``'root'``.
+
+        Returns
+        -------
+        int
+            The number of records written to the file.
+        """
+        return cls.from_path(path, file_format=file_format).write(
+            data,
+            root_tag=root_tag,
+        )

etlplus/file/csv.py

@@ -0,0 +1,82 @@
+"""
+:mod:`etlplus.file.csv` module.
+
+CSV read/write helpers.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import cast
+
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Load CSV content as a list of dictionaries.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the CSV file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the CSV file.
+    """
+    with path.open('r', encoding='utf-8', newline='') as handle:
+        reader: csv.DictReader[str] = csv.DictReader(handle)
+        rows: JSONList = []
+        for row in reader:
+            if not any(row.values()):
+                continue
+            rows.append(cast(JSONDict, dict(row)))
+    return rows
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write CSV rows to ``path`` and return the number of rows.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the CSV file on disk.
+    data : JSONData
+        Data to write as CSV. Should be a list of dictionaries or a
+        single dictionary.
+
+    Returns
+    -------
+    int
+        The number of rows written to the CSV file.
+    """
+    rows: list[JSONDict]
+    if isinstance(data, list):
+        rows = [row for row in data if isinstance(row, dict)]
+    else:
+        rows = [data]
+
+    if not rows:
+        return 0
+
+    fieldnames = sorted({key for row in rows for key in row})
+    with path.open('w', encoding='utf-8', newline='') as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames)
+        writer.writeheader()
+        for row in rows:
+            writer.writerow({field: row.get(field) for field in fieldnames})
+
+    return len(rows)