etlplus 0.9.2__py3-none-any.whl → 0.10.2__py3-none-any.whl

etlplus/enums.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import enum
 import operator as _op
+from pathlib import PurePath
 from statistics import fmean
 from typing import Self
 
@@ -22,9 +23,18 @@ __all__ = [
     # Enums
     'AggregateName',
     'CoercibleStrEnum',
+    'CompressionFormat',
     'DataConnectorType',
+    'FileFormat',
+    'HttpMethod',
     'OperatorName',
     'PipelineStep',
+    # Functions
+    'coerce_compression_format',
+    'coerce_data_connector_type',
+    'coerce_file_format',
+    'coerce_http_method',
+    'infer_file_format_and_compression',
 ]
 
 
@@ -168,6 +178,39 @@ class AggregateName(CoercibleStrEnum):
         return lambda xs, n: (fmean(xs) if xs else 0.0)
 
 
+class CompressionFormat(CoercibleStrEnum):
+    """Supported compression formats for data files."""
+
+    # -- Constants -- #
+
+    GZ = 'gz'
+    ZIP = 'zip'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            # File extensions
+            '.gz': 'gz',
+            '.gzip': 'gz',
+            '.zip': 'zip',
+            # MIME types
+            'application/gzip': 'gz',
+            'application/x-gzip': 'gz',
+            'application/zip': 'zip',
+            'application/x-zip-compressed': 'zip',
+        }
+
+
 class DataConnectorType(CoercibleStrEnum):
     """Supported data connector types."""
 
@@ -199,6 +242,119 @@ class DataConnectorType(CoercibleStrEnum):
         }
 
 
+class FileFormat(CoercibleStrEnum):
+    """Supported file formats for extraction."""
+
+    # -- Constants -- #
+
+    AVRO = 'avro'
+    CSV = 'csv'
+    FEATHER = 'feather'
+    GZ = 'gz'
+    JSON = 'json'
+    NDJSON = 'ndjson'
+    ORC = 'orc'
+    PARQUET = 'parquet'
+    TSV = 'tsv'
+    TXT = 'txt'
+    XLS = 'xls'
+    XLSX = 'xlsx'
+    ZIP = 'zip'
+    XML = 'xml'
+    YAML = 'yaml'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            # Common shorthand
+            'parq': 'parquet',
+            'yml': 'yaml',
+            # File extensions
+            '.avro': 'avro',
+            '.csv': 'csv',
+            '.feather': 'feather',
+            '.gz': 'gz',
+            '.json': 'json',
+            '.jsonl': 'ndjson',
+            '.ndjson': 'ndjson',
+            '.orc': 'orc',
+            '.parquet': 'parquet',
+            '.pq': 'parquet',
+            '.tsv': 'tsv',
+            '.txt': 'txt',
+            '.xls': 'xls',
+            '.xlsx': 'xlsx',
+            '.zip': 'zip',
+            '.xml': 'xml',
+            '.yaml': 'yaml',
+            '.yml': 'yaml',
+            # MIME types
+            'application/avro': 'avro',
+            'application/feather': 'feather',
+            'application/gzip': 'gz',
+            'application/json': 'json',
+            'application/jsonlines': 'ndjson',
+            'application/ndjson': 'ndjson',
+            'application/orc': 'orc',
+            'application/vnd.apache.arrow.file': 'feather',
+            'application/vnd.apache.orc': 'orc',
+            'application/vnd.ms-excel': 'xls',
+            (
+                'application/vnd.openxmlformats-'
+                'officedocument.spreadsheetml.sheet'
+            ): 'xlsx',
+            'application/x-avro': 'avro',
+            'application/x-ndjson': 'ndjson',
+            'application/x-parquet': 'parquet',
+            'application/xml': 'xml',
+            'application/zip': 'zip',
+            'text/csv': 'csv',
+            'text/plain': 'txt',
+            'text/tab-separated-values': 'tsv',
+        }
+
+
+class HttpMethod(CoercibleStrEnum):
+    """Supported HTTP verbs that accept JSON payloads."""
+
+    # -- Constants -- #
+
+    CONNECT = 'connect'
+    DELETE = 'delete'
+    GET = 'get'
+    HEAD = 'head'
+    OPTIONS = 'options'
+    PATCH = 'patch'
+    POST = 'post'
+    PUT = 'put'
+    TRACE = 'trace'
+
+    # -- Getters -- #
+
+    @property
+    def allows_body(self) -> bool:
+        """
+        Whether the method typically allows a request body.
+
+        Notes
+        -----
+        - RFCs do not strictly forbid bodies on some other methods (e.g.,
+          ``DELETE``), but many servers/clients do not expect them. We mark
+          ``POST``, ``PUT``, and ``PATCH`` as True.
+        """
+        return self in {HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH}
+
+
 class OperatorName(CoercibleStrEnum):
     """Supported comparison operators with helpers."""
 
@@ -298,6 +454,13 @@ class PipelineStep(CoercibleStrEnum):
 # SECTION: INTERNAL CONSTANTS ============================================== #
 
 
+# Compression formats that are also file formats.
+_COMPRESSION_FILE_FORMATS: set[FileFormat] = {
+    FileFormat.GZ,
+    FileFormat.ZIP,
+}
+
+
 # Precomputed order index for PipelineStep; avoids recomputing on each access.
 _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
     PipelineStep.FILTER: 0,
@@ -306,3 +469,128 @@ _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
     PipelineStep.SORT: 3,
     PipelineStep.AGGREGATE: 4,
 }
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def coerce_data_connector_type(
+    connector: DataConnectorType | str,
+) -> DataConnectorType:
+    """
+    Normalize textual data connector values to :class:`DataConnectorType`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`DataConnectorType.coerce` going forward.
+    """
+    return DataConnectorType.coerce(connector)
+
+
+def coerce_file_format(
+    file_format: FileFormat | str,
+) -> FileFormat:
+    """
+    Normalize textual file format values to :class:`FileFormat`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`FileFormat.coerce` going forward.
+    """
+    return FileFormat.coerce(file_format)
+
+
+def coerce_compression_format(
+    compression_format: CompressionFormat | str,
+) -> CompressionFormat:
+    """
+    Normalize textual compression format values to :class:`CompressionFormat`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`CompressionFormat.coerce` going forward.
+    """
+    return CompressionFormat.coerce(compression_format)
+
+
+def coerce_http_method(
+    http_method: HttpMethod | str,
+) -> HttpMethod:
+    """
+    Normalize textual HTTP method values to :class:`HttpMethod`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`HttpMethod.coerce` going forward.
+    """
+    return HttpMethod.coerce(http_method)
+
+
+def infer_file_format_and_compression(
+    value: object,
+    filename: object | None = None,
+) -> tuple[FileFormat | None, CompressionFormat | None]:
+    """
+    Infer data format and compression from a filename, extension, or MIME type.
+
+    Parameters
+    ----------
+    value : object
+        A filename, extension, MIME type, or existing enum member.
+    filename : object | None, optional
+        A filename to consult for extension-based inference (e.g. when
+        ``value`` is ``application/octet-stream``).
+
+    Returns
+    -------
+    tuple[FileFormat | None, CompressionFormat | None]
+        The inferred data format and compression, if any.
+    """
+    if isinstance(value, FileFormat):
+        if value in _COMPRESSION_FILE_FORMATS:
+            return None, CompressionFormat.coerce(value.value)
+        return value, None
+    if isinstance(value, CompressionFormat):
+        return None, value
+
+    text = str(value).strip()
+    if not text:
+        return None, None
+
+    normalized = text.casefold()
+    mime = normalized.split(';', 1)[0].strip()
+
+    compression = CompressionFormat.try_coerce(mime)
+    fmt = FileFormat.try_coerce(mime)
+
+    is_mime = mime.startswith(
+        (
+            'application/',
+            'text/',
+            'audio/',
+            'image/',
+            'video/',
+            'multipart/',
+        ),
+    )
+    suffix_source: object | None = filename if filename is not None else text
+    if is_mime and filename is None:
+        suffix_source = None
+
+    suffixes = (
+        PurePath(str(suffix_source)).suffixes
+        if suffix_source is not None
+        else []
+    )
+    if suffixes:
+        normalized_suffixes = [suffix.casefold() for suffix in suffixes]
+        compression = (
+            CompressionFormat.try_coerce(normalized_suffixes[-1])
+            or compression
+        )
+        if compression is not None:
+            normalized_suffixes = normalized_suffixes[:-1]
+        if normalized_suffixes:
+            fmt = FileFormat.try_coerce(normalized_suffixes[-1]) or fmt
+
+    if fmt in _COMPRESSION_FILE_FORMATS:
+        compression = compression or CompressionFormat.coerce(fmt.value)
+        fmt = None
+
+    return fmt, compression

etlplus/{ops/extract.py → extract.py}

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.ops.extract` module.
+:mod:`etlplus.extract` module.
 
 Helpers to extract data from files, databases, and REST APIs.
 """
@@ -10,81 +10,58 @@ from pathlib import Path
 from typing import Any
 from typing import cast
 
-from ..api import HttpMethod
-from ..api.utils import resolve_request
-from ..enums import DataConnectorType
-from ..file import File
-from ..file import FileFormat
-from ..types import JSONData
-from ..types import JSONDict
-from ..types import JSONList
-from ..types import StrPath
+import requests  # type: ignore[import]
+
+from .enums import DataConnectorType
+from .enums import FileFormat
+from .enums import HttpMethod
+from .enums import coerce_data_connector_type
+from .enums import coerce_file_format
+from .file import File
+from .types import JSONData
+from .types import JSONDict
+from .types import JSONList
+from .types import StrPath
 
 # SECTION: FUNCTIONS ======================================================== #
 
 
-def extract_from_api(
-    url: str,
-    method: HttpMethod | str = HttpMethod.GET,
-    **kwargs: Any,
+# -- File Extraction -- #
+
+
+def extract_from_file(
+    file_path: StrPath,
+    file_format: FileFormat | str | None = FileFormat.JSON,
 ) -> JSONData:
     """
-    Extract data from a REST API.
+    Extract (semi-)structured data from a local file.
 
     Parameters
     ----------
-    url : str
-        API endpoint URL.
-    method : HttpMethod | str, optional
-        HTTP method to use. Defaults to ``GET``.
-    **kwargs : Any
-        Extra arguments forwarded to the underlying ``requests`` call
-        (for example, ``timeout``). To use a pre-configured
-        :class:`requests.Session`, provide it via ``session``.
-        When omitted, ``timeout`` defaults to 10 seconds.
+    file_path : StrPath
+        Source file path.
+    file_format : FileFormat | str | None, optional
+        File format to parse. If ``None``, infer from the filename
+        extension. Defaults to `'json'` for backward compatibility when
+        explicitly provided.
 
     Returns
     -------
     JSONData
-        Parsed JSON payload, or a fallback object with raw text.
-
-    Raises
-    ------
-    TypeError
-        If a provided ``session`` does not expose the required HTTP
-        method (for example, ``get``).
+        Parsed data as a mapping or a list of mappings.
     """
-    timeout = kwargs.pop('timeout', None)
-    session = kwargs.pop('session', None)
-    request_callable, timeout, _ = resolve_request(
-        method,
-        session=session,
-        timeout=timeout,
-    )
-    response = request_callable(url, timeout=timeout, **kwargs)
-    response.raise_for_status()
+    path = Path(file_path)
 
-    content_type = response.headers.get('content-type', '').lower()
-    if 'application/json' in content_type:
-        try:
-            payload: Any = response.json()
-        except ValueError:
-            # Malformed JSON despite content-type; fall back to text
-            return {
-                'content': response.text,
-                'content_type': content_type,
-            }
-        if isinstance(payload, dict):
-            return cast(JSONDict, payload)
-        if isinstance(payload, list):
-            if all(isinstance(x, dict) for x in payload):
-                return cast(JSONList, payload)
-            # Coerce non-dict array items into objects for consistency
-            return [{'value': x} for x in payload]
-        # Fallback: wrap scalar JSON
-        return {'value': payload}
+    # If no explicit format is provided, let File infer from extension.
+    if file_format is None:
+        return File(path, None).read()
+    fmt = coerce_file_format(file_format)
 
-    return {'content': response.text, 'content_type': content_type}
+    # Let file module perform existence and format validation.
+    return File(path, fmt).read()
+
+
+# -- Database Extraction (Placeholder) -- #
 
 
 def extract_from_database(
@@ -119,36 +96,77 @@ def extract_from_database(
     ]
 
 
-def extract_from_file(
-    file_path: StrPath,
-    file_format: FileFormat | str | None = FileFormat.JSON,
+# -- REST API Extraction -- #
+
+
+def extract_from_api(
+    url: str,
+    method: HttpMethod | str = HttpMethod.GET,
+    **kwargs: Any,
 ) -> JSONData:
     """
-    Extract (semi-)structured data from a local file.
+    Extract data from a REST API.
 
     Parameters
     ----------
-    file_path : StrPath
-        Source file path.
-    file_format : FileFormat | str | None, optional
-        File format to parse. If ``None``, infer from the filename
-        extension. Defaults to `'json'` for backward compatibility when
-        explicitly provided.
+    url : str
+        API endpoint URL.
+    method : HttpMethod | str, optional
+        HTTP method to use. Defaults to ``GET``.
+    **kwargs : Any
+        Extra arguments forwarded to the underlying ``requests`` call
+        (for example, ``timeout``). To use a pre-configured
+        :class:`requests.Session`, provide it via ``session``.
 
     Returns
     -------
     JSONData
-        Parsed data as a mapping or a list of mappings.
+        Parsed JSON payload, or a fallback object with raw text.
+
+    Raises
+    ------
+    TypeError
+        If a provided ``session`` does not expose the required HTTP
+        method (for example, ``get``).
     """
-    path = Path(file_path)
+    http_method = HttpMethod.coerce(method)
 
-    # If no explicit format is provided, let File infer from extension.
-    if file_format is None:
-        return File(path, None).read()
-    fmt = FileFormat.coerce(file_format)
+    # Apply a conservative timeout to guard against hanging requests.
+    timeout = kwargs.pop('timeout', 10.0)
+    session = kwargs.pop('session', None)
+    requester = session or requests
 
-    # Let file module perform existence and format validation.
-    return File(path, fmt).read()
+    request_callable = getattr(requester, http_method.value, None)
+    if not callable(request_callable):
+        raise TypeError(
+            'Session object must supply a callable'
+            f'"{http_method.value}" method',
+        )
+
+    response = request_callable(url, timeout=timeout, **kwargs)
+    response.raise_for_status()
+
+    content_type = response.headers.get('content-type', '').lower()
+    if 'application/json' in content_type:
+        try:
+            payload: Any = response.json()
+        except ValueError:
+            # Malformed JSON despite content-type; fall back to text
+            return {
+                'content': response.text,
+                'content_type': content_type,
+            }
+        if isinstance(payload, dict):
+            return cast(JSONDict, payload)
+        if isinstance(payload, list):
+            if all(isinstance(x, dict) for x in payload):
+                return cast(JSONList, payload)
+            # Coerce non-dict array items into objects for consistency
+            return [{'value': x} for x in payload]
+        # Fallback: wrap scalar JSON
+        return {'value': payload}
+
+    return {'content': response.text, 'content_type': content_type}
 
 
 # -- Orchestration -- #
@@ -184,7 +202,7 @@ def extract(
     ValueError
         If `source_type` is not one of the supported values.
     """
-    match DataConnectorType.coerce(source_type):
+    match coerce_data_connector_type(source_type):
         case DataConnectorType.FILE:
             # Prefer explicit format if provided, else infer from filename.
             return extract_from_file(source, file_format)
@@ -195,6 +213,6 @@ def extract(
             # ``file_format`` is ignored for APIs.
             return extract_from_api(str(source), **kwargs)
         case _:
-            # :meth:`coerce` already raises for invalid connector types, but
-            # keep explicit guard for defensive programming.
+            # ``coerce_data_connector_type`` covers invalid entries, but keep
+            # explicit guard for defensive programming.
             raise ValueError(f'Invalid source type: {source_type}')