etlplus 0.5.4__py3-none-any.whl

etlplus/enums.py

@@ -0,0 +1,414 @@
+"""
+:mod:`etlplus.enums` module.
+
+Shared enumeration types used across ETLPlus modules.
+"""
+
+from __future__ import annotations
+
+import enum
+import operator as _op
+from statistics import fmean
+from typing import Self
+
+from .types import AggregateFunc
+from .types import OperatorFunc
+from .types import StrStrMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'AggregateName',
+    'CoercibleStrEnum',
+    'DataConnectorType',
+    'FileFormat',
+    'HttpMethod',
+    'OperatorName',
+    'PipelineStep',
+    'coerce_data_connector_type',
+    'coerce_file_format',
+    'coerce_http_method',
+]
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+class CoercibleStrEnum(enum.StrEnum):
+    """
+    StrEnum with ergonomic helpers.
+
+    Provides a DRY, class-level :meth:`coerce` that normalizes inputs and
+    produces consistent, informative error messages. Also exposes
+    :meth:`choices` for UI/validation and :meth:`try_coerce` for soft parsing.
+
+    Notes
+    -----
+    - Values are normalized via ``str(value).strip().casefold()``.
+    - Error messages enumerate allowed values for easier debugging.
+    """
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Subclasses may override this method to provide custom aliases.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {}
+
+    @classmethod
+    def choices(cls) -> tuple[str, ...]:
+        """
+        Return the allowed string values for this enum.
+
+        Returns
+        -------
+        tuple[str, ...]
+            A tuple of allowed string values for this enum.
+        """
+        return tuple(member.value for member in cls)
+
+    @classmethod
+    def coerce(cls, value: Self | str | object) -> Self:
+        """
+        Convert an enum member or string-like input to a member of ``cls``.
+
+        Parameters
+        ----------
+        value : Self | str | object
+            An existing enum member or a text value to normalize.
+
+        Returns
+        -------
+        Self
+            The corresponding enum member.
+
+        Raises
+        ------
+        ValueError
+            If the value cannot be coerced into a valid member.
+        """
+        if isinstance(value, cls):
+            return value
+        try:
+            normalized = str(value).strip().casefold()
+            resolved = cls.aliases().get(normalized, normalized)
+            return cls(resolved)  # type: ignore[arg-type]
+        except (ValueError, TypeError) as e:
+            allowed = ', '.join(cls.choices())
+            raise ValueError(
+                f'Invalid {cls.__name__} value: {value!r}. Allowed: {allowed}',
+            ) from e
+
+    @classmethod
+    def try_coerce(
+        cls,
+        value: object,
+    ) -> Self | None:
+        """
+        Best-effort parse; return ``None`` on failure instead of raising.
+
+        Parameters
+        ----------
+        value : object
+            An existing enum member or a text value to normalize.
+
+        Returns
+        -------
+        Self | None
+            The corresponding enum member, or ``None`` if coercion fails.
+        """
+        try:
+            return cls.coerce(value)
+        except ValueError:
+            return None
+
+
+# SECTION: ENUMS ============================================================ #
+
+
+class AggregateName(CoercibleStrEnum):
+    """Supported aggregations with helpers."""
+
+    # -- Constants -- #
+
+    AVG = 'avg'
+    COUNT = 'count'
+    MAX = 'max'
+    MIN = 'min'
+    SUM = 'sum'
+
+    # -- Class Methods -- #
+
+    @property
+    def func(self) -> AggregateFunc:
+        """
+        Get the aggregation function for this aggregation type.
+
+        Returns
+        -------
+        AggregateFunc
+            The aggregation function corresponding to this aggregation type.
+        """
+        if self is AggregateName.COUNT:
+            return lambda xs, n: n
+        if self is AggregateName.MAX:
+            return lambda xs, n: (max(xs) if xs else None)
+        if self is AggregateName.MIN:
+            return lambda xs, n: (min(xs) if xs else None)
+        if self is AggregateName.SUM:
+            return lambda xs, n: sum(xs)
+
+        # AVG
+        return lambda xs, n: (fmean(xs) if xs else 0.0)
+
+
+class DataConnectorType(CoercibleStrEnum):
+    """Supported data connector types."""
+
+    # -- Constants -- #
+
+    API = 'api'
+    DATABASE = 'database'
+    FILE = 'file'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            'http': 'api',
+            'https': 'api',
+            'rest': 'api',
+            'db': 'database',
+            'filesystem': 'file',
+            'fs': 'file',
+        }
+
+
+class FileFormat(CoercibleStrEnum):
+    """Supported file formats for extraction."""
+
+    # -- Constants -- #
+
+    CSV = 'csv'
+    JSON = 'json'
+    XML = 'xml'
+    YAML = 'yaml'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            # Common shorthand
+            'yml': 'yaml',
+            # MIME types
+            'text/csv': 'csv',
+            'application/json': 'json',
+            'application/xml': 'xml',
+        }
+
+
+class HttpMethod(CoercibleStrEnum):
+    """Supported HTTP verbs that accept JSON payloads."""
+
+    # -- Constants -- #
+
+    CONNECT = 'connect'
+    DELETE = 'delete'
+    GET = 'get'
+    HEAD = 'head'
+    OPTIONS = 'options'
+    PATCH = 'patch'
+    POST = 'post'
+    PUT = 'put'
+    TRACE = 'trace'
+
+    # -- Getters -- #
+
+    @property
+    def allows_body(self) -> bool:
+        """
+        Whether the method typically allows a request body.
+
+        Notes
+        -----
+        - RFCs do not strictly forbid bodies on some other methods (e.g.,
+          ``DELETE``), but many servers/clients do not expect them. We mark
+          ``POST``, ``PUT``, and ``PATCH`` as True.
+        """
+        return self in {HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH}
+
+
+class OperatorName(CoercibleStrEnum):
+    """Supported comparison operators with helpers."""
+
+    # -- Constants -- #
+
+    EQ = 'eq'
+    NE = 'ne'
+    GT = 'gt'
+    GTE = 'gte'
+    LT = 'lt'
+    LTE = 'lte'
+    IN = 'in'
+    CONTAINS = 'contains'
+
+    # -- Getters -- #
+
+    @property
+    def func(self) -> OperatorFunc:
+        """
+        Get the comparison function for this operator.
+
+        Returns
+        -------
+        OperatorFunc
+            The comparison function corresponding to this operator.
+        """
+        match self:
+            case OperatorName.EQ:
+                return _op.eq
+            case OperatorName.NE:
+                return _op.ne
+            case OperatorName.GT:
+                return _op.gt
+            case OperatorName.GTE:
+                return _op.ge
+            case OperatorName.LT:
+                return _op.lt
+            case OperatorName.LTE:
+                return _op.le
+            case OperatorName.IN:
+                return lambda a, b: a in b
+            case OperatorName.CONTAINS:
+                return lambda a, b: b in a
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            '==': 'eq',
+            '=': 'eq',
+            '!=': 'ne',
+            '<>': 'ne',
+            '>=': 'gte',
+            '≥': 'gte',
+            '<=': 'lte',
+            '≤': 'lte',
+            '>': 'gt',
+            '<': 'lt',
+        }
+
+
+class PipelineStep(CoercibleStrEnum):
+    """Pipeline step names as an enum for internal orchestration."""
+
+    # -- Constants -- #
+
+    FILTER = 'filter'
+    MAP = 'map'
+    SELECT = 'select'
+    SORT = 'sort'
+    AGGREGATE = 'aggregate'
+
+    # -- Getters -- #
+
+    @property
+    def order(self) -> int:
+        """
+        Get the execution order of this pipeline step.
+
+        Returns
+        -------
+        int
+            The execution order of this pipeline step.
+        """
+        return _PIPELINE_ORDER_INDEX[self]
+
+
+# SECTION: INTERNAL CONSTANTS ============================================== #
+
+
+# Precomputed order index for PipelineStep; avoids recomputing on each access.
+_PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
+    PipelineStep.FILTER: 0,
+    PipelineStep.MAP: 1,
+    PipelineStep.SELECT: 2,
+    PipelineStep.SORT: 3,
+    PipelineStep.AGGREGATE: 4,
+}
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def coerce_data_connector_type(
+    connector: DataConnectorType | str,
+) -> DataConnectorType:
+    """
+    Normalize textual data connector values to :class:`DataConnectorType`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`DataConnectorType.coerce` going forward.
+    """
+    return DataConnectorType.coerce(connector)
+
+
+def coerce_file_format(
+    file_format: FileFormat | str,
+) -> FileFormat:
+    """
+    Normalize textual file format values to :class:`FileFormat`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`FileFormat.coerce` going forward.
+    """
+    return FileFormat.coerce(file_format)
+
+
+def coerce_http_method(
+    http_method: HttpMethod | str,
+) -> HttpMethod:
+    """
+    Normalize textual HTTP method values to :class:`HttpMethod`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`HttpMethod.coerce` going forward.
+    """
+    return HttpMethod.coerce(http_method)

etlplus/extract.py

@@ -0,0 +1,218 @@
+"""
+:mod:`etlplus.extract` module.
+
+Helpers to extract data from files, databases, and REST APIs.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+import requests  # type: ignore[import]
+
+from .enums import DataConnectorType
+from .enums import FileFormat
+from .enums import HttpMethod
+from .enums import coerce_data_connector_type
+from .enums import coerce_file_format
+from .file import File
+from .types import JSONData
+from .types import JSONDict
+from .types import JSONList
+from .types import StrPath
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+# -- File Extraction -- #
+
+
+def extract_from_file(
+    file_path: StrPath,
+    file_format: FileFormat | str | None = FileFormat.JSON,
+) -> JSONData:
+    """
+    Extract (semi-)structured data from a local file.
+
+    Parameters
+    ----------
+    file_path : StrPath
+        Source file path.
+    file_format : FileFormat | str | None, optional
+        File format to parse. If ``None``, infer from the filename
+        extension. Defaults to `'json'` for backward compatibility when
+        explicitly provided.
+
+    Returns
+    -------
+    JSONData
+        Parsed data as a mapping or a list of mappings.
+    """
+    path = Path(file_path)
+
+    # If no explicit format is provided, let File infer from extension.
+    if file_format is None:
+        return File(path, None).read()
+    fmt = coerce_file_format(file_format)
+
+    # Let file module perform existence and format validation.
+    return File(path, fmt).read()
+
+
+# -- Database Extraction (Placeholder) -- #
+
+
+def extract_from_database(
+    connection_string: str,
+) -> JSONList:
+    """
+    Extract data from a database.
+
+    Notes
+    -----
+    Placeholder implementation. To enable database extraction, install and
+    configure database-specific drivers and query logic.
+
+    Parameters
+    ----------
+    connection_string : str
+        Database connection string.
+
+    Returns
+    -------
+    JSONList
+        Informational message payload.
+    """
+    return [
+        {
+            'message': 'Database extraction not yet implemented',
+            'connection_string': connection_string,
+            'note': (
+                'Install database-specific drivers to enable this feature'
+            ),
+        },
+    ]
+
+
+# -- REST API Extraction -- #
+
+
+def extract_from_api(
+    url: str,
+    method: HttpMethod | str = HttpMethod.GET,
+    **kwargs: Any,
+) -> JSONData:
+    """
+    Extract data from a REST API.
+
+    Parameters
+    ----------
+    url : str
+        API endpoint URL.
+    method : HttpMethod | str, optional
+        HTTP method to use. Defaults to ``GET``.
+    **kwargs : Any
+        Extra arguments forwarded to the underlying ``requests`` call
+        (for example, ``timeout``). To use a pre-configured
+        :class:`requests.Session`, provide it via ``session``.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+
+    Raises
+    ------
+    TypeError
+        If a provided ``session`` does not expose the required HTTP
+        method (for example, ``get``).
+    """
+    http_method = HttpMethod.coerce(method)
+
+    # Apply a conservative timeout to guard against hanging requests.
+    timeout = kwargs.pop('timeout', 10.0)
+    session = kwargs.pop('session', None)
+    requester = session or requests
+
+    request_callable = getattr(requester, http_method.value, None)
+    if not callable(request_callable):
+        raise TypeError(
+            'Session object must supply a callable'
+            f'"{http_method.value}" method',
+        )
+
+    response = request_callable(url, timeout=timeout, **kwargs)
+    response.raise_for_status()
+
+    content_type = response.headers.get('content-type', '').lower()
+    if 'application/json' in content_type:
+        try:
+            payload: Any = response.json()
+        except ValueError:
+            # Malformed JSON despite content-type; fall back to text
+            return {
+                'content': response.text,
+                'content_type': content_type,
+            }
+        if isinstance(payload, dict):
+            return cast(JSONDict, payload)
+        if isinstance(payload, list):
+            if all(isinstance(x, dict) for x in payload):
+                return cast(JSONList, payload)
+            # Coerce non-dict array items into objects for consistency
+            return [{'value': x} for x in payload]
+        # Fallback: wrap scalar JSON
+        return {'value': payload}
+
+    return {'content': response.text, 'content_type': content_type}
+
+
+# -- Orchestration -- #
+
+
+def extract(
+    source_type: DataConnectorType | str,
+    source: StrPath,
+    file_format: FileFormat | str | None = None,
+    **kwargs: Any,
+) -> JSONData:
+    """
+    Extract data from a source (file, database, or API).
+
+    Parameters
+    ----------
+    source_type : DataConnectorType | str
+        Type of data source.
+    source : StrPath
+        Source location (file path, connection string, or API URL).
+    file_format : FileFormat | str | None, optional
+        File format, inferred from filename extension if omitted.
+    **kwargs : Any
+        Additional arguments forwarded to source-specific extractors.
+
+    Returns
+    -------
+    JSONData
+        Extracted data.
+
+    Raises
+    ------
+    ValueError
+        If `source_type` is not one of the supported values.
+    """
+    match coerce_data_connector_type(source_type):
+        case DataConnectorType.FILE:
+            # Prefer explicit format if provided, else infer from filename.
+            return extract_from_file(source, file_format)
+        case DataConnectorType.DATABASE:
+            return extract_from_database(str(source))
+        case DataConnectorType.API:
+            # API extraction always uses an HTTP method; default is GET.
+            # ``file_format`` is ignored for APIs.
+            return extract_from_api(str(source), **kwargs)
+        case _:
+            # ``coerce_data_connector_type`` covers invalid entries, but keep
+            # explicit guard for defensive programming.
+            raise ValueError(f'Invalid source type: {source_type}')