etlplus 0.9.2__py3-none-any.whl → 0.10.1__py3-none-any.whl

etlplus/config/types.py

@@ -0,0 +1,204 @@
+"""
+:mod:`etlplus.config.types` module.
+
+Type aliases and editor-only TypedDicts for :mod:`etlplus.config`.
+
+These types improve IDE autocomplete and static analysis while the runtime
+parsers remain permissive.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- ``*.from_obj`` constructors accept ``Mapping[str, Any]`` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+
+Examples
+--------
+>>> from etlplus.config import Connector
+>>> src: Connector = {
+>>>     "type": "file",
+>>>     "path": "/data/input.csv",
+>>> }
+>>> tgt: Connector = {
+>>>     "type": "database",
+>>>     "connection_string": "postgresql://user:pass@localhost/db",
+>>> }
+>>> from etlplus.api import RetryPolicy
+>>> rp: RetryPolicy = {"max_attempts": 3, "backoff": 0.5}
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from typing import Literal
+from typing import TypedDict
+
+from ..api import PaginationConfigMap
+from ..api import RateLimitConfigMap
+from ..types import StrAnyMap
+
+# SECTION: EXPORTS  ========================================================= #
+
+
+__all__ = [
+    # Type aliases
+    'ConnectorType',
+    # 'PaginationType',
+    # TypedDicts
+    'ApiProfileDefaultsMap',
+    'ApiProfileConfigMap',
+    'ApiConfigMap',
+    'EndpointMap',
+    'ConnectorApiConfigMap',
+    'ConnectorDbConfigMap',
+    'ConnectorFileConfigMap',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+# Literal type for supported connector kinds
+type ConnectorType = Literal['api', 'database', 'file']
+
+# Literal type for supported pagination kinds
+# type PaginationType = Literal['page', 'offset', 'cursor']
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ApiConfigMap(TypedDict, total=False):
+    """
+    Top-level API config shape parsed by ApiConfig.from_obj.
+
+    Either provide a 'base_url' with optional 'headers' and 'endpoints', or
+    provide 'profiles' with at least one profile having a 'base_url'.
+
+    See Also
+    --------
+    - etlplus.config.api.ApiConfig.from_obj: parses this mapping
+    """
+
+    base_url: str
+    headers: StrAnyMap
+    endpoints: Mapping[str, EndpointMap | str]
+    profiles: Mapping[str, ApiProfileConfigMap]
+
+
+class ApiProfileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted for a profile entry under ApiConfigMap.profiles.
+
+    Notes
+    -----
+    `base_url` is required at runtime when profiles are provided.
+
+    See Also
+    --------
+    - etlplus.config.api.ApiProfileConfig.from_obj: parses this mapping
+    """
+
+    base_url: str
+    headers: StrAnyMap
+    base_path: str
+    auth: StrAnyMap
+    defaults: ApiProfileDefaultsMap
+
+
+class ApiProfileDefaultsMap(TypedDict, total=False):
+    """
+    Defaults block available under a profile (all keys optional).
+
+    Notes
+    -----
+    Runtime expects header values to be str; typing remains permissive.
+
+    See Also
+    --------
+    - etlplus.config.api.ApiProfileConfig.from_obj: consumes this block
+    - etlplus.config.pagination.PaginationConfig.from_obj: parses pagination
+    - etlplus.api.rate_limiting.RateLimitConfig.from_obj: parses rate_limit
+    """
+
+    headers: StrAnyMap
+    pagination: PaginationConfigMap | StrAnyMap
+    rate_limit: RateLimitConfigMap | StrAnyMap
+
+
+class ConnectorApiConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by ConnectorApi.from_obj (all keys optional).
+
+    See Also
+    --------
+    - etlplus.config.connector.ConnectorApi.from_obj
+    """
+
+    name: str
+    type: ConnectorType
+    url: str
+    method: str
+    headers: StrAnyMap
+    query_params: StrAnyMap
+    pagination: PaginationConfigMap
+    rate_limit: RateLimitConfigMap
+    api: str
+    endpoint: str
+
+
+class ConnectorDbConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by ConnectorDb.from_obj (all keys optional).
+
+    See Also
+    --------
+    - etlplus.config.connector.ConnectorDb.from_obj
+    """
+
+    name: str
+    type: ConnectorType
+    connection_string: str
+    query: str
+    table: str
+    mode: str
+
+
+class ConnectorFileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by ConnectorFile.from_obj (all keys optional).
+
+    See Also
+    --------
+    - etlplus.config.connector.ConnectorFile.from_obj
+    """
+
+    name: str
+    type: ConnectorType
+    format: str
+    path: str
+    options: StrAnyMap
+
+
+class EndpointMap(TypedDict, total=False):
+    """
+    Shape accepted by EndpointConfig.from_obj.
+
+    One of 'path' or 'url' should be provided.
+
+    See Also
+    --------
+    - etlplus.config.api.EndpointConfig.from_obj: parses this mapping
+    """
+
+    path: str
+    url: str
+    method: str
+    path_params: StrAnyMap
+    query_params: StrAnyMap
+    body: Any
+    pagination: PaginationConfigMap
+    rate_limit: RateLimitConfigMap

etlplus/config/utils.py

@@ -0,0 +1,120 @@
+"""
+:mod:`etlplus.config.utils` module.
+
+A module defining utility helpers for ETL pipeline configuration.
+
+Notes
+-----
+- Inputs to parsers favor ``Mapping[str, Any]`` to remain permissive and
+    avoid unnecessary copies; normalization returns concrete types.
+- Substitution is shallow for strings and recursive for containers.
+- Numeric coercion helpers are intentionally forgiving: invalid values
+    become ``None`` rather than raising.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from collections.abc import Mapping
+from typing import Any
+
+from ..types import StrAnyMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'deep_substitute',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def deep_substitute(
+    value: Any,
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, str] | None,
+) -> Any:
+    """
+    Recursively substitute ``${VAR}`` tokens in nested structures.
+
+    Only strings are substituted; other types are returned as-is.
+
+    Parameters
+    ----------
+    value : Any
+        The value to perform substitutions on.
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, str] | None
+        Mapping of environment variables overriding ``vars_map`` values (higher
+        precedence).
+
+    Returns
+    -------
+    Any
+        New structure with substitutions applied where tokens were found.
+    """
+    substitutions = _prepare_substitutions(vars_map, env_map)
+
+    def _apply(node: Any) -> Any:
+        match node:
+            case str():
+                return _replace_tokens(node, substitutions)
+            case Mapping():
+                return {k: _apply(v) for k, v in node.items()}
+            case list() | tuple() as seq:
+                apply = [_apply(item) for item in seq]
+                return apply if isinstance(seq, list) else tuple(apply)
+            case set():
+                return {_apply(item) for item in node}
+            case frozenset():
+                return frozenset(_apply(item) for item in node)
+            case _:
+                return node
+
+    return _apply(value)
+
+
+# SECTION: INTERNAL FUNCTIONS ============================================== #
+
+
+def _prepare_substitutions(
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, Any] | None,
+) -> tuple[tuple[str, Any], ...]:
+    """Merge variable and environment maps into an ordered substitutions list.
+
+    Parameters
+    ----------
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, Any] | None
+        Environment-backed values that override entries from ``vars_map``.
+
+    Returns
+    -------
+    tuple[tuple[str, Any], ...]
+        Immutable sequence of ``(name, value)`` pairs suitable for token
+        replacement.
+    """
+    if not vars_map and not env_map:
+        return ()
+    merged: dict[str, Any] = {**(vars_map or {}), **(env_map or {})}
+    return tuple(merged.items())
+
+
+def _replace_tokens(
+    text: str,
+    substitutions: Iterable[tuple[str, Any]],
+) -> str:
+    if not substitutions:
+        return text
+    out = text
+    for name, replacement in substitutions:
+        token = f'${{{name}}}'
+        if token in out:
+            out = out.replace(token, str(replacement))
+    return out

etlplus/database/ddl.py

@@ -203,7 +203,7 @@ def load_table_spec(
         raise ValueError('Spec must be .json, .yml, or .yaml')
 
     try:
-        spec = File(spec_path).read()
+        spec = File.read_file(spec_path)
     except ImportError as e:
         if suffix in {'.yml', '.yaml'}:
             raise RuntimeError(

etlplus/database/engine.py

@@ -113,7 +113,7 @@ def load_database_url_from_config(
     ValueError
         If no connection string/URL/DSN is found for the specified entry.
     """
-    cfg = File(Path(path)).read()
+    cfg = File.read_file(Path(path))
     if not isinstance(cfg, Mapping):
         raise TypeError('Database config must be a mapping')
 
@@ -136,25 +136,9 @@ def load_database_url_from_config(
     return url
 
 
-def make_engine(
-    url: str | None = None,
-    **engine_kwargs: Any,
-) -> Engine:
-    """
-    Create a SQLAlchemy Engine, defaulting to env config if no URL given.
-
-    Parameters
-    ----------
-    url : str | None, optional
-        Database URL/DSN string. When omitted, ``DATABASE_URL`` is used.
-    **engine_kwargs : Any
-        Extra keyword arguments forwarded to ``create_engine``.
+def make_engine(url: str | None = None, **engine_kwargs: Any) -> Engine:
+    """Create a SQLAlchemy Engine, defaulting to env config if no URL given."""
 
-    Returns
-    -------
-    Engine
-        Configured SQLAlchemy engine instance.
-    """
     resolved_url = url or DATABASE_URL
     return create_engine(resolved_url, pool_pre_ping=True, **engine_kwargs)
 

etlplus/database/orm.py

@@ -201,14 +201,12 @@ def build_models(
 ) -> ModelRegistry:
     """
     Build SQLAlchemy ORM models from table specifications.
-
     Parameters
     ----------
     specs : list[TableSpec]
         List of table specifications.
     base : type[DeclarativeBase], optional
         Base class for the ORM models (default: :class:`Base`).
-
     Returns
     -------
     ModelRegistry

etlplus/database/schema.py

@@ -260,7 +260,7 @@ def load_table_specs(
     list[TableSpec]
         A list of TableSpec instances parsed from the YAML file.
     """
-    data = File(Path(path)).read()
+    data = File.read_file(Path(path))
     if not data:
         return []
 

etlplus/enums.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import enum
 import operator as _op
+from pathlib import PurePath
 from statistics import fmean
 from typing import Self
 
@@ -22,9 +23,18 @@ __all__ = [
     # Enums
     'AggregateName',
     'CoercibleStrEnum',
+    'CompressionFormat',
     'DataConnectorType',
+    'FileFormat',
+    'HttpMethod',
     'OperatorName',
     'PipelineStep',
+    # Functions
+    'coerce_compression_format',
+    'coerce_data_connector_type',
+    'coerce_file_format',
+    'coerce_http_method',
+    'infer_file_format_and_compression',
 ]
 
 
@@ -168,6 +178,39 @@ class AggregateName(CoercibleStrEnum):
         return lambda xs, n: (fmean(xs) if xs else 0.0)
 
 
+class CompressionFormat(CoercibleStrEnum):
+    """Supported compression formats for data files."""
+
+    # -- Constants -- #
+
+    GZ = 'gz'
+    ZIP = 'zip'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            # File extensions
+            '.gz': 'gz',
+            '.gzip': 'gz',
+            '.zip': 'zip',
+            # MIME types
+            'application/gzip': 'gz',
+            'application/x-gzip': 'gz',
+            'application/zip': 'zip',
+            'application/x-zip-compressed': 'zip',
+        }
+
+
 class DataConnectorType(CoercibleStrEnum):
     """Supported data connector types."""
 
@@ -199,6 +242,119 @@ class DataConnectorType(CoercibleStrEnum):
         }
 
 
+class FileFormat(CoercibleStrEnum):
+    """Supported file formats for extraction."""
+
+    # -- Constants -- #
+
+    AVRO = 'avro'
+    CSV = 'csv'
+    FEATHER = 'feather'
+    GZ = 'gz'
+    JSON = 'json'
+    NDJSON = 'ndjson'
+    ORC = 'orc'
+    PARQUET = 'parquet'
+    TSV = 'tsv'
+    TXT = 'txt'
+    XLS = 'xls'
+    XLSX = 'xlsx'
+    ZIP = 'zip'
+    XML = 'xml'
+    YAML = 'yaml'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            # Common shorthand
+            'parq': 'parquet',
+            'yml': 'yaml',
+            # File extensions
+            '.avro': 'avro',
+            '.csv': 'csv',
+            '.feather': 'feather',
+            '.gz': 'gz',
+            '.json': 'json',
+            '.jsonl': 'ndjson',
+            '.ndjson': 'ndjson',
+            '.orc': 'orc',
+            '.parquet': 'parquet',
+            '.pq': 'parquet',
+            '.tsv': 'tsv',
+            '.txt': 'txt',
+            '.xls': 'xls',
+            '.xlsx': 'xlsx',
+            '.zip': 'zip',
+            '.xml': 'xml',
+            '.yaml': 'yaml',
+            '.yml': 'yaml',
+            # MIME types
+            'application/avro': 'avro',
+            'application/feather': 'feather',
+            'application/gzip': 'gz',
+            'application/json': 'json',
+            'application/jsonlines': 'ndjson',
+            'application/ndjson': 'ndjson',
+            'application/orc': 'orc',
+            'application/vnd.apache.arrow.file': 'feather',
+            'application/vnd.apache.orc': 'orc',
+            'application/vnd.ms-excel': 'xls',
+            (
+                'application/vnd.openxmlformats-'
+                'officedocument.spreadsheetml.sheet'
+            ): 'xlsx',
+            'application/x-avro': 'avro',
+            'application/x-ndjson': 'ndjson',
+            'application/x-parquet': 'parquet',
+            'application/xml': 'xml',
+            'application/zip': 'zip',
+            'text/csv': 'csv',
+            'text/plain': 'txt',
+            'text/tab-separated-values': 'tsv',
+        }
+
+
+class HttpMethod(CoercibleStrEnum):
+    """Supported HTTP verbs that accept JSON payloads."""
+
+    # -- Constants -- #
+
+    CONNECT = 'connect'
+    DELETE = 'delete'
+    GET = 'get'
+    HEAD = 'head'
+    OPTIONS = 'options'
+    PATCH = 'patch'
+    POST = 'post'
+    PUT = 'put'
+    TRACE = 'trace'
+
+    # -- Getters -- #
+
+    @property
+    def allows_body(self) -> bool:
+        """
+        Whether the method typically allows a request body.
+
+        Notes
+        -----
+        - RFCs do not strictly forbid bodies on some other methods (e.g.,
+          ``DELETE``), but many servers/clients do not expect them. We mark
+          ``POST``, ``PUT``, and ``PATCH`` as True.
+        """
+        return self in {HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH}
+
+
 class OperatorName(CoercibleStrEnum):
     """Supported comparison operators with helpers."""
 
@@ -298,6 +454,13 @@ class PipelineStep(CoercibleStrEnum):
 # SECTION: INTERNAL CONSTANTS ============================================== #
 
 
+# Compression formats that are also file formats.
+_COMPRESSION_FILE_FORMATS: set[FileFormat] = {
+    FileFormat.GZ,
+    FileFormat.ZIP,
+}
+
+
 # Precomputed order index for PipelineStep; avoids recomputing on each access.
 _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
     PipelineStep.FILTER: 0,
@@ -306,3 +469,106 @@ _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
     PipelineStep.SORT: 3,
     PipelineStep.AGGREGATE: 4,
 }
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def coerce_data_connector_type(
+    connector: DataConnectorType | str,
+) -> DataConnectorType:
+    """
+    Normalize textual data connector values to :class:`DataConnectorType`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`DataConnectorType.coerce` going forward.
+    """
+    return DataConnectorType.coerce(connector)
+
+
+def coerce_file_format(
+    file_format: FileFormat | str,
+) -> FileFormat:
+    """
+    Normalize textual file format values to :class:`FileFormat`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`FileFormat.coerce` going forward.
+    """
+    return FileFormat.coerce(file_format)
+
+
+def coerce_compression_format(
+    compression_format: CompressionFormat | str,
+) -> CompressionFormat:
+    """
+    Normalize textual compression format values to :class:`CompressionFormat`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`CompressionFormat.coerce` going forward.
+    """
+    return CompressionFormat.coerce(compression_format)
+
+
+def coerce_http_method(
+    http_method: HttpMethod | str,
+) -> HttpMethod:
+    """
+    Normalize textual HTTP method values to :class:`HttpMethod`.
+
+    This thin wrapper is kept for backward compatibility; prefer
+    :meth:`HttpMethod.coerce` going forward.
+    """
+    return HttpMethod.coerce(http_method)
+
+
+def infer_file_format_and_compression(
+    value: object,
+) -> tuple[FileFormat | None, CompressionFormat | None]:
+    """
+    Infer data format and compression from a filename, extension, or MIME type.
+
+    Parameters
+    ----------
+    value : object
+        A filename, extension, MIME type, or existing enum member.
+
+    Returns
+    -------
+    tuple[FileFormat | None, CompressionFormat | None]
+        The inferred data format and compression, if any.
+    """
+    if isinstance(value, FileFormat):
+        if value in _COMPRESSION_FILE_FORMATS:
+            return None, CompressionFormat.coerce(value.value)
+        return value, None
+    if isinstance(value, CompressionFormat):
+        return None, value
+
+    text = str(value).strip()
+    if not text:
+        return None, None
+
+    normalized = text.casefold()
+    mime = normalized.split(';', 1)[0].strip()
+
+    compression = CompressionFormat.try_coerce(mime)
+    fmt = FileFormat.try_coerce(mime)
+
+    suffixes = PurePath(text).suffixes
+    if suffixes:
+        normalized_suffixes = [suffix.casefold() for suffix in suffixes]
+        compression = (
+            CompressionFormat.try_coerce(normalized_suffixes[-1])
+            or compression
+        )
+        if compression is not None:
+            normalized_suffixes = normalized_suffixes[:-1]
+        if normalized_suffixes:
+            fmt = FileFormat.try_coerce(normalized_suffixes[-1]) or fmt
+
+    if fmt in _COMPRESSION_FILE_FORMATS:
+        compression = compression or CompressionFormat.coerce(fmt.value)
+        fmt = None
+
+    return fmt, compression