etlplus 0.9.2__py3-none-any.whl → 0.10.2__py3-none-any.whl

etlplus/config/types.py

@@ -0,0 +1,204 @@
+"""
+:mod:`etlplus.config.types` module.
+
+Type aliases and editor-only TypedDicts for :mod:`etlplus.config`.
+
+These types improve IDE autocomplete and static analysis while the runtime
+parsers remain permissive.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- ``*.from_obj`` constructors accept ``Mapping[str, Any]`` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+
+Examples
+--------
+>>> from etlplus.config import Connector
+>>> src: Connector = {
+>>>     "type": "file",
+>>>     "path": "/data/input.csv",
+>>> }
+>>> tgt: Connector = {
+>>>     "type": "database",
+>>>     "connection_string": "postgresql://user:pass@localhost/db",
+>>> }
+>>> from etlplus.api import RetryPolicy
+>>> rp: RetryPolicy = {"max_attempts": 3, "backoff": 0.5}
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from typing import Literal
+from typing import TypedDict
+
+from ..api import PaginationConfigMap
+from ..api import RateLimitConfigMap
+from ..types import StrAnyMap
+
+# SECTION: EXPORTS  ========================================================= #
+
+
+__all__ = [
+    # Type aliases
+    'ConnectorType',
+    # 'PaginationType',
+    # TypedDicts
+    'ApiProfileDefaultsMap',
+    'ApiProfileConfigMap',
+    'ApiConfigMap',
+    'EndpointMap',
+    'ConnectorApiConfigMap',
+    'ConnectorDbConfigMap',
+    'ConnectorFileConfigMap',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+# Literal type for supported connector kinds
+type ConnectorType = Literal['api', 'database', 'file']
+
+# Literal type for supported pagination kinds
+# type PaginationType = Literal['page', 'offset', 'cursor']
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ApiConfigMap(TypedDict, total=False):
+    """
+    Top-level API config shape parsed by ApiConfig.from_obj.
+
+    Either provide a 'base_url' with optional 'headers' and 'endpoints', or
+    provide 'profiles' with at least one profile having a 'base_url'.
+
+    See Also
+    --------
+    - etlplus.config.api.ApiConfig.from_obj: parses this mapping
+    """
+
+    base_url: str
+    headers: StrAnyMap
+    endpoints: Mapping[str, EndpointMap | str]
+    profiles: Mapping[str, ApiProfileConfigMap]
+
+
+class ApiProfileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted for a profile entry under ApiConfigMap.profiles.
+
+    Notes
+    -----
+    `base_url` is required at runtime when profiles are provided.
+
+    See Also
+    --------
+    - etlplus.config.api.ApiProfileConfig.from_obj: parses this mapping
+    """
+
+    base_url: str
+    headers: StrAnyMap
+    base_path: str
+    auth: StrAnyMap
+    defaults: ApiProfileDefaultsMap
+
+
+class ApiProfileDefaultsMap(TypedDict, total=False):
+    """
+    Defaults block available under a profile (all keys optional).
+
+    Notes
+    -----
+    Runtime expects header values to be str; typing remains permissive.
+
+    See Also
+    --------
+    - etlplus.config.api.ApiProfileConfig.from_obj: consumes this block
+    - etlplus.config.pagination.PaginationConfig.from_obj: parses pagination
+    - etlplus.api.rate_limiting.RateLimitConfig.from_obj: parses rate_limit
+    """
+
+    headers: StrAnyMap
+    pagination: PaginationConfigMap | StrAnyMap
+    rate_limit: RateLimitConfigMap | StrAnyMap
+
+
+class ConnectorApiConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by ConnectorApi.from_obj (all keys optional).
+
+    See Also
+    --------
+    - etlplus.config.connector.ConnectorApi.from_obj
+    """
+
+    name: str
+    type: ConnectorType
+    url: str
+    method: str
+    headers: StrAnyMap
+    query_params: StrAnyMap
+    pagination: PaginationConfigMap
+    rate_limit: RateLimitConfigMap
+    api: str
+    endpoint: str
+
+
+class ConnectorDbConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by ConnectorDb.from_obj (all keys optional).
+
+    See Also
+    --------
+    - etlplus.config.connector.ConnectorDb.from_obj
+    """
+
+    name: str
+    type: ConnectorType
+    connection_string: str
+    query: str
+    table: str
+    mode: str
+
+
+class ConnectorFileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by ConnectorFile.from_obj (all keys optional).
+
+    See Also
+    --------
+    - etlplus.config.connector.ConnectorFile.from_obj
+    """
+
+    name: str
+    type: ConnectorType
+    format: str
+    path: str
+    options: StrAnyMap
+
+
+class EndpointMap(TypedDict, total=False):
+    """
+    Shape accepted by EndpointConfig.from_obj.
+
+    One of 'path' or 'url' should be provided.
+
+    See Also
+    --------
+    - etlplus.config.api.EndpointConfig.from_obj: parses this mapping
+    """
+
+    path: str
+    url: str
+    method: str
+    path_params: StrAnyMap
+    query_params: StrAnyMap
+    body: Any
+    pagination: PaginationConfigMap
+    rate_limit: RateLimitConfigMap

etlplus/config/utils.py

@@ -0,0 +1,120 @@
+"""
+:mod:`etlplus.config.utils` module.
+
+A module defining utility helpers for ETL pipeline configuration.
+
+Notes
+-----
+- Inputs to parsers favor ``Mapping[str, Any]`` to remain permissive and
+    avoid unnecessary copies; normalization returns concrete types.
+- Substitution is shallow for strings and recursive for containers.
+- Numeric coercion helpers are intentionally forgiving: invalid values
+    become ``None`` rather than raising.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from collections.abc import Mapping
+from typing import Any
+
+from ..types import StrAnyMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'deep_substitute',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def deep_substitute(
+    value: Any,
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, str] | None,
+) -> Any:
+    """
+    Recursively substitute ``${VAR}`` tokens in nested structures.
+
+    Only strings are substituted; other types are returned as-is.
+
+    Parameters
+    ----------
+    value : Any
+        The value to perform substitutions on.
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, str] | None
+        Mapping of environment variables overriding ``vars_map`` values (higher
+        precedence).
+
+    Returns
+    -------
+    Any
+        New structure with substitutions applied where tokens were found.
+    """
+    substitutions = _prepare_substitutions(vars_map, env_map)
+
+    def _apply(node: Any) -> Any:
+        match node:
+            case str():
+                return _replace_tokens(node, substitutions)
+            case Mapping():
+                return {k: _apply(v) for k, v in node.items()}
+            case list() | tuple() as seq:
+                apply = [_apply(item) for item in seq]
+                return apply if isinstance(seq, list) else tuple(apply)
+            case set():
+                return {_apply(item) for item in node}
+            case frozenset():
+                return frozenset(_apply(item) for item in node)
+            case _:
+                return node
+
+    return _apply(value)
+
+
+# SECTION: INTERNAL FUNCTIONS ============================================== #
+
+
+def _prepare_substitutions(
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, Any] | None,
+) -> tuple[tuple[str, Any], ...]:
+    """Merge variable and environment maps into an ordered substitutions list.
+
+    Parameters
+    ----------
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, Any] | None
+        Environment-backed values that override entries from ``vars_map``.
+
+    Returns
+    -------
+    tuple[tuple[str, Any], ...]
+        Immutable sequence of ``(name, value)`` pairs suitable for token
+        replacement.
+    """
+    if not vars_map and not env_map:
+        return ()
+    merged: dict[str, Any] = {**(vars_map or {}), **(env_map or {})}
+    return tuple(merged.items())
+
+
+def _replace_tokens(
+    text: str,
+    substitutions: Iterable[tuple[str, Any]],
+) -> str:
+    if not substitutions:
+        return text
+    out = text
+    for name, replacement in substitutions:
+        token = f'${{{name}}}'
+        if token in out:
+            out = out.replace(token, str(replacement))
+    return out

etlplus/database/ddl.py

@@ -203,7 +203,7 @@ def load_table_spec(
         raise ValueError('Spec must be .json, .yml, or .yaml')
 
     try:
-        spec = File(spec_path).read()
+        spec = File.read_file(spec_path)
     except ImportError as e:
         if suffix in {'.yml', '.yaml'}:
             raise RuntimeError(

etlplus/database/engine.py

@@ -113,7 +113,7 @@ def load_database_url_from_config(
     ValueError
         If no connection string/URL/DSN is found for the specified entry.
     """
-    cfg = File(Path(path)).read()
+    cfg = File.read_file(Path(path))
     if not isinstance(cfg, Mapping):
         raise TypeError('Database config must be a mapping')
 
@@ -136,25 +136,9 @@ def load_database_url_from_config(
     return url
 
 
-def make_engine(
-    url: str | None = None,
-    **engine_kwargs: Any,
-) -> Engine:
-    """
-    Create a SQLAlchemy Engine, defaulting to env config if no URL given.
-
-    Parameters
-    ----------
-    url : str | None, optional
-        Database URL/DSN string. When omitted, ``DATABASE_URL`` is used.
-    **engine_kwargs : Any
-        Extra keyword arguments forwarded to ``create_engine``.
+def make_engine(url: str | None = None, **engine_kwargs: Any) -> Engine:
+    """Create a SQLAlchemy Engine, defaulting to env config if no URL given."""
 
-    Returns
-    -------
-    Engine
-        Configured SQLAlchemy engine instance.
-    """
     resolved_url = url or DATABASE_URL
     return create_engine(resolved_url, pool_pre_ping=True, **engine_kwargs)
 

etlplus/database/orm.py

@@ -201,14 +201,12 @@ def build_models(
 ) -> ModelRegistry:
     """
     Build SQLAlchemy ORM models from table specifications.
-
     Parameters
     ----------
     specs : list[TableSpec]
         List of table specifications.
     base : type[DeclarativeBase], optional
         Base class for the ORM models (default: :class:`Base`).
-
     Returns
     -------
     ModelRegistry

etlplus/database/schema.py

@@ -260,7 +260,7 @@ def load_table_specs(
     list[TableSpec]
         A list of TableSpec instances parsed from the YAML file.
     """
-    data = File(Path(path)).read()
+    data = File.read_file(Path(path))
     if not data:
         return []