etlplus 0.15.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

etlplus/README.md

@@ -13,12 +13,12 @@ Back to project overview: see the top-level [README](../README.md).
 
 - [etlplus.api](api/README.md): Lightweight HTTP client and paginated REST helpers
 - [etlplus.file](file/README.md): Unified file format support and helpers
-- [etlplus.config](config/README.md): Configuration helpers for connectors, pipelines, jobs, and
-  profiles
-- [etlplus.cli](cli/README.md): Command-line interface for ETLPlus workflows
+- [etlplus.cli](cli/README.md): Command-line interface definitions for `etlplus`
 - [etlplus.database](database/README.md): Database engine, schema, and ORM helpers
 - [etlplus.templates](templates/README.md): SQL and DDL template helpers
 - [etlplus.validation](validation/README.md): Data validation utilities and helpers
+- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
+  profiles
 
 ## Quickstart
 

etlplus/api/README.md

@@ -22,6 +22,7 @@ Back to project overview: see the top-level [README](../../README.md).
   - [Authentication](#authentication)
   - [Errors and Rate Limiting](#errors-and-rate-limiting)
   - [Types and Transport](#types-and-transport)
+  - [Config Schemas](#config-schemas)
   - [Supporting Modules](#supporting-modules)
   - [Minimal Contract](#minimal-contract)
   - [See also](#see-also)
@@ -225,6 +226,36 @@ providers can fall back to their own defaults. If you already possess a static t
   `etlplus/api/request_manager.py` wraps `requests` sessions plus retry orchestration. Advanced
   users may consult those modules to adapt behavior.
 
+## Config Schemas
+
+`etlplus.api.types` defines TypedDict-based configuration shapes for API profiles and endpoints.
+Runtime parsing remains permissive in `etlplus.api.config`, but these types improve IDE
+autocomplete and static analysis.
+
+Exported types:
+
+- `ApiConfigMap`: top-level API config shape
+- `ApiProfileConfigMap`: per-profile API config shape
+- `ApiProfileDefaultsMap`: defaults block within a profile
+- `EndpointMap`: endpoint config shape
+
+Example:
+
+```python
+from etlplus.api import ApiConfigMap
+
+api_cfg: ApiConfigMap = {
+    "base_url": "https://example.test",
+    "headers": {"Authorization": "Bearer token"},
+    "endpoints": {
+        "users": {
+            "path": "/users",
+            "method": "GET",
+        },
+    },
+}
+```
+
 ## Supporting Modules
 
 - `etlplus.api.types` collects friendly aliases such as `Headers`, `Params`, `Url`, and

etlplus/api/auth.py

@@ -136,7 +136,7 @@ def _truncate(
     limit: int = MAX_LOG_BODY,
 ) -> str:
     """
-    Return ``text`` shortened to ``limit`` characters for logging.
+    Return *text* shortened to *limit* characters for logging.
 
     Parameters
     ----------

etlplus/api/config.py

@@ -3,11 +3,6 @@
 
 Configuration dataclasses for REST API services, profiles, and endpoints.
 
-These models used to live under :mod:`etlplus.config`, but they belong in the
-API layer because they compose runtime types such as
-:class:`etlplus.api.EndpointClient`, :class:`etlplus.api.PaginationConfig`, and
-:class:`etlplus.api.RateLimitConfig`.
-
 Notes
 -----
 - TypedDict references remain editor hints only; :meth:`from_obj` accepts
@@ -41,9 +36,9 @@ from .pagination import PaginationConfig
 from .rate_limiting import RateLimitConfig
 
 if TYPE_CHECKING:
-    from ..config.types import ApiConfigMap
-    from ..config.types import ApiProfileConfigMap
-    from ..config.types import EndpointMap
+    from .types import ApiConfigMap
+    from .types import ApiProfileConfigMap
+    from .types import EndpointMap
 
 
 # SECTION: EXPORTS ========================================================== #
@@ -73,7 +68,7 @@ def _effective_service_defaults(
     fallback_headers: dict[str, str],
 ) -> tuple[str, dict[str, str]]:
     """
-    Return ``(base_url, headers)`` using ``profiles`` when present.
+    Return ``(base_url, headers)`` using *profiles* when present.
 
     Parameters
     ----------
@@ -92,7 +87,7 @@ def _effective_service_defaults(
     Raises
     ------
     TypeError
-        If no profiles are defined and ``fallback_base`` is not a string.
+        If no profiles are defined and *fallback_base* is not a string.
     """
     if profiles:
         name = 'default' if 'default' in profiles else next(iter(profiles))

etlplus/api/endpoint_client.py

@@ -759,8 +759,8 @@ class EndpointClient:
         Raises
         ------
         KeyError
-            If ``endpoint_key`` is unknown or a required placeholder in the
-            path has no corresponding entry in ``path_parameters``.
+            If *endpoint_key* is unknown or a required placeholder in the path
+            has no corresponding entry in *path_parameters*.
         ValueError
             If the path template is invalid.
 
@@ -836,7 +836,7 @@ class EndpointClient:
         """
         Sleep for the specified seconds if positive.
 
-        The optional ``sleeper`` is useful for tests (e.g., pass
+        The optional *sleeper* is useful for tests (e.g., pass
         ``lambda s: None``). Defaults to using time.sleep when not provided.
 
         Parameters
@@ -870,7 +870,7 @@ class EndpointClient:
         rate_limit : RateLimitConfigMap | None
             Client-wide rate limit configuration.
         overrides : RateLimitOverrides, optional
-            Per-call overrides that take precedence over ``rate_limit``.
+            Per-call overrides that take precedence over *rate_limit*.
 
         Returns
         -------

etlplus/api/pagination/config.py

@@ -398,7 +398,7 @@ class PaginationConfig(BoundsWarningsMixin):
         Returns
         -------
         Self | None
-            Parsed pagination configuration, or ``None`` if ``obj`` isn't a
+            Parsed pagination configuration, or ``None`` if *obj* isn't a
             mapping.
 
         Notes

etlplus/api/pagination/paginator.py

@@ -66,14 +66,14 @@ def _resolve_path(
     path: str | None,
 ) -> Any:
     """
-    Resolve dotted ``path`` within ``obj`` or return ``_MISSING``.
+    Resolve dotted *path* within *obj* or return ``_MISSING``.
 
     Parameters
     ----------
     obj : Any
         JSON payload from an API response.
     path : str | None
-        Dotted path to the target value within ``obj``.
+        Dotted path to the target value within *obj*.
 
     Returns
     -------
@@ -665,7 +665,7 @@ class Paginator:
         records_path : str | None
             Optional dotted path to the records within the payload.
         fallback_path : str | None
-            Secondary dotted path consulted when ``records_path`` resolves to
+            Secondary dotted path consulted when *records_path* resolves to
             ``None`` or an empty list.
 
         Returns
@@ -675,9 +675,8 @@ class Paginator:
 
         Notes
         -----
-        Supports dotted path extraction via ``records_path`` and handles
-        lists, mappings, and scalars by coercing non-dict items into
-        ``{"value": x}``.
+        Supports dotted path extraction via *records_path* and handles lists,
+        mappings, and scalars by coercing non-dict items into ``{"value": x}``.
         """
         resolver = partial(_resolve_path, x)
         data = resolver(records_path)
@@ -729,7 +728,7 @@ class Paginator:
         Returns
         -------
         PaginationType | None
-            Detected pagination type, or ``default`` if not found.
+            Detected pagination type, or *default* if not found.
         """
         if not config:
             return default

etlplus/api/rate_limiting/config.py

@@ -84,14 +84,14 @@ def _merge_rate_limit(
     overrides: RateLimitOverrides = None,
 ) -> dict[str, Any]:
     """
-    Merge ``rate_limit`` and ``overrides`` honoring override precedence.
+    Merge *rate_limit* and *overrides* honoring override precedence.
 
     Parameters
     ----------
     rate_limit : StrAnyMap | None
         Base rate-limit configuration.
     overrides : RateLimitOverrides, optional
-        Override configuration with precedence over ``rate_limit``.
+        Override configuration with precedence over *rate_limit*.
 
     Returns
     -------
@@ -274,7 +274,7 @@ class RateLimitConfig(BoundsWarningsMixin):
         rate_limit : StrAnyMap | RateLimitConfig | None, optional
             Base rate-limit configuration to normalize.
         overrides : RateLimitOverrides, optional
-            Override values that take precedence over ``rate_limit``.
+            Override values that take precedence over *rate_limit*.
 
         Returns
         -------
@@ -330,7 +330,7 @@ class RateLimitConfig(BoundsWarningsMixin):
         Returns
         -------
         Self | None
-            Parsed instance, or ``None`` if ``obj`` isn't a mapping.
+            Parsed instance, or ``None`` if *obj* isn't a mapping.
         """
         if obj is None:
             return None

etlplus/api/rate_limiting/rate_limiter.py

@@ -235,7 +235,7 @@ class RateLimiter:
             Base rate-limit configuration. May contain ``"sleep_seconds"`` or
             ``"max_per_sec"``.
         overrides : RateLimitOverrides, optional
-            Optional overrides with the same keys as ``rate_limit``.
+            Optional overrides with the same keys as *rate_limit*.
 
         Returns
         -------

etlplus/api/retry_manager.py

@@ -278,7 +278,7 @@ class RetryManager:
         **kwargs: Any,
     ) -> JSONData:
         """
-        Execute ``func`` with exponential-backoff retries.
+        Execute *func* with exponential-backoff retries.
 
         Parameters
         ----------
@@ -287,7 +287,7 @@ class RetryManager:
         url : str
             URL for the API request.
         **kwargs : Any
-            Additional keyword arguments to pass to ``func``
+            Additional keyword arguments to pass to *func*
 
         Returns
         -------

etlplus/api/transport.py

@@ -309,7 +309,7 @@ def build_session_with_adapters(
     adapters_cfg: Sequence[HTTPAdapterMountConfig],
 ) -> requests.Session:
     """
-    Mount adapters described by ``adapters_cfg`` onto a new session.
+    Mount adapters described by *adapters_cfg* onto a new session.
 
     Ignores invalid adapter configurations so that a usable session is always
     returned.

etlplus/api/types.py

@@ -20,9 +20,11 @@ Examples
 from __future__ import annotations
 
 from collections.abc import Callable
+from collections.abc import Mapping
 from dataclasses import dataclass
 from typing import Any
 from typing import Self
+from typing import TypedDict
 from typing import cast
 
 from ..types import JSONData
@@ -40,6 +42,11 @@ __all__ = [
     'Headers',
     'Params',
     'Url',
+    # Typed Dicts
+    'ApiConfigMap',
+    'ApiProfileConfigMap',
+    'ApiProfileDefaultsMap',
+    'EndpointMap',
 ]
 
 
@@ -49,6 +56,98 @@ __all__ = [
 _UNSET = object()
 
 
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ApiConfigMap(TypedDict, total=False):
+    """
+    Top-level API config shape parsed by
+    :meth:`etlplus.api.config.ApiConfig.from_obj`.
+
+    Either provide a :attr:`base_url` with optional :attr:`headers` and
+    :attr:`endpoints`, or provide :attr:`profiles` with at least one profile
+    having a :attr:`base_url`.
+
+    See Also
+    --------
+    - :class:`etlplus.api.config.ApiConfig`
+    """
+
+    base_url: str
+    headers: StrAnyMap
+    endpoints: Mapping[str, EndpointMap | str]
+    profiles: Mapping[str, ApiProfileConfigMap]
+
+
+class ApiProfileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted for a profile entry under
+    :meth:`etlplus.api.config.ApiConfig.from_obj`.
+
+    Notes
+    -----
+    - :attr:`base_url` is required at runtime when :attr:`profiles` key/value
+        pairs are provided.
+    - :meth:`etlplus.api.config.ApiProfileConfig.from_obj` parses this mapping.
+
+    See Also
+    --------
+    - :class:`etlplus.api.config.ApiProfileConfig`
+    """
+
+    base_url: str
+    headers: StrAnyMap
+    base_path: str
+    auth: StrAnyMap
+    defaults: ApiProfileDefaultsMap
+
+
+class ApiProfileDefaultsMap(TypedDict, total=False):
+    """
+    Defaults block available under a profile (all keys optional).
+
+    Notes
+    -----
+    - Runtime expects header values to be ``str``; typing remains permissive.
+    - :meth:`etlplus.api.config.ApiProfileConfig.from_obj` consumes this block.
+    - :meth:`etlplus.api.pagination.PaginationConfig.from_obj` parses
+        :attr:`pagination`.
+    - :meth:`etlplus.api.rate_limiting.RateLimitConfig.from_obj` parses
+        :attr:`rate_limit`.
+
+    See Also
+    --------
+    - :class:`etlplus.api.config.ApiProfileConfig`
+    - :class:`etlplus.api.pagination.PaginationConfig`
+    - :class:`etlplus.api.rate_limiting.RateLimitConfig`
+    """
+
+    headers: StrAnyMap
+    pagination: StrAnyMap  # PaginationConfigMap | StrAnyMap
+    rate_limit: StrAnyMap  # RateLimitConfigMap | StrAnyMap
+
+
+class EndpointMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`etlplus.api.config.EndpointConfig.from_obj`.
+
+    One of :attr:`path` or :attr:`url` should be provided.
+
+    See Also
+    --------
+    - :class:`etlplus.api.config.EndpointConfig`
+    """
+
+    path: str
+    url: str
+    method: str
+    path_params: StrAnyMap
+    query_params: StrAnyMap
+    body: Any
+    pagination: StrAnyMap  # PaginationConfigMap | StrAnyMap
+    rate_limit: StrAnyMap  # RateLimitConfigMap | StrAnyMap
+
+
 # SECTION: DATA CLASSES ===================================================== #
 
 

etlplus/api/utils.py

@@ -172,7 +172,7 @@ def _coalesce(
     *args: Any,
 ) -> Any | None:
     """
-    Return the first non-``None`` value from ``args``.
+    Return the first non-``None`` value from *args*.
 
     Parameters
     ----------

etlplus/cli/commands.py

@@ -61,6 +61,24 @@ __all__ = ['app']
 
 # SECTION: TYPE ALIASES ==================================================== #
 
+
+JobOption = Annotated[
+    str | None,
+    typer.Option(
+        '-j',
+        '--job',
+        help='Name of the job to run',
+    ),
+]
+
+JobsOption = Annotated[
+    bool,
+    typer.Option(
+        '--jobs',
+        help='List available job names and exit',
+    ),
+]
+
 OperationsOption = Annotated[
     str,
     typer.Option(
@@ -89,6 +107,23 @@ PipelineConfigOption = Annotated[
     ),
 ]
 
+PipelineOption = Annotated[
+    str | None,
+    typer.Option(
+        '-p',
+        '--pipeline',
+        help='Name of the pipeline to run',
+    ),
+]
+
+PipelinesOption = Annotated[
+    bool,
+    typer.Option(
+        '--pipelines',
+        help='List ETL pipelines',
+    ),
+]
+
 RenderConfigOption = Annotated[
     str | None,
     typer.Option(
@@ -193,6 +228,22 @@ SourceTypeOption = Annotated[
     ),
 ]
 
+SourcesOption = Annotated[
+    bool,
+    typer.Option(
+        '--sources',
+        help='List data sources',
+    ),
+]
+
+SummaryOption = Annotated[
+    bool,
+    typer.Option(
+        '--summary',
+        help='Show pipeline summary (name, version, sources, targets, jobs)',
+    ),
+]
+
 TargetArg = Annotated[
     str,
     typer.Argument(
@@ -227,6 +278,22 @@ TargetTypeOption = Annotated[
     ),
 ]
 
+TargetsOption = Annotated[
+    bool,
+    typer.Option(
+        '--targets',
+        help='List data targets',
+    ),
+]
+
+TransformsOption = Annotated[
+    bool,
+    typer.Option(
+        '--transforms',
+        help='List data transforms',
+    ),
+]
+
 
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
@@ -341,36 +408,12 @@ def _root(
 def check_cmd(
     ctx: typer.Context,
     config: PipelineConfigOption,
-    jobs: bool = typer.Option(
-        False,
-        '--jobs',
-        help='List available job names and exit',
-    ),
-    pipelines: bool = typer.Option(
-        False,
-        '--pipelines',
-        help='List ETL pipelines',
-    ),
-    sources: bool = typer.Option(
-        False,
-        '--sources',
-        help='List data sources',
-    ),
-    summary: bool = typer.Option(
-        False,
-        '--summary',
-        help='Show pipeline summary (name, version, sources, targets, jobs)',
-    ),
-    targets: bool = typer.Option(
-        False,
-        '--targets',
-        help='List data targets',
-    ),
-    transforms: bool = typer.Option(
-        False,
-        '--transforms',
-        help='List data transforms',
-    ),
+    jobs: JobsOption = False,
+    pipelines: PipelinesOption = False,
+    sources: SourcesOption = False,
+    summary: SummaryOption = False,
+    targets: TargetsOption = False,
+    transforms: TransformsOption = False,
 ) -> int:
     """
     Inspect a pipeline configuration.
@@ -683,18 +726,8 @@ def render_cmd(
 def run_cmd(
     ctx: typer.Context,
     config: PipelineConfigOption,
-    job: str | None = typer.Option(
-        None,
-        '-j',
-        '--job',
-        help='Name of the job to run',
-    ),
-    pipeline: str | None = typer.Option(
-        None,
-        '-p',
-        '--pipeline',
-        help='Name of the pipeline to run',
-    ),
+    job: JobOption = None,
+    pipeline: PipelineOption = None,
 ) -> int:
     """
     Execute an ETL job or pipeline from a YAML configuration.

etlplus/cli/constants.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 from typing import Final
 
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import FileFormat
 
 # SECTION: EXPORTS ========================================================== #

etlplus/cli/handlers.py

@@ -121,9 +121,12 @@ def _check_sections(
     if targets:
         sections['targets'] = [tgt.name for tgt in cfg.targets]
     if transforms:
-        sections['transforms'] = [
-            getattr(trf, 'name', None) for trf in cfg.transforms
-        ]
+        if isinstance(cfg.transforms, Mapping):
+            sections['transforms'] = list(cfg.transforms)
+        else:
+            sections['transforms'] = [
+                getattr(trf, 'name', None) for trf in cfg.transforms
+            ]
     if not sections:
         sections['jobs'] = _pipeline_summary(cfg)['jobs']
     return sections
@@ -157,6 +160,29 @@ def _pipeline_summary(
     }
 
 
+def _write_file_payload(
+    payload: JSONData,
+    target: str,
+    *,
+    format_hint: str | None,
+) -> None:
+    """
+    Write a JSON-like payload to a file path using an optional format hint.
+
+    Parameters
+    ----------
+    payload : JSONData
+        The structured data to write.
+    target : str
+        File path to write to.
+    format_hint : str | None
+        Optional format hint for :class:`FileFormat`.
+    """
+    file_path = Path(target)
+    file_format = FileFormat.coerce(format_hint) if format_hint else None
+    File(file_path, file_format=file_format).write(payload)
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -479,7 +505,7 @@ def run_handler(
         Name of the job to run. If not provided, runs the entire pipeline.
         Default is ``None``.
     pipeline : str | None, optional
-        Alias for ``job``. Default is ``None``.
+        Alias for *job*. Default is ``None``.
     pretty : bool, optional
         Whether to pretty-print output. Default is ``True``.
 
@@ -572,15 +598,7 @@ def transform_handler(
 
     # TODO: Generalize to handle non-file targets.
     if target and target != '-':
-        # Convert target to Path and target_format to FileFormat if needed
-        file_path = Path(target)
-        file_format = None
-        if target_format is not None:
-            try:
-                file_format = FileFormat(target_format)
-            except ValueError:
-                file_format = None  # or handle error as appropriate
-        File(file_path, file_format=file_format).write(data)
+        _write_file_payload(data, target, format_hint=target_format)
         print(f'Data transformed and saved to {target}')
         return 0
 

etlplus/cli/io.py

@@ -71,7 +71,7 @@ def emit_or_write(
     success_message: str,
 ) -> None:
     """
-    Emit JSON or persist to disk based on ``output_path``.
+    Emit JSON or persist to disk based on *output_path*.
 
     Parameters
     ----------
@@ -122,7 +122,7 @@ def materialize_file_payload(
     format_explicit: bool,
 ) -> JSONData | object:
     """
-    Return structured payloads when ``source`` references a file.
+    Return structured payloads when *source* references a file.
 
     Parameters
     ----------

etlplus/cli/main.py

@@ -44,13 +44,13 @@ def _emit_context_help(
     Returns
     -------
     bool
-        ``True`` when help was emitted, ``False`` when ``ctx`` was ``None``.
+        ``True`` when help was emitted, ``False`` when *ctx* was ``None``.
     """
     if ctx is None:
         return False
 
     with contextlib.redirect_stdout(sys.stderr):
-        ctx.get_help()
+        print(ctx.get_help())
     return True