etlplus 0.14.3__py3-none-any.whl → 0.15.2__py3-none-any.whl

etlplus/README.md

@@ -1,4 +1,4 @@
-# etlplus package
+# `etlplus` Package
 
 The `etlplus` package provides a unified Python API and CLI for ETL operations: extraction,
 validation, transformation, and loading of data from files, APIs, and databases.
@@ -13,12 +13,12 @@ Back to project overview: see the top-level [README](../README.md).
 
 - [etlplus.api](api/README.md): Lightweight HTTP client and paginated REST helpers
 - [etlplus.file](file/README.md): Unified file format support and helpers
-- [etlplus.config](config/README.md): Configuration helpers for connectors, pipelines, jobs, and
-  profiles
-- [etlplus.cli](cli/README.md): Command-line interface for ETLPlus workflows
+- [etlplus.cli](cli/README.md): Command-line interface definitions for `etlplus`
 - [etlplus.database](database/README.md): Database engine, schema, and ORM helpers
 - [etlplus.templates](templates/README.md): SQL and DDL template helpers
 - [etlplus.validation](validation/README.md): Data validation utilities and helpers
+- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
+  profiles
 
 ## Quickstart
 

etlplus/api/README.md

@@ -1,4 +1,4 @@
-# etlplus.api subpackage
+# `etlplus.api` Subpackage
 
 Documentation for the `etlplus.api` subpackage: a lightweight HTTP client and helpers for paginated
 REST endpoints.
@@ -12,7 +12,7 @@ REST endpoints.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.api subpackage](#etlplusapi-subpackage)
+- [`etlplus.api` Subpackage](#etlplusapi-subpackage)
   - [Installation](#installation)
   - [Quickstart](#quickstart)
     - [Overriding Rate Limits Per Call](#overriding-rate-limits-per-call)
@@ -22,6 +22,7 @@ Back to project overview: see the top-level [README](../../README.md).
   - [Authentication](#authentication)
   - [Errors and Rate Limiting](#errors-and-rate-limiting)
   - [Types and Transport](#types-and-transport)
+  - [Config Schemas](#config-schemas)
   - [Supporting Modules](#supporting-modules)
   - [Minimal Contract](#minimal-contract)
   - [See also](#see-also)
@@ -225,6 +226,36 @@ providers can fall back to their own defaults. If you already possess a static t
   `etlplus/api/request_manager.py` wraps `requests` sessions plus retry orchestration. Advanced
   users may consult those modules to adapt behavior.
 
+## Config Schemas
+
+`etlplus.api.types` defines TypedDict-based configuration shapes for API profiles and endpoints.
+Runtime parsing remains permissive in `etlplus.api.config`, but these types improve IDE
+autocomplete and static analysis.
+
+Exported types:
+
+- `ApiConfigMap`: top-level API config shape
+- `ApiProfileConfigMap`: per-profile API config shape
+- `ApiProfileDefaultsMap`: defaults block within a profile
+- `EndpointMap`: endpoint config shape
+
+Example:
+
+```python
+from etlplus.api import ApiConfigMap
+
+api_cfg: ApiConfigMap = {
+    "base_url": "https://example.test",
+    "headers": {"Authorization": "Bearer token"},
+    "endpoints": {
+        "users": {
+            "path": "/users",
+            "method": "GET",
+        },
+    },
+}
+```
+
 ## Supporting Modules
 
 - `etlplus.api.types` collects friendly aliases such as `Headers`, `Params`, `Url`, and

etlplus/api/config.py

@@ -3,11 +3,6 @@
 
 Configuration dataclasses for REST API services, profiles, and endpoints.
 
-These models used to live under :mod:`etlplus.config`, but they belong in the
-API layer because they compose runtime types such as
-:class:`etlplus.api.EndpointClient`, :class:`etlplus.api.PaginationConfig`, and
-:class:`etlplus.api.RateLimitConfig`.
-
 Notes
 -----
 - TypedDict references remain editor hints only; :meth:`from_obj` accepts
@@ -41,9 +36,9 @@ from .pagination import PaginationConfig
 from .rate_limiting import RateLimitConfig
 
 if TYPE_CHECKING:
-    from ..config.types import ApiConfigMap
-    from ..config.types import ApiProfileConfigMap
-    from ..config.types import EndpointMap
+    from .types import ApiConfigMap
+    from .types import ApiProfileConfigMap
+    from .types import EndpointMap
 
 
 # SECTION: EXPORTS ========================================================== #

etlplus/api/types.py

@@ -20,9 +20,11 @@ Examples
 from __future__ import annotations
 
 from collections.abc import Callable
+from collections.abc import Mapping
 from dataclasses import dataclass
 from typing import Any
 from typing import Self
+from typing import TypedDict
 from typing import cast
 
 from ..types import JSONData
@@ -40,6 +42,11 @@ __all__ = [
     'Headers',
     'Params',
     'Url',
+    # Typed Dicts
+    'ApiConfigMap',
+    'ApiProfileConfigMap',
+    'ApiProfileDefaultsMap',
+    'EndpointMap',
 ]
 
 
@@ -49,6 +56,88 @@ __all__ = [
 _UNSET = object()
 
 
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ApiConfigMap(TypedDict, total=False):
+    """
+    Top-level API config shape parsed by ApiConfig.from_obj.
+
+    Either provide a ``base_url`` with optional ``headers`` and ``endpoints``,
+    or provide ``profiles`` with at least one profile having a ``base_url``.
+
+    See Also
+    --------
+    - :class:`etlplus.api.config.ApiConfig`
+    """
+
+    base_url: str
+    headers: StrAnyMap
+    endpoints: Mapping[str, EndpointMap | str]
+    profiles: Mapping[str, ApiProfileConfigMap]
+
+
+class ApiProfileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted for a profile entry under ApiConfigMap.profiles.
+
+    Notes
+    -----
+    ``base_url`` is required at runtime when profiles are provided.
+
+    See Also
+    --------
+    - :class:`etlplus.api.config.ApiProfileConfig`
+    """
+
+    base_url: str
+    headers: StrAnyMap
+    base_path: str
+    auth: StrAnyMap
+    defaults: ApiProfileDefaultsMap
+
+
+class ApiProfileDefaultsMap(TypedDict, total=False):
+    """
+    Defaults block available under a profile (all keys optional).
+
+    Notes
+    -----
+    Runtime expects header values to be str; typing remains permissive.
+
+    See Also
+    --------
+    - :class:`etlplus.api.config.ApiProfileConfig`
+    - :class:`etlplus.api.pagination.PaginationConfig`
+    - :class:`etlplus.api.rate_limiting.RateLimitConfig`
+    """
+
+    headers: StrAnyMap
+    pagination: Any
+    rate_limit: Any
+
+
+class EndpointMap(TypedDict, total=False):
+    """
+    Shape accepted by EndpointConfig.from_obj.
+
+    One of ``path`` or ``url`` should be provided.
+
+    See Also
+    --------
+    - :class:`etlplus.api.config.EndpointConfig`
+    """
+
+    path: str
+    url: str
+    method: str
+    path_params: StrAnyMap
+    query_params: StrAnyMap
+    body: Any
+    pagination: Any
+    rate_limit: Any
+
+
 # SECTION: DATA CLASSES ===================================================== #
 
 

etlplus/api/utils.py

@@ -892,4 +892,8 @@ def resolve_request(
             'Session object must supply a callable '
             f'"{http_method.value}" method',
         )
-    return request_callable, request_timeout, http_method
+    typed_request_callable = cast(
+        Callable[..., requests.Response],
+        request_callable,
+    )
+    return typed_request_callable, request_timeout, http_method

etlplus/cli/README.md

@@ -1,4 +1,4 @@
-# etlplus.cli subpackage
+# `etlplus.cli` Subpackage
 
 Documentation for the `etlplus.cli` subpackage: command-line interface for ETLPlus workflows.
 
@@ -9,7 +9,7 @@ Documentation for the `etlplus.cli` subpackage: command-line interface for ETLPl
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.cli subpackage](#etlpluscli-subpackage)
+- [`etlplus.cli` Subpackage](#etlpluscli-subpackage)
   - [Available Commands](#available-commands)
   - [Command Options](#command-options)
   - [Example: Running a Pipeline](#example-running-a-pipeline)

etlplus/cli/commands.py

@@ -61,6 +61,24 @@ __all__ = ['app']
 
 # SECTION: TYPE ALIASES ==================================================== #
 
+
+JobOption = Annotated[
+    str | None,
+    typer.Option(
+        '-j',
+        '--job',
+        help='Name of the job to run',
+    ),
+]
+
+JobsOption = Annotated[
+    bool,
+    typer.Option(
+        '--jobs',
+        help='List available job names and exit',
+    ),
+]
+
 OperationsOption = Annotated[
     str,
     typer.Option(
@@ -89,6 +107,23 @@ PipelineConfigOption = Annotated[
     ),
 ]
 
+PipelineOption = Annotated[
+    str | None,
+    typer.Option(
+        '-p',
+        '--pipeline',
+        help='Name of the pipeline to run',
+    ),
+]
+
+PipelinesOption = Annotated[
+    bool,
+    typer.Option(
+        '--pipelines',
+        help='List ETL pipelines',
+    ),
+]
+
 RenderConfigOption = Annotated[
     str | None,
     typer.Option(
@@ -193,6 +228,22 @@ SourceTypeOption = Annotated[
     ),
 ]
 
+SourcesOption = Annotated[
+    bool,
+    typer.Option(
+        '--sources',
+        help='List data sources',
+    ),
+]
+
+SummaryOption = Annotated[
+    bool,
+    typer.Option(
+        '--summary',
+        help='Show pipeline summary (name, version, sources, targets, jobs)',
+    ),
+]
+
 TargetArg = Annotated[
     str,
     typer.Argument(
@@ -227,6 +278,22 @@ TargetTypeOption = Annotated[
     ),
 ]
 
+TargetsOption = Annotated[
+    bool,
+    typer.Option(
+        '--targets',
+        help='List data targets',
+    ),
+]
+
+TransformsOption = Annotated[
+    bool,
+    typer.Option(
+        '--transforms',
+        help='List data transforms',
+    ),
+]
+
 
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
@@ -341,36 +408,12 @@ def _root(
 def check_cmd(
     ctx: typer.Context,
     config: PipelineConfigOption,
-    jobs: bool = typer.Option(
-        False,
-        '--jobs',
-        help='List available job names and exit',
-    ),
-    pipelines: bool = typer.Option(
-        False,
-        '--pipelines',
-        help='List ETL pipelines',
-    ),
-    sources: bool = typer.Option(
-        False,
-        '--sources',
-        help='List data sources',
-    ),
-    summary: bool = typer.Option(
-        False,
-        '--summary',
-        help='Show pipeline summary (name, version, sources, targets, jobs)',
-    ),
-    targets: bool = typer.Option(
-        False,
-        '--targets',
-        help='List data targets',
-    ),
-    transforms: bool = typer.Option(
-        False,
-        '--transforms',
-        help='List data transforms',
-    ),
+    jobs: JobsOption = False,
+    pipelines: PipelinesOption = False,
+    sources: SourcesOption = False,
+    summary: SummaryOption = False,
+    targets: TargetsOption = False,
+    transforms: TransformsOption = False,
 ) -> int:
     """
     Inspect a pipeline configuration.
@@ -683,18 +726,8 @@ def render_cmd(
 def run_cmd(
     ctx: typer.Context,
     config: PipelineConfigOption,
-    job: str | None = typer.Option(
-        None,
-        '-j',
-        '--job',
-        help='Name of the job to run',
-    ),
-    pipeline: str | None = typer.Option(
-        None,
-        '-p',
-        '--pipeline',
-        help='Name of the pipeline to run',
-    ),
+    job: JobOption = None,
+    pipeline: PipelineOption = None,
 ) -> int:
     """
     Execute an ETL job or pipeline from a YAML configuration.

etlplus/cli/handlers.py

@@ -14,8 +14,6 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
-from ..config import PipelineConfig
-from ..config import load_pipeline_config
 from ..database import load_table_spec
 from ..database import render_tables
 from ..file import File
@@ -28,6 +26,8 @@ from ..ops import validate
 from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
+from ..workflow import PipelineConfig
+from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -121,9 +121,12 @@ def _check_sections(
     if targets:
         sections['targets'] = [tgt.name for tgt in cfg.targets]
     if transforms:
-        sections['transforms'] = [
-            getattr(trf, 'name', None) for trf in cfg.transforms
-        ]
+        if isinstance(cfg.transforms, Mapping):
+            sections['transforms'] = list(cfg.transforms)
+        else:
+            sections['transforms'] = [
+                getattr(trf, 'name', None) for trf in cfg.transforms
+            ]
     if not sections:
         sections['jobs'] = _pipeline_summary(cfg)['jobs']
     return sections
@@ -157,6 +160,29 @@ def _pipeline_summary(
     }
 
 
+def _write_file_payload(
+    payload: JSONData,
+    target: str,
+    *,
+    format_hint: str | None,
+) -> None:
+    """
+    Write a JSON-like payload to a file path using an optional format hint.
+
+    Parameters
+    ----------
+    payload : JSONData
+        The structured data to write.
+    target : str
+        File path to write to.
+    format_hint : str | None
+        Optional format hint for :class:`FileFormat`.
+    """
+    file_path = Path(target)
+    file_format = FileFormat.coerce(format_hint) if format_hint else None
+    File(file_path, file_format=file_format).write(payload)
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -572,15 +598,7 @@ def transform_handler(
 
     # TODO: Generalize to handle non-file targets.
     if target and target != '-':
-        # Convert target to Path and target_format to FileFormat if needed
-        file_path = Path(target)
-        file_format = None
-        if target_format is not None:
-            try:
-                file_format = FileFormat(target_format)
-            except ValueError:
-                file_format = None  # or handle error as appropriate
-        File(file_path, file_format=file_format).write(data)
+        _write_file_payload(data, target, format_hint=target_format)
         print(f'Data transformed and saved to {target}')
         return 0
 

etlplus/cli/main.py

@@ -50,7 +50,7 @@ def _emit_context_help(
         return False
 
     with contextlib.redirect_stdout(sys.stderr):
-        ctx.get_help()
+        print(ctx.get_help())
     return True
 
 

etlplus/cli/state.py

@@ -15,6 +15,7 @@ from typing import Final
 
 import typer
 
+from ..utils import normalize_str
 from .constants import DATA_CONNECTORS
 
 # SECTION: EXPORTS ========================================================== #
@@ -322,14 +323,10 @@ def validate_choice(
     typer.BadParameter
         If the input value is not in the set of valid choices.
     """
-    v = str(value or '').strip().lower()
-    normalized_choices = {c.lower() for c in choices}
+    v = normalize_str(str(value or ''))
+    normalized_choices = {normalize_str(c): c for c in choices}
     if v in normalized_choices:
-        # Preserve original casing from choices when possible for messages
-        for choice in choices:
-            if choice.lower() == v:
-                return choice
-        return v
+        return normalized_choices[v]
     allowed = ', '.join(sorted(choices))
     raise typer.BadParameter(
         f"Invalid {label} '{value}'. Choose from: {allowed}",

etlplus/database/README.md

@@ -1,4 +1,4 @@
-# etlplus.database subpackage
+# `etlplus.database` Subpackage
 
 Documentation for the `etlplus.database` subpackage: database engine, schema, and ORM helpers.
 
@@ -9,7 +9,7 @@ Documentation for the `etlplus.database` subpackage: database engine, schema, an
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.database subpackage](#etlplusdatabase-subpackage)
+- [`etlplus.database` Subpackage](#etlplusdatabase-subpackage)
   - [Database Engine and Connections](#database-engine-and-connections)
   - [Schema and DDL Helpers](#schema-and-ddl-helpers)
   - [ORM Utilities](#orm-utilities)

etlplus/database/engine.py

@@ -136,9 +136,25 @@ def load_database_url_from_config(
     return url
 
 
-def make_engine(url: str | None = None, **engine_kwargs: Any) -> Engine:
-    """Create a SQLAlchemy Engine, defaulting to env config if no URL given."""
+def make_engine(
+    url: str | None = None,
+    **engine_kwargs: Any,
+) -> Engine:
+    """
+    Create a SQLAlchemy Engine, defaulting to env config if no URL given.
+
+    Parameters
+    ----------
+    url : str | None, optional
+        Database URL/DSN string. When omitted, ``DATABASE_URL`` is used.
+    **engine_kwargs : Any
+        Extra keyword arguments forwarded to ``create_engine``.
 
+    Returns
+    -------
+    Engine
+        Configured SQLAlchemy engine instance.
+    """
     resolved_url = url or DATABASE_URL
     return create_engine(resolved_url, pool_pre_ping=True, **engine_kwargs)
 

etlplus/database/orm.py

@@ -201,12 +201,14 @@ def build_models(
 ) -> ModelRegistry:
     """
     Build SQLAlchemy ORM models from table specifications.
+
     Parameters
     ----------
     specs : list[TableSpec]
         List of table specifications.
     base : type[DeclarativeBase], optional
         Base class for the ORM models (default: :class:`Base`).
+
     Returns
     -------
     ModelRegistry

etlplus/file/README.md

@@ -1,4 +1,4 @@
-# etlplus.file subpackage
+# `etlplus.file` Subpackage
 
 Documentation for the `etlplus.file` subpackage: unified file format support and helpers for reading
 and writing data files.
@@ -11,7 +11,7 @@ and writing data files.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.file subpackage](#etlplusfile-subpackage)
+- [`etlplus.file` Subpackage](#etlplusfile-subpackage)
   - [Supported File Formats](#supported-file-formats)
   - [Inferring File Format and Compression](#inferring-file-format-and-compression)
   - [Reading and Writing Files](#reading-and-writing-files)

etlplus/file/_io.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import csv
 from pathlib import Path
+from typing import Any
 from typing import cast
 
 from ..types import JSONData
@@ -17,6 +18,44 @@ from ..types import JSONList
 # SECTION: FUNCTIONS ======================================================== #
 
 
+def coerce_record_payload(
+    payload: Any,
+    *,
+    format_name: str,
+) -> JSONData:
+    """
+    Validate that ``payload`` is an object or list of objects.
+
+    Parameters
+    ----------
+    payload : Any
+        Parsed payload to validate.
+    format_name : str
+        Human-readable format name for error messages.
+
+    Returns
+    -------
+    JSONData
+        ``payload`` when it is a dict or a list of dicts.
+
+    Raises
+    ------
+    TypeError
+        If the payload is not a dict or list of dicts.
+    """
+    if isinstance(payload, dict):
+        return cast(JSONDict, payload)
+    if isinstance(payload, list):
+        if all(isinstance(item, dict) for item in payload):
+            return cast(JSONList, payload)
+        raise TypeError(
+            f'{format_name} array must contain only objects (dicts)',
+        )
+    raise TypeError(
+        f'{format_name} root must be an object or an array of objects',
+    )
+
+
 def normalize_records(
     data: JSONData,
     format_name: str,

etlplus/file/json.py

@@ -20,12 +20,10 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import cast
 
 from ..types import JSONData
-from ..types import JSONDict
-from ..types import JSONList
 from ..utils import count_records
+from ._io import coerce_record_payload
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -65,17 +63,7 @@ def read(
     with path.open('r', encoding='utf-8') as handle:
         loaded = json.load(handle)
 
-    if isinstance(loaded, dict):
-        return cast(JSONDict, loaded)
-    if isinstance(loaded, list):
-        if all(isinstance(item, dict) for item in loaded):
-            return cast(JSONList, loaded)
-        raise TypeError(
-            'JSON array must contain only objects (dicts) when loading file',
-        )
-    raise TypeError(
-        'JSON root must be an object or an array of objects when loading file',
-    )
+    return coerce_record_payload(loaded, format_name='JSON')
 
 
 def write(