etlplus 0.9.2__py3-none-any.whl → 0.10.1__py3-none-any.whl

etlplus/cli/commands.py

@@ -36,7 +36,7 @@ from typing import cast
 import typer
 
 from .. import __version__
-from ..file import FileFormat
+from ..enums import FileFormat
 from . import handlers
 from .constants import CLI_DESCRIPTION
 from .constants import CLI_EPILOG
@@ -61,24 +61,6 @@ __all__ = ['app']
 
 # SECTION: TYPE ALIASES ==================================================== #
 
-
-JobOption = Annotated[
-    str | None,
-    typer.Option(
-        '-j',
-        '--job',
-        help='Name of the job to run',
-    ),
-]
-
-JobsOption = Annotated[
-    bool,
-    typer.Option(
-        '--jobs',
-        help='List available job names and exit',
-    ),
-]
-
 OperationsOption = Annotated[
     str,
     typer.Option(
@@ -107,23 +89,6 @@ PipelineConfigOption = Annotated[
     ),
 ]
 
-PipelineOption = Annotated[
-    str | None,
-    typer.Option(
-        '-p',
-        '--pipeline',
-        help='Name of the pipeline to run',
-    ),
-]
-
-PipelinesOption = Annotated[
-    bool,
-    typer.Option(
-        '--pipelines',
-        help='List ETL pipelines',
-    ),
-]
-
 RenderConfigOption = Annotated[
     str | None,
     typer.Option(
@@ -228,22 +193,6 @@ SourceTypeOption = Annotated[
     ),
 ]
 
-SourcesOption = Annotated[
-    bool,
-    typer.Option(
-        '--sources',
-        help='List data sources',
-    ),
-]
-
-SummaryOption = Annotated[
-    bool,
-    typer.Option(
-        '--summary',
-        help='Show pipeline summary (name, version, sources, targets, jobs)',
-    ),
-]
-
 TargetArg = Annotated[
     str,
     typer.Argument(
@@ -278,22 +227,6 @@ TargetTypeOption = Annotated[
     ),
 ]
 
-TargetsOption = Annotated[
-    bool,
-    typer.Option(
-        '--targets',
-        help='List data targets',
-    ),
-]
-
-TransformsOption = Annotated[
-    bool,
-    typer.Option(
-        '--transforms',
-        help='List data transforms',
-    ),
-]
-
 
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
@@ -408,12 +341,36 @@ def _root(
 def check_cmd(
     ctx: typer.Context,
     config: PipelineConfigOption,
-    jobs: JobsOption = False,
-    pipelines: PipelinesOption = False,
-    sources: SourcesOption = False,
-    summary: SummaryOption = False,
-    targets: TargetsOption = False,
-    transforms: TransformsOption = False,
+    jobs: bool = typer.Option(
+        False,
+        '--jobs',
+        help='List available job names and exit',
+    ),
+    pipelines: bool = typer.Option(
+        False,
+        '--pipelines',
+        help='List ETL pipelines',
+    ),
+    sources: bool = typer.Option(
+        False,
+        '--sources',
+        help='List data sources',
+    ),
+    summary: bool = typer.Option(
+        False,
+        '--summary',
+        help='Show pipeline summary (name, version, sources, targets, jobs)',
+    ),
+    targets: bool = typer.Option(
+        False,
+        '--targets',
+        help='List data targets',
+    ),
+    transforms: bool = typer.Option(
+        False,
+        '--transforms',
+        help='List data transforms',
+    ),
 ) -> int:
     """
     Inspect a pipeline configuration.
@@ -726,8 +683,18 @@ def render_cmd(
 def run_cmd(
     ctx: typer.Context,
     config: PipelineConfigOption,
-    job: JobOption = None,
-    pipeline: PipelineOption = None,
+    job: str | None = typer.Option(
+        None,
+        '-j',
+        '--job',
+        help='Name of the job to run',
+    ),
+    pipeline: str | None = typer.Option(
+        None,
+        '-p',
+        '--pipeline',
+        help='Name of the pipeline to run',
+    ),
 ) -> int:
     """
     Execute an ETL job or pipeline from a YAML configuration.

etlplus/cli/constants.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 from typing import Final
 
 from ..enums import DataConnectorType
-from ..file import FileFormat
+from ..enums import FileFormat
 
 # SECTION: EXPORTS ========================================================== #
 

etlplus/cli/handlers.py

@@ -14,20 +14,19 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
+from ..config import PipelineConfig
+from ..config import load_pipeline_config
 from ..database import load_table_spec
 from ..database import render_tables
+from ..extract import extract
 from ..file import File
-from ..file import FileFormat
-from ..ops import extract
-from ..ops import load
-from ..ops import run
-from ..ops import transform
-from ..ops import validate
-from ..ops.validate import FieldRules
+from ..load import load
+from ..run import run
+from ..transform import transform
 from ..types import JSONData
 from ..types import TemplateKey
-from ..workflow import PipelineConfig
-from ..workflow import load_pipeline_config
+from ..validate import FieldRules
+from ..validate import validate
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -121,12 +120,9 @@ def _check_sections(
     if targets:
         sections['targets'] = [tgt.name for tgt in cfg.targets]
     if transforms:
-        if isinstance(cfg.transforms, Mapping):
-            sections['transforms'] = list(cfg.transforms)
-        else:
-            sections['transforms'] = [
-                getattr(trf, 'name', None) for trf in cfg.transforms
-            ]
+        sections['transforms'] = [
+            getattr(trf, 'name', None) for trf in cfg.transforms
+        ]
     if not sections:
         sections['jobs'] = _pipeline_summary(cfg)['jobs']
     return sections
@@ -160,29 +156,6 @@ def _pipeline_summary(
     }
 
 
-def _write_file_payload(
-    payload: JSONData,
-    target: str,
-    *,
-    format_hint: str | None,
-) -> None:
-    """
-    Write a JSON-like payload to a file path using an optional format hint.
-
-    Parameters
-    ----------
-    payload : JSONData
-        The structured data to write.
-    target : str
-        File path to write to.
-    format_hint : str | None
-        Optional format hint for :class:`FileFormat`.
-    """
-    file_path = Path(target)
-    file_format = FileFormat.coerce(format_hint) if format_hint else None
-    File(file_path, file_format=file_format).write(payload)
-
-
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -596,9 +569,8 @@ def transform_handler(
 
     data = transform(payload, cast(TransformOperations, operations_payload))
 
-    # TODO: Generalize to handle non-file targets.
     if target and target != '-':
-        _write_file_payload(data, target, format_hint=target_format)
+        File.write_file(target, data, file_format=target_format)
         print(f'Data transformed and saved to {target}')
         return 0
 

etlplus/cli/io.py

@@ -15,8 +15,8 @@ from pathlib import Path
 from typing import Any
 from typing import cast
 
+from ..enums import FileFormat
 from ..file import File
-from ..file import FileFormat
 from ..types import JSONData
 from ..utils import print_json
 
@@ -331,6 +331,6 @@ def write_json_output(
     """
     if not output_path or output_path == '-':
         return False
-    File(Path(output_path), FileFormat.JSON).write(data)
+    File(Path(output_path), FileFormat.JSON).write_json(data)
     print(f'{success_message} {output_path}')
     return True

etlplus/cli/main.py

@@ -50,7 +50,7 @@ def _emit_context_help(
         return False
 
     with contextlib.redirect_stdout(sys.stderr):
-        print(ctx.get_help())
+        ctx.get_help()
     return True
 
 

etlplus/cli/state.py

@@ -15,7 +15,6 @@ from typing import Final
 
 import typer
 
-from ..utils import normalize_str
 from .constants import DATA_CONNECTORS
 
 # SECTION: EXPORTS ========================================================== #
@@ -323,10 +322,14 @@ def validate_choice(
     typer.BadParameter
         If the input value is not in the set of valid choices.
     """
-    v = normalize_str(str(value or ''))
-    normalized_choices = {normalize_str(c): c for c in choices}
+    v = str(value or '').strip().lower()
+    normalized_choices = {c.lower() for c in choices}
     if v in normalized_choices:
-        return normalized_choices[v]
+        # Preserve original casing from choices when possible for messages
+        for choice in choices:
+            if choice.lower() == v:
+                return choice
+        return v
     allowed = ', '.join(sorted(choices))
     raise typer.BadParameter(
         f"Invalid {label} '{value}'. Choose from: {allowed}",

etlplus/{workflow → config}/__init__.py

@@ -1,7 +1,17 @@
 """
-:mod:`etlplus.workflow` package.
+:mod:`etlplus.config` package.
 
-Job workflow helpers.
+Configuration models and helpers for ETLPlus.
+
+This package defines models for data sources/targets ("connectors"), APIs,
+pagination/rate limits, pipeline orchestration, and related utilities. The
+parsers are permissive (accepting ``Mapping[str, Any]``) and normalize to
+concrete types without raising on unknown/optional fields.
+
+Notes
+-----
+- The models use ``@dataclass(slots=True)`` and avoid mutating inputs.
+- TypedDicts are editor/type-checking hints and are not enforced at runtime.
 """
 
 from __future__ import annotations
@@ -11,7 +21,6 @@ from .connector import ConnectorApi
 from .connector import ConnectorDb
 from .connector import ConnectorFile
 from .connector import parse_connector
-from .dag import topological_sort_jobs
 from .jobs import ExtractRef
 from .jobs import JobConfig
 from .jobs import LoadRef
@@ -19,25 +28,29 @@ from .jobs import TransformRef
 from .jobs import ValidationRef
 from .pipeline import PipelineConfig
 from .pipeline import load_pipeline_config
+from .profile import ProfileConfig
+from .types import ConnectorType
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
-    # Data Classes
+    # Connectors
+    'Connector',
+    'ConnectorType',
     'ConnectorApi',
     'ConnectorDb',
     'ConnectorFile',
+    'parse_connector',
+    # Jobs / Refs
     'ExtractRef',
     'JobConfig',
     'LoadRef',
-    'PipelineConfig',
     'TransformRef',
     'ValidationRef',
-    # Functions
+    # Pipeline
+    'PipelineConfig',
     'load_pipeline_config',
-    'parse_connector',
-    'topological_sort_jobs',
-    # Type Aliases
-    'Connector',
+    # Profile
+    'ProfileConfig',
 ]

etlplus/{workflow → config}/connector.py

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.workflow.connector` module.
+:mod:`etlplus.config.connector` module.
 
 A module defining configuration types for data source/target connectors in ETL
 pipelines. A "connector" is any I/O endpoint:
@@ -11,26 +11,25 @@ pipelines. A "connector" is any I/O endpoint:
 
 Examples
 --------
-- Use :class:`ConnectorApi`/:class:`ConnectorFile`/:class:`ConnectorDb` when
-    you want the concrete dataclasses.
-- Use the :class:`Connector` union for typing a value that can be any
-    connector.
-- Use :func:`parse_connector(obj)` to construct a connector instance from a
-    generic mapping that includes a *type* key.
+- Use ``ConnectorApi``/``ConnectorFile``/``ConnectorDb`` when you want the
+  concrete dataclasses.
+- Use the ``Connector`` union for typing a value that can be any connector.
+- Use ``parse_connector(obj)`` to construct a connector instance from a generic
+  mapping that includes a ``type`` key.
 
 Notes
 -----
 - TypedDict shapes are editor hints; runtime parsing remains permissive
-    (from_obj accepts Mapping[str, Any]).
-- TypedDicts referenced in :mod:`etlplus.workflow.types` remain editor hints.
-    Runtime parsing stays permissive and tolerant.
+  (from_obj accepts Mapping[str, Any]).
+- TypedDicts referenced in :mod:`etlplus.config.types` remain editor hints.
+  Runtime parsing stays permissive and tolerant.
 
 See Also
 --------
 - TypedDict shapes for editor hints (not enforced at runtime):
-    :mod:`etlplus.workflow.types.ConnectorApiConfigMap`,
-    :mod:`etlplus.workflow.types.ConnectorDbConfigMap`,
-    :mod:`etlplus.workflow.types.ConnectorFileConfigMap`.
+    :mod:`etlplus.config.types.ConnectorApiConfigMap`,
+    :mod:`etlplus.config.types.ConnectorDbConfigMap`,
+    :mod:`etlplus.config.types.ConnectorFileConfigMap`.
 """
 
 from __future__ import annotations
@@ -60,7 +59,7 @@ if TYPE_CHECKING:  # Editor-only typing hints to avoid runtime imports
 
 
 __all__ = [
-    # Data Classes
+    # Classes
     'ConnectorApi',
     'ConnectorDb',
     'ConnectorFile',
@@ -71,40 +70,6 @@ __all__ = [
 ]
 
 
-# SECTION: INTERNAL FUNCTIONS ============================================== #
-
-
-def _require_name(
-    obj: StrAnyMap,
-    *,
-    kind: str,
-) -> str:
-    """
-    Extract and validate the ``name`` field from connector mappings.
-
-    Parameters
-    ----------
-    obj : StrAnyMap
-        Connector mapping with a ``name`` entry.
-    kind : str
-        Connector kind used in the error message.
-
-    Returns
-    -------
-    str
-        Valid connector name.
-
-    Raises
-    ------
-    TypeError
-        If ``name`` is missing or not a string.
-    """
-    name = obj.get('name')
-    if not isinstance(name, str):
-        raise TypeError(f'Connector{kind} requires a "name" (str)')
-    return name
-
-
 # SECTION: DATA CLASSES ===================================================== #
 
 
@@ -118,12 +83,12 @@ class ConnectorApi:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``'api'``.
+        Connector kind literal, always ``"api"``.
     url : str | None
         Direct absolute URL (when not using ``service``/``endpoint`` refs).
     method : str | None
         Optional HTTP method; typically omitted for sources (defaults to
-        GET) and used for targets (e.g., ``'post'``).
+        GET) and used for targets (e.g., ``"post"``).
     headers : dict[str, str]
         Additional request headers.
     query_params : dict[str, Any]
@@ -146,7 +111,7 @@ class ConnectorApi:
 
     # Direct form
     url: str | None = None
-    # Optional HTTP method; typically omitted for sources (defaults to GET)
+    # Optional HTTP method; typically omitted for sources (defaults to GET
     # at runtime) and used for targets (e.g., 'post', 'put').
     method: str | None = None
     headers: dict[str, str] = field(default_factory=dict)
@@ -185,8 +150,15 @@ class ConnectorApi:
         -------
         Self
             Parsed connector instance.
+
+        Raises
+        ------
+        TypeError
+            If ``name`` is missing or invalid.
         """
-        name = _require_name(obj, kind='Api')
+        name = obj.get('name')
+        if not isinstance(name, str):
+            raise TypeError('ConnectorApi requires a "name" (str)')
         headers = cast_str_dict(obj.get('headers'))
 
         return cls(
@@ -213,7 +185,7 @@ class ConnectorDb:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``'database'``.
+        Connector kind literal, always ``"database"``.
     connection_string : str | None
         Connection string/DSN for the database.
     query : str | None
@@ -221,7 +193,7 @@ class ConnectorDb:
     table : str | None
         Target/source table name (optional).
     mode : str | None
-        Load mode hint (e.g., ``'append'``, ``'replace'``) — future use.
+        Load mode hint (e.g., ``"append"``, ``"replace"``) — future use.
     """
 
     # -- Attributes -- #
@@ -260,8 +232,15 @@ class ConnectorDb:
         -------
         Self
             Parsed connector instance.
+
+        Raises
+        ------
+        TypeError
+            If ``name`` is missing or invalid.
         """
-        name = _require_name(obj, kind='Db')
+        name = obj.get('name')
+        if not isinstance(name, str):
+            raise TypeError('ConnectorDb requires a "name" (str)')
 
         return cls(
             name=name,
@@ -283,9 +262,9 @@ class ConnectorFile:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``'file'``.
+        Connector kind literal, always ``"file"``.
     format : str | None
-        File format (e.g., ``'json'``, ``'csv'``).
+        File format (e.g., ``"json"``, ``"csv"``).
     path : str | None
         File path or URI.
     options : dict[str, Any]
@@ -327,8 +306,15 @@ class ConnectorFile:
         -------
         Self
             Parsed connector instance.
+
+        Raises
+        ------
+        TypeError
+            If ``name`` is missing or invalid.
         """
-        name = _require_name(obj, kind='File')
+        name = obj.get('name')
+        if not isinstance(name, str):
+            raise TypeError('ConnectorFile requires a "name" (str)')
 
         return cls(
             name=name,