etlplus 0.9.1__py3-none-any.whl → 0.9.2__py3-none-any.whl

etlplus/cli/commands.py

@@ -36,7 +36,7 @@ from typing import cast
 import typer
 
 from .. import __version__
-from ..enums import FileFormat
+from ..file import FileFormat
 from . import handlers
 from .constants import CLI_DESCRIPTION
 from .constants import CLI_EPILOG
@@ -61,6 +61,24 @@ __all__ = ['app']
 
 # SECTION: TYPE ALIASES ==================================================== #
 
+
+JobOption = Annotated[
+    str | None,
+    typer.Option(
+        '-j',
+        '--job',
+        help='Name of the job to run',
+    ),
+]
+
+JobsOption = Annotated[
+    bool,
+    typer.Option(
+        '--jobs',
+        help='List available job names and exit',
+    ),
+]
+
 OperationsOption = Annotated[
     str,
     typer.Option(
@@ -89,6 +107,23 @@ PipelineConfigOption = Annotated[
     ),
 ]
 
+PipelineOption = Annotated[
+    str | None,
+    typer.Option(
+        '-p',
+        '--pipeline',
+        help='Name of the pipeline to run',
+    ),
+]
+
+PipelinesOption = Annotated[
+    bool,
+    typer.Option(
+        '--pipelines',
+        help='List ETL pipelines',
+    ),
+]
+
 RenderConfigOption = Annotated[
     str | None,
     typer.Option(
@@ -193,6 +228,22 @@ SourceTypeOption = Annotated[
     ),
 ]
 
+SourcesOption = Annotated[
+    bool,
+    typer.Option(
+        '--sources',
+        help='List data sources',
+    ),
+]
+
+SummaryOption = Annotated[
+    bool,
+    typer.Option(
+        '--summary',
+        help='Show pipeline summary (name, version, sources, targets, jobs)',
+    ),
+]
+
 TargetArg = Annotated[
     str,
     typer.Argument(
@@ -227,6 +278,22 @@ TargetTypeOption = Annotated[
     ),
 ]
 
+TargetsOption = Annotated[
+    bool,
+    typer.Option(
+        '--targets',
+        help='List data targets',
+    ),
+]
+
+TransformsOption = Annotated[
+    bool,
+    typer.Option(
+        '--transforms',
+        help='List data transforms',
+    ),
+]
+
 
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
@@ -341,36 +408,12 @@ def _root(
 def check_cmd(
     ctx: typer.Context,
     config: PipelineConfigOption,
-    jobs: bool = typer.Option(
-        False,
-        '--jobs',
-        help='List available job names and exit',
-    ),
-    pipelines: bool = typer.Option(
-        False,
-        '--pipelines',
-        help='List ETL pipelines',
-    ),
-    sources: bool = typer.Option(
-        False,
-        '--sources',
-        help='List data sources',
-    ),
-    summary: bool = typer.Option(
-        False,
-        '--summary',
-        help='Show pipeline summary (name, version, sources, targets, jobs)',
-    ),
-    targets: bool = typer.Option(
-        False,
-        '--targets',
-        help='List data targets',
-    ),
-    transforms: bool = typer.Option(
-        False,
-        '--transforms',
-        help='List data transforms',
-    ),
+    jobs: JobsOption = False,
+    pipelines: PipelinesOption = False,
+    sources: SourcesOption = False,
+    summary: SummaryOption = False,
+    targets: TargetsOption = False,
+    transforms: TransformsOption = False,
 ) -> int:
     """
     Inspect a pipeline configuration.
@@ -683,18 +726,8 @@ def render_cmd(
 def run_cmd(
     ctx: typer.Context,
     config: PipelineConfigOption,
-    job: str | None = typer.Option(
-        None,
-        '-j',
-        '--job',
-        help='Name of the job to run',
-    ),
-    pipeline: str | None = typer.Option(
-        None,
-        '-p',
-        '--pipeline',
-        help='Name of the pipeline to run',
-    ),
+    job: JobOption = None,
+    pipeline: PipelineOption = None,
 ) -> int:
     """
     Execute an ETL job or pipeline from a YAML configuration.

etlplus/cli/constants.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 from typing import Final
 
 from ..enums import DataConnectorType
-from ..enums import FileFormat
+from ..file import FileFormat
 
 # SECTION: EXPORTS ========================================================== #
 

etlplus/cli/handlers.py

@@ -14,19 +14,20 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
-from ..config import PipelineConfig
-from ..config import load_pipeline_config
 from ..database import load_table_spec
 from ..database import render_tables
-from ..extract import extract
 from ..file import File
-from ..load import load
-from ..run import run
-from ..transform import transform
+from ..file import FileFormat
+from ..ops import extract
+from ..ops import load
+from ..ops import run
+from ..ops import transform
+from ..ops import validate
+from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
-from ..validate import FieldRules
-from ..validate import validate
+from ..workflow import PipelineConfig
+from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -120,9 +121,12 @@ def _check_sections(
     if targets:
         sections['targets'] = [tgt.name for tgt in cfg.targets]
     if transforms:
-        sections['transforms'] = [
-            getattr(trf, 'name', None) for trf in cfg.transforms
-        ]
+        if isinstance(cfg.transforms, Mapping):
+            sections['transforms'] = list(cfg.transforms)
+        else:
+            sections['transforms'] = [
+                getattr(trf, 'name', None) for trf in cfg.transforms
+            ]
     if not sections:
         sections['jobs'] = _pipeline_summary(cfg)['jobs']
     return sections
@@ -156,6 +160,29 @@ def _pipeline_summary(
     }
 
 
+def _write_file_payload(
+    payload: JSONData,
+    target: str,
+    *,
+    format_hint: str | None,
+) -> None:
+    """
+    Write a JSON-like payload to a file path using an optional format hint.
+
+    Parameters
+    ----------
+    payload : JSONData
+        The structured data to write.
+    target : str
+        File path to write to.
+    format_hint : str | None
+        Optional format hint for :class:`FileFormat`.
+    """
+    file_path = Path(target)
+    file_format = FileFormat.coerce(format_hint) if format_hint else None
+    File(file_path, file_format=file_format).write(payload)
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -569,8 +596,9 @@ def transform_handler(
 
     data = transform(payload, cast(TransformOperations, operations_payload))
 
+    # TODO: Generalize to handle non-file targets.
     if target and target != '-':
-        File.write_file(target, data, file_format=target_format)
+        _write_file_payload(data, target, format_hint=target_format)
         print(f'Data transformed and saved to {target}')
         return 0
 

etlplus/cli/io.py

@@ -15,8 +15,8 @@ from pathlib import Path
 from typing import Any
 from typing import cast
 
-from ..enums import FileFormat
 from ..file import File
+from ..file import FileFormat
 from ..types import JSONData
 from ..utils import print_json
 
@@ -331,6 +331,6 @@ def write_json_output(
     """
     if not output_path or output_path == '-':
         return False
-    File(Path(output_path), FileFormat.JSON).write_json(data)
+    File(Path(output_path), FileFormat.JSON).write(data)
     print(f'{success_message} {output_path}')
     return True

etlplus/cli/main.py

@@ -50,7 +50,7 @@ def _emit_context_help(
         return False
 
     with contextlib.redirect_stdout(sys.stderr):
-        ctx.get_help()
+        print(ctx.get_help())
     return True
 
 

etlplus/cli/state.py

@@ -15,6 +15,7 @@ from typing import Final
 
 import typer
 
+from ..utils import normalize_str
 from .constants import DATA_CONNECTORS
 
 # SECTION: EXPORTS ========================================================== #
@@ -322,14 +323,10 @@ def validate_choice(
     typer.BadParameter
         If the input value is not in the set of valid choices.
     """
-    v = str(value or '').strip().lower()
-    normalized_choices = {c.lower() for c in choices}
+    v = normalize_str(str(value or ''))
+    normalized_choices = {normalize_str(c): c for c in choices}
     if v in normalized_choices:
-        # Preserve original casing from choices when possible for messages
-        for choice in choices:
-            if choice.lower() == v:
-                return choice
-        return v
+        return normalized_choices[v]
     allowed = ', '.join(sorted(choices))
     raise typer.BadParameter(
         f"Invalid {label} '{value}'. Choose from: {allowed}",

etlplus/database/README.md

@@ -0,0 +1,48 @@
+# `etlplus.database` Subpackage
+
+Documentation for the `etlplus.database` subpackage: database engine, schema, and ORM helpers.
+
+- Provides database engine and connection management
+- Supports schema definition and DDL generation
+- Includes lightweight ORM utilities for tabular data
+- Exposes type definitions for database objects
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [`etlplus.database` Subpackage](#etlplusdatabase-subpackage)
+  - [Database Engine and Connections](#database-engine-and-connections)
+  - [Schema and DDL Helpers](#schema-and-ddl-helpers)
+  - [ORM Utilities](#orm-utilities)
+  - [Example: Creating a Table](#example-creating-a-table)
+  - [See Also](#see-also)
+
+## Database Engine and Connections
+
+- Manage connections to supported databases
+- Configure engines for different backends
+
+## Schema and DDL Helpers
+
+- Define table schemas and columns
+- Generate DDL statements for supported databases
+
+## ORM Utilities
+
+- Map rows to Python objects
+- Simple CRUD helpers for tabular data
+
+## Example: Creating a Table
+
+```python
+from etlplus.database import Schema, Engine
+
+engine = Engine.connect("sqlite:///example.db")
+schema = Schema.from_dict({"name": "users", "columns": [ ... ]})
+engine.create_table(schema)
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- Schema helpers in [schema.py](schema.py)
+- ORM utilities in [orm.py](orm.py)

etlplus/database/ddl.py

@@ -203,7 +203,7 @@ def load_table_spec(
         raise ValueError('Spec must be .json, .yml, or .yaml')
 
     try:
-        spec = File.read_file(spec_path)
+        spec = File(spec_path).read()
     except ImportError as e:
         if suffix in {'.yml', '.yaml'}:
             raise RuntimeError(

etlplus/database/engine.py

@@ -113,7 +113,7 @@ def load_database_url_from_config(
     ValueError
         If no connection string/URL/DSN is found for the specified entry.
     """
-    cfg = File.read_file(Path(path))
+    cfg = File(Path(path)).read()
     if not isinstance(cfg, Mapping):
         raise TypeError('Database config must be a mapping')
 
@@ -136,9 +136,25 @@ def load_database_url_from_config(
     return url
 
 
-def make_engine(url: str | None = None, **engine_kwargs: Any) -> Engine:
-    """Create a SQLAlchemy Engine, defaulting to env config if no URL given."""
+def make_engine(
+    url: str | None = None,
+    **engine_kwargs: Any,
+) -> Engine:
+    """
+    Create a SQLAlchemy Engine, defaulting to env config if no URL given.
+
+    Parameters
+    ----------
+    url : str | None, optional
+        Database URL/DSN string. When omitted, ``DATABASE_URL`` is used.
+    **engine_kwargs : Any
+        Extra keyword arguments forwarded to ``create_engine``.
 
+    Returns
+    -------
+    Engine
+        Configured SQLAlchemy engine instance.
+    """
     resolved_url = url or DATABASE_URL
     return create_engine(resolved_url, pool_pre_ping=True, **engine_kwargs)
 

etlplus/database/orm.py

@@ -201,12 +201,14 @@ def build_models(
 ) -> ModelRegistry:
     """
     Build SQLAlchemy ORM models from table specifications.
+
     Parameters
     ----------
     specs : list[TableSpec]
         List of table specifications.
     base : type[DeclarativeBase], optional
         Base class for the ORM models (default: :class:`Base`).
+
     Returns
     -------
     ModelRegistry

etlplus/database/schema.py

@@ -260,7 +260,7 @@ def load_table_specs(
     list[TableSpec]
         A list of TableSpec instances parsed from the YAML file.
     """
-    data = File.read_file(Path(path))
+    data = File(Path(path)).read()
     if not data:
         return []
 

etlplus/enums.py

@@ -19,16 +19,12 @@ from .types import StrStrMap
 
 
 __all__ = [
+    # Enums
     'AggregateName',
     'CoercibleStrEnum',
     'DataConnectorType',
-    'FileFormat',
-    'HttpMethod',
     'OperatorName',
     'PipelineStep',
-    'coerce_data_connector_type',
-    'coerce_file_format',
-    'coerce_http_method',
 ]
 
 
@@ -203,119 +199,6 @@ class DataConnectorType(CoercibleStrEnum):
         }
 
 
-class FileFormat(CoercibleStrEnum):
-    """Supported file formats for extraction."""
-
-    # -- Constants -- #
-
-    AVRO = 'avro'
-    CSV = 'csv'
-    FEATHER = 'feather'
-    GZ = 'gz'
-    JSON = 'json'
-    NDJSON = 'ndjson'
-    ORC = 'orc'
-    PARQUET = 'parquet'
-    TSV = 'tsv'
-    TXT = 'txt'
-    XLS = 'xls'
-    XLSX = 'xlsx'
-    ZIP = 'zip'
-    XML = 'xml'
-    YAML = 'yaml'
-
-    # -- Class Methods -- #
-
-    @classmethod
-    def aliases(cls) -> StrStrMap:
-        """
-        Return a mapping of common aliases for each enum member.
-
-        Returns
-        -------
-        StrStrMap
-            A mapping of alias names to their corresponding enum member names.
-        """
-        return {
-            # Common shorthand
-            'parq': 'parquet',
-            'yml': 'yaml',
-            # File extensions
-            '.avro': 'avro',
-            '.csv': 'csv',
-            '.feather': 'feather',
-            '.gz': 'gz',
-            '.json': 'json',
-            '.jsonl': 'ndjson',
-            '.ndjson': 'ndjson',
-            '.orc': 'orc',
-            '.parquet': 'parquet',
-            '.pq': 'parquet',
-            '.tsv': 'tsv',
-            '.txt': 'txt',
-            '.xls': 'xls',
-            '.xlsx': 'xlsx',
-            '.zip': 'zip',
-            '.xml': 'xml',
-            '.yaml': 'yaml',
-            '.yml': 'yaml',
-            # MIME types
-            'application/avro': 'avro',
-            'application/feather': 'feather',
-            'application/gzip': 'gz',
-            'application/json': 'json',
-            'application/jsonlines': 'ndjson',
-            'application/ndjson': 'ndjson',
-            'application/orc': 'orc',
-            'application/vnd.apache.arrow.file': 'feather',
-            'application/vnd.apache.orc': 'orc',
-            'application/vnd.ms-excel': 'xls',
-            (
-                'application/vnd.openxmlformats-'
-                'officedocument.spreadsheetml.sheet'
-            ): 'xlsx',
-            'application/x-avro': 'avro',
-            'application/x-ndjson': 'ndjson',
-            'application/x-parquet': 'parquet',
-            'application/xml': 'xml',
-            'application/zip': 'zip',
-            'text/csv': 'csv',
-            'text/plain': 'txt',
-            'text/tab-separated-values': 'tsv',
-        }
-
-
-class HttpMethod(CoercibleStrEnum):
-    """Supported HTTP verbs that accept JSON payloads."""
-
-    # -- Constants -- #
-
-    CONNECT = 'connect'
-    DELETE = 'delete'
-    GET = 'get'
-    HEAD = 'head'
-    OPTIONS = 'options'
-    PATCH = 'patch'
-    POST = 'post'
-    PUT = 'put'
-    TRACE = 'trace'
-
-    # -- Getters -- #
-
-    @property
-    def allows_body(self) -> bool:
-        """
-        Whether the method typically allows a request body.
-
-        Notes
-        -----
-        - RFCs do not strictly forbid bodies on some other methods (e.g.,
-          ``DELETE``), but many servers/clients do not expect them. We mark
-          ``POST``, ``PUT``, and ``PATCH`` as True.
-        """
-        return self in {HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH}
-
-
 class OperatorName(CoercibleStrEnum):
     """Supported comparison operators with helpers."""
 
@@ -423,42 +306,3 @@ _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
     PipelineStep.SORT: 3,
     PipelineStep.AGGREGATE: 4,
 }
-
-
-# SECTION: FUNCTIONS ======================================================== #
-
-
-def coerce_data_connector_type(
-    connector: DataConnectorType | str,
-) -> DataConnectorType:
-    """
-    Normalize textual data connector values to :class:`DataConnectorType`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`DataConnectorType.coerce` going forward.
-    """
-    return DataConnectorType.coerce(connector)
-
-
-def coerce_file_format(
-    file_format: FileFormat | str,
-) -> FileFormat:
-    """
-    Normalize textual file format values to :class:`FileFormat`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`FileFormat.coerce` going forward.
-    """
-    return FileFormat.coerce(file_format)
-
-
-def coerce_http_method(
-    http_method: HttpMethod | str,
-) -> HttpMethod:
-    """
-    Normalize textual HTTP method values to :class:`HttpMethod`.
-
-    This thin wrapper is kept for backward compatibility; prefer
-    :meth:`HttpMethod.coerce` going forward.
-    """
-    return HttpMethod.coerce(http_method)