etlplus 0.3.25__py3-none-any.whl → 0.4.6__py3-none-any.whl

etlplus/cli/main.py

@@ -0,0 +1,461 @@
+"""
+:mod:`etlplus.cli.main` module.
+
+Entry point helpers for the Typer-powered ``etlplus`` CLI.
+
+This module exposes :func:`main` for the console script as well as
+:func:`create_parser` for callers that still need an ``argparse`` parser.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from typing import Literal
+
+import typer
+
+from .. import __version__
+from ..enums import DataConnectorType
+from ..enums import FileFormat
+from ..utils import json_type
+from .app import PROJECT_URL
+from .app import app
+from .handlers import cmd_extract
+from .handlers import cmd_list
+from .handlers import cmd_load
+from .handlers import cmd_pipeline
+from .handlers import cmd_run
+from .handlers import cmd_transform
+from .handlers import cmd_validate
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'create_parser',
+    'main',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+type FormatContext = Literal['source', 'target']
+
+
+# SECTION: INTERNAL CLASSES ================================================= #
+
+
+class _FormatAction(argparse.Action):
+    """
+    Argparse action that records when ``--source-format`` or
+    ``--target-format`` is provided."""
+
+    def __call__(
+        self,
+        parser: argparse.ArgumentParser,
+        namespace: argparse.Namespace,
+        values: str | Sequence[object] | None,
+        option_string: str | None = None,
+    ) -> None:  # pragma: no cover
+        setattr(namespace, self.dest, values)
+        namespace._format_explicit = True
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _add_config_option(
+    parser: argparse.ArgumentParser,
+    *,
+    required: bool = True,
+) -> None:
+    """Attach the shared ``--config`` option used by legacy commands.
+
+    Parameters
+    ----------
+    parser : argparse.ArgumentParser
+        Parser receiving the option.
+    required : bool, optional
+        Whether the flag must be provided. Defaults to ``True``.
+    """
+
+    parser.add_argument(
+        '--config',
+        required=required,
+        help='Path to pipeline YAML configuration file',
+    )
+
+
+def _add_format_options(
+    parser: argparse.ArgumentParser,
+    *,
+    context: FormatContext,
+) -> None:
+    """
+    Attach shared ``--source-format`` or ``--target-format`` options to
+    extract/load parsers.
+
+    Parameters
+    ----------
+    parser : argparse.ArgumentParser
+        Parser to augment.
+    context : FormatContext
+        Context for the format option: either ``'source'`` or ``'target'``
+    """
+    parser.set_defaults(_format_explicit=False)
+    parser.add_argument(
+        '--source-format',
+        choices=list(FileFormat.choices()),
+        default='json',
+        action=_FormatAction,
+        help=(
+            f'Format of the {context}. Overrides filename-based inference '
+            'when provided.'
+        ),
+    )
+    parser.add_argument(
+        '--target-format',
+        choices=list(FileFormat.choices()),
+        default='json',
+        action=_FormatAction,
+        help=(
+            f'Format of the {context}. Overrides filename-based inference '
+            'when provided.'
+        ),
+    )
+
+
+def _add_boolean_flag(
+    parser: argparse.ArgumentParser,
+    *,
+    name: str,
+    help_text: str,
+) -> None:
+    """Add a toggle that also supports the ``--no-`` prefix via 3.13.
+
+    Parameters
+    ----------
+    parser : argparse.ArgumentParser
+        Parser receiving the flag.
+    name : str
+        Primary flag name without leading dashes.
+    help_text : str
+        Help text rendered in ``--help`` output.
+    """
+
+    parser.add_argument(
+        f'--{name}',
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        help=help_text,
+    )
+
+
+def _cli_description() -> str:
+    return '\n'.join(
+        [
+            'ETLPlus - A Swiss Army knife for simple ETL operations.',
+            '',
+            '    Provide a subcommand and options. Examples:',
+            '',
+            '    etlplus extract file in.csv > out.json',
+            '    etlplus validate in.json --rules \'{"required": ["id"]}\'',
+            (
+                '    etlplus transform --from file in.csv --operations '
+                '\'{"select": ["id"]}\' --to file -o out.json'
+            ),
+            '    etlplus extract in.csv | etlplus load --to file out.json',
+            '',
+            '    Override format inference when extensions are misleading:',
+            '',
+            '    etlplus extract data.txt --source-format csv',
+            '    etlplus load payload.bin --target-format json',
+        ],
+    )
+
+
+def _cli_epilog() -> str:
+    return '\n'.join(
+        [
+            'Tip:',
+            '    --source-format and --target-format override format '
+            'inference based on filename extensions when needed.',
+        ],
+    )
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """
+    Return the legacy :mod:`argparse` parser wired to current handlers.
+
+    Returns
+    -------
+    argparse.ArgumentParser
+        Parser compatible with historical ``etlplus`` entry points.
+    """
+
+    parser = argparse.ArgumentParser(
+        prog='etlplus',
+        description=_cli_description(),
+        epilog=_cli_epilog(),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        '-V',
+        '--version',
+        action='version',
+        version=f'%(prog)s {__version__}',
+    )
+
+    subparsers = parser.add_subparsers(
+        dest='command',
+        help='Available commands',
+    )
+    subparsers.required = True
+
+    extract_parser = subparsers.add_parser(
+        'extract',
+        help='Extract data from sources (files, databases, REST APIs)',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    extract_parser.add_argument(
+        'source_type',
+        choices=list(DataConnectorType.choices()),
+        help='Type of source to extract from',
+    )
+    extract_parser.add_argument(
+        'source',
+        help=(
+            'Source location (file path, database connection string, '
+            'or API URL)'
+        ),
+    )
+    _add_format_options(extract_parser, context='source')
+    extract_parser.set_defaults(func=cmd_extract)
+
+    validate_parser = subparsers.add_parser(
+        'validate',
+        help='Validate data from sources',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    validate_parser.add_argument(
+        'source',
+        help='Data source to validate (file path or JSON string)',
+    )
+    validate_parser.add_argument(
+        '--rules',
+        type=json_type,
+        default={},
+        help='Validation rules as JSON string',
+    )
+    validate_parser.set_defaults(func=cmd_validate)
+
+    transform_parser = subparsers.add_parser(
+        'transform',
+        help='Transform data',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    transform_parser.add_argument(
+        'source',
+        help='Data source to transform (file path or JSON string)',
+    )
+    transform_parser.add_argument(
+        '--operations',
+        type=json_type,
+        default={},
+        help='Transformation operations as JSON string',
+    )
+    transform_parser.add_argument(
+        '--from',
+        dest='from_',
+        choices=list(DataConnectorType.choices()),
+        help='Override the inferred source type (file, database, api).',
+    )
+    transform_parser.add_argument(
+        '--to',
+        dest='to',
+        choices=list(DataConnectorType.choices()),
+        help='Override the inferred target type (file, database, api).',
+    )
+    transform_parser.add_argument(
+        '--source-format',
+        choices=list(FileFormat.choices()),
+        dest='source_format',
+        help=(
+            'Input payload format when SOURCE is - or a literal payload. '
+            'File sources infer format from the extension.'
+        ),
+    )
+    transform_parser.add_argument(
+        '--target-format',
+        dest='target_format',
+        choices=list(FileFormat.choices()),
+        help=(
+            'Output payload format '
+            'when writing to stdout or non-file targets. '
+            'File targets infer format from the extension.'
+        ),
+    )
+    transform_parser.set_defaults(func=cmd_transform)
+
+    load_parser = subparsers.add_parser(
+        'load',
+        help='Load data to targets (files, databases, REST APIs)',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    load_parser.add_argument(
+        'source',
+        help='Data source to load (file path or JSON string)',
+    )
+    load_parser.add_argument(
+        'target_type',
+        choices=list(DataConnectorType.choices()),
+        help='Type of target to load to',
+    )
+    load_parser.add_argument(
+        'target',
+        help=(
+            'Target location (file path, database connection string, '
+            'or API URL)'
+        ),
+    )
+    _add_format_options(load_parser, context='target')
+    load_parser.set_defaults(func=cmd_load)
+
+    pipe_parser = subparsers.add_parser(
+        'pipeline',
+        help=(
+            'Inspect or run pipeline YAML (see '
+            f'{PROJECT_URL}/blob/main/docs/pipeline-guide.md)'
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    _add_config_option(pipe_parser)
+    pipe_parser.add_argument(
+        '--list',
+        action='store_true',
+        help='List available job names and exit',
+    )
+    pipe_parser.add_argument(
+        '--run',
+        metavar='JOB',
+        help='Run a specific job by name',
+    )
+    pipe_parser.set_defaults(func=cmd_pipeline)
+
+    list_parser = subparsers.add_parser(
+        'list',
+        help='List ETL pipeline metadata',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    _add_config_option(list_parser)
+    _add_boolean_flag(
+        list_parser,
+        name='pipelines',
+        help_text='List ETL pipelines',
+    )
+    _add_boolean_flag(
+        list_parser,
+        name='sources',
+        help_text='List data sources',
+    )
+    _add_boolean_flag(
+        list_parser,
+        name='targets',
+        help_text='List data targets',
+    )
+    _add_boolean_flag(
+        list_parser,
+        name='transforms',
+        help_text='List data transforms',
+    )
+    list_parser.set_defaults(func=cmd_list)
+
+    run_parser = subparsers.add_parser(
+        'run',
+        help=(
+            'Run an ETL pipeline '
+            f'(see {PROJECT_URL}/blob/main/docs/run-module.md)'
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    _add_config_option(run_parser)
+    run_parser.add_argument(
+        '-j',
+        '--job',
+        help='Name of the job to run',
+    )
+    run_parser.add_argument(
+        '-p',
+        '--pipeline',
+        help='Name of the pipeline to run',
+    )
+    run_parser.set_defaults(func=cmd_run)
+
+    return parser
+
+
+def main(
+    argv: list[str] | None = None,
+) -> int:
+    """
+    Run the Typer-powered CLI and normalize exit codes.
+
+    Parameters
+    ----------
+    argv : list[str] | None, optional
+        Sequence of command-line arguments excluding the program name. When
+        ``None``, defaults to ``sys.argv[1:]``.
+
+    Returns
+    -------
+    int
+        A conventional POSIX exit code: zero on success, non-zero on error.
+
+    Raises
+    ------
+    SystemExit
+        Re-raises SystemExit exceptions to preserve exit codes.
+
+    Notes
+    -----
+    This function uses Typer (Click) for parsing/dispatch, but preserves the
+    existing `cmd_*` handlers by adapting parsed arguments into an
+    :class:`argparse.Namespace`.
+    """
+    resolved_argv = sys.argv[1:] if argv is None else list(argv)
+    command = typer.main.get_command(app)
+
+    try:
+        result = command.main(
+            args=resolved_argv,
+            prog_name='etlplus',
+            standalone_mode=False,
+        )
+        return int(result or 0)
+
+    except typer.Exit as exc:
+        return int(exc.exit_code)
+
+    except typer.Abort:
+        return 1
+
+    except KeyboardInterrupt:  # pragma: no cover - interactive path
+        # Conventional exit code for SIGINT
+        return 130
+
+    except SystemExit as e:
+        print(f'Error: {e}', file=sys.stderr)
+        raise e
+
+    except (OSError, TypeError, ValueError) as e:
+        print(f'Error: {e}', file=sys.stderr)
+        return 1

{etlplus-0.3.25.dist-info → etlplus-0.4.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.3.25
+Version: 0.4.6
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team
@@ -22,6 +22,7 @@ Requires-Dist: pyodbc>=5.3.0
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: requests>=2.32.5
+Requires-Dist: typer>=0.21.0
 Provides-Extra: dev
 Requires-Dist: black>=25.9.0; extra == "dev"
 Requires-Dist: build>=1.2.2; extra == "dev"
@@ -66,7 +67,7 @@ package and command-line interface for data extraction, validation, transformati
       - [Load Data](#load-data)
     - [Python API](#python-api)
     - [Complete ETL Pipeline Example](#complete-etl-pipeline-example)
-    - [Environment Variables](#environment-variables)
+    - [Format Overrides](#format-overrides)
   - [Transformation Operations](#transformation-operations)
     - [Filter Operations](#filter-operations)
     - [Aggregation Functions](#aggregation-functions)
@@ -78,6 +79,8 @@ package and command-line interface for data extraction, validation, transformati
       - [Test Layers](#test-layers)
     - [Code Coverage](#code-coverage)
     - [Linting](#linting)
+    - [Updating Demo Snippets](#updating-demo-snippets)
+    - [Releasing to PyPI](#releasing-to-pypi)
   - [Links](#links)
   - [License](#license)
   - [Contributing](#contributing)
@@ -168,9 +171,9 @@ etlplus --version
 
 #### Extract Data
 
-Note: For file sources, the format is inferred from the filename extension; the `--format` option is
-ignored.  To treat passing `--format` as an error for file sources, either set
-`ETLPLUS_FORMAT_BEHAVIOR=error` or pass the CLI flag `--strict-format`.
+Note: For file sources, the format is normally inferred from the filename extension. Use
+`--source-format` to override inference when a file lacks an extension or when you want to force a
+specific parser.
 
 Extract from JSON file:
 ```bash
@@ -211,6 +214,20 @@ etlplus validate examples/data/sample.json --rules '{"email": {"type": "string",
 
 #### Transform Data
 
+When piping data through `etlplus transform`, use `--source-format` whenever the SOURCE argument is
+`-` or a literal payload, mirroring the `etlplus extract` semantics. Use `--target-format` to
+control the emitted format for stdout or other non-file outputs, just like `etlplus load`. File
+paths continue to infer formats from their extensions. Use `--from` to override the inferred source
+connector type and `--to` to override the inferred target connector type, matching the `etlplus
+extract`/`etlplus load` behavior.
+
+Transform file inputs while overriding connector types:
+```bash
+etlplus transform --from file examples/data/sample.json \
+  --operations '{"select": ["name", "email"]}' \
+  --to file -o temp/selected_output.json
+```
+
 Filter and select fields:
 ```bash
 etlplus transform '[{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]' \
@@ -234,19 +251,24 @@ etlplus transform examples/data/sample.json --operations '{"map": {"name": "new_
 
 #### Load Data
 
+`etlplus load` consumes JSON from stdin; provide only the target argument plus optional flags.
+
 Load to JSON file:
 ```bash
-etlplus load '{"name": "John", "age": 30}' file temp/sample_output.json
+etlplus extract file examples/data/sample.json \
+  | etlplus load --to file temp/sample_output.json
 ```
 
 Load to CSV file:
 ```bash
-etlplus load '[{"name": "John", "age": 30}]' file temp/sample_output.csv
+etlplus extract file examples/data/sample.csv \
+  | etlplus load --to file temp/sample_output.csv
 ```
 
 Load to REST API:
 ```bash
-etlplus load examples/data/sample.json api https://api.example.com/endpoint
+cat examples/data/sample.json \
+  | etlplus load --to api https://api.example.com/endpoint
 ```
 
 ### Python API
@@ -300,41 +322,28 @@ etlplus validate temp/sample_transformed.json \
   --rules '{"name": {"type": "string", "required": true}, "email": {"type": "string", "required": true}}'
 
 # 4. Load to CSV
-etlplus load temp/sample_transformed.json file temp/sample_output.csv
+cat temp/sample_transformed.json \
+  | etlplus load --to temp/sample_output.csv
 ```
 
-### Environment Variables
-
-ETLPlus honors a small number of environment toggles to refine CLI behavior:
+### Format Overrides
 
-- `ETLPLUS_FORMAT_BEHAVIOR`: controls what happens when `--format` is provided for
-  file sources or targets (extract/load) where the format is inferred from the
-  filename extension.
-  - `error|fail|strict`: treat as error (non-zero exit)
-  - `warn` (default): print a warning to stderr
-  - `ignore|silent`: no message
-- Precedence: the CLI flag `--strict-format` overrides the environment.
+`--source-format` and `--target-format` override whichever format would normally be inferred from a
+file extension. This is useful when an input lacks an extension (for example, `records.txt` that
+actually contains CSV) or when you intentionally want to treat a file as another format.
 
 Examples (zsh):
 
 ```zsh
-# Warn (default)
-etlplus extract file data.csv --format csv
-etlplus load data.json file out.csv --format csv
-
-# Enforce error via environment
-ETLPLUS_FORMAT_BEHAVIOR=error \
-  etlplus extract file data.csv --format csv
-ETLPLUS_FORMAT_BEHAVIOR=error \
-  etlplus load data.json file out.csv --format csv
-
-# Equivalent strict behavior via flag (overrides environment)
-etlplus extract file data.csv --format csv --strict-format
-etlplus load data.json file out.csv --format csv --strict-format
-
-# Recommended: rely on extension, no --format needed for files
-etlplus extract file data.csv
-etlplus load data.json file out.csv
+# Force CSV parsing for an extension-less file
+etlplus extract --from file data.txt --source-format csv
+
+# Write CSV to a file without the .csv suffix
+etlplus load --to file output.bin --target-format csv < data.json
+
+# Leave the flags off when extensions already match the desired format
+etlplus extract --from file data.csv
+etlplus load --to file data.json < data.json
 ```
 
 ## Transformation Operations

{etlplus-0.3.25.dist-info → etlplus-0.4.6.dist-info}/RECORD

@@ -1,7 +1,6 @@
 etlplus/__init__.py,sha256=M2gScnyir6WOMAh_EuoQIiAzdcTls0_5hbd_Q6of8I0,1021
-etlplus/__main__.py,sha256=lSbVOF5Mnd_ljmCqK7nTuF_MRDYTkL73eZEFeUQ_vnI,510
+etlplus/__main__.py,sha256=btoROneNiigyfBU7BSzPKZ1R9gzBMpxcpsbPwmuHwTM,479
 etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
-etlplus/cli.py,sha256=dmMW5dLbFiDRGne97qDsqr3YuU30g_Ekl_vo7bgDLig,21752
 etlplus/enums.py,sha256=V_j18Ud2BCXpFsBk2pZGrvCVrvAMJ7uja1z9fppFGso,10175
 etlplus/extract.py,sha256=f44JdHhNTACxgn44USx05paKTwq7LQY-V4wANCW9hVM,6173
 etlplus/file.py,sha256=RxIAsGDN4f_vNA2B5-ct88JNd_ISAyYbooIRE5DstS8,17972
@@ -31,6 +30,10 @@ etlplus/api/pagination/paginator.py,sha256=wtdY_er4yfjx5yTUQJ1gPq-IuWmpLAHeG5buB
 etlplus/api/rate_limiting/__init__.py,sha256=ZySB1dZettEDnWvI1EHf_TZ9L08M_kKsNR-Y_lbU6kI,1070
 etlplus/api/rate_limiting/config.py,sha256=2b4wIynblN-1EyMqI4aXa71SljzSjXYh5N1Nngr3jOg,9406
 etlplus/api/rate_limiting/rate_limiter.py,sha256=Uxozqd_Ej5Lsj-M-mLT2WexChgWh7x35_YP10yqYPQA,7159
+etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
+etlplus/cli/app.py,sha256=pc9VDUb3Qc8u5-XyDrHJkrSR9D3bq4e9zLbaD8KzyfY,32618
+etlplus/cli/handlers.py,sha256=aI_ZlnJCGGkVnVJJPhmPRCXc31MxtLaOeqqJoo3ci48,15816
+etlplus/cli/main.py,sha256=9hoitdc9FisrXzwZniTglPWwKsODFAW-A-2QQV4NkBs,12565
 etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
 etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
 etlplus/config/jobs.py,sha256=hmzRCqt0OvCEZZR4ONKrd3lvSv0OmayjLc4yOBk3ug8,7399
@@ -40,9 +43,9 @@ etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
 etlplus/config/utils.py,sha256=4SUHMkt5bKBhMhiJm-DrnmE2Q4TfOgdNCKz8PJDS27o,3443
 etlplus/validation/__init__.py,sha256=Pe5Xg1_EA4uiNZGYu5WTF3j7odjmyxnAJ8rcioaplSQ,1254
 etlplus/validation/utils.py,sha256=Mtqg449VIke0ziy_wd2r6yrwJzQkA1iulZC87FzXMjo,10201
-etlplus-0.3.25.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
-etlplus-0.3.25.dist-info/METADATA,sha256=QehDLWQ0MN7sUykhLs9gQgOrT3ru_QgaEFftJCkgNgw,16730
-etlplus-0.3.25.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-etlplus-0.3.25.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
-etlplus-0.3.25.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
-etlplus-0.3.25.dist-info/RECORD,,
+etlplus-0.4.6.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
+etlplus-0.4.6.dist-info/METADATA,sha256=oq0-zYAVKNkZFCBPecObqjJ2LuYcbSX5kVjCErpiDO4,17278
+etlplus-0.4.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+etlplus-0.4.6.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
+etlplus-0.4.6.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
+etlplus-0.4.6.dist-info/RECORD,,