etlplus 0.4.7__tar.gz → 0.4.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {etlplus-0.4.7/etlplus.egg-info → etlplus-0.4.9}/PKG-INFO +14 -1
- {etlplus-0.4.7 → etlplus-0.4.9}/README.md +13 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/docs/pipeline-guide.md +8 -7
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/cli/app.py +19 -3
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/cli/handlers.py +12 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/cli/main.py +15 -2
- {etlplus-0.4.7 → etlplus-0.4.9/etlplus.egg-info}/PKG-INFO +14 -1
- {etlplus-0.4.7 → etlplus-0.4.9}/examples/README.md +4 -4
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/test_i_pipeline_smoke.py +4 -3
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/cli/test_u_cli_app.py +207 -186
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/cli/test_u_cli_handlers.py +58 -35
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/cli/test_u_cli_main.py +88 -68
- {etlplus-0.4.7 → etlplus-0.4.9}/.coveragerc +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/.editorconfig +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/.gitattributes +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/.github/actions/python-bootstrap/action.yml +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/.github/workflows/ci.yml +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/.gitignore +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/.pre-commit-config.yaml +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/.ruff.toml +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/CODE_OF_CONDUCT.md +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/CONTRIBUTING.md +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/DEMO.md +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/LICENSE +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/Makefile +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/REFERENCES.md +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/docs/snippets/installation_version.md +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/__init__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/__main__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/__version__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/README.md +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/__init__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/auth.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/config.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/endpoint_client.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/errors.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/pagination/__init__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/pagination/client.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/pagination/config.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/pagination/paginator.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/rate_limiting/__init__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/rate_limiting/config.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/rate_limiting/rate_limiter.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/request_manager.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/retry_manager.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/transport.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/api/types.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/cli/__init__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/config/__init__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/config/connector.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/config/jobs.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/config/pipeline.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/config/profile.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/config/types.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/config/utils.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/enums.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/extract.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/file.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/load.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/mixins.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/py.typed +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/run.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/run_helpers.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/transform.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/types.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/utils.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/validate.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/validation/__init__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus/validation/utils.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus.egg-info/SOURCES.txt +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus.egg-info/dependency_links.txt +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus.egg-info/entry_points.txt +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus.egg-info/requires.txt +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/etlplus.egg-info/top_level.txt +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/examples/configs/pipeline.yml +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/examples/data/sample.csv +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/examples/data/sample.json +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/examples/data/sample.xml +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/examples/data/sample.xsd +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/examples/data/sample.yaml +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/examples/quickstart_python.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/pyproject.toml +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/pytest.ini +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/setup.cfg +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/setup.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/__init__.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/conftest.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/conftest.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/test_i_cli.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/test_i_examples_data_parity.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/test_i_pagination_strategy.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/test_i_pipeline_yaml_load.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/test_i_run.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/test_i_run_profile_pagination_defaults.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/integration/test_i_run_profile_rate_limit_defaults.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/conftest.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_auth.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_config.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_endpoint_client.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_mocks.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_pagination_client.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_pagination_config.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_paginator.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_rate_limit_config.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_rate_limiter.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_request_manager.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_retry_manager.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_transport.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/api/test_u_types.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/cli/conftest.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/config/test_u_config_utils.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/config/test_u_connector.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/config/test_u_jobs.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/config/test_u_pipeline.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/conftest.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_enums.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_extract.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_file.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_load.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_main.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_mixins.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_run.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_run_helpers.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_transform.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_utils.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_validate.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/test_u_version.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tests/unit/validation/test_u_validation_utils.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tools/run_pipeline.py +0 -0
- {etlplus-0.4.7 → etlplus-0.4.9}/tools/update_demo_snippets.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.9
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -306,6 +306,19 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
|
|
|
306
306
|
- Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
307
307
|
- Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
|
|
308
308
|
|
|
309
|
+
CLI quick reference for pipelines:
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
# List jobs or show a pipeline summary
|
|
313
|
+
etlplus list --config examples/configs/pipeline.yml --jobs
|
|
314
|
+
etlplus list --config examples/configs/pipeline.yml --summary
|
|
315
|
+
|
|
316
|
+
# Run a job
|
|
317
|
+
etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
318
|
+
|
|
319
|
+
# Deprecated shim (will be removed): etlplus pipeline
|
|
320
|
+
```
|
|
321
|
+
|
|
309
322
|
### Complete ETL Pipeline Example
|
|
310
323
|
|
|
311
324
|
```bash
|
|
@@ -264,6 +264,19 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
|
|
|
264
264
|
- Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
265
265
|
- Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
|
|
266
266
|
|
|
267
|
+
CLI quick reference for pipelines:
|
|
268
|
+
|
|
269
|
+
```bash
|
|
270
|
+
# List jobs or show a pipeline summary
|
|
271
|
+
etlplus list --config examples/configs/pipeline.yml --jobs
|
|
272
|
+
etlplus list --config examples/configs/pipeline.yml --summary
|
|
273
|
+
|
|
274
|
+
# Run a job
|
|
275
|
+
etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
276
|
+
|
|
277
|
+
# Deprecated shim (will be removed): etlplus pipeline
|
|
278
|
+
```
|
|
279
|
+
|
|
267
280
|
### Complete ETL Pipeline Example
|
|
268
281
|
|
|
269
282
|
```bash
|
|
@@ -378,31 +378,32 @@ jobs:
|
|
|
378
378
|
Once you have a pipeline YAML, you can run jobs either from the
|
|
379
379
|
command line or directly from Python.
|
|
380
380
|
|
|
381
|
-
### CLI: `etlplus
|
|
381
|
+
### CLI: `etlplus list` (inspect) and `etlplus run` (execute)
|
|
382
382
|
|
|
383
|
-
List jobs
|
|
383
|
+
List jobs or show a summary from a pipeline file:
|
|
384
384
|
|
|
385
385
|
```bash
|
|
386
|
-
etlplus
|
|
386
|
+
etlplus list --config examples/configs/pipeline.yml --jobs
|
|
387
|
+
etlplus list --config examples/configs/pipeline.yml --summary
|
|
387
388
|
```
|
|
388
389
|
|
|
389
390
|
Run a specific job end-to-end (extract → validate → transform → load):
|
|
390
391
|
|
|
391
392
|
```bash
|
|
392
|
-
etlplus pipeline --config examples/configs/pipeline.yml --run file_to_file_customers
|
|
393
|
-
|
|
394
|
-
# Equivalent, using the dedicated run command
|
|
395
393
|
etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
396
394
|
```
|
|
397
395
|
|
|
398
396
|
Notes:
|
|
399
397
|
|
|
400
|
-
-
|
|
398
|
+
- These commands read the same YAML schema described in this guide.
|
|
401
399
|
- Environment-variable substitution (e.g. `${GITHUB_TOKEN}`) is applied the same way as when loading
|
|
402
400
|
configs via the Python API.
|
|
403
401
|
- For more details on the orchestration implementation, see
|
|
404
402
|
[Runner internals: etlplus.run](run-module.md).
|
|
405
403
|
|
|
404
|
+
Deprecated: `etlplus pipeline` is still available as a shim but will be removed in a future release;
|
|
405
|
+
prefer `list` and `run`.
|
|
406
|
+
|
|
406
407
|
### Python: `etlplus.run.run`
|
|
407
408
|
|
|
408
409
|
To trigger a job programmatically, use the high-level runner function exposed by the package:
|
|
@@ -782,8 +782,21 @@ def list_cmd(
|
|
|
782
782
|
'--pipelines',
|
|
783
783
|
help='List ETL pipelines',
|
|
784
784
|
),
|
|
785
|
-
sources: bool = typer.Option(
|
|
786
|
-
|
|
785
|
+
sources: bool = typer.Option(
|
|
786
|
+
False,
|
|
787
|
+
'--sources',
|
|
788
|
+
help='List data sources',
|
|
789
|
+
),
|
|
790
|
+
summary: bool = typer.Option(
|
|
791
|
+
False,
|
|
792
|
+
'--summary',
|
|
793
|
+
help='Show pipeline summary (name, version, sources, targets, jobs)',
|
|
794
|
+
),
|
|
795
|
+
targets: bool = typer.Option(
|
|
796
|
+
False,
|
|
797
|
+
'--targets',
|
|
798
|
+
help='List data targets',
|
|
799
|
+
),
|
|
787
800
|
transforms: bool = typer.Option(
|
|
788
801
|
False,
|
|
789
802
|
'--transforms',
|
|
@@ -805,6 +818,8 @@ def list_cmd(
|
|
|
805
818
|
If True, list ETL pipelines.
|
|
806
819
|
sources : bool, optional
|
|
807
820
|
If True, list data sources.
|
|
821
|
+
summary : bool, optional
|
|
822
|
+
If True, show pipeline summary (name, version, sources, targets, jobs).
|
|
808
823
|
targets : bool, optional
|
|
809
824
|
If True, list data targets.
|
|
810
825
|
transforms : bool, optional
|
|
@@ -820,6 +835,7 @@ def list_cmd(
|
|
|
820
835
|
state,
|
|
821
836
|
command='list',
|
|
822
837
|
config=config,
|
|
838
|
+
summary=summary,
|
|
823
839
|
pipelines=pipelines,
|
|
824
840
|
jobs=jobs,
|
|
825
841
|
sources=sources,
|
|
@@ -953,7 +969,7 @@ def pipeline_cmd(
|
|
|
953
969
|
),
|
|
954
970
|
) -> int:
|
|
955
971
|
"""
|
|
956
|
-
|
|
972
|
+
Deprecated wrapper to inspect or run a pipeline YAML configuration.
|
|
957
973
|
|
|
958
974
|
Parameters
|
|
959
975
|
----------
|
|
@@ -117,6 +117,8 @@ def _list_sections(
|
|
|
117
117
|
Metadata output for the list command.
|
|
118
118
|
"""
|
|
119
119
|
sections: dict[str, Any] = {}
|
|
120
|
+
if getattr(args, 'jobs', False):
|
|
121
|
+
sections['jobs'] = _pipeline_summary(cfg)['jobs']
|
|
120
122
|
if getattr(args, 'pipelines', False):
|
|
121
123
|
sections['pipelines'] = [cfg.name]
|
|
122
124
|
if getattr(args, 'sources', False):
|
|
@@ -598,6 +600,12 @@ def cmd_pipeline(
|
|
|
598
600
|
int
|
|
599
601
|
Zero on success.
|
|
600
602
|
"""
|
|
603
|
+
print(
|
|
604
|
+
'DEPRECATED: use "etlplus list --summary|--jobs" or '
|
|
605
|
+
'"etlplus run --job/--pipeline" instead of "etlplus pipeline".',
|
|
606
|
+
file=sys.stderr,
|
|
607
|
+
)
|
|
608
|
+
|
|
601
609
|
cfg = load_pipeline_config(args.config, substitute=True)
|
|
602
610
|
|
|
603
611
|
list_flag = getattr(args, 'list', False) or getattr(args, 'jobs', False)
|
|
@@ -635,6 +643,10 @@ def cmd_list(args: argparse.Namespace) -> int:
|
|
|
635
643
|
Zero on success.
|
|
636
644
|
"""
|
|
637
645
|
cfg = load_pipeline_config(args.config, substitute=True)
|
|
646
|
+
if getattr(args, 'summary', False):
|
|
647
|
+
print_json(_pipeline_summary(cfg))
|
|
648
|
+
return 0
|
|
649
|
+
|
|
638
650
|
print_json(_list_sections(cfg, args))
|
|
639
651
|
return 0
|
|
640
652
|
|
|
@@ -422,8 +422,9 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
422
422
|
pipe_parser = subparsers.add_parser(
|
|
423
423
|
'pipeline',
|
|
424
424
|
help=(
|
|
425
|
-
'
|
|
426
|
-
|
|
425
|
+
'DEPRECATED: use "list" (for summary/jobs) or "run" (to execute); '
|
|
426
|
+
'see '
|
|
427
|
+
f'{PROJECT_URL}/blob/main/docs/pipeline-guide.md'
|
|
427
428
|
),
|
|
428
429
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
429
430
|
)
|
|
@@ -446,6 +447,11 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
446
447
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
447
448
|
)
|
|
448
449
|
_add_config_option(list_parser)
|
|
450
|
+
_add_boolean_flag(
|
|
451
|
+
list_parser,
|
|
452
|
+
name='jobs',
|
|
453
|
+
help_text='List ETL jobs',
|
|
454
|
+
)
|
|
449
455
|
_add_boolean_flag(
|
|
450
456
|
list_parser,
|
|
451
457
|
name='pipelines',
|
|
@@ -456,6 +462,13 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
456
462
|
name='sources',
|
|
457
463
|
help_text='List data sources',
|
|
458
464
|
)
|
|
465
|
+
_add_boolean_flag(
|
|
466
|
+
list_parser,
|
|
467
|
+
name='summary',
|
|
468
|
+
help_text=(
|
|
469
|
+
'Show pipeline summary (name, version, sources, targets, jobs)'
|
|
470
|
+
),
|
|
471
|
+
)
|
|
459
472
|
_add_boolean_flag(
|
|
460
473
|
list_parser,
|
|
461
474
|
name='targets',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.9
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -306,6 +306,19 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
|
|
|
306
306
|
- Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
307
307
|
- Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
|
|
308
308
|
|
|
309
|
+
CLI quick reference for pipelines:
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
# List jobs or show a pipeline summary
|
|
313
|
+
etlplus list --config examples/configs/pipeline.yml --jobs
|
|
314
|
+
etlplus list --config examples/configs/pipeline.yml --summary
|
|
315
|
+
|
|
316
|
+
# Run a job
|
|
317
|
+
etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
318
|
+
|
|
319
|
+
# Deprecated shim (will be removed): etlplus pipeline
|
|
320
|
+
```
|
|
321
|
+
|
|
309
322
|
### Complete ETL Pipeline Example
|
|
310
323
|
|
|
311
324
|
```bash
|
|
@@ -44,12 +44,12 @@ CLI examples:
|
|
|
44
44
|
|
|
45
45
|
```bash
|
|
46
46
|
# List jobs defined in a pipeline file
|
|
47
|
-
etlplus
|
|
47
|
+
etlplus list --config examples/configs/pipeline.yml --jobs
|
|
48
48
|
|
|
49
|
-
#
|
|
50
|
-
etlplus
|
|
49
|
+
# Show a pipeline summary (name, version, sources, targets, jobs)
|
|
50
|
+
etlplus list --config examples/configs/pipeline.yml --summary
|
|
51
51
|
|
|
52
|
-
#
|
|
52
|
+
# Run a specific job end-to-end
|
|
53
53
|
etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
54
54
|
```
|
|
55
55
|
|
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
:mod:`tests.integration.test_i_pipeline_smoke` module.
|
|
3
3
|
|
|
4
4
|
Pipeline smoke integration test suite exercising a minimal file→file job via
|
|
5
|
-
the CLI. Parametrized to verify both empty
|
|
5
|
+
the CLI (using the deprecated-free path). Parametrized to verify both empty
|
|
6
|
+
and non-empty inputs.
|
|
6
7
|
|
|
7
8
|
Notes
|
|
8
9
|
-----
|
|
9
10
|
- Builds a transient pipeline YAML string per test run.
|
|
10
|
-
- Invokes ``etlplus
|
|
11
|
+
- Invokes ``etlplus run --job <job>`` end-to-end.
|
|
11
12
|
- Validates output file contents against input data shape.
|
|
12
13
|
"""
|
|
13
14
|
|
|
@@ -85,7 +86,7 @@ class TestPipelineSmoke:
|
|
|
85
86
|
|
|
86
87
|
code, out, err = cli_invoke(
|
|
87
88
|
(
|
|
88
|
-
'
|
|
89
|
+
'run',
|
|
89
90
|
'--config',
|
|
90
91
|
str(cfg_path),
|
|
91
92
|
'--job',
|