etlplus 0.16.6__tar.gz → 0.16.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {etlplus-0.16.6 → etlplus-0.16.7}/CONTRIBUTING.md +3 -3
- {etlplus-0.16.6 → etlplus-0.16.7}/DEMO.md +16 -17
- {etlplus-0.16.6/etlplus.egg-info → etlplus-0.16.7}/PKG-INFO +32 -28
- {etlplus-0.16.6 → etlplus-0.16.7}/README.md +31 -27
- {etlplus-0.16.6 → etlplus-0.16.7}/docs/README.md +2 -2
- {etlplus-0.16.6 → etlplus-0.16.7}/docs/pipeline-guide.md +7 -8
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/README.md +4 -4
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/enums.py +2 -2
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/README.md +9 -7
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/main.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/database/README.md +7 -7
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/README.md +7 -5
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/accdb.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/arrow.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/bson.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/cbor.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/cfg.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/conf.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/dat.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/dta.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/duckdb.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/enums.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/fwf.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/hbs.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/hdf5.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/ini.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/ion.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/jinja2.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/log.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/mat.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/mdb.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/msgpack.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/mustache.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/nc.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/numbers.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/ods.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/pb.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/pbf.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/properties.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/proto.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/psv.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/rda.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/rds.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/sas7bdat.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/sav.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/sqlite.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/sylk.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/tab.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/toml.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/vm.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/wks.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/xls.py +1 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/xlsm.py +2 -2
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/xpt.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/zsav.py +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/README.md +10 -9
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/templates/README.md +11 -9
- {etlplus-0.16.6 → etlplus-0.16.7/etlplus.egg-info}/PKG-INFO +32 -28
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/README.md +2 -1
- {etlplus-0.16.6 → etlplus-0.16.7}/.coveragerc +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/.editorconfig +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/.gitattributes +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/.github/actions/python-bootstrap/action.yml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/.github/workflows/ci.yml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/.gitignore +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/.pre-commit-config.yaml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/.ruff.toml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/CODE_OF_CONDUCT.md +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/LICENSE +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/MANIFEST.in +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/Makefile +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/REFERENCES.md +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/SECURITY.md +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/SUPPORT.md +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/docs/snippets/installation_version.md +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/__main__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/__version__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/README.md +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/auth.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/config.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/endpoint_client.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/errors.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/pagination/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/pagination/client.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/pagination/config.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/pagination/paginator.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/rate_limiting/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/rate_limiting/config.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/rate_limiting/rate_limiter.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/request_manager.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/retry_manager.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/transport.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/types.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/api/utils.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/commands.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/constants.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/handlers.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/io.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/options.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/state.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/cli/types.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/config.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/api.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/connector.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/core.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/database.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/enums.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/file.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/types.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/connector/utils.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/database/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/database/ddl.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/database/engine.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/database/orm.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/database/schema.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/database/types.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/enums.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/_imports.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/_io.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/avro.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/core.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/csv.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/feather.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/gz.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/json.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/ndjson.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/orc.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/parquet.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/stub.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/tsv.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/txt.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/xlsx.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/xml.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/yaml.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/file/zip.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/mixins.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/enums.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/extract.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/load.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/run.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/transform.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/types.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/utils.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/ops/validate.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/py.typed +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/templates/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/templates/ddl.sql.j2 +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/templates/view.sql.j2 +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/types.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/utils.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/workflow/README.md +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/workflow/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/workflow/dag.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/workflow/jobs.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus/workflow/profile.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus.egg-info/SOURCES.txt +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus.egg-info/dependency_links.txt +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus.egg-info/entry_points.txt +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus.egg-info/requires.txt +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/etlplus.egg-info/top_level.txt +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/configs/ddl_spec.yml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/configs/pipeline.yml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/data/sample.csv +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/data/sample.json +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/data/sample.xml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/data/sample.xsd +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/data/sample.yaml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/examples/quickstart.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/pyproject.toml +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/pytest.ini +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/setup.cfg +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/setup.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/__init__.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/conftest.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/conftest.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/test_i_cli.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/test_i_config_load.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/test_i_examples_data_parity.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/test_i_pagination_strategy.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/test_i_pipeline_smoke.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/test_i_run.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/test_i_run_profile_pagination_defaults.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/integration/test_i_run_profile_rate_limit_defaults.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/conftest.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_auth.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_config.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_endpoint_client.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_enums.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_mocks.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_pagination_client.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_pagination_config.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_paginator.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_rate_limit_config.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_rate_limiter.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_request_manager.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_retry_manager.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_transport.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_types.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/api/test_u_api_utils.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/cli/conftest.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/cli/test_u_cli_handlers.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/cli/test_u_cli_io.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/cli/test_u_cli_main.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/cli/test_u_cli_state.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/conftest.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/connector/test_u_connector_enums.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/connector/test_u_connector_utils.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/database/test_u_database_ddl.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/database/test_u_database_engine.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/database/test_u_database_orm.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/database/test_u_database_schema.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/file/test_u_file_core.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/file/test_u_file_enums.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/file/test_u_file_yaml.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/ops/test_u_ops_enums.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/ops/test_u_ops_extract.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/ops/test_u_ops_load.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/ops/test_u_ops_run.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/ops/test_u_ops_transform.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/ops/test_u_ops_utils.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/ops/test_u_ops_validate.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/test_u_config.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/test_u_main.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/test_u_mixins.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/test_u_utils.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/test_u_version.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tests/unit/workflow/test_u_workflow_jobs.py +0 -0
- {etlplus-0.16.6 → etlplus-0.16.7}/tools/update_demo_snippets.py +0 -0
|
@@ -54,8 +54,8 @@ but using the stubs can provide a nicer editing experience.
|
|
|
54
54
|
|
|
55
55
|
We optimize for a great editor experience and a permissive runtime:
|
|
56
56
|
|
|
57
|
-
- TypedDicts in `etlplus/
|
|
58
|
-
`total=False` (all keys optional) and are not enforced at runtime.
|
|
57
|
+
- TypedDicts in `etlplus/*/types.py` (for example `etlplus/api/types.py`) are editor/type-checking
|
|
58
|
+
hints. They are intentionally `total=False` (all keys optional) and are not enforced at runtime.
|
|
59
59
|
- Constructors named `*.from_obj` accept `Mapping[str, Any]` and perform tolerant parsing and light
|
|
60
60
|
casting. This keeps runtime permissive while improving autocomplete and static analysis.
|
|
61
61
|
- Prefer `Mapping[str, Any]` for inputs and plain `dict[...]` for internal state/returns. Avoid
|
|
@@ -111,7 +111,7 @@ Use these guidelines to decide whether a test belongs in the unit or integration
|
|
|
111
111
|
- Unit tests (put under `tests/unit/`):
|
|
112
112
|
- Exercise a single function or class in isolation (no orchestration across modules).
|
|
113
113
|
- No real file system or network I/O. Use in-memory data and stubs.
|
|
114
|
-
- Examples in this repo:
|
|
114
|
+
- Examples in this repo: small helpers in `etlplus.utils`, validation and transform functions.
|
|
115
115
|
- Fast and deterministic; rely on `monkeypatch` to stub collaborators.
|
|
116
116
|
|
|
117
117
|
- Integration tests (put under `tests/integration/`):
|
|
@@ -32,10 +32,10 @@ etlplus 0.3.4
|
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
$ etlplus --help
|
|
35
|
-
|
|
35
|
+
Usage: etlplus [OPTIONS] COMMAND [ARGS]...
|
|
36
36
|
|
|
37
37
|
ETLPlus - A Swiss Army knife for enabling simple ETL operations
|
|
38
|
-
|
|
38
|
+
# (Output abbreviated; run `etlplus --help` for the full command list.)
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
## Demo 1: Extract Data from Different Sources
|
|
@@ -43,7 +43,7 @@ ETLPlus - A Swiss Army knife for enabling simple ETL operations
|
|
|
43
43
|
### Extract from JSON
|
|
44
44
|
```bash
|
|
45
45
|
$ echo '{"name": "John", "age": 30}' > sample.json
|
|
46
|
-
$ etlplus extract
|
|
46
|
+
$ etlplus extract sample.json
|
|
47
47
|
{
|
|
48
48
|
"name": "John",
|
|
49
49
|
"age": 30
|
|
@@ -111,8 +111,8 @@ $ etlplus transform --operations '{
|
|
|
111
111
|
|
|
112
112
|
### Sort Data
|
|
113
113
|
```bash
|
|
114
|
-
$ etlplus transform
|
|
115
|
-
|
|
114
|
+
$ etlplus transform \
|
|
115
|
+
--operations '{"sort": {"field": "score", "reverse": true}}' \
|
|
116
116
|
'[{"name": "Charlie", "score": 85}, {"name": "Alice", "score": 95}, {"name": "Bob", "score": 90}]'
|
|
117
117
|
```
|
|
118
118
|
|
|
@@ -133,9 +133,8 @@ $ etlplus transform --operations '{"aggregate": {"field": "sales", "func": "sum"
|
|
|
133
133
|
|
|
134
134
|
### Load to JSON File
|
|
135
135
|
```bash
|
|
136
|
-
$
|
|
137
|
-
|
|
138
|
-
output.json --target-type file
|
|
136
|
+
$ echo '{"name": "John", "status": "active"}' \
|
|
137
|
+
| etlplus load output.json --target-type file
|
|
139
138
|
{
|
|
140
139
|
"status": "success",
|
|
141
140
|
"message": "Data loaded to output.json",
|
|
@@ -145,12 +144,12 @@ $ etlplus load \
|
|
|
145
144
|
|
|
146
145
|
### Load to CSV File
|
|
147
146
|
```bash
|
|
148
|
-
$ etlplus load
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
147
|
+
$ cat << 'JSON' | etlplus load users.csv --target-type file
|
|
148
|
+
[
|
|
149
|
+
{"name": "John", "email": "john@example.com"},
|
|
150
|
+
{"name": "Jane", "email": "jane@example.com"}
|
|
151
|
+
]
|
|
152
|
+
JSON
|
|
154
153
|
{
|
|
155
154
|
"status": "success",
|
|
156
155
|
"message": "Data loaded to users.csv",
|
|
@@ -190,7 +189,7 @@ $ etlplus validate \
|
|
|
190
189
|
transformed.json
|
|
191
190
|
|
|
192
191
|
# Step 4: Load
|
|
193
|
-
$
|
|
192
|
+
$ cat transformed.json | etlplus load final_output.csv
|
|
194
193
|
```
|
|
195
194
|
|
|
196
195
|
## Demo 6: Using Python API
|
|
@@ -199,7 +198,7 @@ $ etlplus load transformed.json file final_output.csv
|
|
|
199
198
|
from etlplus.ops import extract, validate, transform, load
|
|
200
199
|
|
|
201
200
|
# Extract
|
|
202
|
-
data = extract("file", "data.csv",
|
|
201
|
+
data = extract("file", "data.csv", file_format="csv")
|
|
203
202
|
|
|
204
203
|
# Validate
|
|
205
204
|
validation_result = validate(data, {
|
|
@@ -214,7 +213,7 @@ if validation_result["valid"]:
|
|
|
214
213
|
})
|
|
215
214
|
|
|
216
215
|
# Load
|
|
217
|
-
load(transformed, "file", "output.json",
|
|
216
|
+
load(transformed, "file", "output.json", file_format="json")
|
|
218
217
|
print("ETL pipeline completed successfully!")
|
|
219
218
|
else:
|
|
220
219
|
print(f"Validation errors: {validation_result['errors']}")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.16.
|
|
3
|
+
Version: 0.16.7
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -141,7 +141,7 @@ ETLPlus supports Python 3.13 and above.
|
|
|
141
141
|
|
|
142
142
|
- **Extract** data from multiple sources:
|
|
143
143
|
- Files (CSV, JSON, XML, YAML)
|
|
144
|
-
- Databases (connection string support)
|
|
144
|
+
- Databases (connection string support; extract is a placeholder today)
|
|
145
145
|
- REST APIs (GET)
|
|
146
146
|
|
|
147
147
|
- **Validate** data with flexible rules:
|
|
@@ -161,7 +161,7 @@ ETLPlus supports Python 3.13 and above.
|
|
|
161
161
|
|
|
162
162
|
- **Load** data to multiple targets:
|
|
163
163
|
- Files (CSV, JSON, XML, YAML)
|
|
164
|
-
- Databases (connection string support)
|
|
164
|
+
- Databases (connection string support; load is a placeholder today)
|
|
165
165
|
- REST APIs (PATCH, POST, PUT)
|
|
166
166
|
|
|
167
167
|
## Installation
|
|
@@ -188,7 +188,7 @@ etlplus --help
|
|
|
188
188
|
etlplus --version
|
|
189
189
|
|
|
190
190
|
# One-liner: extract CSV, filter, select, and write JSON
|
|
191
|
-
etlplus extract
|
|
191
|
+
etlplus extract examples/data/sample.csv \
|
|
192
192
|
| etlplus transform --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
|
|
193
193
|
- temp/sample_output.json
|
|
194
194
|
```
|
|
@@ -218,8 +218,9 @@ include GET for extract and PATCH/POST/PUT for load.
|
|
|
218
218
|
|
|
219
219
|
### Databases (`database`)
|
|
220
220
|
|
|
221
|
-
Database connectors use connection strings for extraction and loading, and
|
|
222
|
-
|
|
221
|
+
Database connectors use connection strings for extraction and loading, and DDL can be rendered from
|
|
222
|
+
table specs for migrations or schema checks. Database extract/load operations are currently
|
|
223
|
+
placeholders; plan to integrate a database client in your runner.
|
|
223
224
|
|
|
224
225
|
### Files (`file`)
|
|
225
226
|
|
|
@@ -298,7 +299,7 @@ Recognized file formats are listed in the tables below. Support for reading to o
|
|
|
298
299
|
| `numbers` | N | N | Apple Numbers |
|
|
299
300
|
| `ods` | N | N | OpenDocument |
|
|
300
301
|
| `wks` | N | N | Lotus 1-2-3 |
|
|
301
|
-
| `xls` | Y |
|
|
302
|
+
| `xls` | Y | N | Microsoft Excel (BIFF; read-only) |
|
|
302
303
|
| `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
|
|
303
304
|
| `xlsx` | Y | Y | Microsoft Excel (Open XML) |
|
|
304
305
|
|
|
@@ -320,8 +321,8 @@ Recognized file formats are listed in the tables below. Support for reading to o
|
|
|
320
321
|
|
|
321
322
|
#### Logs and Event Streams
|
|
322
323
|
|
|
323
|
-
| Format |
|
|
324
|
-
| --- | --- | --- |
|
|
324
|
+
| Format | Read | Write | Description |
|
|
325
|
+
| --- | --- | --- | --- |
|
|
325
326
|
| `log` | N | N | Generic log file |
|
|
326
327
|
|
|
327
328
|
#### Data Archives
|
|
@@ -354,13 +355,15 @@ etlplus --help
|
|
|
354
355
|
etlplus --version
|
|
355
356
|
```
|
|
356
357
|
|
|
357
|
-
The CLI is implemented with Typer (Click-based).
|
|
358
|
-
on the documented commands/flags and run `etlplus <command>
|
|
358
|
+
The CLI is implemented with Typer (Click-based). The legacy argparse parser has been removed
|
|
359
|
+
(`create_parser` now raises), so rely on the documented commands/flags and run `etlplus <command>
|
|
360
|
+
--help` for current options.
|
|
359
361
|
|
|
360
362
|
**Example error messages:**
|
|
361
363
|
|
|
362
364
|
- If you omit a required argument: `Error: Missing required argument 'SOURCE'.`
|
|
363
|
-
- If you place an option before its argument: `Error: Option '--source-format' must follow the
|
|
365
|
+
- If you place an option before its argument: `Error: Option '--source-format' must follow the
|
|
366
|
+
'SOURCE' argument.`
|
|
364
367
|
|
|
365
368
|
#### Argument Order and Required Options
|
|
366
369
|
|
|
@@ -424,27 +427,27 @@ specific parser.
|
|
|
424
427
|
|
|
425
428
|
Extract from JSON file:
|
|
426
429
|
```bash
|
|
427
|
-
etlplus extract
|
|
430
|
+
etlplus extract examples/data/sample.json
|
|
428
431
|
```
|
|
429
432
|
|
|
430
433
|
Extract from CSV file:
|
|
431
434
|
```bash
|
|
432
|
-
etlplus extract
|
|
435
|
+
etlplus extract examples/data/sample.csv
|
|
433
436
|
```
|
|
434
437
|
|
|
435
438
|
Extract from XML file:
|
|
436
439
|
```bash
|
|
437
|
-
etlplus extract
|
|
440
|
+
etlplus extract examples/data/sample.xml
|
|
438
441
|
```
|
|
439
442
|
|
|
440
443
|
Extract from REST API:
|
|
441
444
|
```bash
|
|
442
|
-
etlplus extract
|
|
445
|
+
etlplus extract https://api.example.com/data
|
|
443
446
|
```
|
|
444
447
|
|
|
445
448
|
Save extracted data to file:
|
|
446
449
|
```bash
|
|
447
|
-
etlplus extract
|
|
450
|
+
etlplus extract examples/data/sample.csv > temp/sample_output.json
|
|
448
451
|
```
|
|
449
452
|
|
|
450
453
|
#### Validate Data
|
|
@@ -510,13 +513,13 @@ etlplus transform \
|
|
|
510
513
|
|
|
511
514
|
Load to JSON file:
|
|
512
515
|
```bash
|
|
513
|
-
etlplus extract
|
|
516
|
+
etlplus extract examples/data/sample.json \
|
|
514
517
|
| etlplus load temp/sample_output.json --target-type file
|
|
515
518
|
```
|
|
516
519
|
|
|
517
520
|
Load to CSV file:
|
|
518
521
|
```bash
|
|
519
|
-
etlplus extract
|
|
522
|
+
etlplus extract examples/data/sample.csv \
|
|
520
523
|
| etlplus load temp/sample_output.csv --target-type file
|
|
521
524
|
```
|
|
522
525
|
|
|
@@ -553,13 +556,13 @@ operations = {
|
|
|
553
556
|
transformed = transform(data, operations)
|
|
554
557
|
|
|
555
558
|
# Load data
|
|
556
|
-
load(transformed, "file", "temp/sample_output.json",
|
|
559
|
+
load(transformed, "file", "temp/sample_output.json", file_format="json")
|
|
557
560
|
```
|
|
558
561
|
|
|
559
562
|
For YAML-driven pipelines executed end-to-end (extract → validate → transform → load), see:
|
|
560
563
|
|
|
561
564
|
- Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
562
|
-
- Runner API and internals:
|
|
565
|
+
- Runner API and internals: see `etlplus.ops.run` docstrings and `docs/pipeline-guide.md`.
|
|
563
566
|
|
|
564
567
|
CLI quick reference for pipelines:
|
|
565
568
|
|
|
@@ -576,7 +579,7 @@ etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
|
576
579
|
|
|
577
580
|
```bash
|
|
578
581
|
# 1. Extract from CSV
|
|
579
|
-
etlplus extract
|
|
582
|
+
etlplus extract examples/data/sample.csv > temp/sample_extracted.json
|
|
580
583
|
|
|
581
584
|
# 2. Transform (filter and select fields)
|
|
582
585
|
etlplus transform \
|
|
@@ -611,7 +614,7 @@ etlplus load output.bin --target-type file --target-format csv < data.json
|
|
|
611
614
|
|
|
612
615
|
# Leave the flags off when extensions already match the desired format
|
|
613
616
|
etlplus extract data.csv --source-type file
|
|
614
|
-
etlplus load
|
|
617
|
+
etlplus load output.json --target-type file < data.json
|
|
615
618
|
```
|
|
616
619
|
|
|
617
620
|
## Transformation Operations
|
|
@@ -706,8 +709,9 @@ Looking for the HTTP client and pagination helpers? See the dedicated docs in
|
|
|
706
709
|
|
|
707
710
|
Curious how the pipeline runner composes API requests, pagination, and load calls?
|
|
708
711
|
|
|
709
|
-
- Runner overview and helpers:
|
|
710
|
-
-
|
|
712
|
+
- Runner overview and helpers: see `etlplus.ops.run` docstrings and
|
|
713
|
+
[`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
714
|
+
- Unified "connector" vocabulary (API/File/DB): `etlplus/connector`
|
|
711
715
|
- API/file targets reuse the same shapes as sources; API targets typically set a `method`.
|
|
712
716
|
|
|
713
717
|
### Running Tests
|
|
@@ -721,7 +725,7 @@ pytest tests/ -v
|
|
|
721
725
|
We split tests into two layers:
|
|
722
726
|
|
|
723
727
|
- **Unit (`tests/unit/`)**: single function or class, no real I/O, fast, uses stubs/monkeypatch
|
|
724
|
-
(e.g.
|
|
728
|
+
(e.g. small helpers in `etlplus.utils`, transform + validate helpers).
|
|
725
729
|
- **Integration (`tests/integration/`)**: end-to-end flows (CLI `main()`, pipeline `run()`,
|
|
726
730
|
pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
|
|
727
731
|
fake clients.
|
|
@@ -808,7 +812,7 @@ Navigate to detailed documentation for each subpackage:
|
|
|
808
812
|
- [etlplus.cli](etlplus/cli/README.md): Command-line interface definitions for `etlplus`
|
|
809
813
|
- [etlplus.database](etlplus/database/README.md): Database engine, schema, and ORM helpers
|
|
810
814
|
- [etlplus.templates](etlplus/templates/README.md): SQL and DDL template helpers
|
|
811
|
-
- [etlplus.
|
|
815
|
+
- [etlplus.ops](etlplus/ops/README.md): Extract/validate/transform/load primitives
|
|
812
816
|
- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
|
|
813
817
|
profiles
|
|
814
818
|
|
|
@@ -824,7 +828,7 @@ Navigate to detailed documentation for each subpackage:
|
|
|
824
828
|
- API client docs: [`etlplus/api/README.md`](etlplus/api/README.md)
|
|
825
829
|
- Examples: [`examples/README.md`](examples/README.md)
|
|
826
830
|
- Pipeline authoring guide: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
827
|
-
- Runner internals: [`docs/
|
|
831
|
+
- Runner internals: see `etlplus.ops.run` docstrings and [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
828
832
|
- Design notes (Mapping inputs, dict outputs): [`docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs`](docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs)
|
|
829
833
|
- Typing philosophy: [`CONTRIBUTING.md#typing-philosophy`](CONTRIBUTING.md#typing-philosophy)
|
|
830
834
|
- Demo and walkthrough: [`DEMO.md`](DEMO.md)
|
|
@@ -91,7 +91,7 @@ ETLPlus supports Python 3.13 and above.
|
|
|
91
91
|
|
|
92
92
|
- **Extract** data from multiple sources:
|
|
93
93
|
- Files (CSV, JSON, XML, YAML)
|
|
94
|
-
- Databases (connection string support)
|
|
94
|
+
- Databases (connection string support; extract is a placeholder today)
|
|
95
95
|
- REST APIs (GET)
|
|
96
96
|
|
|
97
97
|
- **Validate** data with flexible rules:
|
|
@@ -111,7 +111,7 @@ ETLPlus supports Python 3.13 and above.
|
|
|
111
111
|
|
|
112
112
|
- **Load** data to multiple targets:
|
|
113
113
|
- Files (CSV, JSON, XML, YAML)
|
|
114
|
-
- Databases (connection string support)
|
|
114
|
+
- Databases (connection string support; load is a placeholder today)
|
|
115
115
|
- REST APIs (PATCH, POST, PUT)
|
|
116
116
|
|
|
117
117
|
## Installation
|
|
@@ -138,7 +138,7 @@ etlplus --help
|
|
|
138
138
|
etlplus --version
|
|
139
139
|
|
|
140
140
|
# One-liner: extract CSV, filter, select, and write JSON
|
|
141
|
-
etlplus extract
|
|
141
|
+
etlplus extract examples/data/sample.csv \
|
|
142
142
|
| etlplus transform --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
|
|
143
143
|
- temp/sample_output.json
|
|
144
144
|
```
|
|
@@ -168,8 +168,9 @@ include GET for extract and PATCH/POST/PUT for load.
|
|
|
168
168
|
|
|
169
169
|
### Databases (`database`)
|
|
170
170
|
|
|
171
|
-
Database connectors use connection strings for extraction and loading, and
|
|
172
|
-
|
|
171
|
+
Database connectors use connection strings for extraction and loading, and DDL can be rendered from
|
|
172
|
+
table specs for migrations or schema checks. Database extract/load operations are currently
|
|
173
|
+
placeholders; plan to integrate a database client in your runner.
|
|
173
174
|
|
|
174
175
|
### Files (`file`)
|
|
175
176
|
|
|
@@ -248,7 +249,7 @@ Recognized file formats are listed in the tables below. Support for reading to o
|
|
|
248
249
|
| `numbers` | N | N | Apple Numbers |
|
|
249
250
|
| `ods` | N | N | OpenDocument |
|
|
250
251
|
| `wks` | N | N | Lotus 1-2-3 |
|
|
251
|
-
| `xls` | Y |
|
|
252
|
+
| `xls` | Y | N | Microsoft Excel (BIFF; read-only) |
|
|
252
253
|
| `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
|
|
253
254
|
| `xlsx` | Y | Y | Microsoft Excel (Open XML) |
|
|
254
255
|
|
|
@@ -270,8 +271,8 @@ Recognized file formats are listed in the tables below. Support for reading to o
|
|
|
270
271
|
|
|
271
272
|
#### Logs and Event Streams
|
|
272
273
|
|
|
273
|
-
| Format |
|
|
274
|
-
| --- | --- | --- |
|
|
274
|
+
| Format | Read | Write | Description |
|
|
275
|
+
| --- | --- | --- | --- |
|
|
275
276
|
| `log` | N | N | Generic log file |
|
|
276
277
|
|
|
277
278
|
#### Data Archives
|
|
@@ -304,13 +305,15 @@ etlplus --help
|
|
|
304
305
|
etlplus --version
|
|
305
306
|
```
|
|
306
307
|
|
|
307
|
-
The CLI is implemented with Typer (Click-based).
|
|
308
|
-
on the documented commands/flags and run `etlplus <command>
|
|
308
|
+
The CLI is implemented with Typer (Click-based). The legacy argparse parser has been removed
|
|
309
|
+
(`create_parser` now raises), so rely on the documented commands/flags and run `etlplus <command>
|
|
310
|
+
--help` for current options.
|
|
309
311
|
|
|
310
312
|
**Example error messages:**
|
|
311
313
|
|
|
312
314
|
- If you omit a required argument: `Error: Missing required argument 'SOURCE'.`
|
|
313
|
-
- If you place an option before its argument: `Error: Option '--source-format' must follow the
|
|
315
|
+
- If you place an option before its argument: `Error: Option '--source-format' must follow the
|
|
316
|
+
'SOURCE' argument.`
|
|
314
317
|
|
|
315
318
|
#### Argument Order and Required Options
|
|
316
319
|
|
|
@@ -374,27 +377,27 @@ specific parser.
|
|
|
374
377
|
|
|
375
378
|
Extract from JSON file:
|
|
376
379
|
```bash
|
|
377
|
-
etlplus extract
|
|
380
|
+
etlplus extract examples/data/sample.json
|
|
378
381
|
```
|
|
379
382
|
|
|
380
383
|
Extract from CSV file:
|
|
381
384
|
```bash
|
|
382
|
-
etlplus extract
|
|
385
|
+
etlplus extract examples/data/sample.csv
|
|
383
386
|
```
|
|
384
387
|
|
|
385
388
|
Extract from XML file:
|
|
386
389
|
```bash
|
|
387
|
-
etlplus extract
|
|
390
|
+
etlplus extract examples/data/sample.xml
|
|
388
391
|
```
|
|
389
392
|
|
|
390
393
|
Extract from REST API:
|
|
391
394
|
```bash
|
|
392
|
-
etlplus extract
|
|
395
|
+
etlplus extract https://api.example.com/data
|
|
393
396
|
```
|
|
394
397
|
|
|
395
398
|
Save extracted data to file:
|
|
396
399
|
```bash
|
|
397
|
-
etlplus extract
|
|
400
|
+
etlplus extract examples/data/sample.csv > temp/sample_output.json
|
|
398
401
|
```
|
|
399
402
|
|
|
400
403
|
#### Validate Data
|
|
@@ -460,13 +463,13 @@ etlplus transform \
|
|
|
460
463
|
|
|
461
464
|
Load to JSON file:
|
|
462
465
|
```bash
|
|
463
|
-
etlplus extract
|
|
466
|
+
etlplus extract examples/data/sample.json \
|
|
464
467
|
| etlplus load temp/sample_output.json --target-type file
|
|
465
468
|
```
|
|
466
469
|
|
|
467
470
|
Load to CSV file:
|
|
468
471
|
```bash
|
|
469
|
-
etlplus extract
|
|
472
|
+
etlplus extract examples/data/sample.csv \
|
|
470
473
|
| etlplus load temp/sample_output.csv --target-type file
|
|
471
474
|
```
|
|
472
475
|
|
|
@@ -503,13 +506,13 @@ operations = {
|
|
|
503
506
|
transformed = transform(data, operations)
|
|
504
507
|
|
|
505
508
|
# Load data
|
|
506
|
-
load(transformed, "file", "temp/sample_output.json",
|
|
509
|
+
load(transformed, "file", "temp/sample_output.json", file_format="json")
|
|
507
510
|
```
|
|
508
511
|
|
|
509
512
|
For YAML-driven pipelines executed end-to-end (extract → validate → transform → load), see:
|
|
510
513
|
|
|
511
514
|
- Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
512
|
-
- Runner API and internals:
|
|
515
|
+
- Runner API and internals: see `etlplus.ops.run` docstrings and `docs/pipeline-guide.md`.
|
|
513
516
|
|
|
514
517
|
CLI quick reference for pipelines:
|
|
515
518
|
|
|
@@ -526,7 +529,7 @@ etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
|
526
529
|
|
|
527
530
|
```bash
|
|
528
531
|
# 1. Extract from CSV
|
|
529
|
-
etlplus extract
|
|
532
|
+
etlplus extract examples/data/sample.csv > temp/sample_extracted.json
|
|
530
533
|
|
|
531
534
|
# 2. Transform (filter and select fields)
|
|
532
535
|
etlplus transform \
|
|
@@ -561,7 +564,7 @@ etlplus load output.bin --target-type file --target-format csv < data.json
|
|
|
561
564
|
|
|
562
565
|
# Leave the flags off when extensions already match the desired format
|
|
563
566
|
etlplus extract data.csv --source-type file
|
|
564
|
-
etlplus load
|
|
567
|
+
etlplus load output.json --target-type file < data.json
|
|
565
568
|
```
|
|
566
569
|
|
|
567
570
|
## Transformation Operations
|
|
@@ -656,8 +659,9 @@ Looking for the HTTP client and pagination helpers? See the dedicated docs in
|
|
|
656
659
|
|
|
657
660
|
Curious how the pipeline runner composes API requests, pagination, and load calls?
|
|
658
661
|
|
|
659
|
-
- Runner overview and helpers:
|
|
660
|
-
-
|
|
662
|
+
- Runner overview and helpers: see `etlplus.ops.run` docstrings and
|
|
663
|
+
[`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
664
|
+
- Unified "connector" vocabulary (API/File/DB): `etlplus/connector`
|
|
661
665
|
- API/file targets reuse the same shapes as sources; API targets typically set a `method`.
|
|
662
666
|
|
|
663
667
|
### Running Tests
|
|
@@ -671,7 +675,7 @@ pytest tests/ -v
|
|
|
671
675
|
We split tests into two layers:
|
|
672
676
|
|
|
673
677
|
- **Unit (`tests/unit/`)**: single function or class, no real I/O, fast, uses stubs/monkeypatch
|
|
674
|
-
(e.g.
|
|
678
|
+
(e.g. small helpers in `etlplus.utils`, transform + validate helpers).
|
|
675
679
|
- **Integration (`tests/integration/`)**: end-to-end flows (CLI `main()`, pipeline `run()`,
|
|
676
680
|
pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
|
|
677
681
|
fake clients.
|
|
@@ -758,7 +762,7 @@ Navigate to detailed documentation for each subpackage:
|
|
|
758
762
|
- [etlplus.cli](etlplus/cli/README.md): Command-line interface definitions for `etlplus`
|
|
759
763
|
- [etlplus.database](etlplus/database/README.md): Database engine, schema, and ORM helpers
|
|
760
764
|
- [etlplus.templates](etlplus/templates/README.md): SQL and DDL template helpers
|
|
761
|
-
- [etlplus.
|
|
765
|
+
- [etlplus.ops](etlplus/ops/README.md): Extract/validate/transform/load primitives
|
|
762
766
|
- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
|
|
763
767
|
profiles
|
|
764
768
|
|
|
@@ -774,7 +778,7 @@ Navigate to detailed documentation for each subpackage:
|
|
|
774
778
|
- API client docs: [`etlplus/api/README.md`](etlplus/api/README.md)
|
|
775
779
|
- Examples: [`examples/README.md`](examples/README.md)
|
|
776
780
|
- Pipeline authoring guide: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
777
|
-
- Runner internals: [`docs/
|
|
781
|
+
- Runner internals: see `etlplus.ops.run` docstrings and [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
778
782
|
- Design notes (Mapping inputs, dict outputs): [`docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs`](docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs)
|
|
779
783
|
- Typing philosophy: [`CONTRIBUTING.md#typing-philosophy`](CONTRIBUTING.md#typing-philosophy)
|
|
780
784
|
- Demo and walkthrough: [`DEMO.md`](DEMO.md)
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Documentation Notes
|
|
2
2
|
|
|
3
3
|
## CLI Parser Status
|
|
4
|
-
- The CLI is
|
|
5
|
-
|
|
4
|
+
- The CLI is Typer/Click-only. The historical `argparse` parser has been removed; `create_parser`
|
|
5
|
+
now raises to force migration.
|
|
6
6
|
- Downstream tools should invoke the Typer app exported at `etlplus.cli.commands.app` (e.g., `python
|
|
7
7
|
-m etlplus` or `etlplus ...`).
|
|
8
8
|
- Handler functions still accept keyword arguments; the legacy namespace shim is temporary and will
|
|
@@ -7,8 +7,9 @@ ETLPlus focuses on simple, JSON-first ETL. The pipeline file is a declarative de
|
|
|
7
7
|
runner (a script, Makefile, CI job) can parse and execute using ETLPlus primitives: `extract`,
|
|
8
8
|
`validate`, `transform`, and `load`.
|
|
9
9
|
|
|
10
|
-
CLI note: ETLPlus uses Typer for command parsing
|
|
11
|
-
documented `etlplus` commands and flags (check `etlplus
|
|
10
|
+
CLI note: ETLPlus uses Typer for command parsing. The legacy argparse parser has been removed
|
|
11
|
+
(`create_parser` now raises). Use the documented `etlplus` commands and flags (check `etlplus
|
|
12
|
+
--help`) when wiring your runner.
|
|
12
13
|
|
|
13
14
|
## Running a pipeline from YAML (CLI)
|
|
14
15
|
|
|
@@ -270,10 +271,9 @@ up or slowed down.
|
|
|
270
271
|
Format override note:
|
|
271
272
|
|
|
272
273
|
When extracting from file sources, ETLPlus still infers the format from the filename extension
|
|
273
|
-
(`.csv`, `.json`, `.xml`, `.yaml`). However, `--source-format` and `--target-format`
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
to rename the file first.
|
|
274
|
+
(`.csv`, `.json`, `.xml`, `.yaml`). However, `--source-format` and `--target-format` override that
|
|
275
|
+
inference in the Typer CLI. This means you can safely point at files without extensions or with
|
|
276
|
+
misleading suffixes and force the desired parser or writer without renaming the file first.
|
|
277
277
|
|
|
278
278
|
Note: When using a service + endpoint in a source, URL composition (including `base_path`) is
|
|
279
279
|
handled automatically. See “Runner behavior with base_path (sources and targets)” in the APIs
|
|
@@ -430,8 +430,7 @@ Notes:
|
|
|
430
430
|
- These commands read the same YAML schema described in this guide.
|
|
431
431
|
- Environment-variable substitution (e.g. `${GITHUB_TOKEN}`) is applied the same way as when loading
|
|
432
432
|
configs via the Python API.
|
|
433
|
-
- For more details on the orchestration implementation, see
|
|
434
|
-
[Runner internals: etlplus.ops.run](run-module.md).
|
|
433
|
+
- For more details on the orchestration implementation, see the `etlplus.ops.run` docstrings.
|
|
435
434
|
|
|
436
435
|
### Python: `etlplus.ops.run.run`
|
|
437
436
|
|
|
@@ -16,8 +16,8 @@ Back to project overview: see the top-level [README](../README.md).
|
|
|
16
16
|
- [etlplus.cli](cli/README.md): Command-line interface definitions for `etlplus`
|
|
17
17
|
- [etlplus.database](database/README.md): Database engine, schema, and ORM helpers
|
|
18
18
|
- [etlplus.templates](templates/README.md): SQL and DDL template helpers
|
|
19
|
-
- [etlplus.
|
|
20
|
-
- [etlplus.workflow](
|
|
19
|
+
- [etlplus.ops](ops/README.md): Extract/validate/transform/load primitives
|
|
20
|
+
- [etlplus.workflow](workflow/README.md): Helpers for data connectors, pipelines, jobs, and
|
|
21
21
|
profiles
|
|
22
22
|
|
|
23
23
|
## Quickstart
|
|
@@ -36,7 +36,7 @@ load(filtered, "file", "output.json", file_format="json")
|
|
|
36
36
|
Use the provided classes to load and validate configuration files:
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
|
-
from etlplus
|
|
39
|
+
from etlplus import Config
|
|
40
40
|
|
|
41
41
|
cfg = Config.from_yaml("pipeline.yml")
|
|
42
42
|
```
|
|
@@ -47,7 +47,7 @@ cfg = Config.from_yaml("pipeline.yml")
|
|
|
47
47
|
## Example: Loading a Pipeline Config
|
|
48
48
|
|
|
49
49
|
```python
|
|
50
|
-
from etlplus
|
|
50
|
+
from etlplus import Config
|
|
51
51
|
|
|
52
52
|
config = Config.from_yaml("configs/pipeline.yml")
|
|
53
53
|
print(config)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
2
|
:mod:`etlplus.api.enums` module.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
REST API-aligned enums and helpers.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
@@ -21,7 +21,7 @@ __all__ = [
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class HttpMethod(CoercibleStrEnum):
|
|
24
|
-
"""Supported HTTP verbs
|
|
24
|
+
"""Supported HTTP verbs with a helper for request-body allowance."""
|
|
25
25
|
|
|
26
26
|
# -- Constants -- #
|
|
27
27
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Documentation for the `etlplus.cli` subpackage: command-line interface for ETLPlus workflows.
|
|
4
4
|
|
|
5
5
|
- Provides a CLI for running ETL pipelines, jobs, and utilities
|
|
6
|
-
- Supports commands for
|
|
6
|
+
- Supports commands for extracting, transforming, validating data, and running pipelines
|
|
7
7
|
- Includes options for configuration, state, and output control
|
|
8
8
|
- Exposes handlers for custom command integration
|
|
9
9
|
|
|
@@ -17,20 +17,22 @@ Back to project overview: see the top-level [README](../../README.md).
|
|
|
17
17
|
|
|
18
18
|
## Available Commands
|
|
19
19
|
|
|
20
|
+
- **check**: Inspect pipeline configuration (jobs, sources, targets)
|
|
21
|
+
- **extract**: Extract data from files/APIs/databases
|
|
22
|
+
- **transform**: Transform records
|
|
23
|
+
- **load**: Load data to files/APIs/databases
|
|
24
|
+
- **render**: Render SQL DDL from table specs
|
|
25
|
+
- **validate**: Validate data against rules
|
|
20
26
|
- **run**: Execute a pipeline or job
|
|
21
|
-
- **validate**: Validate pipeline or config files
|
|
22
|
-
- **inspect**: Show pipeline/job details
|
|
23
27
|
|
|
24
28
|
## Command Options
|
|
25
29
|
|
|
26
|
-
|
|
27
|
-
- `--state`: Path to state file
|
|
28
|
-
- `--output`: Output file or format
|
|
30
|
+
Use `etlplus <command> --help` for the exact options supported by each command.
|
|
29
31
|
|
|
30
32
|
## Example: Running a Pipeline
|
|
31
33
|
|
|
32
34
|
```bash
|
|
33
|
-
etlplus run --config configs/pipeline.yml --
|
|
35
|
+
etlplus run --config configs/pipeline.yml --job file_to_file_customers
|
|
34
36
|
```
|
|
35
37
|
|
|
36
38
|
## See Also
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
Entry point helpers for the Typer-powered ``etlplus`` CLI.
|
|
5
5
|
|
|
6
6
|
This module exposes :func:`main` for the console script as well as
|
|
7
|
-
:func:`create_parser
|
|
7
|
+
:func:`create_parser`, which is kept for compatibility but now raises.
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
from __future__ import annotations
|