etlplus 0.16.5__tar.gz → 0.16.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {etlplus-0.16.5 → etlplus-0.16.10}/CONTRIBUTING.md +13 -7
- {etlplus-0.16.5 → etlplus-0.16.10}/DEMO.md +16 -17
- {etlplus-0.16.5/etlplus.egg-info → etlplus-0.16.10}/PKG-INFO +34 -29
- {etlplus-0.16.5 → etlplus-0.16.10}/README.md +33 -28
- {etlplus-0.16.5 → etlplus-0.16.10}/docs/README.md +9 -4
- {etlplus-0.16.5 → etlplus-0.16.10}/docs/pipeline-guide.md +39 -16
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/README.md +4 -4
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/enums.py +2 -2
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/README.md +9 -7
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/main.py +1 -22
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/database/README.md +7 -7
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/README.md +7 -5
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/accdb.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/arrow.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/bson.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/cbor.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/cfg.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/conf.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/dat.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/dta.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/duckdb.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/enums.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/fwf.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/hbs.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/hdf5.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/ini.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/ion.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/jinja2.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/log.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/mat.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/mdb.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/msgpack.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/mustache.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/nc.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/numbers.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/ods.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/pb.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/pbf.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/properties.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/proto.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/psv.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/rda.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/rds.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/sas7bdat.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/sav.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/sqlite.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/sylk.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/tab.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/toml.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/vm.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/wks.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/xls.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/xlsm.py +2 -2
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/xpt.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/zsav.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/README.md +10 -9
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/templates/README.md +11 -9
- {etlplus-0.16.5 → etlplus-0.16.10/etlplus.egg-info}/PKG-INFO +34 -29
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus.egg-info/SOURCES.txt +22 -13
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/README.md +2 -1
- etlplus-0.16.10/pytest.ini +13 -0
- etlplus-0.16.10/tests/README.md +38 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/__init__.py +1 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/conftest.py +186 -28
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/integration/conftest.py +3 -6
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/integration/test_i_cli.py +13 -7
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/integration/test_i_config_load.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/integration/test_i_examples_data_parity.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/integration/test_i_pagination_strategy.py +31 -4
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/integration/test_i_run.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/integration/test_i_run_profile_pagination_defaults.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/integration/test_i_run_profile_rate_limit_defaults.py +2 -1
- etlplus-0.16.10/tests/smoke/__init__.py +12 -0
- etlplus-0.16.10/tests/smoke/conftest.py +214 -0
- etlplus-0.16.10/tests/smoke/test_s_cli_check.py +48 -0
- etlplus-0.16.10/tests/smoke/test_s_cli_extract.py +45 -0
- etlplus-0.16.10/tests/smoke/test_s_cli_load.py +57 -0
- etlplus-0.16.10/tests/smoke/test_s_cli_render.py +65 -0
- etlplus-0.16.10/tests/smoke/test_s_cli_transform.py +52 -0
- etlplus-0.16.10/tests/smoke/test_s_cli_validate.py +47 -0
- etlplus-0.16.10/tests/smoke/test_s_pipeline.py +72 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/api/conftest.py +19 -99
- etlplus-0.16.5/tests/unit/api/test_u_auth.py → etlplus-0.16.10/tests/unit/api/test_u_api_auth.py +4 -3
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/api/test_u_api_config.py +2 -1
- etlplus-0.16.5/tests/unit/api/test_u_endpoint_client.py → etlplus-0.16.10/tests/unit/api/test_u_api_endpoint_client.py +8 -3
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/api/test_u_api_enums.py +3 -2
- etlplus-0.16.5/tests/unit/api/test_u_mocks.py → etlplus-0.16.10/tests/unit/api/test_u_api_mocks.py +3 -3
- etlplus-0.16.5/tests/unit/api/test_u_pagination_client.py → etlplus-0.16.10/tests/unit/api/test_u_api_pagination_client.py +3 -2
- etlplus-0.16.5/tests/unit/api/test_u_pagination_config.py → etlplus-0.16.10/tests/unit/api/test_u_api_pagination_config.py +4 -2
- etlplus-0.16.5/tests/unit/api/test_u_paginator.py → etlplus-0.16.10/tests/unit/api/test_u_api_paginator.py +4 -2
- etlplus-0.16.5/tests/unit/api/test_u_rate_limit_config.py → etlplus-0.16.10/tests/unit/api/test_u_api_rate_limit_config.py +4 -2
- etlplus-0.16.5/tests/unit/api/test_u_rate_limiter.py → etlplus-0.16.10/tests/unit/api/test_u_api_rate_limiter.py +5 -3
- etlplus-0.16.5/tests/unit/api/test_u_request_manager.py → etlplus-0.16.10/tests/unit/api/test_u_api_request_manager.py +6 -2
- etlplus-0.16.5/tests/unit/api/test_u_retry_manager.py → etlplus-0.16.10/tests/unit/api/test_u_api_retry_manager.py +3 -2
- etlplus-0.16.5/tests/unit/api/test_u_transport.py → etlplus-0.16.10/tests/unit/api/test_u_api_transport.py +6 -4
- etlplus-0.16.5/tests/unit/api/test_u_types.py → etlplus-0.16.10/tests/unit/api/test_u_api_types.py +3 -2
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/api/test_u_api_utils.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/cli/conftest.py +37 -17
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/cli/test_u_cli_handlers.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/cli/test_u_cli_io.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/cli/test_u_cli_main.py +6 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/cli/test_u_cli_state.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/conftest.py +18 -124
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/connector/test_u_connector_enums.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/connector/test_u_connector_utils.py +3 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/database/test_u_database_ddl.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/database/test_u_database_engine.py +6 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/database/test_u_database_orm.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/database/test_u_database_schema.py +6 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/file/test_u_file_core.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/file/test_u_file_enums.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/file/test_u_file_yaml.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_enums.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_extract.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_load.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_run.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_transform.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_utils.py +7 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_validate.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/test_u_config.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/test_u_main.py +6 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/test_u_mixins.py +3 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/test_u_utils.py +2 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/test_u_version.py +5 -1
- {etlplus-0.16.5 → etlplus-0.16.10}/tests/unit/workflow/test_u_workflow_jobs.py +9 -4
- etlplus-0.16.5/pytest.ini +0 -12
- etlplus-0.16.5/tests/integration/test_i_pipeline_smoke.py +0 -110
- {etlplus-0.16.5 → etlplus-0.16.10}/.coveragerc +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/.editorconfig +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/.gitattributes +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/.github/actions/python-bootstrap/action.yml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/.github/workflows/ci.yml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/.gitignore +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/.pre-commit-config.yaml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/.ruff.toml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/CODE_OF_CONDUCT.md +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/LICENSE +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/MANIFEST.in +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/Makefile +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/REFERENCES.md +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/SECURITY.md +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/SUPPORT.md +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/docs/snippets/installation_version.md +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/__main__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/__version__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/README.md +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/auth.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/config.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/endpoint_client.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/errors.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/pagination/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/pagination/client.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/pagination/config.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/pagination/paginator.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/rate_limiting/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/rate_limiting/config.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/rate_limiting/rate_limiter.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/request_manager.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/retry_manager.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/transport.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/types.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/api/utils.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/commands.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/constants.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/handlers.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/io.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/options.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/state.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/cli/types.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/config.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/api.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/connector.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/core.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/database.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/enums.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/file.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/types.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/connector/utils.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/database/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/database/ddl.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/database/engine.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/database/orm.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/database/schema.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/database/types.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/enums.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/_imports.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/_io.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/avro.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/core.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/csv.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/feather.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/gz.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/json.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/ndjson.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/orc.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/parquet.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/stub.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/tsv.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/txt.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/xlsx.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/xml.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/yaml.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/file/zip.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/mixins.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/enums.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/extract.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/load.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/run.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/transform.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/types.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/utils.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/ops/validate.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/py.typed +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/templates/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/templates/ddl.sql.j2 +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/templates/view.sql.j2 +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/types.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/utils.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/workflow/README.md +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/workflow/__init__.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/workflow/dag.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/workflow/jobs.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus/workflow/profile.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus.egg-info/dependency_links.txt +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus.egg-info/entry_points.txt +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus.egg-info/requires.txt +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/etlplus.egg-info/top_level.txt +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/configs/ddl_spec.yml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/configs/pipeline.yml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/data/sample.csv +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/data/sample.json +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/data/sample.xml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/data/sample.xsd +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/data/sample.yaml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/examples/quickstart.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/pyproject.toml +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/setup.cfg +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/setup.py +0 -0
- {etlplus-0.16.5 → etlplus-0.16.10}/tools/update_demo_snippets.py +0 -0
|
@@ -14,7 +14,7 @@ email, or any other method with the owners of this repository before making a ch
|
|
|
14
14
|
- [Type Checking](#type-checking)
|
|
15
15
|
- [Typing Philosophy](#typing-philosophy)
|
|
16
16
|
- [Testing](#testing)
|
|
17
|
-
- [Unit vs. Integration](#unit-vs-integration)
|
|
17
|
+
- [Unit vs. Smoke vs. Integration](#unit-vs-smoke-vs-integration)
|
|
18
18
|
- [Where to put tests](#where-to-put-tests)
|
|
19
19
|
- [Common patterns](#common-patterns)
|
|
20
20
|
|
|
@@ -54,8 +54,8 @@ but using the stubs can provide a nicer editing experience.
|
|
|
54
54
|
|
|
55
55
|
We optimize for a great editor experience and a permissive runtime:
|
|
56
56
|
|
|
57
|
-
- TypedDicts in `etlplus/
|
|
58
|
-
`total=False` (all keys optional) and are not enforced at runtime.
|
|
57
|
+
- TypedDicts in `etlplus/*/types.py` (for example `etlplus/api/types.py`) are editor/type-checking
|
|
58
|
+
hints. They are intentionally `total=False` (all keys optional) and are not enforced at runtime.
|
|
59
59
|
- Constructors named `*.from_obj` accept `Mapping[str, Any]` and perform tolerant parsing and light
|
|
60
60
|
casting. This keeps runtime permissive while improving autocomplete and static analysis.
|
|
61
61
|
- Prefer `Mapping[str, Any]` for inputs and plain `dict[...]` for internal state/returns. Avoid
|
|
@@ -104,26 +104,32 @@ class ExampleConfig:
|
|
|
104
104
|
|
|
105
105
|
## Testing
|
|
106
106
|
|
|
107
|
-
### Unit vs. Integration
|
|
107
|
+
### Unit vs. Smoke vs. Integration
|
|
108
108
|
|
|
109
|
-
Use these guidelines to decide whether a test belongs in the unit or integration suite:
|
|
109
|
+
Use these guidelines to decide whether a test belongs in the unit, smoke, or integration suite:
|
|
110
110
|
|
|
111
111
|
- Unit tests (put under `tests/unit/`):
|
|
112
112
|
- Exercise a single function or class in isolation (no orchestration across modules).
|
|
113
113
|
- No real file system or network I/O. Use in-memory data and stubs.
|
|
114
|
-
- Examples in this repo:
|
|
114
|
+
- Examples in this repo: small helpers in `etlplus.utils`, validation and transform functions.
|
|
115
115
|
- Fast and deterministic; rely on `monkeypatch` to stub collaborators.
|
|
116
116
|
|
|
117
117
|
- Integration tests (put under `tests/integration/`):
|
|
118
118
|
- Exercise end-to-end flows across modules and boundaries (CLI `main()`, `run()` pipeline orchestration, file connectors, API client pagination wiring).
|
|
119
119
|
- Can use temporary files/directories, and stub network with fakes/mocks.
|
|
120
|
-
- Examples in this repo: CLI end-to-end,
|
|
120
|
+
- Examples in this repo: CLI end-to-end, pagination strategy, runner defaults for pagination/rate limits, target URL composition.
|
|
121
|
+
|
|
122
|
+
- Smoke tests (put under `tests/smoke/`):
|
|
123
|
+
- Minimal end-to-end checks for core flows; very fast and stable.
|
|
124
|
+
- May touch temporary files but avoid external network calls.
|
|
125
|
+
- Examples in this repo: pipeline smoke tests.
|
|
121
126
|
|
|
122
127
|
If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()`, it is integration by default.
|
|
123
128
|
|
|
124
129
|
### Where to put tests
|
|
125
130
|
|
|
126
131
|
- Unit tests live in `tests/unit/` and should import and test specific functions/classes directly.
|
|
132
|
+
- Smoke tests live in `tests/smoke/` and should focus on minimal end-to-end CLI/pipeline checks.
|
|
127
133
|
- Integration tests live in `tests/integration/` and may simulate real usage using CLI argv, temporary files, and fake clients.
|
|
128
134
|
- The `tests/integration/conftest.py` applies `@pytest.mark.integration` to all tests in that folder. You don’t need to add the marker per test.
|
|
129
135
|
- Markers are declared in `pytest.ini`. Avoid introducing ad-hoc markers without adding them there.
|
|
@@ -32,10 +32,10 @@ etlplus 0.3.4
|
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
$ etlplus --help
|
|
35
|
-
|
|
35
|
+
Usage: etlplus [OPTIONS] COMMAND [ARGS]...
|
|
36
36
|
|
|
37
37
|
ETLPlus - A Swiss Army knife for enabling simple ETL operations
|
|
38
|
-
|
|
38
|
+
# (Output abbreviated; run `etlplus --help` for the full command list.)
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
## Demo 1: Extract Data from Different Sources
|
|
@@ -43,7 +43,7 @@ ETLPlus - A Swiss Army knife for enabling simple ETL operations
|
|
|
43
43
|
### Extract from JSON
|
|
44
44
|
```bash
|
|
45
45
|
$ echo '{"name": "John", "age": 30}' > sample.json
|
|
46
|
-
$ etlplus extract
|
|
46
|
+
$ etlplus extract sample.json
|
|
47
47
|
{
|
|
48
48
|
"name": "John",
|
|
49
49
|
"age": 30
|
|
@@ -111,8 +111,8 @@ $ etlplus transform --operations '{
|
|
|
111
111
|
|
|
112
112
|
### Sort Data
|
|
113
113
|
```bash
|
|
114
|
-
$ etlplus transform
|
|
115
|
-
|
|
114
|
+
$ etlplus transform \
|
|
115
|
+
--operations '{"sort": {"field": "score", "reverse": true}}' \
|
|
116
116
|
'[{"name": "Charlie", "score": 85}, {"name": "Alice", "score": 95}, {"name": "Bob", "score": 90}]'
|
|
117
117
|
```
|
|
118
118
|
|
|
@@ -133,9 +133,8 @@ $ etlplus transform --operations '{"aggregate": {"field": "sales", "func": "sum"
|
|
|
133
133
|
|
|
134
134
|
### Load to JSON File
|
|
135
135
|
```bash
|
|
136
|
-
$
|
|
137
|
-
|
|
138
|
-
output.json --target-type file
|
|
136
|
+
$ echo '{"name": "John", "status": "active"}' \
|
|
137
|
+
| etlplus load output.json --target-type file
|
|
139
138
|
{
|
|
140
139
|
"status": "success",
|
|
141
140
|
"message": "Data loaded to output.json",
|
|
@@ -145,12 +144,12 @@ $ etlplus load \
|
|
|
145
144
|
|
|
146
145
|
### Load to CSV File
|
|
147
146
|
```bash
|
|
148
|
-
$ etlplus load
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
147
|
+
$ cat << 'JSON' | etlplus load users.csv --target-type file
|
|
148
|
+
[
|
|
149
|
+
{"name": "John", "email": "john@example.com"},
|
|
150
|
+
{"name": "Jane", "email": "jane@example.com"}
|
|
151
|
+
]
|
|
152
|
+
JSON
|
|
154
153
|
{
|
|
155
154
|
"status": "success",
|
|
156
155
|
"message": "Data loaded to users.csv",
|
|
@@ -190,7 +189,7 @@ $ etlplus validate \
|
|
|
190
189
|
transformed.json
|
|
191
190
|
|
|
192
191
|
# Step 4: Load
|
|
193
|
-
$
|
|
192
|
+
$ cat transformed.json | etlplus load final_output.csv
|
|
194
193
|
```
|
|
195
194
|
|
|
196
195
|
## Demo 6: Using Python API
|
|
@@ -199,7 +198,7 @@ $ etlplus load transformed.json file final_output.csv
|
|
|
199
198
|
from etlplus.ops import extract, validate, transform, load
|
|
200
199
|
|
|
201
200
|
# Extract
|
|
202
|
-
data = extract("file", "data.csv",
|
|
201
|
+
data = extract("file", "data.csv", file_format="csv")
|
|
203
202
|
|
|
204
203
|
# Validate
|
|
205
204
|
validation_result = validate(data, {
|
|
@@ -214,7 +213,7 @@ if validation_result["valid"]:
|
|
|
214
213
|
})
|
|
215
214
|
|
|
216
215
|
# Load
|
|
217
|
-
load(transformed, "file", "output.json",
|
|
216
|
+
load(transformed, "file", "output.json", file_format="json")
|
|
218
217
|
print("ETL pipeline completed successfully!")
|
|
219
218
|
else:
|
|
220
219
|
print(f"Validation errors: {validation_result['errors']}")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.16.
|
|
3
|
+
Version: 0.16.10
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -141,7 +141,7 @@ ETLPlus supports Python 3.13 and above.
|
|
|
141
141
|
|
|
142
142
|
- **Extract** data from multiple sources:
|
|
143
143
|
- Files (CSV, JSON, XML, YAML)
|
|
144
|
-
- Databases (connection string support)
|
|
144
|
+
- Databases (connection string support; extract is a placeholder today)
|
|
145
145
|
- REST APIs (GET)
|
|
146
146
|
|
|
147
147
|
- **Validate** data with flexible rules:
|
|
@@ -161,7 +161,7 @@ ETLPlus supports Python 3.13 and above.
|
|
|
161
161
|
|
|
162
162
|
- **Load** data to multiple targets:
|
|
163
163
|
- Files (CSV, JSON, XML, YAML)
|
|
164
|
-
- Databases (connection string support)
|
|
164
|
+
- Databases (connection string support; load is a placeholder today)
|
|
165
165
|
- REST APIs (PATCH, POST, PUT)
|
|
166
166
|
|
|
167
167
|
## Installation
|
|
@@ -188,7 +188,7 @@ etlplus --help
|
|
|
188
188
|
etlplus --version
|
|
189
189
|
|
|
190
190
|
# One-liner: extract CSV, filter, select, and write JSON
|
|
191
|
-
etlplus extract
|
|
191
|
+
etlplus extract examples/data/sample.csv \
|
|
192
192
|
| etlplus transform --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
|
|
193
193
|
- temp/sample_output.json
|
|
194
194
|
```
|
|
@@ -218,8 +218,9 @@ include GET for extract and PATCH/POST/PUT for load.
|
|
|
218
218
|
|
|
219
219
|
### Databases (`database`)
|
|
220
220
|
|
|
221
|
-
Database connectors use connection strings for extraction and loading, and
|
|
222
|
-
|
|
221
|
+
Database connectors use connection strings for extraction and loading, and DDL can be rendered from
|
|
222
|
+
table specs for migrations or schema checks. Database extract/load operations are currently
|
|
223
|
+
placeholders; plan to integrate a database client in your runner.
|
|
223
224
|
|
|
224
225
|
### Files (`file`)
|
|
225
226
|
|
|
@@ -298,7 +299,7 @@ Recognized file formats are listed in the tables below. Support for reading to o
|
|
|
298
299
|
| `numbers` | N | N | Apple Numbers |
|
|
299
300
|
| `ods` | N | N | OpenDocument |
|
|
300
301
|
| `wks` | N | N | Lotus 1-2-3 |
|
|
301
|
-
| `xls` | Y |
|
|
302
|
+
| `xls` | Y | N | Microsoft Excel (BIFF; read-only) |
|
|
302
303
|
| `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
|
|
303
304
|
| `xlsx` | Y | Y | Microsoft Excel (Open XML) |
|
|
304
305
|
|
|
@@ -320,8 +321,8 @@ Recognized file formats are listed in the tables below. Support for reading to o
|
|
|
320
321
|
|
|
321
322
|
#### Logs and Event Streams
|
|
322
323
|
|
|
323
|
-
| Format |
|
|
324
|
-
| --- | --- | --- |
|
|
324
|
+
| Format | Read | Write | Description |
|
|
325
|
+
| --- | --- | --- | --- |
|
|
325
326
|
| `log` | N | N | Generic log file |
|
|
326
327
|
|
|
327
328
|
#### Data Archives
|
|
@@ -354,13 +355,14 @@ etlplus --help
|
|
|
354
355
|
etlplus --version
|
|
355
356
|
```
|
|
356
357
|
|
|
357
|
-
The CLI is implemented with Typer (Click-based).
|
|
358
|
-
on the documented commands/flags and run `etlplus <command> --help` for current options.
|
|
358
|
+
The CLI is implemented with Typer (Click-based). The legacy argparse parser has been removed, so
|
|
359
|
+
rely on the documented commands/flags and run `etlplus <command> --help` for current options.
|
|
359
360
|
|
|
360
361
|
**Example error messages:**
|
|
361
362
|
|
|
362
363
|
- If you omit a required argument: `Error: Missing required argument 'SOURCE'.`
|
|
363
|
-
- If you place an option before its argument: `Error: Option '--source-format' must follow the
|
|
364
|
+
- If you place an option before its argument: `Error: Option '--source-format' must follow the
|
|
365
|
+
'SOURCE' argument.`
|
|
364
366
|
|
|
365
367
|
#### Argument Order and Required Options
|
|
366
368
|
|
|
@@ -424,27 +426,27 @@ specific parser.
|
|
|
424
426
|
|
|
425
427
|
Extract from JSON file:
|
|
426
428
|
```bash
|
|
427
|
-
etlplus extract
|
|
429
|
+
etlplus extract examples/data/sample.json
|
|
428
430
|
```
|
|
429
431
|
|
|
430
432
|
Extract from CSV file:
|
|
431
433
|
```bash
|
|
432
|
-
etlplus extract
|
|
434
|
+
etlplus extract examples/data/sample.csv
|
|
433
435
|
```
|
|
434
436
|
|
|
435
437
|
Extract from XML file:
|
|
436
438
|
```bash
|
|
437
|
-
etlplus extract
|
|
439
|
+
etlplus extract examples/data/sample.xml
|
|
438
440
|
```
|
|
439
441
|
|
|
440
442
|
Extract from REST API:
|
|
441
443
|
```bash
|
|
442
|
-
etlplus extract
|
|
444
|
+
etlplus extract https://api.example.com/data
|
|
443
445
|
```
|
|
444
446
|
|
|
445
447
|
Save extracted data to file:
|
|
446
448
|
```bash
|
|
447
|
-
etlplus extract
|
|
449
|
+
etlplus extract examples/data/sample.csv > temp/sample_output.json
|
|
448
450
|
```
|
|
449
451
|
|
|
450
452
|
#### Validate Data
|
|
@@ -510,13 +512,13 @@ etlplus transform \
|
|
|
510
512
|
|
|
511
513
|
Load to JSON file:
|
|
512
514
|
```bash
|
|
513
|
-
etlplus extract
|
|
515
|
+
etlplus extract examples/data/sample.json \
|
|
514
516
|
| etlplus load temp/sample_output.json --target-type file
|
|
515
517
|
```
|
|
516
518
|
|
|
517
519
|
Load to CSV file:
|
|
518
520
|
```bash
|
|
519
|
-
etlplus extract
|
|
521
|
+
etlplus extract examples/data/sample.csv \
|
|
520
522
|
| etlplus load temp/sample_output.csv --target-type file
|
|
521
523
|
```
|
|
522
524
|
|
|
@@ -553,13 +555,13 @@ operations = {
|
|
|
553
555
|
transformed = transform(data, operations)
|
|
554
556
|
|
|
555
557
|
# Load data
|
|
556
|
-
load(transformed, "file", "temp/sample_output.json",
|
|
558
|
+
load(transformed, "file", "temp/sample_output.json", file_format="json")
|
|
557
559
|
```
|
|
558
560
|
|
|
559
561
|
For YAML-driven pipelines executed end-to-end (extract → validate → transform → load), see:
|
|
560
562
|
|
|
561
563
|
- Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
562
|
-
- Runner API and internals:
|
|
564
|
+
- Runner API and internals: see `etlplus.ops.run` docstrings and `docs/pipeline-guide.md`.
|
|
563
565
|
|
|
564
566
|
CLI quick reference for pipelines:
|
|
565
567
|
|
|
@@ -576,7 +578,7 @@ etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
|
576
578
|
|
|
577
579
|
```bash
|
|
578
580
|
# 1. Extract from CSV
|
|
579
|
-
etlplus extract
|
|
581
|
+
etlplus extract examples/data/sample.csv > temp/sample_extracted.json
|
|
580
582
|
|
|
581
583
|
# 2. Transform (filter and select fields)
|
|
582
584
|
etlplus transform \
|
|
@@ -611,7 +613,7 @@ etlplus load output.bin --target-type file --target-format csv < data.json
|
|
|
611
613
|
|
|
612
614
|
# Leave the flags off when extensions already match the desired format
|
|
613
615
|
etlplus extract data.csv --source-type file
|
|
614
|
-
etlplus load
|
|
616
|
+
etlplus load output.json --target-type file < data.json
|
|
615
617
|
```
|
|
616
618
|
|
|
617
619
|
## Transformation Operations
|
|
@@ -706,8 +708,9 @@ Looking for the HTTP client and pagination helpers? See the dedicated docs in
|
|
|
706
708
|
|
|
707
709
|
Curious how the pipeline runner composes API requests, pagination, and load calls?
|
|
708
710
|
|
|
709
|
-
- Runner overview and helpers:
|
|
710
|
-
-
|
|
711
|
+
- Runner overview and helpers: see `etlplus.ops.run` docstrings and
|
|
712
|
+
[`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
713
|
+
- Unified "connector" vocabulary (API/File/DB): `etlplus/connector`
|
|
711
714
|
- API/file targets reuse the same shapes as sources; API targets typically set a `method`.
|
|
712
715
|
|
|
713
716
|
### Running Tests
|
|
@@ -718,10 +721,12 @@ pytest tests/ -v
|
|
|
718
721
|
|
|
719
722
|
#### Test Layers
|
|
720
723
|
|
|
721
|
-
We split tests into
|
|
724
|
+
We split tests into three layers:
|
|
722
725
|
|
|
723
726
|
- **Unit (`tests/unit/`)**: single function or class, no real I/O, fast, uses stubs/monkeypatch
|
|
724
|
-
(e.g.
|
|
727
|
+
(e.g. small helpers in `etlplus.utils`, transform + validate helpers).
|
|
728
|
+
- **Smoke (`tests/smoke/`)**: minimal end-to-end checks for core flows; may touch temp files but
|
|
729
|
+
avoids external network calls.
|
|
725
730
|
- **Integration (`tests/integration/`)**: end-to-end flows (CLI `main()`, pipeline `run()`,
|
|
726
731
|
pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
|
|
727
732
|
fake clients.
|
|
@@ -808,7 +813,7 @@ Navigate to detailed documentation for each subpackage:
|
|
|
808
813
|
- [etlplus.cli](etlplus/cli/README.md): Command-line interface definitions for `etlplus`
|
|
809
814
|
- [etlplus.database](etlplus/database/README.md): Database engine, schema, and ORM helpers
|
|
810
815
|
- [etlplus.templates](etlplus/templates/README.md): SQL and DDL template helpers
|
|
811
|
-
- [etlplus.
|
|
816
|
+
- [etlplus.ops](etlplus/ops/README.md): Extract/validate/transform/load primitives
|
|
812
817
|
- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
|
|
813
818
|
profiles
|
|
814
819
|
|
|
@@ -824,7 +829,7 @@ Navigate to detailed documentation for each subpackage:
|
|
|
824
829
|
- API client docs: [`etlplus/api/README.md`](etlplus/api/README.md)
|
|
825
830
|
- Examples: [`examples/README.md`](examples/README.md)
|
|
826
831
|
- Pipeline authoring guide: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
827
|
-
- Runner internals: [`docs/
|
|
832
|
+
- Runner internals: see `etlplus.ops.run` docstrings and [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
828
833
|
- Design notes (Mapping inputs, dict outputs): [`docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs`](docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs)
|
|
829
834
|
- Typing philosophy: [`CONTRIBUTING.md#typing-philosophy`](CONTRIBUTING.md#typing-philosophy)
|
|
830
835
|
- Demo and walkthrough: [`DEMO.md`](DEMO.md)
|
|
@@ -91,7 +91,7 @@ ETLPlus supports Python 3.13 and above.
|
|
|
91
91
|
|
|
92
92
|
- **Extract** data from multiple sources:
|
|
93
93
|
- Files (CSV, JSON, XML, YAML)
|
|
94
|
-
- Databases (connection string support)
|
|
94
|
+
- Databases (connection string support; extract is a placeholder today)
|
|
95
95
|
- REST APIs (GET)
|
|
96
96
|
|
|
97
97
|
- **Validate** data with flexible rules:
|
|
@@ -111,7 +111,7 @@ ETLPlus supports Python 3.13 and above.
|
|
|
111
111
|
|
|
112
112
|
- **Load** data to multiple targets:
|
|
113
113
|
- Files (CSV, JSON, XML, YAML)
|
|
114
|
-
- Databases (connection string support)
|
|
114
|
+
- Databases (connection string support; load is a placeholder today)
|
|
115
115
|
- REST APIs (PATCH, POST, PUT)
|
|
116
116
|
|
|
117
117
|
## Installation
|
|
@@ -138,7 +138,7 @@ etlplus --help
|
|
|
138
138
|
etlplus --version
|
|
139
139
|
|
|
140
140
|
# One-liner: extract CSV, filter, select, and write JSON
|
|
141
|
-
etlplus extract
|
|
141
|
+
etlplus extract examples/data/sample.csv \
|
|
142
142
|
| etlplus transform --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
|
|
143
143
|
- temp/sample_output.json
|
|
144
144
|
```
|
|
@@ -168,8 +168,9 @@ include GET for extract and PATCH/POST/PUT for load.
|
|
|
168
168
|
|
|
169
169
|
### Databases (`database`)
|
|
170
170
|
|
|
171
|
-
Database connectors use connection strings for extraction and loading, and
|
|
172
|
-
|
|
171
|
+
Database connectors use connection strings for extraction and loading, and DDL can be rendered from
|
|
172
|
+
table specs for migrations or schema checks. Database extract/load operations are currently
|
|
173
|
+
placeholders; plan to integrate a database client in your runner.
|
|
173
174
|
|
|
174
175
|
### Files (`file`)
|
|
175
176
|
|
|
@@ -248,7 +249,7 @@ Recognized file formats are listed in the tables below. Support for reading to o
|
|
|
248
249
|
| `numbers` | N | N | Apple Numbers |
|
|
249
250
|
| `ods` | N | N | OpenDocument |
|
|
250
251
|
| `wks` | N | N | Lotus 1-2-3 |
|
|
251
|
-
| `xls` | Y |
|
|
252
|
+
| `xls` | Y | N | Microsoft Excel (BIFF; read-only) |
|
|
252
253
|
| `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
|
|
253
254
|
| `xlsx` | Y | Y | Microsoft Excel (Open XML) |
|
|
254
255
|
|
|
@@ -270,8 +271,8 @@ Recognized file formats are listed in the tables below. Support for reading to o
|
|
|
270
271
|
|
|
271
272
|
#### Logs and Event Streams
|
|
272
273
|
|
|
273
|
-
| Format |
|
|
274
|
-
| --- | --- | --- |
|
|
274
|
+
| Format | Read | Write | Description |
|
|
275
|
+
| --- | --- | --- | --- |
|
|
275
276
|
| `log` | N | N | Generic log file |
|
|
276
277
|
|
|
277
278
|
#### Data Archives
|
|
@@ -304,13 +305,14 @@ etlplus --help
|
|
|
304
305
|
etlplus --version
|
|
305
306
|
```
|
|
306
307
|
|
|
307
|
-
The CLI is implemented with Typer (Click-based).
|
|
308
|
-
on the documented commands/flags and run `etlplus <command> --help` for current options.
|
|
308
|
+
The CLI is implemented with Typer (Click-based). The legacy argparse parser has been removed, so
|
|
309
|
+
rely on the documented commands/flags and run `etlplus <command> --help` for current options.
|
|
309
310
|
|
|
310
311
|
**Example error messages:**
|
|
311
312
|
|
|
312
313
|
- If you omit a required argument: `Error: Missing required argument 'SOURCE'.`
|
|
313
|
-
- If you place an option before its argument: `Error: Option '--source-format' must follow the
|
|
314
|
+
- If you place an option before its argument: `Error: Option '--source-format' must follow the
|
|
315
|
+
'SOURCE' argument.`
|
|
314
316
|
|
|
315
317
|
#### Argument Order and Required Options
|
|
316
318
|
|
|
@@ -374,27 +376,27 @@ specific parser.
|
|
|
374
376
|
|
|
375
377
|
Extract from JSON file:
|
|
376
378
|
```bash
|
|
377
|
-
etlplus extract
|
|
379
|
+
etlplus extract examples/data/sample.json
|
|
378
380
|
```
|
|
379
381
|
|
|
380
382
|
Extract from CSV file:
|
|
381
383
|
```bash
|
|
382
|
-
etlplus extract
|
|
384
|
+
etlplus extract examples/data/sample.csv
|
|
383
385
|
```
|
|
384
386
|
|
|
385
387
|
Extract from XML file:
|
|
386
388
|
```bash
|
|
387
|
-
etlplus extract
|
|
389
|
+
etlplus extract examples/data/sample.xml
|
|
388
390
|
```
|
|
389
391
|
|
|
390
392
|
Extract from REST API:
|
|
391
393
|
```bash
|
|
392
|
-
etlplus extract
|
|
394
|
+
etlplus extract https://api.example.com/data
|
|
393
395
|
```
|
|
394
396
|
|
|
395
397
|
Save extracted data to file:
|
|
396
398
|
```bash
|
|
397
|
-
etlplus extract
|
|
399
|
+
etlplus extract examples/data/sample.csv > temp/sample_output.json
|
|
398
400
|
```
|
|
399
401
|
|
|
400
402
|
#### Validate Data
|
|
@@ -460,13 +462,13 @@ etlplus transform \
|
|
|
460
462
|
|
|
461
463
|
Load to JSON file:
|
|
462
464
|
```bash
|
|
463
|
-
etlplus extract
|
|
465
|
+
etlplus extract examples/data/sample.json \
|
|
464
466
|
| etlplus load temp/sample_output.json --target-type file
|
|
465
467
|
```
|
|
466
468
|
|
|
467
469
|
Load to CSV file:
|
|
468
470
|
```bash
|
|
469
|
-
etlplus extract
|
|
471
|
+
etlplus extract examples/data/sample.csv \
|
|
470
472
|
| etlplus load temp/sample_output.csv --target-type file
|
|
471
473
|
```
|
|
472
474
|
|
|
@@ -503,13 +505,13 @@ operations = {
|
|
|
503
505
|
transformed = transform(data, operations)
|
|
504
506
|
|
|
505
507
|
# Load data
|
|
506
|
-
load(transformed, "file", "temp/sample_output.json",
|
|
508
|
+
load(transformed, "file", "temp/sample_output.json", file_format="json")
|
|
507
509
|
```
|
|
508
510
|
|
|
509
511
|
For YAML-driven pipelines executed end-to-end (extract → validate → transform → load), see:
|
|
510
512
|
|
|
511
513
|
- Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
512
|
-
- Runner API and internals:
|
|
514
|
+
- Runner API and internals: see `etlplus.ops.run` docstrings and `docs/pipeline-guide.md`.
|
|
513
515
|
|
|
514
516
|
CLI quick reference for pipelines:
|
|
515
517
|
|
|
@@ -526,7 +528,7 @@ etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
|
526
528
|
|
|
527
529
|
```bash
|
|
528
530
|
# 1. Extract from CSV
|
|
529
|
-
etlplus extract
|
|
531
|
+
etlplus extract examples/data/sample.csv > temp/sample_extracted.json
|
|
530
532
|
|
|
531
533
|
# 2. Transform (filter and select fields)
|
|
532
534
|
etlplus transform \
|
|
@@ -561,7 +563,7 @@ etlplus load output.bin --target-type file --target-format csv < data.json
|
|
|
561
563
|
|
|
562
564
|
# Leave the flags off when extensions already match the desired format
|
|
563
565
|
etlplus extract data.csv --source-type file
|
|
564
|
-
etlplus load
|
|
566
|
+
etlplus load output.json --target-type file < data.json
|
|
565
567
|
```
|
|
566
568
|
|
|
567
569
|
## Transformation Operations
|
|
@@ -656,8 +658,9 @@ Looking for the HTTP client and pagination helpers? See the dedicated docs in
|
|
|
656
658
|
|
|
657
659
|
Curious how the pipeline runner composes API requests, pagination, and load calls?
|
|
658
660
|
|
|
659
|
-
- Runner overview and helpers:
|
|
660
|
-
-
|
|
661
|
+
- Runner overview and helpers: see `etlplus.ops.run` docstrings and
|
|
662
|
+
[`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
663
|
+
- Unified "connector" vocabulary (API/File/DB): `etlplus/connector`
|
|
661
664
|
- API/file targets reuse the same shapes as sources; API targets typically set a `method`.
|
|
662
665
|
|
|
663
666
|
### Running Tests
|
|
@@ -668,10 +671,12 @@ pytest tests/ -v
|
|
|
668
671
|
|
|
669
672
|
#### Test Layers
|
|
670
673
|
|
|
671
|
-
We split tests into
|
|
674
|
+
We split tests into three layers:
|
|
672
675
|
|
|
673
676
|
- **Unit (`tests/unit/`)**: single function or class, no real I/O, fast, uses stubs/monkeypatch
|
|
674
|
-
(e.g.
|
|
677
|
+
(e.g. small helpers in `etlplus.utils`, transform + validate helpers).
|
|
678
|
+
- **Smoke (`tests/smoke/`)**: minimal end-to-end checks for core flows; may touch temp files but
|
|
679
|
+
avoids external network calls.
|
|
675
680
|
- **Integration (`tests/integration/`)**: end-to-end flows (CLI `main()`, pipeline `run()`,
|
|
676
681
|
pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
|
|
677
682
|
fake clients.
|
|
@@ -758,7 +763,7 @@ Navigate to detailed documentation for each subpackage:
|
|
|
758
763
|
- [etlplus.cli](etlplus/cli/README.md): Command-line interface definitions for `etlplus`
|
|
759
764
|
- [etlplus.database](etlplus/database/README.md): Database engine, schema, and ORM helpers
|
|
760
765
|
- [etlplus.templates](etlplus/templates/README.md): SQL and DDL template helpers
|
|
761
|
-
- [etlplus.
|
|
766
|
+
- [etlplus.ops](etlplus/ops/README.md): Extract/validate/transform/load primitives
|
|
762
767
|
- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
|
|
763
768
|
profiles
|
|
764
769
|
|
|
@@ -774,7 +779,7 @@ Navigate to detailed documentation for each subpackage:
|
|
|
774
779
|
- API client docs: [`etlplus/api/README.md`](etlplus/api/README.md)
|
|
775
780
|
- Examples: [`examples/README.md`](examples/README.md)
|
|
776
781
|
- Pipeline authoring guide: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
777
|
-
- Runner internals: [`docs/
|
|
782
|
+
- Runner internals: see `etlplus.ops.run` docstrings and [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
778
783
|
- Design notes (Mapping inputs, dict outputs): [`docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs`](docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs)
|
|
779
784
|
- Typing philosophy: [`CONTRIBUTING.md#typing-philosophy`](CONTRIBUTING.md#typing-philosophy)
|
|
780
785
|
- Demo and walkthrough: [`DEMO.md`](DEMO.md)
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
# Documentation Notes
|
|
2
2
|
|
|
3
|
+
- [Documentation Notes](#documentation-notes)
|
|
4
|
+
- [CLI Parser Status](#cli-parser-status)
|
|
5
|
+
- [Migration Hints](#migration-hints)
|
|
6
|
+
|
|
3
7
|
## CLI Parser Status
|
|
4
|
-
|
|
5
|
-
|
|
8
|
+
|
|
9
|
+
- The CLI is Typer/Click-only. The historical `argparse` parser has been removed.
|
|
6
10
|
- Downstream tools should invoke the Typer app exported at `etlplus.cli.commands.app` (e.g., `python
|
|
7
11
|
-m etlplus` or `etlplus ...`).
|
|
8
12
|
- Handler functions still accept keyword arguments; the legacy namespace shim is temporary and will
|
|
@@ -10,8 +14,9 @@
|
|
|
10
14
|
handlers with explicit keyword arguments if you integrate programmatically.
|
|
11
15
|
|
|
12
16
|
## Migration Hints
|
|
13
|
-
|
|
14
|
-
|
|
17
|
+
|
|
18
|
+
- Replace any `argparse`-based integrations with Typer invocations (`etlplus` binary or `app`
|
|
19
|
+
directly).
|
|
15
20
|
- If you maintained custom subcommands around the old parser, port them to Typer by attaching to
|
|
16
21
|
`app` or wrapping the `etlplus` executable.
|
|
17
22
|
- Tests and examples now target the Typer surface; expect argparse-focused helpers (e.g., namespace
|