etlplus 0.13.0__tar.gz → 0.15.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {etlplus-0.13.0 → etlplus-0.15.2}/CONTRIBUTING.md +1 -1
- {etlplus-0.13.0 → etlplus-0.15.2}/DEMO.md +1 -1
- {etlplus-0.13.0/etlplus.egg-info → etlplus-0.15.2}/PKG-INFO +7 -7
- {etlplus-0.13.0 → etlplus-0.15.2}/README.md +6 -6
- {etlplus-0.13.0 → etlplus-0.15.2}/docs/pipeline-guide.md +7 -7
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/README.md +5 -5
- etlplus-0.15.2/etlplus/__init__.py +18 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/README.md +33 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/__init__.py +10 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/config.py +39 -28
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/endpoint_client.py +3 -3
- etlplus-0.15.2/etlplus/api/enums.py +51 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/pagination/client.py +1 -1
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/rate_limiting/config.py +13 -1
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/rate_limiting/rate_limiter.py +8 -11
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/request_manager.py +11 -6
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/transport.py +14 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/types.py +96 -6
- etlplus-0.13.0/etlplus/run_helpers.py → etlplus-0.15.2/etlplus/api/utils.py +209 -153
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/README.md +2 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/commands.py +75 -42
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/handlers.py +40 -12
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/main.py +1 -1
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/state.py +4 -7
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/database/README.md +2 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/database/engine.py +18 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/database/orm.py +2 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/enums.py +0 -32
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/README.md +2 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/_io.py +39 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/json.py +2 -14
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/yaml.py +2 -14
- {etlplus-0.13.0/etlplus/validation → etlplus-0.15.2/etlplus/ops}/README.md +2 -2
- etlplus-0.15.2/etlplus/ops/__init__.py +61 -0
- {etlplus-0.13.0/etlplus → etlplus-0.15.2/etlplus/ops}/extract.py +78 -94
- {etlplus-0.13.0/etlplus → etlplus-0.15.2/etlplus/ops}/load.py +73 -93
- {etlplus-0.13.0/etlplus → etlplus-0.15.2/etlplus/ops}/run.py +153 -118
- {etlplus-0.13.0/etlplus → etlplus-0.15.2/etlplus/ops}/transform.py +75 -68
- {etlplus-0.13.0/etlplus/validation → etlplus-0.15.2/etlplus/ops}/utils.py +53 -17
- {etlplus-0.13.0/etlplus → etlplus-0.15.2/etlplus/ops}/validate.py +22 -12
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/templates/README.md +2 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/types.py +5 -4
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/utils.py +136 -2
- {etlplus-0.13.0/etlplus/config → etlplus-0.15.2/etlplus/workflow}/README.md +6 -6
- {etlplus-0.13.0/etlplus/config → etlplus-0.15.2/etlplus/workflow}/__init__.py +10 -23
- {etlplus-0.13.0/etlplus/config → etlplus-0.15.2/etlplus/workflow}/connector.py +58 -44
- {etlplus-0.13.0/etlplus → etlplus-0.15.2/etlplus/workflow}/dag.py +6 -4
- {etlplus-0.13.0/etlplus/config → etlplus-0.15.2/etlplus/workflow}/jobs.py +101 -38
- {etlplus-0.13.0/etlplus/config → etlplus-0.15.2/etlplus/workflow}/pipeline.py +57 -49
- {etlplus-0.13.0/etlplus/config → etlplus-0.15.2/etlplus/workflow}/profile.py +8 -5
- etlplus-0.15.2/etlplus/workflow/types.py +115 -0
- {etlplus-0.13.0 → etlplus-0.15.2/etlplus.egg-info}/PKG-INFO +7 -7
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus.egg-info/SOURCES.txt +29 -29
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/README.md +2 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/configs/pipeline.yml +4 -4
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/quickstart_python.py +5 -5
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/conftest.py +10 -10
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/test_i_pagination_strategy.py +13 -13
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/test_i_pipeline_yaml_load.py +1 -1
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/test_i_run.py +2 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/test_i_run_profile_pagination_defaults.py +1 -1
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/test_i_run_profile_rate_limit_defaults.py +1 -1
- etlplus-0.15.2/tests/unit/api/test_u_api_enums.py +34 -0
- etlplus-0.13.0/tests/unit/test_u_run_helpers.py → etlplus-0.15.2/tests/unit/api/test_u_api_utils.py +17 -15
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/cli/conftest.py +2 -2
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/cli/test_u_cli_handlers.py +3 -1
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/conftest.py +1 -1
- etlplus-0.13.0/tests/unit/test_u_extract.py → etlplus-0.15.2/tests/unit/ops/test_u_ops_extract.py +12 -12
- etlplus-0.13.0/tests/unit/test_u_load.py → etlplus-0.15.2/tests/unit/ops/test_u_ops_load.py +18 -17
- etlplus-0.13.0/tests/unit/test_u_run.py → etlplus-0.15.2/tests/unit/ops/test_u_ops_run.py +4 -4
- etlplus-0.13.0/tests/unit/test_u_transform.py → etlplus-0.15.2/tests/unit/ops/test_u_ops_transform.py +65 -64
- etlplus-0.13.0/tests/unit/validation/test_u_validation_utils.py → etlplus-0.15.2/tests/unit/ops/test_u_ops_utils.py +4 -4
- etlplus-0.13.0/tests/unit/test_u_validate.py → etlplus-0.15.2/tests/unit/ops/test_u_ops_validate.py +9 -9
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/test_u_enums.py +0 -16
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/test_u_utils.py +110 -4
- etlplus-0.13.0/tests/unit/config/test_u_connector.py → etlplus-0.15.2/tests/unit/workflow/test_u_workflow_connector.py +6 -6
- etlplus-0.13.0/tests/unit/config/test_u_jobs.py → etlplus-0.15.2/tests/unit/workflow/test_u_workflow_jobs.py +3 -3
- etlplus-0.13.0/tests/unit/config/test_u_pipeline.py → etlplus-0.15.2/tests/unit/workflow/test_u_workflow_pipeline.py +38 -34
- etlplus-0.13.0/etlplus/__init__.py +0 -43
- etlplus-0.13.0/etlplus/config/types.py +0 -204
- etlplus-0.13.0/etlplus/config/utils.py +0 -120
- etlplus-0.13.0/etlplus/validation/__init__.py +0 -44
- etlplus-0.13.0/tests/unit/config/test_u_config_utils.py +0 -129
- {etlplus-0.13.0 → etlplus-0.15.2}/.coveragerc +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/.editorconfig +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/.gitattributes +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/.github/actions/python-bootstrap/action.yml +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/.github/workflows/ci.yml +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/.gitignore +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/.pre-commit-config.yaml +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/.ruff.toml +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/CODE_OF_CONDUCT.md +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/LICENSE +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/MANIFEST.in +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/Makefile +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/REFERENCES.md +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/SECURITY.md +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/SUPPORT.md +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/docs/README.md +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/docs/snippets/installation_version.md +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/__main__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/__version__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/auth.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/errors.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/pagination/__init__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/pagination/config.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/pagination/paginator.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/rate_limiting/__init__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/api/retry_manager.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/__init__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/constants.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/io.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/options.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/cli/types.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/database/__init__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/database/ddl.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/database/schema.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/database/types.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/__init__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/_imports.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/accdb.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/arrow.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/avro.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/bson.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/cbor.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/cfg.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/conf.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/core.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/csv.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/dat.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/dta.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/duckdb.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/enums.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/feather.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/fwf.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/gz.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/hbs.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/hdf5.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/ini.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/ion.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/jinja2.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/log.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/mat.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/mdb.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/msgpack.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/mustache.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/nc.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/ndjson.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/numbers.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/ods.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/orc.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/parquet.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/pb.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/pbf.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/properties.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/proto.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/psv.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/rda.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/rds.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/sas7bdat.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/sav.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/sqlite.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/stub.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/sylk.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/tab.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/toml.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/tsv.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/txt.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/vm.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/wks.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/xls.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/xlsm.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/xlsx.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/xml.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/xpt.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/zip.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/file/zsav.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/mixins.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/py.typed +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/templates/__init__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/templates/ddl.sql.j2 +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus/templates/view.sql.j2 +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus.egg-info/dependency_links.txt +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus.egg-info/entry_points.txt +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus.egg-info/requires.txt +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/etlplus.egg-info/top_level.txt +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/configs/ddl_spec.yml +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/data/sample.csv +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/data/sample.json +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/data/sample.xml +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/data/sample.xsd +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/examples/data/sample.yaml +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/pyproject.toml +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/pytest.ini +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/setup.cfg +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/setup.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/__init__.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/conftest.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/test_i_cli.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/test_i_examples_data_parity.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/integration/test_i_pipeline_smoke.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/conftest.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_auth.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_config.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_endpoint_client.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_mocks.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_pagination_client.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_pagination_config.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_paginator.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_rate_limit_config.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_rate_limiter.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_request_manager.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_retry_manager.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_transport.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/api/test_u_types.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/cli/test_u_cli_io.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/cli/test_u_cli_main.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/cli/test_u_cli_state.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/database/test_u_database_ddl.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/database/test_u_database_engine.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/database/test_u_database_orm.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/database/test_u_database_schema.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/file/test_u_file_core.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/file/test_u_file_enums.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/file/test_u_file_yaml.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/test_u_main.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/test_u_mixins.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tests/unit/test_u_version.py +0 -0
- {etlplus-0.13.0 → etlplus-0.15.2}/tools/update_demo_snippets.py +0 -0
|
@@ -119,7 +119,7 @@ Use these guidelines to decide whether a test belongs in the unit or integration
|
|
|
119
119
|
- Can use temporary files/directories, and stub network with fakes/mocks.
|
|
120
120
|
- Examples in this repo: CLI end-to-end, pipeline smoke tests, pagination strategy, runner defaults for pagination/rate limits, target URL composition.
|
|
121
121
|
|
|
122
|
-
If a test calls `etlplus.cli.main()` or `etlplus.run.run()`, it is integration by default.
|
|
122
|
+
If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()`, it is integration by default.
|
|
123
123
|
|
|
124
124
|
### Where to put tests
|
|
125
125
|
|
|
@@ -196,7 +196,7 @@ $ etlplus load transformed.json file final_output.csv
|
|
|
196
196
|
## Demo 6: Using Python API
|
|
197
197
|
|
|
198
198
|
```python
|
|
199
|
-
from etlplus import extract, validate, transform, load
|
|
199
|
+
from etlplus.ops import extract, validate, transform, load
|
|
200
200
|
|
|
201
201
|
# Extract
|
|
202
202
|
data = extract("file", "data.csv", format="csv")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.2
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -196,7 +196,7 @@ etlplus extract file examples/data/sample.csv \
|
|
|
196
196
|
[Python API](#python-api):
|
|
197
197
|
|
|
198
198
|
```python
|
|
199
|
-
from etlplus import extract, transform, validate, load
|
|
199
|
+
from etlplus.ops import extract, transform, validate, load
|
|
200
200
|
|
|
201
201
|
data = extract("file", "input.csv")
|
|
202
202
|
ops = {"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}
|
|
@@ -531,7 +531,7 @@ cat examples/data/sample.json \
|
|
|
531
531
|
Use ETLPlus as a Python library:
|
|
532
532
|
|
|
533
533
|
```python
|
|
534
|
-
from etlplus import extract, validate, transform, load
|
|
534
|
+
from etlplus.ops import extract, validate, transform, load
|
|
535
535
|
|
|
536
536
|
# Extract data
|
|
537
537
|
data = extract("file", "data.json")
|
|
@@ -726,7 +726,7 @@ We split tests into two layers:
|
|
|
726
726
|
pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
|
|
727
727
|
fake clients.
|
|
728
728
|
|
|
729
|
-
If a test calls `etlplus.cli.main()` or `etlplus.run.run()` it’s integration by default.
|
|
729
|
+
If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()` it’s integration by default. Full
|
|
730
730
|
criteria: [`CONTRIBUTING.md#testing`](CONTRIBUTING.md#testing).
|
|
731
731
|
|
|
732
732
|
### Code Coverage
|
|
@@ -805,12 +805,12 @@ Navigate to detailed documentation for each subpackage:
|
|
|
805
805
|
|
|
806
806
|
- [etlplus.api](etlplus/api/README.md): Lightweight HTTP client and paginated REST helpers
|
|
807
807
|
- [etlplus.file](etlplus/file/README.md): Unified file format support and helpers
|
|
808
|
-
- [etlplus.
|
|
809
|
-
and profiles
|
|
810
|
-
- [etlplus.cli](etlplus/cli/README.md): Command-line interface for ETLPlus workflows
|
|
808
|
+
- [etlplus.cli](etlplus/cli/README.md): Command-line interface definitions for `etlplus`
|
|
811
809
|
- [etlplus.database](etlplus/database/README.md): Database engine, schema, and ORM helpers
|
|
812
810
|
- [etlplus.templates](etlplus/templates/README.md): SQL and DDL template helpers
|
|
813
811
|
- [etlplus.validation](etlplus/validation/README.md): Data validation utilities and helpers
|
|
812
|
+
- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
|
|
813
|
+
profiles
|
|
814
814
|
|
|
815
815
|
### Community Health
|
|
816
816
|
|
|
@@ -146,7 +146,7 @@ etlplus extract file examples/data/sample.csv \
|
|
|
146
146
|
[Python API](#python-api):
|
|
147
147
|
|
|
148
148
|
```python
|
|
149
|
-
from etlplus import extract, transform, validate, load
|
|
149
|
+
from etlplus.ops import extract, transform, validate, load
|
|
150
150
|
|
|
151
151
|
data = extract("file", "input.csv")
|
|
152
152
|
ops = {"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}
|
|
@@ -481,7 +481,7 @@ cat examples/data/sample.json \
|
|
|
481
481
|
Use ETLPlus as a Python library:
|
|
482
482
|
|
|
483
483
|
```python
|
|
484
|
-
from etlplus import extract, validate, transform, load
|
|
484
|
+
from etlplus.ops import extract, validate, transform, load
|
|
485
485
|
|
|
486
486
|
# Extract data
|
|
487
487
|
data = extract("file", "data.json")
|
|
@@ -676,7 +676,7 @@ We split tests into two layers:
|
|
|
676
676
|
pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
|
|
677
677
|
fake clients.
|
|
678
678
|
|
|
679
|
-
If a test calls `etlplus.cli.main()` or `etlplus.run.run()` it’s integration by default.
|
|
679
|
+
If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()` it’s integration by default. Full
|
|
680
680
|
criteria: [`CONTRIBUTING.md#testing`](CONTRIBUTING.md#testing).
|
|
681
681
|
|
|
682
682
|
### Code Coverage
|
|
@@ -755,12 +755,12 @@ Navigate to detailed documentation for each subpackage:
|
|
|
755
755
|
|
|
756
756
|
- [etlplus.api](etlplus/api/README.md): Lightweight HTTP client and paginated REST helpers
|
|
757
757
|
- [etlplus.file](etlplus/file/README.md): Unified file format support and helpers
|
|
758
|
-
- [etlplus.
|
|
759
|
-
and profiles
|
|
760
|
-
- [etlplus.cli](etlplus/cli/README.md): Command-line interface for ETLPlus workflows
|
|
758
|
+
- [etlplus.cli](etlplus/cli/README.md): Command-line interface definitions for `etlplus`
|
|
761
759
|
- [etlplus.database](etlplus/database/README.md): Database engine, schema, and ORM helpers
|
|
762
760
|
- [etlplus.templates](etlplus/templates/README.md): SQL and DDL template helpers
|
|
763
761
|
- [etlplus.validation](etlplus/validation/README.md): Data validation utilities and helpers
|
|
762
|
+
- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
|
|
763
|
+
profiles
|
|
764
764
|
|
|
765
765
|
### Community Health
|
|
766
766
|
|
|
@@ -281,7 +281,7 @@ section.
|
|
|
281
281
|
|
|
282
282
|
## Validations
|
|
283
283
|
|
|
284
|
-
Validation rule sets map field names to rules, mirroring `etlplus.validate.FieldRules`:
|
|
284
|
+
Validation rule sets map field names to rules, mirroring `etlplus.ops.validate.FieldRules`:
|
|
285
285
|
|
|
286
286
|
```yaml
|
|
287
287
|
validations:
|
|
@@ -297,7 +297,7 @@ validations:
|
|
|
297
297
|
|
|
298
298
|
## Transforms
|
|
299
299
|
|
|
300
|
-
Transformation pipelines follow `etlplus.transform` shapes exactly:
|
|
300
|
+
Transformation pipelines follow `etlplus.ops.transform` shapes exactly:
|
|
301
301
|
|
|
302
302
|
```yaml
|
|
303
303
|
transforms:
|
|
@@ -377,9 +377,9 @@ Details:
|
|
|
377
377
|
- Unknown or malformed entries are skipped rather than failing the whole load (keeping pipeline
|
|
378
378
|
authoring permissive).
|
|
379
379
|
- The connector kind is also available as a type-safe literal in code as
|
|
380
|
-
`etlplus.
|
|
380
|
+
`etlplus.workflow.ConnectorType` (values: `"file" | "database" | "api"`).
|
|
381
381
|
|
|
382
|
-
To add new connector kinds in the future, implement a new dataclass in `etlplus.
|
|
382
|
+
To add new connector kinds in the future, implement a new dataclass in `etlplus.workflow.connector`
|
|
383
383
|
and extend the internal parser to handle its `type` value.
|
|
384
384
|
|
|
385
385
|
## Jobs
|
|
@@ -431,14 +431,14 @@ Notes:
|
|
|
431
431
|
- Environment-variable substitution (e.g. `${GITHUB_TOKEN}`) is applied the same way as when loading
|
|
432
432
|
configs via the Python API.
|
|
433
433
|
- For more details on the orchestration implementation, see
|
|
434
|
-
[Runner internals: etlplus.run](run-module.md).
|
|
434
|
+
[Runner internals: etlplus.ops.run](run-module.md).
|
|
435
435
|
|
|
436
|
-
### Python: `etlplus.run.run`
|
|
436
|
+
### Python: `etlplus.ops.run.run`
|
|
437
437
|
|
|
438
438
|
To trigger a job programmatically, use the high-level runner function exposed by the package:
|
|
439
439
|
|
|
440
440
|
```python
|
|
441
|
-
from etlplus.run import run as run_job
|
|
441
|
+
from etlplus.ops.run import run as run_job
|
|
442
442
|
|
|
443
443
|
result = run_job(
|
|
444
444
|
job="file_to_file_customers",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# etlplus
|
|
1
|
+
# `etlplus` Package
|
|
2
2
|
|
|
3
3
|
The `etlplus` package provides a unified Python API and CLI for ETL operations: extraction,
|
|
4
4
|
validation, transformation, and loading of data from files, APIs, and databases.
|
|
@@ -13,17 +13,17 @@ Back to project overview: see the top-level [README](../README.md).
|
|
|
13
13
|
|
|
14
14
|
- [etlplus.api](api/README.md): Lightweight HTTP client and paginated REST helpers
|
|
15
15
|
- [etlplus.file](file/README.md): Unified file format support and helpers
|
|
16
|
-
- [etlplus.
|
|
17
|
-
profiles
|
|
18
|
-
- [etlplus.cli](cli/README.md): Command-line interface for ETLPlus workflows
|
|
16
|
+
- [etlplus.cli](cli/README.md): Command-line interface definitions for `etlplus`
|
|
19
17
|
- [etlplus.database](database/README.md): Database engine, schema, and ORM helpers
|
|
20
18
|
- [etlplus.templates](templates/README.md): SQL and DDL template helpers
|
|
21
19
|
- [etlplus.validation](validation/README.md): Data validation utilities and helpers
|
|
20
|
+
- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
|
|
21
|
+
profiles
|
|
22
22
|
|
|
23
23
|
## Quickstart
|
|
24
24
|
|
|
25
25
|
```python
|
|
26
|
-
from etlplus import extract, validate, transform, load
|
|
26
|
+
from etlplus.ops import extract, validate, transform, load
|
|
27
27
|
|
|
28
28
|
data = extract("file", "input.csv")
|
|
29
29
|
filtered = transform(data, {"filter": {"field": "age", "op": "gt", "value": 25}})
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus` package.
|
|
3
|
+
|
|
4
|
+
Top-level facade for the ETLPlus toolkit.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from .__version__ import __version__
|
|
8
|
+
|
|
9
|
+
__author__ = 'ETLPlus Team'
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# SECTION: EXPORTS ========================================================== #
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
'__author__',
|
|
17
|
+
'__version__',
|
|
18
|
+
]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# etlplus.api
|
|
1
|
+
# `etlplus.api` Subpackage
|
|
2
2
|
|
|
3
3
|
Documentation for the `etlplus.api` subpackage: a lightweight HTTP client and helpers for paginated
|
|
4
4
|
REST endpoints.
|
|
@@ -12,7 +12,7 @@ REST endpoints.
|
|
|
12
12
|
|
|
13
13
|
Back to project overview: see the top-level [README](../../README.md).
|
|
14
14
|
|
|
15
|
-
- [etlplus.api
|
|
15
|
+
- [`etlplus.api` Subpackage](#etlplusapi-subpackage)
|
|
16
16
|
- [Installation](#installation)
|
|
17
17
|
- [Quickstart](#quickstart)
|
|
18
18
|
- [Overriding Rate Limits Per Call](#overriding-rate-limits-per-call)
|
|
@@ -22,6 +22,7 @@ Back to project overview: see the top-level [README](../../README.md).
|
|
|
22
22
|
- [Authentication](#authentication)
|
|
23
23
|
- [Errors and Rate Limiting](#errors-and-rate-limiting)
|
|
24
24
|
- [Types and Transport](#types-and-transport)
|
|
25
|
+
- [Config Schemas](#config-schemas)
|
|
25
26
|
- [Supporting Modules](#supporting-modules)
|
|
26
27
|
- [Minimal Contract](#minimal-contract)
|
|
27
28
|
- [See also](#see-also)
|
|
@@ -225,6 +226,36 @@ providers can fall back to their own defaults. If you already possess a static t
|
|
|
225
226
|
`etlplus/api/request_manager.py` wraps `requests` sessions plus retry orchestration. Advanced
|
|
226
227
|
users may consult those modules to adapt behavior.
|
|
227
228
|
|
|
229
|
+
## Config Schemas
|
|
230
|
+
|
|
231
|
+
`etlplus.api.types` defines TypedDict-based configuration shapes for API profiles and endpoints.
|
|
232
|
+
Runtime parsing remains permissive in `etlplus.api.config`, but these types improve IDE
|
|
233
|
+
autocomplete and static analysis.
|
|
234
|
+
|
|
235
|
+
Exported types:
|
|
236
|
+
|
|
237
|
+
- `ApiConfigMap`: top-level API config shape
|
|
238
|
+
- `ApiProfileConfigMap`: per-profile API config shape
|
|
239
|
+
- `ApiProfileDefaultsMap`: defaults block within a profile
|
|
240
|
+
- `EndpointMap`: endpoint config shape
|
|
241
|
+
|
|
242
|
+
Example:
|
|
243
|
+
|
|
244
|
+
```python
|
|
245
|
+
from etlplus.api import ApiConfigMap
|
|
246
|
+
|
|
247
|
+
api_cfg: ApiConfigMap = {
|
|
248
|
+
"base_url": "https://example.test",
|
|
249
|
+
"headers": {"Authorization": "Bearer token"},
|
|
250
|
+
"endpoints": {
|
|
251
|
+
"users": {
|
|
252
|
+
"path": "/users",
|
|
253
|
+
"method": "GET",
|
|
254
|
+
},
|
|
255
|
+
},
|
|
256
|
+
}
|
|
257
|
+
```
|
|
258
|
+
|
|
228
259
|
## Supporting Modules
|
|
229
260
|
|
|
230
261
|
- `etlplus.api.types` collects friendly aliases such as `Headers`, `Params`, `Url`, and
|
|
@@ -78,6 +78,7 @@ from .config import ApiConfig
|
|
|
78
78
|
from .config import ApiProfileConfig
|
|
79
79
|
from .config import EndpointConfig
|
|
80
80
|
from .endpoint_client import EndpointClient
|
|
81
|
+
from .enums import HttpMethod
|
|
81
82
|
from .pagination import CursorPaginationConfigMap
|
|
82
83
|
from .pagination import PagePaginationConfigMap
|
|
83
84
|
from .pagination import PaginationClient
|
|
@@ -98,6 +99,10 @@ from .types import Headers
|
|
|
98
99
|
from .types import Params
|
|
99
100
|
from .types import RequestOptions
|
|
100
101
|
from .types import Url
|
|
102
|
+
from .utils import compose_api_request_env
|
|
103
|
+
from .utils import compose_api_target_env
|
|
104
|
+
from .utils import paginate_with_client
|
|
105
|
+
from .utils import resolve_request
|
|
101
106
|
|
|
102
107
|
# SECTION: EXPORTS ========================================================== #
|
|
103
108
|
|
|
@@ -119,9 +124,14 @@ __all__ = [
|
|
|
119
124
|
'RequestOptions',
|
|
120
125
|
'RetryStrategy',
|
|
121
126
|
# Enums
|
|
127
|
+
'HttpMethod',
|
|
122
128
|
'PaginationType',
|
|
123
129
|
# Functions
|
|
124
130
|
'build_http_adapter',
|
|
131
|
+
'compose_api_request_env',
|
|
132
|
+
'compose_api_target_env',
|
|
133
|
+
'paginate_with_client',
|
|
134
|
+
'resolve_request',
|
|
125
135
|
# Type Aliases
|
|
126
136
|
'CursorPaginationConfigMap',
|
|
127
137
|
'Headers',
|
|
@@ -3,11 +3,6 @@
|
|
|
3
3
|
|
|
4
4
|
Configuration dataclasses for REST API services, profiles, and endpoints.
|
|
5
5
|
|
|
6
|
-
These models used to live under :mod:`etlplus.config`, but they belong in the
|
|
7
|
-
API layer because they compose runtime types such as
|
|
8
|
-
:class:`etlplus.api.EndpointClient`, :class:`etlplus.api.PaginationConfig`, and
|
|
9
|
-
:class:`etlplus.api.RateLimitConfig`.
|
|
10
|
-
|
|
11
6
|
Notes
|
|
12
7
|
-----
|
|
13
8
|
- TypedDict references remain editor hints only; :meth:`from_obj` accepts
|
|
@@ -18,6 +13,7 @@ Notes
|
|
|
18
13
|
|
|
19
14
|
from __future__ import annotations
|
|
20
15
|
|
|
16
|
+
from collections.abc import Callable
|
|
21
17
|
from collections.abc import Mapping
|
|
22
18
|
from dataclasses import dataclass
|
|
23
19
|
from dataclasses import field
|
|
@@ -29,20 +25,20 @@ from typing import overload
|
|
|
29
25
|
from urllib.parse import urlsplit
|
|
30
26
|
from urllib.parse import urlunsplit
|
|
31
27
|
|
|
32
|
-
from ..enums import HttpMethod
|
|
33
28
|
from ..types import StrAnyMap
|
|
34
29
|
from ..types import StrStrMap
|
|
35
30
|
from ..utils import cast_str_dict
|
|
36
31
|
from ..utils import coerce_dict
|
|
37
32
|
from ..utils import maybe_mapping
|
|
38
33
|
from .endpoint_client import EndpointClient
|
|
34
|
+
from .enums import HttpMethod
|
|
39
35
|
from .pagination import PaginationConfig
|
|
40
36
|
from .rate_limiting import RateLimitConfig
|
|
41
37
|
|
|
42
38
|
if TYPE_CHECKING:
|
|
43
|
-
from
|
|
44
|
-
from
|
|
45
|
-
from
|
|
39
|
+
from .types import ApiConfigMap
|
|
40
|
+
from .types import ApiProfileConfigMap
|
|
41
|
+
from .types import EndpointMap
|
|
46
42
|
|
|
47
43
|
|
|
48
44
|
# SECTION: EXPORTS ========================================================== #
|
|
@@ -106,6 +102,33 @@ def _effective_service_defaults(
|
|
|
106
102
|
return fallback_base, fallback_headers
|
|
107
103
|
|
|
108
104
|
|
|
105
|
+
def _freeze_mapping(
|
|
106
|
+
mapping: Mapping[Any, Any],
|
|
107
|
+
*,
|
|
108
|
+
key_cast: Callable[[Any], Any] | None = None,
|
|
109
|
+
) -> MappingProxyType:
|
|
110
|
+
"""
|
|
111
|
+
Return an immutable copy of a mapping, optionally normalizing keys.
|
|
112
|
+
|
|
113
|
+
Parameters
|
|
114
|
+
----------
|
|
115
|
+
mapping : Mapping[Any, Any]
|
|
116
|
+
Source mapping to freeze.
|
|
117
|
+
key_cast : Callable[[Any], Any] | None, optional
|
|
118
|
+
Optional key coercion applied to each key.
|
|
119
|
+
|
|
120
|
+
Returns
|
|
121
|
+
-------
|
|
122
|
+
MappingProxyType
|
|
123
|
+
Read-only mapping proxy with normalized keys.
|
|
124
|
+
"""
|
|
125
|
+
if key_cast is None:
|
|
126
|
+
data = dict(mapping)
|
|
127
|
+
else:
|
|
128
|
+
data = {key_cast(key): value for key, value in mapping.items()}
|
|
129
|
+
return MappingProxyType(data)
|
|
130
|
+
|
|
131
|
+
|
|
109
132
|
def _normalize_method(
|
|
110
133
|
value: Any,
|
|
111
134
|
) -> Any | None:
|
|
@@ -232,16 +255,8 @@ class ApiProfileConfig:
|
|
|
232
255
|
# -- Magic Methods (Object Lifecycle) -- #
|
|
233
256
|
|
|
234
257
|
def __post_init__(self) -> None:
|
|
235
|
-
object.__setattr__(
|
|
236
|
-
|
|
237
|
-
'headers',
|
|
238
|
-
MappingProxyType(dict(self.headers)),
|
|
239
|
-
)
|
|
240
|
-
object.__setattr__(
|
|
241
|
-
self,
|
|
242
|
-
'auth',
|
|
243
|
-
MappingProxyType(dict(self.auth)),
|
|
244
|
-
)
|
|
258
|
+
object.__setattr__(self, 'headers', _freeze_mapping(self.headers))
|
|
259
|
+
object.__setattr__(self, 'auth', _freeze_mapping(self.auth))
|
|
245
260
|
|
|
246
261
|
# -- Class Methods -- #
|
|
247
262
|
|
|
@@ -340,20 +355,16 @@ class ApiConfig:
|
|
|
340
355
|
# -- Magic Methods (Object Lifecycle) -- #
|
|
341
356
|
|
|
342
357
|
def __post_init__(self) -> None:
|
|
343
|
-
object.__setattr__(
|
|
344
|
-
self,
|
|
345
|
-
'headers',
|
|
346
|
-
MappingProxyType(dict(self.headers)),
|
|
347
|
-
)
|
|
358
|
+
object.__setattr__(self, 'headers', _freeze_mapping(self.headers))
|
|
348
359
|
object.__setattr__(
|
|
349
360
|
self,
|
|
350
361
|
'endpoints',
|
|
351
|
-
|
|
362
|
+
_freeze_mapping(self.endpoints, key_cast=str),
|
|
352
363
|
)
|
|
353
364
|
object.__setattr__(
|
|
354
365
|
self,
|
|
355
366
|
'profiles',
|
|
356
|
-
|
|
367
|
+
_freeze_mapping(self.profiles, key_cast=str),
|
|
357
368
|
)
|
|
358
369
|
|
|
359
370
|
# -- Internal Instance Methods -- #
|
|
@@ -545,12 +556,12 @@ class EndpointConfig:
|
|
|
545
556
|
object.__setattr__(
|
|
546
557
|
self,
|
|
547
558
|
'path_params',
|
|
548
|
-
|
|
559
|
+
_freeze_mapping(self.path_params),
|
|
549
560
|
)
|
|
550
561
|
object.__setattr__(
|
|
551
562
|
self,
|
|
552
563
|
'query_params',
|
|
553
|
-
|
|
564
|
+
_freeze_mapping(self.query_params),
|
|
554
565
|
)
|
|
555
566
|
|
|
556
567
|
# -- Class Methods -- #
|
|
@@ -455,7 +455,7 @@ class EndpointClient:
|
|
|
455
455
|
-------
|
|
456
456
|
JSONData
|
|
457
457
|
Parsed JSON payload or fallback structure matching
|
|
458
|
-
:func:`etlplus.extract.extract_from_api` semantics.
|
|
458
|
+
:func:`etlplus.ops.extract.extract_from_api` semantics.
|
|
459
459
|
"""
|
|
460
460
|
return self._request_manager.get(url, **kwargs)
|
|
461
461
|
|
|
@@ -479,7 +479,7 @@ class EndpointClient:
|
|
|
479
479
|
-------
|
|
480
480
|
JSONData
|
|
481
481
|
Parsed JSON payload or fallback structure matching
|
|
482
|
-
:func:`etlplus.extract.extract_from_api` semantics.
|
|
482
|
+
:func:`etlplus.ops.extract.extract_from_api` semantics.
|
|
483
483
|
"""
|
|
484
484
|
return self._request_manager.post(url, **kwargs)
|
|
485
485
|
|
|
@@ -506,7 +506,7 @@ class EndpointClient:
|
|
|
506
506
|
-------
|
|
507
507
|
JSONData
|
|
508
508
|
Parsed JSON payload or fallback structure matching
|
|
509
|
-
:func:`etlplus.extract.extract_from_api` semantics.
|
|
509
|
+
:func:`etlplus.ops.extract.extract_from_api` semantics.
|
|
510
510
|
"""
|
|
511
511
|
return self._request_manager.request(method, url, **kwargs)
|
|
512
512
|
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.api.enums` module.
|
|
3
|
+
|
|
4
|
+
File-specific REST API-aligned enums and helpers.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from ..enums import CoercibleStrEnum
|
|
10
|
+
|
|
11
|
+
# SECTION: EXPORTS ========================================================= #
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
# Enums
|
|
16
|
+
'HttpMethod',
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# SECTION: ENUMS ============================================================ #
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class HttpMethod(CoercibleStrEnum):
|
|
24
|
+
"""Supported HTTP verbs that accept JSON payloads."""
|
|
25
|
+
|
|
26
|
+
# -- Constants -- #
|
|
27
|
+
|
|
28
|
+
CONNECT = 'connect'
|
|
29
|
+
DELETE = 'delete'
|
|
30
|
+
GET = 'get'
|
|
31
|
+
HEAD = 'head'
|
|
32
|
+
OPTIONS = 'options'
|
|
33
|
+
PATCH = 'patch'
|
|
34
|
+
POST = 'post'
|
|
35
|
+
PUT = 'put'
|
|
36
|
+
TRACE = 'trace'
|
|
37
|
+
|
|
38
|
+
# -- Getters -- #
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def allows_body(self) -> bool:
|
|
42
|
+
"""
|
|
43
|
+
Whether the method typically allows a request body.
|
|
44
|
+
|
|
45
|
+
Notes
|
|
46
|
+
-----
|
|
47
|
+
- RFCs do not strictly forbid bodies on some other methods (e.g.,
|
|
48
|
+
``DELETE``), but many servers/clients do not expect them. We mark
|
|
49
|
+
``POST``, ``PUT``, and ``PATCH`` as True.
|
|
50
|
+
"""
|
|
51
|
+
return self in {HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""
|
|
2
|
-
:mod:`etlplus.api.rate_limiting.
|
|
2
|
+
:mod:`etlplus.api.rate_limiting.config` module.
|
|
3
3
|
|
|
4
4
|
Rate limiting configuration primitives.
|
|
5
5
|
|
|
@@ -268,6 +268,18 @@ class RateLimitConfig(BoundsWarningsMixin):
|
|
|
268
268
|
) -> Self:
|
|
269
269
|
"""
|
|
270
270
|
Normalize rate-limit config and overrides into a single instance.
|
|
271
|
+
|
|
272
|
+
Parameters
|
|
273
|
+
----------
|
|
274
|
+
rate_limit : StrAnyMap | RateLimitConfig | None, optional
|
|
275
|
+
Base rate-limit configuration to normalize.
|
|
276
|
+
overrides : RateLimitOverrides, optional
|
|
277
|
+
Override values that take precedence over ``rate_limit``.
|
|
278
|
+
|
|
279
|
+
Returns
|
|
280
|
+
-------
|
|
281
|
+
Self
|
|
282
|
+
Normalized rate-limit configuration.
|
|
271
283
|
"""
|
|
272
284
|
normalized = _coerce_rate_limit_map(rate_limit)
|
|
273
285
|
cfg = _merge_rate_limit(normalized, overrides)
|
|
@@ -20,6 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
|
|
21
21
|
import time
|
|
22
22
|
from dataclasses import dataclass
|
|
23
|
+
from typing import Self
|
|
23
24
|
|
|
24
25
|
from ...utils import to_float
|
|
25
26
|
from ...utils import to_positive_float
|
|
@@ -143,13 +144,13 @@ class RateLimiter:
|
|
|
143
144
|
# -- Class Methods -- #
|
|
144
145
|
|
|
145
146
|
@classmethod
|
|
146
|
-
def disabled(cls) ->
|
|
147
|
+
def disabled(cls) -> Self:
|
|
147
148
|
"""
|
|
148
149
|
Create a limiter that never sleeps.
|
|
149
150
|
|
|
150
151
|
Returns
|
|
151
152
|
-------
|
|
152
|
-
|
|
153
|
+
Self
|
|
153
154
|
Instance with rate limiting disabled.
|
|
154
155
|
"""
|
|
155
156
|
return cls(sleep_seconds=0.0)
|
|
@@ -158,7 +159,7 @@ class RateLimiter:
|
|
|
158
159
|
def fixed(
|
|
159
160
|
cls,
|
|
160
161
|
seconds: float,
|
|
161
|
-
) ->
|
|
162
|
+
) -> Self:
|
|
162
163
|
"""
|
|
163
164
|
Create a limiter with a fixed non-negative delay.
|
|
164
165
|
|
|
@@ -170,7 +171,7 @@ class RateLimiter:
|
|
|
170
171
|
|
|
171
172
|
Returns
|
|
172
173
|
-------
|
|
173
|
-
|
|
174
|
+
Self
|
|
174
175
|
Instance with the specified delay.
|
|
175
176
|
"""
|
|
176
177
|
value = to_float(seconds, 0.0, minimum=0.0) or 0.0
|
|
@@ -181,7 +182,7 @@ class RateLimiter:
|
|
|
181
182
|
def from_config(
|
|
182
183
|
cls,
|
|
183
184
|
cfg: RateLimitInput,
|
|
184
|
-
) ->
|
|
185
|
+
) -> Self:
|
|
185
186
|
"""
|
|
186
187
|
Build a :class:`RateLimiter` from a configuration mapping.
|
|
187
188
|
|
|
@@ -201,12 +202,10 @@ class RateLimiter:
|
|
|
201
202
|
|
|
202
203
|
Returns
|
|
203
204
|
-------
|
|
204
|
-
|
|
205
|
+
Self
|
|
205
206
|
Instance with normalized ``sleep_seconds`` and ``max_per_sec``.
|
|
206
207
|
"""
|
|
207
208
|
config = RateLimitConfig.from_inputs(rate_limit=cfg)
|
|
208
|
-
if config is None:
|
|
209
|
-
return cls.disabled()
|
|
210
209
|
|
|
211
210
|
# RateLimiter.__post_init__ will normalize and enforce invariants.
|
|
212
211
|
return cls(**config.as_mapping())
|
|
@@ -261,6 +260,4 @@ class RateLimiter:
|
|
|
261
260
|
rate_limit=rate_limit,
|
|
262
261
|
overrides=overrides,
|
|
263
262
|
)
|
|
264
|
-
|
|
265
|
-
return 0.0
|
|
266
|
-
return float(config.sleep_seconds)
|
|
263
|
+
return float(config.sleep_seconds) if config.sleep_seconds else 0.0
|
|
@@ -14,6 +14,7 @@ from collections.abc import Sequence
|
|
|
14
14
|
from dataclasses import dataclass
|
|
15
15
|
from dataclasses import field
|
|
16
16
|
from functools import partial
|
|
17
|
+
from types import TracebackType
|
|
17
18
|
from typing import Any
|
|
18
19
|
from typing import cast
|
|
19
20
|
|
|
@@ -137,7 +138,7 @@ class RequestManager:
|
|
|
137
138
|
self,
|
|
138
139
|
exc_type: type[BaseException] | None,
|
|
139
140
|
exc: BaseException | None,
|
|
140
|
-
tb:
|
|
141
|
+
tb: TracebackType | None,
|
|
141
142
|
) -> None:
|
|
142
143
|
"""
|
|
143
144
|
Exit the runtime context and close owned sessions.
|
|
@@ -148,7 +149,7 @@ class RequestManager:
|
|
|
148
149
|
Exception type if raised, else ``None``.
|
|
149
150
|
exc : BaseException | None
|
|
150
151
|
Exception instance if raised, else ``None``.
|
|
151
|
-
tb :
|
|
152
|
+
tb : TracebackType | None
|
|
152
153
|
Traceback if an exception was raised, else ``None``.
|
|
153
154
|
"""
|
|
154
155
|
if self._ctx_session is None:
|
|
@@ -275,7 +276,7 @@ class RequestManager:
|
|
|
275
276
|
|
|
276
277
|
try:
|
|
277
278
|
policy = self.retry
|
|
278
|
-
if
|
|
279
|
+
if policy is None:
|
|
279
280
|
try:
|
|
280
281
|
return fetch(url, **call_kwargs)
|
|
281
282
|
except requests.RequestException as exc: # pragma: no cover
|
|
@@ -438,9 +439,13 @@ class RequestManager:
|
|
|
438
439
|
if isinstance(payload, dict):
|
|
439
440
|
return cast(JSONDict, payload)
|
|
440
441
|
if isinstance(payload, list):
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
442
|
+
out: list[JSONDict] = []
|
|
443
|
+
for item in payload:
|
|
444
|
+
if isinstance(item, dict):
|
|
445
|
+
out.append(cast(JSONDict, item))
|
|
446
|
+
else:
|
|
447
|
+
out.append({'value': item})
|
|
448
|
+
return cast(JSONData, out)
|
|
444
449
|
return {'value': payload}
|
|
445
450
|
return {
|
|
446
451
|
'content': response.text,
|