etlplus 0.15.5__tar.gz → 0.16.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {etlplus-0.15.5 → etlplus-0.16.5}/CONTRIBUTING.md +1 -1
- {etlplus-0.15.5/etlplus.egg-info → etlplus-0.16.5}/PKG-INFO +1 -1
- {etlplus-0.15.5 → etlplus-0.16.5}/docs/pipeline-guide.md +4 -4
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/README.md +22 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/__init__.py +2 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/__init__.py +14 -14
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/auth.py +9 -6
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/config.py +6 -6
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/endpoint_client.py +16 -16
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/errors.py +4 -4
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/pagination/__init__.py +6 -6
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/pagination/config.py +11 -9
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/rate_limiting/__init__.py +2 -2
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/rate_limiting/config.py +10 -10
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/rate_limiting/rate_limiter.py +2 -2
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/request_manager.py +4 -4
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/retry_manager.py +6 -6
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/transport.py +10 -10
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/types.py +47 -26
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/utils.py +49 -49
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/commands.py +22 -22
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/constants.py +1 -1
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/handlers.py +12 -13
- etlplus-0.15.5/etlplus/workflow/pipeline.py → etlplus-0.16.5/etlplus/config.py +52 -89
- etlplus-0.16.5/etlplus/connector/__init__.py +43 -0
- etlplus-0.16.5/etlplus/connector/api.py +161 -0
- etlplus-0.16.5/etlplus/connector/connector.py +26 -0
- etlplus-0.16.5/etlplus/connector/core.py +132 -0
- etlplus-0.16.5/etlplus/connector/database.py +122 -0
- etlplus-0.16.5/etlplus/connector/enums.py +52 -0
- etlplus-0.16.5/etlplus/connector/file.py +120 -0
- etlplus-0.16.5/etlplus/connector/types.py +40 -0
- etlplus-0.16.5/etlplus/connector/utils.py +122 -0
- etlplus-0.16.5/etlplus/enums.py +144 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/ops/__init__.py +1 -0
- {etlplus-0.15.5/etlplus → etlplus-0.16.5/etlplus/ops}/enums.py +5 -140
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/ops/extract.py +210 -23
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/ops/load.py +141 -35
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/ops/run.py +89 -104
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/ops/transform.py +46 -27
- etlplus-0.16.5/etlplus/ops/types.py +147 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/ops/utils.py +5 -5
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/ops/validate.py +13 -13
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/types.py +5 -102
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/workflow/README.md +0 -24
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/workflow/__init__.py +2 -15
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/workflow/dag.py +23 -1
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/workflow/jobs.py +15 -28
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/workflow/profile.py +4 -2
- {etlplus-0.15.5 → etlplus-0.16.5/etlplus.egg-info}/PKG-INFO +1 -1
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus.egg-info/SOURCES.txt +19 -9
- {etlplus-0.15.5 → etlplus-0.16.5}/examples/README.md +1 -1
- etlplus-0.15.5/examples/quickstart_python.py → etlplus-0.16.5/examples/quickstart.py +25 -4
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/integration/conftest.py +35 -38
- etlplus-0.15.5/tests/integration/test_i_pipeline_yaml_load.py → etlplus-0.16.5/tests/integration/test_i_config_load.py +9 -9
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/integration/test_i_pagination_strategy.py +2 -2
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/integration/test_i_run.py +3 -3
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/integration/test_i_run_profile_pagination_defaults.py +3 -3
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/integration/test_i_run_profile_rate_limit_defaults.py +2 -2
- etlplus-0.15.5/tests/unit/api/test_u_config.py → etlplus-0.16.5/tests/unit/api/test_u_api_config.py +1 -1
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_api_utils.py +5 -5
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_endpoint_client.py +23 -20
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_paginator.py +8 -8
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_rate_limiter.py +6 -6
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_retry_manager.py +2 -2
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/cli/conftest.py +3 -3
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/cli/test_u_cli_handlers.py +21 -21
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/conftest.py +37 -32
- etlplus-0.16.5/tests/unit/connector/test_u_connector_enums.py +47 -0
- etlplus-0.15.5/tests/unit/workflow/test_u_workflow_connector.py → etlplus-0.16.5/tests/unit/connector/test_u_connector_utils.py +6 -6
- etlplus-0.15.5/tests/unit/test_u_enums.py → etlplus-0.16.5/tests/unit/ops/test_u_ops_enums.py +8 -35
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/ops/test_u_ops_load.py +1 -6
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/ops/test_u_ops_run.py +40 -37
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/ops/test_u_ops_transform.py +3 -3
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/ops/test_u_ops_utils.py +13 -13
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/ops/test_u_ops_validate.py +5 -5
- etlplus-0.15.5/tests/unit/workflow/test_u_workflow_pipeline.py → etlplus-0.16.5/tests/unit/test_u_config.py +29 -30
- etlplus-0.15.5/etlplus/workflow/connector.py +0 -386
- etlplus-0.15.5/etlplus/workflow/types.py +0 -115
- {etlplus-0.15.5 → etlplus-0.16.5}/.coveragerc +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/.editorconfig +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/.gitattributes +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/.github/actions/python-bootstrap/action.yml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/.github/workflows/ci.yml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/.gitignore +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/.pre-commit-config.yaml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/.ruff.toml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/CODE_OF_CONDUCT.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/DEMO.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/LICENSE +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/MANIFEST.in +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/Makefile +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/README.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/REFERENCES.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/SECURITY.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/SUPPORT.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/docs/README.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/docs/snippets/installation_version.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/__main__.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/__version__.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/README.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/enums.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/pagination/client.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/api/pagination/paginator.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/README.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/__init__.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/io.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/main.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/options.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/state.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/cli/types.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/database/README.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/database/__init__.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/database/ddl.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/database/engine.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/database/orm.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/database/schema.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/database/types.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/README.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/__init__.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/_imports.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/_io.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/accdb.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/arrow.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/avro.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/bson.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/cbor.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/cfg.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/conf.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/core.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/csv.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/dat.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/dta.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/duckdb.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/enums.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/feather.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/fwf.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/gz.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/hbs.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/hdf5.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/ini.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/ion.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/jinja2.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/json.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/log.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/mat.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/mdb.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/msgpack.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/mustache.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/nc.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/ndjson.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/numbers.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/ods.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/orc.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/parquet.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/pb.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/pbf.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/properties.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/proto.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/psv.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/rda.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/rds.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/sas7bdat.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/sav.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/sqlite.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/stub.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/sylk.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/tab.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/toml.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/tsv.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/txt.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/vm.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/wks.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/xls.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/xlsm.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/xlsx.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/xml.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/xpt.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/yaml.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/zip.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/file/zsav.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/mixins.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/ops/README.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/py.typed +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/templates/README.md +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/templates/__init__.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/templates/ddl.sql.j2 +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/templates/view.sql.j2 +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus/utils.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus.egg-info/dependency_links.txt +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus.egg-info/entry_points.txt +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus.egg-info/requires.txt +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/etlplus.egg-info/top_level.txt +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/examples/configs/ddl_spec.yml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/examples/configs/pipeline.yml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/examples/data/sample.csv +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/examples/data/sample.json +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/examples/data/sample.xml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/examples/data/sample.xsd +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/examples/data/sample.yaml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/pyproject.toml +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/pytest.ini +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/setup.cfg +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/setup.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/__init__.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/conftest.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/integration/test_i_cli.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/integration/test_i_examples_data_parity.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/integration/test_i_pipeline_smoke.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/conftest.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_api_enums.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_auth.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_mocks.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_pagination_client.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_pagination_config.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_rate_limit_config.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_request_manager.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_transport.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/api/test_u_types.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/cli/test_u_cli_io.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/cli/test_u_cli_main.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/cli/test_u_cli_state.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/database/test_u_database_ddl.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/database/test_u_database_engine.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/database/test_u_database_orm.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/database/test_u_database_schema.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/file/test_u_file_core.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/file/test_u_file_enums.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/file/test_u_file_yaml.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/ops/test_u_ops_extract.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/test_u_main.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/test_u_mixins.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/test_u_utils.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/test_u_version.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tests/unit/workflow/test_u_workflow_jobs.py +0 -0
- {etlplus-0.15.5 → etlplus-0.16.5}/tools/update_demo_snippets.py +0 -0
|
@@ -133,5 +133,5 @@ If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()`, it is integrati
|
|
|
133
133
|
- CLI tests: monkeypatch `sys.argv` and call `etlplus.cli.main()`; capture output with `capsys`.
|
|
134
134
|
- File I/O: use `tmp_path` / `TemporaryDirectory()`; never write to the repo tree.
|
|
135
135
|
- API flows: stub `EndpointClient` or transport layer via `monkeypatch` to avoid real HTTP.
|
|
136
|
-
- Runner tests: monkeypatch `
|
|
136
|
+
- Runner tests: monkeypatch `load_config` to inject an in-memory `Config`.
|
|
137
137
|
- Keep tests small and focused; prefer one behavior per test with clear assertions.
|
|
@@ -377,9 +377,9 @@ Details:
|
|
|
377
377
|
- Unknown or malformed entries are skipped rather than failing the whole load (keeping pipeline
|
|
378
378
|
authoring permissive).
|
|
379
379
|
- The connector kind is also available as a type-safe literal in code as
|
|
380
|
-
`etlplus.
|
|
380
|
+
`etlplus.connector.ConnectorType` (values: `"file" | "database" | "api"`).
|
|
381
381
|
|
|
382
|
-
To add new connector kinds in the future, implement a new dataclass in `etlplus.
|
|
382
|
+
To add new connector kinds in the future, implement a new dataclass in `etlplus.connector`
|
|
383
383
|
and extend the internal parser to handle its `type` value.
|
|
384
384
|
|
|
385
385
|
## Jobs
|
|
@@ -497,8 +497,8 @@ For the HTTP client and pagination API, see `etlplus/api/README.md`.
|
|
|
497
497
|
|
|
498
498
|
## Design notes: Mapping inputs, dict outputs
|
|
499
499
|
|
|
500
|
-
ETLPlus config constructors (e.g., `ApiConfig.from_obj`, `
|
|
501
|
-
|
|
500
|
+
ETLPlus config constructors (e.g., `ApiConfig.from_obj`, `Config.from_dict`) accept `Mapping[str,
|
|
501
|
+
Any]` rather than `dict[str, Any]` for inputs. Why?
|
|
502
502
|
|
|
503
503
|
- Flexibility: callers can pass any mapping-like object (e.g., YAML loaders that return custom
|
|
504
504
|
mappings) without copying into a `dict` first.
|
|
@@ -31,6 +31,28 @@ assert validate(filtered, {"age": {"type": "number", "min": 0}})["valid"]
|
|
|
31
31
|
load(filtered, "file", "output.json", file_format="json")
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
+
## Loading and Validating Configs
|
|
35
|
+
|
|
36
|
+
Use the provided classes to load and validate configuration files:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from etlplus.workflow import Config
|
|
40
|
+
|
|
41
|
+
cfg = Config.from_yaml("pipeline.yml")
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
- Supports YAML and JSON formats
|
|
45
|
+
- Validates against expected schema
|
|
46
|
+
|
|
47
|
+
## Example: Loading a Pipeline Config
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from etlplus.workflow import Config
|
|
51
|
+
|
|
52
|
+
config = Config.from_yaml("configs/pipeline.yml")
|
|
53
|
+
print(config)
|
|
54
|
+
```
|
|
55
|
+
|
|
34
56
|
## See Also
|
|
35
57
|
|
|
36
58
|
- [Top-level project README](../README.md)
|
|
@@ -5,6 +5,7 @@ Top-level facade for the ETLPlus toolkit.
|
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
7
|
from .__version__ import __version__
|
|
8
|
+
from .config import Config
|
|
8
9
|
|
|
9
10
|
__author__ = 'ETLPlus Team'
|
|
10
11
|
|
|
@@ -15,4 +16,5 @@ __author__ = 'ETLPlus Team'
|
|
|
15
16
|
__all__ = [
|
|
16
17
|
'__author__',
|
|
17
18
|
'__version__',
|
|
19
|
+
'Config',
|
|
18
20
|
]
|
|
@@ -79,21 +79,21 @@ from .config import ApiProfileConfig
|
|
|
79
79
|
from .config import EndpointConfig
|
|
80
80
|
from .endpoint_client import EndpointClient
|
|
81
81
|
from .enums import HttpMethod
|
|
82
|
-
from .pagination import
|
|
83
|
-
from .pagination import
|
|
82
|
+
from .pagination import CursorPaginationConfigDict
|
|
83
|
+
from .pagination import PagePaginationConfigDict
|
|
84
84
|
from .pagination import PaginationClient
|
|
85
85
|
from .pagination import PaginationConfig
|
|
86
|
-
from .pagination import
|
|
86
|
+
from .pagination import PaginationConfigDict
|
|
87
87
|
from .pagination import PaginationType
|
|
88
88
|
from .pagination import Paginator
|
|
89
89
|
from .rate_limiting import RateLimitConfig
|
|
90
|
-
from .rate_limiting import
|
|
90
|
+
from .rate_limiting import RateLimitConfigDict
|
|
91
91
|
from .rate_limiting import RateLimiter
|
|
92
92
|
from .retry_manager import RetryManager
|
|
93
|
-
from .retry_manager import
|
|
93
|
+
from .retry_manager import RetryPolicyDict
|
|
94
94
|
from .retry_manager import RetryStrategy
|
|
95
|
-
from .transport import
|
|
96
|
-
from .transport import
|
|
95
|
+
from .transport import HTTPAdapterMountConfigDict
|
|
96
|
+
from .transport import HTTPAdapterRetryConfigDict
|
|
97
97
|
from .transport import build_http_adapter
|
|
98
98
|
from .types import Headers
|
|
99
99
|
from .types import Params
|
|
@@ -133,14 +133,14 @@ __all__ = [
|
|
|
133
133
|
'paginate_with_client',
|
|
134
134
|
'resolve_request',
|
|
135
135
|
# Type Aliases
|
|
136
|
-
'
|
|
136
|
+
'CursorPaginationConfigDict',
|
|
137
137
|
'Headers',
|
|
138
|
-
'
|
|
139
|
-
'
|
|
140
|
-
'
|
|
141
|
-
'
|
|
138
|
+
'HTTPAdapterMountConfigDict',
|
|
139
|
+
'HTTPAdapterRetryConfigDict',
|
|
140
|
+
'PagePaginationConfigDict',
|
|
141
|
+
'PaginationConfigDict',
|
|
142
142
|
'Params',
|
|
143
|
-
'
|
|
144
|
-
'
|
|
143
|
+
'RateLimitConfigDict',
|
|
144
|
+
'RetryPolicyDict',
|
|
145
145
|
'Url',
|
|
146
146
|
]
|
|
@@ -76,13 +76,16 @@ FORM_HEADERS = MappingProxyType(
|
|
|
76
76
|
# SECTION: TYPED DICTS ====================================================== #
|
|
77
77
|
|
|
78
78
|
|
|
79
|
-
class
|
|
79
|
+
class _TokenResponseDict(TypedDict):
|
|
80
80
|
"""Minimal shape of an OAuth token response body."""
|
|
81
81
|
|
|
82
82
|
access_token: str
|
|
83
83
|
expires_in: int | float
|
|
84
84
|
|
|
85
85
|
|
|
86
|
+
# SECTION: PROTOCOLS ======================================================== #
|
|
87
|
+
|
|
88
|
+
|
|
86
89
|
class _TokenHttpClient(Protocol):
|
|
87
90
|
"""Protocol for objects that expose a ``post`` helper like ``requests``."""
|
|
88
91
|
|
|
@@ -284,7 +287,7 @@ class EndpointCredentialsBearer(AuthBase):
|
|
|
284
287
|
def _parse_token_response(
|
|
285
288
|
self,
|
|
286
289
|
resp: Response,
|
|
287
|
-
) ->
|
|
290
|
+
) -> _TokenResponseDict:
|
|
288
291
|
"""
|
|
289
292
|
Validate the JSON token response and return a typed mapping.
|
|
290
293
|
|
|
@@ -295,7 +298,7 @@ class EndpointCredentialsBearer(AuthBase):
|
|
|
295
298
|
|
|
296
299
|
Returns
|
|
297
300
|
-------
|
|
298
|
-
|
|
301
|
+
_TokenResponseDict
|
|
299
302
|
Parsed token response mapping.
|
|
300
303
|
|
|
301
304
|
Raises
|
|
@@ -335,15 +338,15 @@ class EndpointCredentialsBearer(AuthBase):
|
|
|
335
338
|
except (TypeError, ValueError):
|
|
336
339
|
ttl = float(DEFAULT_TOKEN_TTL)
|
|
337
340
|
|
|
338
|
-
return
|
|
341
|
+
return _TokenResponseDict(access_token=token, expires_in=ttl)
|
|
339
342
|
|
|
340
|
-
def _request_token(self) ->
|
|
343
|
+
def _request_token(self) -> _TokenResponseDict:
|
|
341
344
|
"""
|
|
342
345
|
Execute the OAuth2 token request and parse the response.
|
|
343
346
|
|
|
344
347
|
Returns
|
|
345
348
|
-------
|
|
346
|
-
|
|
349
|
+
_TokenResponseDict
|
|
347
350
|
Parsed token response mapping.
|
|
348
351
|
|
|
349
352
|
Raises
|
|
@@ -36,9 +36,9 @@ from .pagination import PaginationConfig
|
|
|
36
36
|
from .rate_limiting import RateLimitConfig
|
|
37
37
|
|
|
38
38
|
if TYPE_CHECKING:
|
|
39
|
-
from .types import
|
|
40
|
-
from .types import
|
|
41
|
-
from .types import
|
|
39
|
+
from .types import ApiConfigDict
|
|
40
|
+
from .types import ApiProfileConfigDict
|
|
41
|
+
from .types import EndpointConfigDict
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
# SECTION: EXPORTS ========================================================== #
|
|
@@ -264,7 +264,7 @@ class ApiProfileConfig:
|
|
|
264
264
|
@overload
|
|
265
265
|
def from_obj(
|
|
266
266
|
cls,
|
|
267
|
-
obj:
|
|
267
|
+
obj: ApiProfileConfigDict,
|
|
268
268
|
) -> Self: ...
|
|
269
269
|
|
|
270
270
|
@classmethod
|
|
@@ -460,7 +460,7 @@ class ApiConfig:
|
|
|
460
460
|
@overload
|
|
461
461
|
def from_obj(
|
|
462
462
|
cls,
|
|
463
|
-
obj:
|
|
463
|
+
obj: ApiConfigDict,
|
|
464
464
|
) -> Self: ...
|
|
465
465
|
|
|
466
466
|
@classmethod
|
|
@@ -577,7 +577,7 @@ class EndpointConfig:
|
|
|
577
577
|
@overload
|
|
578
578
|
def from_obj(
|
|
579
579
|
cls,
|
|
580
|
-
obj:
|
|
580
|
+
obj: EndpointConfigDict,
|
|
581
581
|
) -> Self: ...
|
|
582
582
|
|
|
583
583
|
@classmethod
|
|
@@ -11,8 +11,8 @@ Notes
|
|
|
11
11
|
-----
|
|
12
12
|
- Retry-related types live in :mod:`etlplus.api.retry_manager`.
|
|
13
13
|
- Pagination requires a ``PaginationConfig``; see
|
|
14
|
-
:class:`
|
|
15
|
-
the accepted shapes.
|
|
14
|
+
:class:`PagePaginationConfigDict` and :class:`CursorPaginationConfigDict`
|
|
15
|
+
for the accepted shapes.
|
|
16
16
|
|
|
17
17
|
Examples
|
|
18
18
|
--------
|
|
@@ -65,14 +65,14 @@ from .errors import PaginationError
|
|
|
65
65
|
from .pagination import PaginationClient
|
|
66
66
|
from .pagination import PaginationInput
|
|
67
67
|
from .pagination import Paginator
|
|
68
|
-
from .rate_limiting import
|
|
68
|
+
from .rate_limiting import RateLimitConfigDict
|
|
69
69
|
from .rate_limiting import RateLimiter
|
|
70
70
|
from .rate_limiting import RateLimitOverrides
|
|
71
71
|
from .request_manager import RequestManager
|
|
72
72
|
from .retry_manager import RetryManager
|
|
73
|
-
from .retry_manager import
|
|
73
|
+
from .retry_manager import RetryPolicyDict
|
|
74
74
|
from .retry_manager import RetryStrategy
|
|
75
|
-
from .transport import
|
|
75
|
+
from .transport import HTTPAdapterMountConfigDict
|
|
76
76
|
from .types import RequestOptions
|
|
77
77
|
from .types import Url
|
|
78
78
|
|
|
@@ -101,13 +101,13 @@ class EndpointClient:
|
|
|
101
101
|
base_path : str | None, optional
|
|
102
102
|
Optional base path prefix (``/v2``) prepended to all endpoint
|
|
103
103
|
paths when building URLs.
|
|
104
|
-
retry :
|
|
104
|
+
retry : RetryPolicyDict | None, optional
|
|
105
105
|
Optional retry policy. When provided, failed requests matching
|
|
106
106
|
``retry_on`` statuses are retried with full jitter.
|
|
107
107
|
retry_network_errors : bool, optional
|
|
108
108
|
When ``True``, also retry on network errors (timeouts, connection
|
|
109
109
|
resets). Defaults to ``False``.
|
|
110
|
-
rate_limit :
|
|
110
|
+
rate_limit : RateLimitConfigDict | None, optional
|
|
111
111
|
Optional client-wide rate limit used to derive an inter-request
|
|
112
112
|
delay when an explicit ``sleep_seconds`` isn't supplied.
|
|
113
113
|
session : requests.Session | None, optional
|
|
@@ -115,7 +115,7 @@ class EndpointClient:
|
|
|
115
115
|
session_factory : Callable[[], requests.Session] | None, optional
|
|
116
116
|
Factory used to lazily create a session. Ignored if ``session`` is
|
|
117
117
|
provided.
|
|
118
|
-
session_adapters : Sequence[
|
|
118
|
+
session_adapters : Sequence[HTTPAdapterMountConfigDict] | None, optional
|
|
119
119
|
Adapter mount configuration(s) used to build a session lazily when
|
|
120
120
|
neither ``session`` nor ``session_factory`` is supplied.
|
|
121
121
|
|
|
@@ -128,17 +128,17 @@ class EndpointClient:
|
|
|
128
128
|
(``MappingProxyType``).
|
|
129
129
|
base_path : str | None
|
|
130
130
|
Optional base path prefix appended after ``base_url``.
|
|
131
|
-
retry :
|
|
131
|
+
retry : RetryPolicyDict | None
|
|
132
132
|
Retry policy reference (may be ``None``).
|
|
133
133
|
retry_network_errors : bool
|
|
134
134
|
Whether network errors are retried in addition to HTTP statuses.
|
|
135
|
-
rate_limit :
|
|
135
|
+
rate_limit : RateLimitConfigDict | None
|
|
136
136
|
Client-wide rate limit configuration (may be ``None``).
|
|
137
137
|
session : requests.Session | None
|
|
138
138
|
Explicit HTTP session used for requests when provided.
|
|
139
139
|
session_factory : Callable[[], requests.Session] | None
|
|
140
140
|
Lazily invoked factory producing a session when needed.
|
|
141
|
-
session_adapters : Sequence[
|
|
141
|
+
session_adapters : Sequence[HTTPAdapterMountConfigDict] | None
|
|
142
142
|
Adapter mount configuration(s) for connection pooling / retries.
|
|
143
143
|
DEFAULT_PAGE_PARAM : ClassVar[str]
|
|
144
144
|
Default page parameter name.
|
|
@@ -212,10 +212,10 @@ class EndpointClient:
|
|
|
212
212
|
base_path: str | None = None
|
|
213
213
|
|
|
214
214
|
# Optional retry configuration (constructor parameter; object is frozen)
|
|
215
|
-
retry:
|
|
215
|
+
retry: RetryPolicyDict | None = None
|
|
216
216
|
retry_network_errors: bool = False
|
|
217
217
|
# Optional client-wide rate limit configuration
|
|
218
|
-
rate_limit:
|
|
218
|
+
rate_limit: RateLimitConfigDict | None = None
|
|
219
219
|
|
|
220
220
|
# Optional HTTP session or factory
|
|
221
221
|
session: requests.Session | None = None
|
|
@@ -225,7 +225,7 @@ class EndpointClient:
|
|
|
225
225
|
# and connection pooling. If provided and neither `session` nor
|
|
226
226
|
# `session_factory` is supplied, a factory is synthesized to create a
|
|
227
227
|
# Session and mount the configured adapters lazily.
|
|
228
|
-
session_adapters: Sequence[
|
|
228
|
+
session_adapters: Sequence[HTTPAdapterMountConfigDict] | None = None
|
|
229
229
|
|
|
230
230
|
# Internal: context-managed session and ownership flag.
|
|
231
231
|
_request_manager: RequestManager = field(
|
|
@@ -857,7 +857,7 @@ class EndpointClient:
|
|
|
857
857
|
@staticmethod
|
|
858
858
|
def _resolve_sleep_seconds(
|
|
859
859
|
explicit: float,
|
|
860
|
-
rate_limit:
|
|
860
|
+
rate_limit: RateLimitConfigDict | None,
|
|
861
861
|
overrides: RateLimitOverrides = None,
|
|
862
862
|
) -> float:
|
|
863
863
|
"""
|
|
@@ -867,7 +867,7 @@ class EndpointClient:
|
|
|
867
867
|
----------
|
|
868
868
|
explicit : float
|
|
869
869
|
Explicit sleep seconds provided by the caller.
|
|
870
|
-
rate_limit :
|
|
870
|
+
rate_limit : RateLimitConfigDict | None
|
|
871
871
|
Client-wide rate limit configuration.
|
|
872
872
|
overrides : RateLimitOverrides, optional
|
|
873
873
|
Per-call overrides that take precedence over *rate_limit*.
|
|
@@ -30,7 +30,7 @@ from typing import Any
|
|
|
30
30
|
import requests # type: ignore[import]
|
|
31
31
|
|
|
32
32
|
if TYPE_CHECKING: # pragma: no cover - typing only
|
|
33
|
-
from .retry_manager import
|
|
33
|
+
from .retry_manager import RetryPolicyDict
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
# SECTION: EXPORTS ========================================================== #
|
|
@@ -57,7 +57,7 @@ class ApiRequestError(requests.RequestException):
|
|
|
57
57
|
Number of attempts performed (defaults to ``1``).
|
|
58
58
|
retried : bool, optional
|
|
59
59
|
Whether any retry attempts were made.
|
|
60
|
-
retry_policy :
|
|
60
|
+
retry_policy : RetryPolicyDict | None, optional
|
|
61
61
|
The retry policy in effect, if any.
|
|
62
62
|
cause : Exception | None, optional
|
|
63
63
|
Original underlying exception.
|
|
@@ -72,7 +72,7 @@ class ApiRequestError(requests.RequestException):
|
|
|
72
72
|
Number of attempts performed.
|
|
73
73
|
retried : bool
|
|
74
74
|
Whether any retry attempts were made.
|
|
75
|
-
retry_policy :
|
|
75
|
+
retry_policy : RetryPolicyDict | None
|
|
76
76
|
The retry policy in effect, if any.
|
|
77
77
|
cause : Exception | None
|
|
78
78
|
Original underlying exception.
|
|
@@ -97,7 +97,7 @@ class ApiRequestError(requests.RequestException):
|
|
|
97
97
|
status: int | None = None
|
|
98
98
|
attempts: int = 1
|
|
99
99
|
retried: bool = False
|
|
100
|
-
retry_policy:
|
|
100
|
+
retry_policy: RetryPolicyDict | None = None
|
|
101
101
|
cause: Exception | None = None
|
|
102
102
|
|
|
103
103
|
# -- Magic Methods (Object Representation) -- #
|
|
@@ -20,10 +20,10 @@ Notes
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
22
|
from .client import PaginationClient
|
|
23
|
-
from .config import
|
|
24
|
-
from .config import
|
|
23
|
+
from .config import CursorPaginationConfigDict
|
|
24
|
+
from .config import PagePaginationConfigDict
|
|
25
25
|
from .config import PaginationConfig
|
|
26
|
-
from .config import
|
|
26
|
+
from .config import PaginationConfigDict
|
|
27
27
|
from .config import PaginationInput
|
|
28
28
|
from .config import PaginationType
|
|
29
29
|
from .paginator import Paginator
|
|
@@ -40,8 +40,8 @@ __all__ = [
|
|
|
40
40
|
# Enums
|
|
41
41
|
'PaginationType',
|
|
42
42
|
# Type Aliases
|
|
43
|
-
'
|
|
44
|
-
'
|
|
43
|
+
'CursorPaginationConfigDict',
|
|
44
|
+
'PagePaginationConfigDict',
|
|
45
45
|
'PaginationInput',
|
|
46
|
-
'
|
|
46
|
+
'PaginationConfigDict',
|
|
47
47
|
]
|
|
@@ -8,7 +8,7 @@ by :mod:`etlplus.api.pagination`. It exposes:
|
|
|
8
8
|
|
|
9
9
|
- :class:`PaginationType` – enumeration of supported pagination modes.
|
|
10
10
|
- :class:`PaginationConfig` – normalized configuration container.
|
|
11
|
-
- ``*
|
|
11
|
+
- ``*PaginationConfigDict`` TypedDicts – loose, user-facing config mappings.
|
|
12
12
|
|
|
13
13
|
Notes
|
|
14
14
|
-----
|
|
@@ -44,11 +44,11 @@ __all__ = [
|
|
|
44
44
|
# Enums
|
|
45
45
|
'PaginationType',
|
|
46
46
|
# Type Aliases
|
|
47
|
-
'
|
|
47
|
+
'PaginationConfigDict',
|
|
48
48
|
'PaginationInput',
|
|
49
49
|
# Typed Dicts
|
|
50
|
-
'
|
|
51
|
-
'
|
|
50
|
+
'CursorPaginationConfigDict',
|
|
51
|
+
'PagePaginationConfigDict',
|
|
52
52
|
]
|
|
53
53
|
|
|
54
54
|
|
|
@@ -74,7 +74,7 @@ class PaginationType(CoercibleStrEnum):
|
|
|
74
74
|
# SECTION: TYPED DICTS ====================================================== #
|
|
75
75
|
|
|
76
76
|
|
|
77
|
-
class
|
|
77
|
+
class CursorPaginationConfigDict(TypedDict, total=False):
|
|
78
78
|
"""
|
|
79
79
|
Configuration mapping for cursor-based REST API response pagination.
|
|
80
80
|
|
|
@@ -131,7 +131,7 @@ class CursorPaginationConfigMap(TypedDict, total=False):
|
|
|
131
131
|
limit_param: str
|
|
132
132
|
|
|
133
133
|
|
|
134
|
-
class
|
|
134
|
+
class PagePaginationConfigDict(TypedDict, total=False):
|
|
135
135
|
"""
|
|
136
136
|
Configuration mapping for page-based and offset-based REST API response
|
|
137
137
|
pagination.
|
|
@@ -379,7 +379,7 @@ class PaginationConfig(BoundsWarningsMixin):
|
|
|
379
379
|
@overload
|
|
380
380
|
def from_obj(
|
|
381
381
|
cls,
|
|
382
|
-
obj:
|
|
382
|
+
obj: PaginationConfigDict,
|
|
383
383
|
) -> Self: ...
|
|
384
384
|
|
|
385
385
|
@classmethod
|
|
@@ -429,12 +429,14 @@ class PaginationConfig(BoundsWarningsMixin):
|
|
|
429
429
|
# SECTION: TYPE ALIASES ===================================================== #
|
|
430
430
|
|
|
431
431
|
|
|
432
|
-
type
|
|
432
|
+
type PaginationConfigDict = (
|
|
433
|
+
PagePaginationConfigDict | CursorPaginationConfigDict
|
|
434
|
+
)
|
|
433
435
|
|
|
434
436
|
# External callers may pass either a raw mapping-shaped config or an already
|
|
435
437
|
# constructed PaginationConfig instance, or omit pagination entirely. Accept a
|
|
436
438
|
# loose mapping here to reflect the runtime behavior while still providing
|
|
437
439
|
# stronger TypedDict hints for common shapes.
|
|
438
440
|
type PaginationInput = (
|
|
439
|
-
|
|
441
|
+
PaginationConfigDict | PaginationConfig | StrAnyMap | None
|
|
440
442
|
)
|
|
@@ -19,7 +19,7 @@ Notes
|
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
21
|
from .config import RateLimitConfig
|
|
22
|
-
from .config import
|
|
22
|
+
from .config import RateLimitConfigDict
|
|
23
23
|
from .config import RateLimitOverrides
|
|
24
24
|
from .rate_limiter import RateLimiter
|
|
25
25
|
|
|
@@ -34,5 +34,5 @@ __all__ = [
|
|
|
34
34
|
# Type Aliases
|
|
35
35
|
'RateLimitOverrides',
|
|
36
36
|
# Type Dicts
|
|
37
|
-
'
|
|
37
|
+
'RateLimitConfigDict',
|
|
38
38
|
]
|
|
@@ -41,7 +41,7 @@ __all__ = [
|
|
|
41
41
|
# Type Aliases
|
|
42
42
|
'RateLimitOverrides',
|
|
43
43
|
# Typed Dicts
|
|
44
|
-
'
|
|
44
|
+
'RateLimitConfigDict',
|
|
45
45
|
]
|
|
46
46
|
|
|
47
47
|
|
|
@@ -50,9 +50,9 @@ __all__ = [
|
|
|
50
50
|
|
|
51
51
|
def _coerce_rate_limit_map(
|
|
52
52
|
rate_limit: StrAnyMap | RateLimitConfig | None,
|
|
53
|
-
) ->
|
|
53
|
+
) -> RateLimitConfigDict | None:
|
|
54
54
|
"""
|
|
55
|
-
Normalize user inputs into a :class:`
|
|
55
|
+
Normalize user inputs into a :class:`RateLimitConfigDict`.
|
|
56
56
|
|
|
57
57
|
This helper is the single entry point for converting loosely-typed
|
|
58
58
|
configuration into the canonical mapping consumed by downstream
|
|
@@ -65,7 +65,7 @@ def _coerce_rate_limit_map(
|
|
|
65
65
|
|
|
66
66
|
Returns
|
|
67
67
|
-------
|
|
68
|
-
|
|
68
|
+
RateLimitConfigDict | None
|
|
69
69
|
Normalized mapping, or ``None`` if input couldn't be parsed.
|
|
70
70
|
"""
|
|
71
71
|
if rate_limit is None:
|
|
@@ -133,7 +133,7 @@ def _normalized_rate_values(
|
|
|
133
133
|
# SECTION: TYPED DICTS ====================================================== #
|
|
134
134
|
|
|
135
135
|
|
|
136
|
-
class
|
|
136
|
+
class RateLimitConfigDict(TypedDict, total=False):
|
|
137
137
|
"""
|
|
138
138
|
Configuration mapping for HTTP request rate limits.
|
|
139
139
|
|
|
@@ -149,7 +149,7 @@ class RateLimitConfigMap(TypedDict, total=False):
|
|
|
149
149
|
|
|
150
150
|
Examples
|
|
151
151
|
--------
|
|
152
|
-
>>> rl:
|
|
152
|
+
>>> rl: RateLimitConfigDict = {'max_per_sec': 4}
|
|
153
153
|
... # sleep ~= 0.25s between calls
|
|
154
154
|
"""
|
|
155
155
|
|
|
@@ -197,9 +197,9 @@ class RateLimitConfig(BoundsWarningsMixin):
|
|
|
197
197
|
|
|
198
198
|
# -- Instance Methods -- #
|
|
199
199
|
|
|
200
|
-
def as_mapping(self) ->
|
|
200
|
+
def as_mapping(self) -> RateLimitConfigDict:
|
|
201
201
|
"""Return a normalized mapping consumable by rate-limit helpers."""
|
|
202
|
-
cfg:
|
|
202
|
+
cfg: RateLimitConfigDict = {}
|
|
203
203
|
if (sleep := to_float(self.sleep_seconds)) is not None:
|
|
204
204
|
cfg['sleep_seconds'] = sleep
|
|
205
205
|
if (rate := to_float(self.max_per_sec)) is not None:
|
|
@@ -309,7 +309,7 @@ class RateLimitConfig(BoundsWarningsMixin):
|
|
|
309
309
|
@overload
|
|
310
310
|
def from_obj(
|
|
311
311
|
cls,
|
|
312
|
-
obj:
|
|
312
|
+
obj: RateLimitConfigDict,
|
|
313
313
|
) -> Self: ...
|
|
314
314
|
|
|
315
315
|
@classmethod
|
|
@@ -352,4 +352,4 @@ class RateLimitConfig(BoundsWarningsMixin):
|
|
|
352
352
|
type RateLimitInput = StrAnyMap | RateLimitConfig | None
|
|
353
353
|
|
|
354
354
|
# Optional mapping of rate-limit fields to override values.
|
|
355
|
-
type RateLimitOverrides =
|
|
355
|
+
type RateLimitOverrides = RateLimitConfigDict | None
|
|
@@ -25,7 +25,7 @@ from typing import Self
|
|
|
25
25
|
from ...utils import to_float
|
|
26
26
|
from ...utils import to_positive_float
|
|
27
27
|
from .config import RateLimitConfig
|
|
28
|
-
from .config import
|
|
28
|
+
from .config import RateLimitConfigDict
|
|
29
29
|
from .config import RateLimitInput
|
|
30
30
|
from .config import RateLimitOverrides
|
|
31
31
|
|
|
@@ -38,7 +38,7 @@ __all__ = [
|
|
|
38
38
|
# Data Classes
|
|
39
39
|
'RateLimitConfig',
|
|
40
40
|
# Typed Dicts
|
|
41
|
-
'
|
|
41
|
+
'RateLimitConfigDict',
|
|
42
42
|
]
|
|
43
43
|
|
|
44
44
|
|
|
@@ -28,7 +28,7 @@ from .errors import ApiAuthError
|
|
|
28
28
|
from .errors import ApiRequestError
|
|
29
29
|
from .retry_manager import RetryInput
|
|
30
30
|
from .retry_manager import RetryManager
|
|
31
|
-
from .transport import
|
|
31
|
+
from .transport import HTTPAdapterMountConfigDict
|
|
32
32
|
from .transport import build_session_with_adapters
|
|
33
33
|
|
|
34
34
|
# SECTION: TYPE ALIASES ==================================================== #
|
|
@@ -68,7 +68,7 @@ class RequestManager:
|
|
|
68
68
|
``None``.
|
|
69
69
|
retry_cap : float, optional
|
|
70
70
|
Maximum backoff cap in seconds. Default is 30.0.
|
|
71
|
-
session_adapters : Sequence[
|
|
71
|
+
session_adapters : Sequence[HTTPAdapterMountConfigDict] | None, optional
|
|
72
72
|
Adapter mount configurations used when lazily building a session via
|
|
73
73
|
:func:`etlplus.api.transport.build_session_with_adapters`.
|
|
74
74
|
|
|
@@ -86,7 +86,7 @@ class RequestManager:
|
|
|
86
86
|
Optional factory for creating sessions.
|
|
87
87
|
retry_cap : float
|
|
88
88
|
Maximum backoff cap in seconds for :class:`RetryManager` sleeps.
|
|
89
|
-
session_adapters : Sequence[
|
|
89
|
+
session_adapters : Sequence[HTTPAdapterMountConfigDict] | None
|
|
90
90
|
Adapter mount configurations used when lazily building a session.
|
|
91
91
|
"""
|
|
92
92
|
|
|
@@ -98,7 +98,7 @@ class RequestManager:
|
|
|
98
98
|
session: requests.Session | None = None
|
|
99
99
|
session_factory: Callable[[], requests.Session] | None = None
|
|
100
100
|
retry_cap: float = 30.0
|
|
101
|
-
session_adapters: Sequence[
|
|
101
|
+
session_adapters: Sequence[HTTPAdapterMountConfigDict] | None = None
|
|
102
102
|
|
|
103
103
|
def __post_init__(self) -> None:
|
|
104
104
|
if self.session_adapters:
|
|
@@ -47,7 +47,7 @@ __all__ = [
|
|
|
47
47
|
'RetryStrategy',
|
|
48
48
|
'RetryManager',
|
|
49
49
|
# Typed Dicts
|
|
50
|
-
'
|
|
50
|
+
'RetryPolicyDict',
|
|
51
51
|
# Type Aliases
|
|
52
52
|
'RetryInput',
|
|
53
53
|
]
|
|
@@ -69,7 +69,7 @@ DEFAULT_RETRY_STATUS_CODES: Final[frozenset[int]] = frozenset(
|
|
|
69
69
|
# SECTION: TYPED DICTS ====================================================== #
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
class
|
|
72
|
+
class RetryPolicyDict(TypedDict, total=False):
|
|
73
73
|
"""
|
|
74
74
|
Optional retry policy for HTTP requests.
|
|
75
75
|
|
|
@@ -100,7 +100,7 @@ class RetryPolicy(TypedDict, total=False):
|
|
|
100
100
|
# SECTION: TYPE ALIASES ===================================================== #
|
|
101
101
|
|
|
102
102
|
|
|
103
|
-
type RetryInput =
|
|
103
|
+
type RetryInput = RetryPolicyDict | None
|
|
104
104
|
|
|
105
105
|
|
|
106
106
|
# SECTION: DATA CLASSES ===================================================== #
|
|
@@ -108,7 +108,7 @@ type RetryInput = RetryPolicy | None
|
|
|
108
108
|
|
|
109
109
|
@dataclass(frozen=True, slots=True)
|
|
110
110
|
class RetryStrategy:
|
|
111
|
-
"""Normalized retry settings derived from a :class:`
|
|
111
|
+
"""Normalized retry settings derived from a :class:`RetryPolicyDict`."""
|
|
112
112
|
|
|
113
113
|
# -- Attributes -- #
|
|
114
114
|
|
|
@@ -171,7 +171,7 @@ class RetryManager:
|
|
|
171
171
|
Default HTTP status codes considered retryable.
|
|
172
172
|
DEFAULT_CAP : ClassVar[float]
|
|
173
173
|
Default maximum sleep seconds for jittered backoff.
|
|
174
|
-
policy :
|
|
174
|
+
policy : RetryPolicyDict
|
|
175
175
|
Retry policy configuration.
|
|
176
176
|
retry_network_errors : bool
|
|
177
177
|
Whether to retry on network errors (timeouts, connection errors).
|
|
@@ -191,7 +191,7 @@ class RetryManager:
|
|
|
191
191
|
|
|
192
192
|
# -- Instance Attributes-- #
|
|
193
193
|
|
|
194
|
-
policy:
|
|
194
|
+
policy: RetryPolicyDict
|
|
195
195
|
retry_network_errors: bool = False
|
|
196
196
|
cap: float = DEFAULT_CAP
|
|
197
197
|
sleeper: Sleeper = time.sleep
|