etlplus 0.15.5__tar.gz → 0.16.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. {etlplus-0.15.5 → etlplus-0.16.10}/CONTRIBUTING.md +14 -8
  2. {etlplus-0.15.5 → etlplus-0.16.10}/DEMO.md +16 -17
  3. {etlplus-0.15.5/etlplus.egg-info → etlplus-0.16.10}/PKG-INFO +34 -29
  4. {etlplus-0.15.5 → etlplus-0.16.10}/README.md +33 -28
  5. {etlplus-0.15.5 → etlplus-0.16.10}/docs/README.md +9 -4
  6. {etlplus-0.15.5 → etlplus-0.16.10}/docs/pipeline-guide.md +43 -20
  7. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/README.md +24 -2
  8. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/__init__.py +2 -0
  9. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/__init__.py +14 -14
  10. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/auth.py +9 -6
  11. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/config.py +6 -6
  12. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/endpoint_client.py +16 -16
  13. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/enums.py +2 -2
  14. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/errors.py +4 -4
  15. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/pagination/__init__.py +6 -6
  16. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/pagination/config.py +11 -9
  17. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/rate_limiting/__init__.py +2 -2
  18. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/rate_limiting/config.py +10 -10
  19. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/rate_limiting/rate_limiter.py +2 -2
  20. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/request_manager.py +4 -4
  21. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/retry_manager.py +6 -6
  22. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/transport.py +10 -10
  23. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/types.py +47 -26
  24. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/utils.py +49 -49
  25. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/README.md +9 -7
  26. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/commands.py +22 -22
  27. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/constants.py +1 -1
  28. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/handlers.py +12 -13
  29. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/main.py +1 -22
  30. etlplus-0.15.5/etlplus/workflow/pipeline.py → etlplus-0.16.10/etlplus/config.py +52 -89
  31. etlplus-0.16.10/etlplus/connector/__init__.py +43 -0
  32. etlplus-0.16.10/etlplus/connector/api.py +161 -0
  33. etlplus-0.16.10/etlplus/connector/connector.py +26 -0
  34. etlplus-0.16.10/etlplus/connector/core.py +132 -0
  35. etlplus-0.16.10/etlplus/connector/database.py +122 -0
  36. etlplus-0.16.10/etlplus/connector/enums.py +52 -0
  37. etlplus-0.16.10/etlplus/connector/file.py +120 -0
  38. etlplus-0.16.10/etlplus/connector/types.py +40 -0
  39. etlplus-0.16.10/etlplus/connector/utils.py +122 -0
  40. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/database/README.md +7 -7
  41. etlplus-0.16.10/etlplus/enums.py +144 -0
  42. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/README.md +7 -5
  43. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/accdb.py +2 -1
  44. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/arrow.py +2 -1
  45. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/bson.py +2 -1
  46. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/cbor.py +2 -1
  47. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/cfg.py +1 -1
  48. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/conf.py +1 -1
  49. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/dat.py +1 -1
  50. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/dta.py +1 -1
  51. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/duckdb.py +2 -1
  52. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/enums.py +1 -1
  53. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/fwf.py +2 -1
  54. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/hbs.py +2 -1
  55. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/hdf5.py +2 -1
  56. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/ini.py +2 -1
  57. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/ion.py +1 -1
  58. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/jinja2.py +2 -1
  59. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/log.py +1 -1
  60. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/mat.py +1 -1
  61. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/mdb.py +2 -1
  62. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/msgpack.py +2 -1
  63. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/mustache.py +2 -1
  64. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/nc.py +1 -1
  65. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/numbers.py +2 -1
  66. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/ods.py +2 -1
  67. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/pb.py +2 -1
  68. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/pbf.py +2 -1
  69. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/properties.py +2 -1
  70. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/proto.py +2 -1
  71. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/psv.py +2 -1
  72. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/rda.py +2 -1
  73. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/rds.py +1 -1
  74. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/sas7bdat.py +2 -1
  75. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/sav.py +1 -1
  76. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/sqlite.py +2 -1
  77. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/sylk.py +2 -1
  78. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/tab.py +2 -1
  79. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/toml.py +2 -1
  80. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/vm.py +2 -1
  81. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/wks.py +2 -1
  82. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/xls.py +1 -1
  83. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/xlsm.py +2 -2
  84. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/xpt.py +2 -1
  85. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/zsav.py +2 -1
  86. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/ops/README.md +10 -9
  87. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/ops/__init__.py +1 -0
  88. {etlplus-0.15.5/etlplus → etlplus-0.16.10/etlplus/ops}/enums.py +5 -140
  89. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/ops/extract.py +210 -23
  90. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/ops/load.py +141 -35
  91. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/ops/run.py +89 -104
  92. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/ops/transform.py +46 -27
  93. etlplus-0.16.10/etlplus/ops/types.py +147 -0
  94. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/ops/utils.py +5 -5
  95. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/ops/validate.py +13 -13
  96. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/templates/README.md +11 -9
  97. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/types.py +5 -102
  98. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/workflow/README.md +0 -24
  99. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/workflow/__init__.py +2 -15
  100. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/workflow/dag.py +23 -1
  101. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/workflow/jobs.py +15 -28
  102. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/workflow/profile.py +4 -2
  103. {etlplus-0.15.5 → etlplus-0.16.10/etlplus.egg-info}/PKG-INFO +34 -29
  104. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus.egg-info/SOURCES.txt +41 -22
  105. {etlplus-0.15.5 → etlplus-0.16.10}/examples/README.md +3 -2
  106. etlplus-0.15.5/examples/quickstart_python.py → etlplus-0.16.10/examples/quickstart.py +25 -4
  107. etlplus-0.16.10/pytest.ini +13 -0
  108. etlplus-0.16.10/tests/README.md +38 -0
  109. {etlplus-0.15.5 → etlplus-0.16.10}/tests/__init__.py +1 -1
  110. {etlplus-0.15.5 → etlplus-0.16.10}/tests/conftest.py +186 -28
  111. {etlplus-0.15.5 → etlplus-0.16.10}/tests/integration/conftest.py +38 -44
  112. {etlplus-0.15.5 → etlplus-0.16.10}/tests/integration/test_i_cli.py +13 -7
  113. etlplus-0.15.5/tests/integration/test_i_pipeline_yaml_load.py → etlplus-0.16.10/tests/integration/test_i_config_load.py +11 -10
  114. {etlplus-0.15.5 → etlplus-0.16.10}/tests/integration/test_i_examples_data_parity.py +5 -1
  115. {etlplus-0.15.5 → etlplus-0.16.10}/tests/integration/test_i_pagination_strategy.py +33 -6
  116. {etlplus-0.15.5 → etlplus-0.16.10}/tests/integration/test_i_run.py +8 -4
  117. {etlplus-0.15.5 → etlplus-0.16.10}/tests/integration/test_i_run_profile_pagination_defaults.py +5 -4
  118. {etlplus-0.15.5 → etlplus-0.16.10}/tests/integration/test_i_run_profile_rate_limit_defaults.py +4 -3
  119. etlplus-0.16.10/tests/smoke/__init__.py +12 -0
  120. etlplus-0.16.10/tests/smoke/conftest.py +214 -0
  121. etlplus-0.16.10/tests/smoke/test_s_cli_check.py +48 -0
  122. etlplus-0.16.10/tests/smoke/test_s_cli_extract.py +45 -0
  123. etlplus-0.16.10/tests/smoke/test_s_cli_load.py +57 -0
  124. etlplus-0.16.10/tests/smoke/test_s_cli_render.py +65 -0
  125. etlplus-0.16.10/tests/smoke/test_s_cli_transform.py +52 -0
  126. etlplus-0.16.10/tests/smoke/test_s_cli_validate.py +47 -0
  127. etlplus-0.16.10/tests/smoke/test_s_pipeline.py +72 -0
  128. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/api/conftest.py +19 -99
  129. etlplus-0.15.5/tests/unit/api/test_u_auth.py → etlplus-0.16.10/tests/unit/api/test_u_api_auth.py +4 -3
  130. etlplus-0.15.5/tests/unit/api/test_u_config.py → etlplus-0.16.10/tests/unit/api/test_u_api_config.py +3 -2
  131. etlplus-0.15.5/tests/unit/api/test_u_endpoint_client.py → etlplus-0.16.10/tests/unit/api/test_u_api_endpoint_client.py +31 -23
  132. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/api/test_u_api_enums.py +3 -2
  133. etlplus-0.15.5/tests/unit/api/test_u_mocks.py → etlplus-0.16.10/tests/unit/api/test_u_api_mocks.py +3 -3
  134. etlplus-0.15.5/tests/unit/api/test_u_pagination_client.py → etlplus-0.16.10/tests/unit/api/test_u_api_pagination_client.py +3 -2
  135. etlplus-0.15.5/tests/unit/api/test_u_pagination_config.py → etlplus-0.16.10/tests/unit/api/test_u_api_pagination_config.py +4 -2
  136. etlplus-0.15.5/tests/unit/api/test_u_paginator.py → etlplus-0.16.10/tests/unit/api/test_u_api_paginator.py +12 -10
  137. etlplus-0.15.5/tests/unit/api/test_u_rate_limit_config.py → etlplus-0.16.10/tests/unit/api/test_u_api_rate_limit_config.py +4 -2
  138. etlplus-0.15.5/tests/unit/api/test_u_rate_limiter.py → etlplus-0.16.10/tests/unit/api/test_u_api_rate_limiter.py +11 -9
  139. etlplus-0.15.5/tests/unit/api/test_u_request_manager.py → etlplus-0.16.10/tests/unit/api/test_u_api_request_manager.py +6 -2
  140. etlplus-0.15.5/tests/unit/api/test_u_retry_manager.py → etlplus-0.16.10/tests/unit/api/test_u_api_retry_manager.py +5 -4
  141. etlplus-0.15.5/tests/unit/api/test_u_transport.py → etlplus-0.16.10/tests/unit/api/test_u_api_transport.py +6 -4
  142. etlplus-0.15.5/tests/unit/api/test_u_types.py → etlplus-0.16.10/tests/unit/api/test_u_api_types.py +3 -2
  143. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/api/test_u_api_utils.py +10 -6
  144. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/cli/conftest.py +40 -20
  145. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/cli/test_u_cli_handlers.py +23 -22
  146. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/cli/test_u_cli_io.py +2 -1
  147. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/cli/test_u_cli_main.py +6 -1
  148. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/cli/test_u_cli_state.py +2 -1
  149. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/conftest.py +50 -151
  150. etlplus-0.16.10/tests/unit/connector/test_u_connector_enums.py +48 -0
  151. etlplus-0.15.5/tests/unit/workflow/test_u_workflow_connector.py → etlplus-0.16.10/tests/unit/connector/test_u_connector_utils.py +9 -7
  152. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/database/test_u_database_ddl.py +2 -1
  153. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/database/test_u_database_engine.py +6 -1
  154. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/database/test_u_database_orm.py +2 -1
  155. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/database/test_u_database_schema.py +6 -1
  156. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/file/test_u_file_core.py +5 -1
  157. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/file/test_u_file_enums.py +5 -1
  158. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/file/test_u_file_yaml.py +5 -1
  159. etlplus-0.15.5/tests/unit/test_u_enums.py → etlplus-0.16.10/tests/unit/ops/test_u_ops_enums.py +10 -36
  160. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_extract.py +5 -1
  161. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_load.py +6 -7
  162. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_run.py +45 -38
  163. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_transform.py +8 -4
  164. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_utils.py +20 -13
  165. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/ops/test_u_ops_validate.py +7 -6
  166. etlplus-0.15.5/tests/unit/workflow/test_u_workflow_pipeline.py → etlplus-0.16.10/tests/unit/test_u_config.py +34 -31
  167. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/test_u_main.py +6 -1
  168. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/test_u_mixins.py +3 -1
  169. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/test_u_utils.py +2 -1
  170. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/test_u_version.py +5 -1
  171. {etlplus-0.15.5 → etlplus-0.16.10}/tests/unit/workflow/test_u_workflow_jobs.py +9 -4
  172. etlplus-0.15.5/etlplus/workflow/connector.py +0 -386
  173. etlplus-0.15.5/etlplus/workflow/types.py +0 -115
  174. etlplus-0.15.5/pytest.ini +0 -12
  175. etlplus-0.15.5/tests/integration/test_i_pipeline_smoke.py +0 -110
  176. {etlplus-0.15.5 → etlplus-0.16.10}/.coveragerc +0 -0
  177. {etlplus-0.15.5 → etlplus-0.16.10}/.editorconfig +0 -0
  178. {etlplus-0.15.5 → etlplus-0.16.10}/.gitattributes +0 -0
  179. {etlplus-0.15.5 → etlplus-0.16.10}/.github/actions/python-bootstrap/action.yml +0 -0
  180. {etlplus-0.15.5 → etlplus-0.16.10}/.github/workflows/ci.yml +0 -0
  181. {etlplus-0.15.5 → etlplus-0.16.10}/.gitignore +0 -0
  182. {etlplus-0.15.5 → etlplus-0.16.10}/.pre-commit-config.yaml +0 -0
  183. {etlplus-0.15.5 → etlplus-0.16.10}/.ruff.toml +0 -0
  184. {etlplus-0.15.5 → etlplus-0.16.10}/CODE_OF_CONDUCT.md +0 -0
  185. {etlplus-0.15.5 → etlplus-0.16.10}/LICENSE +0 -0
  186. {etlplus-0.15.5 → etlplus-0.16.10}/MANIFEST.in +0 -0
  187. {etlplus-0.15.5 → etlplus-0.16.10}/Makefile +0 -0
  188. {etlplus-0.15.5 → etlplus-0.16.10}/REFERENCES.md +0 -0
  189. {etlplus-0.15.5 → etlplus-0.16.10}/SECURITY.md +0 -0
  190. {etlplus-0.15.5 → etlplus-0.16.10}/SUPPORT.md +0 -0
  191. {etlplus-0.15.5 → etlplus-0.16.10}/docs/snippets/installation_version.md +0 -0
  192. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/__main__.py +0 -0
  193. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/__version__.py +0 -0
  194. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/README.md +0 -0
  195. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/pagination/client.py +0 -0
  196. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/api/pagination/paginator.py +0 -0
  197. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/__init__.py +0 -0
  198. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/io.py +0 -0
  199. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/options.py +0 -0
  200. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/state.py +0 -0
  201. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/cli/types.py +0 -0
  202. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/database/__init__.py +0 -0
  203. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/database/ddl.py +0 -0
  204. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/database/engine.py +0 -0
  205. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/database/orm.py +0 -0
  206. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/database/schema.py +0 -0
  207. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/database/types.py +0 -0
  208. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/__init__.py +0 -0
  209. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/_imports.py +0 -0
  210. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/_io.py +0 -0
  211. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/avro.py +0 -0
  212. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/core.py +0 -0
  213. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/csv.py +0 -0
  214. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/feather.py +0 -0
  215. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/gz.py +0 -0
  216. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/json.py +0 -0
  217. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/ndjson.py +0 -0
  218. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/orc.py +0 -0
  219. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/parquet.py +0 -0
  220. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/stub.py +0 -0
  221. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/tsv.py +0 -0
  222. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/txt.py +0 -0
  223. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/xlsx.py +0 -0
  224. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/xml.py +0 -0
  225. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/yaml.py +0 -0
  226. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/file/zip.py +0 -0
  227. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/mixins.py +0 -0
  228. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/py.typed +0 -0
  229. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/templates/__init__.py +0 -0
  230. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/templates/ddl.sql.j2 +0 -0
  231. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/templates/view.sql.j2 +0 -0
  232. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus/utils.py +0 -0
  233. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus.egg-info/dependency_links.txt +0 -0
  234. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus.egg-info/entry_points.txt +0 -0
  235. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus.egg-info/requires.txt +0 -0
  236. {etlplus-0.15.5 → etlplus-0.16.10}/etlplus.egg-info/top_level.txt +0 -0
  237. {etlplus-0.15.5 → etlplus-0.16.10}/examples/configs/ddl_spec.yml +0 -0
  238. {etlplus-0.15.5 → etlplus-0.16.10}/examples/configs/pipeline.yml +0 -0
  239. {etlplus-0.15.5 → etlplus-0.16.10}/examples/data/sample.csv +0 -0
  240. {etlplus-0.15.5 → etlplus-0.16.10}/examples/data/sample.json +0 -0
  241. {etlplus-0.15.5 → etlplus-0.16.10}/examples/data/sample.xml +0 -0
  242. {etlplus-0.15.5 → etlplus-0.16.10}/examples/data/sample.xsd +0 -0
  243. {etlplus-0.15.5 → etlplus-0.16.10}/examples/data/sample.yaml +0 -0
  244. {etlplus-0.15.5 → etlplus-0.16.10}/pyproject.toml +0 -0
  245. {etlplus-0.15.5 → etlplus-0.16.10}/setup.cfg +0 -0
  246. {etlplus-0.15.5 → etlplus-0.16.10}/setup.py +0 -0
  247. {etlplus-0.15.5 → etlplus-0.16.10}/tools/update_demo_snippets.py +0 -0
@@ -14,7 +14,7 @@ email, or any other method with the owners of this repository before making a ch
14
14
  - [Type Checking](#type-checking)
15
15
  - [Typing Philosophy](#typing-philosophy)
16
16
  - [Testing](#testing)
17
- - [Unit vs. Integration](#unit-vs-integration)
17
+ - [Unit vs. Smoke vs. Integration](#unit-vs-smoke-vs-integration)
18
18
  - [Where to put tests](#where-to-put-tests)
19
19
  - [Common patterns](#common-patterns)
20
20
 
@@ -54,8 +54,8 @@ but using the stubs can provide a nicer editing experience.
54
54
 
55
55
  We optimize for a great editor experience and a permissive runtime:
56
56
 
57
- - TypedDicts in `etlplus/config/types.py` are editor/type-checking hints. They are intentionally
58
- `total=False` (all keys optional) and are not enforced at runtime.
57
+ - TypedDicts in `etlplus/*/types.py` (for example `etlplus/api/types.py`) are editor/type-checking
58
+ hints. They are intentionally `total=False` (all keys optional) and are not enforced at runtime.
59
59
  - Constructors named `*.from_obj` accept `Mapping[str, Any]` and perform tolerant parsing and light
60
60
  casting. This keeps runtime permissive while improving autocomplete and static analysis.
61
61
  - Prefer `Mapping[str, Any]` for inputs and plain `dict[...]` for internal state/returns. Avoid
@@ -104,26 +104,32 @@ class ExampleConfig:
104
104
 
105
105
  ## Testing
106
106
 
107
- ### Unit vs. Integration
107
+ ### Unit vs. Smoke vs. Integration
108
108
 
109
- Use these guidelines to decide whether a test belongs in the unit or integration suite:
109
+ Use these guidelines to decide whether a test belongs in the unit, smoke, or integration suite:
110
110
 
111
111
  - Unit tests (put under `tests/unit/`):
112
112
  - Exercise a single function or class in isolation (no orchestration across modules).
113
113
  - No real file system or network I/O. Use in-memory data and stubs.
114
- - Examples in this repo: `etlplus.cli.create_parser`, small helpers in `etlplus.utils`, validation and transform functions.
114
+ - Examples in this repo: small helpers in `etlplus.utils`, validation and transform functions.
115
115
  - Fast and deterministic; rely on `monkeypatch` to stub collaborators.
116
116
 
117
117
  - Integration tests (put under `tests/integration/`):
118
118
  - Exercise end-to-end flows across modules and boundaries (CLI `main()`, `run()` pipeline orchestration, file connectors, API client pagination wiring).
119
119
  - Can use temporary files/directories, and stub network with fakes/mocks.
120
- - Examples in this repo: CLI end-to-end, pipeline smoke tests, pagination strategy, runner defaults for pagination/rate limits, target URL composition.
120
+ - Examples in this repo: CLI end-to-end, pagination strategy, runner defaults for pagination/rate limits, target URL composition.
121
+
122
+ - Smoke tests (put under `tests/smoke/`):
123
+ - Minimal end-to-end checks for core flows; very fast and stable.
124
+ - May touch temporary files but avoid external network calls.
125
+ - Examples in this repo: pipeline smoke tests.
121
126
 
122
127
  If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()`, it is integration by default.
123
128
 
124
129
  ### Where to put tests
125
130
 
126
131
  - Unit tests live in `tests/unit/` and should import and test specific functions/classes directly.
132
+ - Smoke tests live in `tests/smoke/` and should focus on minimal end-to-end CLI/pipeline checks.
127
133
  - Integration tests live in `tests/integration/` and may simulate real usage using CLI argv, temporary files, and fake clients.
128
134
  - The `tests/integration/conftest.py` applies `@pytest.mark.integration` to all tests in that folder. You don’t need to add the marker per test.
129
135
  - Markers are declared in `pytest.ini`. Avoid introducing ad-hoc markers without adding them there.
@@ -133,5 +139,5 @@ If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()`, it is integrati
133
139
  - CLI tests: monkeypatch `sys.argv` and call `etlplus.cli.main()`; capture output with `capsys`.
134
140
  - File I/O: use `tmp_path` / `TemporaryDirectory()`; never write to the repo tree.
135
141
  - API flows: stub `EndpointClient` or transport layer via `monkeypatch` to avoid real HTTP.
136
- - Runner tests: monkeypatch `load_pipeline_config` to inject an in-memory `PipelineConfig`.
142
+ - Runner tests: monkeypatch `load_config` to inject an in-memory `Config`.
137
143
  - Keep tests small and focused; prefer one behavior per test with clear assertions.
@@ -32,10 +32,10 @@ etlplus 0.3.4
32
32
 
33
33
  ```bash
34
34
  $ etlplus --help
35
- usage: etlplus [-h] [--version] {extract,validate,transform,load} ...
35
+ Usage: etlplus [OPTIONS] COMMAND [ARGS]...
36
36
 
37
37
  ETLPlus - A Swiss Army knife for enabling simple ETL operations
38
- ...
38
+ # (Output abbreviated; run `etlplus --help` for the full command list.)
39
39
  ```
40
40
 
41
41
  ## Demo 1: Extract Data from Different Sources
@@ -43,7 +43,7 @@ ETLPlus - A Swiss Army knife for enabling simple ETL operations
43
43
  ### Extract from JSON
44
44
  ```bash
45
45
  $ echo '{"name": "John", "age": 30}' > sample.json
46
- $ etlplus extract file sample.json
46
+ $ etlplus extract sample.json
47
47
  {
48
48
  "name": "John",
49
49
  "age": 30
@@ -111,8 +111,8 @@ $ etlplus transform --operations '{
111
111
 
112
112
  ### Sort Data
113
113
  ```bash
114
- $ etlplus transform -\
115
- -operations '{"sort": {"field": "score", "reverse": true}}' \
114
+ $ etlplus transform \
115
+ --operations '{"sort": {"field": "score", "reverse": true}}' \
116
116
  '[{"name": "Charlie", "score": 85}, {"name": "Alice", "score": 95}, {"name": "Bob", "score": 90}]'
117
117
  ```
118
118
 
@@ -133,9 +133,8 @@ $ etlplus transform --operations '{"aggregate": {"field": "sales", "func": "sum"
133
133
 
134
134
  ### Load to JSON File
135
135
  ```bash
136
- $ etlplus load \
137
- '{"name": "John", "status": "active"}' \
138
- output.json --target-type file
136
+ $ echo '{"name": "John", "status": "active"}' \
137
+ | etlplus load output.json --target-type file
139
138
  {
140
139
  "status": "success",
141
140
  "message": "Data loaded to output.json",
@@ -145,12 +144,12 @@ $ etlplus load \
145
144
 
146
145
  ### Load to CSV File
147
146
  ```bash
148
- $ etlplus load \
149
- '[
150
- {"name": "John", "email": "john@example.com"},
151
- {"name": "Jane", "email": "jane@example.com"}
152
- ]' \
153
- users.csv --target-type file
147
+ $ cat << 'JSON' | etlplus load users.csv --target-type file
148
+ [
149
+ {"name": "John", "email": "john@example.com"},
150
+ {"name": "Jane", "email": "jane@example.com"}
151
+ ]
152
+ JSON
154
153
  {
155
154
  "status": "success",
156
155
  "message": "Data loaded to users.csv",
@@ -190,7 +189,7 @@ $ etlplus validate \
190
189
  transformed.json
191
190
 
192
191
  # Step 4: Load
193
- $ etlplus load transformed.json file final_output.csv
192
+ $ cat transformed.json | etlplus load final_output.csv
194
193
  ```
195
194
 
196
195
  ## Demo 6: Using Python API
@@ -199,7 +198,7 @@ $ etlplus load transformed.json file final_output.csv
199
198
  from etlplus.ops import extract, validate, transform, load
200
199
 
201
200
  # Extract
202
- data = extract("file", "data.csv", format="csv")
201
+ data = extract("file", "data.csv", file_format="csv")
203
202
 
204
203
  # Validate
205
204
  validation_result = validate(data, {
@@ -214,7 +213,7 @@ if validation_result["valid"]:
214
213
  })
215
214
 
216
215
  # Load
217
- load(transformed, "file", "output.json", format="json")
216
+ load(transformed, "file", "output.json", file_format="json")
218
217
  print("ETL pipeline completed successfully!")
219
218
  else:
220
219
  print(f"Validation errors: {validation_result['errors']}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: etlplus
3
- Version: 0.15.5
3
+ Version: 0.16.10
4
4
  Summary: A Swiss Army knife for simple ETL operations
5
5
  Home-page: https://github.com/Dagitali/ETLPlus
6
6
  Author: ETLPlus Team
@@ -141,7 +141,7 @@ ETLPlus supports Python 3.13 and above.
141
141
 
142
142
  - **Extract** data from multiple sources:
143
143
  - Files (CSV, JSON, XML, YAML)
144
- - Databases (connection string support)
144
+ - Databases (connection string support; extract is a placeholder today)
145
145
  - REST APIs (GET)
146
146
 
147
147
  - **Validate** data with flexible rules:
@@ -161,7 +161,7 @@ ETLPlus supports Python 3.13 and above.
161
161
 
162
162
  - **Load** data to multiple targets:
163
163
  - Files (CSV, JSON, XML, YAML)
164
- - Databases (connection string support)
164
+ - Databases (connection string support; load is a placeholder today)
165
165
  - REST APIs (PATCH, POST, PUT)
166
166
 
167
167
  ## Installation
@@ -188,7 +188,7 @@ etlplus --help
188
188
  etlplus --version
189
189
 
190
190
  # One-liner: extract CSV, filter, select, and write JSON
191
- etlplus extract file examples/data/sample.csv \
191
+ etlplus extract examples/data/sample.csv \
192
192
  | etlplus transform --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
193
193
  - temp/sample_output.json
194
194
  ```
@@ -218,8 +218,9 @@ include GET for extract and PATCH/POST/PUT for load.
218
218
 
219
219
  ### Databases (`database`)
220
220
 
221
- Database connectors use connection strings for extraction and loading, and
222
- DDL can be rendered from table specs for migrations or schema checks.
221
+ Database connectors use connection strings for extraction and loading, and DDL can be rendered from
222
+ table specs for migrations or schema checks. Database extract/load operations are currently
223
+ placeholders; plan to integrate a database client in your runner.
223
224
 
224
225
  ### Files (`file`)
225
226
 
@@ -298,7 +299,7 @@ Recognized file formats are listed in the tables below. Support for reading to o
298
299
  | `numbers` | N | N | Apple Numbers |
299
300
  | `ods` | N | N | OpenDocument |
300
301
  | `wks` | N | N | Lotus 1-2-3 |
301
- | `xls` | Y | Y | Microsoft Excel (BIFF) |
302
+ | `xls` | Y | N | Microsoft Excel (BIFF; read-only) |
302
303
  | `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
303
304
  | `xlsx` | Y | Y | Microsoft Excel (Open XML) |
304
305
 
@@ -320,8 +321,8 @@ Recognized file formats are listed in the tables below. Support for reading to o
320
321
 
321
322
  #### Logs and Event Streams
322
323
 
323
- | Format | Supported | Description |
324
- | --- | --- | --- |
324
+ | Format | Read | Write | Description |
325
+ | --- | --- | --- | --- |
325
326
  | `log` | N | N | Generic log file |
326
327
 
327
328
  #### Data Archives
@@ -354,13 +355,14 @@ etlplus --help
354
355
  etlplus --version
355
356
  ```
356
357
 
357
- The CLI is implemented with Typer (Click-based). There is no argparse compatibility layer, so rely
358
- on the documented commands/flags and run `etlplus <command> --help` for current options.
358
+ The CLI is implemented with Typer (Click-based). The legacy argparse parser has been removed, so
359
+ rely on the documented commands/flags and run `etlplus <command> --help` for current options.
359
360
 
360
361
  **Example error messages:**
361
362
 
362
363
  - If you omit a required argument: `Error: Missing required argument 'SOURCE'.`
363
- - If you place an option before its argument: `Error: Option '--source-format' must follow the 'SOURCE' argument.`
364
+ - If you place an option before its argument: `Error: Option '--source-format' must follow the
365
+ 'SOURCE' argument.`
364
366
 
365
367
  #### Argument Order and Required Options
366
368
 
@@ -424,27 +426,27 @@ specific parser.
424
426
 
425
427
  Extract from JSON file:
426
428
  ```bash
427
- etlplus extract file examples/data/sample.json
429
+ etlplus extract examples/data/sample.json
428
430
  ```
429
431
 
430
432
  Extract from CSV file:
431
433
  ```bash
432
- etlplus extract file examples/data/sample.csv
434
+ etlplus extract examples/data/sample.csv
433
435
  ```
434
436
 
435
437
  Extract from XML file:
436
438
  ```bash
437
- etlplus extract file examples/data/sample.xml
439
+ etlplus extract examples/data/sample.xml
438
440
  ```
439
441
 
440
442
  Extract from REST API:
441
443
  ```bash
442
- etlplus extract api https://api.example.com/data
444
+ etlplus extract https://api.example.com/data
443
445
  ```
444
446
 
445
447
  Save extracted data to file:
446
448
  ```bash
447
- etlplus extract file examples/data/sample.csv > temp/sample_output.json
449
+ etlplus extract examples/data/sample.csv > temp/sample_output.json
448
450
  ```
449
451
 
450
452
  #### Validate Data
@@ -510,13 +512,13 @@ etlplus transform \
510
512
 
511
513
  Load to JSON file:
512
514
  ```bash
513
- etlplus extract file examples/data/sample.json \
515
+ etlplus extract examples/data/sample.json \
514
516
  | etlplus load temp/sample_output.json --target-type file
515
517
  ```
516
518
 
517
519
  Load to CSV file:
518
520
  ```bash
519
- etlplus extract file examples/data/sample.csv \
521
+ etlplus extract examples/data/sample.csv \
520
522
  | etlplus load temp/sample_output.csv --target-type file
521
523
  ```
522
524
 
@@ -553,13 +555,13 @@ operations = {
553
555
  transformed = transform(data, operations)
554
556
 
555
557
  # Load data
556
- load(transformed, "file", "temp/sample_output.json", format="json")
558
+ load(transformed, "file", "temp/sample_output.json", file_format="json")
557
559
  ```
558
560
 
559
561
  For YAML-driven pipelines executed end-to-end (extract → validate → transform → load), see:
560
562
 
561
563
  - Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
562
- - Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
564
+ - Runner API and internals: see `etlplus.ops.run` docstrings and `docs/pipeline-guide.md`.
563
565
 
564
566
  CLI quick reference for pipelines:
565
567
 
@@ -576,7 +578,7 @@ etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
576
578
 
577
579
  ```bash
578
580
  # 1. Extract from CSV
579
- etlplus extract file examples/data/sample.csv > temp/sample_extracted.json
581
+ etlplus extract examples/data/sample.csv > temp/sample_extracted.json
580
582
 
581
583
  # 2. Transform (filter and select fields)
582
584
  etlplus transform \
@@ -611,7 +613,7 @@ etlplus load output.bin --target-type file --target-format csv < data.json
611
613
 
612
614
  # Leave the flags off when extensions already match the desired format
613
615
  etlplus extract data.csv --source-type file
614
- etlplus load data.json --target-type file < data.json
616
+ etlplus load output.json --target-type file < data.json
615
617
  ```
616
618
 
617
619
  ## Transformation Operations
@@ -706,8 +708,9 @@ Looking for the HTTP client and pagination helpers? See the dedicated docs in
706
708
 
707
709
  Curious how the pipeline runner composes API requests, pagination, and load calls?
708
710
 
709
- - Runner overview and helpers: [`docs/run-module.md`](docs/run-module.md)
710
- - Unified "connector" vocabulary (API/File/DB): `etlplus/config/connector.py`
711
+ - Runner overview and helpers: see `etlplus.ops.run` docstrings and
712
+ [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
713
+ - Unified "connector" vocabulary (API/File/DB): `etlplus/connector`
711
714
  - API/file targets reuse the same shapes as sources; API targets typically set a `method`.
712
715
 
713
716
  ### Running Tests
@@ -718,10 +721,12 @@ pytest tests/ -v
718
721
 
719
722
  #### Test Layers
720
723
 
721
- We split tests into two layers:
724
+ We split tests into three layers:
722
725
 
723
726
  - **Unit (`tests/unit/`)**: single function or class, no real I/O, fast, uses stubs/monkeypatch
724
- (e.g. `etlplus.cli.create_parser`, transform + validate helpers).
727
+ (e.g. small helpers in `etlplus.utils`, transform + validate helpers).
728
+ - **Smoke (`tests/smoke/`)**: minimal end-to-end checks for core flows; may touch temp files but
729
+ avoids external network calls.
725
730
  - **Integration (`tests/integration/`)**: end-to-end flows (CLI `main()`, pipeline `run()`,
726
731
  pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
727
732
  fake clients.
@@ -808,7 +813,7 @@ Navigate to detailed documentation for each subpackage:
808
813
  - [etlplus.cli](etlplus/cli/README.md): Command-line interface definitions for `etlplus`
809
814
  - [etlplus.database](etlplus/database/README.md): Database engine, schema, and ORM helpers
810
815
  - [etlplus.templates](etlplus/templates/README.md): SQL and DDL template helpers
811
- - [etlplus.validation](etlplus/validation/README.md): Data validation utilities and helpers
816
+ - [etlplus.ops](etlplus/ops/README.md): Extract/validate/transform/load primitives
812
817
  - [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
813
818
  profiles
814
819
 
@@ -824,7 +829,7 @@ Navigate to detailed documentation for each subpackage:
824
829
  - API client docs: [`etlplus/api/README.md`](etlplus/api/README.md)
825
830
  - Examples: [`examples/README.md`](examples/README.md)
826
831
  - Pipeline authoring guide: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
827
- - Runner internals: [`docs/run-module.md`](docs/run-module.md)
832
+ - Runner internals: see `etlplus.ops.run` docstrings and [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
828
833
  - Design notes (Mapping inputs, dict outputs): [`docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs`](docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs)
829
834
  - Typing philosophy: [`CONTRIBUTING.md#typing-philosophy`](CONTRIBUTING.md#typing-philosophy)
830
835
  - Demo and walkthrough: [`DEMO.md`](DEMO.md)
@@ -91,7 +91,7 @@ ETLPlus supports Python 3.13 and above.
91
91
 
92
92
  - **Extract** data from multiple sources:
93
93
  - Files (CSV, JSON, XML, YAML)
94
- - Databases (connection string support)
94
+ - Databases (connection string support; extract is a placeholder today)
95
95
  - REST APIs (GET)
96
96
 
97
97
  - **Validate** data with flexible rules:
@@ -111,7 +111,7 @@ ETLPlus supports Python 3.13 and above.
111
111
 
112
112
  - **Load** data to multiple targets:
113
113
  - Files (CSV, JSON, XML, YAML)
114
- - Databases (connection string support)
114
+ - Databases (connection string support; load is a placeholder today)
115
115
  - REST APIs (PATCH, POST, PUT)
116
116
 
117
117
  ## Installation
@@ -138,7 +138,7 @@ etlplus --help
138
138
  etlplus --version
139
139
 
140
140
  # One-liner: extract CSV, filter, select, and write JSON
141
- etlplus extract file examples/data/sample.csv \
141
+ etlplus extract examples/data/sample.csv \
142
142
  | etlplus transform --operations '{"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}' \
143
143
  - temp/sample_output.json
144
144
  ```
@@ -168,8 +168,9 @@ include GET for extract and PATCH/POST/PUT for load.
168
168
 
169
169
  ### Databases (`database`)
170
170
 
171
- Database connectors use connection strings for extraction and loading, and
172
- DDL can be rendered from table specs for migrations or schema checks.
171
+ Database connectors use connection strings for extraction and loading, and DDL can be rendered from
172
+ table specs for migrations or schema checks. Database extract/load operations are currently
173
+ placeholders; plan to integrate a database client in your runner.
173
174
 
174
175
  ### Files (`file`)
175
176
 
@@ -248,7 +249,7 @@ Recognized file formats are listed in the tables below. Support for reading to o
248
249
  | `numbers` | N | N | Apple Numbers |
249
250
  | `ods` | N | N | OpenDocument |
250
251
  | `wks` | N | N | Lotus 1-2-3 |
251
- | `xls` | Y | Y | Microsoft Excel (BIFF) |
252
+ | `xls` | Y | N | Microsoft Excel (BIFF; read-only) |
252
253
  | `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
253
254
  | `xlsx` | Y | Y | Microsoft Excel (Open XML) |
254
255
 
@@ -270,8 +271,8 @@ Recognized file formats are listed in the tables below. Support for reading to o
270
271
 
271
272
  #### Logs and Event Streams
272
273
 
273
- | Format | Supported | Description |
274
- | --- | --- | --- |
274
+ | Format | Read | Write | Description |
275
+ | --- | --- | --- | --- |
275
276
  | `log` | N | N | Generic log file |
276
277
 
277
278
  #### Data Archives
@@ -304,13 +305,14 @@ etlplus --help
304
305
  etlplus --version
305
306
  ```
306
307
 
307
- The CLI is implemented with Typer (Click-based). There is no argparse compatibility layer, so rely
308
- on the documented commands/flags and run `etlplus <command> --help` for current options.
308
+ The CLI is implemented with Typer (Click-based). The legacy argparse parser has been removed, so
309
+ rely on the documented commands/flags and run `etlplus <command> --help` for current options.
309
310
 
310
311
  **Example error messages:**
311
312
 
312
313
  - If you omit a required argument: `Error: Missing required argument 'SOURCE'.`
313
- - If you place an option before its argument: `Error: Option '--source-format' must follow the 'SOURCE' argument.`
314
+ - If you place an option before its argument: `Error: Option '--source-format' must follow the
315
+ 'SOURCE' argument.`
314
316
 
315
317
  #### Argument Order and Required Options
316
318
 
@@ -374,27 +376,27 @@ specific parser.
374
376
 
375
377
  Extract from JSON file:
376
378
  ```bash
377
- etlplus extract file examples/data/sample.json
379
+ etlplus extract examples/data/sample.json
378
380
  ```
379
381
 
380
382
  Extract from CSV file:
381
383
  ```bash
382
- etlplus extract file examples/data/sample.csv
384
+ etlplus extract examples/data/sample.csv
383
385
  ```
384
386
 
385
387
  Extract from XML file:
386
388
  ```bash
387
- etlplus extract file examples/data/sample.xml
389
+ etlplus extract examples/data/sample.xml
388
390
  ```
389
391
 
390
392
  Extract from REST API:
391
393
  ```bash
392
- etlplus extract api https://api.example.com/data
394
+ etlplus extract https://api.example.com/data
393
395
  ```
394
396
 
395
397
  Save extracted data to file:
396
398
  ```bash
397
- etlplus extract file examples/data/sample.csv > temp/sample_output.json
399
+ etlplus extract examples/data/sample.csv > temp/sample_output.json
398
400
  ```
399
401
 
400
402
  #### Validate Data
@@ -460,13 +462,13 @@ etlplus transform \
460
462
 
461
463
  Load to JSON file:
462
464
  ```bash
463
- etlplus extract file examples/data/sample.json \
465
+ etlplus extract examples/data/sample.json \
464
466
  | etlplus load temp/sample_output.json --target-type file
465
467
  ```
466
468
 
467
469
  Load to CSV file:
468
470
  ```bash
469
- etlplus extract file examples/data/sample.csv \
471
+ etlplus extract examples/data/sample.csv \
470
472
  | etlplus load temp/sample_output.csv --target-type file
471
473
  ```
472
474
 
@@ -503,13 +505,13 @@ operations = {
503
505
  transformed = transform(data, operations)
504
506
 
505
507
  # Load data
506
- load(transformed, "file", "temp/sample_output.json", format="json")
508
+ load(transformed, "file", "temp/sample_output.json", file_format="json")
507
509
  ```
508
510
 
509
511
  For YAML-driven pipelines executed end-to-end (extract → validate → transform → load), see:
510
512
 
511
513
  - Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
512
- - Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
514
+ - Runner API and internals: see `etlplus.ops.run` docstrings and `docs/pipeline-guide.md`.
513
515
 
514
516
  CLI quick reference for pipelines:
515
517
 
@@ -526,7 +528,7 @@ etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
526
528
 
527
529
  ```bash
528
530
  # 1. Extract from CSV
529
- etlplus extract file examples/data/sample.csv > temp/sample_extracted.json
531
+ etlplus extract examples/data/sample.csv > temp/sample_extracted.json
530
532
 
531
533
  # 2. Transform (filter and select fields)
532
534
  etlplus transform \
@@ -561,7 +563,7 @@ etlplus load output.bin --target-type file --target-format csv < data.json
561
563
 
562
564
  # Leave the flags off when extensions already match the desired format
563
565
  etlplus extract data.csv --source-type file
564
- etlplus load data.json --target-type file < data.json
566
+ etlplus load output.json --target-type file < data.json
565
567
  ```
566
568
 
567
569
  ## Transformation Operations
@@ -656,8 +658,9 @@ Looking for the HTTP client and pagination helpers? See the dedicated docs in
656
658
 
657
659
  Curious how the pipeline runner composes API requests, pagination, and load calls?
658
660
 
659
- - Runner overview and helpers: [`docs/run-module.md`](docs/run-module.md)
660
- - Unified "connector" vocabulary (API/File/DB): `etlplus/config/connector.py`
661
+ - Runner overview and helpers: see `etlplus.ops.run` docstrings and
662
+ [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
663
+ - Unified "connector" vocabulary (API/File/DB): `etlplus/connector`
661
664
  - API/file targets reuse the same shapes as sources; API targets typically set a `method`.
662
665
 
663
666
  ### Running Tests
@@ -668,10 +671,12 @@ pytest tests/ -v
668
671
 
669
672
  #### Test Layers
670
673
 
671
- We split tests into two layers:
674
+ We split tests into three layers:
672
675
 
673
676
  - **Unit (`tests/unit/`)**: single function or class, no real I/O, fast, uses stubs/monkeypatch
674
- (e.g. `etlplus.cli.create_parser`, transform + validate helpers).
677
+ (e.g. small helpers in `etlplus.utils`, transform + validate helpers).
678
+ - **Smoke (`tests/smoke/`)**: minimal end-to-end checks for core flows; may touch temp files but
679
+ avoids external network calls.
675
680
  - **Integration (`tests/integration/`)**: end-to-end flows (CLI `main()`, pipeline `run()`,
676
681
  pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
677
682
  fake clients.
@@ -758,7 +763,7 @@ Navigate to detailed documentation for each subpackage:
758
763
  - [etlplus.cli](etlplus/cli/README.md): Command-line interface definitions for `etlplus`
759
764
  - [etlplus.database](etlplus/database/README.md): Database engine, schema, and ORM helpers
760
765
  - [etlplus.templates](etlplus/templates/README.md): SQL and DDL template helpers
761
- - [etlplus.validation](etlplus/validation/README.md): Data validation utilities and helpers
766
+ - [etlplus.ops](etlplus/ops/README.md): Extract/validate/transform/load primitives
762
767
  - [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
763
768
  profiles
764
769
 
@@ -774,7 +779,7 @@ Navigate to detailed documentation for each subpackage:
774
779
  - API client docs: [`etlplus/api/README.md`](etlplus/api/README.md)
775
780
  - Examples: [`examples/README.md`](examples/README.md)
776
781
  - Pipeline authoring guide: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
777
- - Runner internals: [`docs/run-module.md`](docs/run-module.md)
782
+ - Runner internals: see `etlplus.ops.run` docstrings and [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
778
783
  - Design notes (Mapping inputs, dict outputs): [`docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs`](docs/pipeline-guide.md#design-notes-mapping-inputs-dict-outputs)
779
784
  - Typing philosophy: [`CONTRIBUTING.md#typing-philosophy`](CONTRIBUTING.md#typing-philosophy)
780
785
  - Demo and walkthrough: [`DEMO.md`](DEMO.md)
@@ -1,8 +1,12 @@
1
1
  # Documentation Notes
2
2
 
3
+ - [Documentation Notes](#documentation-notes)
4
+ - [CLI Parser Status](#cli-parser-status)
5
+ - [Migration Hints](#migration-hints)
6
+
3
7
  ## CLI Parser Status
4
- - The CLI is now Typer/Click-only. The historical `argparse` parser and `create_parser` entrypoint
5
- are deprecated and no longer supported for new integrations.
8
+
9
+ - The CLI is Typer/Click-only. The historical `argparse` parser has been removed.
6
10
  - Downstream tools should invoke the Typer app exported at `etlplus.cli.commands.app` (e.g., `python
7
11
  -m etlplus` or `etlplus ...`).
8
12
  - Handler functions still accept keyword arguments; the legacy namespace shim is temporary and will
@@ -10,8 +14,9 @@
10
14
  handlers with explicit keyword arguments if you integrate programmatically.
11
15
 
12
16
  ## Migration Hints
13
- - Replace any imports of `etlplus.cli.main.create_parser` with Typer invocations (`etlplus` binary
14
- or `app` directly).
17
+
18
+ - Replace any `argparse`-based integrations with Typer invocations (`etlplus` binary or `app`
19
+ directly).
15
20
  - If you maintained custom subcommands around the old parser, port them to Typer by attaching to
16
21
  `app` or wrapping the `etlplus` executable.
17
22
  - Tests and examples now target the Typer surface; expect argparse-focused helpers (e.g., namespace