etlplus 0.3.8__tar.gz → 0.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. etlplus-0.5.1/.coveragerc +21 -0
  2. {etlplus-0.3.8 → etlplus-0.5.1}/.github/actions/python-bootstrap/action.yml +5 -1
  3. etlplus-0.5.1/.github/workflows/ci.yml +160 -0
  4. {etlplus-0.3.8 → etlplus-0.5.1}/.pre-commit-config.yaml +5 -1
  5. {etlplus-0.3.8 → etlplus-0.5.1}/DEMO.md +5 -5
  6. etlplus-0.5.1/MANIFEST.in +12 -0
  7. {etlplus-0.3.8 → etlplus-0.5.1}/Makefile +1 -1
  8. {etlplus-0.3.8/etlplus.egg-info → etlplus-0.5.1}/PKG-INFO +58 -36
  9. {etlplus-0.3.8 → etlplus-0.5.1}/README.md +56 -35
  10. {etlplus-0.3.8 → etlplus-0.5.1}/docs/pipeline-guide.md +14 -13
  11. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/__main__.py +1 -2
  12. etlplus-0.5.1/etlplus/cli/__init__.py +15 -0
  13. etlplus-0.5.1/etlplus/cli/app.py +1367 -0
  14. etlplus-0.5.1/etlplus/cli/handlers.py +771 -0
  15. etlplus-0.5.1/etlplus/cli/main.py +616 -0
  16. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/config/pipeline.py +11 -0
  17. etlplus-0.5.1/etlplus/ddl.py +197 -0
  18. etlplus-0.5.1/etlplus/templates/__init__.py +5 -0
  19. etlplus-0.5.1/etlplus/templates/ddl.sql.j2 +128 -0
  20. etlplus-0.5.1/etlplus/templates/view.sql.j2 +69 -0
  21. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/transform.py +12 -0
  22. {etlplus-0.3.8 → etlplus-0.5.1/etlplus.egg-info}/PKG-INFO +58 -36
  23. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus.egg-info/SOURCES.txt +24 -2
  24. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus.egg-info/requires.txt +1 -0
  25. {etlplus-0.3.8 → etlplus-0.5.1}/examples/README.md +4 -4
  26. etlplus-0.5.1/examples/configs/ddl_spec.yml +67 -0
  27. {etlplus-0.3.8 → etlplus-0.5.1}/pyproject.toml +2 -1
  28. {etlplus-0.3.8 → etlplus-0.5.1}/setup.py +6 -1
  29. etlplus-0.5.1/tests/conftest.py +210 -0
  30. {etlplus-0.3.8 → etlplus-0.5.1}/tests/integration/conftest.py +105 -16
  31. etlplus-0.5.1/tests/integration/test_i_cli.py +172 -0
  32. {etlplus-0.3.8 → etlplus-0.5.1}/tests/integration/test_i_examples_data_parity.py +5 -0
  33. etlplus-0.5.1/tests/integration/test_i_pagination_strategy.py +556 -0
  34. {etlplus-0.3.8 → etlplus-0.5.1}/tests/integration/test_i_pipeline_smoke.py +46 -40
  35. {etlplus-0.3.8 → etlplus-0.5.1}/tests/integration/test_i_pipeline_yaml_load.py +6 -0
  36. etlplus-0.5.1/tests/integration/test_i_run.py +69 -0
  37. {etlplus-0.3.8 → etlplus-0.5.1}/tests/integration/test_i_run_profile_pagination_defaults.py +11 -7
  38. {etlplus-0.3.8 → etlplus-0.5.1}/tests/integration/test_i_run_profile_rate_limit_defaults.py +6 -0
  39. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/conftest.py +42 -15
  40. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_auth.py +114 -124
  41. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_config.py +60 -16
  42. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_endpoint_client.py +456 -276
  43. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_pagination_client.py +6 -1
  44. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_pagination_config.py +5 -0
  45. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_paginator.py +6 -1
  46. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_rate_limit_config.py +5 -0
  47. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_rate_limiter.py +6 -1
  48. etlplus-0.5.1/tests/unit/api/test_u_request_manager.py +349 -0
  49. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_retry_manager.py +6 -0
  50. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_transport.py +53 -1
  51. etlplus-0.5.1/tests/unit/api/test_u_types.py +135 -0
  52. etlplus-0.5.1/tests/unit/cli/conftest.py +29 -0
  53. etlplus-0.5.1/tests/unit/cli/test_u_cli_app.py +582 -0
  54. etlplus-0.5.1/tests/unit/cli/test_u_cli_handlers.py +945 -0
  55. etlplus-0.5.1/tests/unit/cli/test_u_cli_main.py +287 -0
  56. etlplus-0.5.1/tests/unit/config/test_u_config_utils.py +129 -0
  57. etlplus-0.5.1/tests/unit/config/test_u_connector.py +119 -0
  58. etlplus-0.5.1/tests/unit/config/test_u_jobs.py +131 -0
  59. etlplus-0.5.1/tests/unit/config/test_u_pipeline.py +315 -0
  60. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/conftest.py +30 -30
  61. etlplus-0.5.1/tests/unit/test_u_enums.py +135 -0
  62. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/test_u_extract.py +213 -1
  63. etlplus-0.5.1/tests/unit/test_u_file.py +296 -0
  64. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/test_u_load.py +205 -4
  65. etlplus-0.5.1/tests/unit/test_u_main.py +58 -0
  66. etlplus-0.5.1/tests/unit/test_u_mixins.py +47 -0
  67. etlplus-0.5.1/tests/unit/test_u_run.py +602 -0
  68. etlplus-0.5.1/tests/unit/test_u_run_helpers.py +385 -0
  69. etlplus-0.5.1/tests/unit/test_u_transform.py +860 -0
  70. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/test_u_utils.py +94 -5
  71. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/test_u_validate.py +42 -2
  72. etlplus-0.5.1/tests/unit/test_u_version.py +53 -0
  73. etlplus-0.3.8/.github/workflows/ci.yml +0 -121
  74. etlplus-0.3.8/etlplus/cli.py +0 -868
  75. etlplus-0.3.8/tests/conftest.py +0 -11
  76. etlplus-0.3.8/tests/integration/test_i_cli.py +0 -348
  77. etlplus-0.3.8/tests/integration/test_i_pagination_strategy.py +0 -452
  78. etlplus-0.3.8/tests/integration/test_i_run.py +0 -133
  79. etlplus-0.3.8/tests/unit/api/test_u_request_manager.py +0 -134
  80. etlplus-0.3.8/tests/unit/config/test_u_connector.py +0 -54
  81. etlplus-0.3.8/tests/unit/config/test_u_pipeline.py +0 -194
  82. etlplus-0.3.8/tests/unit/test_u_cli.py +0 -124
  83. etlplus-0.3.8/tests/unit/test_u_file.py +0 -100
  84. etlplus-0.3.8/tests/unit/test_u_transform.py +0 -483
  85. {etlplus-0.3.8 → etlplus-0.5.1}/.editorconfig +0 -0
  86. {etlplus-0.3.8 → etlplus-0.5.1}/.gitattributes +0 -0
  87. {etlplus-0.3.8 → etlplus-0.5.1}/.gitignore +0 -0
  88. {etlplus-0.3.8 → etlplus-0.5.1}/.ruff.toml +0 -0
  89. {etlplus-0.3.8 → etlplus-0.5.1}/CODE_OF_CONDUCT.md +0 -0
  90. {etlplus-0.3.8 → etlplus-0.5.1}/CONTRIBUTING.md +0 -0
  91. {etlplus-0.3.8 → etlplus-0.5.1}/LICENSE +0 -0
  92. {etlplus-0.3.8 → etlplus-0.5.1}/REFERENCES.md +0 -0
  93. {etlplus-0.3.8 → etlplus-0.5.1}/docs/snippets/installation_version.md +0 -0
  94. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/__init__.py +0 -0
  95. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/__version__.py +0 -0
  96. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/README.md +0 -0
  97. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/__init__.py +0 -0
  98. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/auth.py +0 -0
  99. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/config.py +0 -0
  100. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/endpoint_client.py +0 -0
  101. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/errors.py +0 -0
  102. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/pagination/__init__.py +0 -0
  103. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/pagination/client.py +0 -0
  104. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/pagination/config.py +0 -0
  105. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/pagination/paginator.py +0 -0
  106. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/rate_limiting/__init__.py +0 -0
  107. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/rate_limiting/config.py +0 -0
  108. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/rate_limiting/rate_limiter.py +0 -0
  109. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/request_manager.py +0 -0
  110. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/retry_manager.py +0 -0
  111. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/transport.py +0 -0
  112. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/api/types.py +0 -0
  113. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/config/__init__.py +0 -0
  114. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/config/connector.py +0 -0
  115. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/config/jobs.py +0 -0
  116. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/config/profile.py +0 -0
  117. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/config/types.py +0 -0
  118. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/config/utils.py +0 -0
  119. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/enums.py +0 -0
  120. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/extract.py +0 -0
  121. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/file.py +0 -0
  122. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/load.py +0 -0
  123. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/mixins.py +0 -0
  124. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/py.typed +0 -0
  125. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/run.py +0 -0
  126. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/run_helpers.py +0 -0
  127. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/types.py +0 -0
  128. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/utils.py +0 -0
  129. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/validate.py +0 -0
  130. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/validation/__init__.py +0 -0
  131. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus/validation/utils.py +0 -0
  132. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus.egg-info/dependency_links.txt +0 -0
  133. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus.egg-info/entry_points.txt +0 -0
  134. {etlplus-0.3.8 → etlplus-0.5.1}/etlplus.egg-info/top_level.txt +0 -0
  135. {etlplus-0.3.8 → etlplus-0.5.1}/examples/configs/pipeline.yml +0 -0
  136. {etlplus-0.3.8 → etlplus-0.5.1}/examples/data/sample.csv +0 -0
  137. {etlplus-0.3.8 → etlplus-0.5.1}/examples/data/sample.json +0 -0
  138. {etlplus-0.3.8 → etlplus-0.5.1}/examples/data/sample.xml +0 -0
  139. {etlplus-0.3.8 → etlplus-0.5.1}/examples/data/sample.xsd +0 -0
  140. {etlplus-0.3.8 → etlplus-0.5.1}/examples/data/sample.yaml +0 -0
  141. {etlplus-0.3.8 → etlplus-0.5.1}/examples/quickstart_python.py +0 -0
  142. {etlplus-0.3.8 → etlplus-0.5.1}/pytest.ini +0 -0
  143. {etlplus-0.3.8 → etlplus-0.5.1}/setup.cfg +0 -0
  144. {etlplus-0.3.8 → etlplus-0.5.1}/tests/__init__.py +0 -0
  145. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/api/test_u_mocks.py +0 -0
  146. {etlplus-0.3.8 → etlplus-0.5.1}/tests/unit/validation/test_u_validation_utils.py +0 -0
  147. {etlplus-0.3.8 → etlplus-0.5.1}/tools/run_pipeline.py +0 -0
  148. {etlplus-0.3.8 → etlplus-0.5.1}/tools/update_demo_snippets.py +0 -0
@@ -0,0 +1,21 @@
1
+ # .coveragerc
2
+ # ETLPlus
3
+ #
4
+ # Copyright © 2025 Dagitali LLC. All rights reserved.
5
+ #
6
+ # An optional pytest-cov configuration file. Limits coverage measurement to the
7
+ # ETLPlus package and ignore test modules.
8
+ #
9
+ # See:
10
+ # 1. https://pytest-cov.readthedocs.io/en/latest/config.html
11
+
12
+ [run]
13
+ source = etlplus
14
+ branch = true
15
+ omit =
16
+ tests/*
17
+ */tests/*
18
+
19
+ [report]
20
+ skip_covered = true
21
+ show_missing = true
@@ -5,6 +5,10 @@
5
5
  #
6
6
  # A GitHub Actions action to set up Python and install Python package
7
7
  # dependencies.
8
+ #
9
+ # Notes
10
+ # - External GitHub Actions action references are each pinned to specific SHA
11
+ # to limit supply-chain risk.
8
12
 
9
13
  ---
10
14
 
@@ -25,7 +29,7 @@ inputs:
25
29
  runs:
26
30
  using: composite
27
31
  steps:
28
- - uses: actions/setup-python@v5
32
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # Pinned v5
29
33
  with:
30
34
  python-version: ${{ inputs.python-version }}
31
35
 
@@ -0,0 +1,160 @@
1
+ # ci.yml
2
+ # ETLPlus
3
+ #
4
+ # Copyright © 2025 Dagitali LLC. All rights reserved.
5
+ #
6
+ # A GitHub Actions workflow configuration file for Continuous Integration (CI).
7
+ #
8
+ # Notes
9
+ # - The workflow includes jobs for linting, testing, building distributions,
10
+ # and publishing releases to GitHub and PyPI.
11
+ # - The workflow is hardened by removing global permissions and setting per-job
12
+ # scopes.
13
+ # - To harden workflow security, global permissions are absent, and per-job
14
+ # permissions are set as needed.
15
+ # - External GitHub Actions action references are each pinned to specific SHA
16
+ # to limit supply-chain risk.
17
+
18
+ ---
19
+
20
+ name: CI
21
+
22
+ on:
23
+ push:
24
+ branches:
25
+ # GitFlow branches
26
+ - main
27
+ - develop
28
+ - '**/feature/**'
29
+ - '**/bugfix/**'
30
+ - '**/release/**'
31
+ - '**/hotfix/**'
32
+
33
+ # Extended branches
34
+ - '**/chore/**'
35
+ - '**/ci/**'
36
+ - '**/docs/**'
37
+ tags: [ 'v*.*.*' ]
38
+ pull_request:
39
+ branches: [ main, develop ]
40
+
41
+ permissions: {}
42
+
43
+ jobs:
44
+ lint:
45
+ runs-on: ubuntu-latest
46
+ strategy: &python-matrix
47
+ matrix:
48
+ python-version: ['3.13', '3.14']
49
+ permissions: &permissions_read
50
+ contents: read
51
+ steps:
52
+ - &checkout_step
53
+ name: Checkout repository
54
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # Pinned v4
55
+ with:
56
+ fetch-depth: 0
57
+ - uses: ./.github/actions/python-bootstrap
58
+ with:
59
+ python-version: ${{ matrix.python-version }}
60
+ python-bootstrap: ".[dev]"
61
+ - name: Ruff check
62
+ run: |
63
+ ruff version
64
+ ruff check .
65
+ ruff format --check .
66
+
67
+ test:
68
+ runs-on: ubuntu-latest
69
+ strategy: *python-matrix
70
+ permissions: *permissions_read
71
+ steps:
72
+ - *checkout_step
73
+ - uses: ./.github/actions/python-bootstrap
74
+ with:
75
+ python-version: ${{ matrix.python-version }}
76
+ python-bootstrap: "-e .[dev,yaml]"
77
+ - name: Run tests (with coverage)
78
+ run: |
79
+ pytest -q \
80
+ --cov \
81
+ --cov-branch \
82
+ --cov-config=.coveragerc \
83
+ --cov-report=term-missing \
84
+ --cov-report=xml \
85
+ tests/
86
+
87
+ - name: Upload coverage reports to Codecov
88
+ if: matrix.python-version == '3.13'
89
+ uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # Pinned v5.5.2
90
+ with:
91
+ fail_ci_if_error: true
92
+ files: coverage.xml
93
+ flags: unit
94
+ name: etlplus
95
+ token: ${{ secrets.CODECOV_TOKEN }} # Omit for public repo
96
+ verbose: true
97
+
98
+ build:
99
+ name: Build distributions
100
+ runs-on: ubuntu-latest
101
+ if: &release_tag_condition >
102
+ startsWith(github.ref, 'refs/tags/v') ||
103
+ startsWith(github.ref, 'refs/tags/rc')
104
+ needs: [lint, test]
105
+ permissions: *permissions_read
106
+ steps:
107
+ - *checkout_step
108
+ - uses: ./.github/actions/python-bootstrap
109
+ with:
110
+ python-version: '3.13'
111
+ python-bootstrap: build
112
+ - name: Build distributions
113
+ run: |
114
+ python -m build
115
+ - name: Upload distributions
116
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # Pinned v4
117
+ with:
118
+ name: dist-artifacts
119
+ path: dist/*
120
+ if-no-files-found: error
121
+
122
+ release:
123
+ name: Publish GitHub Release
124
+ runs-on: ubuntu-latest
125
+ if: *release_tag_condition
126
+ needs: build
127
+ permissions:
128
+ contents: write
129
+ steps:
130
+ - *checkout_step
131
+ - &download_dist_step
132
+ name: Download distributions
133
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # Pinned v4
134
+ with:
135
+ name: dist-artifacts
136
+ path: dist
137
+ - name: Publish GitHub release
138
+ uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # Pinned v2
139
+ with:
140
+ files: dist/*
141
+ generate_release_notes: true
142
+
143
+ publish:
144
+ name: Publish to PyPI
145
+ runs-on: ubuntu-latest
146
+ if: *release_tag_condition
147
+ needs: build
148
+ permissions:
149
+ contents: read
150
+ id-token: write
151
+ environment:
152
+ name: pypi
153
+ url: https://pypi.org/project/etlplus/
154
+ steps:
155
+ - *checkout_step
156
+ - *download_dist_step
157
+ - name: Publish to PyPI
158
+ uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # Pinned v1
159
+ with:
160
+ verbose: true
@@ -159,7 +159,11 @@ repos:
159
159
  rev: v1.19.0
160
160
  hooks:
161
161
  - id: mypy
162
- args: [--ignore-missing-imports, --install-types, --non-interactive]
162
+ args:
163
+ - --cache-dir=.mypy_cache/pre-commit
164
+ - --ignore-missing-imports
165
+ - --install-types
166
+ - --non-interactive
163
167
 
164
168
  - repo: https://github.com/pycqa/flake8
165
169
  rev: 7.3.0
@@ -58,7 +58,7 @@ John Doe,30,New York
58
58
  Jane Smith,25,Los Angeles
59
59
  CSVDATA
60
60
 
61
- $ etlplus extract file users.csv --format csv
61
+ $ etlplus extract users.csv
62
62
  [
63
63
  {
64
64
  "name": "John Doe",
@@ -151,7 +151,7 @@ $ etlplus load '{"name": "John", "status": "active"}' file output.json
151
151
  $ etlplus load '[
152
152
  {"name": "John", "email": "john@example.com"},
153
153
  {"name": "Jane", "email": "jane@example.com"}
154
- ]' file users.csv --format csv
154
+ ]' --to users.csv
155
155
  {
156
156
  "status": "success",
157
157
  "message": "Data loaded to users.csv",
@@ -170,14 +170,14 @@ This example shows a complete ETL workflow:
170
170
 
171
171
  ```bash
172
172
  # Step 1: Extract
173
- $ etlplus extract file raw_data.csv --format csv -o extracted.json
173
+ $ etlplus extract raw_data.csv > extracted.json
174
174
 
175
175
  # Step 2: Transform
176
- $ etlplus transform extracted.json \
176
+ $ etlplus transform --from extracted.json \
177
177
  --operations '{
178
178
  "filter": {"field": "age", "op": "gte", "value": 18},
179
179
  "select": ["name", "email", "age"]
180
- }' -o transformed.json
180
+ }' --to transformed.json
181
181
 
182
182
  # Step 3: Validate
183
183
  $ etlplus validate transformed.json \
@@ -0,0 +1,12 @@
1
+ # MANIFEST.in
2
+ # ETLPlus
3
+ #
4
+ # Copyright © 2026 Dagitali LLC. All rights reserved.
5
+ #
6
+ # Contains commands that allow lists of files to be discovered and manipulated.
7
+ #
8
+ # See:
9
+ # 1. https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html
10
+
11
+ # Include Jinja template files in the etlplus package
12
+ recursive-include etlplus/templates *.j2
@@ -253,7 +253,7 @@ venv: ## Create the virtual environment (at $(VENV_DIR))
253
253
  else \
254
254
  $(call ECHO_INFO, "Using existing venv: $(VENV_DIR)"); \
255
255
  fi
256
- @$(PYTHON) -m pip install --upgrade pip setuptool wheel >/dev/null
256
+ @$(PYTHON) -m pip install --upgrade pip setuptools wheel >/dev/null
257
257
  @$(call ECHO_OK,"venv ready")
258
258
 
259
259
  ##@ CI
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: etlplus
3
- Version: 0.3.8
3
+ Version: 0.5.1
4
4
  Summary: A Swiss Army knife for simple ETL operations
5
5
  Home-page: https://github.com/Dagitali/ETLPlus
6
6
  Author: ETLPlus Team
@@ -22,6 +22,7 @@ Requires-Dist: pyodbc>=5.3.0
22
22
  Requires-Dist: python-dotenv>=1.2.1
23
23
  Requires-Dist: pandas>=2.3.3
24
24
  Requires-Dist: requests>=2.32.5
25
+ Requires-Dist: typer>=0.21.0
25
26
  Provides-Extra: dev
26
27
  Requires-Dist: black>=25.9.0; extra == "dev"
27
28
  Requires-Dist: build>=1.2.2; extra == "dev"
@@ -66,7 +67,7 @@ package and command-line interface for data extraction, validation, transformati
66
67
  - [Load Data](#load-data)
67
68
  - [Python API](#python-api)
68
69
  - [Complete ETL Pipeline Example](#complete-etl-pipeline-example)
69
- - [Environment Variables](#environment-variables)
70
+ - [Format Overrides](#format-overrides)
70
71
  - [Transformation Operations](#transformation-operations)
71
72
  - [Filter Operations](#filter-operations)
72
73
  - [Aggregation Functions](#aggregation-functions)
@@ -78,6 +79,8 @@ package and command-line interface for data extraction, validation, transformati
78
79
  - [Test Layers](#test-layers)
79
80
  - [Code Coverage](#code-coverage)
80
81
  - [Linting](#linting)
82
+ - [Updating Demo Snippets](#updating-demo-snippets)
83
+ - [Releasing to PyPI](#releasing-to-pypi)
81
84
  - [Links](#links)
82
85
  - [License](#license)
83
86
  - [Contributing](#contributing)
@@ -168,9 +171,9 @@ etlplus --version
168
171
 
169
172
  #### Extract Data
170
173
 
171
- Note: For file sources, the format is inferred from the filename extension; the `--format` option is
172
- ignored. To treat passing `--format` as an error for file sources, either set
173
- `ETLPLUS_FORMAT_BEHAVIOR=error` or pass the CLI flag `--strict-format`.
174
+ Note: For file sources, the format is normally inferred from the filename extension. Use
175
+ `--source-format` to override inference when a file lacks an extension or when you want to force a
176
+ specific parser.
174
177
 
175
178
  Extract from JSON file:
176
179
  ```bash
@@ -211,6 +214,20 @@ etlplus validate examples/data/sample.json --rules '{"email": {"type": "string",
211
214
 
212
215
  #### Transform Data
213
216
 
217
+ When piping data through `etlplus transform`, use `--source-format` whenever the SOURCE argument is
218
+ `-` or a literal payload, mirroring the `etlplus extract` semantics. Use `--target-format` to
219
+ control the emitted format for stdout or other non-file outputs, just like `etlplus load`. File
220
+ paths continue to infer formats from their extensions. Use `--from` to override the inferred source
221
+ connector type and `--to` to override the inferred target connector type, matching the `etlplus
222
+ extract`/`etlplus load` behavior.
223
+
224
+ Transform file inputs while overriding connector types:
225
+ ```bash
226
+ etlplus transform --from file examples/data/sample.json \
227
+ --operations '{"select": ["name", "email"]}' \
228
+ --to file -o temp/selected_output.json
229
+ ```
230
+
214
231
  Filter and select fields:
215
232
  ```bash
216
233
  etlplus transform '[{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]' \
@@ -234,19 +251,24 @@ etlplus transform examples/data/sample.json --operations '{"map": {"name": "new_
234
251
 
235
252
  #### Load Data
236
253
 
254
+ `etlplus load` consumes JSON from stdin; provide only the target argument plus optional flags.
255
+
237
256
  Load to JSON file:
238
257
  ```bash
239
- etlplus load '{"name": "John", "age": 30}' file temp/sample_output.json
258
+ etlplus extract file examples/data/sample.json \
259
+ | etlplus load --to file temp/sample_output.json
240
260
  ```
241
261
 
242
262
  Load to CSV file:
243
263
  ```bash
244
- etlplus load '[{"name": "John", "age": 30}]' file temp/sample_output.csv
264
+ etlplus extract file examples/data/sample.csv \
265
+ | etlplus load --to file temp/sample_output.csv
245
266
  ```
246
267
 
247
268
  Load to REST API:
248
269
  ```bash
249
- etlplus load examples/data/sample.json api https://api.example.com/endpoint
270
+ cat examples/data/sample.json \
271
+ | etlplus load --to api https://api.example.com/endpoint
250
272
  ```
251
273
 
252
274
  ### Python API
@@ -284,6 +306,19 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
284
306
  - Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
285
307
  - Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
286
308
 
309
+ CLI quick reference for pipelines:
310
+
311
+ ```bash
312
+ # List jobs or show a pipeline summary
313
+ etlplus list --config examples/configs/pipeline.yml --jobs
314
+ etlplus list --config examples/configs/pipeline.yml --summary
315
+
316
+ # Run a job
317
+ etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
318
+
319
+ # Deprecated shim (will be removed): etlplus pipeline
320
+ ```
321
+
287
322
  ### Complete ETL Pipeline Example
288
323
 
289
324
  ```bash
@@ -300,41 +335,28 @@ etlplus validate temp/sample_transformed.json \
300
335
  --rules '{"name": {"type": "string", "required": true}, "email": {"type": "string", "required": true}}'
301
336
 
302
337
  # 4. Load to CSV
303
- etlplus load temp/sample_transformed.json file temp/sample_output.csv
338
+ cat temp/sample_transformed.json \
339
+ | etlplus load --to temp/sample_output.csv
304
340
  ```
305
341
 
306
- ### Environment Variables
342
+ ### Format Overrides
307
343
 
308
- ETLPlus honors a small number of environment toggles to refine CLI behavior:
309
-
310
- - `ETLPLUS_FORMAT_BEHAVIOR`: controls what happens when `--format` is provided for
311
- file sources or targets (extract/load) where the format is inferred from the
312
- filename extension.
313
- - `error|fail|strict`: treat as error (non-zero exit)
314
- - `warn` (default): print a warning to stderr
315
- - `ignore|silent`: no message
316
- - Precedence: the CLI flag `--strict-format` overrides the environment.
344
+ `--source-format` and `--target-format` override whichever format would normally be inferred from a
345
+ file extension. This is useful when an input lacks an extension (for example, `records.txt` that
346
+ actually contains CSV) or when you intentionally want to treat a file as another format.
317
347
 
318
348
  Examples (zsh):
319
349
 
320
350
  ```zsh
321
- # Warn (default)
322
- etlplus extract file data.csv --format csv
323
- etlplus load data.json file out.csv --format csv
324
-
325
- # Enforce error via environment
326
- ETLPLUS_FORMAT_BEHAVIOR=error \
327
- etlplus extract file data.csv --format csv
328
- ETLPLUS_FORMAT_BEHAVIOR=error \
329
- etlplus load data.json file out.csv --format csv
330
-
331
- # Equivalent strict behavior via flag (overrides environment)
332
- etlplus extract file data.csv --format csv --strict-format
333
- etlplus load data.json file out.csv --format csv --strict-format
334
-
335
- # Recommended: rely on extension, no --format needed for files
336
- etlplus extract file data.csv
337
- etlplus load data.json file out.csv
351
+ # Force CSV parsing for an extension-less file
352
+ etlplus extract --from file data.txt --source-format csv
353
+
354
+ # Write CSV to a file without the .csv suffix
355
+ etlplus load --to file output.bin --target-format csv < data.json
356
+
357
+ # Leave the flags off when extensions already match the desired format
358
+ etlplus extract --from file data.csv
359
+ etlplus load --to file data.json < data.json
338
360
  ```
339
361
 
340
362
  ## Transformation Operations
@@ -25,7 +25,7 @@ package and command-line interface for data extraction, validation, transformati
25
25
  - [Load Data](#load-data)
26
26
  - [Python API](#python-api)
27
27
  - [Complete ETL Pipeline Example](#complete-etl-pipeline-example)
28
- - [Environment Variables](#environment-variables)
28
+ - [Format Overrides](#format-overrides)
29
29
  - [Transformation Operations](#transformation-operations)
30
30
  - [Filter Operations](#filter-operations)
31
31
  - [Aggregation Functions](#aggregation-functions)
@@ -37,6 +37,8 @@ package and command-line interface for data extraction, validation, transformati
37
37
  - [Test Layers](#test-layers)
38
38
  - [Code Coverage](#code-coverage)
39
39
  - [Linting](#linting)
40
+ - [Updating Demo Snippets](#updating-demo-snippets)
41
+ - [Releasing to PyPI](#releasing-to-pypi)
40
42
  - [Links](#links)
41
43
  - [License](#license)
42
44
  - [Contributing](#contributing)
@@ -127,9 +129,9 @@ etlplus --version
127
129
 
128
130
  #### Extract Data
129
131
 
130
- Note: For file sources, the format is inferred from the filename extension; the `--format` option is
131
- ignored. To treat passing `--format` as an error for file sources, either set
132
- `ETLPLUS_FORMAT_BEHAVIOR=error` or pass the CLI flag `--strict-format`.
132
+ Note: For file sources, the format is normally inferred from the filename extension. Use
133
+ `--source-format` to override inference when a file lacks an extension or when you want to force a
134
+ specific parser.
133
135
 
134
136
  Extract from JSON file:
135
137
  ```bash
@@ -170,6 +172,20 @@ etlplus validate examples/data/sample.json --rules '{"email": {"type": "string",
170
172
 
171
173
  #### Transform Data
172
174
 
175
+ When piping data through `etlplus transform`, use `--source-format` whenever the SOURCE argument is
176
+ `-` or a literal payload, mirroring the `etlplus extract` semantics. Use `--target-format` to
177
+ control the emitted format for stdout or other non-file outputs, just like `etlplus load`. File
178
+ paths continue to infer formats from their extensions. Use `--from` to override the inferred source
179
+ connector type and `--to` to override the inferred target connector type, matching the `etlplus
180
+ extract`/`etlplus load` behavior.
181
+
182
+ Transform file inputs while overriding connector types:
183
+ ```bash
184
+ etlplus transform --from file examples/data/sample.json \
185
+ --operations '{"select": ["name", "email"]}' \
186
+ --to file -o temp/selected_output.json
187
+ ```
188
+
173
189
  Filter and select fields:
174
190
  ```bash
175
191
  etlplus transform '[{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]' \
@@ -193,19 +209,24 @@ etlplus transform examples/data/sample.json --operations '{"map": {"name": "new_
193
209
 
194
210
  #### Load Data
195
211
 
212
+ `etlplus load` consumes JSON from stdin; provide only the target argument plus optional flags.
213
+
196
214
  Load to JSON file:
197
215
  ```bash
198
- etlplus load '{"name": "John", "age": 30}' file temp/sample_output.json
216
+ etlplus extract file examples/data/sample.json \
217
+ | etlplus load --to file temp/sample_output.json
199
218
  ```
200
219
 
201
220
  Load to CSV file:
202
221
  ```bash
203
- etlplus load '[{"name": "John", "age": 30}]' file temp/sample_output.csv
222
+ etlplus extract file examples/data/sample.csv \
223
+ | etlplus load --to file temp/sample_output.csv
204
224
  ```
205
225
 
206
226
  Load to REST API:
207
227
  ```bash
208
- etlplus load examples/data/sample.json api https://api.example.com/endpoint
228
+ cat examples/data/sample.json \
229
+ | etlplus load --to api https://api.example.com/endpoint
209
230
  ```
210
231
 
211
232
  ### Python API
@@ -243,6 +264,19 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
243
264
  - Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
244
265
  - Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
245
266
 
267
+ CLI quick reference for pipelines:
268
+
269
+ ```bash
270
+ # List jobs or show a pipeline summary
271
+ etlplus list --config examples/configs/pipeline.yml --jobs
272
+ etlplus list --config examples/configs/pipeline.yml --summary
273
+
274
+ # Run a job
275
+ etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
276
+
277
+ # Deprecated shim (will be removed): etlplus pipeline
278
+ ```
279
+
246
280
  ### Complete ETL Pipeline Example
247
281
 
248
282
  ```bash
@@ -259,41 +293,28 @@ etlplus validate temp/sample_transformed.json \
259
293
  --rules '{"name": {"type": "string", "required": true}, "email": {"type": "string", "required": true}}'
260
294
 
261
295
  # 4. Load to CSV
262
- etlplus load temp/sample_transformed.json file temp/sample_output.csv
296
+ cat temp/sample_transformed.json \
297
+ | etlplus load --to temp/sample_output.csv
263
298
  ```
264
299
 
265
- ### Environment Variables
300
+ ### Format Overrides
266
301
 
267
- ETLPlus honors a small number of environment toggles to refine CLI behavior:
268
-
269
- - `ETLPLUS_FORMAT_BEHAVIOR`: controls what happens when `--format` is provided for
270
- file sources or targets (extract/load) where the format is inferred from the
271
- filename extension.
272
- - `error|fail|strict`: treat as error (non-zero exit)
273
- - `warn` (default): print a warning to stderr
274
- - `ignore|silent`: no message
275
- - Precedence: the CLI flag `--strict-format` overrides the environment.
302
+ `--source-format` and `--target-format` override whichever format would normally be inferred from a
303
+ file extension. This is useful when an input lacks an extension (for example, `records.txt` that
304
+ actually contains CSV) or when you intentionally want to treat a file as another format.
276
305
 
277
306
  Examples (zsh):
278
307
 
279
308
  ```zsh
280
- # Warn (default)
281
- etlplus extract file data.csv --format csv
282
- etlplus load data.json file out.csv --format csv
283
-
284
- # Enforce error via environment
285
- ETLPLUS_FORMAT_BEHAVIOR=error \
286
- etlplus extract file data.csv --format csv
287
- ETLPLUS_FORMAT_BEHAVIOR=error \
288
- etlplus load data.json file out.csv --format csv
289
-
290
- # Equivalent strict behavior via flag (overrides environment)
291
- etlplus extract file data.csv --format csv --strict-format
292
- etlplus load data.json file out.csv --format csv --strict-format
293
-
294
- # Recommended: rely on extension, no --format needed for files
295
- etlplus extract file data.csv
296
- etlplus load data.json file out.csv
309
+ # Force CSV parsing for an extension-less file
310
+ etlplus extract --from file data.txt --source-format csv
311
+
312
+ # Write CSV to a file without the .csv suffix
313
+ etlplus load --to file output.bin --target-format csv < data.json
314
+
315
+ # Leave the flags off when extensions already match the desired format
316
+ etlplus extract --from file data.csv
317
+ etlplus load --to file data.json < data.json
297
318
  ```
298
319
 
299
320
  ## Transformation Operations
@@ -245,13 +245,13 @@ job. Those values are merged into the client configuration and forwarded to
245
245
  `EndpointClient.paginate(..., rate_limit_overrides=...)`, ensuring only that job’s paginator is sped
246
246
  up or slowed down.
247
247
 
248
- Environment / format inference note:
248
+ Format override note:
249
249
 
250
- When extracting from file sources, ETLPlus infers the format from the filename extension (e.g.
251
- `.csv`, `.json`, `.xml`, `.yaml`). Passing an explicit CLI `--format` for files is ignored unless
252
- strict mode is enabled. To enforce an error if contributors still specify a redundant `--format`
253
- flag in scripts or CI runners, set the environment variable `ETLPLUS_FORMAT_BEHAVIOR=error` or use
254
- the CLI flag `--strict-format`. This keeps pipelines cleaner by relying on naming conventions.
250
+ When extracting from file sources, ETLPlus still infers the format from the filename extension
251
+ (`.csv`, `.json`, `.xml`, `.yaml`). However, `--source-format` and `--target-format` now override
252
+ that inference for both Typer- and argparse-based CLIs. This means you can safely point at files
253
+ without/extensions or with misleading suffixes and force the desired parser or writer without having
254
+ to rename the file first.
255
255
 
256
256
  Note: When using a service + endpoint in a source, URL composition (including `base_path`) is
257
257
  handled automatically. See “Runner behavior with base_path (sources and targets)” in the APIs
@@ -378,31 +378,32 @@ jobs:
378
378
  Once you have a pipeline YAML, you can run jobs either from the
379
379
  command line or directly from Python.
380
380
 
381
- ### CLI: `etlplus pipeline` and `etlplus run`
381
+ ### CLI: `etlplus list` (inspect) and `etlplus run` (execute)
382
382
 
383
- List jobs defined in a pipeline file:
383
+ List jobs or show a summary from a pipeline file:
384
384
 
385
385
  ```bash
386
- etlplus pipeline --config examples/configs/pipeline.yml --list
386
+ etlplus list --config examples/configs/pipeline.yml --jobs
387
+ etlplus list --config examples/configs/pipeline.yml --summary
387
388
  ```
388
389
 
389
390
  Run a specific job end-to-end (extract → validate → transform → load):
390
391
 
391
392
  ```bash
392
- etlplus pipeline --config examples/configs/pipeline.yml --run file_to_file_customers
393
-
394
- # Equivalent, using the dedicated run command
395
393
  etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
396
394
  ```
397
395
 
398
396
  Notes:
399
397
 
400
- - Both commands read the same YAML schema described in this guide.
398
+ - These commands read the same YAML schema described in this guide.
401
399
  - Environment-variable substitution (e.g. `${GITHUB_TOKEN}`) is applied the same way as when loading
402
400
  configs via the Python API.
403
401
  - For more details on the orchestration implementation, see
404
402
  [Runner internals: etlplus.run](run-module.md).
405
403
 
404
+ Deprecated: `etlplus pipeline` is still available as a shim but will be removed in a future release;
405
+ prefer `list` and `run`.
406
+
406
407
  ### Python: `etlplus.run.run`
407
408
 
408
409
  To trigger a job programmatically, use the high-level runner function exposed by the package:
@@ -11,8 +11,7 @@ from .cli import main
11
11
 
12
12
 
13
13
  def _run() -> int:
14
- """Return the exit status from :func:`etlplus.cli.main`."""
15
-
14
+ """Return the exit status."""
16
15
  return main()
17
16
 
18
17