etlplus 0.12.12__py3-none-any.whl → 0.14.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/README.md +1 -1
- etlplus/__init__.py +1 -26
- etlplus/api/__init__.py +10 -0
- etlplus/api/config.py +36 -20
- etlplus/api/endpoint_client.py +3 -3
- etlplus/api/enums.py +51 -0
- etlplus/api/pagination/client.py +1 -1
- etlplus/api/rate_limiting/config.py +13 -1
- etlplus/api/rate_limiting/rate_limiter.py +8 -11
- etlplus/api/request_manager.py +11 -6
- etlplus/api/transport.py +14 -2
- etlplus/api/types.py +7 -6
- etlplus/{run_helpers.py → api/utils.py} +205 -153
- etlplus/cli/handlers.py +17 -7
- etlplus/config/jobs.py +14 -4
- etlplus/dag.py +103 -0
- etlplus/enums.py +0 -32
- etlplus/file/enums.py +1 -1
- etlplus/{validation → ops}/README.md +2 -2
- etlplus/ops/__init__.py +61 -0
- etlplus/{extract.py → ops/extract.py} +78 -94
- etlplus/{load.py → ops/load.py} +73 -93
- etlplus/{run.py → ops/run.py} +140 -110
- etlplus/{transform.py → ops/transform.py} +75 -68
- etlplus/{validation → ops}/utils.py +80 -15
- etlplus/{validate.py → ops/validate.py} +19 -9
- etlplus/types.py +2 -2
- {etlplus-0.12.12.dist-info → etlplus-0.14.3.dist-info}/METADATA +91 -60
- {etlplus-0.12.12.dist-info → etlplus-0.14.3.dist-info}/RECORD +33 -31
- etlplus/validation/__init__.py +0 -44
- {etlplus-0.12.12.dist-info → etlplus-0.14.3.dist-info}/WHEEL +0 -0
- {etlplus-0.12.12.dist-info → etlplus-0.14.3.dist-info}/entry_points.txt +0 -0
- {etlplus-0.12.12.dist-info → etlplus-0.14.3.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.12.12.dist-info → etlplus-0.14.3.dist-info}/top_level.txt +0 -0
etlplus/types.py
CHANGED
|
@@ -193,8 +193,8 @@ type AggregateSpec = StrAnyMap
|
|
|
193
193
|
|
|
194
194
|
# -- Pipelines-- #
|
|
195
195
|
|
|
196
|
-
# Unified pipeline step spec consumed by :mod:`etlplus.transform`.
|
|
197
|
-
type StepSpec = FilterSpec | MapSpec | SelectSpec | SortSpec
|
|
196
|
+
# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
|
|
197
|
+
type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
|
|
198
198
|
|
|
199
199
|
# Collections of steps
|
|
200
200
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.14.3
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -79,8 +79,10 @@ package and command-line interface for data extraction, validation, transformati
|
|
|
79
79
|
- [Binary Serialization and Interchange](#binary-serialization-and-interchange)
|
|
80
80
|
- [Databases and Embedded Storage](#databases-and-embedded-storage)
|
|
81
81
|
- [Spreadsheets](#spreadsheets)
|
|
82
|
-
- [
|
|
82
|
+
- [Statistical / Scientific / Numeric Computing](#statistical--scientific--numeric-computing)
|
|
83
83
|
- [Logs and Event Streams](#logs-and-event-streams)
|
|
84
|
+
- [Data Archives](#data-archives)
|
|
85
|
+
- [Templates](#templates)
|
|
84
86
|
- [Usage](#usage)
|
|
85
87
|
- [Command Line Interface](#command-line-interface)
|
|
86
88
|
- [Argument Order and Required Options](#argument-order-and-required-options)
|
|
@@ -194,7 +196,7 @@ etlplus extract file examples/data/sample.csv \
|
|
|
194
196
|
[Python API](#python-api):
|
|
195
197
|
|
|
196
198
|
```python
|
|
197
|
-
from etlplus import extract, transform, validate, load
|
|
199
|
+
from etlplus.ops import extract, transform, validate, load
|
|
198
200
|
|
|
199
201
|
data = extract("file", "input.csv")
|
|
200
202
|
ops = {"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}
|
|
@@ -221,93 +223,122 @@ DDL can be rendered from table specs for migrations or schema checks.
|
|
|
221
223
|
|
|
222
224
|
### Files (`file`)
|
|
223
225
|
|
|
224
|
-
|
|
226
|
+
Recognized file formats are listed in the tables below. Support for reading to or writing from a recognized file format is marked as:
|
|
225
227
|
|
|
226
228
|
- **Y**: implemented (may require optional dependencies)
|
|
227
229
|
- **N**: stubbed or not yet implemented
|
|
228
230
|
|
|
229
231
|
#### Stubbed / Placeholder
|
|
230
232
|
|
|
231
|
-
| Format |
|
|
232
|
-
| --- | --- | --- |
|
|
233
|
+
| Format | Read | Write | Description |
|
|
234
|
+
| --- | --- | --- | --- |
|
|
233
235
|
| `stub` | N | Placeholder format for tests and future connectors. |
|
|
234
236
|
|
|
235
237
|
#### Tabular & Delimited Text
|
|
236
238
|
|
|
237
|
-
| Format |
|
|
238
|
-
| --- | --- | --- |
|
|
239
|
-
| `csv` | Y | Comma-Separated Values |
|
|
240
|
-
| `
|
|
241
|
-
| `
|
|
242
|
-
| `psv` | N | Pipe-Separated Values |
|
|
243
|
-
| `tab` | N | Often synonymous with TSV |
|
|
244
|
-
| `tsv` | Y | Tab-Separated Values |
|
|
245
|
-
| `txt` | Y | Plain text, often delimited or fixed-width |
|
|
239
|
+
| Format | Read | Write | Description |
|
|
240
|
+
| --- | --- | --- | --- |
|
|
241
|
+
| `csv` | Y | Y | Comma-Separated Values |
|
|
242
|
+
| `dat` | N | N | Generic data file, often delimited or fixed-width |
|
|
243
|
+
| `fwf` | N | N | Fixed-Width Fields |
|
|
244
|
+
| `psv` | N | N | Pipe-Separated Values |
|
|
245
|
+
| `tab` | N | N | Often synonymous with TSV |
|
|
246
|
+
| `tsv` | Y | Y | Tab-Separated Values |
|
|
247
|
+
| `txt` | Y | Y | Plain text, often delimited or fixed-width |
|
|
246
248
|
|
|
247
249
|
#### Semi-Structured Text
|
|
248
250
|
|
|
249
|
-
| Format |
|
|
250
|
-
| --- | --- | --- |
|
|
251
|
-
| `cfg` | N | Config-style key-value pairs |
|
|
252
|
-
| `conf` | N | Config-style key-value pairs |
|
|
253
|
-
| `ini` | N | Config-style key-value pairs |
|
|
254
|
-
| `json` | Y | JavaScript Object Notation |
|
|
255
|
-
| `ndjson` | Y | Newline-Delimited JSON |
|
|
256
|
-
| `properties` | N | Java-style key-value pairs |
|
|
257
|
-
| `toml` | N | Tom's Obvious Minimal Language |
|
|
258
|
-
| `xml` | Y | Extensible Markup Language |
|
|
259
|
-
| `yaml` | Y | YAML Ain't Markup Language |
|
|
251
|
+
| Format | Read | Write | Description |
|
|
252
|
+
| --- | --- | --- | --- |
|
|
253
|
+
| `cfg` | N | N | Config-style key-value pairs |
|
|
254
|
+
| `conf` | N | N | Config-style key-value pairs |
|
|
255
|
+
| `ini` | N | N | Config-style key-value pairs |
|
|
256
|
+
| `json` | Y | Y | JavaScript Object Notation |
|
|
257
|
+
| `ndjson` | Y | Y | Newline-Delimited JSON |
|
|
258
|
+
| `properties` | N | N | Java-style key-value pairs |
|
|
259
|
+
| `toml` | N | N | Tom's Obvious Minimal Language |
|
|
260
|
+
| `xml` | Y | Y | Extensible Markup Language |
|
|
261
|
+
| `yaml` | Y | Y | YAML Ain't Markup Language |
|
|
260
262
|
|
|
261
263
|
#### Columnar / Analytics-Friendly
|
|
262
264
|
|
|
263
|
-
| Format |
|
|
264
|
-
| --- | --- | --- |
|
|
265
|
-
| `arrow` | N | Apache Arrow IPC |
|
|
266
|
-
| `feather` | Y | Apache Arrow Feather |
|
|
267
|
-
| `orc` | Y | Optimized Row Columnar; common in Hadoop |
|
|
268
|
-
| `parquet` | Y | Apache Parquet; common in Big Data |
|
|
265
|
+
| Format | Read | Write | Description |
|
|
266
|
+
| --- | --- | --- | --- |
|
|
267
|
+
| `arrow` | N | N | Apache Arrow IPC |
|
|
268
|
+
| `feather` | Y | Y | Apache Arrow Feather |
|
|
269
|
+
| `orc` | Y | Y | Optimized Row Columnar; common in Hadoop |
|
|
270
|
+
| `parquet` | Y | Y | Apache Parquet; common in Big Data |
|
|
269
271
|
|
|
270
272
|
#### Binary Serialization and Interchange
|
|
271
273
|
|
|
272
|
-
| Format |
|
|
273
|
-
| --- | --- | --- |
|
|
274
|
-
| `avro` | Y | Apache Avro |
|
|
275
|
-
| `bson` | N | Binary JSON; common with MongoDB exports/dumps |
|
|
276
|
-
| `cbor` | N | Concise Binary Object Representation |
|
|
277
|
-
| `ion` | N | Amazon Ion |
|
|
278
|
-
| `msgpack` | N | MessagePack |
|
|
279
|
-
| `pb` | N | Protocol Buffers (Google Protobuf) |
|
|
280
|
-
| `pbf` | N | Protocolbuffer Binary Format; often for GIS data |
|
|
281
|
-
| `proto` | N | Protocol Buffers schema; often in .pb / .bin |
|
|
274
|
+
| Format | Read | Write | Description |
|
|
275
|
+
| --- | --- | --- | --- |
|
|
276
|
+
| `avro` | Y | Y | Apache Avro |
|
|
277
|
+
| `bson` | N | N | Binary JSON; common with MongoDB exports/dumps |
|
|
278
|
+
| `cbor` | N | N | Concise Binary Object Representation |
|
|
279
|
+
| `ion` | N | N | Amazon Ion |
|
|
280
|
+
| `msgpack` | N | N | MessagePack |
|
|
281
|
+
| `pb` | N | N | Protocol Buffers (Google Protobuf) |
|
|
282
|
+
| `pbf` | N | N | Protocolbuffer Binary Format; often for GIS data |
|
|
283
|
+
| `proto` | N | N | Protocol Buffers schema; often in .pb / .bin |
|
|
282
284
|
|
|
283
285
|
#### Databases and Embedded Storage
|
|
284
286
|
|
|
285
|
-
| Format |
|
|
286
|
-
| --- | --- | --- |
|
|
287
|
-
| `accdb` | N | Microsoft Access
|
|
288
|
-
| `duckdb` | N |
|
|
289
|
-
| `mdb` | N | Microsoft Access
|
|
290
|
-
| `sqlite` | N |
|
|
287
|
+
| Format | Read | Write | Description |
|
|
288
|
+
| --- | --- | --- | --- |
|
|
289
|
+
| `accdb` | N | N | Microsoft Access (newer format) |
|
|
290
|
+
| `duckdb` | N | N | DuckDB |
|
|
291
|
+
| `mdb` | N | N | Microsoft Access (older format) |
|
|
292
|
+
| `sqlite` | N | N | SQLite |
|
|
291
293
|
|
|
292
294
|
#### Spreadsheets
|
|
293
295
|
|
|
296
|
+
| Format | Read | Write | Description |
|
|
297
|
+
| --- | --- | --- | --- |
|
|
298
|
+
| `numbers` | N | N | Apple Numbers |
|
|
299
|
+
| `ods` | N | N | OpenDocument |
|
|
300
|
+
| `wks` | N | N | Lotus 1-2-3 |
|
|
301
|
+
| `xls` | Y | Y | Microsoft Excel (BIFF) |
|
|
302
|
+
| `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
|
|
303
|
+
| `xlsx` | Y | Y | Microsoft Excel (Open XML) |
|
|
304
|
+
|
|
305
|
+
#### Statistical / Scientific / Numeric Computing
|
|
306
|
+
|
|
307
|
+
| Format | Read | Write | Description |
|
|
308
|
+
| --- | --- | --- | --- |
|
|
309
|
+
| `dta` | N | N | Stata |
|
|
310
|
+
| `hdf5` | N | N | Hierarchical Data Format |
|
|
311
|
+
| `mat` | N | N | MATLAB |
|
|
312
|
+
| `nc` | N | N | NetCDF |
|
|
313
|
+
| `rda` | N | N | RData workspace/object |
|
|
314
|
+
| `rds` | N | N | R data |
|
|
315
|
+
| `sas7bdat` | N | N | SAS data |
|
|
316
|
+
| `sav` | N | N | SPSS data |
|
|
317
|
+
| `sylk` | N | N | Symbolic Link |
|
|
318
|
+
| `xpt` | N | N | SAS Transport |
|
|
319
|
+
| `zsav` | N | N | Compressed SPSS data |
|
|
320
|
+
|
|
321
|
+
#### Logs and Event Streams
|
|
322
|
+
|
|
294
323
|
| Format | Supported | Description |
|
|
295
324
|
| --- | --- | --- |
|
|
296
|
-
| `
|
|
297
|
-
| `xlsx` | Y | Microsoft Excel (Open XML) |
|
|
325
|
+
| `log` | N | N | Generic log file |
|
|
298
326
|
|
|
299
327
|
#### Data Archives
|
|
300
328
|
|
|
301
|
-
| Format |
|
|
302
|
-
| --- | --- | --- |
|
|
303
|
-
| `gz` | Y | Gzip-compressed file |
|
|
304
|
-
| `zip` | Y | ZIP archive |
|
|
329
|
+
| Format | Read | Write | Description |
|
|
330
|
+
| --- | --- | --- | --- |
|
|
331
|
+
| `gz` | Y | Y | Gzip-compressed file |
|
|
332
|
+
| `zip` | Y | Y | ZIP archive |
|
|
305
333
|
|
|
306
|
-
####
|
|
334
|
+
#### Templates
|
|
307
335
|
|
|
308
|
-
| Format |
|
|
309
|
-
| --- | --- | --- |
|
|
310
|
-
| `
|
|
336
|
+
| Format | Read | Write | Description |
|
|
337
|
+
| --- | --- | --- | --- |
|
|
338
|
+
| `hbs` | N | N | Handlebars |
|
|
339
|
+
| `jinja2` | N | N | Jinja2 |
|
|
340
|
+
| `mustache` | N | N | Mustache |
|
|
341
|
+
| `vm` | N | N | Apache Velocity |
|
|
311
342
|
|
|
312
343
|
## Usage
|
|
313
344
|
|
|
@@ -500,7 +531,7 @@ cat examples/data/sample.json \
|
|
|
500
531
|
Use ETLPlus as a Python library:
|
|
501
532
|
|
|
502
533
|
```python
|
|
503
|
-
from etlplus import extract, validate, transform, load
|
|
534
|
+
from etlplus.ops import extract, validate, transform, load
|
|
504
535
|
|
|
505
536
|
# Extract data
|
|
506
537
|
data = extract("file", "data.json")
|
|
@@ -695,7 +726,7 @@ We split tests into two layers:
|
|
|
695
726
|
pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
|
|
696
727
|
fake clients.
|
|
697
728
|
|
|
698
|
-
If a test calls `etlplus.cli.main()` or `etlplus.run.run()` it’s integration by default.
|
|
729
|
+
If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()` it’s integration by default. Full
|
|
699
730
|
criteria: [`CONTRIBUTING.md#testing`](CONTRIBUTING.md#testing).
|
|
700
731
|
|
|
701
732
|
### Code Coverage
|
|
@@ -1,40 +1,37 @@
|
|
|
1
|
-
etlplus/README.md,sha256=
|
|
2
|
-
etlplus/__init__.py,sha256=
|
|
1
|
+
etlplus/README.md,sha256=HwDt6eh6p4422FOJhrpUBhfPGC82QzpsJNecs3zEtUU,1459
|
|
2
|
+
etlplus/__init__.py,sha256=mgTP4PJmRmsEjTCAizzzdtzAmhuHtarmPzphzdjvLgM,277
|
|
3
3
|
etlplus/__main__.py,sha256=btoROneNiigyfBU7BSzPKZ1R9gzBMpxcpsbPwmuHwTM,479
|
|
4
4
|
etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
|
|
5
|
-
etlplus/
|
|
6
|
-
etlplus/
|
|
7
|
-
etlplus/load.py,sha256=aufl-2CpuI_J1hKBY1uFsoVf9Gfl9bKQjs233dYFf00,8631
|
|
5
|
+
etlplus/dag.py,sha256=4EYmBsJax3y4clHv10jjdp3GrBBD_WblvtxUb_JxGCQ,2464
|
|
6
|
+
etlplus/enums.py,sha256=ZxObavNilITJNT4Mf2hBy1s4kKtTFUTmrc8_whkxKNg,7541
|
|
8
7
|
etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
|
|
9
8
|
etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
-
etlplus/
|
|
11
|
-
etlplus/run_helpers.py,sha256=bj6MkaeFxjl3CeKG1HoXKx5DwAlXNERVW-GX-z1P_qQ,24373
|
|
12
|
-
etlplus/transform.py,sha256=uAUVDDHYCgx7GpVez9IK3OAZM-CnCuMa9iox3vwGGJA,25296
|
|
13
|
-
etlplus/types.py,sha256=1hsDlnF6r76zAwaUYay-i6pCM-Y0IU5nP7Crj8PLCQ4,6157
|
|
9
|
+
etlplus/types.py,sha256=LhyKU67CW7CcLEshE_1OIvEqWr-QVMJVYByf1221miY,6161
|
|
14
10
|
etlplus/utils.py,sha256=BMLTWAvCJj3zLEcffBgURYnu0UGhhXsfH2WWpAt7fV8,13363
|
|
15
|
-
etlplus/validate.py,sha256=7rJoEI_SIILdPpoBqqh2UJqg9oeReDz34mYSlc3t7Qg,12989
|
|
16
11
|
etlplus/api/README.md,sha256=aGTbcL-EqaiMTS7GToibmeTIzjX-viP2OtUDfIJnZEo,7913
|
|
17
|
-
etlplus/api/__init__.py,sha256=
|
|
12
|
+
etlplus/api/__init__.py,sha256=PK2lQv1FbsE7ZZS_ejevFZQSuOUHGApBc22YfHAzMqA,4615
|
|
18
13
|
etlplus/api/auth.py,sha256=GOO5on-LoMS1GXTAhtK9rFcfpjbBcNeA6NE5UZwIq0g,12158
|
|
19
|
-
etlplus/api/config.py,sha256=
|
|
20
|
-
etlplus/api/endpoint_client.py,sha256=
|
|
14
|
+
etlplus/api/config.py,sha256=Xi1_FUaogQxAAx7LUTGwYr5vEB_SIFvjU_xgMGF2Fsc,17913
|
|
15
|
+
etlplus/api/endpoint_client.py,sha256=OPAvw3NkpSzeITqgRP1reyqX9Nc_XC8lAj6Yp7nV4Tw,30705
|
|
16
|
+
etlplus/api/enums.py,sha256=Tvkru6V8fzQh2JM2FDLcA_yaPENOKz5JgzxLhieqEvc,1141
|
|
21
17
|
etlplus/api/errors.py,sha256=XjI2xW-sypMUNUbqfc2S57-IGyWnH3oCDFhCmKYYI_Q,4648
|
|
22
|
-
etlplus/api/request_manager.py,sha256=
|
|
18
|
+
etlplus/api/request_manager.py,sha256=fhzPV5x7DqpKqoLvfDR8GKhBX_QBMtvZsRXxVnQQElY,18674
|
|
23
19
|
etlplus/api/retry_manager.py,sha256=0GDhJVyIlb1Ww35JUWlYoa8QYUPjKLBtxQeZj3TdLbY,11306
|
|
24
|
-
etlplus/api/transport.py,sha256=
|
|
25
|
-
etlplus/api/types.py,sha256=
|
|
20
|
+
etlplus/api/transport.py,sha256=DVWzWFZLfQQXaTJ60w-SzdXRxiOE8llPprpUB5CtSyg,9410
|
|
21
|
+
etlplus/api/types.py,sha256=ijHgREN5akEFF3th2yNdWbhJhXvAxh_24qQgg_sMB4w,4653
|
|
22
|
+
etlplus/api/utils.py,sha256=I-19UPmBuOTLJs3fuCkCPbYUA4B3bXHmnRWSACP3n28,26134
|
|
26
23
|
etlplus/api/pagination/__init__.py,sha256=a4UX2J0AG8RMvmHt_CCofUm5vSmFo6GAfkb8XnSXypM,1395
|
|
27
|
-
etlplus/api/pagination/client.py,sha256=
|
|
24
|
+
etlplus/api/pagination/client.py,sha256=yMEpWqRxTCD4zRc9OYtEyUtShpGH5atiHFEAt95v2FE,5394
|
|
28
25
|
etlplus/api/pagination/config.py,sha256=3dXDJ-nMbO9Zk6i344n4roBFbUlHsa294D1_plPmm6E,13579
|
|
29
26
|
etlplus/api/pagination/paginator.py,sha256=wtdY_er4yfjx5yTUQJ1gPq-IuWmpLAHeG5buBQZJm54,24453
|
|
30
27
|
etlplus/api/rate_limiting/__init__.py,sha256=ZySB1dZettEDnWvI1EHf_TZ9L08M_kKsNR-Y_lbU6kI,1070
|
|
31
|
-
etlplus/api/rate_limiting/config.py,sha256=
|
|
32
|
-
etlplus/api/rate_limiting/rate_limiter.py,sha256=
|
|
28
|
+
etlplus/api/rate_limiting/config.py,sha256=eB-dSOG-c5awYeEpnDsQJ5pO8BJtEpGYxOunBetWiB0,9775
|
|
29
|
+
etlplus/api/rate_limiting/rate_limiter.py,sha256=heAasm1KLeBFmoJKPz6W-eQbNuahMgG1E9-z8zGEUro,7035
|
|
33
30
|
etlplus/cli/README.md,sha256=rl9VYNH5MluV0rh-eP7TbxJZ5BTMEIaksxhl_JXpYio,1233
|
|
34
31
|
etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
|
|
35
32
|
etlplus/cli/commands.py,sha256=g8_m3A8HEMyTRu2HctNiRoi2gtB5oSZCUEcyq-PIXos,24669
|
|
36
33
|
etlplus/cli/constants.py,sha256=E6Uy4WauLa_0zkzxqImXh-bb1gKdb9sBZQVc8QOzr2Q,1943
|
|
37
|
-
etlplus/cli/handlers.py,sha256=
|
|
34
|
+
etlplus/cli/handlers.py,sha256=gbW1jH348QmX65g4gffxrtMt2obFIw8X0dEKnwXQKPQ,18170
|
|
38
35
|
etlplus/cli/io.py,sha256=EFaBPYaBOyOllfMQWXgTjy-MPiGfNejicpD7ROrPyAE,7840
|
|
39
36
|
etlplus/cli/main.py,sha256=IgeqxypixfwLHR-QcpgVMQ7vMZ865bXOh2oO9v-BWeM,5234
|
|
40
37
|
etlplus/cli/options.py,sha256=vfXT3YLh7wG1iC-aTdSg6ItMC8l6n0Lozmy53XjqLbA,1199
|
|
@@ -43,7 +40,7 @@ etlplus/cli/types.py,sha256=tclhKVJXDqHzlTQBYKARfqMgDOcuBJ-Zej2pvFy96WM,652
|
|
|
43
40
|
etlplus/config/README.md,sha256=ot6oFZxTz4x83mj1_FrQ13dO0z2QkRFDnkCkx7NPsSs,1636
|
|
44
41
|
etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
|
|
45
42
|
etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
|
|
46
|
-
etlplus/config/jobs.py,sha256=
|
|
43
|
+
etlplus/config/jobs.py,sha256=oa2rNwacy2b_1HP_iFDLarGnn812ZVP2k5cHt4eqBIg,7843
|
|
47
44
|
etlplus/config/pipeline.py,sha256=m4Jh0ctFcKrIx6zR7LEC0sYY5wq0o8NqOruWPlz6qmA,9494
|
|
48
45
|
etlplus/config/profile.py,sha256=Ss2zedQGjkaGSpvBLTD4SZaWViMJ7TJPLB8Q2_BTpPg,1898
|
|
49
46
|
etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
|
|
@@ -71,7 +68,7 @@ etlplus/file/csv.py,sha256=6zXt7OKXm_6k8MrDyw8DdEwpQQrmrxG6myrDplF87_E,1744
|
|
|
71
68
|
etlplus/file/dat.py,sha256=j-GpY49SmkZtDUzZK6CbrHY9k6N83pyGcMqVGgJZ9cs,1642
|
|
72
69
|
etlplus/file/dta.py,sha256=cEprcahuYEncDYEBZiEoHyg-1jgBsr9eCHPLdI-naXM,1616
|
|
73
70
|
etlplus/file/duckdb.py,sha256=hQ8PWcvYILpkgPEtWeqbT_0yhQpJN9bJh1OwQQCcRD4,1631
|
|
74
|
-
etlplus/file/enums.py,sha256=
|
|
71
|
+
etlplus/file/enums.py,sha256=d3VGF1IbI2Vye3A9WLeMyo5dqlo2Q1TrUp9GaG7a62g,11067
|
|
75
72
|
etlplus/file/feather.py,sha256=U19T5V_08ooK1STclE9nDvsnUFzu7nvQvi6J09iC-_0,2669
|
|
76
73
|
etlplus/file/fwf.py,sha256=WLbvL94cyLCOYfhABJvoIQc1fljtz3yOuA7X4Fc4QGo,1589
|
|
77
74
|
etlplus/file/gz.py,sha256=NKsvIV7TIWn8USbvuZmRH9hr6OrXh4TzTfDykHD41Kk,2631
|
|
@@ -118,16 +115,21 @@ etlplus/file/xpt.py,sha256=JLqOkZ60awNsPXSqLKcPUwqZLPhPR05zk4EVRdEfvoU,1702
|
|
|
118
115
|
etlplus/file/yaml.py,sha256=8BEqCELaTQ_nRu1iksLBHVj19P6JmcUf5__fe0yVigw,2449
|
|
119
116
|
etlplus/file/zip.py,sha256=nd26V3S0edklriKnKOGDTLlO8RBXTda_zLLEQrJgKL4,4185
|
|
120
117
|
etlplus/file/zsav.py,sha256=2WxjXamvzj0adDbWGMWM3-cj_LsCRjyZz4J907lNkPk,1664
|
|
118
|
+
etlplus/ops/README.md,sha256=8omi7DYZhelc26JKk8Cm8QR8I3OGwziysPj1ivx41iQ,1380
|
|
119
|
+
etlplus/ops/__init__.py,sha256=NIIr2f-AZj5B0piBt6gjv46Yn0SzGYxEe6BPoopRh38,1702
|
|
120
|
+
etlplus/ops/extract.py,sha256=OJozX25qTjME7m9aTdVPJScT3GHHgopGM8uHo_rHm1Y,5783
|
|
121
|
+
etlplus/ops/load.py,sha256=RgEKnVygRN2cPDpzihU8UsIhd4eVoj0cPe0jBuNC0u4,8328
|
|
122
|
+
etlplus/ops/run.py,sha256=4ACM-bHoiBDbUdF9DnEJYFsIvKjZobW0nXTbVHCg4Xo,13393
|
|
123
|
+
etlplus/ops/transform.py,sha256=1P43WYUaw872JDU86FhbbbkP8xnBWpgEPn2Q-z4ywls,25421
|
|
124
|
+
etlplus/ops/utils.py,sha256=tbV-s3NjIGO-GfdyJFxPYIYihPibNo-4TbdRq9vWVWs,11480
|
|
125
|
+
etlplus/ops/validate.py,sha256=gLa5zcwhxGbaIhH-OqTwDsQAgAYZSajLVGwwyeT2OZk,13257
|
|
121
126
|
etlplus/templates/README.md,sha256=kHSZ8FWcrlrcWz0hBIbho-k1Bi-PL-DQ7g02o-g70c8,1355
|
|
122
127
|
etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
|
|
123
128
|
etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
|
|
124
129
|
etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
|
|
125
|
-
etlplus/
|
|
126
|
-
etlplus/
|
|
127
|
-
etlplus/
|
|
128
|
-
etlplus-0.
|
|
129
|
-
etlplus-0.
|
|
130
|
-
etlplus-0.
|
|
131
|
-
etlplus-0.12.12.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
132
|
-
etlplus-0.12.12.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
133
|
-
etlplus-0.12.12.dist-info/RECORD,,
|
|
130
|
+
etlplus-0.14.3.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
|
|
131
|
+
etlplus-0.14.3.dist-info/METADATA,sha256=0I15VHV30nTVM3CMhDZZEImeC6LV_9jtF9Fnbi3rSWg,28115
|
|
132
|
+
etlplus-0.14.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
133
|
+
etlplus-0.14.3.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
134
|
+
etlplus-0.14.3.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
135
|
+
etlplus-0.14.3.dist-info/RECORD,,
|
etlplus/validation/__init__.py
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.validation` package.
|
|
3
|
-
|
|
4
|
-
Conditional validation utilities used across the ETL pipeline.
|
|
5
|
-
|
|
6
|
-
The package intentionally exposes a single helper, :func:`maybe_validate`, to
|
|
7
|
-
keep the public API compact and predictable. Supporting logic lives in
|
|
8
|
-
``etlplus.validation.utils`` where validation configuration is normalized,
|
|
9
|
-
reducing the likelihood of phase/option mismatches.
|
|
10
|
-
|
|
11
|
-
Examples
|
|
12
|
-
--------
|
|
13
|
-
>>> from etlplus.validation import maybe_validate
|
|
14
|
-
>>> payload = {'name': 'Alice'}
|
|
15
|
-
>>> rules = {'required': ['name']}
|
|
16
|
-
>>> def validator(data, config):
|
|
17
|
-
... missing = [field for field in config['required'] if field not in data]
|
|
18
|
-
... return {'valid': not missing, 'errors': missing, 'data': data}
|
|
19
|
-
>>> maybe_validate(
|
|
20
|
-
... payload,
|
|
21
|
-
... when='both',
|
|
22
|
-
... enabled=True,
|
|
23
|
-
... rules=rules,
|
|
24
|
-
... phase='before_transform',
|
|
25
|
-
... severity='warn',
|
|
26
|
-
... validate_fn=validator,
|
|
27
|
-
... print_json_fn=lambda message: message,
|
|
28
|
-
... )
|
|
29
|
-
{'name': 'Alice'}
|
|
30
|
-
|
|
31
|
-
See Also
|
|
32
|
-
--------
|
|
33
|
-
- :mod:`etlplus.validation.utils` for implementation details and helper
|
|
34
|
-
utilities.
|
|
35
|
-
"""
|
|
36
|
-
|
|
37
|
-
from __future__ import annotations
|
|
38
|
-
|
|
39
|
-
from .utils import maybe_validate
|
|
40
|
-
|
|
41
|
-
# SECTION: EXPORTS ========================================================== #
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
__all__ = ['maybe_validate']
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|