datatoolpack 0.10.0__tar.gz → 0.10.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/PKG-INFO +107 -7
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/README.md +106 -6
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/datatoolpack/__init__.py +1 -1
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/datatoolpack/client.py +1 -1
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/datatoolpack.egg-info/PKG-INFO +107 -7
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/setup.py +1 -1
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/datatoolpack.egg-info/SOURCES.txt +0 -0
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/datatoolpack.egg-info/dependency_links.txt +0 -0
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/datatoolpack.egg-info/requires.txt +0 -0
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/datatoolpack.egg-info/top_level.txt +0 -0
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/pyproject.toml +0 -0
- {datatoolpack-0.10.0 → datatoolpack-0.10.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datatoolpack
|
|
3
|
-
Version: 0.10.
|
|
3
|
+
Version: 0.10.1
|
|
4
4
|
Summary: Official Python SDK for the AutoData ML data preparation pipeline API
|
|
5
5
|
Home-page: https://autodata.datatoolpack.com
|
|
6
6
|
Author: AutoData Team
|
|
@@ -62,7 +62,7 @@ with AutoDataClient(api_key="dtpk_YOUR_API_KEY") as client:
|
|
|
62
62
|
result = client.process(
|
|
63
63
|
file_path="data.csv",
|
|
64
64
|
target_columns=["price"],
|
|
65
|
-
output_rows=
|
|
65
|
+
output_rows=10000,
|
|
66
66
|
)
|
|
67
67
|
print(result["files"])
|
|
68
68
|
```
|
|
@@ -162,7 +162,7 @@ client.close()
|
|
|
162
162
|
result = client.process(
|
|
163
163
|
file_path="data.csv", # Path to input file (CSV, XLSX, Parquet, ...)
|
|
164
164
|
target_columns=["price"], # y-column(s) for ML
|
|
165
|
-
output_rows=
|
|
165
|
+
output_rows=10000, # Target row count in the synthetic output (default 10000)
|
|
166
166
|
tools={ # Toggle pipeline steps (all optional)
|
|
167
167
|
"anomaly": False, # Anomaly detection (off by default)
|
|
168
168
|
"dtc": True, # Data Type Conversion
|
|
@@ -199,7 +199,7 @@ result = client.process(
|
|
|
199
199
|
{"name": "dsg.csv", "url": "/download/.../dsg.csv", "size": 2097152, "description": "..."},
|
|
200
200
|
{"name": "dsm_train.csv", ...},
|
|
201
201
|
],
|
|
202
|
-
"row_count":
|
|
202
|
+
"row_count": 10000,
|
|
203
203
|
"duration_seconds": 42.1,
|
|
204
204
|
}
|
|
205
205
|
```
|
|
@@ -289,6 +289,30 @@ client.download_file("/api/v1/download/abc123.../dsg.csv", "dsg.csv")
|
|
|
289
289
|
|
|
290
290
|
---
|
|
291
291
|
|
|
292
|
+
## Feature Selection (F4)
|
|
293
|
+
|
|
294
|
+
Rank a completed session's columns by predictive importance — **without re-running the pipeline**. Handy for trimming wide datasets before training.
|
|
295
|
+
|
|
296
|
+
### `client.recommend_features(session_id, target_columns, methods, top_k)`
|
|
297
|
+
|
|
298
|
+
```python
|
|
299
|
+
ranking = client.recommend_features(
|
|
300
|
+
session_id="abc123...", # a completed session you own
|
|
301
|
+
target_columns=["price"], # rank features against these target(s)
|
|
302
|
+
methods=["mutual_information"], # one or more methods (see below); default ["mutual_information"]
|
|
303
|
+
top_k=20, # number of top features to return (default 20)
|
|
304
|
+
)
|
|
305
|
+
# {
|
|
306
|
+
# "ranking": [{"feature": "sqft", "score": 0.81}, ...], # combined across methods
|
|
307
|
+
# "per_method": {"mutual_information": [...], "shap_importance": [...]},
|
|
308
|
+
# "warnings": [...],
|
|
309
|
+
# }
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
**Available methods:** `variance_threshold`, `correlation_pruning`, `mutual_information`, `rfe`, `shap_importance`, `pca`. Pass several to blend their rankings, e.g. `methods=["mutual_information", "shap_importance"]`. (SHAP-based ranking uses a fast LightGBM model on the server; it degrades gracefully to `mutual_information` if those libraries are unavailable.)
|
|
313
|
+
|
|
314
|
+
---
|
|
315
|
+
|
|
292
316
|
## API Keys & Usage
|
|
293
317
|
|
|
294
318
|
### `client.list_keys()` — List API keys
|
|
@@ -336,6 +360,8 @@ Read data directly from databases and cloud storage instead of uploading files.
|
|
|
336
360
|
| `kafka` | Apache Kafka | `bootstrap_servers`, `topic`, `group_id` |
|
|
337
361
|
| `kinesis` | Amazon Kinesis | `stream_name`, `region`, `access_key_id`, `secret_access_key` |
|
|
338
362
|
|
|
363
|
+
> **More connectors:** AutoData supports 30+ source types. Beyond the common ones above you can also pass `oracle`, `redshift`, `cassandra`, `dynamodb`, `elasticsearch`, `clickhouse`, `timescaledb`, `synapse`, `azure_blob`, `adls`, `sap_hana`, `mqtt`, `opcua`, `influxdb`, `pi_web`, `salesforce`, `hubspot`, `stripe`, `netsuite`, `google_sheets`, `ga4`, `sharepoint`, and `http`/`rest` as `connector_type`. See **Dashboard → Documentation → Connector Data Sources** for each type's required `secrets` fields.
|
|
364
|
+
|
|
339
365
|
### Quick connector example
|
|
340
366
|
|
|
341
367
|
```python
|
|
@@ -363,7 +389,7 @@ with AutoDataClient(api_key="dtpk_...") as client:
|
|
|
363
389
|
table="orders",
|
|
364
390
|
target_columns=["price"],
|
|
365
391
|
secrets={"connection_string": "postgresql://user:pass@host/db"},
|
|
366
|
-
output_rows=
|
|
392
|
+
output_rows=10000,
|
|
367
393
|
)
|
|
368
394
|
print(result["files"])
|
|
369
395
|
```
|
|
@@ -452,7 +478,7 @@ result = client.process_from_connector(
|
|
|
452
478
|
table="orders",
|
|
453
479
|
target_columns=["price"],
|
|
454
480
|
secrets={"connection_string": "postgresql://..."},
|
|
455
|
-
output_rows=
|
|
481
|
+
output_rows=10000,
|
|
456
482
|
incremental=True, # Only fetch new rows since last sync
|
|
457
483
|
type_casts={"age": "int"}, # Override column types before processing
|
|
458
484
|
wait=True,
|
|
@@ -474,7 +500,7 @@ client.write_output(
|
|
|
474
500
|
if_exists="replace", # "replace", "append", "fail", or "merge"
|
|
475
501
|
merge_keys=["id"], # column names for merge matching (when if_exists='merge')
|
|
476
502
|
)
|
|
477
|
-
# {"success": True, "rows_written":
|
|
503
|
+
# {"success": True, "rows_written": 10000, "table": "ml_prepared_data"}
|
|
478
504
|
```
|
|
479
505
|
|
|
480
506
|
---
|
|
@@ -639,6 +665,65 @@ client.delete_scheduled_run(run_id="run-id")
|
|
|
639
665
|
|
|
640
666
|
---
|
|
641
667
|
|
|
668
|
+
## Triggers (F7)
|
|
669
|
+
|
|
670
|
+
Auto-start a pipeline when an external event fires — an inbound webhook, a connector watermark advancing, another pipeline completing, or a quality alert. Triggers are feature-flagged on the server (`FEATURE_FLAG_TRIGGERS`); if the flag is off, these endpoints return `404`.
|
|
671
|
+
|
|
672
|
+
### `client.create_trigger(name, trigger_type, config, target_pipeline_config, enabled)`
|
|
673
|
+
|
|
674
|
+
```python
|
|
675
|
+
trigger = client.create_trigger(
|
|
676
|
+
name="Reprocess on new S3 batch",
|
|
677
|
+
trigger_type="inbound_webhook", # inbound_webhook | watermark_advance | pipeline_completion | quality_alert
|
|
678
|
+
config={}, # type-specific (see table below)
|
|
679
|
+
target_pipeline_config={ # the pipeline to run when the trigger fires
|
|
680
|
+
"source_type": "s3",
|
|
681
|
+
"credential_id": "cred-uuid",
|
|
682
|
+
"table": "incoming/sales.csv",
|
|
683
|
+
"target_columns": ["revenue"],
|
|
684
|
+
"output_rows": 10000,
|
|
685
|
+
},
|
|
686
|
+
enabled=True,
|
|
687
|
+
)
|
|
688
|
+
# For inbound_webhook the secret is returned ONLY once — save it now:
|
|
689
|
+
print(trigger["webhook_secret"], trigger["inbound_url"])
|
|
690
|
+
```
|
|
691
|
+
|
|
692
|
+
**Trigger types & their `config`:**
|
|
693
|
+
|
|
694
|
+
| `trigger_type` | `config` keys |
|
|
695
|
+
|-----------------------|---------------|
|
|
696
|
+
| `inbound_webhook` | optional `{"hmac_key": "..."}` — require HMAC-SHA256-signed requests |
|
|
697
|
+
| `watermark_advance` | `{"credential_id", "table", "watermark_column", "min_advance"}` |
|
|
698
|
+
| `pipeline_completion` | `{}` — chain off another pipeline finishing |
|
|
699
|
+
| `quality_alert` | `{}` — fire when a quality-alert rule trips |
|
|
700
|
+
|
|
701
|
+
```python
|
|
702
|
+
# List / fetch / update / delete
|
|
703
|
+
triggers = client.list_triggers()
|
|
704
|
+
t = client.get_trigger(trigger_id)
|
|
705
|
+
client.update_trigger(trigger_id, enabled=False) # name | enabled | config | target_pipeline_config
|
|
706
|
+
client.delete_trigger(trigger_id)
|
|
707
|
+
|
|
708
|
+
# Dry-run: validate config + target WITHOUT firing
|
|
709
|
+
report = client.test_trigger(trigger_id)
|
|
710
|
+
# {"would_fire": True, "checks": {...}}
|
|
711
|
+
```
|
|
712
|
+
|
|
713
|
+
**Firing an inbound webhook** from your own system:
|
|
714
|
+
|
|
715
|
+
```bash
|
|
716
|
+
curl -X POST "https://autodata.datatoolpack.com/trigger/inbound/<webhook_secret>" \
|
|
717
|
+
-H "Content-Type: application/json" \
|
|
718
|
+
-d '{"any": "payload"}'
|
|
719
|
+
# If you set an hmac_key, also sign the raw body:
|
|
720
|
+
# X-Signature: sha256=<hex HMAC-SHA256 of the request body>
|
|
721
|
+
```
|
|
722
|
+
|
|
723
|
+
Inbound firing is rate-limited and HMAC-verified server-side; repeated failures auto-disable the trigger.
|
|
724
|
+
|
|
725
|
+
---
|
|
726
|
+
|
|
642
727
|
## Folder Listeners
|
|
643
728
|
|
|
644
729
|
Automatically trigger pipelines when new files appear in cloud storage:
|
|
@@ -819,6 +904,11 @@ with AutoDataClient() as client:
|
|
|
819
904
|
| `list_keys()` | List all API keys for the account |
|
|
820
905
|
| `get_usage()` | Get credit usage statistics |
|
|
821
906
|
|
|
907
|
+
### Feature Selection
|
|
908
|
+
| Method | Description |
|
|
909
|
+
|--------|-------------|
|
|
910
|
+
| `recommend_features(session_id, target_columns, ...)` | Rank a completed session's features (F4) |
|
|
911
|
+
|
|
822
912
|
### Connectors
|
|
823
913
|
| Method | Description |
|
|
824
914
|
|--------|-------------|
|
|
@@ -866,6 +956,16 @@ with AutoDataClient() as client:
|
|
|
866
956
|
| `toggle_scheduled_run(run_id)` | Enable/disable a scheduled run |
|
|
867
957
|
| `run_scheduled_now(run_id)` | Trigger a scheduled run immediately |
|
|
868
958
|
|
|
959
|
+
### Triggers
|
|
960
|
+
| Method | Description |
|
|
961
|
+
|--------|-------------|
|
|
962
|
+
| `list_triggers()` | List your triggers |
|
|
963
|
+
| `create_trigger(name, trigger_type, ...)` | Create an event trigger (F7) |
|
|
964
|
+
| `get_trigger(trigger_id)` | Fetch one trigger |
|
|
965
|
+
| `update_trigger(trigger_id, ...)` | Update a trigger |
|
|
966
|
+
| `delete_trigger(trigger_id)` | Delete a trigger |
|
|
967
|
+
| `test_trigger(trigger_id)` | Dry-run a trigger without firing |
|
|
968
|
+
|
|
869
969
|
### Folder Listeners
|
|
870
970
|
| Method | Description |
|
|
871
971
|
|--------|-------------|
|
|
@@ -25,7 +25,7 @@ with AutoDataClient(api_key="dtpk_YOUR_API_KEY") as client:
|
|
|
25
25
|
result = client.process(
|
|
26
26
|
file_path="data.csv",
|
|
27
27
|
target_columns=["price"],
|
|
28
|
-
output_rows=
|
|
28
|
+
output_rows=10000,
|
|
29
29
|
)
|
|
30
30
|
print(result["files"])
|
|
31
31
|
```
|
|
@@ -125,7 +125,7 @@ client.close()
|
|
|
125
125
|
result = client.process(
|
|
126
126
|
file_path="data.csv", # Path to input file (CSV, XLSX, Parquet, ...)
|
|
127
127
|
target_columns=["price"], # y-column(s) for ML
|
|
128
|
-
output_rows=
|
|
128
|
+
output_rows=10000, # Target row count in the synthetic output (default 10000)
|
|
129
129
|
tools={ # Toggle pipeline steps (all optional)
|
|
130
130
|
"anomaly": False, # Anomaly detection (off by default)
|
|
131
131
|
"dtc": True, # Data Type Conversion
|
|
@@ -162,7 +162,7 @@ result = client.process(
|
|
|
162
162
|
{"name": "dsg.csv", "url": "/download/.../dsg.csv", "size": 2097152, "description": "..."},
|
|
163
163
|
{"name": "dsm_train.csv", ...},
|
|
164
164
|
],
|
|
165
|
-
"row_count":
|
|
165
|
+
"row_count": 10000,
|
|
166
166
|
"duration_seconds": 42.1,
|
|
167
167
|
}
|
|
168
168
|
```
|
|
@@ -252,6 +252,30 @@ client.download_file("/api/v1/download/abc123.../dsg.csv", "dsg.csv")
|
|
|
252
252
|
|
|
253
253
|
---
|
|
254
254
|
|
|
255
|
+
## Feature Selection (F4)
|
|
256
|
+
|
|
257
|
+
Rank a completed session's columns by predictive importance — **without re-running the pipeline**. Handy for trimming wide datasets before training.
|
|
258
|
+
|
|
259
|
+
### `client.recommend_features(session_id, target_columns, methods, top_k)`
|
|
260
|
+
|
|
261
|
+
```python
|
|
262
|
+
ranking = client.recommend_features(
|
|
263
|
+
session_id="abc123...", # a completed session you own
|
|
264
|
+
target_columns=["price"], # rank features against these target(s)
|
|
265
|
+
methods=["mutual_information"], # one or more methods (see below); default ["mutual_information"]
|
|
266
|
+
top_k=20, # number of top features to return (default 20)
|
|
267
|
+
)
|
|
268
|
+
# {
|
|
269
|
+
# "ranking": [{"feature": "sqft", "score": 0.81}, ...], # combined across methods
|
|
270
|
+
# "per_method": {"mutual_information": [...], "shap_importance": [...]},
|
|
271
|
+
# "warnings": [...],
|
|
272
|
+
# }
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
**Available methods:** `variance_threshold`, `correlation_pruning`, `mutual_information`, `rfe`, `shap_importance`, `pca`. Pass several to blend their rankings, e.g. `methods=["mutual_information", "shap_importance"]`. (SHAP-based ranking uses a fast LightGBM model on the server; it degrades gracefully to `mutual_information` if those libraries are unavailable.)
|
|
276
|
+
|
|
277
|
+
---
|
|
278
|
+
|
|
255
279
|
## API Keys & Usage
|
|
256
280
|
|
|
257
281
|
### `client.list_keys()` — List API keys
|
|
@@ -299,6 +323,8 @@ Read data directly from databases and cloud storage instead of uploading files.
|
|
|
299
323
|
| `kafka` | Apache Kafka | `bootstrap_servers`, `topic`, `group_id` |
|
|
300
324
|
| `kinesis` | Amazon Kinesis | `stream_name`, `region`, `access_key_id`, `secret_access_key` |
|
|
301
325
|
|
|
326
|
+
> **More connectors:** AutoData supports 30+ source types. Beyond the common ones above you can also pass `oracle`, `redshift`, `cassandra`, `dynamodb`, `elasticsearch`, `clickhouse`, `timescaledb`, `synapse`, `azure_blob`, `adls`, `sap_hana`, `mqtt`, `opcua`, `influxdb`, `pi_web`, `salesforce`, `hubspot`, `stripe`, `netsuite`, `google_sheets`, `ga4`, `sharepoint`, and `http`/`rest` as `connector_type`. See **Dashboard → Documentation → Connector Data Sources** for each type's required `secrets` fields.
|
|
327
|
+
|
|
302
328
|
### Quick connector example
|
|
303
329
|
|
|
304
330
|
```python
|
|
@@ -326,7 +352,7 @@ with AutoDataClient(api_key="dtpk_...") as client:
|
|
|
326
352
|
table="orders",
|
|
327
353
|
target_columns=["price"],
|
|
328
354
|
secrets={"connection_string": "postgresql://user:pass@host/db"},
|
|
329
|
-
output_rows=
|
|
355
|
+
output_rows=10000,
|
|
330
356
|
)
|
|
331
357
|
print(result["files"])
|
|
332
358
|
```
|
|
@@ -415,7 +441,7 @@ result = client.process_from_connector(
|
|
|
415
441
|
table="orders",
|
|
416
442
|
target_columns=["price"],
|
|
417
443
|
secrets={"connection_string": "postgresql://..."},
|
|
418
|
-
output_rows=
|
|
444
|
+
output_rows=10000,
|
|
419
445
|
incremental=True, # Only fetch new rows since last sync
|
|
420
446
|
type_casts={"age": "int"}, # Override column types before processing
|
|
421
447
|
wait=True,
|
|
@@ -437,7 +463,7 @@ client.write_output(
|
|
|
437
463
|
if_exists="replace", # "replace", "append", "fail", or "merge"
|
|
438
464
|
merge_keys=["id"], # column names for merge matching (when if_exists='merge')
|
|
439
465
|
)
|
|
440
|
-
# {"success": True, "rows_written":
|
|
466
|
+
# {"success": True, "rows_written": 10000, "table": "ml_prepared_data"}
|
|
441
467
|
```
|
|
442
468
|
|
|
443
469
|
---
|
|
@@ -602,6 +628,65 @@ client.delete_scheduled_run(run_id="run-id")
|
|
|
602
628
|
|
|
603
629
|
---
|
|
604
630
|
|
|
631
|
+
## Triggers (F7)
|
|
632
|
+
|
|
633
|
+
Auto-start a pipeline when an external event fires — an inbound webhook, a connector watermark advancing, another pipeline completing, or a quality alert. Triggers are feature-flagged on the server (`FEATURE_FLAG_TRIGGERS`); if the flag is off, these endpoints return `404`.
|
|
634
|
+
|
|
635
|
+
### `client.create_trigger(name, trigger_type, config, target_pipeline_config, enabled)`
|
|
636
|
+
|
|
637
|
+
```python
|
|
638
|
+
trigger = client.create_trigger(
|
|
639
|
+
name="Reprocess on new S3 batch",
|
|
640
|
+
trigger_type="inbound_webhook", # inbound_webhook | watermark_advance | pipeline_completion | quality_alert
|
|
641
|
+
config={}, # type-specific (see table below)
|
|
642
|
+
target_pipeline_config={ # the pipeline to run when the trigger fires
|
|
643
|
+
"source_type": "s3",
|
|
644
|
+
"credential_id": "cred-uuid",
|
|
645
|
+
"table": "incoming/sales.csv",
|
|
646
|
+
"target_columns": ["revenue"],
|
|
647
|
+
"output_rows": 10000,
|
|
648
|
+
},
|
|
649
|
+
enabled=True,
|
|
650
|
+
)
|
|
651
|
+
# For inbound_webhook the secret is returned ONLY once — save it now:
|
|
652
|
+
print(trigger["webhook_secret"], trigger["inbound_url"])
|
|
653
|
+
```
|
|
654
|
+
|
|
655
|
+
**Trigger types & their `config`:**
|
|
656
|
+
|
|
657
|
+
| `trigger_type` | `config` keys |
|
|
658
|
+
|-----------------------|---------------|
|
|
659
|
+
| `inbound_webhook` | optional `{"hmac_key": "..."}` — require HMAC-SHA256-signed requests |
|
|
660
|
+
| `watermark_advance` | `{"credential_id", "table", "watermark_column", "min_advance"}` |
|
|
661
|
+
| `pipeline_completion` | `{}` — chain off another pipeline finishing |
|
|
662
|
+
| `quality_alert` | `{}` — fire when a quality-alert rule trips |
|
|
663
|
+
|
|
664
|
+
```python
|
|
665
|
+
# List / fetch / update / delete
|
|
666
|
+
triggers = client.list_triggers()
|
|
667
|
+
t = client.get_trigger(trigger_id)
|
|
668
|
+
client.update_trigger(trigger_id, enabled=False) # name | enabled | config | target_pipeline_config
|
|
669
|
+
client.delete_trigger(trigger_id)
|
|
670
|
+
|
|
671
|
+
# Dry-run: validate config + target WITHOUT firing
|
|
672
|
+
report = client.test_trigger(trigger_id)
|
|
673
|
+
# {"would_fire": True, "checks": {...}}
|
|
674
|
+
```
|
|
675
|
+
|
|
676
|
+
**Firing an inbound webhook** from your own system:
|
|
677
|
+
|
|
678
|
+
```bash
|
|
679
|
+
curl -X POST "https://autodata.datatoolpack.com/trigger/inbound/<webhook_secret>" \
|
|
680
|
+
-H "Content-Type: application/json" \
|
|
681
|
+
-d '{"any": "payload"}'
|
|
682
|
+
# If you set an hmac_key, also sign the raw body:
|
|
683
|
+
# X-Signature: sha256=<hex HMAC-SHA256 of the request body>
|
|
684
|
+
```
|
|
685
|
+
|
|
686
|
+
Inbound firing is rate-limited and HMAC-verified server-side; repeated failures auto-disable the trigger.
|
|
687
|
+
|
|
688
|
+
---
|
|
689
|
+
|
|
605
690
|
## Folder Listeners
|
|
606
691
|
|
|
607
692
|
Automatically trigger pipelines when new files appear in cloud storage:
|
|
@@ -782,6 +867,11 @@ with AutoDataClient() as client:
|
|
|
782
867
|
| `list_keys()` | List all API keys for the account |
|
|
783
868
|
| `get_usage()` | Get credit usage statistics |
|
|
784
869
|
|
|
870
|
+
### Feature Selection
|
|
871
|
+
| Method | Description |
|
|
872
|
+
|--------|-------------|
|
|
873
|
+
| `recommend_features(session_id, target_columns, ...)` | Rank a completed session's features (F4) |
|
|
874
|
+
|
|
785
875
|
### Connectors
|
|
786
876
|
| Method | Description |
|
|
787
877
|
|--------|-------------|
|
|
@@ -829,6 +919,16 @@ with AutoDataClient() as client:
|
|
|
829
919
|
| `toggle_scheduled_run(run_id)` | Enable/disable a scheduled run |
|
|
830
920
|
| `run_scheduled_now(run_id)` | Trigger a scheduled run immediately |
|
|
831
921
|
|
|
922
|
+
### Triggers
|
|
923
|
+
| Method | Description |
|
|
924
|
+
|--------|-------------|
|
|
925
|
+
| `list_triggers()` | List your triggers |
|
|
926
|
+
| `create_trigger(name, trigger_type, ...)` | Create an event trigger (F7) |
|
|
927
|
+
| `get_trigger(trigger_id)` | Fetch one trigger |
|
|
928
|
+
| `update_trigger(trigger_id, ...)` | Update a trigger |
|
|
929
|
+
| `delete_trigger(trigger_id)` | Delete a trigger |
|
|
930
|
+
| `test_trigger(trigger_id)` | Dry-run a trigger without firing |
|
|
931
|
+
|
|
832
932
|
### Folder Listeners
|
|
833
933
|
| Method | Description |
|
|
834
934
|
|--------|-------------|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datatoolpack
|
|
3
|
-
Version: 0.10.
|
|
3
|
+
Version: 0.10.1
|
|
4
4
|
Summary: Official Python SDK for the AutoData ML data preparation pipeline API
|
|
5
5
|
Home-page: https://autodata.datatoolpack.com
|
|
6
6
|
Author: AutoData Team
|
|
@@ -62,7 +62,7 @@ with AutoDataClient(api_key="dtpk_YOUR_API_KEY") as client:
|
|
|
62
62
|
result = client.process(
|
|
63
63
|
file_path="data.csv",
|
|
64
64
|
target_columns=["price"],
|
|
65
|
-
output_rows=
|
|
65
|
+
output_rows=10000,
|
|
66
66
|
)
|
|
67
67
|
print(result["files"])
|
|
68
68
|
```
|
|
@@ -162,7 +162,7 @@ client.close()
|
|
|
162
162
|
result = client.process(
|
|
163
163
|
file_path="data.csv", # Path to input file (CSV, XLSX, Parquet, ...)
|
|
164
164
|
target_columns=["price"], # y-column(s) for ML
|
|
165
|
-
output_rows=
|
|
165
|
+
output_rows=10000, # Target row count in the synthetic output (default 10000)
|
|
166
166
|
tools={ # Toggle pipeline steps (all optional)
|
|
167
167
|
"anomaly": False, # Anomaly detection (off by default)
|
|
168
168
|
"dtc": True, # Data Type Conversion
|
|
@@ -199,7 +199,7 @@ result = client.process(
|
|
|
199
199
|
{"name": "dsg.csv", "url": "/download/.../dsg.csv", "size": 2097152, "description": "..."},
|
|
200
200
|
{"name": "dsm_train.csv", ...},
|
|
201
201
|
],
|
|
202
|
-
"row_count":
|
|
202
|
+
"row_count": 10000,
|
|
203
203
|
"duration_seconds": 42.1,
|
|
204
204
|
}
|
|
205
205
|
```
|
|
@@ -289,6 +289,30 @@ client.download_file("/api/v1/download/abc123.../dsg.csv", "dsg.csv")
|
|
|
289
289
|
|
|
290
290
|
---
|
|
291
291
|
|
|
292
|
+
## Feature Selection (F4)
|
|
293
|
+
|
|
294
|
+
Rank a completed session's columns by predictive importance — **without re-running the pipeline**. Handy for trimming wide datasets before training.
|
|
295
|
+
|
|
296
|
+
### `client.recommend_features(session_id, target_columns, methods, top_k)`
|
|
297
|
+
|
|
298
|
+
```python
|
|
299
|
+
ranking = client.recommend_features(
|
|
300
|
+
session_id="abc123...", # a completed session you own
|
|
301
|
+
target_columns=["price"], # rank features against these target(s)
|
|
302
|
+
methods=["mutual_information"], # one or more methods (see below); default ["mutual_information"]
|
|
303
|
+
top_k=20, # number of top features to return (default 20)
|
|
304
|
+
)
|
|
305
|
+
# {
|
|
306
|
+
# "ranking": [{"feature": "sqft", "score": 0.81}, ...], # combined across methods
|
|
307
|
+
# "per_method": {"mutual_information": [...], "shap_importance": [...]},
|
|
308
|
+
# "warnings": [...],
|
|
309
|
+
# }
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
**Available methods:** `variance_threshold`, `correlation_pruning`, `mutual_information`, `rfe`, `shap_importance`, `pca`. Pass several to blend their rankings, e.g. `methods=["mutual_information", "shap_importance"]`. (SHAP-based ranking uses a fast LightGBM model on the server; it degrades gracefully to `mutual_information` if those libraries are unavailable.)
|
|
313
|
+
|
|
314
|
+
---
|
|
315
|
+
|
|
292
316
|
## API Keys & Usage
|
|
293
317
|
|
|
294
318
|
### `client.list_keys()` — List API keys
|
|
@@ -336,6 +360,8 @@ Read data directly from databases and cloud storage instead of uploading files.
|
|
|
336
360
|
| `kafka` | Apache Kafka | `bootstrap_servers`, `topic`, `group_id` |
|
|
337
361
|
| `kinesis` | Amazon Kinesis | `stream_name`, `region`, `access_key_id`, `secret_access_key` |
|
|
338
362
|
|
|
363
|
+
> **More connectors:** AutoData supports 30+ source types. Beyond the common ones above you can also pass `oracle`, `redshift`, `cassandra`, `dynamodb`, `elasticsearch`, `clickhouse`, `timescaledb`, `synapse`, `azure_blob`, `adls`, `sap_hana`, `mqtt`, `opcua`, `influxdb`, `pi_web`, `salesforce`, `hubspot`, `stripe`, `netsuite`, `google_sheets`, `ga4`, `sharepoint`, and `http`/`rest` as `connector_type`. See **Dashboard → Documentation → Connector Data Sources** for each type's required `secrets` fields.
|
|
364
|
+
|
|
339
365
|
### Quick connector example
|
|
340
366
|
|
|
341
367
|
```python
|
|
@@ -363,7 +389,7 @@ with AutoDataClient(api_key="dtpk_...") as client:
|
|
|
363
389
|
table="orders",
|
|
364
390
|
target_columns=["price"],
|
|
365
391
|
secrets={"connection_string": "postgresql://user:pass@host/db"},
|
|
366
|
-
output_rows=
|
|
392
|
+
output_rows=10000,
|
|
367
393
|
)
|
|
368
394
|
print(result["files"])
|
|
369
395
|
```
|
|
@@ -452,7 +478,7 @@ result = client.process_from_connector(
|
|
|
452
478
|
table="orders",
|
|
453
479
|
target_columns=["price"],
|
|
454
480
|
secrets={"connection_string": "postgresql://..."},
|
|
455
|
-
output_rows=
|
|
481
|
+
output_rows=10000,
|
|
456
482
|
incremental=True, # Only fetch new rows since last sync
|
|
457
483
|
type_casts={"age": "int"}, # Override column types before processing
|
|
458
484
|
wait=True,
|
|
@@ -474,7 +500,7 @@ client.write_output(
|
|
|
474
500
|
if_exists="replace", # "replace", "append", "fail", or "merge"
|
|
475
501
|
merge_keys=["id"], # column names for merge matching (when if_exists='merge')
|
|
476
502
|
)
|
|
477
|
-
# {"success": True, "rows_written":
|
|
503
|
+
# {"success": True, "rows_written": 10000, "table": "ml_prepared_data"}
|
|
478
504
|
```
|
|
479
505
|
|
|
480
506
|
---
|
|
@@ -639,6 +665,65 @@ client.delete_scheduled_run(run_id="run-id")
|
|
|
639
665
|
|
|
640
666
|
---
|
|
641
667
|
|
|
668
|
+
## Triggers (F7)
|
|
669
|
+
|
|
670
|
+
Auto-start a pipeline when an external event fires — an inbound webhook, a connector watermark advancing, another pipeline completing, or a quality alert. Triggers are feature-flagged on the server (`FEATURE_FLAG_TRIGGERS`); if the flag is off, these endpoints return `404`.
|
|
671
|
+
|
|
672
|
+
### `client.create_trigger(name, trigger_type, config, target_pipeline_config, enabled)`
|
|
673
|
+
|
|
674
|
+
```python
|
|
675
|
+
trigger = client.create_trigger(
|
|
676
|
+
name="Reprocess on new S3 batch",
|
|
677
|
+
trigger_type="inbound_webhook", # inbound_webhook | watermark_advance | pipeline_completion | quality_alert
|
|
678
|
+
config={}, # type-specific (see table below)
|
|
679
|
+
target_pipeline_config={ # the pipeline to run when the trigger fires
|
|
680
|
+
"source_type": "s3",
|
|
681
|
+
"credential_id": "cred-uuid",
|
|
682
|
+
"table": "incoming/sales.csv",
|
|
683
|
+
"target_columns": ["revenue"],
|
|
684
|
+
"output_rows": 10000,
|
|
685
|
+
},
|
|
686
|
+
enabled=True,
|
|
687
|
+
)
|
|
688
|
+
# For inbound_webhook the secret is returned ONLY once — save it now:
|
|
689
|
+
print(trigger["webhook_secret"], trigger["inbound_url"])
|
|
690
|
+
```
|
|
691
|
+
|
|
692
|
+
**Trigger types & their `config`:**
|
|
693
|
+
|
|
694
|
+
| `trigger_type` | `config` keys |
|
|
695
|
+
|-----------------------|---------------|
|
|
696
|
+
| `inbound_webhook` | optional `{"hmac_key": "..."}` — require HMAC-SHA256-signed requests |
|
|
697
|
+
| `watermark_advance` | `{"credential_id", "table", "watermark_column", "min_advance"}` |
|
|
698
|
+
| `pipeline_completion` | `{}` — chain off another pipeline finishing |
|
|
699
|
+
| `quality_alert` | `{}` — fire when a quality-alert rule trips |
|
|
700
|
+
|
|
701
|
+
```python
|
|
702
|
+
# List / fetch / update / delete
|
|
703
|
+
triggers = client.list_triggers()
|
|
704
|
+
t = client.get_trigger(trigger_id)
|
|
705
|
+
client.update_trigger(trigger_id, enabled=False) # name | enabled | config | target_pipeline_config
|
|
706
|
+
client.delete_trigger(trigger_id)
|
|
707
|
+
|
|
708
|
+
# Dry-run: validate config + target WITHOUT firing
|
|
709
|
+
report = client.test_trigger(trigger_id)
|
|
710
|
+
# {"would_fire": True, "checks": {...}}
|
|
711
|
+
```
|
|
712
|
+
|
|
713
|
+
**Firing an inbound webhook** from your own system:
|
|
714
|
+
|
|
715
|
+
```bash
|
|
716
|
+
curl -X POST "https://autodata.datatoolpack.com/trigger/inbound/<webhook_secret>" \
|
|
717
|
+
-H "Content-Type: application/json" \
|
|
718
|
+
-d '{"any": "payload"}'
|
|
719
|
+
# If you set an hmac_key, also sign the raw body:
|
|
720
|
+
# X-Signature: sha256=<hex HMAC-SHA256 of the request body>
|
|
721
|
+
```
|
|
722
|
+
|
|
723
|
+
Inbound firing is rate-limited and HMAC-verified server-side; repeated failures auto-disable the trigger.
|
|
724
|
+
|
|
725
|
+
---
|
|
726
|
+
|
|
642
727
|
## Folder Listeners
|
|
643
728
|
|
|
644
729
|
Automatically trigger pipelines when new files appear in cloud storage:
|
|
@@ -819,6 +904,11 @@ with AutoDataClient() as client:
|
|
|
819
904
|
| `list_keys()` | List all API keys for the account |
|
|
820
905
|
| `get_usage()` | Get credit usage statistics |
|
|
821
906
|
|
|
907
|
+
### Feature Selection
|
|
908
|
+
| Method | Description |
|
|
909
|
+
|--------|-------------|
|
|
910
|
+
| `recommend_features(session_id, target_columns, ...)` | Rank a completed session's features (F4) |
|
|
911
|
+
|
|
822
912
|
### Connectors
|
|
823
913
|
| Method | Description |
|
|
824
914
|
|--------|-------------|
|
|
@@ -866,6 +956,16 @@ with AutoDataClient() as client:
|
|
|
866
956
|
| `toggle_scheduled_run(run_id)` | Enable/disable a scheduled run |
|
|
867
957
|
| `run_scheduled_now(run_id)` | Trigger a scheduled run immediately |
|
|
868
958
|
|
|
959
|
+
### Triggers
|
|
960
|
+
| Method | Description |
|
|
961
|
+
|--------|-------------|
|
|
962
|
+
| `list_triggers()` | List your triggers |
|
|
963
|
+
| `create_trigger(name, trigger_type, ...)` | Create an event trigger (F7) |
|
|
964
|
+
| `get_trigger(trigger_id)` | Fetch one trigger |
|
|
965
|
+
| `update_trigger(trigger_id, ...)` | Update a trigger |
|
|
966
|
+
| `delete_trigger(trigger_id)` | Delete a trigger |
|
|
967
|
+
| `test_trigger(trigger_id)` | Dry-run a trigger without firing |
|
|
968
|
+
|
|
869
969
|
### Folder Listeners
|
|
870
970
|
| Method | Description |
|
|
871
971
|
|--------|-------------|
|
|
@@ -7,7 +7,7 @@ with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
|
|
|
7
7
|
|
|
8
8
|
setup(
|
|
9
9
|
name="datatoolpack",
|
|
10
|
-
version="0.10.
|
|
10
|
+
version="0.10.1",
|
|
11
11
|
description="Official Python SDK for the AutoData ML data preparation pipeline API",
|
|
12
12
|
long_description=long_description,
|
|
13
13
|
long_description_content_type="text/markdown",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|