datatoolpack 0.6.0__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/PKG-INFO +1 -1
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/datatoolpack/__init__.py +1 -1
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/datatoolpack/client.py +352 -53
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/datatoolpack.egg-info/PKG-INFO +1 -1
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/setup.py +1 -1
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/README.md +0 -0
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/datatoolpack.egg-info/SOURCES.txt +0 -0
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/datatoolpack.egg-info/dependency_links.txt +0 -0
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/datatoolpack.egg-info/requires.txt +0 -0
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/datatoolpack.egg-info/top_level.txt +0 -0
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/pyproject.toml +0 -0
- {datatoolpack-0.6.0 → datatoolpack-0.7.2}/setup.cfg +0 -0
|
@@ -24,9 +24,6 @@ import time
|
|
|
24
24
|
import zipfile
|
|
25
25
|
from typing import Dict, List, Optional, Union
|
|
26
26
|
|
|
27
|
-
import requests
|
|
28
|
-
|
|
29
|
-
|
|
30
27
|
try:
|
|
31
28
|
import requests
|
|
32
29
|
except ImportError as exc: # pragma: no cover
|
|
@@ -147,10 +144,20 @@ class AutoDataClient:
|
|
|
147
144
|
# ------------------------------------------------------------------
|
|
148
145
|
|
|
149
146
|
def _url(self, path: str) -> str:
|
|
147
|
+
"""Build URL for the versioned API. Use this for everything except
|
|
148
|
+
the handful of endpoints that are only registered under the
|
|
149
|
+
unversioned ``/api/`` prefix (see :meth:`_api`).
|
|
150
|
+
"""
|
|
150
151
|
return f"{self.base_url}/api/v1{path}"
|
|
151
152
|
|
|
152
153
|
def _api(self, path: str) -> str:
|
|
153
|
-
"""Build URL for endpoints under /api/ (without v1 prefix).
|
|
154
|
+
"""Build URL for endpoints under /api/ (without v1 prefix).
|
|
155
|
+
|
|
156
|
+
Currently this is required for ``/api/scheduled-runs*`` only —
|
|
157
|
+
those routes are intentionally NOT dual-registered on /api/v1/.
|
|
158
|
+
Don't 'fix' a `_api` call to `_url` without verifying the route
|
|
159
|
+
exists under /api/v1/, or you'll silently 404.
|
|
160
|
+
"""
|
|
154
161
|
return f"{self.base_url}/api{path}"
|
|
155
162
|
|
|
156
163
|
@staticmethod
|
|
@@ -214,6 +221,7 @@ class AutoDataClient:
|
|
|
214
221
|
auto_download: bool = True,
|
|
215
222
|
output_preferences: Optional[List[str]] = None,
|
|
216
223
|
compressed: bool = True,
|
|
224
|
+
return_dataframes: bool = False,
|
|
217
225
|
) -> Dict:
|
|
218
226
|
"""Upload a data file and start the AutoData pipeline.
|
|
219
227
|
|
|
@@ -296,7 +304,7 @@ class AutoDataClient:
|
|
|
296
304
|
session_id, poll_interval=poll_interval
|
|
297
305
|
)
|
|
298
306
|
except KeyboardInterrupt:
|
|
299
|
-
print("\nInterrupted
|
|
307
|
+
print("\nInterrupted - cancelling job on server...")
|
|
300
308
|
self.cancel(session_id)
|
|
301
309
|
raise
|
|
302
310
|
|
|
@@ -307,6 +315,12 @@ class AutoDataClient:
|
|
|
307
315
|
output_preferences=output_preferences,
|
|
308
316
|
compressed=compressed,
|
|
309
317
|
)
|
|
318
|
+
# New in 0.7.0: return the output files as DataFrames so callers can
|
|
319
|
+
# skip the "download to disk, then pd.read_csv()" two-step. Gated on
|
|
320
|
+
# an opt-in flag to keep the default light (no pandas import).
|
|
321
|
+
if return_dataframes:
|
|
322
|
+
final = dict(final) # don't mutate the cached result object
|
|
323
|
+
final["dataframes"] = self.get_dataframes(session_id, names=output_preferences)
|
|
310
324
|
return final
|
|
311
325
|
|
|
312
326
|
def get_status(self, session_id: str) -> Dict:
|
|
@@ -342,7 +356,7 @@ class AutoDataClient:
|
|
|
342
356
|
self._raise_for_error(r)
|
|
343
357
|
return r.json().get("cancelled", False)
|
|
344
358
|
except AutoDataError as e:
|
|
345
|
-
print(f"Warning: cancel failed
|
|
359
|
+
print(f"Warning: cancel failed - {e}")
|
|
346
360
|
return False
|
|
347
361
|
|
|
348
362
|
def wait_for_completion(
|
|
@@ -370,7 +384,11 @@ class AutoDataClient:
|
|
|
370
384
|
AutoDataError: On processing failure, cancellation, timeout, or
|
|
371
385
|
detected stall.
|
|
372
386
|
"""
|
|
373
|
-
|
|
387
|
+
# Use ASCII-only progress output. The previous version used unicode
|
|
388
|
+
# glyphs (U+2713 check, U+2026 ellipsis, U+2014 em-dash) which crash
|
|
389
|
+
# on Windows consoles using cp1252 — the default for Python on
|
|
390
|
+
# Windows. ASCII keeps the SDK portable.
|
|
391
|
+
print(f"Waiting for session {session_id}...")
|
|
374
392
|
start = time.monotonic()
|
|
375
393
|
last_change = start
|
|
376
394
|
last_signature: tuple = ()
|
|
@@ -381,7 +399,7 @@ class AutoDataClient:
|
|
|
381
399
|
msg = data.get("message", "")
|
|
382
400
|
|
|
383
401
|
if status == "completed":
|
|
384
|
-
print(f"\r
|
|
402
|
+
print(f"\r[OK] Completed ({pct}%): {msg} ")
|
|
385
403
|
return self.get_result(session_id)
|
|
386
404
|
if status == "error":
|
|
387
405
|
raise AutoDataError(f"Processing failed: {msg}")
|
|
@@ -399,7 +417,7 @@ class AutoDataClient:
|
|
|
399
417
|
raise AutoDataError(
|
|
400
418
|
f"Processing stalled: no progress for {stall_timeout:.0f}s "
|
|
401
419
|
f"(last status={status!r}, message={msg!r}). "
|
|
402
|
-
"The worker may have crashed
|
|
420
|
+
"The worker may have crashed - please retry."
|
|
403
421
|
)
|
|
404
422
|
|
|
405
423
|
if timeout is not None and now - start > timeout:
|
|
@@ -407,7 +425,7 @@ class AutoDataClient:
|
|
|
407
425
|
f"wait_for_completion timed out after {timeout:.0f}s"
|
|
408
426
|
)
|
|
409
427
|
|
|
410
|
-
print(f"\r {pct:3d}%
|
|
428
|
+
print(f"\r {pct:3d}% - {msg[:60]:<60}", end="", flush=True)
|
|
411
429
|
time.sleep(poll_interval)
|
|
412
430
|
|
|
413
431
|
# ------------------------------------------------------------------
|
|
@@ -556,7 +574,7 @@ class AutoDataClient:
|
|
|
556
574
|
last_err = err
|
|
557
575
|
if attempt < retries:
|
|
558
576
|
backoff = 2 ** attempt
|
|
559
|
-
print(f"\nDownload attempt {attempt + 1} failed ({err}); retrying in {backoff}s
|
|
577
|
+
print(f"\nDownload attempt {attempt + 1} failed ({err}); retrying in {backoff}s...")
|
|
560
578
|
time.sleep(backoff)
|
|
561
579
|
continue
|
|
562
580
|
raise
|
|
@@ -564,6 +582,125 @@ class AutoDataClient:
|
|
|
564
582
|
if last_err:
|
|
565
583
|
raise last_err
|
|
566
584
|
|
|
585
|
+
# ------------------------------------------------------------------
|
|
586
|
+
# DataFrame-native helpers
|
|
587
|
+
# ------------------------------------------------------------------
|
|
588
|
+
# These let callers skip the "download to disk, then pd.read_csv()" two-step.
|
|
589
|
+
# pandas is imported lazily so the SDK stays light when a caller doesn't
|
|
590
|
+
# need it (the package install_requires only `requests`).
|
|
591
|
+
|
|
592
|
+
@staticmethod
|
|
593
|
+
def _require_pandas():
|
|
594
|
+
try:
|
|
595
|
+
import pandas as pd # noqa: F401
|
|
596
|
+
return pd
|
|
597
|
+
except ImportError as e:
|
|
598
|
+
raise AutoDataError(
|
|
599
|
+
"pandas is required for DataFrame helpers — install it with `pip install pandas` "
|
|
600
|
+
"or use download_file()/download_results() to get raw files instead."
|
|
601
|
+
) from e
|
|
602
|
+
|
|
603
|
+
def get_dataframe(
|
|
604
|
+
self,
|
|
605
|
+
session_id: str,
|
|
606
|
+
output_name: str,
|
|
607
|
+
*,
|
|
608
|
+
timeout: Optional[float] = None,
|
|
609
|
+
**read_csv_kwargs,
|
|
610
|
+
):
|
|
611
|
+
"""Fetch one output file and return it as a pandas DataFrame.
|
|
612
|
+
|
|
613
|
+
Args:
|
|
614
|
+
session_id: Completed processing session.
|
|
615
|
+
output_name: File base name (e.g. ``"dsg_output"``, ``"mdh_output"``)
|
|
616
|
+
or the full filename (``"dsg_output.csv"``). ``.csv``
|
|
617
|
+
is appended if missing.
|
|
618
|
+
timeout: Per-request timeout; defaults to the SDK's normal
|
|
619
|
+
download timeout (15 min).
|
|
620
|
+
read_csv_kwargs: Forwarded to ``pandas.read_csv`` (e.g. ``dtype``,
|
|
621
|
+
``parse_dates``, ``usecols``).
|
|
622
|
+
|
|
623
|
+
Returns:
|
|
624
|
+
A pandas.DataFrame.
|
|
625
|
+
|
|
626
|
+
Raises:
|
|
627
|
+
AutoDataError: if pandas isn't installed, the file is missing or
|
|
628
|
+
empty (server failed to generate it), or the
|
|
629
|
+
download was truncated.
|
|
630
|
+
"""
|
|
631
|
+
pd = self._require_pandas()
|
|
632
|
+
fname = output_name if output_name.lower().endswith(
|
|
633
|
+
(".csv", ".json", ".parquet", ".feather", ".orc")
|
|
634
|
+
) else f"{output_name}.csv"
|
|
635
|
+
url = self._url(f"/download/{session_id}/{fname}")
|
|
636
|
+
if not url.startswith("http"):
|
|
637
|
+
url = f"{self.base_url}{url}"
|
|
638
|
+
r = self._request(
|
|
639
|
+
"GET", url, stream=True,
|
|
640
|
+
timeout=timeout if timeout is not None else max(self.timeout, 900),
|
|
641
|
+
)
|
|
642
|
+
self._raise_for_error(r)
|
|
643
|
+
cl = r.headers.get("Content-Length")
|
|
644
|
+
if cl == "0":
|
|
645
|
+
raise AutoDataError(
|
|
646
|
+
f"Server returned an empty file for {fname} — the output likely "
|
|
647
|
+
"failed to generate. Retry the run or check /result/<session_id>."
|
|
648
|
+
)
|
|
649
|
+
# Buffer into memory — pandas.read_csv needs a seekable stream and
|
|
650
|
+
# output files are bounded by the server-side row ceiling anyway.
|
|
651
|
+
body = r.content
|
|
652
|
+
if not body:
|
|
653
|
+
raise AutoDataError(f"Server returned 0 bytes for {fname}")
|
|
654
|
+
import io as _io
|
|
655
|
+
buf = _io.BytesIO(body)
|
|
656
|
+
low = fname.lower()
|
|
657
|
+
if low.endswith(".parquet"):
|
|
658
|
+
return pd.read_parquet(buf)
|
|
659
|
+
if low.endswith(".feather"):
|
|
660
|
+
return pd.read_feather(buf)
|
|
661
|
+
if low.endswith(".orc"):
|
|
662
|
+
return pd.read_orc(buf)
|
|
663
|
+
if low.endswith(".json"):
|
|
664
|
+
return pd.read_json(buf, **read_csv_kwargs)
|
|
665
|
+
return pd.read_csv(buf, **read_csv_kwargs)
|
|
666
|
+
|
|
667
|
+
def get_dataframes(
|
|
668
|
+
self,
|
|
669
|
+
session_id: str,
|
|
670
|
+
names: Optional[List[str]] = None,
|
|
671
|
+
*,
|
|
672
|
+
timeout: Optional[float] = None,
|
|
673
|
+
) -> Dict:
|
|
674
|
+
"""Fetch multiple output files and return a ``{name: DataFrame}`` dict.
|
|
675
|
+
|
|
676
|
+
Args:
|
|
677
|
+
session_id: Completed processing session.
|
|
678
|
+
names: Output base names to fetch. ``None`` means every file
|
|
679
|
+
the server lists for the session (skipping PDF
|
|
680
|
+
reports and .json metadata files, which aren't
|
|
681
|
+
naturally a DataFrame).
|
|
682
|
+
timeout: Per-file timeout; see ``get_dataframe``.
|
|
683
|
+
|
|
684
|
+
Returns:
|
|
685
|
+
Dict of ``{file_stem: pandas.DataFrame}``.
|
|
686
|
+
"""
|
|
687
|
+
self._require_pandas()
|
|
688
|
+
if names is None:
|
|
689
|
+
result = self.get_result(session_id)
|
|
690
|
+
names = []
|
|
691
|
+
for f in result.get("files", []):
|
|
692
|
+
fn = f.get("name", "")
|
|
693
|
+
low = fn.lower()
|
|
694
|
+
if not low or low.endswith((".pdf", ".json", ".md")):
|
|
695
|
+
continue
|
|
696
|
+
names.append(fn)
|
|
697
|
+
out: Dict = {}
|
|
698
|
+
for n in names:
|
|
699
|
+
# Strip the extension so callers can access via the logical name.
|
|
700
|
+
key = n.rsplit(".", 1)[0] if "." in n else n
|
|
701
|
+
out[key] = self.get_dataframe(session_id, n, timeout=timeout)
|
|
702
|
+
return out
|
|
703
|
+
|
|
567
704
|
# ------------------------------------------------------------------
|
|
568
705
|
# Account / API key management
|
|
569
706
|
# ------------------------------------------------------------------
|
|
@@ -604,9 +741,25 @@ class AutoDataClient:
|
|
|
604
741
|
"""Test connectivity to an external data source.
|
|
605
742
|
|
|
606
743
|
Args:
|
|
607
|
-
connector_type: One of
|
|
608
|
-
|
|
609
|
-
|
|
744
|
+
connector_type: One of the supported types:
|
|
745
|
+
|
|
746
|
+
Databases / warehouses:
|
|
747
|
+
``sql`` (Postgres/MySQL/MSSQL via SQLAlchemy URL),
|
|
748
|
+
``snowflake``, ``bigquery``, ``redshift``, ``oracle``,
|
|
749
|
+
``databricks``, ``fabric`` (Microsoft Fabric).
|
|
750
|
+
|
|
751
|
+
NoSQL:
|
|
752
|
+
``mongodb``, ``cassandra``, ``dynamodb``, ``elasticsearch``.
|
|
753
|
+
|
|
754
|
+
Object storage / data lakes:
|
|
755
|
+
``s3``, ``gcs``, ``azure_blob``, ``adls_gen2`` (Azure
|
|
756
|
+
Data Lake Storage Gen2), ``delta`` (Delta Lake).
|
|
757
|
+
|
|
758
|
+
Streaming:
|
|
759
|
+
``kafka``, ``kinesis``.
|
|
760
|
+
|
|
761
|
+
Generic:
|
|
762
|
+
``http`` (REST/HTTP API), ``mock`` (test).
|
|
610
763
|
secrets: Inline connection secrets (e.g. connection_string,
|
|
611
764
|
bucket, access keys).
|
|
612
765
|
credential_id: ID of a saved credential on the server
|
|
@@ -634,10 +787,24 @@ class AutoDataClient:
|
|
|
634
787
|
secrets: Optional[Dict] = None,
|
|
635
788
|
credential_id: Optional[str] = None,
|
|
636
789
|
) -> List[str]:
|
|
637
|
-
"""List available tables / files in a data source.
|
|
790
|
+
"""List available tables / files / indexes / topics in a data source.
|
|
791
|
+
|
|
792
|
+
What "discover" returns is dialect-specific:
|
|
793
|
+
* SQL warehouses (``sql``, ``snowflake``, ``redshift``, ``oracle``,
|
|
794
|
+
``databricks``, ``fabric``, ``bigquery``) → table + view names.
|
|
795
|
+
* NoSQL (``mongodb``, ``cassandra``, ``dynamodb``,
|
|
796
|
+
``elasticsearch``) → collection / table / index names.
|
|
797
|
+
* Object stores (``s3``, ``gcs``, ``azure_blob``, ``adls_gen2``)
|
|
798
|
+
→ object keys under ``prefix`` (capped at 500), or bucket /
|
|
799
|
+
container names if no bucket is set on the credential.
|
|
800
|
+
* Streaming (``kafka``, ``kinesis``) → topic / stream names.
|
|
801
|
+
* ``http`` → a single virtual table (the URL itself).
|
|
802
|
+
* ``delta`` → the table's path.
|
|
803
|
+
|
|
804
|
+
See :meth:`test_connector` for the full list of supported types.
|
|
638
805
|
|
|
639
806
|
Returns:
|
|
640
|
-
List of
|
|
807
|
+
List of object names (strings).
|
|
641
808
|
"""
|
|
642
809
|
body: Dict = {}
|
|
643
810
|
if credential_id:
|
|
@@ -660,7 +827,26 @@ class AutoDataClient:
|
|
|
660
827
|
credential_id: Optional[str] = None,
|
|
661
828
|
custom_query: Optional[str] = None,
|
|
662
829
|
) -> Dict:
|
|
663
|
-
"""Preview columns and row count of a table / file.
|
|
830
|
+
"""Preview columns and row count of a table / file / index.
|
|
831
|
+
|
|
832
|
+
Notes per connector family:
|
|
833
|
+
* SQL warehouses honour ``custom_query`` (read-only SELECT/WITH
|
|
834
|
+
only — INSERT/UPDATE/DELETE are rejected at the server).
|
|
835
|
+
* Object stores (``s3``, ``gcs``, ``azure_blob``, ``adls_gen2``)
|
|
836
|
+
auto-detect file format from extension: ``.csv`` reads the
|
|
837
|
+
header row, ``.parquet`` reads schema, ``.json``/``.jsonl``
|
|
838
|
+
samples the first record.
|
|
839
|
+
* Cassandra ``row_count`` is intentionally ``None`` — COUNT(*)
|
|
840
|
+
requires ALLOW FILTERING and is too expensive for a preview.
|
|
841
|
+
* DynamoDB returns key attributes from describe_table plus any
|
|
842
|
+
extras found in a single Scan(Limit=1) sample.
|
|
843
|
+
* Elasticsearch samples one document to extract the actual
|
|
844
|
+
field set (mappings alone miss dynamic fields).
|
|
845
|
+
* HTTP auto-detects JSON envelopes; pass ``json_path`` in the
|
|
846
|
+
credential's secrets to point at a nested array (e.g.
|
|
847
|
+
``"data.results"``).
|
|
848
|
+
|
|
849
|
+
See :meth:`test_connector` for the full list of supported types.
|
|
664
850
|
|
|
665
851
|
Returns:
|
|
666
852
|
Dict with ``columns`` (list[str]) and ``row_count`` (int or None).
|
|
@@ -699,6 +885,7 @@ class AutoDataClient:
|
|
|
699
885
|
compressed: bool = True,
|
|
700
886
|
incremental: bool = False,
|
|
701
887
|
type_casts: Optional[Dict[str, str]] = None,
|
|
888
|
+
return_dataframes: bool = False,
|
|
702
889
|
) -> Dict:
|
|
703
890
|
"""Run the AutoData pipeline on data from a connector source.
|
|
704
891
|
|
|
@@ -788,7 +975,7 @@ class AutoDataClient:
|
|
|
788
975
|
try:
|
|
789
976
|
final = self.wait_for_completion(session_id, poll_interval=poll_interval)
|
|
790
977
|
except KeyboardInterrupt:
|
|
791
|
-
print("\nInterrupted
|
|
978
|
+
print("\nInterrupted - cancelling job on server...")
|
|
792
979
|
self.cancel(session_id)
|
|
793
980
|
raise
|
|
794
981
|
|
|
@@ -799,6 +986,9 @@ class AutoDataClient:
|
|
|
799
986
|
output_preferences=output_preferences,
|
|
800
987
|
compressed=compressed,
|
|
801
988
|
)
|
|
989
|
+
if return_dataframes:
|
|
990
|
+
final = dict(final)
|
|
991
|
+
final["dataframes"] = self.get_dataframes(session_id, names=output_preferences)
|
|
802
992
|
return final
|
|
803
993
|
|
|
804
994
|
def write_output(
|
|
@@ -865,10 +1055,19 @@ class AutoDataClient:
|
|
|
865
1055
|
) -> Dict:
|
|
866
1056
|
"""Save a new credential on the server for later reuse.
|
|
867
1057
|
|
|
1058
|
+
The server validates ``secrets`` at save-time against the connector
|
|
1059
|
+
type's required-field schema (e.g. for ``oracle`` it enforces that
|
|
1060
|
+
you supplied a connection_string OR host+user+password+service_name).
|
|
1061
|
+
A bad credential raises :class:`AutoDataError` with a clear message
|
|
1062
|
+
instead of failing later in a polling loop.
|
|
1063
|
+
|
|
868
1064
|
Args:
|
|
869
1065
|
name: Human-readable label.
|
|
870
|
-
connector_type:
|
|
871
|
-
|
|
1066
|
+
connector_type: One of the supported connector types — see
|
|
1067
|
+
:meth:`test_connector` for the full list.
|
|
1068
|
+
secrets: Connection secrets dict. Required fields vary
|
|
1069
|
+
by connector type; see the dashboard's quick-
|
|
1070
|
+
connect form for the exact field names.
|
|
872
1071
|
|
|
873
1072
|
Returns:
|
|
874
1073
|
Dict with the saved credential metadata including ``id``.
|
|
@@ -1281,43 +1480,112 @@ class AutoDataClient:
|
|
|
1281
1480
|
self,
|
|
1282
1481
|
name: str,
|
|
1283
1482
|
source_type: str,
|
|
1284
|
-
|
|
1285
|
-
y_columns: Union[str, List[str]],
|
|
1483
|
+
folder_path: Optional[str] = None,
|
|
1484
|
+
y_columns: Optional[Union[str, List[str]]] = None,
|
|
1485
|
+
*,
|
|
1486
|
+
bucket_name: Optional[str] = None,
|
|
1487
|
+
prefix: Optional[str] = None,
|
|
1286
1488
|
credential_id: Optional[str] = None,
|
|
1287
|
-
secrets: Optional[Dict] = None,
|
|
1288
1489
|
pipeline_config: Optional[Dict] = None,
|
|
1289
1490
|
output_rows: int = 10000,
|
|
1491
|
+
poll_interval_seconds: int = 60,
|
|
1492
|
+
allowed_extensions: str = ".csv,.parquet,.json",
|
|
1493
|
+
max_file_size_mb: int = 500,
|
|
1494
|
+
cooldown_seconds: int = 60,
|
|
1495
|
+
max_files_per_hour: int = 10,
|
|
1496
|
+
enabled: bool = True,
|
|
1497
|
+
auto_sink_config: Optional[Dict] = None,
|
|
1498
|
+
watch_path: Optional[str] = None, # deprecated alias of folder_path
|
|
1290
1499
|
) -> Dict:
|
|
1291
|
-
"""Create a folder listener
|
|
1500
|
+
"""Create a folder listener that watches a source for new files and
|
|
1501
|
+
triggers the AutoData pipeline on each one.
|
|
1292
1502
|
|
|
1293
1503
|
Args:
|
|
1294
|
-
name:
|
|
1295
|
-
source_type:
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1504
|
+
name: Human-readable listener name.
|
|
1505
|
+
source_type: ``'local'``, ``'s3'``, ``'gcs'``,
|
|
1506
|
+
``'azure_blob'``, ``'adls_gen2'``, or
|
|
1507
|
+
``'sftp'``. (Database-style connectors
|
|
1508
|
+
like ``oracle``/``cassandra`` aren't
|
|
1509
|
+
listener sources — use a scheduled run
|
|
1510
|
+
with a watermark instead.)
|
|
1511
|
+
folder_path: Local path to watch. Required for
|
|
1512
|
+
``source_type='local'``. For cloud sources
|
|
1513
|
+
it's auto-derived from ``bucket_name``.
|
|
1514
|
+
For ``'sftp'`` it's auto-set to the user's
|
|
1515
|
+
inbox directory.
|
|
1516
|
+
y_columns: Target column name(s) for ML. Required for
|
|
1517
|
+
the listener to actually fire the pipeline
|
|
1518
|
+
(the server stores it but won't trigger
|
|
1519
|
+
without it).
|
|
1520
|
+
bucket_name: Bucket / container name. **Required** for
|
|
1521
|
+
s3 / gcs / azure_blob sources.
|
|
1522
|
+
prefix: Optional object prefix filter for cloud
|
|
1523
|
+
sources.
|
|
1524
|
+
credential_id: ID of a saved credential on the server.
|
|
1525
|
+
**Required** for cloud and SFTP sources.
|
|
1526
|
+
Use :meth:`save_credential` first.
|
|
1527
|
+
pipeline_config: Full pipeline config dict. Recognised keys
|
|
1528
|
+
include ``output_number``, ``text_mode``,
|
|
1529
|
+
``text_cleaning``, ``mdh_mode``,
|
|
1530
|
+
``excluded_columns``, ``task_types``,
|
|
1531
|
+
``zscore_limit``, ``similarity_p``,
|
|
1532
|
+
``dsg_mode``, ``enable_anomaly_detection``,
|
|
1533
|
+
``enable_dtc``/``mdh``/``dor``/``cds``/``dsm``/``dsg``,
|
|
1534
|
+
``webhook_ids``.
|
|
1535
|
+
output_rows: Convenience for ``pipeline_config['output_number']``.
|
|
1536
|
+
Ignored if ``pipeline_config`` already sets it.
|
|
1537
|
+
poll_interval_seconds: How often the watcher checks for new files
|
|
1538
|
+
(default 60).
|
|
1539
|
+
allowed_extensions: Comma-separated list, e.g.
|
|
1540
|
+
``'.csv,.parquet'``.
|
|
1541
|
+
max_file_size_mb: Skip files larger than this (default 500).
|
|
1542
|
+
cooldown_seconds: Min seconds between pipeline triggers
|
|
1543
|
+
(default 60).
|
|
1544
|
+
max_files_per_hour: Hard cap on triggers per hour (default 10).
|
|
1545
|
+
enabled: Start watching immediately (default True).
|
|
1546
|
+
auto_sink_config: Optional ``{enabled, connector_type,
|
|
1547
|
+
credential_id, table_name, if_exists}``
|
|
1548
|
+
to auto-write each output to a sink.
|
|
1303
1549
|
|
|
1304
1550
|
Returns:
|
|
1305
|
-
Dict with the created listener metadata
|
|
1551
|
+
Dict with the created listener metadata under ``listener``.
|
|
1306
1552
|
"""
|
|
1307
|
-
|
|
1553
|
+
# Backwards compat: accept the old `watch_path` kwarg.
|
|
1554
|
+
if folder_path is None and watch_path is not None:
|
|
1555
|
+
folder_path = watch_path
|
|
1556
|
+
if y_columns is None:
|
|
1557
|
+
raise ValueError("y_columns is required to create a folder listener")
|
|
1558
|
+
|
|
1559
|
+
y_cols = [y_columns] if isinstance(y_columns, str) else list(y_columns)
|
|
1560
|
+
|
|
1561
|
+
# Fold output_rows into pipeline_config — the server's watcher reads
|
|
1562
|
+
# `output_number` from there, not from a top-level `output_rows` key.
|
|
1563
|
+
cfg = dict(pipeline_config or {})
|
|
1564
|
+
cfg.setdefault("output_number", str(output_rows))
|
|
1565
|
+
|
|
1308
1566
|
body: Dict = {
|
|
1309
1567
|
"name": name,
|
|
1310
1568
|
"source_type": source_type,
|
|
1311
|
-
"watch_path": watch_path,
|
|
1312
1569
|
"y_columns": y_cols,
|
|
1313
|
-
"
|
|
1570
|
+
"pipeline_config": cfg,
|
|
1571
|
+
"poll_interval_seconds": poll_interval_seconds,
|
|
1572
|
+
"allowed_extensions": allowed_extensions,
|
|
1573
|
+
"max_file_size_mb": max_file_size_mb,
|
|
1574
|
+
"cooldown_seconds": cooldown_seconds,
|
|
1575
|
+
"max_files_per_hour": max_files_per_hour,
|
|
1576
|
+
"enabled": enabled,
|
|
1314
1577
|
}
|
|
1578
|
+
if folder_path:
|
|
1579
|
+
body["folder_path"] = folder_path
|
|
1580
|
+
if bucket_name:
|
|
1581
|
+
body["bucket_name"] = bucket_name
|
|
1582
|
+
if prefix:
|
|
1583
|
+
body["prefix"] = prefix
|
|
1315
1584
|
if credential_id:
|
|
1316
1585
|
body["credential_id"] = credential_id
|
|
1317
|
-
if
|
|
1318
|
-
body["
|
|
1319
|
-
|
|
1320
|
-
body["pipeline_config"] = pipeline_config
|
|
1586
|
+
if auto_sink_config is not None:
|
|
1587
|
+
body["auto_sink_config"] = auto_sink_config
|
|
1588
|
+
|
|
1321
1589
|
r = self._request("POST", self._url("/listeners/folder"), json=body)
|
|
1322
1590
|
self._raise_for_error(r)
|
|
1323
1591
|
return r.json()
|
|
@@ -1325,34 +1593,65 @@ class AutoDataClient:
|
|
|
1325
1593
|
def update_listener(
|
|
1326
1594
|
self,
|
|
1327
1595
|
listener_id: str,
|
|
1596
|
+
*,
|
|
1328
1597
|
name: Optional[str] = None,
|
|
1329
|
-
|
|
1598
|
+
folder_path: Optional[str] = None,
|
|
1599
|
+
y_columns: Optional[Union[str, List[str]]] = None,
|
|
1600
|
+
bucket_name: Optional[str] = None,
|
|
1601
|
+
prefix: Optional[str] = None,
|
|
1602
|
+
credential_id: Optional[str] = None,
|
|
1330
1603
|
pipeline_config: Optional[Dict] = None,
|
|
1604
|
+
poll_interval_seconds: Optional[int] = None,
|
|
1605
|
+
allowed_extensions: Optional[str] = None,
|
|
1606
|
+
max_file_size_mb: Optional[int] = None,
|
|
1607
|
+
cooldown_seconds: Optional[int] = None,
|
|
1608
|
+
max_files_per_hour: Optional[int] = None,
|
|
1331
1609
|
enabled: Optional[bool] = None,
|
|
1610
|
+
auto_sink_config: Optional[Dict] = None,
|
|
1611
|
+
watch_path: Optional[str] = None, # deprecated alias of folder_path
|
|
1332
1612
|
) -> Dict:
|
|
1333
1613
|
"""Update a folder listener.
|
|
1334
1614
|
|
|
1335
1615
|
Only provided fields are updated; omitted fields remain unchanged.
|
|
1616
|
+
Setting ``enabled`` toggles the background watcher thread.
|
|
1336
1617
|
|
|
1337
|
-
|
|
1338
|
-
listener_id: ID of the listener to update.
|
|
1339
|
-
name: New listener name.
|
|
1340
|
-
watch_path: New folder path or URI to watch.
|
|
1341
|
-
pipeline_config: Updated pipeline tool toggles.
|
|
1342
|
-
enabled: Enable or disable the listener.
|
|
1343
|
-
|
|
1344
|
-
Returns:
|
|
1345
|
-
Dict with the updated listener metadata.
|
|
1618
|
+
See :meth:`create_listener` for field semantics.
|
|
1346
1619
|
"""
|
|
1620
|
+
if folder_path is None and watch_path is not None:
|
|
1621
|
+
folder_path = watch_path
|
|
1622
|
+
|
|
1347
1623
|
body: Dict = {}
|
|
1348
1624
|
if name is not None:
|
|
1349
1625
|
body["name"] = name
|
|
1350
|
-
if
|
|
1351
|
-
body["
|
|
1626
|
+
if folder_path is not None:
|
|
1627
|
+
body["folder_path"] = folder_path
|
|
1628
|
+
if y_columns is not None:
|
|
1629
|
+
body["y_columns"] = (
|
|
1630
|
+
[y_columns] if isinstance(y_columns, str) else list(y_columns)
|
|
1631
|
+
)
|
|
1632
|
+
if bucket_name is not None:
|
|
1633
|
+
body["bucket_name"] = bucket_name
|
|
1634
|
+
if prefix is not None:
|
|
1635
|
+
body["prefix"] = prefix
|
|
1636
|
+
if credential_id is not None:
|
|
1637
|
+
body["credential_id"] = credential_id
|
|
1352
1638
|
if pipeline_config is not None:
|
|
1353
1639
|
body["pipeline_config"] = pipeline_config
|
|
1640
|
+
if poll_interval_seconds is not None:
|
|
1641
|
+
body["poll_interval_seconds"] = poll_interval_seconds
|
|
1642
|
+
if allowed_extensions is not None:
|
|
1643
|
+
body["allowed_extensions"] = allowed_extensions
|
|
1644
|
+
if max_file_size_mb is not None:
|
|
1645
|
+
body["max_file_size_mb"] = max_file_size_mb
|
|
1646
|
+
if cooldown_seconds is not None:
|
|
1647
|
+
body["cooldown_seconds"] = cooldown_seconds
|
|
1648
|
+
if max_files_per_hour is not None:
|
|
1649
|
+
body["max_files_per_hour"] = max_files_per_hour
|
|
1354
1650
|
if enabled is not None:
|
|
1355
1651
|
body["enabled"] = enabled
|
|
1652
|
+
if auto_sink_config is not None:
|
|
1653
|
+
body["auto_sink_config"] = auto_sink_config
|
|
1654
|
+
|
|
1356
1655
|
r = self._request(
|
|
1357
1656
|
"PUT", self._url(f"/listeners/folder/{listener_id}"), json=body
|
|
1358
1657
|
)
|
|
@@ -7,7 +7,7 @@ with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
|
|
|
7
7
|
|
|
8
8
|
setup(
|
|
9
9
|
name="datatoolpack",
|
|
10
|
-
version="0.
|
|
10
|
+
version="0.7.2",
|
|
11
11
|
description="Official Python SDK for the AutoData ML data preparation pipeline API",
|
|
12
12
|
long_description=long_description,
|
|
13
13
|
long_description_content_type="text/markdown",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|