datatoolpack 0.6.0__tar.gz → 0.7.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datatoolpack
3
- Version: 0.6.0
3
+ Version: 0.7.2
4
4
  Summary: Official Python SDK for the AutoData ML data preparation pipeline API
5
5
  Home-page: https://autodata.datatoolpack.com
6
6
  Author: AutoData Team
@@ -1,4 +1,4 @@
1
1
  from .client import AutoDataClient, AutoDataError
2
2
 
3
- __version__ = "0.6.0"
3
+ __version__ = "0.7.2"
4
4
  __all__ = ["AutoDataClient", "AutoDataError", "__version__"]
@@ -24,9 +24,6 @@ import time
24
24
  import zipfile
25
25
  from typing import Dict, List, Optional, Union
26
26
 
27
- import requests
28
-
29
-
30
27
  try:
31
28
  import requests
32
29
  except ImportError as exc: # pragma: no cover
@@ -147,10 +144,20 @@ class AutoDataClient:
147
144
  # ------------------------------------------------------------------
148
145
 
149
146
  def _url(self, path: str) -> str:
147
+ """Build URL for the versioned API. Use this for everything except
148
+ the handful of endpoints that are only registered under the
149
+ unversioned ``/api/`` prefix (see :meth:`_api`).
150
+ """
150
151
  return f"{self.base_url}/api/v1{path}"
151
152
 
152
153
  def _api(self, path: str) -> str:
153
- """Build URL for endpoints under /api/ (without v1 prefix)."""
154
+ """Build URL for endpoints under /api/ (without v1 prefix).
155
+
156
+ Currently this is required for ``/api/scheduled-runs*`` only —
157
+ those routes are intentionally NOT dual-registered on /api/v1/.
158
+ Don't 'fix' a `_api` call to `_url` without verifying the route
159
+ exists under /api/v1/, or you'll silently 404.
160
+ """
154
161
  return f"{self.base_url}/api{path}"
155
162
 
156
163
  @staticmethod
@@ -214,6 +221,7 @@ class AutoDataClient:
214
221
  auto_download: bool = True,
215
222
  output_preferences: Optional[List[str]] = None,
216
223
  compressed: bool = True,
224
+ return_dataframes: bool = False,
217
225
  ) -> Dict:
218
226
  """Upload a data file and start the AutoData pipeline.
219
227
 
@@ -296,7 +304,7 @@ class AutoDataClient:
296
304
  session_id, poll_interval=poll_interval
297
305
  )
298
306
  except KeyboardInterrupt:
299
- print("\nInterrupted cancelling job on server")
307
+ print("\nInterrupted - cancelling job on server...")
300
308
  self.cancel(session_id)
301
309
  raise
302
310
 
@@ -307,6 +315,12 @@ class AutoDataClient:
307
315
  output_preferences=output_preferences,
308
316
  compressed=compressed,
309
317
  )
318
+ # New in 0.7.0: return the output files as DataFrames so callers can
319
+ # skip the "download to disk, then pd.read_csv()" two-step. Gated on
320
+ # an opt-in flag to keep the default light (no pandas import).
321
+ if return_dataframes:
322
+ final = dict(final) # don't mutate the cached result object
323
+ final["dataframes"] = self.get_dataframes(session_id, names=output_preferences)
310
324
  return final
311
325
 
312
326
  def get_status(self, session_id: str) -> Dict:
@@ -342,7 +356,7 @@ class AutoDataClient:
342
356
  self._raise_for_error(r)
343
357
  return r.json().get("cancelled", False)
344
358
  except AutoDataError as e:
345
- print(f"Warning: cancel failed {e}")
359
+ print(f"Warning: cancel failed - {e}")
346
360
  return False
347
361
 
348
362
  def wait_for_completion(
@@ -370,7 +384,11 @@ class AutoDataClient:
370
384
  AutoDataError: On processing failure, cancellation, timeout, or
371
385
  detected stall.
372
386
  """
373
- print(f"Waiting for session {session_id}…")
387
+ # Use ASCII-only progress output. The previous version used unicode
388
+ # glyphs (U+2713 check, U+2026 ellipsis, U+2014 em-dash) which crash
389
+ # on Windows consoles using cp1252 — the default for Python on
390
+ # Windows. ASCII keeps the SDK portable.
391
+ print(f"Waiting for session {session_id}...")
374
392
  start = time.monotonic()
375
393
  last_change = start
376
394
  last_signature: tuple = ()
@@ -381,7 +399,7 @@ class AutoDataClient:
381
399
  msg = data.get("message", "")
382
400
 
383
401
  if status == "completed":
384
- print(f"\r Completed ({pct}%): {msg} ")
402
+ print(f"\r[OK] Completed ({pct}%): {msg} ")
385
403
  return self.get_result(session_id)
386
404
  if status == "error":
387
405
  raise AutoDataError(f"Processing failed: {msg}")
@@ -399,7 +417,7 @@ class AutoDataClient:
399
417
  raise AutoDataError(
400
418
  f"Processing stalled: no progress for {stall_timeout:.0f}s "
401
419
  f"(last status={status!r}, message={msg!r}). "
402
- "The worker may have crashed please retry."
420
+ "The worker may have crashed - please retry."
403
421
  )
404
422
 
405
423
  if timeout is not None and now - start > timeout:
@@ -407,7 +425,7 @@ class AutoDataClient:
407
425
  f"wait_for_completion timed out after {timeout:.0f}s"
408
426
  )
409
427
 
410
- print(f"\r {pct:3d}% {msg[:60]:<60}", end="", flush=True)
428
+ print(f"\r {pct:3d}% - {msg[:60]:<60}", end="", flush=True)
411
429
  time.sleep(poll_interval)
412
430
 
413
431
  # ------------------------------------------------------------------
@@ -556,7 +574,7 @@ class AutoDataClient:
556
574
  last_err = err
557
575
  if attempt < retries:
558
576
  backoff = 2 ** attempt
559
- print(f"\nDownload attempt {attempt + 1} failed ({err}); retrying in {backoff}s")
577
+ print(f"\nDownload attempt {attempt + 1} failed ({err}); retrying in {backoff}s...")
560
578
  time.sleep(backoff)
561
579
  continue
562
580
  raise
@@ -564,6 +582,125 @@ class AutoDataClient:
564
582
  if last_err:
565
583
  raise last_err
566
584
 
585
+ # ------------------------------------------------------------------
586
+ # DataFrame-native helpers
587
+ # ------------------------------------------------------------------
588
+ # These let callers skip the "download to disk, then pd.read_csv()" two-step.
589
+ # pandas is imported lazily so the SDK stays light when a caller doesn't
590
+ # need it (the package install_requires only `requests`).
591
+
592
+ @staticmethod
593
+ def _require_pandas():
594
+ try:
595
+ import pandas as pd # noqa: F401
596
+ return pd
597
+ except ImportError as e:
598
+ raise AutoDataError(
599
+ "pandas is required for DataFrame helpers — install it with `pip install pandas` "
600
+ "or use download_file()/download_results() to get raw files instead."
601
+ ) from e
602
+
603
+ def get_dataframe(
604
+ self,
605
+ session_id: str,
606
+ output_name: str,
607
+ *,
608
+ timeout: Optional[float] = None,
609
+ **read_csv_kwargs,
610
+ ):
611
+ """Fetch one output file and return it as a pandas DataFrame.
612
+
613
+ Args:
614
+ session_id: Completed processing session.
615
+ output_name: File base name (e.g. ``"dsg_output"``, ``"mdh_output"``)
616
+ or the full filename (``"dsg_output.csv"``). ``.csv``
617
+ is appended if missing.
618
+ timeout: Per-request timeout; defaults to the SDK's normal
619
+ download timeout (15 min).
620
+ read_csv_kwargs: Forwarded to ``pandas.read_csv`` (e.g. ``dtype``,
621
+ ``parse_dates``, ``usecols``).
622
+
623
+ Returns:
624
+ A pandas.DataFrame.
625
+
626
+ Raises:
627
+ AutoDataError: if pandas isn't installed, the file is missing or
628
+ empty (server failed to generate it), or the
629
+ download was truncated.
630
+ """
631
+ pd = self._require_pandas()
632
+ fname = output_name if output_name.lower().endswith(
633
+ (".csv", ".json", ".parquet", ".feather", ".orc")
634
+ ) else f"{output_name}.csv"
635
+ url = self._url(f"/download/{session_id}/{fname}")
636
+ if not url.startswith("http"):
637
+ url = f"{self.base_url}{url}"
638
+ r = self._request(
639
+ "GET", url, stream=True,
640
+ timeout=timeout if timeout is not None else max(self.timeout, 900),
641
+ )
642
+ self._raise_for_error(r)
643
+ cl = r.headers.get("Content-Length")
644
+ if cl == "0":
645
+ raise AutoDataError(
646
+ f"Server returned an empty file for {fname} — the output likely "
647
+ "failed to generate. Retry the run or check /result/<session_id>."
648
+ )
649
+ # Buffer into memory — pandas.read_csv needs a seekable stream and
650
+ # output files are bounded by the server-side row ceiling anyway.
651
+ body = r.content
652
+ if not body:
653
+ raise AutoDataError(f"Server returned 0 bytes for {fname}")
654
+ import io as _io
655
+ buf = _io.BytesIO(body)
656
+ low = fname.lower()
657
+ if low.endswith(".parquet"):
658
+ return pd.read_parquet(buf)
659
+ if low.endswith(".feather"):
660
+ return pd.read_feather(buf)
661
+ if low.endswith(".orc"):
662
+ return pd.read_orc(buf)
663
+ if low.endswith(".json"):
664
+ return pd.read_json(buf, **read_csv_kwargs)
665
+ return pd.read_csv(buf, **read_csv_kwargs)
666
+
667
+ def get_dataframes(
668
+ self,
669
+ session_id: str,
670
+ names: Optional[List[str]] = None,
671
+ *,
672
+ timeout: Optional[float] = None,
673
+ ) -> Dict:
674
+ """Fetch multiple output files and return a ``{name: DataFrame}`` dict.
675
+
676
+ Args:
677
+ session_id: Completed processing session.
678
+ names: Output base names to fetch. ``None`` means every file
679
+ the server lists for the session (skipping PDF
680
+ reports and .json metadata files, which aren't
681
+ naturally a DataFrame).
682
+ timeout: Per-file timeout; see ``get_dataframe``.
683
+
684
+ Returns:
685
+ Dict of ``{file_stem: pandas.DataFrame}``.
686
+ """
687
+ self._require_pandas()
688
+ if names is None:
689
+ result = self.get_result(session_id)
690
+ names = []
691
+ for f in result.get("files", []):
692
+ fn = f.get("name", "")
693
+ low = fn.lower()
694
+ if not low or low.endswith((".pdf", ".json", ".md")):
695
+ continue
696
+ names.append(fn)
697
+ out: Dict = {}
698
+ for n in names:
699
+ # Strip the extension so callers can access via the logical name.
700
+ key = n.rsplit(".", 1)[0] if "." in n else n
701
+ out[key] = self.get_dataframe(session_id, n, timeout=timeout)
702
+ return out
703
+
567
704
  # ------------------------------------------------------------------
568
705
  # Account / API key management
569
706
  # ------------------------------------------------------------------
@@ -604,9 +741,25 @@ class AutoDataClient:
604
741
  """Test connectivity to an external data source.
605
742
 
606
743
  Args:
607
- connector_type: One of ``sql``, ``snowflake``, ``bigquery``,
608
- ``mongodb``, ``s3``, ``gcs``, ``databricks``,
609
- ``delta``, ``fabric``, ``kafka``, ``kinesis``.
744
+ connector_type: One of the supported types:
745
+
746
+ Databases / warehouses:
747
+ ``sql`` (Postgres/MySQL/MSSQL via SQLAlchemy URL),
748
+ ``snowflake``, ``bigquery``, ``redshift``, ``oracle``,
749
+ ``databricks``, ``fabric`` (Microsoft Fabric).
750
+
751
+ NoSQL:
752
+ ``mongodb``, ``cassandra``, ``dynamodb``, ``elasticsearch``.
753
+
754
+ Object storage / data lakes:
755
+ ``s3``, ``gcs``, ``azure_blob``, ``adls_gen2`` (Azure
756
+ Data Lake Storage Gen2), ``delta`` (Delta Lake).
757
+
758
+ Streaming:
759
+ ``kafka``, ``kinesis``.
760
+
761
+ Generic:
762
+ ``http`` (REST/HTTP API), ``mock`` (test).
610
763
  secrets: Inline connection secrets (e.g. connection_string,
611
764
  bucket, access keys).
612
765
  credential_id: ID of a saved credential on the server
@@ -634,10 +787,24 @@ class AutoDataClient:
634
787
  secrets: Optional[Dict] = None,
635
788
  credential_id: Optional[str] = None,
636
789
  ) -> List[str]:
637
- """List available tables / files in a data source.
790
+ """List available tables / files / indexes / topics in a data source.
791
+
792
+ What "discover" returns is dialect-specific:
793
+ * SQL warehouses (``sql``, ``snowflake``, ``redshift``, ``oracle``,
794
+ ``databricks``, ``fabric``, ``bigquery``) → table + view names.
795
+ * NoSQL (``mongodb``, ``cassandra``, ``dynamodb``,
796
+ ``elasticsearch``) → collection / table / index names.
797
+ * Object stores (``s3``, ``gcs``, ``azure_blob``, ``adls_gen2``)
798
+ → object keys under ``prefix`` (capped at 500), or bucket /
799
+ container names if no bucket is set on the credential.
800
+ * Streaming (``kafka``, ``kinesis``) → topic / stream names.
801
+ * ``http`` → a single virtual table (the URL itself).
802
+ * ``delta`` → the table's path.
803
+
804
+ See :meth:`test_connector` for the full list of supported types.
638
805
 
639
806
  Returns:
640
- List of table or object names.
807
+ List of object names (strings).
641
808
  """
642
809
  body: Dict = {}
643
810
  if credential_id:
@@ -660,7 +827,26 @@ class AutoDataClient:
660
827
  credential_id: Optional[str] = None,
661
828
  custom_query: Optional[str] = None,
662
829
  ) -> Dict:
663
- """Preview columns and row count of a table / file.
830
+ """Preview columns and row count of a table / file / index.
831
+
832
+ Notes per connector family:
833
+ * SQL warehouses honour ``custom_query`` (read-only SELECT/WITH
834
+ only — INSERT/UPDATE/DELETE are rejected at the server).
835
+ * Object stores (``s3``, ``gcs``, ``azure_blob``, ``adls_gen2``)
836
+ auto-detect file format from extension: ``.csv`` reads the
837
+ header row, ``.parquet`` reads schema, ``.json``/``.jsonl``
838
+ samples the first record.
839
+ * Cassandra ``row_count`` is intentionally ``None`` — COUNT(*)
840
+ requires ALLOW FILTERING and is too expensive for a preview.
841
+ * DynamoDB returns key attributes from describe_table plus any
842
+ extras found in a single Scan(Limit=1) sample.
843
+ * Elasticsearch samples one document to extract the actual
844
+ field set (mappings alone miss dynamic fields).
845
+ * HTTP auto-detects JSON envelopes; pass ``json_path`` in the
846
+ credential's secrets to point at a nested array (e.g.
847
+ ``"data.results"``).
848
+
849
+ See :meth:`test_connector` for the full list of supported types.
664
850
 
665
851
  Returns:
666
852
  Dict with ``columns`` (list[str]) and ``row_count`` (int or None).
@@ -699,6 +885,7 @@ class AutoDataClient:
699
885
  compressed: bool = True,
700
886
  incremental: bool = False,
701
887
  type_casts: Optional[Dict[str, str]] = None,
888
+ return_dataframes: bool = False,
702
889
  ) -> Dict:
703
890
  """Run the AutoData pipeline on data from a connector source.
704
891
 
@@ -788,7 +975,7 @@ class AutoDataClient:
788
975
  try:
789
976
  final = self.wait_for_completion(session_id, poll_interval=poll_interval)
790
977
  except KeyboardInterrupt:
791
- print("\nInterrupted cancelling job on server")
978
+ print("\nInterrupted - cancelling job on server...")
792
979
  self.cancel(session_id)
793
980
  raise
794
981
 
@@ -799,6 +986,9 @@ class AutoDataClient:
799
986
  output_preferences=output_preferences,
800
987
  compressed=compressed,
801
988
  )
989
+ if return_dataframes:
990
+ final = dict(final)
991
+ final["dataframes"] = self.get_dataframes(session_id, names=output_preferences)
802
992
  return final
803
993
 
804
994
  def write_output(
@@ -865,10 +1055,19 @@ class AutoDataClient:
865
1055
  ) -> Dict:
866
1056
  """Save a new credential on the server for later reuse.
867
1057
 
1058
+ The server validates ``secrets`` at save-time against the connector
1059
+ type's required-field schema (e.g. for ``oracle`` it enforces that
1060
+ you supplied a connection_string OR host+user+password+service_name).
1061
+ A bad credential raises :class:`AutoDataError` with a clear message
1062
+ instead of failing later in a polling loop.
1063
+
868
1064
  Args:
869
1065
  name: Human-readable label.
870
- connector_type: Connector type (``sql``, ``s3``, …).
871
- secrets: Connection secrets dict.
1066
+ connector_type: One of the supported connector types — see
1067
+ :meth:`test_connector` for the full list.
1068
+ secrets: Connection secrets dict. Required fields vary
1069
+ by connector type; see the dashboard's quick-
1070
+ connect form for the exact field names.
872
1071
 
873
1072
  Returns:
874
1073
  Dict with the saved credential metadata including ``id``.
@@ -1281,43 +1480,112 @@ class AutoDataClient:
1281
1480
  self,
1282
1481
  name: str,
1283
1482
  source_type: str,
1284
- watch_path: str,
1285
- y_columns: Union[str, List[str]],
1483
+ folder_path: Optional[str] = None,
1484
+ y_columns: Optional[Union[str, List[str]]] = None,
1485
+ *,
1486
+ bucket_name: Optional[str] = None,
1487
+ prefix: Optional[str] = None,
1286
1488
  credential_id: Optional[str] = None,
1287
- secrets: Optional[Dict] = None,
1288
1489
  pipeline_config: Optional[Dict] = None,
1289
1490
  output_rows: int = 10000,
1491
+ poll_interval_seconds: int = 60,
1492
+ allowed_extensions: str = ".csv,.parquet,.json",
1493
+ max_file_size_mb: int = 500,
1494
+ cooldown_seconds: int = 60,
1495
+ max_files_per_hour: int = 10,
1496
+ enabled: bool = True,
1497
+ auto_sink_config: Optional[Dict] = None,
1498
+ watch_path: Optional[str] = None, # deprecated alias of folder_path
1290
1499
  ) -> Dict:
1291
- """Create a folder listener.
1500
+ """Create a folder listener that watches a source for new files and
1501
+ triggers the AutoData pipeline on each one.
1292
1502
 
1293
1503
  Args:
1294
- name: Human-readable listener name.
1295
- source_type: ``'s3'``, ``'gcs'``, ``'azure_blob'``,
1296
- ``'local'``, or ``'sftp'``.
1297
- watch_path: Folder path or URI to watch for new files.
1298
- y_columns: Target column name(s) for ML.
1299
- credential_id: ID of a saved credential on the server.
1300
- secrets: Inline connection secrets (alternative to credential_id).
1301
- pipeline_config: Pipeline tool toggles (e.g. ``{"enable_dsg": False}``).
1302
- output_rows: Target row count in output dataset.
1504
+ name: Human-readable listener name.
1505
+ source_type: ``'local'``, ``'s3'``, ``'gcs'``,
1506
+ ``'azure_blob'``, ``'adls_gen2'``, or
1507
+ ``'sftp'``. (Database-style connectors
1508
+ like ``oracle``/``cassandra`` aren't
1509
+ listener sources use a scheduled run
1510
+ with a watermark instead.)
1511
+ folder_path: Local path to watch. Required for
1512
+ ``source_type='local'``. For cloud sources
1513
+ it's auto-derived from ``bucket_name``.
1514
+ For ``'sftp'`` it's auto-set to the user's
1515
+ inbox directory.
1516
+ y_columns: Target column name(s) for ML. Required for
1517
+ the listener to actually fire the pipeline
1518
+ (the server stores it but won't trigger
1519
+ without it).
1520
+ bucket_name: Bucket / container name. **Required** for
1521
+ s3 / gcs / azure_blob sources.
1522
+ prefix: Optional object prefix filter for cloud
1523
+ sources.
1524
+ credential_id: ID of a saved credential on the server.
1525
+ **Required** for cloud and SFTP sources.
1526
+ Use :meth:`save_credential` first.
1527
+ pipeline_config: Full pipeline config dict. Recognised keys
1528
+ include ``output_number``, ``text_mode``,
1529
+ ``text_cleaning``, ``mdh_mode``,
1530
+ ``excluded_columns``, ``task_types``,
1531
+ ``zscore_limit``, ``similarity_p``,
1532
+ ``dsg_mode``, ``enable_anomaly_detection``,
1533
+ ``enable_dtc``/``mdh``/``dor``/``cds``/``dsm``/``dsg``,
1534
+ ``webhook_ids``.
1535
+ output_rows: Convenience for ``pipeline_config['output_number']``.
1536
+ Ignored if ``pipeline_config`` already sets it.
1537
+ poll_interval_seconds: How often the watcher checks for new files
1538
+ (default 60).
1539
+ allowed_extensions: Comma-separated list, e.g.
1540
+ ``'.csv,.parquet'``.
1541
+ max_file_size_mb: Skip files larger than this (default 500).
1542
+ cooldown_seconds: Min seconds between pipeline triggers
1543
+ (default 60).
1544
+ max_files_per_hour: Hard cap on triggers per hour (default 10).
1545
+ enabled: Start watching immediately (default True).
1546
+ auto_sink_config: Optional ``{enabled, connector_type,
1547
+ credential_id, table_name, if_exists}``
1548
+ to auto-write each output to a sink.
1303
1549
 
1304
1550
  Returns:
1305
- Dict with the created listener metadata.
1551
+ Dict with the created listener metadata under ``listener``.
1306
1552
  """
1307
- y_cols = [y_columns] if isinstance(y_columns, str) else y_columns
1553
+ # Backwards compat: accept the old `watch_path` kwarg.
1554
+ if folder_path is None and watch_path is not None:
1555
+ folder_path = watch_path
1556
+ if y_columns is None:
1557
+ raise ValueError("y_columns is required to create a folder listener")
1558
+
1559
+ y_cols = [y_columns] if isinstance(y_columns, str) else list(y_columns)
1560
+
1561
+ # Fold output_rows into pipeline_config — the server's watcher reads
1562
+ # `output_number` from there, not from a top-level `output_rows` key.
1563
+ cfg = dict(pipeline_config or {})
1564
+ cfg.setdefault("output_number", str(output_rows))
1565
+
1308
1566
  body: Dict = {
1309
1567
  "name": name,
1310
1568
  "source_type": source_type,
1311
- "watch_path": watch_path,
1312
1569
  "y_columns": y_cols,
1313
- "output_rows": output_rows,
1570
+ "pipeline_config": cfg,
1571
+ "poll_interval_seconds": poll_interval_seconds,
1572
+ "allowed_extensions": allowed_extensions,
1573
+ "max_file_size_mb": max_file_size_mb,
1574
+ "cooldown_seconds": cooldown_seconds,
1575
+ "max_files_per_hour": max_files_per_hour,
1576
+ "enabled": enabled,
1314
1577
  }
1578
+ if folder_path:
1579
+ body["folder_path"] = folder_path
1580
+ if bucket_name:
1581
+ body["bucket_name"] = bucket_name
1582
+ if prefix:
1583
+ body["prefix"] = prefix
1315
1584
  if credential_id:
1316
1585
  body["credential_id"] = credential_id
1317
- if secrets:
1318
- body["inline_secrets"] = secrets
1319
- if pipeline_config:
1320
- body["pipeline_config"] = pipeline_config
1586
+ if auto_sink_config is not None:
1587
+ body["auto_sink_config"] = auto_sink_config
1588
+
1321
1589
  r = self._request("POST", self._url("/listeners/folder"), json=body)
1322
1590
  self._raise_for_error(r)
1323
1591
  return r.json()
@@ -1325,34 +1593,65 @@ class AutoDataClient:
1325
1593
  def update_listener(
1326
1594
  self,
1327
1595
  listener_id: str,
1596
+ *,
1328
1597
  name: Optional[str] = None,
1329
- watch_path: Optional[str] = None,
1598
+ folder_path: Optional[str] = None,
1599
+ y_columns: Optional[Union[str, List[str]]] = None,
1600
+ bucket_name: Optional[str] = None,
1601
+ prefix: Optional[str] = None,
1602
+ credential_id: Optional[str] = None,
1330
1603
  pipeline_config: Optional[Dict] = None,
1604
+ poll_interval_seconds: Optional[int] = None,
1605
+ allowed_extensions: Optional[str] = None,
1606
+ max_file_size_mb: Optional[int] = None,
1607
+ cooldown_seconds: Optional[int] = None,
1608
+ max_files_per_hour: Optional[int] = None,
1331
1609
  enabled: Optional[bool] = None,
1610
+ auto_sink_config: Optional[Dict] = None,
1611
+ watch_path: Optional[str] = None, # deprecated alias of folder_path
1332
1612
  ) -> Dict:
1333
1613
  """Update a folder listener.
1334
1614
 
1335
1615
  Only provided fields are updated; omitted fields remain unchanged.
1616
+ Setting ``enabled`` toggles the background watcher thread.
1336
1617
 
1337
- Args:
1338
- listener_id: ID of the listener to update.
1339
- name: New listener name.
1340
- watch_path: New folder path or URI to watch.
1341
- pipeline_config: Updated pipeline tool toggles.
1342
- enabled: Enable or disable the listener.
1343
-
1344
- Returns:
1345
- Dict with the updated listener metadata.
1618
+ See :meth:`create_listener` for field semantics.
1346
1619
  """
1620
+ if folder_path is None and watch_path is not None:
1621
+ folder_path = watch_path
1622
+
1347
1623
  body: Dict = {}
1348
1624
  if name is not None:
1349
1625
  body["name"] = name
1350
- if watch_path is not None:
1351
- body["watch_path"] = watch_path
1626
+ if folder_path is not None:
1627
+ body["folder_path"] = folder_path
1628
+ if y_columns is not None:
1629
+ body["y_columns"] = (
1630
+ [y_columns] if isinstance(y_columns, str) else list(y_columns)
1631
+ )
1632
+ if bucket_name is not None:
1633
+ body["bucket_name"] = bucket_name
1634
+ if prefix is not None:
1635
+ body["prefix"] = prefix
1636
+ if credential_id is not None:
1637
+ body["credential_id"] = credential_id
1352
1638
  if pipeline_config is not None:
1353
1639
  body["pipeline_config"] = pipeline_config
1640
+ if poll_interval_seconds is not None:
1641
+ body["poll_interval_seconds"] = poll_interval_seconds
1642
+ if allowed_extensions is not None:
1643
+ body["allowed_extensions"] = allowed_extensions
1644
+ if max_file_size_mb is not None:
1645
+ body["max_file_size_mb"] = max_file_size_mb
1646
+ if cooldown_seconds is not None:
1647
+ body["cooldown_seconds"] = cooldown_seconds
1648
+ if max_files_per_hour is not None:
1649
+ body["max_files_per_hour"] = max_files_per_hour
1354
1650
  if enabled is not None:
1355
1651
  body["enabled"] = enabled
1652
+ if auto_sink_config is not None:
1653
+ body["auto_sink_config"] = auto_sink_config
1654
+
1356
1655
  r = self._request(
1357
1656
  "PUT", self._url(f"/listeners/folder/{listener_id}"), json=body
1358
1657
  )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datatoolpack
3
- Version: 0.6.0
3
+ Version: 0.7.2
4
4
  Summary: Official Python SDK for the AutoData ML data preparation pipeline API
5
5
  Home-page: https://autodata.datatoolpack.com
6
6
  Author: AutoData Team
@@ -7,7 +7,7 @@ with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
7
7
 
8
8
  setup(
9
9
  name="datatoolpack",
10
- version="0.6.0",
10
+ version="0.7.2",
11
11
  description="Official Python SDK for the AutoData ML data preparation pipeline API",
12
12
  long_description=long_description,
13
13
  long_description_content_type="text/markdown",
File without changes
File without changes