datatoolpack 0.7.0__tar.gz → 0.7.2__tar.gz

{datatoolpack-0.7.0 → datatoolpack-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatoolpack
-Version: 0.7.0
+Version: 0.7.2
 Summary: Official Python SDK for the AutoData ML data preparation pipeline API
 Home-page: https://autodata.datatoolpack.com
 Author: AutoData Team

{datatoolpack-0.7.0 → datatoolpack-0.7.2}/datatoolpack/__init__.py

@@ -1,4 +1,4 @@
 from .client import AutoDataClient, AutoDataError
 
-__version__ = "0.7.0"
+__version__ = "0.7.2"
 __all__ = ["AutoDataClient", "AutoDataError", "__version__"]

{datatoolpack-0.7.0 → datatoolpack-0.7.2}/datatoolpack/client.py

@@ -24,9 +24,6 @@ import time
 import zipfile
 from typing import Dict, List, Optional, Union
 
-import requests
-
-
 try:
     import requests
 except ImportError as exc:  # pragma: no cover
@@ -147,10 +144,20 @@ class AutoDataClient:
     # ------------------------------------------------------------------
 
     def _url(self, path: str) -> str:
+        """Build URL for the versioned API. Use this for everything except
+        the handful of endpoints that are only registered under the
+        unversioned ``/api/`` prefix (see :meth:`_api`).
+        """
         return f"{self.base_url}/api/v1{path}"
 
     def _api(self, path: str) -> str:
-        """Build URL for endpoints under /api/ (without v1 prefix)."""
+        """Build URL for endpoints under /api/ (without v1 prefix).
+
+        Currently this is required for ``/api/scheduled-runs*`` only —
+        those routes are intentionally NOT dual-registered on /api/v1/.
+        Don't 'fix' a `_api` call to `_url` without verifying the route
+        exists under /api/v1/, or you'll silently 404.
+        """
         return f"{self.base_url}/api{path}"
 
     @staticmethod
@@ -297,7 +304,7 @@ class AutoDataClient:
                 session_id, poll_interval=poll_interval
             )
         except KeyboardInterrupt:
-            print("\nInterrupted — cancelling job on server…")
+            print("\nInterrupted - cancelling job on server...")
             self.cancel(session_id)
             raise
 
@@ -349,7 +356,7 @@ class AutoDataClient:
             self._raise_for_error(r)
             return r.json().get("cancelled", False)
         except AutoDataError as e:
-            print(f"Warning: cancel failed — {e}")
+            print(f"Warning: cancel failed - {e}")
             return False
 
     def wait_for_completion(
@@ -377,7 +384,11 @@ class AutoDataClient:
             AutoDataError: On processing failure, cancellation, timeout, or
                            detected stall.
         """
-        print(f"Waiting for session {session_id}…")
+        # Use ASCII-only progress output. The previous version used unicode
+        # glyphs (U+2713 check, U+2026 ellipsis, U+2014 em-dash) which crash
+        # on Windows consoles using cp1252 — the default for Python on
+        # Windows. ASCII keeps the SDK portable.
+        print(f"Waiting for session {session_id}...")
         start = time.monotonic()
         last_change = start
         last_signature: tuple = ()
@@ -388,7 +399,7 @@ class AutoDataClient:
             msg = data.get("message", "")
 
             if status == "completed":
-                print(f"\r✓ Completed ({pct}%): {msg}     ")
+                print(f"\r[OK] Completed ({pct}%): {msg}     ")
                 return self.get_result(session_id)
             if status == "error":
                 raise AutoDataError(f"Processing failed: {msg}")
@@ -406,7 +417,7 @@ class AutoDataClient:
                 raise AutoDataError(
                     f"Processing stalled: no progress for {stall_timeout:.0f}s "
                     f"(last status={status!r}, message={msg!r}). "
-                    "The worker may have crashed — please retry."
+                    "The worker may have crashed - please retry."
                 )
 
             if timeout is not None and now - start > timeout:
@@ -414,7 +425,7 @@ class AutoDataClient:
                     f"wait_for_completion timed out after {timeout:.0f}s"
                 )
 
-            print(f"\r  {pct:3d}% — {msg[:60]:<60}", end="", flush=True)
+            print(f"\r  {pct:3d}% - {msg[:60]:<60}", end="", flush=True)
             time.sleep(poll_interval)
 
     # ------------------------------------------------------------------
@@ -563,7 +574,7 @@ class AutoDataClient:
                 last_err = err
                 if attempt < retries:
                     backoff = 2 ** attempt
-                    print(f"\nDownload attempt {attempt + 1} failed ({err}); retrying in {backoff}s…")
+                    print(f"\nDownload attempt {attempt + 1} failed ({err}); retrying in {backoff}s...")
                     time.sleep(backoff)
                     continue
                 raise
@@ -730,9 +741,25 @@ class AutoDataClient:
         """Test connectivity to an external data source.
 
         Args:
-            connector_type: One of ``sql``, ``snowflake``, ``bigquery``,
-                            ``mongodb``, ``s3``, ``gcs``, ``databricks``,
-                            ``delta``, ``fabric``, ``kafka``, ``kinesis``.
+            connector_type: One of the supported types:
+
+                Databases / warehouses:
+                  ``sql`` (Postgres/MySQL/MSSQL via SQLAlchemy URL),
+                  ``snowflake``, ``bigquery``, ``redshift``, ``oracle``,
+                  ``databricks``, ``fabric`` (Microsoft Fabric).
+
+                NoSQL:
+                  ``mongodb``, ``cassandra``, ``dynamodb``, ``elasticsearch``.
+
+                Object storage / data lakes:
+                  ``s3``, ``gcs``, ``azure_blob``, ``adls_gen2`` (Azure
+                  Data Lake Storage Gen2), ``delta`` (Delta Lake).
+
+                Streaming:
+                  ``kafka``, ``kinesis``.
+
+                Generic:
+                  ``http`` (REST/HTTP API), ``mock`` (test).
             secrets:        Inline connection secrets (e.g. connection_string,
                             bucket, access keys).
             credential_id:  ID of a saved credential on the server
@@ -760,10 +787,24 @@ class AutoDataClient:
         secrets: Optional[Dict] = None,
         credential_id: Optional[str] = None,
     ) -> List[str]:
-        """List available tables / files in a data source.
+        """List available tables / files / indexes / topics in a data source.
+
+        What "discover" returns is dialect-specific:
+          * SQL warehouses (``sql``, ``snowflake``, ``redshift``, ``oracle``,
+            ``databricks``, ``fabric``, ``bigquery``) → table + view names.
+          * NoSQL (``mongodb``, ``cassandra``, ``dynamodb``,
+            ``elasticsearch``) → collection / table / index names.
+          * Object stores (``s3``, ``gcs``, ``azure_blob``, ``adls_gen2``)
+            → object keys under ``prefix`` (capped at 500), or bucket /
+            container names if no bucket is set on the credential.
+          * Streaming (``kafka``, ``kinesis``) → topic / stream names.
+          * ``http`` → a single virtual table (the URL itself).
+          * ``delta`` → the table's path.
+
+        See :meth:`test_connector` for the full list of supported types.
 
         Returns:
-            List of table or object names.
+            List of object names (strings).
         """
         body: Dict = {}
         if credential_id:
@@ -786,7 +827,26 @@ class AutoDataClient:
         credential_id: Optional[str] = None,
         custom_query: Optional[str] = None,
     ) -> Dict:
-        """Preview columns and row count of a table / file.
+        """Preview columns and row count of a table / file / index.
+
+        Notes per connector family:
+          * SQL warehouses honour ``custom_query`` (read-only SELECT/WITH
+            only — INSERT/UPDATE/DELETE are rejected at the server).
+          * Object stores (``s3``, ``gcs``, ``azure_blob``, ``adls_gen2``)
+            auto-detect file format from extension: ``.csv`` reads the
+            header row, ``.parquet`` reads schema, ``.json``/``.jsonl``
+            samples the first record.
+          * Cassandra ``row_count`` is intentionally ``None`` — COUNT(*)
+            requires ALLOW FILTERING and is too expensive for a preview.
+          * DynamoDB returns key attributes from describe_table plus any
+            extras found in a single Scan(Limit=1) sample.
+          * Elasticsearch samples one document to extract the actual
+            field set (mappings alone miss dynamic fields).
+          * HTTP auto-detects JSON envelopes; pass ``json_path`` in the
+            credential's secrets to point at a nested array (e.g.
+            ``"data.results"``).
+
+        See :meth:`test_connector` for the full list of supported types.
 
         Returns:
             Dict with ``columns`` (list[str]) and ``row_count`` (int or None).
@@ -915,7 +975,7 @@ class AutoDataClient:
         try:
             final = self.wait_for_completion(session_id, poll_interval=poll_interval)
         except KeyboardInterrupt:
-            print("\nInterrupted — cancelling job on server…")
+            print("\nInterrupted - cancelling job on server...")
             self.cancel(session_id)
             raise
 
@@ -995,10 +1055,19 @@ class AutoDataClient:
     ) -> Dict:
         """Save a new credential on the server for later reuse.
 
+        The server validates ``secrets`` at save-time against the connector
+        type's required-field schema (e.g. for ``oracle`` it enforces that
+        you supplied a connection_string OR host+user+password+service_name).
+        A bad credential raises :class:`AutoDataError` with a clear message
+        instead of failing later in a polling loop.
+
         Args:
             name:           Human-readable label.
-            connector_type: Connector type (``sql``, ``s3``, …).
-            secrets:        Connection secrets dict.
+            connector_type: One of the supported connector types — see
+                            :meth:`test_connector` for the full list.
+            secrets:        Connection secrets dict. Required fields vary
+                            by connector type; see the dashboard's quick-
+                            connect form for the exact field names.
 
         Returns:
             Dict with the saved credential metadata including ``id``.
@@ -1411,43 +1480,112 @@ class AutoDataClient:
         self,
         name: str,
         source_type: str,
-        watch_path: str,
-        y_columns: Union[str, List[str]],
+        folder_path: Optional[str] = None,
+        y_columns: Optional[Union[str, List[str]]] = None,
+        *,
+        bucket_name: Optional[str] = None,
+        prefix: Optional[str] = None,
         credential_id: Optional[str] = None,
-        secrets: Optional[Dict] = None,
         pipeline_config: Optional[Dict] = None,
         output_rows: int = 10000,
+        poll_interval_seconds: int = 60,
+        allowed_extensions: str = ".csv,.parquet,.json",
+        max_file_size_mb: int = 500,
+        cooldown_seconds: int = 60,
+        max_files_per_hour: int = 10,
+        enabled: bool = True,
+        auto_sink_config: Optional[Dict] = None,
+        watch_path: Optional[str] = None,  # deprecated alias of folder_path
     ) -> Dict:
-        """Create a folder listener.
+        """Create a folder listener that watches a source for new files and
+        triggers the AutoData pipeline on each one.
 
         Args:
-            name:            Human-readable listener name.
-            source_type:     ``'s3'``, ``'gcs'``, ``'azure_blob'``,
-                             ``'local'``, or ``'sftp'``.
-            watch_path:      Folder path or URI to watch for new files.
-            y_columns:       Target column name(s) for ML.
-            credential_id:   ID of a saved credential on the server.
-            secrets:         Inline connection secrets (alternative to credential_id).
-            pipeline_config: Pipeline tool toggles (e.g. ``{"enable_dsg": False}``).
-            output_rows:     Target row count in output dataset.
+            name:                  Human-readable listener name.
+            source_type:           ``'local'``, ``'s3'``, ``'gcs'``,
+                                   ``'azure_blob'``, ``'adls_gen2'``, or
+                                   ``'sftp'``. (Database-style connectors
+                                   like ``oracle``/``cassandra`` aren't
+                                   listener sources — use a scheduled run
+                                   with a watermark instead.)
+            folder_path:           Local path to watch. Required for
+                                   ``source_type='local'``. For cloud sources
+                                   it's auto-derived from ``bucket_name``.
+                                   For ``'sftp'`` it's auto-set to the user's
+                                   inbox directory.
+            y_columns:             Target column name(s) for ML. Required for
+                                   the listener to actually fire the pipeline
+                                   (the server stores it but won't trigger
+                                   without it).
+            bucket_name:           Bucket / container name. **Required** for
+                                   s3 / gcs / azure_blob sources.
+            prefix:                Optional object prefix filter for cloud
+                                   sources.
+            credential_id:         ID of a saved credential on the server.
+                                   **Required** for cloud and SFTP sources.
+                                   Use :meth:`save_credential` first.
+            pipeline_config:       Full pipeline config dict. Recognised keys
+                                   include ``output_number``, ``text_mode``,
+                                   ``text_cleaning``, ``mdh_mode``,
+                                   ``excluded_columns``, ``task_types``,
+                                   ``zscore_limit``, ``similarity_p``,
+                                   ``dsg_mode``, ``enable_anomaly_detection``,
+                                   ``enable_dtc``/``mdh``/``dor``/``cds``/``dsm``/``dsg``,
+                                   ``webhook_ids``.
+            output_rows:           Convenience for ``pipeline_config['output_number']``.
+                                   Ignored if ``pipeline_config`` already sets it.
+            poll_interval_seconds: How often the watcher checks for new files
+                                   (default 60).
+            allowed_extensions:    Comma-separated list, e.g.
+                                   ``'.csv,.parquet'``.
+            max_file_size_mb:      Skip files larger than this (default 500).
+            cooldown_seconds:      Min seconds between pipeline triggers
+                                   (default 60).
+            max_files_per_hour:    Hard cap on triggers per hour (default 10).
+            enabled:               Start watching immediately (default True).
+            auto_sink_config:      Optional ``{enabled, connector_type,
+                                   credential_id, table_name, if_exists}``
+                                   to auto-write each output to a sink.
 
         Returns:
-            Dict with the created listener metadata.
+            Dict with the created listener metadata under ``listener``.
         """
-        y_cols = [y_columns] if isinstance(y_columns, str) else y_columns
+        # Backwards compat: accept the old `watch_path` kwarg.
+        if folder_path is None and watch_path is not None:
+            folder_path = watch_path
+        if y_columns is None:
+            raise ValueError("y_columns is required to create a folder listener")
+
+        y_cols = [y_columns] if isinstance(y_columns, str) else list(y_columns)
+
+        # Fold output_rows into pipeline_config — the server's watcher reads
+        # `output_number` from there, not from a top-level `output_rows` key.
+        cfg = dict(pipeline_config or {})
+        cfg.setdefault("output_number", str(output_rows))
+
         body: Dict = {
             "name": name,
             "source_type": source_type,
-            "watch_path": watch_path,
             "y_columns": y_cols,
-            "output_rows": output_rows,
+            "pipeline_config": cfg,
+            "poll_interval_seconds": poll_interval_seconds,
+            "allowed_extensions": allowed_extensions,
+            "max_file_size_mb": max_file_size_mb,
+            "cooldown_seconds": cooldown_seconds,
+            "max_files_per_hour": max_files_per_hour,
+            "enabled": enabled,
         }
+        if folder_path:
+            body["folder_path"] = folder_path
+        if bucket_name:
+            body["bucket_name"] = bucket_name
+        if prefix:
+            body["prefix"] = prefix
         if credential_id:
             body["credential_id"] = credential_id
-        if secrets:
-            body["inline_secrets"] = secrets
-        if pipeline_config:
-            body["pipeline_config"] = pipeline_config
+        if auto_sink_config is not None:
+            body["auto_sink_config"] = auto_sink_config
+
         r = self._request("POST", self._url("/listeners/folder"), json=body)
         self._raise_for_error(r)
         return r.json()
@@ -1455,34 +1593,65 @@ class AutoDataClient:
     def update_listener(
         self,
         listener_id: str,
+        *,
         name: Optional[str] = None,
-        watch_path: Optional[str] = None,
+        folder_path: Optional[str] = None,
+        y_columns: Optional[Union[str, List[str]]] = None,
+        bucket_name: Optional[str] = None,
+        prefix: Optional[str] = None,
+        credential_id: Optional[str] = None,
         pipeline_config: Optional[Dict] = None,
+        poll_interval_seconds: Optional[int] = None,
+        allowed_extensions: Optional[str] = None,
+        max_file_size_mb: Optional[int] = None,
+        cooldown_seconds: Optional[int] = None,
+        max_files_per_hour: Optional[int] = None,
         enabled: Optional[bool] = None,
+        auto_sink_config: Optional[Dict] = None,
+        watch_path: Optional[str] = None,  # deprecated alias of folder_path
     ) -> Dict:
         """Update a folder listener.
 
         Only provided fields are updated; omitted fields remain unchanged.
+        Setting ``enabled`` toggles the background watcher thread.
 
-        Args:
-            listener_id:     ID of the listener to update.
-            name:            New listener name.
-            watch_path:      New folder path or URI to watch.
-            pipeline_config: Updated pipeline tool toggles.
-            enabled:         Enable or disable the listener.
-
-        Returns:
-            Dict with the updated listener metadata.
+        See :meth:`create_listener` for field semantics.
         """
+        if folder_path is None and watch_path is not None:
+            folder_path = watch_path
+
         body: Dict = {}
         if name is not None:
             body["name"] = name
-        if watch_path is not None:
-            body["watch_path"] = watch_path
+        if folder_path is not None:
+            body["folder_path"] = folder_path
+        if y_columns is not None:
+            body["y_columns"] = (
+                [y_columns] if isinstance(y_columns, str) else list(y_columns)
+            )
+        if bucket_name is not None:
+            body["bucket_name"] = bucket_name
+        if prefix is not None:
+            body["prefix"] = prefix
+        if credential_id is not None:
+            body["credential_id"] = credential_id
         if pipeline_config is not None:
             body["pipeline_config"] = pipeline_config
+        if poll_interval_seconds is not None:
+            body["poll_interval_seconds"] = poll_interval_seconds
+        if allowed_extensions is not None:
+            body["allowed_extensions"] = allowed_extensions
+        if max_file_size_mb is not None:
+            body["max_file_size_mb"] = max_file_size_mb
+        if cooldown_seconds is not None:
+            body["cooldown_seconds"] = cooldown_seconds
+        if max_files_per_hour is not None:
+            body["max_files_per_hour"] = max_files_per_hour
         if enabled is not None:
             body["enabled"] = enabled
+        if auto_sink_config is not None:
+            body["auto_sink_config"] = auto_sink_config
+
         r = self._request(
             "PUT", self._url(f"/listeners/folder/{listener_id}"), json=body
         )

{datatoolpack-0.7.0 → datatoolpack-0.7.2}/datatoolpack.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatoolpack
-Version: 0.7.0
+Version: 0.7.2
 Summary: Official Python SDK for the AutoData ML data preparation pipeline API
 Home-page: https://autodata.datatoolpack.com
 Author: AutoData Team

{datatoolpack-0.7.0 → datatoolpack-0.7.2}/setup.py

@@ -7,7 +7,7 @@ with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
 
 setup(
     name="datatoolpack",
-    version="0.7.0",
+    version="0.7.2",
     description="Official Python SDK for the AutoData ML data preparation pipeline API",
     long_description=long_description,
     long_description_content_type="text/markdown",