datatoolpack 0.3.0__tar.gz → 0.4.0__tar.gz

{datatoolpack-0.3.0 → datatoolpack-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatoolpack
-Version: 0.3.0
+Version: 0.4.0
 Summary: Official Python SDK for the AutoData ML data preparation pipeline API
 Home-page: https://autodata.datatoolpack.com
 Author: AutoData Team
@@ -500,6 +500,115 @@ with AutoDataClient(api_key="dtpk_...") as client:
 
 ---
 
+## Quality Alerts
+
+Monitor pipeline metrics and get notified when thresholds are breached:
+
+```python
+# Create a quality alert rule
+rule = client.create_quality_alert(
+    name="High row loss",
+    metric="row_loss_pct",    # row_loss_pct, null_pct, column_drop_count, duration_seconds
+    operator=">",             # >, <, >=, <=, ==
+    threshold=10.0,
+    severity="critical",      # warning or critical
+    stage="mdh",              # optional: anomaly, dtc, mdh, cds, dsm, dsg
+)
+
+# List rules
+rules = client.list_quality_alerts()
+
+# Get fired alert events
+events = client.get_alert_events(session_id="optional-filter")
+
+# Delete a rule
+client.delete_quality_alert(rule_id=rule["rule"]["id"])
+```
+
+## Sync Watermarks
+
+Track incremental sync progress for connector-based pipelines:
+
+```python
+# List all watermarks
+watermarks = client.list_watermarks()
+
+# Reset a watermark (re-sync from beginning)
+client.reset_watermark(watermark_id="wm-123")
+```
+
+## Scheduled Runs
+
+Automate recurring pipeline executions:
+
+```python
+# Create an interval-based schedule (every 24 hours)
+schedule = client.create_scheduled_run(
+    name="Daily ETL",
+    schedule_type="interval",
+    interval_minutes=1440,
+    connector_type="snowflake",
+    table="orders",
+    credential_id="cred-id",
+    y_columns=["target"],
+)
+
+# Create with cron expression
+schedule = client.create_scheduled_run(
+    name="Weekday ETL",
+    schedule_type="cron",
+    cron_expression="0 8 * * 1-5",
+    connector_type="snowflake",
+    table="orders",
+    credential_id="cred-id",
+    y_columns=["target"],
+)
+
+# List and delete
+schedules = client.list_scheduled_runs()
+client.delete_scheduled_run(run_id="run-id")
+```
+
+## Folder Listeners
+
+Automatically trigger pipelines when new files appear:
+
+```python
+# Create an S3 folder listener
+listener = client.create_listener(
+    name="S3 Ingest",
+    source_type="s3",            # s3, gcs, azure_blob, local, sftp
+    folder_path="s3://bucket/incoming/",
+    credential_id="cred-id",
+    y_columns=["target"],
+    pipeline_config={"enable_dsg": False},
+)
+
+# List and delete
+listeners = client.list_listeners()
+client.delete_listener(listener_id="listener-id")
+```
+
+## Pipeline Retry
+
+Retry a failed pipeline from its last checkpoint:
+
+```python
+result = client.retry_session(session_id="failed-session-id")
+print(result)  # {'success': True, 'session_id': '...', 'message': 'Retrying from last checkpoint'}
+```
+
+## Worker Status
+
+Check the processing worker fleet status:
+
+```python
+status = client.worker_status()
+print(status)  # {'backend': 'local', 'active_jobs': 2, 'queue_size': 0, ...}
+```
+
+---
+
 ## Requirements
 
 - Python >= 3.8

{datatoolpack-0.3.0 → datatoolpack-0.4.0}/README.md

@@ -463,6 +463,115 @@ with AutoDataClient(api_key="dtpk_...") as client:
 
 ---
 
+## Quality Alerts
+
+Monitor pipeline metrics and get notified when thresholds are breached:
+
+```python
+# Create a quality alert rule
+rule = client.create_quality_alert(
+    name="High row loss",
+    metric="row_loss_pct",    # row_loss_pct, null_pct, column_drop_count, duration_seconds
+    operator=">",             # >, <, >=, <=, ==
+    threshold=10.0,
+    severity="critical",      # warning or critical
+    stage="mdh",              # optional: anomaly, dtc, mdh, cds, dsm, dsg
+)
+
+# List rules
+rules = client.list_quality_alerts()
+
+# Get fired alert events
+events = client.get_alert_events(session_id="optional-filter")
+
+# Delete a rule
+client.delete_quality_alert(rule_id=rule["rule"]["id"])
+```
+
+## Sync Watermarks
+
+Track incremental sync progress for connector-based pipelines:
+
+```python
+# List all watermarks
+watermarks = client.list_watermarks()
+
+# Reset a watermark (re-sync from beginning)
+client.reset_watermark(watermark_id="wm-123")
+```
+
+## Scheduled Runs
+
+Automate recurring pipeline executions:
+
+```python
+# Create an interval-based schedule (every 24 hours)
+schedule = client.create_scheduled_run(
+    name="Daily ETL",
+    schedule_type="interval",
+    interval_minutes=1440,
+    connector_type="snowflake",
+    table="orders",
+    credential_id="cred-id",
+    y_columns=["target"],
+)
+
+# Create with cron expression
+schedule = client.create_scheduled_run(
+    name="Weekday ETL",
+    schedule_type="cron",
+    cron_expression="0 8 * * 1-5",
+    connector_type="snowflake",
+    table="orders",
+    credential_id="cred-id",
+    y_columns=["target"],
+)
+
+# List and delete
+schedules = client.list_scheduled_runs()
+client.delete_scheduled_run(run_id="run-id")
+```
+
+## Folder Listeners
+
+Automatically trigger pipelines when new files appear:
+
+```python
+# Create an S3 folder listener
+listener = client.create_listener(
+    name="S3 Ingest",
+    source_type="s3",            # s3, gcs, azure_blob, local, sftp
+    folder_path="s3://bucket/incoming/",
+    credential_id="cred-id",
+    y_columns=["target"],
+    pipeline_config={"enable_dsg": False},
+)
+
+# List and delete
+listeners = client.list_listeners()
+client.delete_listener(listener_id="listener-id")
+```
+
+## Pipeline Retry
+
+Retry a failed pipeline from its last checkpoint:
+
+```python
+result = client.retry_session(session_id="failed-session-id")
+print(result)  # {'success': True, 'session_id': '...', 'message': 'Retrying from last checkpoint'}
+```
+
+## Worker Status
+
+Check the processing worker fleet status:
+
+```python
+status = client.worker_status()
+print(status)  # {'backend': 'local', 'active_jobs': 2, 'queue_size': 0, ...}
+```
+
+---
+
 ## Requirements
 
 - Python >= 3.8

{datatoolpack-0.3.0 → datatoolpack-0.4.0}/autodata/client.py

@@ -655,7 +655,7 @@ class AutoDataClient:
         result = r.json()
         session_id = result.get("session_id")
 
-        if not wait:
+        if not wait or not session_id:
             return result
 
         try:
@@ -934,6 +934,136 @@ class AutoDataClient:
         self._raise_for_error(r)
         return r.json()
 
+    # ------------------------------------------------------------------
+    # Quality Alerts
+    # ------------------------------------------------------------------
+
+    def list_quality_alerts(self) -> List[Dict]:
+        """List quality alert rules for the authenticated user."""
+        r = self._request("GET", self._url("/quality-alerts"))
+        self._raise_for_error(r)
+        return r.json().get("rules", [])
+
+    def create_quality_alert(
+        self,
+        name: str,
+        metric: str,
+        operator: str,
+        threshold: float,
+        severity: str = "warning",
+        stage: Optional[str] = None,
+    ) -> Dict:
+        """Create a quality alert rule.
+
+        Args:
+            name:      Rule name.
+            metric:    One of row_loss_pct, null_pct, column_drop_count, duration_seconds.
+            operator:  One of >, <, >=, <=, ==.
+            threshold: Numeric threshold value.
+            severity:  'warning' or 'critical'.
+            stage:     Optional pipeline stage (anomaly, dtc, mdh, cds, dsm, dsg).
+        """
+        body: Dict = {"name": name, "metric": metric, "operator": operator,
+                       "threshold": threshold, "severity": severity}
+        if stage:
+            body["stage"] = stage
+        r = self._request("POST", self._url("/quality-alerts"), json=body)
+        self._raise_for_error(r)
+        return r.json()
+
+    def delete_quality_alert(self, rule_id: str) -> Dict:
+        """Delete a quality alert rule."""
+        r = self._request("DELETE", self._url(f"/quality-alerts/{rule_id}"))
+        self._raise_for_error(r)
+        return r.json()
+
+    def get_alert_events(self, session_id: Optional[str] = None) -> List[Dict]:
+        """List recent quality alert events."""
+        params = {}
+        if session_id:
+            params["session_id"] = session_id
+        r = self._request("GET", self._url("/quality-alerts/events"), params=params)
+        self._raise_for_error(r)
+        return r.json().get("events", [])
+
+    # ------------------------------------------------------------------
+    # Scheduled Runs
+    # ------------------------------------------------------------------
+
+    def list_scheduled_runs(self) -> List[Dict]:
+        """List all scheduled runs."""
+        r = self._request("GET", f"{self.base_url}/api/scheduled-runs")
+        self._raise_for_error(r)
+        return r.json().get("scheduled_runs", [])
+
+    def create_scheduled_run(self, **kwargs) -> Dict:
+        """Create a scheduled run.
+
+        Kwargs:
+            name, schedule_type ('interval' or 'cron'), interval_minutes,
+            cron_expression, connector_type, table, credential_id, y_columns, etc.
+        """
+        r = self._request("POST", f"{self.base_url}/api/scheduled-runs", json=kwargs)
+        self._raise_for_error(r)
+        return r.json()
+
+    def delete_scheduled_run(self, run_id: str) -> Dict:
+        """Delete a scheduled run."""
+        r = self._request("DELETE", self._api(f"/scheduled-runs/{run_id}"))
+        self._raise_for_error(r)
+        return r.json()
+
+    # ------------------------------------------------------------------
+    # Folder Listeners
+    # ------------------------------------------------------------------
+
+    def list_listeners(self) -> List[Dict]:
+        """List folder listeners."""
+        r = self._request("GET", self._url("/listeners/folder"))
+        self._raise_for_error(r)
+        return r.json().get("listeners", [])
+
+    def create_listener(self, **kwargs) -> Dict:
+        """Create a folder listener.
+
+        Kwargs:
+            name, source_type ('s3', 'gcs', 'azure_blob', 'local', 'sftp'),
+            watch_path, credential_id, y_columns, pipeline_config, etc.
+        """
+        r = self._request("POST", self._url("/listeners/folder"), json=kwargs)
+        self._raise_for_error(r)
+        return r.json()
+
+    def delete_listener(self, listener_id: str) -> Dict:
+        """Delete a folder listener."""
+        r = self._request("DELETE", self._url(f"/listeners/folder/{listener_id}"))
+        self._raise_for_error(r)
+        return r.json()
+
+    # ------------------------------------------------------------------
+    # Pipeline Retry
+    # ------------------------------------------------------------------
+
+    def retry_session(self, session_id: str) -> Dict:
+        """Retry a failed pipeline session from its last checkpoint.
+
+        Args:
+            session_id: ID of the failed session to retry.
+        """
+        r = self._request("POST", self._url(f"/sessions/{session_id}/retry"))
+        self._raise_for_error(r)
+        return r.json()
+
+    # ------------------------------------------------------------------
+    # Worker Status
+    # ------------------------------------------------------------------
+
+    def worker_status(self) -> Dict:
+        """Get the worker fleet status (queue size, active workers, etc.)."""
+        r = self._request("GET", self._url("/workers/status"))
+        self._raise_for_error(r)
+        return r.json()
+
     # ------------------------------------------------------------------
     # Backward-compatibility alias
     # ------------------------------------------------------------------

{datatoolpack-0.3.0 → datatoolpack-0.4.0}/datatoolpack.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatoolpack
-Version: 0.3.0
+Version: 0.4.0
 Summary: Official Python SDK for the AutoData ML data preparation pipeline API
 Home-page: https://autodata.datatoolpack.com
 Author: AutoData Team
@@ -500,6 +500,115 @@ with AutoDataClient(api_key="dtpk_...") as client:
 
 ---
 
+## Quality Alerts
+
+Monitor pipeline metrics and get notified when thresholds are breached:
+
+```python
+# Create a quality alert rule
+rule = client.create_quality_alert(
+    name="High row loss",
+    metric="row_loss_pct",    # row_loss_pct, null_pct, column_drop_count, duration_seconds
+    operator=">",             # >, <, >=, <=, ==
+    threshold=10.0,
+    severity="critical",      # warning or critical
+    stage="mdh",              # optional: anomaly, dtc, mdh, cds, dsm, dsg
+)
+
+# List rules
+rules = client.list_quality_alerts()
+
+# Get fired alert events
+events = client.get_alert_events(session_id="optional-filter")
+
+# Delete a rule
+client.delete_quality_alert(rule_id=rule["rule"]["id"])
+```
+
+## Sync Watermarks
+
+Track incremental sync progress for connector-based pipelines:
+
+```python
+# List all watermarks
+watermarks = client.list_watermarks()
+
+# Reset a watermark (re-sync from beginning)
+client.reset_watermark(watermark_id="wm-123")
+```
+
+## Scheduled Runs
+
+Automate recurring pipeline executions:
+
+```python
+# Create an interval-based schedule (every 24 hours)
+schedule = client.create_scheduled_run(
+    name="Daily ETL",
+    schedule_type="interval",
+    interval_minutes=1440,
+    connector_type="snowflake",
+    table="orders",
+    credential_id="cred-id",
+    y_columns=["target"],
+)
+
+# Create with cron expression
+schedule = client.create_scheduled_run(
+    name="Weekday ETL",
+    schedule_type="cron",
+    cron_expression="0 8 * * 1-5",
+    connector_type="snowflake",
+    table="orders",
+    credential_id="cred-id",
+    y_columns=["target"],
+)
+
+# List and delete
+schedules = client.list_scheduled_runs()
+client.delete_scheduled_run(run_id="run-id")
+```
+
+## Folder Listeners
+
+Automatically trigger pipelines when new files appear:
+
+```python
+# Create an S3 folder listener
+listener = client.create_listener(
+    name="S3 Ingest",
+    source_type="s3",            # s3, gcs, azure_blob, local, sftp
+    folder_path="s3://bucket/incoming/",
+    credential_id="cred-id",
+    y_columns=["target"],
+    pipeline_config={"enable_dsg": False},
+)
+
+# List and delete
+listeners = client.list_listeners()
+client.delete_listener(listener_id="listener-id")
+```
+
+## Pipeline Retry
+
+Retry a failed pipeline from its last checkpoint:
+
+```python
+result = client.retry_session(session_id="failed-session-id")
+print(result)  # {'success': True, 'session_id': '...', 'message': 'Retrying from last checkpoint'}
+```
+
+## Worker Status
+
+Check the processing worker fleet status:
+
+```python
+status = client.worker_status()
+print(status)  # {'backend': 'local', 'active_jobs': 2, 'queue_size': 0, ...}
+```
+
+---
+
 ## Requirements
 
 - Python >= 3.8

{datatoolpack-0.3.0 → datatoolpack-0.4.0}/setup.py

@@ -7,7 +7,7 @@ with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
 
 setup(
     name="datatoolpack",
-    version="0.3.0",
+    version="0.4.0",
     description="Official Python SDK for the AutoData ML data preparation pipeline API",
     long_description=long_description,
     long_description_content_type="text/markdown",