datatoolpack 0.5.2__tar.gz → 0.7.0__tar.gz

{datatoolpack-0.5.2 → datatoolpack-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatoolpack
-Version: 0.5.2
+Version: 0.7.0
 Summary: Official Python SDK for the AutoData ML data preparation pipeline API
 Home-page: https://autodata.datatoolpack.com
 Author: AutoData Team

{datatoolpack-0.5.2 → datatoolpack-0.7.0}/datatoolpack/__init__.py

@@ -1,4 +1,4 @@
 from .client import AutoDataClient, AutoDataError
 
-__version__ = "0.5.2"
+__version__ = "0.7.0"
 __all__ = ["AutoDataClient", "AutoDataError", "__version__"]

{datatoolpack-0.5.2 → datatoolpack-0.7.0}/datatoolpack/client.py

@@ -24,6 +24,8 @@ import time
 import zipfile
 from typing import Dict, List, Optional, Union
 
+import requests
+
 
 try:
     import requests
@@ -212,6 +214,7 @@ class AutoDataClient:
         auto_download: bool = True,
         output_preferences: Optional[List[str]] = None,
         compressed: bool = True,
+        return_dataframes: bool = False,
     ) -> Dict:
         """Upload a data file and start the AutoData pipeline.
 
@@ -305,6 +308,12 @@ class AutoDataClient:
                 output_preferences=output_preferences,
                 compressed=compressed,
             )
+        # New in 0.7.0: return the output files as DataFrames so callers can
+        # skip the "download to disk, then pd.read_csv()" two-step. Gated on
+        # an opt-in flag to keep the default light (no pandas import).
+        if return_dataframes:
+            final = dict(final)  # don't mutate the cached result object
+            final["dataframes"] = self.get_dataframes(session_id, names=output_preferences)
         return final
 
     def get_status(self, session_id: str) -> Dict:
@@ -344,16 +353,34 @@ class AutoDataClient:
             return False
 
     def wait_for_completion(
-        self, session_id: str, poll_interval: int = 2
+        self,
+        session_id: str,
+        poll_interval: int = 2,
+        timeout: Optional[float] = None,
+        stall_timeout: float = 600.0,
     ) -> Dict:
         """Block until a session reaches *completed*, *error*, or *cancelled*.
 
         Prints progress updates to stdout.
 
+        Args:
+            session_id:    Session to wait for.
+            poll_interval: Seconds between status polls (default 2).
+            timeout:       Hard ceiling in seconds for the whole wait. ``None``
+                           disables. Raises ``AutoDataError`` if exceeded.
+            stall_timeout: If progress (percent + message) doesn't change for
+                           this many seconds AND status is still 'unknown' /
+                           'processing', treat as a stalled worker and raise.
+                           Default 10 minutes.
+
         Raises:
-            AutoDataError: On processing failure or cancellation.
+            AutoDataError: On processing failure, cancellation, timeout, or
+                           detected stall.
         """
         print(f"Waiting for session {session_id}…")
+        start = time.monotonic()
+        last_change = start
+        last_signature: tuple = ()
         while True:
             data = self.get_status(session_id)
             status = data.get("status", "unknown")
@@ -363,11 +390,30 @@ class AutoDataClient:
             if status == "completed":
                 print(f"\r✓ Completed ({pct}%): {msg}     ")
                 return self.get_result(session_id)
-            elif status == "error":
+            if status == "error":
                 raise AutoDataError(f"Processing failed: {msg}")
-            elif status == "cancelled":
+            if status == "cancelled":
                 raise AutoDataError("Processing was cancelled")
 
+            # Detect stalled workers: status stays 'unknown'/'processing' with
+            # no progress change. Prevents infinite loops when cpu_worker died.
+            sig = (status, pct, msg)
+            now = time.monotonic()
+            if sig != last_signature:
+                last_signature = sig
+                last_change = now
+            elif now - last_change > stall_timeout:
+                raise AutoDataError(
+                    f"Processing stalled: no progress for {stall_timeout:.0f}s "
+                    f"(last status={status!r}, message={msg!r}). "
+                    "The worker may have crashed — please retry."
+                )
+
+            if timeout is not None and now - start > timeout:
+                raise AutoDataError(
+                    f"wait_for_completion timed out after {timeout:.0f}s"
+                )
+
             print(f"\r  {pct:3d}% — {msg[:60]:<60}", end="", flush=True)
             time.sleep(poll_interval)
 
@@ -411,16 +457,32 @@ class AutoDataClient:
                 "POST",
                 self._url(f"/download-archive/{session_id}"),
                 json=body,
-                timeout=max(self.timeout, 300),
-                stream=True,
+                timeout=max(self.timeout, 900),
             )
             self._raise_for_error(r)
+            content = r.content or b""
+            if not content:
+                raise AutoDataError(
+                    f"Server returned an empty archive for session {session_id} "
+                    "(the outputs may have expired or failed to generate — please retry the run)."
+                )
             try:
-                z = zipfile.ZipFile(io.BytesIO(r.content))
+                z = zipfile.ZipFile(io.BytesIO(content))
+                members = z.namelist()
+                if not members:
+                    raise AutoDataError(
+                        f"Archive for session {session_id} is empty "
+                        "(no output files found on the server)."
+                    )
                 z.extractall(download_path)
-                print(f"Extracted results to {download_path}")
+                print(f"Extracted {len(members)} file(s) to {download_path}")
             except zipfile.BadZipFile:
-                raise AutoDataError("Server returned an invalid ZIP archive")
+                # Surface the body head to aid debugging — often an HTML error page.
+                preview = content[:200].decode("utf-8", errors="replace")
+                raise AutoDataError(
+                    f"Server returned an invalid ZIP archive "
+                    f"(Content-Type={r.headers.get('Content-Type')!r}, head={preview!r})"
+                )
         else:
             result = self.get_result(session_id)
             for f in result.get("files", []):
@@ -434,12 +496,26 @@ class AutoDataClient:
 
         return os.path.abspath(download_path)
 
-    def download_file(self, url: str, output_path: str) -> None:
+    def download_file(
+        self,
+        url: str,
+        output_path: str,
+        timeout: Optional[float] = None,
+        retries: int = 2,
+    ) -> None:
         """Download a single file by URL (absolute or server-relative).
 
+        Validates Content-Length when the server provides it, and retries on
+        transient network errors. Partial downloads are detected and raised
+        rather than silently succeeding.
+
         Args:
             url:         Full URL or path starting with ``/``.
             output_path: Local file path to write to.
+            timeout:     Per-attempt timeout in seconds. Default scales with
+                         the client's configured timeout; pass an explicit
+                         value for very large files on slow networks.
+            retries:     Number of retry attempts on partial / network errors.
         """
         if not url.startswith("http"):
             url = f"{self.base_url}{url}"
@@ -447,14 +523,172 @@ class AutoDataClient:
         # Ensure parent directory exists
         os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
 
+        effective_timeout = timeout if timeout is not None else max(self.timeout, 900)
+        last_err: Optional[Exception] = None
+
+        for attempt in range(retries + 1):
+            bytes_written = 0
+            expected: Optional[int] = None
+            try:
+                r = self._request(
+                    "GET", url, stream=True, timeout=effective_timeout
+                )
+                self._raise_for_error(r)
+                # Reject 0-byte responses so the caller notices instead of
+                # silently producing an empty file.
+                cl = r.headers.get("Content-Length")
+                if cl is not None:
+                    try:
+                        expected = int(cl)
+                    except (TypeError, ValueError):
+                        expected = None
+                if expected == 0:
+                    raise AutoDataError(
+                        f"Server returned an empty file for {url} "
+                        "(the output may have failed to generate — please retry the run)."
+                    )
+                with open(output_path, "wb") as fh:
+                    for chunk in r.iter_content(chunk_size=65536):
+                        if not chunk:
+                            continue
+                        fh.write(chunk)
+                        bytes_written += len(chunk)
+                if expected is not None and bytes_written != expected:
+                    raise AutoDataError(
+                        f"Partial download: received {bytes_written} of {expected} bytes"
+                    )
+                print(f"Downloaded to {output_path} ({bytes_written} bytes)")
+                return
+            except (requests.ConnectionError, requests.Timeout, AutoDataError) as err:
+                last_err = err
+                if attempt < retries:
+                    backoff = 2 ** attempt
+                    print(f"\nDownload attempt {attempt + 1} failed ({err}); retrying in {backoff}s…")
+                    time.sleep(backoff)
+                    continue
+                raise
+        # Unreachable — loop either returns or re-raises
+        if last_err:
+            raise last_err
+
+    # ------------------------------------------------------------------
+    # DataFrame-native helpers
+    # ------------------------------------------------------------------
+    # These let callers skip the "download to disk, then pd.read_csv()" two-step.
+    # pandas is imported lazily so the SDK stays light when a caller doesn't
+    # need it (the package install_requires only `requests`).
+
+    @staticmethod
+    def _require_pandas():
+        try:
+            import pandas as pd  # noqa: F401
+            return pd
+        except ImportError as e:
+            raise AutoDataError(
+                "pandas is required for DataFrame helpers — install it with `pip install pandas` "
+                "or use download_file()/download_results() to get raw files instead."
+            ) from e
+
+    def get_dataframe(
+        self,
+        session_id: str,
+        output_name: str,
+        *,
+        timeout: Optional[float] = None,
+        **read_csv_kwargs,
+    ):
+        """Fetch one output file and return it as a pandas DataFrame.
+
+        Args:
+            session_id:      Completed processing session.
+            output_name:     File base name (e.g. ``"dsg_output"``, ``"mdh_output"``)
+                             or the full filename (``"dsg_output.csv"``). ``.csv``
+                             is appended if missing.
+            timeout:         Per-request timeout; defaults to the SDK's normal
+                             download timeout (15 min).
+            read_csv_kwargs: Forwarded to ``pandas.read_csv`` (e.g. ``dtype``,
+                             ``parse_dates``, ``usecols``).
+
+        Returns:
+            A pandas.DataFrame.
+
+        Raises:
+            AutoDataError: if pandas isn't installed, the file is missing or
+                           empty (server failed to generate it), or the
+                           download was truncated.
+        """
+        pd = self._require_pandas()
+        fname = output_name if output_name.lower().endswith(
+            (".csv", ".json", ".parquet", ".feather", ".orc")
+        ) else f"{output_name}.csv"
+        url = self._url(f"/download/{session_id}/{fname}")
+        if not url.startswith("http"):
+            url = f"{self.base_url}{url}"
         r = self._request(
-            "GET", url, stream=True, timeout=max(self.timeout, 300)
+            "GET", url, stream=True,
+            timeout=timeout if timeout is not None else max(self.timeout, 900),
         )
         self._raise_for_error(r)
-        with open(output_path, "wb") as fh:
-            for chunk in r.iter_content(chunk_size=65536):
-                fh.write(chunk)
-        print(f"Downloaded to {output_path}")
+        cl = r.headers.get("Content-Length")
+        if cl == "0":
+            raise AutoDataError(
+                f"Server returned an empty file for {fname} — the output likely "
+                "failed to generate. Retry the run or check /result/<session_id>."
+            )
+        # Buffer into memory — pandas.read_csv needs a seekable stream and
+        # output files are bounded by the server-side row ceiling anyway.
+        body = r.content
+        if not body:
+            raise AutoDataError(f"Server returned 0 bytes for {fname}")
+        import io as _io
+        buf = _io.BytesIO(body)
+        low = fname.lower()
+        if low.endswith(".parquet"):
+            return pd.read_parquet(buf)
+        if low.endswith(".feather"):
+            return pd.read_feather(buf)
+        if low.endswith(".orc"):
+            return pd.read_orc(buf)
+        if low.endswith(".json"):
+            return pd.read_json(buf, **read_csv_kwargs)
+        return pd.read_csv(buf, **read_csv_kwargs)
+
+    def get_dataframes(
+        self,
+        session_id: str,
+        names: Optional[List[str]] = None,
+        *,
+        timeout: Optional[float] = None,
+    ) -> Dict:
+        """Fetch multiple output files and return a ``{name: DataFrame}`` dict.
+
+        Args:
+            session_id: Completed processing session.
+            names:      Output base names to fetch. ``None`` means every file
+                        the server lists for the session (skipping PDF
+                        reports and .json metadata files, which aren't
+                        naturally a DataFrame).
+            timeout:    Per-file timeout; see ``get_dataframe``.
+
+        Returns:
+            Dict of ``{file_stem: pandas.DataFrame}``.
+        """
+        self._require_pandas()
+        if names is None:
+            result = self.get_result(session_id)
+            names = []
+            for f in result.get("files", []):
+                fn = f.get("name", "")
+                low = fn.lower()
+                if not low or low.endswith((".pdf", ".json", ".md")):
+                    continue
+                names.append(fn)
+        out: Dict = {}
+        for n in names:
+            # Strip the extension so callers can access via the logical name.
+            key = n.rsplit(".", 1)[0] if "." in n else n
+            out[key] = self.get_dataframe(session_id, n, timeout=timeout)
+        return out
 
     # ------------------------------------------------------------------
     # Account / API key management
@@ -591,6 +825,7 @@ class AutoDataClient:
         compressed: bool = True,
         incremental: bool = False,
         type_casts: Optional[Dict[str, str]] = None,
+        return_dataframes: bool = False,
     ) -> Dict:
         """Run the AutoData pipeline on data from a connector source.
 
@@ -691,6 +926,9 @@ class AutoDataClient:
                 output_preferences=output_preferences,
                 compressed=compressed,
             )
+        if return_dataframes:
+            final = dict(final)
+            final["dataframes"] = self.get_dataframes(session_id, names=output_preferences)
         return final
 
     def write_output(

{datatoolpack-0.5.2 → datatoolpack-0.7.0}/datatoolpack.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatoolpack
-Version: 0.5.2
+Version: 0.7.0
 Summary: Official Python SDK for the AutoData ML data preparation pipeline API
 Home-page: https://autodata.datatoolpack.com
 Author: AutoData Team

{datatoolpack-0.5.2 → datatoolpack-0.7.0}/setup.py

@@ -7,7 +7,7 @@ with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
 
 setup(
     name="datatoolpack",
-    version="0.5.2",
+    version="0.7.0",
     description="Official Python SDK for the AutoData ML data preparation pipeline API",
     long_description=long_description,
     long_description_content_type="text/markdown",