thordata-sdk 1.2.0__tar.gz → 1.3.0__tar.gz

{thordata_sdk-1.2.0/src/thordata_sdk.egg-info → thordata_sdk-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thordata-sdk
-Version: 1.2.0
+Version: 1.3.0
 Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
 Author-email: Thordata Developer Team <support@thordata.com>
 License: MIT

{thordata_sdk-1.2.0 → thordata_sdk-1.3.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "thordata-sdk"
-version = "1.2.0"
+version = "1.3.0"
 description = "The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network."
 readme = "README.md"
 requires-python = ">=3.9"

{thordata_sdk-1.2.0 → thordata_sdk-1.3.0}/src/thordata/__init__.py

@@ -35,7 +35,7 @@ Async Usage:
     >>> asyncio.run(main())
 """
 
-__version__ = "1.2.0"
+__version__ = "1.3.0"
 __author__ = "Thordata Developer Team"
 __email__ = "support@thordata.com"
 

{thordata_sdk-1.2.0 → thordata_sdk-1.3.0}/src/thordata/async_client.py

@@ -652,7 +652,7 @@ class AsyncThordataClient:
                 self._builder_url, data=payload, headers=headers
             ) as response:
                 response.raise_for_status()
-                data = await response.json()
+                data = await response.json(content_type=None)
 
                 code = data.get("code")
                 if code != 200:
@@ -763,7 +763,7 @@ class AsyncThordataClient:
                 self._status_url, data=payload, headers=headers
             ) as response:
                 response.raise_for_status()
-                data = await response.json()
+                data = await response.json(content_type=None)
 
                 if isinstance(data, dict):
                     code = data.get("code")
@@ -826,7 +826,7 @@ class AsyncThordataClient:
             async with session.post(
                 self._download_url, data=payload, headers=headers
             ) as response:
-                data = await response.json()
+                data = await response.json(content_type=None)
                 code = data.get("code")
 
                 if code == 200 and data.get("data"):
@@ -879,7 +879,7 @@ class AsyncThordataClient:
                 timeout=self._api_timeout,
             ) as response:
                 response.raise_for_status()
-                data = await response.json()
+                data = await response.json(content_type=None)
 
                 code = data.get("code")
                 if code != 200:
@@ -933,6 +933,64 @@ class AsyncThordataClient:
 
         raise TimeoutError(f"Task {task_id} did not complete within {max_wait} seconds")
 
+    async def run_task(
+        self,
+        file_name: str,
+        spider_id: str,
+        spider_name: str,
+        parameters: dict[str, Any],
+        universal_params: dict[str, Any] | None = None,
+        *,
+        max_wait: float = 600.0,
+        initial_poll_interval: float = 2.0,
+        max_poll_interval: float = 10.0,
+        include_errors: bool = True,
+    ) -> str:
+        """
+        Async high-level wrapper to Run a Web Scraper task and wait for result.
+
+        Lifecycle: Create -> Poll (Backoff) -> Get Download URL.
+
+        Returns:
+            str: The download URL.
+        """
+        # 1. Create Task
+        config = ScraperTaskConfig(
+            file_name=file_name,
+            spider_id=spider_id,
+            spider_name=spider_name,
+            parameters=parameters,
+            universal_params=universal_params,
+            include_errors=include_errors,
+        )
+        task_id = await self.create_scraper_task_advanced(config)
+        logger.info(f"Async Task created: {task_id}. Polling...")
+
+        # 2. Poll Status
+        import time
+
+        start_time = time.monotonic()
+        current_poll = initial_poll_interval
+
+        while (time.monotonic() - start_time) < max_wait:
+            status = await self.get_task_status(task_id)
+            status_lower = status.lower()
+
+            if status_lower in {"ready", "success", "finished"}:
+                logger.info(f"Task {task_id} ready.")
+                # 3. Get Result
+                return await self.get_task_result(task_id)
+
+            if status_lower in {"failed", "error", "cancelled"}:
+                raise ThordataNetworkError(
+                    f"Task {task_id} failed with status: {status}"
+                )
+
+            await asyncio.sleep(current_poll)
+            current_poll = min(current_poll * 1.5, max_poll_interval)
+
+        raise ThordataTimeoutError(f"Async Task {task_id} timed out after {max_wait}s")
+
     # =========================================================================
     # Proxy Account Management Methods
     # =========================================================================

{thordata_sdk-1.2.0 → thordata_sdk-1.3.0}/src/thordata/client.py

@@ -1380,6 +1380,83 @@ class ThordataClient:
             time.sleep(poll_interval)
         raise TimeoutError(f"Task {task_id} timeout")
 
+    def run_task(
+        self,
+        file_name: str,
+        spider_id: str,
+        spider_name: str,
+        parameters: dict[str, Any],
+        universal_params: dict[str, Any] | None = None,
+        *,
+        max_wait: float = 600.0,
+        initial_poll_interval: float = 2.0,
+        max_poll_interval: float = 10.0,
+        include_errors: bool = True,
+    ) -> str:
+        """
+        High-level wrapper to Run a Web Scraper task and wait for the result download URL.
+
+        This method handles the entire lifecycle:
+        1. Create Task
+        2. Poll status (with exponential backoff)
+        3. Get download URL when ready
+
+        Args:
+            file_name: Name for the output file.
+            spider_id: Spider identifier from Dashboard.
+            spider_name: Spider name (target domain).
+            parameters: Spider-specific parameters.
+            universal_params: Global spider settings.
+            max_wait: Maximum seconds to wait for task completion (default 600).
+            initial_poll_interval: Starting poll interval in seconds.
+            max_poll_interval: Maximum poll interval cap.
+            include_errors: Whether to include error logs in the task result.
+
+        Returns:
+            str: The download URL for the task result (default JSON).
+
+        Raises:
+            ThordataTimeoutError: If task takes longer than max_wait.
+            ThordataAPIError: If task fails or is cancelled.
+        """
+        import time
+
+        # 1. Create Task
+        config = ScraperTaskConfig(
+            file_name=file_name,
+            spider_id=spider_id,
+            spider_name=spider_name,
+            parameters=parameters,
+            universal_params=universal_params,
+            include_errors=include_errors,
+        )
+        task_id = self.create_scraper_task_advanced(config)
+        logger.info(f"Task created successfully: {task_id}. Waiting for completion...")
+
+        # 2. Poll Status (Smart Backoff)
+        start_time = time.monotonic()
+        current_poll = initial_poll_interval
+
+        while (time.monotonic() - start_time) < max_wait:
+            status = self.get_task_status(task_id)
+            status_lower = status.lower()
+
+            if status_lower in {"ready", "success", "finished"}:
+                logger.info(f"Task {task_id} finished. Status: {status}")
+                # 3. Get Result
+                return self.get_task_result(task_id)
+
+            if status_lower in {"failed", "error", "cancelled"}:
+                raise ThordataNetworkError(
+                    f"Task {task_id} ended with failed status: {status}"
+                )
+
+            # Wait and increase interval (capped)
+            time.sleep(current_poll)
+            current_poll = min(current_poll * 1.5, max_poll_interval)
+
+        raise ThordataTimeoutError(f"Task {task_id} timed out after {max_wait} seconds")
+
     # =========================================================================
     # Account / Locations / Utils
     # =========================================================================

{thordata_sdk-1.2.0 → thordata_sdk-1.3.0/src/thordata_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thordata-sdk
-Version: 1.2.0
+Version: 1.3.0
 Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
 Author-email: Thordata Developer Team <support@thordata.com>
 License: MIT

{thordata_sdk-1.2.0 → thordata_sdk-1.3.0}/tests/test_async_client_errors.py

@@ -28,7 +28,7 @@ class DummyAsyncResponse:
     def raise_for_status(self) -> None:
         pass
 
-    async def json(self) -> dict[str, Any]:
+    async def json(self, **kwargs: Any) -> dict[str, Any]:
         return self._json_data
 
     async def read(self) -> bytes: