thordata-sdk 1.6.0__tar.gz → 1.7.0__tar.gz

{thordata_sdk-1.6.0/src/thordata_sdk.egg-info → thordata_sdk-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thordata-sdk
-Version: 1.6.0
+Version: 1.7.0
 Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
 Author-email: Thordata Developer Team <support@thordata.com>
 License: MIT

{thordata_sdk-1.6.0 → thordata_sdk-1.7.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "thordata-sdk"
-version = "1.6.0"
+version = "1.7.0"
 description = "The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -115,6 +115,7 @@ check_untyped_defs = false
 strict_optional = false
 show_error_codes = true
 ignore_missing_imports = true
+follow_imports = "skip"
 
 [[tool.mypy.overrides]]
 module = ["aiohttp.*", "requests.*"]

{thordata_sdk-1.6.0 → thordata_sdk-1.7.0}/src/thordata/async_client.py

@@ -293,28 +293,36 @@ class AsyncThordataClient:
         url: str,
         *,
         js_render: bool = False,
-        output_format: str = "html",
+        output_format: str | list[str] = "html",
         country: str | None = None,
         block_resources: str | None = None,
+        clean_content: str | None = None,
         wait: int | None = None,
         wait_for: str | None = None,
+        follow_redirect: bool | None = None,
+        headers: list[dict[str, str]] | None = None,
+        cookies: list[dict[str, str]] | None = None,
         **kwargs: Any,
-    ) -> str | bytes:
+    ) -> str | bytes | dict[str, str | bytes]:
         request = UniversalScrapeRequest(
             url=url,
             js_render=js_render,
             output_format=output_format,
             country=country,
             block_resources=block_resources,
+            clean_content=clean_content,
             wait=wait,
             wait_for=wait_for,
+            follow_redirect=follow_redirect,
+            headers=headers,
+            cookies=cookies,
             extra_params=kwargs,
         )
         return await self.universal_scrape_advanced(request)
 
     async def universal_scrape_advanced(
         self, request: UniversalScrapeRequest
-    ) -> str | bytes:
+    ) -> str | bytes | dict[str, str | bytes]:
         if not self.scraper_token:
             raise ThordataConfigError("scraper_token required")
         payload = request.to_payload()
@@ -327,9 +335,17 @@ class AsyncThordataClient:
         try:
             resp_json = await response.json()
         except ValueError:
-            if request.output_format.lower() == "png":
-                return await response.read()
-            return await response.text()
+            # If not JSON, return raw content based on format
+            if isinstance(request.output_format, list) or (
+                isinstance(request.output_format, str) and "," in request.output_format
+            ):
+                return {"raw": await response.read()}
+            fmt = (
+                request.output_format.lower()
+                if isinstance(request.output_format, str)
+                else str(request.output_format).lower()
+            )
+            return await response.read() if fmt == "png" else await response.text()
 
         if isinstance(resp_json, dict):
             code = resp_json.get("code")
@@ -337,6 +353,27 @@ class AsyncThordataClient:
                 msg = extract_error_message(resp_json)
                 raise_for_code(f"Universal Error: {msg}", code=code, payload=resp_json)
 
+        # Handle multiple output formats
+        if isinstance(request.output_format, list) or (
+            isinstance(request.output_format, str) and "," in request.output_format
+        ):
+            result: dict[str, str | bytes] = {}
+            formats = (
+                request.output_format
+                if isinstance(request.output_format, list)
+                else [f.strip() for f in request.output_format.split(",")]
+            )
+
+            for fmt in formats:
+                fmt_lower = fmt.lower()
+                if fmt_lower == "html" and "html" in resp_json:
+                    result["html"] = resp_json["html"]
+                elif fmt_lower == "png" and "png" in resp_json:
+                    result["png"] = decode_base64_image(resp_json["png"])
+
+            if result:
+                return result
+
         if "html" in resp_json:
             return resp_json["html"]
         if "png" in resp_json:

{thordata_sdk-1.6.0 → thordata_sdk-1.7.0}/src/thordata/client.py

@@ -53,6 +53,7 @@ from .serp_engines import SerpNamespace
 # Import Types (Modernized)
 from .types import (
     CommonSettings,
+    DataFormat,
     ProxyConfig,
     ProxyProduct,
     ProxyServer,
@@ -364,26 +365,36 @@ class ThordataClient:
         url: str,
         *,
         js_render: bool = False,
-        output_format: str = "html",
+        output_format: str | list[str] = "html",
         country: str | None = None,
         block_resources: str | None = None,
+        clean_content: str | None = None,
         wait: int | None = None,
         wait_for: str | None = None,
+        follow_redirect: bool | None = None,
+        headers: list[dict[str, str]] | None = None,
+        cookies: list[dict[str, str]] | None = None,
         **kwargs: Any,
-    ) -> str | bytes:
+    ) -> str | bytes | dict[str, str | bytes]:
         request = UniversalScrapeRequest(
             url=url,
             js_render=js_render,
             output_format=output_format,
             country=country,
             block_resources=block_resources,
+            clean_content=clean_content,
             wait=wait,
             wait_for=wait_for,
+            follow_redirect=follow_redirect,
+            headers=headers,
+            cookies=cookies,
             extra_params=kwargs,
         )
         return self.universal_scrape_advanced(request)
 
-    def universal_scrape_advanced(self, request: UniversalScrapeRequest) -> str | bytes:
+    def universal_scrape_advanced(
+        self, request: UniversalScrapeRequest
+    ) -> str | bytes | dict[str, str | bytes]:
         if not self.scraper_token:
             raise ThordataConfigError("scraper_token required")
 
@@ -648,6 +659,7 @@ class ThordataClient:
         include_errors: bool = True,
         task_type: str = "web",
         common_settings: CommonSettings | None = None,
+        data_format: DataFormat | str | None = None,
     ) -> str:
         import time
 
@@ -671,6 +683,7 @@ class ThordataClient:
                 parameters=parameters,
                 universal_params=universal_params,
                 include_errors=include_errors,
+                data_format=data_format,
             )
             task_id = self.create_scraper_task_advanced(config)
 
@@ -1212,12 +1225,22 @@ class ThordataClient:
     # =========================================================================
 
     def _process_universal_response(
-        self, response: requests.Response, output_format: str
-    ) -> str | bytes:
+        self, response: requests.Response, output_format: str | list[str]
+    ) -> str | bytes | dict[str, str | bytes]:
+        """Process universal scrape response. Returns single value or dict if multiple formats requested."""
         try:
             resp_json = response.json()
         except ValueError:
-            return response.content if output_format.lower() == "png" else response.text
+            # If not JSON, return raw content based on format
+            if isinstance(output_format, list):
+                # Multiple formats requested but got non-JSON response
+                return {"raw": response.content}
+            fmt = (
+                output_format.lower()
+                if isinstance(output_format, str)
+                else str(output_format).lower()
+            )
+            return response.content if fmt == "png" else response.text
 
         if isinstance(resp_json, dict):
             code = resp_json.get("code")
@@ -1225,6 +1248,29 @@ class ThordataClient:
                 msg = extract_error_message(resp_json)
                 raise_for_code(f"Universal Error: {msg}", code=code, payload=resp_json)
 
+        # Handle multiple output formats
+        if isinstance(output_format, list) or (
+            isinstance(output_format, str) and "," in output_format
+        ):
+            result: dict[str, str | bytes] = {}
+            formats = (
+                output_format
+                if isinstance(output_format, list)
+                else [f.strip() for f in output_format.split(",")]
+            )
+
+            for fmt in formats:
+                fmt_lower = fmt.lower()
+                if fmt_lower == "html" and "html" in resp_json:
+                    result["html"] = resp_json["html"]
+                elif fmt_lower == "png" and "png" in resp_json:
+                    result["png"] = decode_base64_image(resp_json["png"])
+
+            # If we got results, return dict; otherwise return single value for backward compatibility
+            if result:
+                return result
+
+        # Single format (backward compatibility)
         if "html" in resp_json:
             return resp_json["html"]
         if "png" in resp_json:

{thordata_sdk-1.6.0 → thordata_sdk-1.7.0}/src/thordata/exceptions.py

@@ -390,7 +390,16 @@ def is_retryable_exception(exc: Exception) -> bool:
     try:
         import requests
 
-        if isinstance(exc, (requests.Timeout, requests.ConnectionError)):
+        # requests exposes SSLError under requests.exceptions.SSLError (not requests.SSLError)
+        ssl_error = getattr(getattr(requests, "exceptions", None), "SSLError", None)
+        retryable: tuple[type[BaseException], ...] = (
+            requests.Timeout,
+            requests.ConnectionError,
+        )
+        if ssl_error is not None:
+            retryable = retryable + (ssl_error,)
+
+        if isinstance(exc, retryable):
             return True
     except ImportError:
         pass

{thordata_sdk-1.6.0 → thordata_sdk-1.7.0}/src/thordata/types/serp.py

@@ -117,7 +117,7 @@ class SerpRequest(ThordataBaseConfig):
     render_js: bool | None = None
     no_cache: bool | None = None
 
-    # Output
+    # Output format: "json" (json=1), "html" (json=3), "light_json" (json=4), or "both" (json=2)
     output_format: str = "json"
 
     # Advanced Google
@@ -155,13 +155,17 @@ class SerpRequest(ThordataBaseConfig):
         }
 
         # JSON output handling
+        # Dashboard mapping: json=1 (json), json=3 (html), json=4 (light json), json=2 (both)
         fmt = self.output_format.lower()
         if fmt == "json":
             payload["json"] = "1"
         elif fmt == "html":
-            pass  # No json param means HTML
+            payload["json"] = "3"
+        elif fmt in ("light_json", "light-json", "lightjson"):
+            payload["json"] = "4"
         elif fmt in ("2", "both", "json+html"):
             payload["json"] = "2"
+        # If no json param is set, default to HTML (legacy behavior)
 
         # Query param handling
         if engine == "yandex":

{thordata_sdk-1.6.0 → thordata_sdk-1.7.0}/src/thordata/types/task.py

@@ -8,6 +8,7 @@ import json
 from dataclasses import dataclass
 from enum import Enum
 from typing import Any
+from urllib.parse import unquote
 
 from .common import CommonSettings, ThordataBaseConfig
 
@@ -49,6 +50,52 @@ class DataFormat(str, Enum):
     XLSX = "xlsx"
 
 
+def _normalize_url_value(value: Any) -> Any:
+    if not isinstance(value, str):
+        return value
+    # Decode all percent-encoded characters to match Dashboard format
+    # Dashboard expects URLs in their raw/decoded form, not URL-encoded
+    # This ensures API/SDK submissions match manual Dashboard input exactly
+    try:
+        # Check if URL contains any percent-encoded characters
+        if "%" in value:
+            # Fully decode the URL to match Dashboard format
+            decoded = unquote(value)
+            # If decoding changed the value, use decoded version
+            # This handles cases like %26 -> &, %3A -> :, %2F -> /, etc.
+            if decoded != value:
+                return decoded
+    except Exception:
+        # If decoding fails, return original value
+        pass
+    return value
+
+
+def _normalize_parameters(params: dict[str, Any]) -> dict[str, Any]:
+    # All parameter keys that contain URLs and should be normalized
+    # This ensures API/SDK submissions match Dashboard format exactly
+    url_keys = {
+        "url",
+        "domain",
+        "profileurl",
+        "posturl",
+        "seller_url",
+        # Additional URL-related keys that may be used
+        "link",
+        "href",
+        "page_url",
+        "product_url",
+        "category_url",
+    }
+    out: dict[str, Any] = {}
+    for k, v in params.items():
+        if k in url_keys:
+            out[k] = _normalize_url_value(v)
+        else:
+            out[k] = v
+    return out
+
+
 @dataclass
 class ScraperTaskConfig(ThordataBaseConfig):
     file_name: str
@@ -57,13 +104,18 @@ class ScraperTaskConfig(ThordataBaseConfig):
     parameters: dict[str, Any] | list[dict[str, Any]]
     universal_params: dict[str, Any] | None = None
     include_errors: bool = True
+    data_format: DataFormat | str | None = (
+        None  # Support json, csv, xlsx output formats
+    )
 
     def to_payload(self) -> dict[str, Any]:
-        # Handle batch parameters: if list, use as is; if dict, wrap in list
+        # Normalize parameters: decode percent-encoded URLs to reduce API/Dashboard divergence
         if isinstance(self.parameters, list):
-            params_json = json.dumps(self.parameters)
+            normalized_list = [_normalize_parameters(p) for p in self.parameters]
+            params_json = json.dumps(normalized_list)
         else:
-            params_json = json.dumps([self.parameters])
+            normalized_one = _normalize_parameters(self.parameters)
+            params_json = json.dumps([normalized_one])
 
         payload: dict[str, Any] = {
             "file_name": self.file_name,
@@ -74,6 +126,14 @@ class ScraperTaskConfig(ThordataBaseConfig):
         }
         if self.universal_params:
             payload["spider_universal"] = json.dumps(self.universal_params)
+        # Add data_format if specified (for json/csv/xlsx output)
+        if self.data_format:
+            fmt = (
+                self.data_format.value
+                if isinstance(self.data_format, DataFormat)
+                else str(self.data_format).lower()
+            )
+            payload["data_format"] = fmt
         return payload
 
 
@@ -87,7 +147,6 @@ class VideoTaskConfig(ThordataBaseConfig):
     include_errors: bool = True
 
     def to_payload(self) -> dict[str, Any]:
-        # Handle batch parameters
         if isinstance(self.parameters, list):
             params_json = json.dumps(self.parameters)
         else:
@@ -99,13 +158,8 @@ class VideoTaskConfig(ThordataBaseConfig):
             "spider_name": self.spider_name,
             "spider_parameters": params_json,
             "spider_errors": "true" if self.include_errors else "false",
-            # v2.0 Doc explicitly requires 'spider_universal' key for video tasks too sometimes,
-            # but usually it's passed as 'common_settings' or 'spider_universal'.
-            # Sticking to original models.py key logic for now to ensure stability.
             "spider_universal": self.common_settings.to_json(),
         }
-        # Note: If API expects 'common_settings' key specifically, adjust here.
-        # Based on v2 context, video builder often uses spider_universal.
         return payload
 
 

{thordata_sdk-1.6.0 → thordata_sdk-1.7.0}/src/thordata/types/universal.py

@@ -15,12 +15,15 @@ from .common import ThordataBaseConfig
 class UniversalScrapeRequest(ThordataBaseConfig):
     url: str
     js_render: bool = False
-    output_format: str = "html"  # 'html' or 'png'
+    output_format: str | list[str] = (
+        "html"  # 'html', 'png', or ['png', 'html'] for both
+    )
     country: str | None = None
-    block_resources: str | None = None  # 'script,image'
+    block_resources: str | None = None  # 'script,image,video'
     clean_content: str | None = None  # 'js,css'
     wait: int | None = None  # ms
     wait_for: str | None = None  # selector
+    follow_redirect: bool | None = None  # Follow redirects
 
     # Headers/Cookies must be serialized to JSON in payload
     headers: list[dict[str, str]] | None = None
@@ -29,12 +32,26 @@ class UniversalScrapeRequest(ThordataBaseConfig):
     extra_params: dict[str, Any] = field(default_factory=dict)
 
     def __post_init__(self) -> None:
+        # Normalize output_format to list for easier handling
+        if isinstance(self.output_format, str):
+            formats = [f.strip().lower() for f in self.output_format.split(",")]
+        else:
+            formats = [
+                f.lower() if isinstance(f, str) else str(f).lower()
+                for f in self.output_format
+            ]
+
         valid_formats = {"html", "png"}
-        if self.output_format.lower() not in valid_formats:
+        invalid = [f for f in formats if f not in valid_formats]
+        if invalid:
             raise ValueError(
-                f"Invalid output_format: {self.output_format}. Must be one of: {valid_formats}"
+                f"Invalid output_format: {invalid}. Must be one or more of: {valid_formats}. "
+                f"Use comma-separated string like 'png,html' or list ['png', 'html'] for multiple formats."
             )
 
+        # Store as list for to_payload
+        self._output_formats = formats
+
         if self.wait is not None and (self.wait < 0 or self.wait > 100000):
             raise ValueError("wait must be between 0 and 100000 milliseconds")
 
@@ -42,9 +59,22 @@ class UniversalScrapeRequest(ThordataBaseConfig):
         payload: dict[str, Any] = {
             "url": self.url,
             "js_render": "True" if self.js_render else "False",
-            "type": self.output_format.lower(),
         }
 
+        # Handle output format: support single or multiple formats (e.g., "png,html")
+        if hasattr(self, "_output_formats") and self._output_formats:
+            if len(self._output_formats) == 1:
+                payload["type"] = self._output_formats[0]
+            else:
+                # Multiple formats: join with comma (e.g., "png,html")
+                payload["type"] = ",".join(self._output_formats)
+        else:
+            # Fallback for backward compatibility
+            if isinstance(self.output_format, str):
+                payload["type"] = self.output_format.lower()
+            else:
+                payload["type"] = ",".join([str(f).lower() for f in self.output_format])
+
         if self.country:
             payload["country"] = self.country.lower()
         if self.block_resources:
@@ -55,6 +85,8 @@ class UniversalScrapeRequest(ThordataBaseConfig):
             payload["wait"] = str(self.wait)
         if self.wait_for:
             payload["wait_for"] = self.wait_for
+        if self.follow_redirect is not None:
+            payload["follow_redirect"] = "True" if self.follow_redirect else "False"
 
         # Serialize complex objects as JSON strings
         if self.headers:

{thordata_sdk-1.6.0 → thordata_sdk-1.7.0/src/thordata_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thordata-sdk
-Version: 1.6.0
+Version: 1.7.0
 Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
 Author-email: Thordata Developer Team <support@thordata.com>
 License: MIT