thordata-sdk 1.6.0__py3-none-any.whl → 1.8.0__py3-none-any.whl

thordata/__init__.py

@@ -5,7 +5,7 @@ Official Python client for Thordata's Proxy Network, SERP API,
 Universal Scraping API (Web Unlocker), and Web Scraper API.
 """
 
-__version__ = "1.6.0"
+__version__ = "1.8.0"
 __author__ = "Thordata Developer Team/Kael Odin"
 __email__ = "support@thordata.com"
 

thordata/async_client.py

@@ -244,6 +244,7 @@ class AsyncThordataClient:
         render_js: bool | None = None,
         no_cache: bool | None = None,
         output_format: str = "json",
+        ai_overview: bool = False,
         **kwargs: Any,
     ) -> dict[str, Any]:
         engine_str = engine.value if isinstance(engine, Engine) else engine.lower()
@@ -258,13 +259,14 @@ class AsyncThordataClient:
             render_js=render_js,
             no_cache=no_cache,
             output_format=output_format,
+            ai_overview=ai_overview,
             extra_params=kwargs,
         )
         return await self.serp_search_advanced(request)
 
     async def serp_search_advanced(self, request: SerpRequest) -> dict[str, Any]:
         if not self.scraper_token:
-            raise ThordataConfigError("scraper_token required")
+            raise ThordataConfigError("scraper_token is required for SERP API")
         payload = request.to_payload()
         headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
         logger.info(f"Async SERP: {request.engine} - {request.query}")
@@ -293,30 +295,38 @@ class AsyncThordataClient:
         url: str,
         *,
         js_render: bool = False,
-        output_format: str = "html",
+        output_format: str | list[str] = "html",
         country: str | None = None,
         block_resources: str | None = None,
+        clean_content: str | None = None,
         wait: int | None = None,
         wait_for: str | None = None,
+        follow_redirect: bool | None = None,
+        headers: list[dict[str, str]] | None = None,
+        cookies: list[dict[str, str]] | None = None,
         **kwargs: Any,
-    ) -> str | bytes:
+    ) -> str | bytes | dict[str, str | bytes]:
         request = UniversalScrapeRequest(
             url=url,
             js_render=js_render,
             output_format=output_format,
             country=country,
             block_resources=block_resources,
+            clean_content=clean_content,
             wait=wait,
             wait_for=wait_for,
+            follow_redirect=follow_redirect,
+            headers=headers,
+            cookies=cookies,
             extra_params=kwargs,
         )
         return await self.universal_scrape_advanced(request)
 
     async def universal_scrape_advanced(
         self, request: UniversalScrapeRequest
-    ) -> str | bytes:
+    ) -> str | bytes | dict[str, str | bytes]:
         if not self.scraper_token:
-            raise ThordataConfigError("scraper_token required")
+            raise ThordataConfigError("scraper_token is required for Universal API")
         payload = request.to_payload()
         headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
 
@@ -327,9 +337,17 @@ class AsyncThordataClient:
         try:
             resp_json = await response.json()
         except ValueError:
-            if request.output_format.lower() == "png":
-                return await response.read()
-            return await response.text()
+            # If not JSON, return raw content based on format
+            if isinstance(request.output_format, list) or (
+                isinstance(request.output_format, str) and "," in request.output_format
+            ):
+                return {"raw": await response.read()}
+            fmt = (
+                request.output_format.lower()
+                if isinstance(request.output_format, str)
+                else str(request.output_format).lower()
+            )
+            return await response.read() if fmt == "png" else await response.text()
 
         if isinstance(resp_json, dict):
             code = resp_json.get("code")
@@ -337,6 +355,27 @@ class AsyncThordataClient:
                 msg = extract_error_message(resp_json)
                 raise_for_code(f"Universal Error: {msg}", code=code, payload=resp_json)
 
+        # Handle multiple output formats
+        if isinstance(request.output_format, list) or (
+            isinstance(request.output_format, str) and "," in request.output_format
+        ):
+            result: dict[str, str | bytes] = {}
+            formats = (
+                request.output_format
+                if isinstance(request.output_format, list)
+                else [f.strip() for f in request.output_format.split(",")]
+            )
+
+            for fmt in formats:
+                fmt_lower = fmt.lower()
+                if fmt_lower == "html" and "html" in resp_json:
+                    result["html"] = resp_json["html"]
+                elif fmt_lower == "png" and "png" in resp_json:
+                    result["png"] = decode_base64_image(resp_json["png"])
+
+            if result:
+                return result
+
         if "html" in resp_json:
             return resp_json["html"]
         if "png" in resp_json:
@@ -411,7 +450,7 @@ class AsyncThordataClient:
     async def create_scraper_task_advanced(self, config: ScraperTaskConfig) -> str:
         self._require_public_credentials()
         if not self.scraper_token:
-            raise ThordataConfigError("scraper_token required")
+            raise ThordataConfigError("scraper_token is required for Task Builder")
         payload = config.to_payload()
         headers = build_builder_headers(
             self.scraper_token, str(self.public_token), str(self.public_key)
@@ -449,7 +488,9 @@ class AsyncThordataClient:
     async def create_video_task_advanced(self, config: VideoTaskConfig) -> str:
         self._require_public_credentials()
         if not self.scraper_token:
-            raise ThordataConfigError("scraper_token required")
+            raise ThordataConfigError(
+                "scraper_token is required for Video Task Builder"
+            )
         payload = config.to_payload()
         headers = build_builder_headers(
             self.scraper_token, str(self.public_token), str(self.public_key)
@@ -1067,3 +1108,28 @@ class AsyncThordataClient:
         safe_user = quote(final_user, safe="")
         safe_pass = quote(pwd, safe="")
         return f"wss://{safe_user}:{safe_pass}@ws-browser.thordata.com"
+
+    @property
+    def browser(self):
+        """Get a browser session for automation.
+
+        Requires playwright: pip install thordata[browser]
+
+        Returns:
+            BrowserSession instance
+
+        Example:
+            async with AsyncThordataClient() as client:
+                session = client.browser
+                await session.navigate("https://example.com")
+                snapshot = await session.snapshot()
+        """
+        try:
+            from .browser import BrowserSession
+
+            return BrowserSession(self)
+        except ImportError as e:
+            raise ImportError(
+                "Playwright is required for browser automation. "
+                "Install it with: pip install thordata[browser]"
+            ) from e

thordata/browser/__init__.py

@@ -0,0 +1,16 @@
+"""Browser automation module for Thordata Scraping Browser.
+
+This module provides high-level browser automation capabilities using Playwright.
+Requires optional dependency: pip install thordata[browser]
+"""
+
+from __future__ import annotations
+
+try:
+    from .exceptions import BrowserConnectionError, BrowserError
+    from .session import BrowserSession
+
+    __all__ = ["BrowserSession", "BrowserError", "BrowserConnectionError"]
+except ImportError:
+    # Playwright not installed
+    __all__ = []

thordata/browser/exceptions.py

@@ -0,0 +1,23 @@
+"""Browser automation exceptions."""
+
+from __future__ import annotations
+
+from ..exceptions import ThordataError
+
+
+class BrowserError(ThordataError):
+    """Base exception for browser automation errors."""
+
+    pass
+
+
+class BrowserConnectionError(BrowserError):
+    """Raised when browser connection fails."""
+
+    pass
+
+
+class BrowserSessionError(BrowserError):
+    """Raised when browser session operations fail."""
+
+    pass

thordata/browser/session.py

@@ -0,0 +1,469 @@
+"""Browser session management for Thordata Scraping Browser.
+
+This module provides a high-level wrapper around Playwright connected to
+Thordata's Scraping Browser.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+from urllib.parse import urlparse
+
+try:
+    from playwright.async_api import Browser, Page, Playwright, async_playwright
+except ImportError as e:
+    raise ImportError(
+        "Playwright is required for browser automation. "
+        "Install it with: pip install thordata[browser]"
+    ) from e
+
+from ..async_client import AsyncThordataClient
+from .exceptions import BrowserConnectionError, BrowserSessionError
+
+logger = logging.getLogger(__name__)
+
+
+class BrowserSession:
+    """Domain-aware browser session wrapper for Thordata Scraping Browser."""
+
+    def __init__(
+        self,
+        client: AsyncThordataClient,
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
+        """Initialize browser session.
+
+        Args:
+            client: AsyncThordataClient instance
+            username: Browser username (optional, can use env var)
+            password: Browser password (optional, can use env var)
+        """
+        self._client = client
+        self._username = username
+        self._password = password
+        self._playwright: Playwright | None = None
+        self._browsers: dict[str, Browser] = {}
+        self._pages: dict[str, Page] = {}
+        self._current_domain: str = "default"
+
+    @staticmethod
+    def _get_domain(url: str) -> str:
+        """Extract domain from URL."""
+        try:
+            parsed = urlparse(url)
+            return parsed.hostname or "default"
+        except Exception:
+            return "default"
+
+    async def _ensure_playwright(self) -> Playwright:
+        """Ensure Playwright is started."""
+        if self._playwright is None:
+            self._playwright = await async_playwright().start()
+        return self._playwright
+
+    async def get_browser(self, domain: str = "default") -> Browser:
+        """Get or create a browser instance for a given domain."""
+        existing = self._browsers.get(domain)
+        if existing and existing.is_connected():
+            return existing
+
+        # Clean up stale browser/page
+        if existing is not None:
+            logger.info("Browser for domain %s disconnected, recreating", domain)
+            self._browsers.pop(domain, None)
+            self._pages.pop(domain, None)
+
+        playwright = await self._ensure_playwright()
+
+        logger.info("Connecting to Thordata Scraping Browser for domain %s", domain)
+
+        # Get browser credentials
+        import os
+
+        user = self._username or os.getenv("THORDATA_BROWSER_USERNAME")
+        pwd = self._password or os.getenv("THORDATA_BROWSER_PASSWORD")
+
+        if not user or not pwd:
+            raise BrowserConnectionError(
+                "Missing browser credentials. Set THORDATA_BROWSER_USERNAME and "
+                "THORDATA_BROWSER_PASSWORD or pass them to BrowserSession."
+            )
+
+        # Retry logic with exponential backoff
+        max_retries = 3
+        last_error = None
+
+        for attempt in range(max_retries):
+            try:
+                ws_url = self._client.get_browser_connection_url(
+                    username=user, password=pwd
+                )
+                logger.debug(
+                    "Attempt %d/%d: Connecting to %s...",
+                    attempt + 1,
+                    max_retries,
+                    ws_url[:50],
+                )
+                browser = await playwright.chromium.connect_over_cdp(ws_url)
+                logger.info("Successfully connected to browser for domain %s", domain)
+                self._browsers[domain] = browser
+                return browser
+            except Exception as e:
+                last_error = e
+                logger.warning(
+                    "Browser connection attempt %d/%d failed: %s",
+                    attempt + 1,
+                    max_retries,
+                    e,
+                )
+
+                if attempt < max_retries - 1:
+                    import asyncio
+
+                    wait_time = 2**attempt  # Exponential backoff: 1s, 2s, 4s
+                    logger.info("Retrying in %d seconds...", wait_time)
+                    await asyncio.sleep(wait_time)
+
+        # If all retries failed, raise the last error
+        raise BrowserConnectionError(
+            f"Failed to connect to Thordata Scraping Browser after {max_retries} attempts. "
+            f"Last error: {last_error}"
+        ) from last_error
+
+    async def get_page(self, url: str | None = None) -> Page:
+        """Get or create a page for the current (or provided) domain."""
+        if url:
+            self._current_domain = self._get_domain(url)
+        domain = self._current_domain
+
+        existing = self._pages.get(domain)
+        if existing and not existing.is_closed():
+            return existing
+
+        browser = await self.get_browser(domain)
+        contexts = browser.contexts
+        if not contexts:
+            context = await browser.new_context()
+        else:
+            context = contexts[0]
+
+        pages = context.pages
+        if pages:
+            page = pages[0]
+        else:
+            page = await context.new_page()
+
+        self._pages[domain] = page
+        return page
+
+    async def navigate(self, url: str, timeout: int = 120000) -> dict[str, Any]:
+        """Navigate to a URL.
+
+        Args:
+            url: Target URL
+            timeout: Navigation timeout in milliseconds
+
+        Returns:
+            Dictionary with url and title
+        """
+        page = await self.get_page(url)
+        if page.url != url:
+            await page.goto(url, timeout=timeout)
+        title = await page.title()
+        return {"url": page.url, "title": title}
+
+    async def snapshot(
+        self, filtered: bool = True, max_items: int = 80
+    ) -> dict[str, Any]:
+        """Capture an ARIA-like snapshot of the current page.
+
+        Args:
+            filtered: Whether to filter to interactive elements only
+            max_items: Maximum number of elements to include
+
+        Returns:
+            Dictionary with url, title, and aria_snapshot
+        """
+        page = await self.get_page()
+        full_snapshot = await self._get_interactive_snapshot(page)
+
+        if not filtered:
+            return {
+                "url": page.url,
+                "title": await page.title(),
+                "aria_snapshot": full_snapshot,
+            }
+
+        # Filter and limit
+        filtered_snapshot = self._filter_snapshot(full_snapshot)
+        filtered_snapshot = self._limit_snapshot_items(
+            filtered_snapshot, max_items=max_items
+        )
+
+        return {
+            "url": page.url,
+            "title": await page.title(),
+            "aria_snapshot": filtered_snapshot,
+        }
+
+    async def click_ref(
+        self, ref: str, wait_for_navigation_ms: int | None = None
+    ) -> dict[str, Any]:
+        """Click an element by its ref ID.
+
+        Args:
+            ref: The ref ID from snapshot (e.g., "1" or "dom-1")
+            wait_for_navigation_ms: Optional wait time in ms to detect navigation
+
+        Returns:
+            Dictionary with click result information
+        """
+        page = await self.get_page()
+        url_before = page.url
+
+        try:
+            locator = page.locator(f'[data-fastmcp-ref="{ref}"]').first
+            await locator.click(timeout=5000)
+
+            # Check for navigation if requested
+            did_navigate = False
+            url_after = url_before
+            if wait_for_navigation_ms and wait_for_navigation_ms > 0:
+                import asyncio
+
+                await asyncio.sleep(wait_for_navigation_ms / 1000)
+                url_after = page.url
+                did_navigate = url_after != url_before
+
+            return {
+                "message": "Successfully clicked element",
+                "ref": ref,
+                "url_before": url_before,
+                "url_after": url_after,
+                "did_navigate": did_navigate,
+            }
+        except Exception as e:
+            raise BrowserSessionError(f"Failed to click element: {e}") from e
+
+    async def type_ref(
+        self, ref: str, text: str, submit: bool = False
+    ) -> dict[str, Any]:
+        """Type text into an element by its ref ID.
+
+        Args:
+            ref: The ref ID from snapshot
+            text: Text to type
+            submit: Whether to press Enter after typing
+
+        Returns:
+            Dictionary with type result information
+        """
+        page = await self.get_page()
+        url_before = page.url
+
+        try:
+            locator = page.locator(f'[data-fastmcp-ref="{ref}"]').first
+            await locator.fill(text)
+            if submit:
+                await locator.press("Enter")
+
+            return {
+                "message": "Typed into element" + (" and submitted" if submit else ""),
+                "ref": ref,
+                "url_before": url_before,
+                "url_after": page.url,
+            }
+        except Exception as e:
+            raise BrowserSessionError(f"Failed to type into element: {e}") from e
+
+    async def screenshot_page(self, full_page: bool = False) -> bytes:
+        """Take a screenshot of the current page.
+
+        Args:
+            full_page: Whether to capture full page or viewport only
+
+        Returns:
+            Screenshot as bytes (PNG format)
+        """
+        page = await self.get_page()
+        return await page.screenshot(full_page=full_page)
+
+    async def get_html(self, full_page: bool = False) -> str:
+        """Get the HTML content of the current page.
+
+        Args:
+            full_page: Whether to get full page HTML or body only
+
+        Returns:
+            HTML content as string
+        """
+        page = await self.get_page()
+        if full_page:
+            return await page.content()
+        else:
+            try:
+                return await page.evaluate("document.body.innerHTML")
+            except Exception:
+                return await page.content()
+
+    async def scroll(self) -> dict[str, Any]:
+        """Scroll to the bottom of the page.
+
+        Returns:
+            Dictionary with scroll result
+        """
+        page = await self.get_page()
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        return {"message": "Scrolled to bottom"}
+
+    async def go_back(self) -> dict[str, Any]:
+        """Navigate back in browser history.
+
+        Returns:
+            Dictionary with new URL
+        """
+        page = await self.get_page()
+        await page.go_back()
+        return {"url": page.url}
+
+    async def _get_interactive_snapshot(self, page: Page) -> str:
+        """Generate a text snapshot of interactive elements with refs."""
+        script = """
+        () => {
+            function getSnapshot() {
+                const lines = [];
+                let refCounter = 0;
+
+                function normalizeRole(tag, explicitRole) {
+                    const role = (explicitRole || '').toLowerCase();
+                    const t = (tag || '').toLowerCase();
+                    if (role) return role;
+                    if (t === 'a') return 'link';
+                    if (t === 'button') return 'button';
+                    if (t === 'input') return 'textbox';
+                    if (t === 'select') return 'combobox';
+                    if (t === 'textarea') return 'textbox';
+                    return t;
+                }
+
+                function traverse(node) {
+                    if (node.nodeType === Node.ELEMENT_NODE) {
+                        const tag = node.tagName.toLowerCase();
+                        const interactiveTag = ['a', 'button', 'input', 'select', 'textarea'].includes(tag);
+                        const role = normalizeRole(tag, node.getAttribute('role'));
+                        const interactiveRole = ['button', 'link', 'textbox', 'searchbox', 'combobox', 'checkbox', 'radio', 'switch', 'tab', 'menuitem', 'option'].includes(role);
+
+                        if (interactiveTag || interactiveRole) {
+                            if (!node.dataset.fastmcpRef) {
+                                node.dataset.fastmcpRef = (++refCounter).toString();
+                            }
+                            let name = node.innerText || node.getAttribute('aria-label') || '';
+                            name = (name || '').replace(/\\s+/g, ' ').trim().substring(0, 80);
+
+                            lines.push(`- ${role} "${name}" [ref=${node.dataset.fastmcpRef}]`);
+                            if (node.href) {
+                                lines.push(`  /url: "${node.href}"`);
+                            }
+                        }
+                    }
+
+                    node.childNodes.forEach(child => traverse(child));
+                }
+
+                traverse(document.body);
+                return lines.join('\\n');
+            }
+            return getSnapshot();
+        }
+        """
+        return await page.evaluate(script)
+
+    @staticmethod
+    def _filter_snapshot(snapshot_text: str) -> str:
+        """Filter snapshot to interactive elements only."""
+        import re
+
+        lines = snapshot_text.split("\n")
+        filtered = []
+        i = 0
+        while i < len(lines):
+            line = lines[i]
+            trimmed = line.strip()
+
+            if not trimmed or not trimmed.startswith("-"):
+                i += 1
+                continue
+
+            # Extract role
+            role_match = re.match(r"^-\s+([a-zA-Z]+)", trimmed)
+            if not role_match:
+                i += 1
+                continue
+
+            role = role_match.group(1).lower()
+            interactive_roles = {
+                "button",
+                "link",
+                "textbox",
+                "searchbox",
+                "combobox",
+                "checkbox",
+                "radio",
+                "switch",
+                "tab",
+                "menuitem",
+                "option",
+            }
+
+            if role in interactive_roles:
+                filtered.append(line)
+                # Include next line if it's a URL
+                if i + 1 < len(lines) and "/url:" in lines[i + 1]:
+                    filtered.append(lines[i + 1])
+                    i += 1
+
+            i += 1
+
+        return "\n".join(filtered)
+
+    @staticmethod
+    def _limit_snapshot_items(text: str, *, max_items: int) -> str:
+        """Limit snapshot to the first N interactive element blocks."""
+        if max_items <= 0:
+            return ""
+        if not text:
+            return text
+
+        lines = text.splitlines()
+        out: list[str] = []
+        items = 0
+        for line in lines:
+            if line.startswith("- ") or line.startswith("["):
+                if items >= max_items:
+                    break
+                items += 1
+            if items > 0:
+                out.append(line)
+        return "\n".join(out).strip()
+
+    async def close(self) -> None:
+        """Cleanly close all pages, browsers, and Playwright."""
+        import contextlib
+
+        for page in list(self._pages.values()):
+            with contextlib.suppress(Exception):
+                await page.close()
+        self._pages.clear()
+
+        for browser in list(self._browsers.values()):
+            with contextlib.suppress(Exception):
+                await browser.close()
+        self._browsers.clear()
+
+        if self._playwright is not None:
+            try:
+                await self._playwright.stop()
+            finally:
+                self._playwright = None

thordata/client.py

@@ -53,6 +53,7 @@ from .serp_engines import SerpNamespace
 # Import Types (Modernized)
 from .types import (
     CommonSettings,
+    DataFormat,
     ProxyConfig,
     ProxyProduct,
     ProxyServer,
@@ -308,6 +309,7 @@ class ThordataClient:
         render_js: bool | None = None,
         no_cache: bool | None = None,
         output_format: str = "json",
+        ai_overview: bool = False,
         **kwargs: Any,
     ) -> dict[str, Any]:
         engine_str = engine.value if isinstance(engine, Engine) else engine.lower()
@@ -323,6 +325,7 @@ class ThordataClient:
             render_js=render_js,
             no_cache=no_cache,
             output_format=output_format,
+            ai_overview=ai_overview,
             extra_params=kwargs,
         )
         return self.serp_search_advanced(request)
@@ -364,28 +367,38 @@ class ThordataClient:
         url: str,
         *,
         js_render: bool = False,
-        output_format: str = "html",
+        output_format: str | list[str] = "html",
         country: str | None = None,
         block_resources: str | None = None,
+        clean_content: str | None = None,
         wait: int | None = None,
         wait_for: str | None = None,
+        follow_redirect: bool | None = None,
+        headers: list[dict[str, str]] | None = None,
+        cookies: list[dict[str, str]] | None = None,
         **kwargs: Any,
-    ) -> str | bytes:
+    ) -> str | bytes | dict[str, str | bytes]:
         request = UniversalScrapeRequest(
             url=url,
             js_render=js_render,
             output_format=output_format,
             country=country,
             block_resources=block_resources,
+            clean_content=clean_content,
             wait=wait,
             wait_for=wait_for,
+            follow_redirect=follow_redirect,
+            headers=headers,
+            cookies=cookies,
             extra_params=kwargs,
         )
         return self.universal_scrape_advanced(request)
 
-    def universal_scrape_advanced(self, request: UniversalScrapeRequest) -> str | bytes:
+    def universal_scrape_advanced(
+        self, request: UniversalScrapeRequest
+    ) -> str | bytes | dict[str, str | bytes]:
         if not self.scraper_token:
-            raise ThordataConfigError("scraper_token required")
+            raise ThordataConfigError("scraper_token is required for Universal API")
 
         payload = request.to_payload()
         headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
@@ -648,6 +661,7 @@ class ThordataClient:
         include_errors: bool = True,
         task_type: str = "web",
         common_settings: CommonSettings | None = None,
+        data_format: DataFormat | str | None = None,
     ) -> str:
         import time
 
@@ -671,6 +685,7 @@ class ThordataClient:
                 parameters=parameters,
                 universal_params=universal_params,
                 include_errors=include_errors,
+                data_format=data_format,
             )
             task_id = self.create_scraper_task_advanced(config)
 
@@ -1212,12 +1227,22 @@ class ThordataClient:
     # =========================================================================
 
     def _process_universal_response(
-        self, response: requests.Response, output_format: str
-    ) -> str | bytes:
+        self, response: requests.Response, output_format: str | list[str]
+    ) -> str | bytes | dict[str, str | bytes]:
+        """Process universal scrape response. Returns single value or dict if multiple formats requested."""
         try:
             resp_json = response.json()
         except ValueError:
-            return response.content if output_format.lower() == "png" else response.text
+            # If not JSON, return raw content based on format
+            if isinstance(output_format, list):
+                # Multiple formats requested but got non-JSON response
+                return {"raw": response.content}
+            fmt = (
+                output_format.lower()
+                if isinstance(output_format, str)
+                else str(output_format).lower()
+            )
+            return response.content if fmt == "png" else response.text
 
         if isinstance(resp_json, dict):
             code = resp_json.get("code")
@@ -1225,6 +1250,29 @@ class ThordataClient:
                 msg = extract_error_message(resp_json)
                 raise_for_code(f"Universal Error: {msg}", code=code, payload=resp_json)
 
+        # Handle multiple output formats
+        if isinstance(output_format, list) or (
+            isinstance(output_format, str) and "," in output_format
+        ):
+            result: dict[str, str | bytes] = {}
+            formats = (
+                output_format
+                if isinstance(output_format, list)
+                else [f.strip() for f in output_format.split(",")]
+            )
+
+            for fmt in formats:
+                fmt_lower = fmt.lower()
+                if fmt_lower == "html" and "html" in resp_json:
+                    result["html"] = resp_json["html"]
+                elif fmt_lower == "png" and "png" in resp_json:
+                    result["png"] = decode_base64_image(resp_json["png"])
+
+            # If we got results, return dict; otherwise return single value for backward compatibility
+            if result:
+                return result
+
+        # Single format (backward compatibility)
         if "html" in resp_json:
             return resp_json["html"]
         if "png" in resp_json:

thordata/exceptions.py

@@ -390,7 +390,16 @@ def is_retryable_exception(exc: Exception) -> bool:
     try:
         import requests
 
-        if isinstance(exc, (requests.Timeout, requests.ConnectionError)):
+        # requests exposes SSLError under requests.exceptions.SSLError (not requests.SSLError)
+        ssl_error = getattr(getattr(requests, "exceptions", None), "SSLError", None)
+        retryable: tuple[type[BaseException], ...] = (
+            requests.Timeout,
+            requests.ConnectionError,
+        )
+        if ssl_error is not None:
+            retryable = retryable + (ssl_error,)
+
+        if isinstance(exc, retryable):
             return True
     except ImportError:
         pass

thordata/types/serp.py

@@ -14,6 +14,7 @@ from .common import ThordataBaseConfig
 class Engine(str, Enum):
     # Google
     GOOGLE = "google"
+    GOOGLE_AI_MODE = "google_ai_mode"
     GOOGLE_NEWS = "google_news"
     GOOGLE_SHOPPING = "google_shopping"
     GOOGLE_VIDEOS = "google_videos"
@@ -21,10 +22,18 @@ class Engine(str, Enum):
     GOOGLE_MAPS = "google_maps"
     GOOGLE_JOBS = "google_jobs"
     GOOGLE_PLAY = "google_play"
+    GOOGLE_PLAY_PRODUCT = "google_play_product"
+    GOOGLE_PLAY_GAMES = "google_play_games"
+    GOOGLE_PLAY_MOVIES = "google_play_movies"
+    GOOGLE_PLAY_BOOKS = "google_play_books"
     GOOGLE_TRENDS = "google_trends"
     GOOGLE_SCHOLAR = "google_scholar"
+    GOOGLE_SCHOLAR_CITE = "google_scholar_cite"
+    GOOGLE_SCHOLAR_AUTHOR = "google_scholar_author"
     GOOGLE_PATENTS = "google_patents"
+    GOOGLE_PATENTS_DETAILS = "google_patents_details"
     GOOGLE_FINANCE = "google_finance"
+    GOOGLE_FINANCE_MARKETS = "google_finance_markets"
     GOOGLE_FLIGHTS = "google_flights"
     GOOGLE_LENS = "google_lens"
     GOOGLE_HOTELS = "google_hotels"
@@ -40,7 +49,7 @@ class Engine(str, Enum):
     # Others
     YANDEX = "yandex"
     DUCKDUCKGO = "duckduckgo"
-    BAIDU = "baidu"
+    BAIDU = "baidu"  # Deprecated: Not supported by Dashboard
 
     # Legacy / Compatibility Aliases
     GOOGLE_SEARCH = "google_search"
@@ -117,12 +126,14 @@ class SerpRequest(ThordataBaseConfig):
     render_js: bool | None = None
     no_cache: bool | None = None
 
-    # Output
+    # Output format: "json" (json=1), "html" (json=3), "light_json" (json=4)
+    # Note: "both" (json=2) format is not supported by Dashboard
     output_format: str = "json"
 
     # Advanced Google
     ludocid: str | None = None
     kgmid: str | None = None
+    ai_overview: bool = False  # Only supported for engine=google
 
     # Pass-through for any other param
     extra_params: dict[str, Any] = field(default_factory=dict)
@@ -155,13 +166,26 @@ class SerpRequest(ThordataBaseConfig):
         }
 
         # JSON output handling
+        # Dashboard mapping: json=1 (json), json=3 (html), json=4 (light json)
+        # Note: json=2 (both) format is not supported by Dashboard
         fmt = self.output_format.lower()
         if fmt == "json":
             payload["json"] = "1"
         elif fmt == "html":
-            pass  # No json param means HTML
+            payload["json"] = "3"
+        elif fmt in ("light_json", "light-json", "lightjson"):
+            payload["json"] = "4"
         elif fmt in ("2", "both", "json+html"):
+            import warnings
+
+            warnings.warn(
+                "The 'both' output format (json=2) is not supported by Dashboard. "
+                "Use 'json' or 'html' instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
             payload["json"] = "2"
+        # If no json param is set, default to HTML (legacy behavior)
 
         # Query param handling
         if engine == "yandex":
@@ -219,6 +243,14 @@ class SerpRequest(ThordataBaseConfig):
         if self.kgmid:
             payload["kgmid"] = self.kgmid
 
+        # AI Overview (only for Google engine)
+        if self.ai_overview:
+            if engine != "google":
+                raise ValueError(
+                    "ai_overview parameter is only supported for engine=google"
+                )
+            payload["ai_overview"] = "true"
+
         # Merge extras
         payload.update(self.extra_params)
         return payload

thordata/types/task.py

@@ -8,6 +8,7 @@ import json
 from dataclasses import dataclass
 from enum import Enum
 from typing import Any
+from urllib.parse import unquote
 
 from .common import CommonSettings, ThordataBaseConfig
 
@@ -49,6 +50,52 @@ class DataFormat(str, Enum):
     XLSX = "xlsx"
 
 
+def _normalize_url_value(value: Any) -> Any:
+    if not isinstance(value, str):
+        return value
+    # Decode all percent-encoded characters to match Dashboard format
+    # Dashboard expects URLs in their raw/decoded form, not URL-encoded
+    # This ensures API/SDK submissions match manual Dashboard input exactly
+    try:
+        # Check if URL contains any percent-encoded characters
+        if "%" in value:
+            # Fully decode the URL to match Dashboard format
+            decoded = unquote(value)
+            # If decoding changed the value, use decoded version
+            # This handles cases like %26 -> &, %3A -> :, %2F -> /, etc.
+            if decoded != value:
+                return decoded
+    except Exception:
+        # If decoding fails, return original value
+        pass
+    return value
+
+
+def _normalize_parameters(params: dict[str, Any]) -> dict[str, Any]:
+    # All parameter keys that contain URLs and should be normalized
+    # This ensures API/SDK submissions match Dashboard format exactly
+    url_keys = {
+        "url",
+        "domain",
+        "profileurl",
+        "posturl",
+        "seller_url",
+        # Additional URL-related keys that may be used
+        "link",
+        "href",
+        "page_url",
+        "product_url",
+        "category_url",
+    }
+    out: dict[str, Any] = {}
+    for k, v in params.items():
+        if k in url_keys:
+            out[k] = _normalize_url_value(v)
+        else:
+            out[k] = v
+    return out
+
+
 @dataclass
 class ScraperTaskConfig(ThordataBaseConfig):
     file_name: str
@@ -57,13 +104,18 @@ class ScraperTaskConfig(ThordataBaseConfig):
     parameters: dict[str, Any] | list[dict[str, Any]]
     universal_params: dict[str, Any] | None = None
     include_errors: bool = True
+    data_format: DataFormat | str | None = (
+        None  # Support json, csv, xlsx output formats
+    )
 
     def to_payload(self) -> dict[str, Any]:
-        # Handle batch parameters: if list, use as is; if dict, wrap in list
+        # Normalize parameters: decode percent-encoded URLs to reduce API/Dashboard divergence
         if isinstance(self.parameters, list):
-            params_json = json.dumps(self.parameters)
+            normalized_list = [_normalize_parameters(p) for p in self.parameters]
+            params_json = json.dumps(normalized_list)
         else:
-            params_json = json.dumps([self.parameters])
+            normalized_one = _normalize_parameters(self.parameters)
+            params_json = json.dumps([normalized_one])
 
         payload: dict[str, Any] = {
             "file_name": self.file_name,
@@ -74,6 +126,14 @@ class ScraperTaskConfig(ThordataBaseConfig):
         }
         if self.universal_params:
             payload["spider_universal"] = json.dumps(self.universal_params)
+        # Add data_format if specified (for json/csv/xlsx output)
+        if self.data_format:
+            fmt = (
+                self.data_format.value
+                if isinstance(self.data_format, DataFormat)
+                else str(self.data_format).lower()
+            )
+            payload["data_format"] = fmt
         return payload
 
 
@@ -87,7 +147,6 @@ class VideoTaskConfig(ThordataBaseConfig):
     include_errors: bool = True
 
     def to_payload(self) -> dict[str, Any]:
-        # Handle batch parameters
         if isinstance(self.parameters, list):
             params_json = json.dumps(self.parameters)
         else:
@@ -99,13 +158,8 @@ class VideoTaskConfig(ThordataBaseConfig):
             "spider_name": self.spider_name,
             "spider_parameters": params_json,
             "spider_errors": "true" if self.include_errors else "false",
-            # v2.0 Doc explicitly requires 'spider_universal' key for video tasks too sometimes,
-            # but usually it's passed as 'common_settings' or 'spider_universal'.
-            # Sticking to original models.py key logic for now to ensure stability.
             "spider_universal": self.common_settings.to_json(),
         }
-        # Note: If API expects 'common_settings' key specifically, adjust here.
-        # Based on v2 context, video builder often uses spider_universal.
         return payload
 
 

thordata/types/universal.py

@@ -15,12 +15,15 @@ from .common import ThordataBaseConfig
 class UniversalScrapeRequest(ThordataBaseConfig):
     url: str
     js_render: bool = False
-    output_format: str = "html"  # 'html' or 'png'
+    output_format: str | list[str] = (
+        "html"  # 'html', 'png', or ['png', 'html'] for both
+    )
     country: str | None = None
-    block_resources: str | None = None  # 'script,image'
+    block_resources: str | None = None  # 'script,image,video'
     clean_content: str | None = None  # 'js,css'
     wait: int | None = None  # ms
     wait_for: str | None = None  # selector
+    follow_redirect: bool | None = None  # Follow redirects
 
     # Headers/Cookies must be serialized to JSON in payload
     headers: list[dict[str, str]] | None = None
@@ -29,12 +32,26 @@ class UniversalScrapeRequest(ThordataBaseConfig):
     extra_params: dict[str, Any] = field(default_factory=dict)
 
     def __post_init__(self) -> None:
+        # Normalize output_format to list for easier handling
+        if isinstance(self.output_format, str):
+            formats = [f.strip().lower() for f in self.output_format.split(",")]
+        else:
+            formats = [
+                f.lower() if isinstance(f, str) else str(f).lower()
+                for f in self.output_format
+            ]
+
         valid_formats = {"html", "png"}
-        if self.output_format.lower() not in valid_formats:
+        invalid = [f for f in formats if f not in valid_formats]
+        if invalid:
             raise ValueError(
-                f"Invalid output_format: {self.output_format}. Must be one of: {valid_formats}"
+                f"Invalid output_format: {invalid}. Must be one or more of: {valid_formats}. "
+                f"Use comma-separated string like 'png,html' or list ['png', 'html'] for multiple formats."
             )
 
+        # Store as list for to_payload
+        self._output_formats = formats
+
         if self.wait is not None and (self.wait < 0 or self.wait > 100000):
             raise ValueError("wait must be between 0 and 100000 milliseconds")
 
@@ -42,9 +59,22 @@ class UniversalScrapeRequest(ThordataBaseConfig):
         payload: dict[str, Any] = {
             "url": self.url,
             "js_render": "True" if self.js_render else "False",
-            "type": self.output_format.lower(),
         }
 
+        # Handle output format: support single or multiple formats (e.g., "png,html")
+        if hasattr(self, "_output_formats") and self._output_formats:
+            if len(self._output_formats) == 1:
+                payload["type"] = self._output_formats[0]
+            else:
+                # Multiple formats: join with comma (e.g., "png,html")
+                payload["type"] = ",".join(self._output_formats)
+        else:
+            # Fallback for backward compatibility
+            if isinstance(self.output_format, str):
+                payload["type"] = self.output_format.lower()
+            else:
+                payload["type"] = ",".join([str(f).lower() for f in self.output_format])
+
         if self.country:
             payload["country"] = self.country.lower()
         if self.block_resources:
@@ -55,6 +85,8 @@ class UniversalScrapeRequest(ThordataBaseConfig):
             payload["wait"] = str(self.wait)
         if self.wait_for:
             payload["wait_for"] = self.wait_for
+        if self.follow_redirect is not None:
+            payload["follow_redirect"] = "True" if self.follow_redirect else "False"
 
         # Serialize complex objects as JSON strings
         if self.headers:

{thordata_sdk-1.6.0.dist-info → thordata_sdk-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thordata-sdk
-Version: 1.6.0
+Version: 1.8.0
 Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
 Author-email: Thordata Developer Team <support@thordata.com>
 License: MIT
@@ -40,6 +40,8 @@ Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: mypy>=1.0.0; extra == "dev"
 Requires-Dist: types-requests>=2.28.0; extra == "dev"
 Requires-Dist: aioresponses>=0.7.6; extra == "dev"
+Provides-Extra: browser
+Requires-Dist: playwright>=1.40.0; extra == "browser"
 Dynamic: license-file
 
 # Thordata Python SDK
@@ -63,7 +65,7 @@ Dynamic: license-file
 
 ## 📖 Introduction
 
-The **Thordata Python SDK v1.6.0** is a production-ready wrapper for Thordata's AI data infrastructure. It is architected for high reliability, strict type safety, and maximum performance.
+The **Thordata Python SDK v1.8.0** is a production-ready wrapper for Thordata's AI data infrastructure. It is architected for high reliability, strict type safety, and maximum performance.
 
 **Why v1.6.0?**
 *   **🛡️ Bulletproof Networking**: Custom core handles `HTTP`, `HTTPS`, and `SOCKS5h` (Remote DNS) tunneling, solving common SSL/TLS handshake issues in complex network environments.

{thordata_sdk-1.6.0.dist-info → thordata_sdk-1.8.0.dist-info}/RECORD

@@ -1,14 +1,17 @@
-thordata/__init__.py,sha256=FMOku6d17GrFjiJlRhvkx-JmhLLD7VlaADLC3FP6hHg,2287
+thordata/__init__.py,sha256=easXVqOOb6hZk0NtsaSJ0lOCzmKu6O24-Y68fukCwcY,2287
 thordata/_utils.py,sha256=Acr_6sHgdZXU7SQozd6FEYTZV6iHw__nlhpBTDwb66U,4917
-thordata/async_client.py,sha256=F_t5EeYUM8BYM9tOQb2lzrcO81whGfO1g53Qagxcyq8,39713
+thordata/async_client.py,sha256=3tb9zg7eqTxxaC46Npei21a-oeLgSi3aaUTwqJygp88,42308
 thordata/async_unlimited.py,sha256=kzTksFkN21rDM21Pwy3hcayjfyGYNGGyGR3fRLtZC6I,4510
-thordata/client.py,sha256=fG7X9JpFS0HKlWZl_6R_Phzt_o2hV25rVUUyCXhioYM,56991
+thordata/client.py,sha256=DARFPtxJdkz0tjh8cKjP0nnwN-oQEkPZXclseCUJ2LY,59074
 thordata/enums.py,sha256=dO5QWpPFLpYP2GfLAdoFtxMTemhGNdr_NPqBoYfSFkk,764
-thordata/exceptions.py,sha256=foAtH5U2pLUXM6u1C_63AVVh4-afuwt5y5MO7jDF0s8,11585
+thordata/exceptions.py,sha256=ntiq3F5sxAiEDmCnlcfS2GNb3Qa7DpRvMrhmgXhAGIg,11947
 thordata/models.py,sha256=7GshQklo5aqke_ZQ2QIXiz9Ac5v6IRtvjWIjsBKEq6A,853
 thordata/retry.py,sha256=X6Sa5IIb5EWD5fUJjKyhvWJyWQGPVgxLB3-vKoWfa5Q,11453
 thordata/serp_engines.py,sha256=iuMWncelcGOskCHXFzpcPMMTL5qfiLkazHB1uj3zpZo,5985
 thordata/unlimited.py,sha256=RzrtwcotYlbOWuSLysDyI75IkMVL7ygdfE9HKNoe02M,6087
+thordata/browser/__init__.py,sha256=nu4cUeQDOhBV4LIJmgOZlsSdSXkmuuGhdXQBHo2yCvU,498
+thordata/browser/exceptions.py,sha256=IRHmH-4dbXWbV6wDMKxDUsKuFGRseZdsYBgspzhcsyM,428
+thordata/browser/session.py,sha256=FuwdUNpEzCIBVBv60gQO2c7bZ3S-MZ8bhGScG_NtDVQ,15686
 thordata/core/__init__.py,sha256=EFT6mZpSdec_7uFUpSpDDHVwbTxy314uxJC_uprR6J4,500
 thordata/core/async_http_client.py,sha256=KKsmhXN6bWRTDFvqa0H-WRf4R-TWH8WSgpDBRv6TEvg,3052
 thordata/core/http_client.py,sha256=8lSwclmVweM-Go1qMW36zYnMKAUT_9RyDdPF7qMS4-Y,2280
@@ -25,11 +28,11 @@ thordata/tools/video.py,sha256=HUFqdue-dtWmTVlYtmf5ffzuYDIzw5l3wk3Vr7AXQW0,4689
 thordata/types/__init__.py,sha256=hlLt5UCVm7QdeOCN5_YWXS4Vy8tJUhIp0XbWjAoQiQg,1357
 thordata/types/common.py,sha256=hkTZ1QtokpE1yT9BvTmYfQz9AUjeCIIPvjib2pnq_Ag,2818
 thordata/types/proxy.py,sha256=IU45wQHCBOIlbdcCN9veypAkDT0q9NIikLu674CudOU,10438
-thordata/types/serp.py,sha256=NO52I1NprjVBgKQe4o2xEp82a3Oy9wCBYG-2Q0oegnU,5817
-thordata/types/task.py,sha256=b9TzcFigWUJDsr2t1hvaDv_CU1xk2d2cMrthmwPn7VU,4602
-thordata/types/universal.py,sha256=Kw8lf_2ElXIfylsNfVosLE1MvlEQkryv4fWEaQw6ecg,2161
-thordata_sdk-1.6.0.dist-info/licenses/LICENSE,sha256=bAxpWgQIzb-5jl3nhLdOwOJ_vlbHLtSG7yev2B7vioY,1088
-thordata_sdk-1.6.0.dist-info/METADATA,sha256=bBy6xzDLWZ9l5bGLu0Jh91X9GtYVjlKCtpp13OZchmU,9308
-thordata_sdk-1.6.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-thordata_sdk-1.6.0.dist-info/top_level.txt,sha256=Z8R_07m0lXCCSb1hapL9_nxMtyO3rf_9wOvq4n9u2Hg,9
-thordata_sdk-1.6.0.dist-info/RECORD,,
+thordata/types/serp.py,sha256=2pOw2mrl1RzMZMSR9ZJCZ2ggYj8f4zRg0TKy8l2ye0o,7340
+thordata/types/task.py,sha256=PVKNyzXZHNunThVb5dwHDtMZ9WmFpbaePDnkeV754CQ,6264
+thordata/types/universal.py,sha256=8OIZs239fBxzSuLEe3VB9qCp1ddN5XKAXbpVm9MJuls,3631
+thordata_sdk-1.8.0.dist-info/licenses/LICENSE,sha256=bAxpWgQIzb-5jl3nhLdOwOJ_vlbHLtSG7yev2B7vioY,1088
+thordata_sdk-1.8.0.dist-info/METADATA,sha256=OI-f2KDEjN96Kt8KtZcbtPWcSJse34MfdlWmUSuNp0s,9386
+thordata_sdk-1.8.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+thordata_sdk-1.8.0.dist-info/top_level.txt,sha256=Z8R_07m0lXCCSb1hapL9_nxMtyO3rf_9wOvq4n9u2Hg,9
+thordata_sdk-1.8.0.dist-info/RECORD,,