thordata-sdk 0.2.4__py3-none-any.whl → 1.2.0__py3-none-any.whl

thordata/__init__.py

@@ -0,0 +1,151 @@
+"""
+Thordata Python SDK
+
+Official Python client for Thordata's Proxy Network, SERP API,
+Universal Scraping API (Web Unlocker), and Web Scraper API.
+
+Basic Usage:
+    >>> from thordata import ThordataClient
+    >>>
+    >>> client = ThordataClient(
+    ...     scraper_token="your_token",
+    ...     public_token="your_public_token",
+    ...     public_key="your_public_key"
+    ... )
+    >>>
+    >>> # Proxy request
+    >>> response = client.get("https://httpbin.org/ip")
+    >>>
+    >>> # SERP search
+    >>> results = client.serp_search("python tutorial", engine="google")
+    >>>
+    >>> # Universal scrape
+    >>> html = client.universal_scrape("https://example.com", js_render=True)
+
+Async Usage:
+    >>> from thordata import AsyncThordataClient
+    >>> import asyncio
+    >>>
+    >>> async def main():
+    ...     async with AsyncThordataClient(
+    ...         scraper_token="your_token"
+    ...     ) as client:
+    ...         response = await client.get("https://httpbin.org/ip")
+    >>>
+    >>> asyncio.run(main())
+"""
+
+__version__ = "1.2.0"
+__author__ = "Thordata Developer Team"
+__email__ = "support@thordata.com"
+
+# Main clients
+from .async_client import AsyncThordataClient
+from .client import ThordataClient
+
+# Enums
+from .enums import (
+    BingSearchType,
+    Continent,
+    Country,
+    DataFormat,
+    Device,
+    Engine,
+    GoogleSearchType,
+    GoogleTbm,
+    OutputFormat,
+    ProxyHost,
+    ProxyPort,
+    ProxyType,
+    SessionType,
+    TaskStatus,
+    TimeRange,
+)
+
+# Exceptions
+from .exceptions import (
+    ThordataAPIError,
+    ThordataAuthError,
+    ThordataConfigError,
+    ThordataError,
+    ThordataNetworkError,
+    ThordataNotCollectedError,
+    ThordataRateLimitError,
+    ThordataServerError,
+    ThordataTimeoutError,
+    ThordataValidationError,
+)
+
+# Models
+from .models import (
+    CommonSettings,
+    ProxyConfig,
+    ProxyProduct,
+    ProxyServer,
+    ProxyUser,
+    ProxyUserList,
+    ScraperTaskConfig,
+    SerpRequest,
+    StaticISPProxy,
+    StickySession,
+    TaskStatusResponse,
+    UniversalScrapeRequest,
+    UsageStatistics,
+    VideoTaskConfig,
+)
+
+# Retry utilities
+from .retry import RetryConfig
+
+# Public API
+__all__ = [
+    # Version
+    "__version__",
+    # Clients
+    "ThordataClient",
+    "AsyncThordataClient",
+    # Enums
+    "Engine",
+    "GoogleSearchType",
+    "BingSearchType",
+    "ProxyType",
+    "SessionType",
+    "Continent",
+    "Country",
+    "OutputFormat",
+    "DataFormat",
+    "TaskStatus",
+    "Device",
+    "TimeRange",
+    "ProxyHost",
+    "ProxyPort",
+    "GoogleTbm",
+    # Models
+    "ProxyConfig",
+    "ProxyProduct",
+    "ProxyServer",
+    "ProxyUser",
+    "ProxyUserList",
+    "UsageStatistics",
+    "StaticISPProxy",
+    "StickySession",
+    "SerpRequest",
+    "UniversalScrapeRequest",
+    "ScraperTaskConfig",
+    "CommonSettings",
+    "VideoTaskConfig",
+    "TaskStatusResponse",
+    # Exceptions
+    "ThordataError",
+    "ThordataConfigError",
+    "ThordataNetworkError",
+    "ThordataTimeoutError",
+    "ThordataAPIError",
+    "ThordataAuthError",
+    "ThordataRateLimitError",
+    "ThordataServerError",
+    "ThordataValidationError",
+    "ThordataNotCollectedError",
+    # Retry
+    "RetryConfig",
+]

thordata/_example_utils.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+try:
+    from dotenv import load_dotenv
+except Exception:  # pragma: no cover
+    load_dotenv = None
+
+
+def load_env() -> None:
+    """Load .env from repo root if python-dotenv is installed."""
+    if load_dotenv is None:
+        return
+    repo_root = Path(__file__).resolve().parents[2]
+    load_dotenv(dotenv_path=repo_root / ".env")
+
+
+def env(name: str) -> str:
+    return (os.getenv(name) or "").strip()
+
+
+def skip_if_missing(required: Iterable[str], *, tip: str | None = None) -> bool:
+    missing = [k for k in required if not env(k)]
+    if not missing:
+        return False
+    print("Skipping live example: missing env:", ", ".join(missing))
+    if tip:
+        print(tip)
+    else:
+        print("Tip: copy .env.example to .env and fill values, then re-run.")
+    return True
+
+
+def parse_json_env(name: str, default: str = "{}") -> Any:
+    raw = env(name) or default
+    return json.loads(raw)
+
+
+def normalize_task_parameters(raw: Any) -> dict[str, Any]:
+    """Accept {..} or [{..}] and return a single dict for create_scraper_task(parameters=...)."""
+    if isinstance(raw, list):
+        if not raw:
+            raise ValueError("Task parameters JSON array must not be empty")
+        raw = raw[0]
+    if not isinstance(raw, dict):
+        raise ValueError("Task parameters must be a JSON object (or array of objects)")
+    return raw
+
+
+def output_dir() -> Path:
+    """Return output dir for examples; defaults to examples/output (ignored by git)."""
+    repo_root = Path(__file__).resolve().parents[2]
+    d = env("THORDATA_OUTPUT_DIR") or str(repo_root / "examples" / "output")
+    p = Path(d)
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def write_text(filename: str, content: str) -> Path:
+    p = output_dir() / filename
+    p.write_text(content, encoding="utf-8", errors="replace")
+    return p
+
+
+def write_json(filename: str, data: Any) -> Path:
+    p = output_dir() / filename
+    p.write_text(
+        json.dumps(data, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+        errors="replace",
+    )
+    return p

thordata/_utils.py

@@ -0,0 +1,190 @@
+"""
+Internal utility functions for the Thordata Python SDK.
+
+These are not part of the public API and may change without notice.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import platform
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def parse_json_response(data: Any) -> Any:
+    """
+    Parse a response that might be double-encoded JSON.
+
+    Some API endpoints return JSON as a string inside JSON.
+
+    Args:
+        data: The response data to parse.
+
+    Returns:
+        Parsed data.
+    """
+    if isinstance(data, str):
+        try:
+            return json.loads(data)
+        except json.JSONDecodeError:
+            return data
+    return data
+
+
+def decode_base64_image(png_str: str) -> bytes:
+    """
+    Decode a base64-encoded PNG image.
+
+    Handles Data URI scheme (data:image/png;base64,...) and fixes padding.
+
+    Args:
+        png_str: Base64-encoded string, possibly with Data URI prefix.
+
+    Returns:
+        Decoded PNG bytes.
+
+    Raises:
+        ValueError: If the string is empty or cannot be decoded.
+    """
+    if not png_str:
+        raise ValueError("Empty PNG data received")
+
+    # Remove Data URI scheme if present
+    if "," in png_str:
+        png_str = png_str.split(",", 1)[1]
+
+    # Clean up whitespace
+    png_str = png_str.replace("\n", "").replace("\r", "").replace(" ", "")
+
+    # Fix Base64 padding
+    missing_padding = len(png_str) % 4
+    if missing_padding:
+        png_str += "=" * (4 - missing_padding)
+
+    try:
+        return base64.b64decode(png_str)
+    except Exception as e:
+        raise ValueError(f"Failed to decode base64 image: {e}") from e
+
+
+def build_auth_headers(token: str, mode: str = "bearer") -> dict[str, str]:
+    """
+    Build authorization headers for API requests.
+
+    Supports two modes:
+    - bearer: Authorization: Bearer <token> (Thordata Docs examples)
+    - header_token: token: <token> (Interface documentation)
+
+    Args:
+        token: The scraper token.
+        mode: Authentication mode ("bearer" or "header_token").
+
+    Returns:
+        Headers dict with Authorization/token and Content-Type.
+    """
+    headers = {
+        "Content-Type": "application/x-www-form-urlencoded",
+    }
+
+    if mode == "bearer":
+        headers["Authorization"] = f"Bearer {token}"
+    elif mode == "header_token":
+        headers["token"] = token
+    else:
+        # Fallback to bearer for compatibility
+        headers["Authorization"] = f"Bearer {token}"
+
+    return headers
+
+
+def build_builder_headers(
+    scraper_token: str,
+    public_token: str,
+    public_key: str,
+) -> dict[str, str]:
+    """
+    Build headers for Web Scraper builder API.
+
+    Builder requires THREE auth headers per official docs:
+    - token: public token
+    - key: public key
+    - Authorization: Bearer scraper_token
+
+    Args:
+        scraper_token: The scraper API token.
+        public_token: The public API token.
+        public_key: The public API key.
+
+    Returns:
+        Headers dict with all required auth headers.
+    """
+    return {
+        "token": public_token,
+        "key": public_key,
+        "Authorization": f"Bearer {scraper_token}",
+        "Content-Type": "application/x-www-form-urlencoded",
+    }
+
+
+def build_public_api_headers(public_token: str, public_key: str) -> dict[str, str]:
+    """
+    Build headers for public API requests (task status, locations, etc.)
+
+    Args:
+        public_token: The public API token.
+        public_key: The public API key.
+
+    Returns:
+        Headers dict with token, key, and Content-Type.
+    """
+    return {
+        "token": public_token,
+        "key": public_key,
+        "Content-Type": "application/x-www-form-urlencoded",
+    }
+
+
+def extract_error_message(payload: Any) -> str:
+    """
+    Extract a human-readable error message from an API response.
+
+    Args:
+        payload: The API response payload.
+
+    Returns:
+        Error message string.
+    """
+    if isinstance(payload, dict):
+        # Try common error message fields
+        for key in ("msg", "message", "error", "detail", "description"):
+            if key in payload:
+                return str(payload[key])
+
+        # Fall back to full payload
+        return str(payload)
+
+    return str(payload)
+
+
+def build_user_agent(sdk_version: str, http_client: str) -> str:
+    """
+    Build a standardized User-Agent for the SDK.
+    Format: thordata-python-sdk/{version} python/{py_ver} ({system}/{release}; {machine})
+    """
+    py_ver = platform.python_version()
+    system = platform.system() or "unknown"
+    release = platform.release() or "unknown"
+    machine = platform.machine() or "unknown"
+
+    # Clean up strings to avoid UA parsing issues (remove newlines, etc)
+    system = system.replace(";", "").strip()
+
+    return (
+        f"thordata-python-sdk/{sdk_version} "
+        f"python/{py_ver} "
+        f"({system}/{release}; {machine}; {http_client})"
+    )