thordata-sdk 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

thordata/enums.py

@@ -34,12 +34,13 @@ class Continent(str, Enum):
 class ProxyHost(str, Enum):
     """
     Available proxy gateway hosts.
+
+    Note: Dashboard provides user-specific hosts like {shard}.{region}.thordata.net
     """
 
     DEFAULT = "pr.thordata.net"
     NORTH_AMERICA = "t.na.thordata.net"
     EUROPE = "t.eu.thordata.net"
-    GATE = "gate.thordata.com"
 
 
 class ProxyPort(IntEnum):
@@ -47,11 +48,10 @@ class ProxyPort(IntEnum):
     Available proxy gateway ports.
     """
 
-    DEFAULT = 9999
+    RESIDENTIAL = 9999
     MOBILE = 5555
     DATACENTER = 7777
     ISP = 6666
-    ALTERNATIVE = 22225
 
 
 # =============================================================================

thordata/exceptions.py

@@ -222,7 +222,8 @@ class ThordataNotCollectedError(ThordataAPIError):
     This error is often transient and typically safe to retry.
     """
 
-    HTTP_STATUS_CODES = {300}
+    API_CODES = {300}
+    HTTP_STATUS_CODES: Set[int] = set()
 
     @property
     def is_retryable(self) -> bool:
@@ -262,8 +263,17 @@ def raise_for_code(
         ThordataValidationError: For 400/422 codes.
         ThordataAPIError: For all other error codes.
     """
-    # Use the code from payload if status_code not available
-    effective_code = status_code or code
+    # Determine the effective error code.
+    # Prefer payload `code` when present and not success (200),
+    # otherwise fall back to HTTP status when it indicates an error.
+    effective_code: Optional[int] = None
+
+    if code is not None and code != 200:
+        effective_code = code
+    elif status_code is not None and status_code != 200:
+        effective_code = status_code
+    else:
+        effective_code = code if code is not None else status_code
 
     kwargs = {
         "status_code": status_code,
@@ -272,8 +282,9 @@ def raise_for_code(
         "request_id": request_id,
     }
 
-    # Not collected (often retryable, not billed)
-    if effective_code in ThordataNotCollectedError.HTTP_STATUS_CODES:
+    # Not collected (API payload code 300, often retryable, not billed)
+    # Check this FIRST since 300 is in API_CODES, not HTTP_STATUS_CODES
+    if effective_code in ThordataNotCollectedError.API_CODES:
         raise ThordataNotCollectedError(message, **kwargs)
 
     # Auth errors

thordata/models.py

@@ -795,6 +795,126 @@ class ScraperTaskConfig:
         return payload
 
 
+@dataclass
+class CommonSettings:
+    """
+    Common settings for YouTube video/audio downloads.
+
+    Used by /video_builder endpoint as `common_settings` parameter.
+    Also known as `spider_universal` in some documentation.
+
+    Args:
+        resolution: Video resolution (360p/480p/720p/1080p/1440p/2160p).
+        audio_format: Audio format (opus/mp3).
+        bitrate: Audio bitrate (48/64/128/160/256/320 or with Kbps suffix).
+        is_subtitles: Whether to download subtitles ("true"/"false").
+        subtitles_language: Subtitle language code (e.g., "en", "zh-Hans").
+
+    Example for video:
+        >>> settings = CommonSettings(
+        ...     resolution="1080p",
+        ...     is_subtitles="true",
+        ...     subtitles_language="en"
+        ... )
+
+    Example for audio:
+        >>> settings = CommonSettings(
+        ...     audio_format="mp3",
+        ...     bitrate="320",
+        ...     is_subtitles="true",
+        ...     subtitles_language="en"
+        ... )
+    """
+
+    # Video settings
+    resolution: Optional[str] = None
+
+    # Audio settings
+    audio_format: Optional[str] = None
+    bitrate: Optional[str] = None
+
+    # Subtitle settings (used by both video and audio)
+    is_subtitles: Optional[str] = None
+    subtitles_language: Optional[str] = None
+
+    # Valid values for validation
+    VALID_RESOLUTIONS = {"360p", "480p", "720p", "1080p", "1440p", "2160p"}
+    VALID_AUDIO_FORMATS = {"opus", "mp3"}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary, excluding None values."""
+        result = {}
+        if self.resolution is not None:
+            result["resolution"] = self.resolution
+        if self.audio_format is not None:
+            result["audio_format"] = self.audio_format
+        if self.bitrate is not None:
+            result["bitrate"] = self.bitrate
+        if self.is_subtitles is not None:
+            result["is_subtitles"] = self.is_subtitles
+        if self.subtitles_language is not None:
+            result["subtitles_language"] = self.subtitles_language
+        return result
+
+    def to_json(self) -> str:
+        """Convert to JSON string for form submission."""
+        return json.dumps(self.to_dict())
+
+
+@dataclass
+class VideoTaskConfig:
+    """
+    Configuration for creating a YouTube video/audio download task.
+
+    Uses the /video_builder endpoint.
+
+    Args:
+        file_name: Name for the output file. Supports {{TasksID}}, {{VideoID}}.
+        spider_id: Spider identifier (e.g., "youtube_video_by-url", "youtube_audio_by-url").
+        spider_name: Spider name (typically "youtube.com").
+        parameters: Spider-specific parameters (e.g., video URL).
+        common_settings: Video/audio settings (resolution, format, subtitles).
+        include_errors: Include error details in output.
+
+    Example:
+        >>> config = VideoTaskConfig(
+        ...     file_name="{{VideoID}}",
+        ...     spider_id="youtube_video_by-url",
+        ...     spider_name="youtube.com",
+        ...     parameters={"url": "https://www.youtube.com/watch?v=xxx"},
+        ...     common_settings=CommonSettings(
+        ...         resolution="1080p",
+        ...         is_subtitles="true",
+        ...         subtitles_language="en"
+        ...     )
+        ... )
+    """
+
+    file_name: str
+    spider_id: str
+    spider_name: str
+    parameters: Dict[str, Any]
+    common_settings: CommonSettings
+    include_errors: bool = True
+
+    def to_payload(self) -> Dict[str, Any]:
+        """
+        Convert to API request payload.
+
+        Returns:
+            Dictionary ready to be sent to the video_builder API.
+        """
+        payload: Dict[str, Any] = {
+            "file_name": self.file_name,
+            "spider_id": self.spider_id,
+            "spider_name": self.spider_name,
+            "spider_parameters": json.dumps([self.parameters]),
+            "spider_errors": "true" if self.include_errors else "false",
+            "common_settings": self.common_settings.to_json(),
+        }
+        return payload
+
+
 # =============================================================================
 # Response Models
 # =============================================================================
@@ -838,3 +958,177 @@ class TaskStatusResponse:
         """Check if the task failed."""
         failure_statuses = {"failed", "error"}
         return self.status.lower() in failure_statuses
+
+
+@dataclass
+class UsageStatistics:
+    """
+    Response model for account usage statistics.
+
+    Attributes:
+        total_usage_traffic: Total traffic used (KB).
+        traffic_balance: Remaining traffic balance (KB).
+        query_days: Number of days in the query range.
+        range_usage_traffic: Traffic used in the specified date range (KB).
+        data: Daily usage breakdown.
+    """
+
+    total_usage_traffic: float
+    traffic_balance: float
+    query_days: int
+    range_usage_traffic: float
+    data: List[Dict[str, Any]]
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "UsageStatistics":
+        """Create from API response dict."""
+        return cls(
+            total_usage_traffic=float(data.get("total_usage_traffic", 0)),
+            traffic_balance=float(data.get("traffic_balance", 0)),
+            query_days=int(data.get("query_days", 0)),
+            range_usage_traffic=float(data.get("range_usage_traffic", 0)),
+            data=data.get("data", []),
+        )
+
+    def total_usage_gb(self) -> float:
+        """Get total usage in GB."""
+        return self.total_usage_traffic / (1024 * 1024)
+
+    def balance_gb(self) -> float:
+        """Get balance in GB."""
+        return self.traffic_balance / (1024 * 1024)
+
+    def range_usage_gb(self) -> float:
+        """Get range usage in GB."""
+        return self.range_usage_traffic / (1024 * 1024)
+
+
+@dataclass
+class ProxyUser:
+    """
+    Proxy user (sub-account) information.
+
+    Attributes:
+        username: User's username.
+        password: User's password.
+        status: User status (True=enabled, False=disabled).
+        traffic_limit: Traffic limit in MB (0 = unlimited).
+        usage_traffic: Traffic used in KB.
+    """
+
+    username: str
+    password: str
+    status: bool
+    traffic_limit: int
+    usage_traffic: float
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyUser":
+        """Create from API response dict."""
+        return cls(
+            username=data.get("username", ""),
+            password=data.get("password", ""),
+            status=data.get("status") in (True, "true", 1),
+            traffic_limit=int(data.get("traffic_limit", 0)),
+            usage_traffic=float(data.get("usage_traffic", 0)),
+        )
+
+    def usage_gb(self) -> float:
+        """Get usage in GB."""
+        return self.usage_traffic / (1024 * 1024)
+
+    def limit_gb(self) -> float:
+        """Get limit in GB (0 means unlimited)."""
+        if self.traffic_limit == 0:
+            return 0
+        return self.traffic_limit / 1024
+
+
+@dataclass
+class ProxyUserList:
+    """
+    Response model for proxy user list.
+
+    Attributes:
+        limit: Total traffic limit (KB).
+        remaining_limit: Remaining traffic limit (KB).
+        user_count: Number of users.
+        users: List of proxy users.
+    """
+
+    limit: float
+    remaining_limit: float
+    user_count: int
+    users: List[ProxyUser]
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyUserList":
+        """Create from API response dict."""
+        user_list = data.get("list", [])
+        users = [ProxyUser.from_dict(u) for u in user_list]
+
+        return cls(
+            limit=float(data.get("limit", 0)),
+            remaining_limit=float(data.get("remaining_limit", 0)),
+            user_count=int(data.get("user_count", len(users))),
+            users=users,
+        )
+
+
+@dataclass
+class ProxyServer:
+    """
+    ISP or Datacenter proxy server information.
+
+    Attributes:
+        ip: Proxy server IP address.
+        port: Proxy server port.
+        username: Authentication username.
+        password: Authentication password.
+        expiration_time: Expiration timestamp (Unix timestamp or datetime string).
+        region: Server region (optional).
+    """
+
+    ip: str
+    port: int
+    username: str
+    password: str
+    expiration_time: Optional[Union[int, str]] = None
+    region: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyServer":
+        """Create from API response dict."""
+        return cls(
+            ip=data.get("ip", ""),
+            port=int(data.get("port", 0)),
+            username=data.get("username", data.get("user", "")),
+            password=data.get("password", data.get("pwd", "")),
+            expiration_time=data.get("expiration_time", data.get("expireTime")),
+            region=data.get("region"),
+        )
+
+    def to_proxy_url(self, protocol: str = "http") -> str:
+        """
+        Build proxy URL for this server.
+
+        Args:
+            protocol: Proxy protocol (http/https/socks5).
+
+        Returns:
+            Complete proxy URL.
+        """
+        return f"{protocol}://{self.username}:{self.password}@{self.ip}:{self.port}"
+
+    def is_expired(self) -> bool:
+        """Check if proxy has expired (if expiration_time is available)."""
+        if self.expiration_time is None:
+            return False
+
+        import time
+
+        if isinstance(self.expiration_time, int):
+            return time.time() > self.expiration_time
+
+        # String timestamp handling would need datetime parsing
+        return False

thordata/retry.py

@@ -64,7 +64,10 @@ class RetryConfig:
 
     # Status codes to retry on (5xx server errors + 429 rate limit)
     retry_on_status_codes: Set[int] = field(
-        default_factory=lambda: {300, 429, 500, 502, 503, 504}
+        default_factory=lambda: {429, 500, 502, 503, 504}
+    )
+    retry_on_api_codes: Set[int] = field(
+        default_factory=lambda: {300}  # API response body code
     )
 
     # Exception types to always retry on

thordata_sdk-0.8.0.dist-info/METADATA

@@ -0,0 +1,212 @@
+Metadata-Version: 2.4
+Name: thordata-sdk
+Version: 0.8.0
+Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
+Author-email: Thordata Developer Team <support@thordata.com>
+License: MIT
+Project-URL: Homepage, https://www.thordata.com
+Project-URL: Documentation, https://github.com/Thordata/thordata-python-sdk#readme
+Project-URL: Source, https://github.com/Thordata/thordata-python-sdk
+Project-URL: Tracker, https://github.com/Thordata/thordata-python-sdk/issues
+Project-URL: Changelog, https://github.com/Thordata/thordata-python-sdk/blob/main/CHANGELOG.md
+Keywords: web scraping,proxy,residential proxy,datacenter proxy,ai,llm,data-mining,serp,thordata,web scraper,anti-bot bypass
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: Proxy Servers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.25.0
+Requires-Dist: aiohttp>=3.9.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-httpserver>=1.0.0; extra == "dev"
+Requires-Dist: python-dotenv>=1.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-requests>=2.28.0; extra == "dev"
+Requires-Dist: aioresponses>=0.7.6; extra == "dev"
+Dynamic: license-file
+
+# Thordata Python SDK
+
+<div align="center">
+
+**Official Python client for Thordata's Proxy Network, SERP API, Web Unlocker, and Web Scraper API.**
+
+*Async-ready, type-safe, built for AI agents and large-scale data collection.*
+
+[![PyPI](https://img.shields.io/pypi/v/thordata-sdk?color=blue)](https://pypi.org/project/thordata-sdk/)
+[![Python](https://img.shields.io/badge/python-3.9+-blue)](https://python.org)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+[Documentation](https://doc.thordata.com) • [Dashboard](https://www.thordata.com) • [Examples](examples/)
+
+</div>
+
+---
+
+## ✨ Features
+
+- 🌐 **Proxy Network**: Residential, Mobile, Datacenter, ISP proxies with geo-targeting
+- 🔍 **SERP API**: Google, Bing, Yandex, DuckDuckGo search results
+- 🔓 **Web Unlocker**: Bypass Cloudflare, CAPTCHAs, anti-bot systems
+- 🕷️ **Web Scraper API**: Async task-based scraping (Text & Video/Audio)
+- 📊 **Account Management**: Usage stats, sub-users, IP whitelist
+- ⚡ **Async Support**: Full async/await support with aiohttp
+- 🔄 **Auto Retry**: Configurable retry with exponential backoff
+
+---
+
+## 📦 Installation
+
+```bash
+pip install thordata-sdk
+```
+
+---
+
+## 🔐 Configuration
+
+Set environment variables:
+
+```bash
+# Required for Scraper APIs (SERP, Universal, Tasks)
+export THORDATA_SCRAPER_TOKEN=your_token
+
+# Required for Public/Location APIs (Dashboard -> My Account)
+export THORDATA_PUBLIC_TOKEN=your_public_token
+export THORDATA_PUBLIC_KEY=your_public_key
+
+# Required for Public API NEW (Dashboard -> Public API NEW)
+# If not set, SDK falls back to PUBLIC_TOKEN/KEY
+export THORDATA_SIGN=your_sign
+export THORDATA_API_KEY=your_api_key
+```
+
+---
+
+## 🚀 Quick Start
+
+```python
+from thordata import ThordataClient, Engine
+
+# Initialize (reads from env vars)
+client = ThordataClient(
+    scraper_token="your_token", 
+    public_token="pub_token", 
+    public_key="pub_key"
+)
+
+# SERP Search
+results = client.serp_search("python tutorial", engine=Engine.GOOGLE)
+print(f"Found {len(results.get('organic', []))} results")
+
+# Universal Scrape
+html = client.universal_scrape("https://httpbin.org/html")
+print(html[:100])
+```
+
+---
+
+## 📖 Feature Guide
+
+### SERP API
+
+```python
+from thordata import SerpRequest
+
+# Advanced search
+results = client.serp_search_advanced(SerpRequest(
+    query="pizza",
+    engine="google_local",
+    country="us",
+    location="New York",
+    num=10
+))
+```
+
+### Web Scraper API (Async Tasks)
+
+**Create Task:**
+```python
+task_id = client.create_scraper_task(
+    file_name="my_task",
+    spider_id="universal",
+    spider_name="universal",
+    parameters={"url": "https://example.com"}
+)
+```
+
+**Video Download (New):**
+```python
+from thordata import CommonSettings
+
+task_id = client.create_video_task(
+    file_name="{{VideoID}}",
+    spider_id="youtube_video_by-url",
+    spider_name="youtube.com",
+    parameters={"url": "https://youtube.com/watch?v=..."},
+    common_settings=CommonSettings(resolution="1080p")
+)
+```
+
+**Wait & Download:**
+```python
+status = client.wait_for_task(task_id)
+if status == "ready":
+    url = client.get_task_result(task_id)
+    print(url)
+```
+
+### Account Management
+
+```python
+# Usage Statistics
+stats = client.get_usage_statistics("2024-01-01", "2024-01-31")
+print(f"Balance: {stats.balance_gb():.2f} GB")
+
+# Proxy Users
+users = client.list_proxy_users()
+print(f"Sub-users: {users.user_count}")
+
+# Whitelist IP
+client.add_whitelist_ip("1.2.3.4")
+```
+
+### Proxy Network
+
+```python
+from thordata import ProxyConfig
+
+# Generate Proxy URL
+proxy_url = client.build_proxy_url(
+    username="proxy_user",
+    password="proxy_pass",
+    country="us",
+    city="ny"
+)
+
+# Use with requests
+import requests
+requests.get("https://httpbin.org/ip", proxies={"http": proxy_url, "https": proxy_url})
+```
+
+---
+
+## 📄 License
+
+MIT License

thordata_sdk-0.8.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+thordata/__init__.py,sha256=yaIxW1T_nsCeiPE6iIHunjRzPrtbiN0BciveICgL4dM,3195
+thordata/_utils.py,sha256=epF-ewHyk7McdejlhHNAfxhIQ8sN3TlIjUJ9H4HOaUE,5254
+thordata/async_client.py,sha256=tC9y1wmcO6RsXCysBo0a0GNRZR3QQjJlCmEwG5HVukQ,53169
+thordata/client.py,sha256=VN5Jm3er7fdZDfT2G9g4siBSYNo0ZWj4WOi6TAiAZcE,59638
+thordata/demo.py,sha256=zmG4I4cHXnbmQfbr063SeRK7_9IXrfof9QFoGqGTVm8,3806
+thordata/enums.py,sha256=MpZnS9_8sg2vtcFqM6UicB94cKZm5R1t83L3ejNSbLs,8502
+thordata/exceptions.py,sha256=IgMsFuh49cPxU5YofsKP1UhP5A_snhtuN6xD1yZWLiI,10018
+thordata/models.py,sha256=NG4wn1bq4-FC4Aex8vwBOldiHovwg0JzhdtBsI1mL_8,36118
+thordata/retry.py,sha256=nkh17ca2TIEcTc-uNo-xcNdJPuxZ_VGlMbC70X6p-_Q,11518
+thordata_sdk-0.8.0.dist-info/licenses/LICENSE,sha256=bAxpWgQIzb-5jl3nhLdOwOJ_vlbHLtSG7yev2B7vioY,1088
+thordata_sdk-0.8.0.dist-info/METADATA,sha256=IgL554I6mzya9FdbqCxKdvO3r-bywiHJjZi1xdk8W48,5850
+thordata_sdk-0.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thordata_sdk-0.8.0.dist-info/top_level.txt,sha256=Z8R_07m0lXCCSb1hapL9_nxMtyO3rf_9wOvq4n9u2Hg,9
+thordata_sdk-0.8.0.dist-info/RECORD,,

thordata/parameters.py

@@ -1,53 +0,0 @@
-# src/thordata/parameters.py
-
-from typing import Any, Dict
-
-
-def normalize_serp_params(engine: str, query: str, **kwargs) -> Dict[str, Any]:
-    """
-    Normalizes parameters across different search engines to ensure a unified API surface.
-
-    Args:
-        engine (str): The search engine to use (e.g., 'google', 'yandex').
-        query (str): The search query string.
-        **kwargs: Additional parameters to pass to the API.
-
-    Returns:
-        Dict[str, Any]: The constructed payload for the API request.
-    """
-    # 1. Base parameters
-    payload = {
-        "num": str(kwargs.get("num", 10)),  # Default to 10 results
-        "json": "1",  # Force JSON response
-        "engine": engine,
-    }
-
-    # 2. Handle Query Parameter Differences (Yandex uses 'text', others use 'q')
-    if engine == "yandex":
-        payload["text"] = query
-        # Set default URL for Yandex if not provided
-        if "url" not in kwargs:
-            payload["url"] = "yandex.com"
-    else:
-        payload["q"] = query
-
-        # 3. Handle Default URLs for other engines
-        if "url" not in kwargs:
-            defaults = {
-                "google": "google.com",
-                "bing": "bing.com",
-                "duckduckgo": "duckduckgo.com",
-                "baidu": "baidu.com",
-            }
-            if engine in defaults:
-                payload["url"] = defaults[engine]
-
-    # 4. Passthrough for all other user-provided arguments
-    # This allows support for engine-specific parameters (e.g., tbm, uule, gl)
-    # without explicitly defining them all.
-    protected_keys = {"num", "engine", "q", "text"}
-    for key, value in kwargs.items():
-        if key not in protected_keys:
-            payload[key] = value
-
-    return payload