thordata-sdk 0.7.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

thordata/enums.py

@@ -34,12 +34,13 @@ class Continent(str, Enum):
 class ProxyHost(str, Enum):
     """
     Available proxy gateway hosts.
+
+    Note: Dashboard provides user-specific hosts like {shard}.{region}.thordata.net
     """
 
     DEFAULT = "pr.thordata.net"
     NORTH_AMERICA = "t.na.thordata.net"
     EUROPE = "t.eu.thordata.net"
-    GATE = "gate.thordata.com"
 
 
 class ProxyPort(IntEnum):
@@ -47,11 +48,10 @@ class ProxyPort(IntEnum):
     Available proxy gateway ports.
     """
 
-    DEFAULT = 9999
+    RESIDENTIAL = 9999
     MOBILE = 5555
     DATACENTER = 7777
     ISP = 6666
-    ALTERNATIVE = 22225
 
 
 # =============================================================================

thordata/exceptions.py

@@ -222,7 +222,8 @@ class ThordataNotCollectedError(ThordataAPIError):
     This error is often transient and typically safe to retry.
     """
 
-    HTTP_STATUS_CODES = {300}
+    API_CODES = {300}
+    HTTP_STATUS_CODES: Set[int] = set()
 
     @property
     def is_retryable(self) -> bool:
@@ -262,8 +263,17 @@ def raise_for_code(
         ThordataValidationError: For 400/422 codes.
         ThordataAPIError: For all other error codes.
     """
-    # Use the code from payload if status_code not available
-    effective_code = status_code or code
+    # Determine the effective error code.
+    # Prefer payload `code` when present and not success (200),
+    # otherwise fall back to HTTP status when it indicates an error.
+    effective_code: Optional[int] = None
+
+    if code is not None and code != 200:
+        effective_code = code
+    elif status_code is not None and status_code != 200:
+        effective_code = status_code
+    else:
+        effective_code = code if code is not None else status_code
 
     kwargs = {
         "status_code": status_code,
@@ -272,8 +282,9 @@ def raise_for_code(
         "request_id": request_id,
     }
 
-    # Not collected (often retryable, not billed)
-    if effective_code in ThordataNotCollectedError.HTTP_STATUS_CODES:
+    # Not collected (API payload code 300, often retryable, not billed)
+    # Check this FIRST since 300 is in API_CODES, not HTTP_STATUS_CODES
+    if effective_code in ThordataNotCollectedError.API_CODES:
         raise ThordataNotCollectedError(message, **kwargs)
 
     # Auth errors

thordata/models.py

@@ -26,11 +26,14 @@ from __future__ import annotations
 
 import json
 import re
+import ssl
 import uuid
 from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
+import urllib3
+
 # =============================================================================
 # Proxy Product Types
 # =============================================================================
@@ -137,6 +140,7 @@ class ProxyConfig:
         if self.host is None:
             # Set host based on product type
             host_map = {
+                # User&Pass auth entry (docs examples use t.pr.thordata.net for authenticated proxy)
                 ProxyProduct.RESIDENTIAL: "t.pr.thordata.net",
                 ProxyProduct.DATACENTER: "dc.pr.thordata.net",
                 ProxyProduct.MOBILE: "m.pr.thordata.net",
@@ -233,6 +237,14 @@ class ProxyConfig:
         username = self.build_username()
         return f"{self.protocol}://{username}:{self.password}@{self.host}:{self.port}"
 
+    def build_proxy_endpoint(self) -> str:
+        """Proxy endpoint without credentials, for HTTPS proxy managers."""
+        return f"{self.protocol}://{self.host}:{self.port}"
+
+    def build_proxy_basic_auth(self) -> str:
+        """Basic auth string 'username:password' for Proxy-Authorization."""
+        return f"{self.build_username()}:{self.password}"
+
     def to_proxies_dict(self) -> Dict[str, str]:
         """
         Build a proxies dict suitable for the requests library.
@@ -264,6 +276,39 @@ class ProxyConfig:
             ) from e
 
 
+@dataclass
+class WhitelistProxyConfig:
+    """
+    Proxy config for IP-whitelist authentication mode (no username/password).
+
+    In whitelist mode, you do NOT pass proxy auth.
+    You only connect to the proxy entry node (host:port).
+
+    Examples (from docs):
+      - Global random: pr.thordata.net:9999
+      - Country nodes: us-pr.thordata.net:10000, etc.
+    """
+
+    host: str = "pr.thordata.net"
+    port: int = 9999
+    protocol: str = "http"  # use http for proxy scheme; target URL can still be https
+
+    def __post_init__(self) -> None:
+        if self.protocol not in ("http", "https"):
+            raise ValueError("protocol must be 'http' or 'https'")
+
+    def build_proxy_url(self) -> str:
+        return f"{self.protocol}://{self.host}:{self.port}"
+
+    def to_proxies_dict(self) -> Dict[str, str]:
+        url = self.build_proxy_url()
+        return {"http": url, "https": url}
+
+    def to_aiohttp_config(self) -> tuple:
+        # aiohttp: proxy_auth should be None in whitelist mode
+        return self.build_proxy_url(), None
+
+
 @dataclass
 class StaticISPProxy:
     """
@@ -545,23 +590,28 @@ class SerpRequest:
         payload: Dict[str, Any] = {
             "engine": engine,
             "num": str(self.num),
-            # output_format: json=1 for JSON, json=0 for raw HTML
-            "json": "1" if self.output_format.lower() == "json" else "0",
         }
 
+        fmt = self.output_format.lower()
+        if fmt == "json":
+            payload["json"] = "1"
+        elif fmt == "html":
+            # omit "json" to get raw HTML (per docs: no json -> HTML)
+            pass
+        else:
+            # keep backward compatibility: if user passes "2"/"both"/etc.
+            if fmt in ("2", "both", "json+html", "json_html"):
+                payload["json"] = "2"
+
         # Handle query parameter (Yandex uses 'text', others use 'q')
         if engine == "yandex":
             payload["text"] = self.query
         else:
             payload["q"] = self.query
 
-        # Set URL / domain based on google_domain or engine default
+        # Domain overrides (preferred by docs)
         if self.google_domain:
-            # 显式设置 google_domain 参数，同时设置 url
             payload["google_domain"] = self.google_domain
-            payload["url"] = self.google_domain
-        elif engine in self.ENGINE_URLS:
-            payload["url"] = self.ENGINE_URLS[engine]
 
         # Pagination
         if self.start > 0:
@@ -795,6 +845,126 @@ class ScraperTaskConfig:
         return payload
 
 
+@dataclass
+class CommonSettings:
+    """
+    Common settings for YouTube video/audio downloads.
+
+    Used by /video_builder endpoint as `common_settings` parameter.
+    Also known as `spider_universal` in some documentation.
+
+    Args:
+        resolution: Video resolution (360p/480p/720p/1080p/1440p/2160p).
+        audio_format: Audio format (opus/mp3).
+        bitrate: Audio bitrate (48/64/128/160/256/320 or with Kbps suffix).
+        is_subtitles: Whether to download subtitles ("true"/"false").
+        subtitles_language: Subtitle language code (e.g., "en", "zh-Hans").
+
+    Example for video:
+        >>> settings = CommonSettings(
+        ...     resolution="1080p",
+        ...     is_subtitles="true",
+        ...     subtitles_language="en"
+        ... )
+
+    Example for audio:
+        >>> settings = CommonSettings(
+        ...     audio_format="mp3",
+        ...     bitrate="320",
+        ...     is_subtitles="true",
+        ...     subtitles_language="en"
+        ... )
+    """
+
+    # Video settings
+    resolution: Optional[str] = None
+
+    # Audio settings
+    audio_format: Optional[str] = None
+    bitrate: Optional[str] = None
+
+    # Subtitle settings (used by both video and audio)
+    is_subtitles: Optional[str] = None
+    subtitles_language: Optional[str] = None
+
+    # Valid values for validation
+    VALID_RESOLUTIONS = {"360p", "480p", "720p", "1080p", "1440p", "2160p"}
+    VALID_AUDIO_FORMATS = {"opus", "mp3"}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary, excluding None values."""
+        result = {}
+        if self.resolution is not None:
+            result["resolution"] = self.resolution
+        if self.audio_format is not None:
+            result["audio_format"] = self.audio_format
+        if self.bitrate is not None:
+            result["bitrate"] = self.bitrate
+        if self.is_subtitles is not None:
+            result["is_subtitles"] = self.is_subtitles
+        if self.subtitles_language is not None:
+            result["subtitles_language"] = self.subtitles_language
+        return result
+
+    def to_json(self) -> str:
+        """Convert to JSON string for form submission."""
+        return json.dumps(self.to_dict())
+
+
+@dataclass
+class VideoTaskConfig:
+    """
+    Configuration for creating a YouTube video/audio download task.
+
+    Uses the /video_builder endpoint.
+
+    Args:
+        file_name: Name for the output file. Supports {{TasksID}}, {{VideoID}}.
+        spider_id: Spider identifier (e.g., "youtube_video_by-url", "youtube_audio_by-url").
+        spider_name: Spider name (typically "youtube.com").
+        parameters: Spider-specific parameters (e.g., video URL).
+        common_settings: Video/audio settings (resolution, format, subtitles).
+        include_errors: Include error details in output.
+
+    Example:
+        >>> config = VideoTaskConfig(
+        ...     file_name="{{VideoID}}",
+        ...     spider_id="youtube_video_by-url",
+        ...     spider_name="youtube.com",
+        ...     parameters={"url": "https://www.youtube.com/watch?v=xxx"},
+        ...     common_settings=CommonSettings(
+        ...         resolution="1080p",
+        ...         is_subtitles="true",
+        ...         subtitles_language="en"
+        ...     )
+        ... )
+    """
+
+    file_name: str
+    spider_id: str
+    spider_name: str
+    parameters: Dict[str, Any]
+    common_settings: CommonSettings
+    include_errors: bool = True
+
+    def to_payload(self) -> Dict[str, Any]:
+        """
+        Convert to API request payload.
+
+        Returns:
+            Dictionary ready to be sent to the video_builder API.
+        """
+        payload: Dict[str, Any] = {
+            "file_name": self.file_name,
+            "spider_id": self.spider_id,
+            "spider_name": self.spider_name,
+            "spider_parameters": json.dumps([self.parameters]),
+            "spider_errors": "true" if self.include_errors else "false",
+            "common_settings": self.common_settings.to_json(),
+        }
+        return payload
+
+
 # =============================================================================
 # Response Models
 # =============================================================================
@@ -838,3 +1008,177 @@ class TaskStatusResponse:
         """Check if the task failed."""
         failure_statuses = {"failed", "error"}
         return self.status.lower() in failure_statuses
+
+
+@dataclass
+class UsageStatistics:
+    """
+    Response model for account usage statistics.
+
+    Attributes:
+        total_usage_traffic: Total traffic used (KB).
+        traffic_balance: Remaining traffic balance (KB).
+        query_days: Number of days in the query range.
+        range_usage_traffic: Traffic used in the specified date range (KB).
+        data: Daily usage breakdown.
+    """
+
+    total_usage_traffic: float
+    traffic_balance: float
+    query_days: int
+    range_usage_traffic: float
+    data: List[Dict[str, Any]]
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "UsageStatistics":
+        """Create from API response dict."""
+        return cls(
+            total_usage_traffic=float(data.get("total_usage_traffic", 0)),
+            traffic_balance=float(data.get("traffic_balance", 0)),
+            query_days=int(data.get("query_days", 0)),
+            range_usage_traffic=float(data.get("range_usage_traffic", 0)),
+            data=data.get("data", []),
+        )
+
+    def total_usage_gb(self) -> float:
+        """Get total usage in GB."""
+        return self.total_usage_traffic / (1024 * 1024)
+
+    def balance_gb(self) -> float:
+        """Get balance in GB."""
+        return self.traffic_balance / (1024 * 1024)
+
+    def range_usage_gb(self) -> float:
+        """Get range usage in GB."""
+        return self.range_usage_traffic / (1024 * 1024)
+
+
+@dataclass
+class ProxyUser:
+    """
+    Proxy user (sub-account) information.
+
+    Attributes:
+        username: User's username.
+        password: User's password.
+        status: User status (True=enabled, False=disabled).
+        traffic_limit: Traffic limit in MB (0 = unlimited).
+        usage_traffic: Traffic used in KB.
+    """
+
+    username: str
+    password: str
+    status: bool
+    traffic_limit: int
+    usage_traffic: float
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyUser":
+        """Create from API response dict."""
+        return cls(
+            username=data.get("username", ""),
+            password=data.get("password", ""),
+            status=data.get("status") in (True, "true", 1),
+            traffic_limit=int(data.get("traffic_limit", 0)),
+            usage_traffic=float(data.get("usage_traffic", 0)),
+        )
+
+    def usage_gb(self) -> float:
+        """Get usage in GB."""
+        return self.usage_traffic / (1024 * 1024)
+
+    def limit_gb(self) -> float:
+        """Get limit in GB (0 means unlimited)."""
+        if self.traffic_limit == 0:
+            return 0
+        return self.traffic_limit / 1024
+
+
+@dataclass
+class ProxyUserList:
+    """
+    Response model for proxy user list.
+
+    Attributes:
+        limit: Total traffic limit (KB).
+        remaining_limit: Remaining traffic limit (KB).
+        user_count: Number of users.
+        users: List of proxy users.
+    """
+
+    limit: float
+    remaining_limit: float
+    user_count: int
+    users: List[ProxyUser]
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyUserList":
+        """Create from API response dict."""
+        user_list = data.get("list", [])
+        users = [ProxyUser.from_dict(u) for u in user_list]
+
+        return cls(
+            limit=float(data.get("limit", 0)),
+            remaining_limit=float(data.get("remaining_limit", 0)),
+            user_count=int(data.get("user_count", len(users))),
+            users=users,
+        )
+
+
+@dataclass
+class ProxyServer:
+    """
+    ISP or Datacenter proxy server information.
+
+    Attributes:
+        ip: Proxy server IP address.
+        port: Proxy server port.
+        username: Authentication username.
+        password: Authentication password.
+        expiration_time: Expiration timestamp (Unix timestamp or datetime string).
+        region: Server region (optional).
+    """
+
+    ip: str
+    port: int
+    username: str
+    password: str
+    expiration_time: Optional[Union[int, str]] = None
+    region: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyServer":
+        """Create from API response dict."""
+        return cls(
+            ip=data.get("ip", ""),
+            port=int(data.get("port", 0)),
+            username=data.get("username", data.get("user", "")),
+            password=data.get("password", data.get("pwd", "")),
+            expiration_time=data.get("expiration_time", data.get("expireTime")),
+            region=data.get("region"),
+        )
+
+    def to_proxy_url(self, protocol: str = "http") -> str:
+        """
+        Build proxy URL for this server.
+
+        Args:
+            protocol: Proxy protocol (http/https/socks5).
+
+        Returns:
+            Complete proxy URL.
+        """
+        return f"{protocol}://{self.username}:{self.password}@{self.ip}:{self.port}"
+
+    def is_expired(self) -> bool:
+        """Check if proxy has expired (if expiration_time is available)."""
+        if self.expiration_time is None:
+            return False
+
+        import time
+
+        if isinstance(self.expiration_time, int):
+            return time.time() > self.expiration_time
+
+        # String timestamp handling would need datetime parsing
+        return False

thordata/retry.py

@@ -16,6 +16,7 @@ Example:
 
 from __future__ import annotations
 
+import inspect
 import logging
 import random
 import time
@@ -64,7 +65,10 @@ class RetryConfig:
 
     # Status codes to retry on (5xx server errors + 429 rate limit)
     retry_on_status_codes: Set[int] = field(
-        default_factory=lambda: {300, 429, 500, 502, 503, 504}
+        default_factory=lambda: {429, 500, 502, 503, 504}
+    )
+    retry_on_api_codes: Set[int] = field(
+        default_factory=lambda: {300}  # API response body code
     )
 
     # Exception types to always retry on
@@ -198,8 +202,6 @@ def with_retry(
 
         @wraps(func)
         async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
-            import asyncio
-
             last_exception: Optional[Exception] = None
 
             for attempt in range(config.max_retries + 1):
@@ -235,7 +237,7 @@ def with_retry(
         # Check if the function is async
         import asyncio
 
-        if asyncio.iscoroutinefunction(func):
+        if inspect.iscoroutinefunction(func):
             return async_wrapper
         return sync_wrapper