thordata-sdk 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

thordata/enums.py

@@ -34,12 +34,13 @@ class Continent(str, Enum):
 class ProxyHost(str, Enum):
     """
     Available proxy gateway hosts.
+
+    Note: Dashboard provides user-specific hosts like {shard}.{region}.thordata.net
     """
 
     DEFAULT = "pr.thordata.net"
     NORTH_AMERICA = "t.na.thordata.net"
     EUROPE = "t.eu.thordata.net"
-    GATE = "gate.thordata.com"
 
 
 class ProxyPort(IntEnum):
@@ -47,11 +48,10 @@ class ProxyPort(IntEnum):
     Available proxy gateway ports.
     """
 
-    DEFAULT = 9999
+    RESIDENTIAL = 9999
     MOBILE = 5555
     DATACENTER = 7777
     ISP = 6666
-    ALTERNATIVE = 22225
 
 
 # =============================================================================
@@ -62,33 +62,90 @@ class ProxyPort(IntEnum):
 class Engine(str, Enum):
     """
     Supported search engines for SERP API.
+
+    Engine naming convention:
+    - Base search: {engine} for basic web search (google, bing, yandex, duckduckgo)
+    - Verticals: {engine}_{vertical} (e.g., google_news, bing_images)
+    - Sub-verticals: {engine}_{vertical}_{sub} (e.g., google_scholar_cite)
     """
 
+    # ===================
+    # Google
+    # ===================
     GOOGLE = "google"
+    GOOGLE_SEARCH = "google_search"
+    GOOGLE_AI_MODE = "google_ai_mode"
+    GOOGLE_WEB = "google_web"
+    GOOGLE_SHOPPING = "google_shopping"
+    GOOGLE_LOCAL = "google_local"
+    GOOGLE_VIDEOS = "google_videos"
+    GOOGLE_NEWS = "google_news"
+    GOOGLE_FLIGHTS = "google_flights"
+    GOOGLE_IMAGES = "google_images"
+    GOOGLE_LENS = "google_lens"
+    GOOGLE_TRENDS = "google_trends"
+    GOOGLE_HOTELS = "google_hotels"
+    GOOGLE_PLAY = "google_play"
+    GOOGLE_JOBS = "google_jobs"
+    GOOGLE_SCHOLAR = "google_scholar"
+    GOOGLE_SCHOLAR_CITE = "google_scholar_cite"
+    GOOGLE_SCHOLAR_AUTHOR = "google_scholar_author"
+    GOOGLE_MAPS = "google_maps"
+    GOOGLE_FINANCE = "google_finance"
+    GOOGLE_FINANCE_MARKETS = "google_finance_markets"
+    GOOGLE_PATENTS = "google_patents"
+    GOOGLE_PATENTS_DETAILS = "google_patents_details"
+
+    # ===================
+    # Bing
+    # ===================
     BING = "bing"
+    BING_SEARCH = "bing_search"
+    BING_IMAGES = "bing_images"
+    BING_VIDEOS = "bing_videos"
+    BING_NEWS = "bing_news"
+    BING_MAPS = "bing_maps"
+    BING_SHOPPING = "bing_shopping"
+
+    # ===================
+    # Yandex
+    # ===================
     YANDEX = "yandex"
+    YANDEX_SEARCH = "yandex_search"
+
+    # ===================
+    # DuckDuckGo
+    # ===================
     DUCKDUCKGO = "duckduckgo"
-    BAIDU = "baidu"
-    YAHOO = "yahoo"
-    NAVER = "naver"
+    DUCKDUCKGO_SEARCH = "duckduckgo_search"
 
 
 class GoogleSearchType(str, Enum):
     """
     Search types specific to Google.
+
+    These map to the second part of Google engine names.
+    For example, GOOGLE + NEWS = google_news
     """
 
     SEARCH = "search"
-    MAPS = "maps"
+    AI_MODE = "ai_mode"
+    WEB = "web"
     SHOPPING = "shopping"
+    LOCAL = "local"
+    VIDEOS = "videos"
     NEWS = "news"
+    FLIGHTS = "flights"
     IMAGES = "images"
-    VIDEOS = "videos"
-    SCHOLAR = "scholar"
-    PATENTS = "patents"
+    LENS = "lens"
+    TRENDS = "trends"
+    HOTELS = "hotels"
+    PLAY = "play"
     JOBS = "jobs"
-    FLIGHTS = "flights"
+    SCHOLAR = "scholar"
+    MAPS = "maps"
     FINANCE = "finance"
+    PATENTS = "patents"
 
 
 class BingSearchType(str, Enum):
@@ -101,6 +158,20 @@ class BingSearchType(str, Enum):
     VIDEOS = "videos"
     NEWS = "news"
     MAPS = "maps"
+    SHOPPING = "shopping"
+
+
+class GoogleTbm(str, Enum):
+    """
+    Google tbm (to be matched) parameter values.
+
+    Only available when using specific Google engines that support tbm.
+    """
+
+    NEWS = "nws"
+    SHOPPING = "shop"
+    IMAGES = "isch"
+    VIDEOS = "vid"
 
 
 class Device(str, Enum):
@@ -159,13 +230,12 @@ class SessionType(str, Enum):
 class OutputFormat(str, Enum):
     """
     Output formats for Universal Scraping API.
+
+    Currently supported: html, png
     """
 
     HTML = "html"
     PNG = "png"
-    PDF = "pdf"
-    MARKDOWN = "markdown"
-    TEXT = "text"
 
 
 class DataFormat(str, Enum):
@@ -222,7 +292,7 @@ class TaskStatus(str, Enum):
 
 
 # =============================================================================
-# Country Enum (常用国家)
+# Country Enum (Common Countries)
 # =============================================================================
 
 
@@ -306,7 +376,6 @@ def normalize_enum_value(value: object, enum_class: type) -> str:
     Safely convert an enum or string to its string value.
     """
     if isinstance(value, enum_class):
-        # value is an enum member, get its .value
         return str(getattr(value, "value", value)).lower()
     if isinstance(value, str):
         return value.lower()

thordata/exceptions.py

@@ -222,7 +222,8 @@ class ThordataNotCollectedError(ThordataAPIError):
     This error is often transient and typically safe to retry.
     """
 
-    HTTP_STATUS_CODES = {300}
+    API_CODES = {300}
+    HTTP_STATUS_CODES: Set[int] = set()
 
     @property
     def is_retryable(self) -> bool:
@@ -262,8 +263,17 @@ def raise_for_code(
         ThordataValidationError: For 400/422 codes.
         ThordataAPIError: For all other error codes.
     """
-    # Use the code from payload if status_code not available
-    effective_code = status_code or code
+    # Determine the effective error code.
+    # Prefer payload `code` when present and not success (200),
+    # otherwise fall back to HTTP status when it indicates an error.
+    effective_code: Optional[int] = None
+
+    if code is not None and code != 200:
+        effective_code = code
+    elif status_code is not None and status_code != 200:
+        effective_code = status_code
+    else:
+        effective_code = code if code is not None else status_code
 
     kwargs = {
         "status_code": status_code,
@@ -272,8 +282,9 @@ def raise_for_code(
         "request_id": request_id,
     }
 
-    # Not collected (often retryable, not billed)
-    if effective_code in ThordataNotCollectedError.HTTP_STATUS_CODES:
+    # Not collected (API payload code 300, often retryable, not billed)
+    # Check this FIRST since 300 is in API_CODES, not HTTP_STATUS_CODES
+    if effective_code in ThordataNotCollectedError.API_CODES:
         raise ThordataNotCollectedError(message, **kwargs)
 
     # Auth errors

thordata/models.py

@@ -795,6 +795,126 @@ class ScraperTaskConfig:
         return payload
 
 
+@dataclass
+class CommonSettings:
+    """
+    Common settings for YouTube video/audio downloads.
+
+    Used by /video_builder endpoint as `common_settings` parameter.
+    Also known as `spider_universal` in some documentation.
+
+    Args:
+        resolution: Video resolution (360p/480p/720p/1080p/1440p/2160p).
+        audio_format: Audio format (opus/mp3).
+        bitrate: Audio bitrate (48/64/128/160/256/320 or with Kbps suffix).
+        is_subtitles: Whether to download subtitles ("true"/"false").
+        subtitles_language: Subtitle language code (e.g., "en", "zh-Hans").
+
+    Example for video:
+        >>> settings = CommonSettings(
+        ...     resolution="1080p",
+        ...     is_subtitles="true",
+        ...     subtitles_language="en"
+        ... )
+
+    Example for audio:
+        >>> settings = CommonSettings(
+        ...     audio_format="mp3",
+        ...     bitrate="320",
+        ...     is_subtitles="true",
+        ...     subtitles_language="en"
+        ... )
+    """
+
+    # Video settings
+    resolution: Optional[str] = None
+
+    # Audio settings
+    audio_format: Optional[str] = None
+    bitrate: Optional[str] = None
+
+    # Subtitle settings (used by both video and audio)
+    is_subtitles: Optional[str] = None
+    subtitles_language: Optional[str] = None
+
+    # Valid values for validation
+    VALID_RESOLUTIONS = {"360p", "480p", "720p", "1080p", "1440p", "2160p"}
+    VALID_AUDIO_FORMATS = {"opus", "mp3"}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary, excluding None values."""
+        result = {}
+        if self.resolution is not None:
+            result["resolution"] = self.resolution
+        if self.audio_format is not None:
+            result["audio_format"] = self.audio_format
+        if self.bitrate is not None:
+            result["bitrate"] = self.bitrate
+        if self.is_subtitles is not None:
+            result["is_subtitles"] = self.is_subtitles
+        if self.subtitles_language is not None:
+            result["subtitles_language"] = self.subtitles_language
+        return result
+
+    def to_json(self) -> str:
+        """Convert to JSON string for form submission."""
+        return json.dumps(self.to_dict())
+
+
+@dataclass
+class VideoTaskConfig:
+    """
+    Configuration for creating a YouTube video/audio download task.
+
+    Uses the /video_builder endpoint.
+
+    Args:
+        file_name: Name for the output file. Supports {{TasksID}}, {{VideoID}}.
+        spider_id: Spider identifier (e.g., "youtube_video_by-url", "youtube_audio_by-url").
+        spider_name: Spider name (typically "youtube.com").
+        parameters: Spider-specific parameters (e.g., video URL).
+        common_settings: Video/audio settings (resolution, format, subtitles).
+        include_errors: Include error details in output.
+
+    Example:
+        >>> config = VideoTaskConfig(
+        ...     file_name="{{VideoID}}",
+        ...     spider_id="youtube_video_by-url",
+        ...     spider_name="youtube.com",
+        ...     parameters={"url": "https://www.youtube.com/watch?v=xxx"},
+        ...     common_settings=CommonSettings(
+        ...         resolution="1080p",
+        ...         is_subtitles="true",
+        ...         subtitles_language="en"
+        ...     )
+        ... )
+    """
+
+    file_name: str
+    spider_id: str
+    spider_name: str
+    parameters: Dict[str, Any]
+    common_settings: CommonSettings
+    include_errors: bool = True
+
+    def to_payload(self) -> Dict[str, Any]:
+        """
+        Convert to API request payload.
+
+        Returns:
+            Dictionary ready to be sent to the video_builder API.
+        """
+        payload: Dict[str, Any] = {
+            "file_name": self.file_name,
+            "spider_id": self.spider_id,
+            "spider_name": self.spider_name,
+            "spider_parameters": json.dumps([self.parameters]),
+            "spider_errors": "true" if self.include_errors else "false",
+            "common_settings": self.common_settings.to_json(),
+        }
+        return payload
+
+
 # =============================================================================
 # Response Models
 # =============================================================================
@@ -838,3 +958,177 @@ class TaskStatusResponse:
         """Check if the task failed."""
         failure_statuses = {"failed", "error"}
         return self.status.lower() in failure_statuses
+
+
+@dataclass
+class UsageStatistics:
+    """
+    Response model for account usage statistics.
+
+    Attributes:
+        total_usage_traffic: Total traffic used (KB).
+        traffic_balance: Remaining traffic balance (KB).
+        query_days: Number of days in the query range.
+        range_usage_traffic: Traffic used in the specified date range (KB).
+        data: Daily usage breakdown.
+    """
+
+    total_usage_traffic: float
+    traffic_balance: float
+    query_days: int
+    range_usage_traffic: float
+    data: List[Dict[str, Any]]
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "UsageStatistics":
+        """Create from API response dict."""
+        return cls(
+            total_usage_traffic=float(data.get("total_usage_traffic", 0)),
+            traffic_balance=float(data.get("traffic_balance", 0)),
+            query_days=int(data.get("query_days", 0)),
+            range_usage_traffic=float(data.get("range_usage_traffic", 0)),
+            data=data.get("data", []),
+        )
+
+    def total_usage_gb(self) -> float:
+        """Get total usage in GB."""
+        return self.total_usage_traffic / (1024 * 1024)
+
+    def balance_gb(self) -> float:
+        """Get balance in GB."""
+        return self.traffic_balance / (1024 * 1024)
+
+    def range_usage_gb(self) -> float:
+        """Get range usage in GB."""
+        return self.range_usage_traffic / (1024 * 1024)
+
+
+@dataclass
+class ProxyUser:
+    """
+    Proxy user (sub-account) information.
+
+    Attributes:
+        username: User's username.
+        password: User's password.
+        status: User status (True=enabled, False=disabled).
+        traffic_limit: Traffic limit in MB (0 = unlimited).
+        usage_traffic: Traffic used in KB.
+    """
+
+    username: str
+    password: str
+    status: bool
+    traffic_limit: int
+    usage_traffic: float
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyUser":
+        """Create from API response dict."""
+        return cls(
+            username=data.get("username", ""),
+            password=data.get("password", ""),
+            status=data.get("status") in (True, "true", 1),
+            traffic_limit=int(data.get("traffic_limit", 0)),
+            usage_traffic=float(data.get("usage_traffic", 0)),
+        )
+
+    def usage_gb(self) -> float:
+        """Get usage in GB."""
+        return self.usage_traffic / (1024 * 1024)
+
+    def limit_gb(self) -> float:
+        """Get limit in GB (0 means unlimited)."""
+        if self.traffic_limit == 0:
+            return 0
+        return self.traffic_limit / 1024
+
+
+@dataclass
+class ProxyUserList:
+    """
+    Response model for proxy user list.
+
+    Attributes:
+        limit: Total traffic limit (KB).
+        remaining_limit: Remaining traffic limit (KB).
+        user_count: Number of users.
+        users: List of proxy users.
+    """
+
+    limit: float
+    remaining_limit: float
+    user_count: int
+    users: List[ProxyUser]
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyUserList":
+        """Create from API response dict."""
+        user_list = data.get("list", [])
+        users = [ProxyUser.from_dict(u) for u in user_list]
+
+        return cls(
+            limit=float(data.get("limit", 0)),
+            remaining_limit=float(data.get("remaining_limit", 0)),
+            user_count=int(data.get("user_count", len(users))),
+            users=users,
+        )
+
+
+@dataclass
+class ProxyServer:
+    """
+    ISP or Datacenter proxy server information.
+
+    Attributes:
+        ip: Proxy server IP address.
+        port: Proxy server port.
+        username: Authentication username.
+        password: Authentication password.
+        expiration_time: Expiration timestamp (Unix timestamp or datetime string).
+        region: Server region (optional).
+    """
+
+    ip: str
+    port: int
+    username: str
+    password: str
+    expiration_time: Optional[Union[int, str]] = None
+    region: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ProxyServer":
+        """Create from API response dict."""
+        return cls(
+            ip=data.get("ip", ""),
+            port=int(data.get("port", 0)),
+            username=data.get("username", data.get("user", "")),
+            password=data.get("password", data.get("pwd", "")),
+            expiration_time=data.get("expiration_time", data.get("expireTime")),
+            region=data.get("region"),
+        )
+
+    def to_proxy_url(self, protocol: str = "http") -> str:
+        """
+        Build proxy URL for this server.
+
+        Args:
+            protocol: Proxy protocol (http/https/socks5).
+
+        Returns:
+            Complete proxy URL.
+        """
+        return f"{protocol}://{self.username}:{self.password}@{self.ip}:{self.port}"
+
+    def is_expired(self) -> bool:
+        """Check if proxy has expired (if expiration_time is available)."""
+        if self.expiration_time is None:
+            return False
+
+        import time
+
+        if isinstance(self.expiration_time, int):
+            return time.time() > self.expiration_time
+
+        # String timestamp handling would need datetime parsing
+        return False

thordata/retry.py

@@ -64,7 +64,10 @@ class RetryConfig:
 
     # Status codes to retry on (5xx server errors + 429 rate limit)
     retry_on_status_codes: Set[int] = field(
-        default_factory=lambda: {300, 429, 500, 502, 503, 504}
+        default_factory=lambda: {429, 500, 502, 503, 504}
+    )
+    retry_on_api_codes: Set[int] = field(
+        default_factory=lambda: {300}  # API response body code
     )
 
     # Exception types to always retry on