thordata-sdk 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

thordata/models.py

@@ -26,13 +26,10 @@ from __future__ import annotations
 
 import json
 import re
-import ssl
 import uuid
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Any, Dict, List, Optional, Union
-
-import urllib3
+from typing import Any
 
 # =============================================================================
 # Proxy Product Types
@@ -111,21 +108,21 @@ class ProxyConfig:
 
     username: str
     password: str
-    product: Union[ProxyProduct, str] = ProxyProduct.RESIDENTIAL
-    host: Optional[str] = None
-    port: Optional[int] = None
+    product: ProxyProduct | str = ProxyProduct.RESIDENTIAL
+    host: str | None = None
+    port: int | None = None
     protocol: str = "http"
 
     # Geo-targeting
-    continent: Optional[str] = None
-    country: Optional[str] = None
-    state: Optional[str] = None
-    city: Optional[str] = None
-    asn: Optional[str] = None
+    continent: str | None = None
+    country: str | None = None
+    state: str | None = None
+    city: str | None = None
+    asn: str | None = None
 
     # Session control
-    session_id: Optional[str] = None
-    session_duration: Optional[int] = None  # minutes, 1-90
+    session_id: str | None = None
+    session_duration: int | None = None  # minutes, 1-90
 
     # Valid continent codes
     VALID_CONTINENTS = {"af", "an", "as", "eu", "na", "oc", "sa"}
@@ -245,7 +242,7 @@ class ProxyConfig:
         """Basic auth string 'username:password' for Proxy-Authorization."""
         return f"{self.build_username()}:{self.password}"
 
-    def to_proxies_dict(self) -> Dict[str, str]:
+    def to_proxies_dict(self) -> dict[str, str]:
         """
         Build a proxies dict suitable for the requests library.
 
@@ -300,7 +297,7 @@ class WhitelistProxyConfig:
     def build_proxy_url(self) -> str:
         return f"{self.protocol}://{self.host}:{self.port}"
 
-    def to_proxies_dict(self) -> Dict[str, str]:
+    def to_proxies_dict(self) -> dict[str, str]:
         url = self.build_proxy_url()
         return {"http": url, "https": url}
 
@@ -358,7 +355,7 @@ class StaticISPProxy:
             f"{self.protocol}://{self.username}:{self.password}@{self.host}:{self.port}"
         )
 
-    def to_proxies_dict(self) -> Dict[str, str]:
+    def to_proxies_dict(self) -> dict[str, str]:
         """
         Build a proxies dict suitable for the requests library.
 
@@ -387,7 +384,7 @@ class StaticISPProxy:
             ) from e
 
     @classmethod
-    def from_env(cls) -> "StaticISPProxy":
+    def from_env(cls) -> StaticISPProxy:
         """
         Create StaticISPProxy from environment variables.
 
@@ -513,39 +510,39 @@ class SerpRequest:
     start: int = 0
 
     # Localization
-    country: Optional[str] = None  # 'gl' for Google
-    language: Optional[str] = None  # 'hl' for Google
-    google_domain: Optional[str] = None
-    countries_filter: Optional[str] = None  # 'cr' parameter
-    languages_filter: Optional[str] = None  # 'lr' parameter
+    country: str | None = None  # 'gl' for Google
+    language: str | None = None  # 'hl' for Google
+    google_domain: str | None = None
+    countries_filter: str | None = None  # 'cr' parameter
+    languages_filter: str | None = None  # 'lr' parameter
 
     # Geo-targeting
-    location: Optional[str] = None
-    uule: Optional[str] = None  # Encoded location
+    location: str | None = None
+    uule: str | None = None  # Encoded location
 
     # Search type
-    search_type: Optional[str] = None  # tbm parameter (isch, shop, nws, vid, ...)
+    search_type: str | None = None  # tbm parameter (isch, shop, nws, vid, ...)
 
     # Filters
-    safe_search: Optional[bool] = None
-    time_filter: Optional[str] = None  # tbs parameter (time part)
+    safe_search: bool | None = None
+    time_filter: str | None = None  # tbs parameter (time part)
     no_autocorrect: bool = False  # nfpr parameter
-    filter_duplicates: Optional[bool] = None  # filter parameter
+    filter_duplicates: bool | None = None  # filter parameter
 
     # Device & Rendering
-    device: Optional[str] = None  # 'desktop', 'mobile', 'tablet'
-    render_js: Optional[bool] = None  # render_js parameter
-    no_cache: Optional[bool] = None  # no_cache parameter
+    device: str | None = None  # 'desktop', 'mobile', 'tablet'
+    render_js: bool | None = None  # render_js parameter
+    no_cache: bool | None = None  # no_cache parameter
 
     # Output format
     output_format: str = "json"  # 'json' or 'html'
 
     # Advanced Google parameters
-    ludocid: Optional[str] = None  # Google Place ID
-    kgmid: Optional[str] = None  # Knowledge Graph ID
+    ludocid: str | None = None  # Google Place ID
+    kgmid: str | None = None  # Knowledge Graph ID
 
     # Pass-through
-    extra_params: Dict[str, Any] = field(default_factory=dict)
+    extra_params: dict[str, Any] = field(default_factory=dict)
 
     # Search type mappings for tbm parameter
     SEARCH_TYPE_MAP = {
@@ -578,7 +575,7 @@ class SerpRequest:
         "baidu": "baidu.com",
     }
 
-    def to_payload(self) -> Dict[str, Any]:
+    def to_payload(self) -> dict[str, Any]:
         """
         Convert to API request payload.
 
@@ -587,7 +584,7 @@ class SerpRequest:
         """
         engine = self.engine.lower()
 
-        payload: Dict[str, Any] = {
+        payload: dict[str, Any] = {
             "engine": engine,
             "num": str(self.num),
         }
@@ -722,14 +719,14 @@ class UniversalScrapeRequest:
     url: str
     js_render: bool = False
     output_format: str = "html"  # 'html' or 'png'
-    country: Optional[str] = None
-    block_resources: Optional[str] = None  # e.g., 'script', 'image', 'script,image'
-    clean_content: Optional[str] = None  # e.g., 'js', 'css', 'js,css'
-    wait: Optional[int] = None  # Milliseconds, max 100000
-    wait_for: Optional[str] = None  # CSS selector
-    headers: Optional[List[Dict[str, str]]] = None  # [{"name": "...", "value": "..."}]
-    cookies: Optional[List[Dict[str, str]]] = None  # [{"name": "...", "value": "..."}]
-    extra_params: Dict[str, Any] = field(default_factory=dict)  # 这个必须用 field()
+    country: str | None = None
+    block_resources: str | None = None  # e.g., 'script', 'image', 'script,image'
+    clean_content: str | None = None  # e.g., 'js', 'css', 'js,css'
+    wait: int | None = None  # Milliseconds, max 100000
+    wait_for: str | None = None  # CSS selector
+    headers: list[dict[str, str]] | None = None  # [{"name": "...", "value": "..."}]
+    cookies: list[dict[str, str]] | None = None  # [{"name": "...", "value": "..."}]
+    extra_params: dict[str, Any] = field(default_factory=dict)  # 这个必须用 field()
 
     def __post_init__(self) -> None:
         """Validate configuration."""
@@ -745,14 +742,14 @@ class UniversalScrapeRequest:
                 f"wait must be between 0 and 100000 milliseconds, got {self.wait}"
             )
 
-    def to_payload(self) -> Dict[str, Any]:
+    def to_payload(self) -> dict[str, Any]:
         """
         Convert to API request payload.
 
         Returns:
             Dictionary ready to be sent to the Universal API.
         """
-        payload: Dict[str, Any] = {
+        payload: dict[str, Any] = {
             "url": self.url,
             "js_render": "True" if self.js_render else "False",
             "type": self.output_format.lower(),
@@ -820,18 +817,18 @@ class ScraperTaskConfig:
     file_name: str
     spider_id: str
     spider_name: str
-    parameters: Dict[str, Any]
-    universal_params: Optional[Dict[str, Any]] = None
+    parameters: dict[str, Any]
+    universal_params: dict[str, Any] | None = None
     include_errors: bool = True
 
-    def to_payload(self) -> Dict[str, Any]:
+    def to_payload(self) -> dict[str, Any]:
         """
         Convert to API request payload.
 
         Returns:
             Dictionary ready to be sent to the Web Scraper API.
         """
-        payload: Dict[str, Any] = {
+        payload: dict[str, Any] = {
             "file_name": self.file_name,
             "spider_id": self.spider_id,
             "spider_name": self.spider_name,
@@ -877,21 +874,21 @@ class CommonSettings:
     """
 
     # Video settings
-    resolution: Optional[str] = None
+    resolution: str | None = None
 
     # Audio settings
-    audio_format: Optional[str] = None
-    bitrate: Optional[str] = None
+    audio_format: str | None = None
+    bitrate: str | None = None
 
     # Subtitle settings (used by both video and audio)
-    is_subtitles: Optional[str] = None
-    subtitles_language: Optional[str] = None
+    is_subtitles: str | None = None
+    subtitles_language: str | None = None
 
     # Valid values for validation
     VALID_RESOLUTIONS = {"360p", "480p", "720p", "1080p", "1440p", "2160p"}
     VALID_AUDIO_FORMATS = {"opus", "mp3"}
 
-    def to_dict(self) -> Dict[str, Any]:
+    def to_dict(self) -> dict[str, Any]:
         """Convert to dictionary, excluding None values."""
         result = {}
         if self.resolution is not None:
@@ -943,18 +940,18 @@ class VideoTaskConfig:
     file_name: str
     spider_id: str
     spider_name: str
-    parameters: Dict[str, Any]
+    parameters: dict[str, Any]
     common_settings: CommonSettings
     include_errors: bool = True
 
-    def to_payload(self) -> Dict[str, Any]:
+    def to_payload(self) -> dict[str, Any]:
         """
         Convert to API request payload.
 
         Returns:
             Dictionary ready to be sent to the video_builder API.
         """
-        payload: Dict[str, Any] = {
+        payload: dict[str, Any] = {
             "file_name": self.file_name,
             "spider_id": self.spider_id,
             "spider_name": self.spider_name,
@@ -984,8 +981,8 @@ class TaskStatusResponse:
 
     task_id: str
     status: str
-    progress: Optional[int] = None
-    message: Optional[str] = None
+    progress: int | None = None
+    message: str | None = None
 
     def is_complete(self) -> bool:
         """Check if the task has completed (success or failure)."""
@@ -1027,10 +1024,10 @@ class UsageStatistics:
     traffic_balance: float
     query_days: int
     range_usage_traffic: float
-    data: List[Dict[str, Any]]
+    data: list[dict[str, Any]]
 
     @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "UsageStatistics":
+    def from_dict(cls, data: dict[str, Any]) -> UsageStatistics:
         """Create from API response dict."""
         return cls(
             total_usage_traffic=float(data.get("total_usage_traffic", 0)),
@@ -1073,7 +1070,7 @@ class ProxyUser:
     usage_traffic: float
 
     @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "ProxyUser":
+    def from_dict(cls, data: dict[str, Any]) -> ProxyUser:
         """Create from API response dict."""
         return cls(
             username=data.get("username", ""),
@@ -1109,10 +1106,10 @@ class ProxyUserList:
     limit: float
     remaining_limit: float
     user_count: int
-    users: List[ProxyUser]
+    users: list[ProxyUser]
 
     @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "ProxyUserList":
+    def from_dict(cls, data: dict[str, Any]) -> ProxyUserList:
         """Create from API response dict."""
         user_list = data.get("list", [])
         users = [ProxyUser.from_dict(u) for u in user_list]
@@ -1143,11 +1140,11 @@ class ProxyServer:
     port: int
     username: str
     password: str
-    expiration_time: Optional[Union[int, str]] = None
-    region: Optional[str] = None
+    expiration_time: int | str | None = None
+    region: str | None = None
 
     @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "ProxyServer":
+    def from_dict(cls, data: dict[str, Any]) -> ProxyServer:
         """Create from API response dict."""
         return cls(
             ip=data.get("ip", ""),

thordata/retry.py

@@ -22,7 +22,7 @@ import random
 import time
 from dataclasses import dataclass, field
 from functools import wraps
-from typing import Any, Callable, Optional, Set, Tuple
+from typing import Any, Callable
 
 from .exceptions import (
     ThordataNetworkError,
@@ -64,15 +64,15 @@ class RetryConfig:
     jitter_factor: float = 0.1
 
     # Status codes to retry on (5xx server errors + 429 rate limit)
-    retry_on_status_codes: Set[int] = field(
+    retry_on_status_codes: set[int] = field(
         default_factory=lambda: {429, 500, 502, 503, 504}
     )
-    retry_on_api_codes: Set[int] = field(
+    retry_on_api_codes: set[int] = field(
         default_factory=lambda: {300}  # API response body code
     )
 
     # Exception types to always retry on
-    retry_on_exceptions: Tuple[type, ...] = field(
+    retry_on_exceptions: tuple[type, ...] = field(
         default_factory=lambda: (
             ThordataNetworkError,
             ThordataServerError,
@@ -104,7 +104,7 @@ class RetryConfig:
         return delay
 
     def should_retry(
-        self, exception: Exception, attempt: int, status_code: Optional[int] = None
+        self, exception: Exception, attempt: int, status_code: int | None = None
     ) -> bool:
         """
         Determine if a request should be retried.
@@ -138,8 +138,8 @@ class RetryConfig:
 
 
 def with_retry(
-    config: Optional[RetryConfig] = None,
-    on_retry: Optional[Callable[[int, Exception, float], None]] = None,
+    config: RetryConfig | None = None,
+    on_retry: Callable[[int, Exception, float], None] | None = None,
 ) -> Callable:
     """
     Decorator to add retry logic to a function.
@@ -168,7 +168,7 @@ def with_retry(
     def decorator(func: Callable) -> Callable:
         @wraps(func)
         def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
-            last_exception: Optional[Exception] = None
+            last_exception: Exception | None = None
 
             for attempt in range(config.max_retries + 1):
                 try:
@@ -202,7 +202,7 @@ def with_retry(
 
         @wraps(func)
         async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
-            last_exception: Optional[Exception] = None
+            last_exception: Exception | None = None
 
             for attempt in range(config.max_retries + 1):
                 try:
@@ -244,7 +244,7 @@ def with_retry(
     return decorator
 
 
-def _extract_status_code(exception: Exception) -> Optional[int]:
+def _extract_status_code(exception: Exception) -> int | None:
     """
     Extract HTTP status code from various exception types.
 
@@ -302,10 +302,10 @@ class RetryableRequest:
         ...             retry.wait()
     """
 
-    def __init__(self, config: Optional[RetryConfig] = None) -> None:
+    def __init__(self, config: RetryConfig | None = None) -> None:
         self.config = config or RetryConfig()
         self.attempt = 0
-        self.last_exception: Optional[Exception] = None
+        self.last_exception: Exception | None = None
 
     def __enter__(self) -> RetryableRequest:
         return self
@@ -314,7 +314,7 @@ class RetryableRequest:
         pass
 
     def should_continue(
-        self, exception: Exception, status_code: Optional[int] = None
+        self, exception: Exception, status_code: int | None = None
     ) -> bool:
         """
         Check if we should continue retrying.

thordata_sdk-1.1.0.dist-info/METADATA

@@ -0,0 +1,271 @@
+Metadata-Version: 2.4
+Name: thordata-sdk
+Version: 1.1.0
+Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
+Author-email: Thordata Developer Team <support@thordata.com>
+License: MIT
+Project-URL: Homepage, https://www.thordata.com
+Project-URL: Documentation, https://github.com/Thordata/thordata-python-sdk#readme
+Project-URL: Source, https://github.com/Thordata/thordata-python-sdk
+Project-URL: Tracker, https://github.com/Thordata/thordata-python-sdk/issues
+Project-URL: Changelog, https://github.com/Thordata/thordata-python-sdk/blob/main/CHANGELOG.md
+Keywords: web scraping,proxy,residential proxy,datacenter proxy,ai,llm,data-mining,serp,thordata,web scraper,anti-bot bypass
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: Proxy Servers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.25.0
+Requires-Dist: aiohttp>=3.9.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-httpserver>=1.0.0; extra == "dev"
+Requires-Dist: python-dotenv>=1.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-requests>=2.28.0; extra == "dev"
+Requires-Dist: aioresponses>=0.7.6; extra == "dev"
+Dynamic: license-file
+
+# Thordata Python SDK
+
+<div align="center">
+
+**Official Python Client for Thordata APIs**
+
+*Proxy Network • SERP API • Web Unlocker • Web Scraper API*
+
+[![PyPI version](https://img.shields.io/pypi/v/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+</div>
+
+---
+
+## 📦 Installation
+
+```bash
+pip install thordata-sdk
+```
+
+Optional dependencies for Scraping Browser examples:
+```bash
+pip install playwright
+```
+
+## 🔐 Configuration
+
+Set the following environment variables (recommended):
+
+```bash
+# Required for SERP, Universal, and Proxy Network
+export THORDATA_SCRAPER_TOKEN="your_scraper_token"
+
+# Required for Web Scraper Tasks & Account Management
+export THORDATA_PUBLIC_TOKEN="your_public_token"
+export THORDATA_PUBLIC_KEY="your_public_key"
+
+# Optional: Default Proxy Credentials (for Proxy Network)
+export THORDATA_RESIDENTIAL_USERNAME="user"
+export THORDATA_RESIDENTIAL_PASSWORD="pass"
+```
+
+## 🚀 Quick Start
+
+```python
+from thordata import ThordataClient
+
+# Initialize (credentials loaded from env)
+client = ThordataClient(scraper_token="...") 
+
+# 1. SERP Search
+print("--- SERP Search ---")
+results = client.serp_search("python tutorial", engine="google")
+print(f"Title: {results['organic'][0]['title']}")
+
+# 2. Universal Scrape (Web Unlocker)
+print("\n--- Universal Scrape ---")
+html = client.universal_scrape("https://httpbin.org/html")
+print(f"HTML Length: {len(html)}")
+```
+
+## 📚 Core Features
+
+### 🌐 Proxy Network
+
+Easily generate proxy URLs with geo-targeting and sticky sessions. The SDK handles connection pooling automatically.
+
+```python
+from thordata import ProxyConfig, ProxyProduct
+
+# Create a proxy configuration
+proxy = ProxyConfig(
+    username="user",
+    password="pass",
+    product=ProxyProduct.RESIDENTIAL,
+    country="us",
+    city="new_york",
+    session_id="session123",
+    session_duration=10  # Sticky for 10 mins
+)
+
+# Use with the client (high performance)
+response = client.get("https://httpbin.org/ip", proxy_config=proxy)
+print(response.json())
+
+# Or get the URL string for other libs (requests, scrapy, etc.)
+proxy_url = proxy.build_proxy_url()
+print(f"Proxy URL: {proxy_url}")
+```
+
+### 🔍 SERP API
+
+Real-time search results from Google, Bing, Yandex, etc.
+
+```python
+from thordata import SerpRequest, Engine
+
+# Simple
+results = client.serp_search(
+    query="pizza near me",
+    engine=Engine.GOOGLE_MAPS,
+    country="us"
+)
+
+# Advanced (Strongly Typed)
+request = SerpRequest(
+    query="AI news",
+    engine="google_news",
+    num=50,
+    time_filter="week",
+    location="San Francisco",
+    render_js=True
+)
+results = client.serp_search_advanced(request)
+```
+
+### 🔓 Universal Scraping API (Web Unlocker)
+
+Bypass Cloudflare, CAPTCHAs, and antibot systems.
+
+```python
+html = client.universal_scrape(
+    url="https://example.com/protected",
+    js_render=True,
+    wait_for=".content",
+    country="gb",
+    output_format="html"
+)
+```
+
+### 🕷️ Web Scraper API (Async Tasks)
+
+Manage asynchronous scraping tasks for massive scale.
+
+```python
+# 1. Create Task
+task_id = client.create_scraper_task(
+    file_name="my_task",
+    spider_id="universal",
+    spider_name="universal",
+    parameters={"url": "https://example.com"}
+)
+print(f"Task Created: {task_id}")
+
+# 2. Wait for Completion
+status = client.wait_for_task(task_id, max_wait=600)
+
+# 3. Get Result
+if status == "ready":
+    download_url = client.get_task_result(task_id)
+    print(f"Result: {download_url}")
+```
+
+### 📹 Video/Audio Tasks
+
+Download content from YouTube and other supported platforms.
+
+```python
+from thordata import CommonSettings
+
+task_id = client.create_video_task(
+    file_name="video_{{VideoID}}",
+    spider_id="youtube_video_by-url",
+    spider_name="youtube.com",
+    parameters={"url": "https://youtube.com/watch?v=..."},
+    common_settings=CommonSettings(resolution="1080p")
+)
+```
+
+### 📊 Account Management
+
+Access usage statistics, manage sub-users, and whitelist IPs.
+
+```python
+# Get Usage Stats
+stats = client.get_usage_statistics("2024-01-01", "2024-01-31")
+print(f"Balance: {stats.balance_gb():.2f} GB")
+
+# List Proxy Users
+users = client.list_proxy_users()
+print(f"Active Sub-users: {users.user_count}")
+
+# Whitelist IP
+client.add_whitelist_ip("1.2.3.4")
+```
+
+## ⚙️ Advanced Usage
+
+### Async Client
+
+For high-concurrency applications, use `AsyncThordataClient`.
+
+```python
+import asyncio
+from thordata import AsyncThordataClient
+
+async def main():
+    async with AsyncThordataClient(scraper_token="...") as client:
+        # SERP
+        results = await client.serp_search("async python")
+        
+        # Universal
+        html = await client.universal_scrape("https://example.com")
+
+asyncio.run(main())
+```
+
+Note: `AsyncThordataClient` does not support HTTPS proxy tunneling (TLS-in-TLS) due to `aiohttp` limitations. For proxy network requests, use the sync client.
+
+### Custom Retry Configuration
+
+```python
+from thordata import RetryConfig
+
+retry = RetryConfig(
+    max_retries=5,
+    backoff_factor=1.5,
+    retry_on_status_codes={429, 500, 502, 503, 504}
+)
+
+client = ThordataClient(..., retry_config=retry)
+```
+
+## 📄 License
+
+MIT License