thordata-sdk 1.0.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

thordata/__init__.py

@@ -35,7 +35,7 @@ Async Usage:
     >>> asyncio.run(main())
 """
 
-__version__ = "1.0.0"
+__version__ = "1.1.0"
 __author__ = "Thordata Developer Team"
 __email__ = "support@thordata.com"
 

thordata/_example_utils.py

@@ -2,8 +2,9 @@ from __future__ import annotations
 
 import json
 import os
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Any, Iterable, Optional
+from typing import Any
 
 try:
     from dotenv import load_dotenv
@@ -23,7 +24,7 @@ def env(name: str) -> str:
     return (os.getenv(name) or "").strip()
 
 
-def skip_if_missing(required: Iterable[str], *, tip: Optional[str] = None) -> bool:
+def skip_if_missing(required: Iterable[str], *, tip: str | None = None) -> bool:
     missing = [k for k in required if not env(k)]
     if not missing:
         return False

thordata/_utils.py

@@ -9,7 +9,8 @@ from __future__ import annotations
 import base64
 import json
 import logging
-from typing import Any, Dict
+import platform
+from typing import Any
 
 logger = logging.getLogger(__name__)
 
@@ -70,7 +71,7 @@ def decode_base64_image(png_str: str) -> bytes:
         raise ValueError(f"Failed to decode base64 image: {e}") from e
 
 
-def build_auth_headers(token: str, mode: str = "bearer") -> Dict[str, str]:
+def build_auth_headers(token: str, mode: str = "bearer") -> dict[str, str]:
     """
     Build authorization headers for API requests.
 
@@ -104,7 +105,7 @@ def build_builder_headers(
     scraper_token: str,
     public_token: str,
     public_key: str,
-) -> Dict[str, str]:
+) -> dict[str, str]:
     """
     Build headers for Web Scraper builder API.
 
@@ -129,7 +130,7 @@ def build_builder_headers(
     }
 
 
-def build_public_api_headers(public_token: str, public_key: str) -> Dict[str, str]:
+def build_public_api_headers(public_token: str, public_key: str) -> dict[str, str]:
     """
     Build headers for public API requests (task status, locations, etc.)
 
@@ -171,17 +172,19 @@ def extract_error_message(payload: Any) -> str:
 
 def build_user_agent(sdk_version: str, http_client: str) -> str:
     """
-    Build a default User-Agent for the SDK.
-
-    Args:
-        sdk_version: SDK version string.
-        http_client: "requests" or "aiohttp" (or any identifier).
-
-    Returns:
-        A User-Agent string.
+    Build a standardized User-Agent for the SDK.
+    Format: thordata-python-sdk/{version} python/{py_ver} ({system}/{release}; {machine})
     """
-    import platform
-
-    py = platform.python_version()
-    system = platform.system()
-    return f"thordata-python-sdk/{sdk_version} (python {py}; {system}; {http_client})"
+    py_ver = platform.python_version()
+    system = platform.system() or "unknown"
+    release = platform.release() or "unknown"
+    machine = platform.machine() or "unknown"
+
+    # Clean up strings to avoid UA parsing issues (remove newlines, etc)
+    system = system.replace(";", "").strip()
+
+    return (
+        f"thordata-python-sdk/{sdk_version} "
+        f"python/{py_ver} "
+        f"({system}/{release}; {machine}; {http_client})"
+    )

thordata/async_client.py

@@ -26,7 +26,7 @@ import asyncio
 import logging
 import os
 from datetime import date
-from typing import Any, Dict, List, Optional, Union
+from typing import Any
 
 import aiohttp
 
@@ -58,7 +58,6 @@ from .models import (
     UniversalScrapeRequest,
     UsageStatistics,
     VideoTaskConfig,
-    WhitelistProxyConfig,
 )
 from .retry import RetryConfig
 
@@ -98,18 +97,18 @@ class AsyncThordataClient:
     def __init__(
         self,
         scraper_token: str,
-        public_token: Optional[str] = None,
-        public_key: Optional[str] = None,
+        public_token: str | None = None,
+        public_key: str | None = None,
         proxy_host: str = "pr.thordata.net",
         proxy_port: int = 9999,
         timeout: int = 30,
         api_timeout: int = 60,
-        retry_config: Optional[RetryConfig] = None,
+        retry_config: RetryConfig | None = None,
         auth_mode: str = "bearer",
-        scraperapi_base_url: Optional[str] = None,
-        universalapi_base_url: Optional[str] = None,
-        web_scraper_api_base_url: Optional[str] = None,
-        locations_base_url: Optional[str] = None,
+        scraperapi_base_url: str | None = None,
+        universalapi_base_url: str | None = None,
+        web_scraper_api_base_url: str | None = None,
+        locations_base_url: str | None = None,
     ) -> None:
         """Initialize the Async Thordata Client."""
         if not scraper_token:
@@ -202,7 +201,7 @@ class AsyncThordataClient:
         self._proxy_expiration_url = f"{proxy_api_base}/proxy/expiration-time"
 
         # Session initialized lazily
-        self._session: Optional[aiohttp.ClientSession] = None
+        self._session: aiohttp.ClientSession | None = None
 
     async def __aenter__(self) -> AsyncThordataClient:
         """Async context manager entry."""
@@ -241,7 +240,7 @@ class AsyncThordataClient:
         self,
         url: str,
         *,
-        proxy_config: Optional[ProxyConfig] = None,
+        proxy_config: ProxyConfig | None = None,
         **kwargs: Any,
     ) -> aiohttp.ClientResponse:
         """
@@ -297,7 +296,7 @@ class AsyncThordataClient:
         self,
         url: str,
         *,
-        proxy_config: Optional[ProxyConfig] = None,
+        proxy_config: ProxyConfig | None = None,
         **kwargs: Any,
     ) -> aiohttp.ClientResponse:
         """
@@ -357,17 +356,17 @@ class AsyncThordataClient:
         self,
         query: str,
         *,
-        engine: Union[Engine, str] = Engine.GOOGLE,
+        engine: Engine | str = Engine.GOOGLE,
         num: int = 10,
-        country: Optional[str] = None,
-        language: Optional[str] = None,
-        search_type: Optional[str] = None,
-        device: Optional[str] = None,
-        render_js: Optional[bool] = None,
-        no_cache: Optional[bool] = None,
+        country: str | None = None,
+        language: str | None = None,
+        search_type: str | None = None,
+        device: str | None = None,
+        render_js: bool | None = None,
+        no_cache: bool | None = None,
         output_format: str = "json",
         **kwargs: Any,
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         """
         Execute an async SERP search.
 
@@ -447,7 +446,7 @@ class AsyncThordataClient:
                 original_error=e,
             ) from e
 
-    async def serp_search_advanced(self, request: SerpRequest) -> Dict[str, Any]:
+    async def serp_search_advanced(self, request: SerpRequest) -> dict[str, Any]:
         """
         Execute an async SERP search using a SerpRequest object.
         """
@@ -505,12 +504,12 @@ class AsyncThordataClient:
         *,
         js_render: bool = False,
         output_format: str = "html",
-        country: Optional[str] = None,
-        block_resources: Optional[str] = None,
-        wait: Optional[int] = None,
-        wait_for: Optional[str] = None,
+        country: str | None = None,
+        block_resources: str | None = None,
+        wait: int | None = None,
+        wait_for: str | None = None,
         **kwargs: Any,
-    ) -> Union[str, bytes]:
+    ) -> str | bytes:
         """
         Async scrape using Universal API (Web Unlocker).
 
@@ -541,7 +540,7 @@ class AsyncThordataClient:
 
     async def universal_scrape_advanced(
         self, request: UniversalScrapeRequest
-    ) -> Union[str, bytes]:
+    ) -> str | bytes:
         """
         Async scrape using a UniversalScrapeRequest object.
         """
@@ -600,8 +599,8 @@ class AsyncThordataClient:
         file_name: str,
         spider_id: str,
         spider_name: str,
-        parameters: Dict[str, Any],
-        universal_params: Optional[Dict[str, Any]] = None,
+        parameters: dict[str, Any],
+        universal_params: dict[str, Any] | None = None,
     ) -> str:
         """
         Create an async Web Scraper task.
@@ -659,7 +658,7 @@ class AsyncThordataClient:
         file_name: str,
         spider_id: str,
         spider_name: str,
-        parameters: Dict[str, Any],
+        parameters: dict[str, Any],
         common_settings: CommonSettings,
     ) -> str:
         """
@@ -828,7 +827,7 @@ class AsyncThordataClient:
         self,
         page: int = 1,
         size: int = 20,
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         """
         List all Web Scraper tasks.
 
@@ -845,7 +844,7 @@ class AsyncThordataClient:
         headers = build_public_api_headers(
             self.public_token or "", self.public_key or ""
         )
-        payload: Dict[str, Any] = {}
+        payload: dict[str, Any] = {}
         if page:
             payload["page"] = str(page)
         if size:
@@ -921,8 +920,8 @@ class AsyncThordataClient:
 
     async def get_usage_statistics(
         self,
-        from_date: Union[str, date],
-        to_date: Union[str, date],
+        from_date: str | date,
+        to_date: str | date,
     ) -> UsageStatistics:
         """
         Get account usage statistics for a date range.
@@ -989,7 +988,7 @@ class AsyncThordataClient:
                 f"Usage statistics failed: {e}", original_error=e
             ) from e
 
-    async def get_residential_balance(self) -> Dict[str, Any]:
+    async def get_residential_balance(self) -> dict[str, Any]:
         """
         Get residential proxy balance.
 
@@ -1030,9 +1029,9 @@ class AsyncThordataClient:
 
     async def get_residential_usage(
         self,
-        start_time: Union[str, int],
-        end_time: Union[str, int],
-    ) -> Dict[str, Any]:
+        start_time: str | int,
+        end_time: str | int,
+    ) -> dict[str, Any]:
         """
         Get residential proxy usage records.
 
@@ -1071,7 +1070,7 @@ class AsyncThordataClient:
             ) from e
 
     async def list_proxy_users(
-        self, proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL
+        self, proxy_type: ProxyType | int = ProxyType.RESIDENTIAL
     ) -> ProxyUserList:
         """List all proxy users (sub-accounts)."""
 
@@ -1126,10 +1125,10 @@ class AsyncThordataClient:
         self,
         username: str,
         password: str,
-        proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
+        proxy_type: ProxyType | int = ProxyType.RESIDENTIAL,
         traffic_limit: int = 0,
         status: bool = True,
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         """Create a new proxy user (sub-account)."""
         self._require_public_credentials()
         session = self._get_session()
@@ -1181,9 +1180,9 @@ class AsyncThordataClient:
     async def add_whitelist_ip(
         self,
         ip: str,
-        proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
+        proxy_type: ProxyType | int = ProxyType.RESIDENTIAL,
         status: bool = True,
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         """
         Add an IP to the whitelist for IP authentication.
         """
@@ -1237,7 +1236,7 @@ class AsyncThordataClient:
     async def list_proxy_servers(
         self,
         proxy_type: int,
-    ) -> List[ProxyServer]:
+    ) -> list[ProxyServer]:
         """
         List ISP or Datacenter proxy servers.
         """
@@ -1290,7 +1289,7 @@ class AsyncThordataClient:
                 f"List servers failed: {e}", original_error=e
             ) from e
 
-    async def get_isp_regions(self) -> List[Dict[str, Any]]:
+    async def get_isp_regions(self) -> list[dict[str, Any]]:
         """
         Get available ISP proxy regions.
 
@@ -1329,7 +1328,7 @@ class AsyncThordataClient:
                 f"Get ISP regions failed: {e}", original_error=e
             ) from e
 
-    async def list_isp_proxies(self) -> List[Dict[str, Any]]:
+    async def list_isp_proxies(self) -> list[dict[str, Any]]:
         """
         List ISP proxies.
 
@@ -1368,7 +1367,7 @@ class AsyncThordataClient:
                 f"List ISP proxies failed: {e}", original_error=e
             ) from e
 
-    async def get_wallet_balance(self) -> Dict[str, Any]:
+    async def get_wallet_balance(self) -> dict[str, Any]:
         """
         Get wallet balance for ISP proxies.
 
@@ -1409,9 +1408,9 @@ class AsyncThordataClient:
 
     async def get_proxy_expiration(
         self,
-        ips: Union[str, List[str]],
+        ips: str | list[str],
         proxy_type: int,
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         """
         Get expiration time for specific proxy IPs.
         """
@@ -1465,8 +1464,8 @@ class AsyncThordataClient:
     # =========================================================================
 
     async def list_countries(
-        self, proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL
-    ) -> List[Dict[str, Any]]:
+        self, proxy_type: ProxyType | int = ProxyType.RESIDENTIAL
+    ) -> list[dict[str, Any]]:
         """List supported countries."""
         return await self._get_locations(
             "countries",
@@ -1478,8 +1477,8 @@ class AsyncThordataClient:
     async def list_states(
         self,
         country_code: str,
-        proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
-    ) -> List[Dict[str, Any]]:
+        proxy_type: ProxyType | int = ProxyType.RESIDENTIAL,
+    ) -> list[dict[str, Any]]:
         """List supported states for a country."""
         return await self._get_locations(
             "states",
@@ -1492,9 +1491,9 @@ class AsyncThordataClient:
     async def list_cities(
         self,
         country_code: str,
-        state_code: Optional[str] = None,
-        proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
-    ) -> List[Dict[str, Any]]:
+        state_code: str | None = None,
+        proxy_type: ProxyType | int = ProxyType.RESIDENTIAL,
+    ) -> list[dict[str, Any]]:
         """List supported cities."""
         kwargs = {
             "proxy_type": (
@@ -1510,8 +1509,8 @@ class AsyncThordataClient:
     async def list_asn(
         self,
         country_code: str,
-        proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
-    ) -> List[Dict[str, Any]]:
+        proxy_type: ProxyType | int = ProxyType.RESIDENTIAL,
+    ) -> list[dict[str, Any]]:
         """List supported ASNs."""
         return await self._get_locations(
             "asn",
@@ -1523,7 +1522,7 @@ class AsyncThordataClient:
 
     async def _get_locations(
         self, endpoint: str, **kwargs: Any
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Internal async locations API call."""
         self._require_public_credentials()
 
@@ -1540,24 +1539,26 @@ class AsyncThordataClient:
         logger.debug(f"Async Locations API: {url}")
 
         # Create temporary session for this request (no proxy needed)
-        async with aiohttp.ClientSession(trust_env=True) as temp_session:
-            async with temp_session.get(url, params=params) as response:
-                response.raise_for_status()
-                data = await response.json()
-
-                if isinstance(data, dict):
-                    code = data.get("code")
-                    if code is not None and code != 200:
-                        msg = data.get("msg", "")
-                        raise RuntimeError(
-                            f"Locations API error ({endpoint}): code={code}, msg={msg}"
-                        )
-                    return data.get("data") or []
+        async with (
+            aiohttp.ClientSession(trust_env=True) as temp_session,
+            temp_session.get(url, params=params) as response,
+        ):
+            response.raise_for_status()
+            data = await response.json()
+
+            if isinstance(data, dict):
+                code = data.get("code")
+                if code is not None and code != 200:
+                    msg = data.get("msg", "")
+                    raise RuntimeError(
+                        f"Locations API error ({endpoint}): code={code}, msg={msg}"
+                    )
+                return data.get("data") or []
 
-                if isinstance(data, list):
-                    return data
+            if isinstance(data, list):
+                return data
 
-                return []
+            return []
 
     # =========================================================================
     # Helper Methods
@@ -1573,7 +1574,7 @@ class AsyncThordataClient:
 
     def _get_proxy_endpoint_overrides(
         self, product: ProxyProduct
-    ) -> tuple[Optional[str], Optional[int], str]:
+    ) -> tuple[str | None, int | None, str]:
         prefix = product.value.upper()
 
         host = os.getenv(f"THORDATA_{prefix}_PROXY_HOST") or os.getenv(
@@ -1588,7 +1589,7 @@ class AsyncThordataClient:
             or "http"
         )
 
-        port: Optional[int] = None
+        port: int | None = None
         if port_raw:
             try:
                 port = int(port_raw)
@@ -1597,7 +1598,7 @@ class AsyncThordataClient:
 
         return host or None, port, protocol
 
-    def _get_default_proxy_config_from_env(self) -> Optional[ProxyConfig]:
+    def _get_default_proxy_config_from_env(self) -> ProxyConfig | None:
         u = os.getenv("THORDATA_RESIDENTIAL_USERNAME")
         p = os.getenv("THORDATA_RESIDENTIAL_PASSWORD")
         if u and p:
@@ -1645,7 +1646,7 @@ class AsyncThordataClient:
 
         return None
 
-    def _build_gateway_headers(self) -> Dict[str, str]:
+    def _build_gateway_headers(self) -> dict[str, str]:
         """
         Headers for gateway-style endpoints.