thordata-sdk 0.8.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

thordata/__init__.py

@@ -35,7 +35,7 @@ Async Usage:
     >>> asyncio.run(main())
 """
 
-__version__ = "0.8.0"
+__version__ = "1.0.0"
 __author__ = "Thordata Developer Team"
 __email__ = "support@thordata.com"
 

thordata/_example_utils.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+try:
+    from dotenv import load_dotenv
+except Exception:  # pragma: no cover
+    load_dotenv = None
+
+
+def load_env() -> None:
+    """Load .env from repo root if python-dotenv is installed."""
+    if load_dotenv is None:
+        return
+    repo_root = Path(__file__).resolve().parents[2]
+    load_dotenv(dotenv_path=repo_root / ".env")
+
+
+def env(name: str) -> str:
+    return (os.getenv(name) or "").strip()
+
+
+def skip_if_missing(required: Iterable[str], *, tip: Optional[str] = None) -> bool:
+    missing = [k for k in required if not env(k)]
+    if not missing:
+        return False
+    print("Skipping live example: missing env:", ", ".join(missing))
+    if tip:
+        print(tip)
+    else:
+        print("Tip: copy .env.example to .env and fill values, then re-run.")
+    return True
+
+
+def parse_json_env(name: str, default: str = "{}") -> Any:
+    raw = env(name) or default
+    return json.loads(raw)
+
+
+def normalize_task_parameters(raw: Any) -> dict[str, Any]:
+    """Accept {..} or [{..}] and return a single dict for create_scraper_task(parameters=...)."""
+    if isinstance(raw, list):
+        if not raw:
+            raise ValueError("Task parameters JSON array must not be empty")
+        raw = raw[0]
+    if not isinstance(raw, dict):
+        raise ValueError("Task parameters must be a JSON object (or array of objects)")
+    return raw
+
+
+def output_dir() -> Path:
+    """Return output dir for examples; defaults to examples/output (ignored by git)."""
+    repo_root = Path(__file__).resolve().parents[2]
+    d = env("THORDATA_OUTPUT_DIR") or str(repo_root / "examples" / "output")
+    p = Path(d)
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def write_text(filename: str, content: str) -> Path:
+    p = output_dir() / filename
+    p.write_text(content, encoding="utf-8", errors="replace")
+    return p
+
+
+def write_json(filename: str, data: Any) -> Path:
+    p = output_dir() / filename
+    p.write_text(
+        json.dumps(data, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+        errors="replace",
+    )
+    return p

thordata/_utils.py

@@ -185,23 +185,3 @@ def build_user_agent(sdk_version: str, http_client: str) -> str:
     py = platform.python_version()
     system = platform.system()
     return f"thordata-python-sdk/{sdk_version} (python {py}; {system}; {http_client})"
-
-
-def build_sign_headers(sign: str, api_key: str) -> Dict[str, str]:
-    """
-    Build headers for Public API NEW (sign + apiKey authentication).
-
-    This is a different authentication system from token+key.
-
-    Args:
-        sign: The sign value from Dashboard (immutable).
-        api_key: The apiKey value from Dashboard (can be changed).
-
-    Returns:
-        Headers dict with sign, apiKey, and Content-Type.
-    """
-    return {
-        "sign": sign,
-        "apiKey": api_key,
-        "Content-Type": "application/x-www-form-urlencoded",
-    }

thordata/async_client.py

@@ -25,7 +25,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import os
-from datetime import date, datetime
+from datetime import date
 from typing import Any, Dict, List, Optional, Union
 
 import aiohttp
@@ -35,7 +35,6 @@ from ._utils import (
     build_auth_headers,
     build_builder_headers,
     build_public_api_headers,
-    build_sign_headers,
     build_user_agent,
     decode_base64_image,
     extract_error_message,
@@ -51,14 +50,15 @@ from .exceptions import (
 from .models import (
     CommonSettings,
     ProxyConfig,
+    ProxyProduct,
     ProxyServer,
-    ProxyUser,
     ProxyUserList,
     ScraperTaskConfig,
     SerpRequest,
     UniversalScrapeRequest,
     UsageStatistics,
     VideoTaskConfig,
+    WhitelistProxyConfig,
 )
 from .retry import RetryConfig
 
@@ -100,8 +100,6 @@ class AsyncThordataClient:
         scraper_token: str,
         public_token: Optional[str] = None,
         public_key: Optional[str] = None,
-        sign: Optional[str] = None,
-        api_key: Optional[str] = None,
         proxy_host: str = "pr.thordata.net",
         proxy_port: int = 9999,
         timeout: int = 30,
@@ -121,18 +119,9 @@ class AsyncThordataClient:
         self.public_token = public_token
         self.public_key = public_key
 
-        # Automatic Fallback Logic: If sign/api_key is not provided, try using public_token/key
-        self.sign = sign or os.getenv("THORDATA_SIGN") or self.public_token
-        self.api_key = api_key or os.getenv("THORDATA_API_KEY") or self.public_key
-
-        # Public API authentication
-        self.sign = sign or os.getenv("THORDATA_SIGN")
-        self.api_key = api_key or os.getenv("THORDATA_API_KEY")
-
         # Proxy configuration
         self._proxy_host = proxy_host
         self._proxy_port = proxy_port
-        self._default_timeout = aiohttp.ClientTimeout(total=timeout)
 
         # Timeout configuration
         self._default_timeout = aiohttp.ClientTimeout(total=timeout)
@@ -141,19 +130,13 @@ class AsyncThordataClient:
         # Retry configuration
         self._retry_config = retry_config or RetryConfig()
 
-        # Authentication mode
+        # Authentication mode (for scraping APIs)
         self._auth_mode = auth_mode.lower()
         if self._auth_mode not in ("bearer", "header_token"):
             raise ThordataConfigError(
                 f"Invalid auth_mode: {auth_mode}. Must be 'bearer' or 'header_token'."
             )
 
-        # Pre-calculate proxy auth
-        self._proxy_url = f"http://{proxy_host}:{proxy_port}"
-        self._proxy_auth = aiohttp.BasicAuth(
-            login=f"td-customer-{scraper_token}", password=""
-        )
-
         # Base URLs (allow override via args or env vars for testing and custom routing)
         scraperapi_base = (
             scraperapi_base_url
@@ -179,6 +162,7 @@ class AsyncThordataClient:
             or self.LOCATIONS_URL
         ).rstrip("/")
 
+        # Keep these env overrides for now
         gateway_base = os.getenv(
             "THORDATA_GATEWAY_BASE_URL", "https://api.thordata.com/api/gateway"
         )
@@ -193,8 +177,11 @@ class AsyncThordataClient:
         self._builder_url = f"{scraperapi_base}/builder"
         self._video_builder_url = f"{scraperapi_base}/video_builder"
         self._universal_url = f"{universalapi_base}/request"
+
         self._status_url = f"{web_scraper_api_base}/tasks-status"
         self._download_url = f"{web_scraper_api_base}/tasks-download"
+        self._list_url = f"{web_scraper_api_base}/tasks-list"
+
         self._locations_base_url = locations_base
         self._usage_stats_url = (
             f"{locations_base.replace('/locations', '')}/account/usage-statistics"
@@ -202,16 +189,17 @@ class AsyncThordataClient:
         self._proxy_users_url = (
             f"{locations_base.replace('/locations', '')}/proxy-users"
         )
+
         whitelist_base = os.getenv(
             "THORDATA_WHITELIST_BASE_URL", "https://api.thordata.com/api"
         )
         self._whitelist_url = f"{whitelist_base}/whitelisted-ips"
+
         proxy_api_base = os.getenv(
             "THORDATA_PROXY_API_BASE_URL", "https://api.thordata.com/api"
         )
         self._proxy_list_url = f"{proxy_api_base}/proxy/proxy-list"
         self._proxy_expiration_url = f"{proxy_api_base}/proxy/expiration-time"
-        self._list_url = f"{web_scraper_api_base}/tasks-list"
 
         # Session initialized lazily
         self._session: Optional[aiohttp.ClientSession] = None
@@ -271,11 +259,26 @@ class AsyncThordataClient:
 
         logger.debug(f"Async Proxy GET: {url}")
 
-        if proxy_config:
-            proxy_url, proxy_auth = proxy_config.to_aiohttp_config()
-        else:
-            proxy_url = self._proxy_url
-            proxy_auth = self._proxy_auth
+        if proxy_config is None:
+            proxy_config = self._get_default_proxy_config_from_env()
+
+        if proxy_config is None:
+            raise ThordataConfigError(
+                "Proxy credentials are missing. "
+                "Pass proxy_config=ProxyConfig(username=..., password=..., product=...) "
+                "or set THORDATA_RESIDENTIAL_USERNAME/THORDATA_RESIDENTIAL_PASSWORD (or DATACENTER/MOBILE)."
+            )
+
+        # aiohttp has limited support for "https://" proxies (TLS to proxy / TLS-in-TLS).
+        # Your account's proxy endpoint requires HTTPS proxy, so we explicitly block here
+        # to avoid confusing "it always fails" behavior.
+        if getattr(proxy_config, "protocol", "http").lower() == "https":
+            raise ThordataConfigError(
+                "Proxy Network requires an HTTPS proxy endpoint (TLS to proxy) for your account. "
+                "aiohttp support for 'https://' proxies is limited and may fail. "
+                "Please use ThordataClient.get/post (sync client) for Proxy Network requests."
+            )
+        proxy_url, proxy_auth = proxy_config.to_aiohttp_config()
 
         try:
             return await session.get(
@@ -312,11 +315,26 @@ class AsyncThordataClient:
 
         logger.debug(f"Async Proxy POST: {url}")
 
-        if proxy_config:
-            proxy_url, proxy_auth = proxy_config.to_aiohttp_config()
-        else:
-            proxy_url = self._proxy_url
-            proxy_auth = self._proxy_auth
+        if proxy_config is None:
+            proxy_config = self._get_default_proxy_config_from_env()
+
+        if proxy_config is None:
+            raise ThordataConfigError(
+                "Proxy credentials are missing. "
+                "Pass proxy_config=ProxyConfig(username=..., password=..., product=...) "
+                "or set THORDATA_RESIDENTIAL_USERNAME/THORDATA_RESIDENTIAL_PASSWORD (or DATACENTER/MOBILE)."
+            )
+
+        # aiohttp has limited support for "https://" proxies (TLS to proxy / TLS-in-TLS).
+        # Your account's proxy endpoint requires HTTPS proxy, so we explicitly block here
+        # to avoid confusing "it always fails" behavior.
+        if getattr(proxy_config, "protocol", "http").lower() == "https":
+            raise ThordataConfigError(
+                "Proxy Network requires an HTTPS proxy endpoint (TLS to proxy) for your account. "
+                "aiohttp support for 'https://' proxies is limited and may fail. "
+                "Please use ThordataClient.get/post (sync client) for Proxy Network requests."
+            )
+        proxy_url, proxy_auth = proxy_config.to_aiohttp_config()
 
         try:
             return await session.post(
@@ -973,23 +991,14 @@ class AsyncThordataClient:
 
     async def get_residential_balance(self) -> Dict[str, Any]:
         """
-        Get residential proxy balance (Public API NEW).
-
-        Requires sign and apiKey credentials.
+        Get residential proxy balance.
 
-        Returns:
-            Dict with 'balance' (bytes) and 'expire_time' (timestamp).
+        Uses public_token/public_key.
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW. "
-                "Set THORDATA_SIGN and THORDATA_API_KEY environment variables."
-            )
-
         session = self._get_session()
-        headers = build_sign_headers(self.sign, self.api_key)
+        headers = self._build_gateway_headers()
 
-        logger.info("Async getting residential proxy balance (API NEW)")
+        logger.info("Async getting residential proxy balance")
 
         try:
             async with session.post(
@@ -1025,26 +1034,13 @@ class AsyncThordataClient:
         end_time: Union[str, int],
     ) -> Dict[str, Any]:
         """
-        Get residential proxy usage records (Public API NEW).
-
-        Args:
-            start_time: Start timestamp (Unix timestamp or YYYY-MM-DD HH:MM:SS).
-            end_time: End timestamp (Unix timestamp or YYYY-MM-DD HH:MM:SS).
+        Get residential proxy usage records.
 
-        Returns:
-            Dict with usage data including 'all_flow', 'all_used_flow', 'data' list.
+        Uses public_token/public_key.
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW."
-            )
-
         session = self._get_session()
-        headers = build_sign_headers(self.sign, self.api_key)
-        payload = {
-            "start_time": str(start_time),
-            "end_time": str(end_time),
-        }
+        headers = self._build_gateway_headers()
+        payload = {"start_time": str(start_time), "end_time": str(end_time)}
 
         logger.info(f"Async getting residential usage: {start_time} to {end_time}")
 
@@ -1296,20 +1292,14 @@ class AsyncThordataClient:
 
     async def get_isp_regions(self) -> List[Dict[str, Any]]:
         """
-        Get available ISP proxy regions (Public API NEW).
+        Get available ISP proxy regions.
 
-        Returns:
-            List of regions with id, continent, country, city, num, pricing.
+        Uses public_token/public_key.
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW."
-            )
-
         session = self._get_session()
-        headers = build_sign_headers(self.sign, self.api_key)
+        headers = self._build_gateway_headers()
 
-        logger.info("Async getting ISP regions (API NEW)")
+        logger.info("Async getting ISP regions")
 
         try:
             async with session.post(
@@ -1341,20 +1331,14 @@ class AsyncThordataClient:
 
     async def list_isp_proxies(self) -> List[Dict[str, Any]]:
         """
-        List ISP proxies (Public API NEW).
+        List ISP proxies.
 
-        Returns:
-            List of ISP proxies with ip, port, user, pwd, startTime, expireTime.
+        Uses public_token/public_key.
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW."
-            )
-
         session = self._get_session()
-        headers = build_sign_headers(self.sign, self.api_key)
+        headers = self._build_gateway_headers()
 
-        logger.info("Async listing ISP proxies (API NEW)")
+        logger.info("Async listing ISP proxies")
 
         try:
             async with session.post(
@@ -1386,20 +1370,14 @@ class AsyncThordataClient:
 
     async def get_wallet_balance(self) -> Dict[str, Any]:
         """
-        Get wallet balance for ISP proxies (Public API NEW).
+        Get wallet balance for ISP proxies.
 
-        Returns:
-            Dict with 'walletBalance'.
+        Uses public_token/public_key.
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW."
-            )
-
         session = self._get_session()
-        headers = build_sign_headers(self.sign, self.api_key)
+        headers = self._build_gateway_headers()
 
-        logger.info("Async getting wallet balance (API NEW)")
+        logger.info("Async getting wallet balance")
 
         try:
             async with session.post(
@@ -1592,3 +1570,86 @@ class AsyncThordataClient:
                 "public_token and public_key are required for this operation. "
                 "Please provide them when initializing AsyncThordataClient."
             )
+
+    def _get_proxy_endpoint_overrides(
+        self, product: ProxyProduct
+    ) -> tuple[Optional[str], Optional[int], str]:
+        prefix = product.value.upper()
+
+        host = os.getenv(f"THORDATA_{prefix}_PROXY_HOST") or os.getenv(
+            "THORDATA_PROXY_HOST"
+        )
+        port_raw = os.getenv(f"THORDATA_{prefix}_PROXY_PORT") or os.getenv(
+            "THORDATA_PROXY_PORT"
+        )
+        protocol = (
+            os.getenv(f"THORDATA_{prefix}_PROXY_PROTOCOL")
+            or os.getenv("THORDATA_PROXY_PROTOCOL")
+            or "http"
+        )
+
+        port: Optional[int] = None
+        if port_raw:
+            try:
+                port = int(port_raw)
+            except ValueError:
+                port = None
+
+        return host or None, port, protocol
+
+    def _get_default_proxy_config_from_env(self) -> Optional[ProxyConfig]:
+        u = os.getenv("THORDATA_RESIDENTIAL_USERNAME")
+        p = os.getenv("THORDATA_RESIDENTIAL_PASSWORD")
+        if u and p:
+            host, port, protocol = self._get_proxy_endpoint_overrides(
+                ProxyProduct.RESIDENTIAL
+            )
+            return ProxyConfig(
+                username=u,
+                password=p,
+                product=ProxyProduct.RESIDENTIAL,
+                host=host,
+                port=port,
+                protocol=protocol,
+            )
+
+        u = os.getenv("THORDATA_DATACENTER_USERNAME")
+        p = os.getenv("THORDATA_DATACENTER_PASSWORD")
+        if u and p:
+            host, port, protocol = self._get_proxy_endpoint_overrides(
+                ProxyProduct.DATACENTER
+            )
+            return ProxyConfig(
+                username=u,
+                password=p,
+                product=ProxyProduct.DATACENTER,
+                host=host,
+                port=port,
+                protocol=protocol,
+            )
+
+        u = os.getenv("THORDATA_MOBILE_USERNAME")
+        p = os.getenv("THORDATA_MOBILE_PASSWORD")
+        if u and p:
+            host, port, protocol = self._get_proxy_endpoint_overrides(
+                ProxyProduct.MOBILE
+            )
+            return ProxyConfig(
+                username=u,
+                password=p,
+                product=ProxyProduct.MOBILE,
+                host=host,
+                port=port,
+                protocol=protocol,
+            )
+
+        return None
+
+    def _build_gateway_headers(self) -> Dict[str, str]:
+        """
+        Headers for gateway-style endpoints.
+
+        Per our SDK rule: ONLY public_token/public_key exist.
+        """
+        self._require_public_credentials()
+        return build_public_api_headers(self.public_token or "", self.public_key or "")

thordata/client.py

@@ -25,17 +25,19 @@ from __future__ import annotations
 
 import logging
 import os
-from datetime import date, datetime
+import ssl
+from datetime import date
 from typing import Any, Dict, List, Optional, Union
+from urllib.parse import urlencode
 
 import requests
+import urllib3
 
 from . import __version__ as _sdk_version
 from ._utils import (
     build_auth_headers,
     build_builder_headers,
     build_public_api_headers,
-    build_sign_headers,
     build_user_agent,
     decode_base64_image,
     extract_error_message,
@@ -53,13 +55,13 @@ from .models import (
     ProxyConfig,
     ProxyProduct,
     ProxyServer,
-    ProxyUser,
     ProxyUserList,
     ScraperTaskConfig,
     SerpRequest,
     UniversalScrapeRequest,
     UsageStatistics,
     VideoTaskConfig,
+    WhitelistProxyConfig,
 )
 from .retry import RetryConfig, with_retry
 
@@ -104,8 +106,6 @@ class ThordataClient:
         scraper_token: str,
         public_token: Optional[str] = None,
         public_key: Optional[str] = None,
-        sign: Optional[str] = None,
-        api_key: Optional[str] = None,
         proxy_host: str = "pr.thordata.net",
         proxy_port: int = 9999,
         timeout: int = 30,
@@ -121,22 +121,14 @@ class ThordataClient:
         if not scraper_token:
             raise ThordataConfigError("scraper_token is required")
 
+        # Core credentials
         self.scraper_token = scraper_token
         self.public_token = public_token
         self.public_key = public_key
 
-        # Automatic Fallback Logic: If sign/api_key is not provided, try using public_token/key
-        self.sign = sign or os.getenv("THORDATA_SIGN") or self.public_token
-        self.api_key = api_key or os.getenv("THORDATA_API_KEY") or self.public_key
-
-        # Public API authentication
-        self.sign = sign or os.getenv("THORDATA_SIGN")
-        self.api_key = api_key or os.getenv("THORDATA_API_KEY")
-
         # Proxy configuration
         self._proxy_host = proxy_host
         self._proxy_port = proxy_port
-        self._default_timeout = timeout
 
         # Timeout configuration
         self._default_timeout = timeout
@@ -145,19 +137,14 @@ class ThordataClient:
         # Retry configuration
         self._retry_config = retry_config or RetryConfig()
 
-        # Authentication mode
+        # Authentication mode (for scraping APIs)
         self._auth_mode = auth_mode.lower()
         if self._auth_mode not in ("bearer", "header_token"):
             raise ThordataConfigError(
                 f"Invalid auth_mode: {auth_mode}. Must be 'bearer' or 'header_token'."
             )
 
-        # Build default proxy URL (for basic usage)
-        self._default_proxy_url = (
-            f"http://td-customer-{self.scraper_token}:@{proxy_host}:{proxy_port}"
-        )
-
-        # Sessions:
+        # NOTE:
         # - _proxy_session: used for proxy network traffic to target sites
         # - _api_session: used for Thordata APIs (SERP/Universal/Tasks/Locations)
         #
@@ -165,14 +152,9 @@ class ThordataClient:
         # so developers can rely on system proxy settings (e.g., Clash) via env vars.
         self._proxy_session = requests.Session()
         self._proxy_session.trust_env = False
-        self._proxy_session.proxies = {
-            "http": self._default_proxy_url,
-            "https": self._default_proxy_url,
-        }
 
         self._api_session = requests.Session()
         self._api_session.trust_env = True
-
         self._api_session.headers.update(
             {"User-Agent": build_user_agent(_sdk_version, "requests")}
         )
@@ -202,13 +184,13 @@ class ThordataClient:
             or self.LOCATIONS_URL
         ).rstrip("/")
 
+        # These URLs exist in your codebase; keep them for now (even if your org later migrates fully to openapi)
         gateway_base = os.getenv(
             "THORDATA_GATEWAY_BASE_URL", "https://api.thordata.com/api/gateway"
         )
         child_base = os.getenv(
             "THORDATA_CHILD_BASE_URL", "https://api.thordata.com/api/child"
         )
-
         self._gateway_base_url = gateway_base
         self._child_base_url = child_base
 
@@ -216,25 +198,31 @@ class ThordataClient:
         self._builder_url = f"{scraperapi_base}/builder"
         self._video_builder_url = f"{scraperapi_base}/video_builder"
         self._universal_url = f"{universalapi_base}/request"
+
         self._status_url = f"{web_scraper_api_base}/tasks-status"
         self._download_url = f"{web_scraper_api_base}/tasks-download"
+        self._list_url = f"{web_scraper_api_base}/tasks-list"
+
         self._locations_base_url = locations_base
+
+        # These 2 lines keep your existing behavior (derive account endpoints from locations_base)
         self._usage_stats_url = (
             f"{locations_base.replace('/locations', '')}/account/usage-statistics"
         )
         self._proxy_users_url = (
             f"{locations_base.replace('/locations', '')}/proxy-users"
         )
+
         whitelist_base = os.getenv(
             "THORDATA_WHITELIST_BASE_URL", "https://api.thordata.com/api"
         )
         self._whitelist_url = f"{whitelist_base}/whitelisted-ips"
+
         proxy_api_base = os.getenv(
             "THORDATA_PROXY_API_BASE_URL", "https://api.thordata.com/api"
         )
         self._proxy_list_url = f"{proxy_api_base}/proxy/proxy-list"
         self._proxy_expiration_url = f"{proxy_api_base}/proxy/expiration-time"
-        self._list_url = f"{web_scraper_api_base}/tasks-list"
 
     # =========================================================================
     # Proxy Network Methods (Pure proxy network request functions)
@@ -277,11 +265,37 @@ class ThordataClient:
 
         timeout = timeout or self._default_timeout
 
-        if proxy_config:
-            proxies = proxy_config.to_proxies_dict()
-            kwargs["proxies"] = proxies
+        if proxy_config is None:
+            proxy_config = self._get_default_proxy_config_from_env()
 
-        return self._request_with_retry("GET", url, timeout=timeout, **kwargs)
+        if proxy_config is None:
+            raise ThordataConfigError(
+                "Proxy credentials are missing. "
+                "Pass proxy_config=ProxyConfig(username=..., password=..., product=...) "
+                "or set THORDATA_RESIDENTIAL_USERNAME/THORDATA_RESIDENTIAL_PASSWORD (or DATACENTER/MOBILE)."
+            )
+
+        kwargs["proxies"] = proxy_config.to_proxies_dict()
+
+        @with_retry(self._retry_config)
+        def _do() -> requests.Response:
+            return self._proxy_request_with_proxy_manager(
+                "GET",
+                url,
+                proxy_config=proxy_config,
+                timeout=timeout,
+                headers=kwargs.pop("headers", None),
+                params=kwargs.pop("params", None),
+            )
+
+        try:
+            return _do()
+        except requests.Timeout as e:
+            raise ThordataTimeoutError(
+                f"Request timed out: {e}", original_error=e
+            ) from e
+        except Exception as e:
+            raise ThordataNetworkError(f"Request failed: {e}", original_error=e) from e
 
     def post(
         self,
@@ -307,11 +321,38 @@ class ThordataClient:
 
         timeout = timeout or self._default_timeout
 
-        if proxy_config:
-            proxies = proxy_config.to_proxies_dict()
-            kwargs["proxies"] = proxies
+        if proxy_config is None:
+            proxy_config = self._get_default_proxy_config_from_env()
 
-        return self._request_with_retry("POST", url, timeout=timeout, **kwargs)
+        if proxy_config is None:
+            raise ThordataConfigError(
+                "Proxy credentials are missing. "
+                "Pass proxy_config=ProxyConfig(username=..., password=..., product=...) "
+                "or set THORDATA_RESIDENTIAL_USERNAME/THORDATA_RESIDENTIAL_PASSWORD (or DATACENTER/MOBILE)."
+            )
+
+        kwargs["proxies"] = proxy_config.to_proxies_dict()
+
+        @with_retry(self._retry_config)
+        def _do() -> requests.Response:
+            return self._proxy_request_with_proxy_manager(
+                "POST",
+                url,
+                proxy_config=proxy_config,
+                timeout=timeout,
+                headers=kwargs.pop("headers", None),
+                params=kwargs.pop("params", None),
+                data=kwargs.pop("data", None),
+            )
+
+        try:
+            return _do()
+        except requests.Timeout as e:
+            raise ThordataTimeoutError(
+                f"Request timed out: {e}", original_error=e
+            ) from e
+        except Exception as e:
+            raise ThordataNetworkError(f"Request failed: {e}", original_error=e) from e
 
     def build_proxy_url(
         self,
@@ -1185,27 +1226,13 @@ class ThordataClient:
 
     def get_residential_balance(self) -> Dict[str, Any]:
         """
-        Get residential proxy balance (Public API NEW).
-
-        Requires sign and apiKey credentials.
+        Get residential proxy balance.
 
-        Returns:
-            Dict with 'balance' (bytes) and 'expire_time' (timestamp).
-
-        Example:
-            >>> result = client.get_residential_balance()
-            >>> balance_gb = result['balance'] / (1024**3)
-            >>> print(f"Balance: {balance_gb:.2f} GB")
+        Uses public_token/public_key (Dashboard -> My account -> API).
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW. "
-                "Set THORDATA_SIGN and THORDATA_API_KEY environment variables."
-            )
-
-        headers = build_sign_headers(self.sign, self.api_key)
+        headers = self._build_gateway_headers()
 
-        logger.info("Getting residential proxy balance (API NEW)")
+        logger.info("Getting residential proxy balance")
 
         response = self._api_request_with_retry(
             "POST",
@@ -1230,32 +1257,12 @@ class ThordataClient:
         end_time: Union[str, int],
     ) -> Dict[str, Any]:
         """
-        Get residential proxy usage records (Public API NEW).
-
-        Args:
-            start_time: Start timestamp (Unix timestamp or YYYY-MM-DD HH:MM:SS).
-            end_time: End timestamp (Unix timestamp or YYYY-MM-DD HH:MM:SS).
+        Get residential proxy usage records.
 
-        Returns:
-            Dict with usage data including 'all_flow', 'all_used_flow', 'data' list.
-
-        Example:
-            >>> import time
-            >>> end = int(time.time())
-            >>> start = end - 7*24*3600  # Last 7 days
-            >>> usage = client.get_residential_usage(start, end)
-            >>> print(f"Total used: {usage['all_used_flow'] / (1024**3):.2f} GB")
+        Uses public_token/public_key (Dashboard -> My account -> API).
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW."
-            )
-
-        headers = build_sign_headers(self.sign, self.api_key)
-        payload = {
-            "start_time": str(start_time),
-            "end_time": str(end_time),
-        }
+        headers = self._build_gateway_headers()
+        payload = {"start_time": str(start_time), "end_time": str(end_time)}
 
         logger.info(f"Getting residential usage: {start_time} to {end_time}")
 
@@ -1516,24 +1523,13 @@ class ThordataClient:
 
     def get_isp_regions(self) -> List[Dict[str, Any]]:
         """
-        Get available ISP proxy regions (Public API NEW).
+        Get available ISP proxy regions.
 
-        Returns:
-            List of regions with id, continent, country, city, num, pricing.
-
-        Example:
-            >>> regions = client.get_isp_regions()
-            >>> for region in regions:
-            ...     print(f"{region['country']}/{region['city']}: {region['num']} IPs")
+        Uses public_token/public_key (Dashboard -> My account -> API).
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW."
-            )
-
-        headers = build_sign_headers(self.sign, self.api_key)
+        headers = self._build_gateway_headers()
 
-        logger.info("Getting ISP regions (API NEW)")
+        logger.info("Getting ISP regions")
 
         response = self._api_request_with_retry(
             "POST",
@@ -1554,24 +1550,13 @@ class ThordataClient:
 
     def list_isp_proxies(self) -> List[Dict[str, Any]]:
         """
-        List ISP proxies (Public API NEW).
-
-        Returns:
-            List of ISP proxies with ip, port, user, pwd, startTime, expireTime.
+        List ISP proxies.
 
-        Example:
-            >>> proxies = client.list_isp_proxies()
-            >>> for proxy in proxies:
-            ...     print(f"{proxy['ip']}:{proxy['port']} - expires: {proxy['expireTime']}")
+        Uses public_token/public_key (Dashboard -> My account -> API).
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW."
-            )
-
-        headers = build_sign_headers(self.sign, self.api_key)
+        headers = self._build_gateway_headers()
 
-        logger.info("Listing ISP proxies (API NEW)")
+        logger.info("Listing ISP proxies")
 
         response = self._api_request_with_retry(
             "POST",
@@ -1592,23 +1577,13 @@ class ThordataClient:
 
     def get_wallet_balance(self) -> Dict[str, Any]:
         """
-        Get wallet balance for ISP proxies (Public API NEW).
+        Get wallet balance for ISP proxies.
 
-        Returns:
-            Dict with 'walletBalance'.
-
-        Example:
-            >>> result = client.get_wallet_balance()
-            >>> print(f"Wallet: ${result['walletBalance']}")
+        Uses public_token/public_key (Dashboard -> My account -> API).
         """
-        if not self.sign or not self.api_key:
-            raise ThordataConfigError(
-                "sign and api_key are required for Public API NEW."
-            )
-
-        headers = build_sign_headers(self.sign, self.api_key)
+        headers = self._build_gateway_headers()
 
-        logger.info("Getting wallet balance (API NEW)")
+        logger.info("Getting wallet balance")
 
         response = self._api_request_with_retry(
             "POST",
@@ -1827,6 +1802,185 @@ class ThordataClient:
                 "Please provide them when initializing ThordataClient."
             )
 
+    def _get_proxy_endpoint_overrides(
+        self, product: ProxyProduct
+    ) -> tuple[Optional[str], Optional[int], str]:
+        """
+        Read proxy endpoint overrides from env.
+
+        Priority:
+        1) THORDATA_<PRODUCT>_PROXY_HOST/PORT/PROTOCOL
+        2) THORDATA_PROXY_HOST/PORT/PROTOCOL
+        3) defaults (host/port None => ProxyConfig will use its product defaults)
+        """
+        prefix = product.value.upper()  # RESIDENTIAL / DATACENTER / MOBILE / ISP
+
+        host = os.getenv(f"THORDATA_{prefix}_PROXY_HOST") or os.getenv(
+            "THORDATA_PROXY_HOST"
+        )
+        port_raw = os.getenv(f"THORDATA_{prefix}_PROXY_PORT") or os.getenv(
+            "THORDATA_PROXY_PORT"
+        )
+        protocol = (
+            os.getenv(f"THORDATA_{prefix}_PROXY_PROTOCOL")
+            or os.getenv("THORDATA_PROXY_PROTOCOL")
+            or "http"
+        )
+
+        port: Optional[int] = None
+        if port_raw:
+            try:
+                port = int(port_raw)
+            except ValueError:
+                port = None
+
+        return host or None, port, protocol
+
+    def _get_default_proxy_config_from_env(self) -> Optional[ProxyConfig]:
+        """
+        Try to build a default ProxyConfig from env vars.
+
+        Priority order:
+        1) Residential
+        2) Datacenter
+        3) Mobile
+        """
+        # Residential
+        u = os.getenv("THORDATA_RESIDENTIAL_USERNAME")
+        p = os.getenv("THORDATA_RESIDENTIAL_PASSWORD")
+        if u and p:
+            host, port, protocol = self._get_proxy_endpoint_overrides(
+                ProxyProduct.RESIDENTIAL
+            )
+            return ProxyConfig(
+                username=u,
+                password=p,
+                product=ProxyProduct.RESIDENTIAL,
+                host=host,
+                port=port,
+                protocol=protocol,
+            )
+
+        # Datacenter
+        u = os.getenv("THORDATA_DATACENTER_USERNAME")
+        p = os.getenv("THORDATA_DATACENTER_PASSWORD")
+        if u and p:
+            host, port, protocol = self._get_proxy_endpoint_overrides(
+                ProxyProduct.DATACENTER
+            )
+            return ProxyConfig(
+                username=u,
+                password=p,
+                product=ProxyProduct.DATACENTER,
+                host=host,
+                port=port,
+                protocol=protocol,
+            )
+
+        # Mobile
+        u = os.getenv("THORDATA_MOBILE_USERNAME")
+        p = os.getenv("THORDATA_MOBILE_PASSWORD")
+        if u and p:
+            host, port, protocol = self._get_proxy_endpoint_overrides(
+                ProxyProduct.MOBILE
+            )
+            return ProxyConfig(
+                username=u,
+                password=p,
+                product=ProxyProduct.MOBILE,
+                host=host,
+                port=port,
+                protocol=protocol,
+            )
+
+        return None
+
+    def _build_gateway_headers(self) -> Dict[str, str]:
+        """
+        Build headers for legacy gateway-style endpoints.
+
+        IMPORTANT:
+        - SDK does NOT expose "sign/apiKey" as a separate credential model.
+        - Values ALWAYS come from public_token/public_key.
+        - Some backend endpoints may still expect header field names "sign" and "apiKey".
+        """
+        self._require_public_credentials()
+        return {
+            "sign": self.public_token or "",
+            "apiKey": self.public_key or "",
+            "Content-Type": "application/x-www-form-urlencoded",
+        }
+
+    def _proxy_request_with_proxy_manager(
+        self,
+        method: str,
+        url: str,
+        *,
+        proxy_config: ProxyConfig,
+        timeout: int,
+        headers: Optional[Dict[str, str]] = None,
+        params: Optional[Dict[str, Any]] = None,
+        data: Any = None,
+    ) -> requests.Response:
+        """
+        Proxy Network request implemented via urllib3.ProxyManager.
+
+        This is required to reliably support HTTPS proxy endpoints like:
+        https://<endpoint>.pr.thordata.net:9999
+        """
+        # Build final URL (include query params)
+        req = requests.Request(method=method.upper(), url=url, params=params)
+        prepped = self._proxy_session.prepare_request(req)
+        final_url = prepped.url or url
+
+        proxy_url = proxy_config.build_proxy_endpoint()
+        proxy_headers = urllib3.make_headers(
+            proxy_basic_auth=proxy_config.build_proxy_basic_auth()
+        )
+
+        pm = urllib3.ProxyManager(
+            proxy_url,
+            proxy_headers=proxy_headers,
+            proxy_ssl_context=(
+                ssl.create_default_context()
+                if proxy_url.startswith("https://")
+                else None
+            ),
+        )
+
+        # Encode form data if dict
+        body = None
+        req_headers = dict(headers or {})
+        if data is not None:
+            if isinstance(data, dict):
+                # form-urlencoded
+                body = urlencode({k: str(v) for k, v in data.items()})
+                req_headers.setdefault(
+                    "Content-Type", "application/x-www-form-urlencoded"
+                )
+            else:
+                body = data
+
+        http_resp = pm.request(
+            method.upper(),
+            final_url,
+            body=body,
+            headers=req_headers or None,
+            timeout=urllib3.Timeout(connect=timeout, read=timeout),
+            retries=False,
+            preload_content=True,
+        )
+
+        # Convert urllib3 response -> requests.Response (keep your API stable)
+        r = requests.Response()
+        r.status_code = int(getattr(http_resp, "status", 0) or 0)
+        r._content = http_resp.data or b""
+        r.url = final_url
+        r.headers = requests.structures.CaseInsensitiveDict(
+            dict(http_resp.headers or {})
+        )
+        return r
+
     def _request_with_retry(
         self, method: str, url: str, **kwargs: Any
     ) -> requests.Response:

thordata/models.py

@@ -26,11 +26,14 @@ from __future__ import annotations
 
 import json
 import re
+import ssl
 import uuid
 from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
+import urllib3
+
 # =============================================================================
 # Proxy Product Types
 # =============================================================================
@@ -137,6 +140,7 @@ class ProxyConfig:
         if self.host is None:
             # Set host based on product type
             host_map = {
+                # User&Pass auth entry (docs examples use t.pr.thordata.net for authenticated proxy)
                 ProxyProduct.RESIDENTIAL: "t.pr.thordata.net",
                 ProxyProduct.DATACENTER: "dc.pr.thordata.net",
                 ProxyProduct.MOBILE: "m.pr.thordata.net",
@@ -233,6 +237,14 @@ class ProxyConfig:
         username = self.build_username()
         return f"{self.protocol}://{username}:{self.password}@{self.host}:{self.port}"
 
+    def build_proxy_endpoint(self) -> str:
+        """Proxy endpoint without credentials, for HTTPS proxy managers."""
+        return f"{self.protocol}://{self.host}:{self.port}"
+
+    def build_proxy_basic_auth(self) -> str:
+        """Basic auth string 'username:password' for Proxy-Authorization."""
+        return f"{self.build_username()}:{self.password}"
+
     def to_proxies_dict(self) -> Dict[str, str]:
         """
         Build a proxies dict suitable for the requests library.
@@ -264,6 +276,39 @@ class ProxyConfig:
             ) from e
 
 
+@dataclass
+class WhitelistProxyConfig:
+    """
+    Proxy config for IP-whitelist authentication mode (no username/password).
+
+    In whitelist mode, you do NOT pass proxy auth.
+    You only connect to the proxy entry node (host:port).
+
+    Examples (from docs):
+      - Global random: pr.thordata.net:9999
+      - Country nodes: us-pr.thordata.net:10000, etc.
+    """
+
+    host: str = "pr.thordata.net"
+    port: int = 9999
+    protocol: str = "http"  # use http for proxy scheme; target URL can still be https
+
+    def __post_init__(self) -> None:
+        if self.protocol not in ("http", "https"):
+            raise ValueError("protocol must be 'http' or 'https'")
+
+    def build_proxy_url(self) -> str:
+        return f"{self.protocol}://{self.host}:{self.port}"
+
+    def to_proxies_dict(self) -> Dict[str, str]:
+        url = self.build_proxy_url()
+        return {"http": url, "https": url}
+
+    def to_aiohttp_config(self) -> tuple:
+        # aiohttp: proxy_auth should be None in whitelist mode
+        return self.build_proxy_url(), None
+
+
 @dataclass
 class StaticISPProxy:
     """
@@ -545,23 +590,28 @@ class SerpRequest:
         payload: Dict[str, Any] = {
             "engine": engine,
             "num": str(self.num),
-            # output_format: json=1 for JSON, json=0 for raw HTML
-            "json": "1" if self.output_format.lower() == "json" else "0",
         }
 
+        fmt = self.output_format.lower()
+        if fmt == "json":
+            payload["json"] = "1"
+        elif fmt == "html":
+            # omit "json" to get raw HTML (per docs: no json -> HTML)
+            pass
+        else:
+            # keep backward compatibility: if user passes "2"/"both"/etc.
+            if fmt in ("2", "both", "json+html", "json_html"):
+                payload["json"] = "2"
+
         # Handle query parameter (Yandex uses 'text', others use 'q')
         if engine == "yandex":
             payload["text"] = self.query
         else:
             payload["q"] = self.query
 
-        # Set URL / domain based on google_domain or engine default
+        # Domain overrides (preferred by docs)
         if self.google_domain:
-            # 显式设置 google_domain 参数，同时设置 url
             payload["google_domain"] = self.google_domain
-            payload["url"] = self.google_domain
-        elif engine in self.ENGINE_URLS:
-            payload["url"] = self.ENGINE_URLS[engine]
 
         # Pagination
         if self.start > 0:

thordata/retry.py

@@ -16,6 +16,7 @@ Example:
 
 from __future__ import annotations
 
+import inspect
 import logging
 import random
 import time
@@ -201,8 +202,6 @@ def with_retry(
 
         @wraps(func)
         async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
-            import asyncio
-
             last_exception: Optional[Exception] = None
 
             for attempt in range(config.max_retries + 1):
@@ -238,7 +237,7 @@ def with_retry(
         # Check if the function is async
         import asyncio
 
-        if asyncio.iscoroutinefunction(func):
+        if inspect.iscoroutinefunction(func):
             return async_wrapper
         return sync_wrapper
 

{thordata_sdk-0.8.0.dist-info → thordata_sdk-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thordata-sdk
-Version: 0.8.0
+Version: 1.0.0
 Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
 Author-email: Thordata Developer Team <support@thordata.com>
 License: MIT
@@ -87,14 +87,10 @@ Set environment variables:
 # Required for Scraper APIs (SERP, Universal, Tasks)
 export THORDATA_SCRAPER_TOKEN=your_token
 
-# Required for Public/Location APIs (Dashboard -> My Account)
+# Public/Location APIs (Dashboard -> My account -> API)
 export THORDATA_PUBLIC_TOKEN=your_public_token
 export THORDATA_PUBLIC_KEY=your_public_key
 
-# Required for Public API NEW (Dashboard -> Public API NEW)
-# If not set, SDK falls back to PUBLIC_TOKEN/KEY
-export THORDATA_SIGN=your_sign
-export THORDATA_API_KEY=your_api_key
 ```
 
 ---

thordata_sdk-1.0.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+thordata/__init__.py,sha256=MILcOkXK1A3U7pCNZxGqVGFMEvdfMJC-Eki2QaNlCdc,3195
+thordata/_example_utils.py,sha256=a7hSQwxS9OqOatvRYm2NkZRakSJ186tNWbHpJ7WskFc,2185
+thordata/_utils.py,sha256=oMPjR6wSmNonU5dJP6NHgYyV0BDO8b_0zxtKuck1htg,4701
+thordata/async_client.py,sha256=XSsT3ariDWSvP32ny_aqf5dTN56T7yMTraaJOgeDi-g,55416
+thordata/client.py,sha256=mUp6pwZKLca9tNljV_Gv4leOrMJ4LQsveiqdKroYp3I,64494
+thordata/demo.py,sha256=zmG4I4cHXnbmQfbr063SeRK7_9IXrfof9QFoGqGTVm8,3806
+thordata/enums.py,sha256=MpZnS9_8sg2vtcFqM6UicB94cKZm5R1t83L3ejNSbLs,8502
+thordata/exceptions.py,sha256=IgMsFuh49cPxU5YofsKP1UhP5A_snhtuN6xD1yZWLiI,10018
+thordata/models.py,sha256=X7wxS6kk18OGX-OLvCdxl0rhpQrAAM3WYLRnjzrWcOM,37690
+thordata/retry.py,sha256=vb05YexCHjiiZTSm8_eK-_3BsCecplGpFjQ3XBwQ8FY,11505
+thordata_sdk-1.0.0.dist-info/licenses/LICENSE,sha256=bAxpWgQIzb-5jl3nhLdOwOJ_vlbHLtSG7yev2B7vioY,1088
+thordata_sdk-1.0.0.dist-info/METADATA,sha256=qI0Weaz5DOzafF6CWv8gKBkozlhX7HVc4-qFQC66sgY,5667
+thordata_sdk-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thordata_sdk-1.0.0.dist-info/top_level.txt,sha256=Z8R_07m0lXCCSb1hapL9_nxMtyO3rf_9wOvq4n9u2Hg,9
+thordata_sdk-1.0.0.dist-info/RECORD,,

thordata_sdk-0.8.0.dist-info/RECORD

@@ -1,14 +0,0 @@
-thordata/__init__.py,sha256=yaIxW1T_nsCeiPE6iIHunjRzPrtbiN0BciveICgL4dM,3195
-thordata/_utils.py,sha256=epF-ewHyk7McdejlhHNAfxhIQ8sN3TlIjUJ9H4HOaUE,5254
-thordata/async_client.py,sha256=tC9y1wmcO6RsXCysBo0a0GNRZR3QQjJlCmEwG5HVukQ,53169
-thordata/client.py,sha256=VN5Jm3er7fdZDfT2G9g4siBSYNo0ZWj4WOi6TAiAZcE,59638
-thordata/demo.py,sha256=zmG4I4cHXnbmQfbr063SeRK7_9IXrfof9QFoGqGTVm8,3806
-thordata/enums.py,sha256=MpZnS9_8sg2vtcFqM6UicB94cKZm5R1t83L3ejNSbLs,8502
-thordata/exceptions.py,sha256=IgMsFuh49cPxU5YofsKP1UhP5A_snhtuN6xD1yZWLiI,10018
-thordata/models.py,sha256=NG4wn1bq4-FC4Aex8vwBOldiHovwg0JzhdtBsI1mL_8,36118
-thordata/retry.py,sha256=nkh17ca2TIEcTc-uNo-xcNdJPuxZ_VGlMbC70X6p-_Q,11518
-thordata_sdk-0.8.0.dist-info/licenses/LICENSE,sha256=bAxpWgQIzb-5jl3nhLdOwOJ_vlbHLtSG7yev2B7vioY,1088
-thordata_sdk-0.8.0.dist-info/METADATA,sha256=IgL554I6mzya9FdbqCxKdvO3r-bywiHJjZi1xdk8W48,5850
-thordata_sdk-0.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thordata_sdk-0.8.0.dist-info/top_level.txt,sha256=Z8R_07m0lXCCSb1hapL9_nxMtyO3rf_9wOvq4n9u2Hg,9
-thordata_sdk-0.8.0.dist-info/RECORD,,