thordata-sdk 0.2.4__py3-none-any.whl → 1.2.0__py3-none-any.whl

thordata_sdk-1.2.0.dist-info/METADATA

@@ -0,0 +1,208 @@
+Metadata-Version: 2.4
+Name: thordata-sdk
+Version: 1.2.0
+Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
+Author-email: Thordata Developer Team <support@thordata.com>
+License: MIT
+Project-URL: Homepage, https://www.thordata.com
+Project-URL: Documentation, https://github.com/Thordata/thordata-python-sdk#readme
+Project-URL: Source, https://github.com/Thordata/thordata-python-sdk
+Project-URL: Tracker, https://github.com/Thordata/thordata-python-sdk/issues
+Project-URL: Changelog, https://github.com/Thordata/thordata-python-sdk/blob/main/CHANGELOG.md
+Keywords: web scraping,proxy,residential proxy,datacenter proxy,ai,llm,data-mining,serp,thordata,web scraper,anti-bot bypass
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: Proxy Servers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.25.0
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: PySocks>=1.7.1
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-httpserver>=1.0.0; extra == "dev"
+Requires-Dist: python-dotenv>=1.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-requests>=2.28.0; extra == "dev"
+Requires-Dist: aioresponses>=0.7.6; extra == "dev"
+Dynamic: license-file
+
+# Thordata Python SDK
+
+<div align="center">
+
+<img src="https://img.shields.io/badge/Thordata-AI%20Infrastructure-blue?style=for-the-badge" alt="Thordata Logo">
+
+**The Official Python Client for Thordata APIs**
+
+*Proxy Network • SERP API • Web Unlocker • Web Scraper API*
+
+[![PyPI version](https://img.shields.io/pypi/v/thordata-sdk.svg?style=flat-square)](https://pypi.org/project/thordata-sdk/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/thordata-sdk.svg?style=flat-square)](https://pypi.org/project/thordata-sdk/)
+[![License](https://img.shields.io/badge/license-MIT-green?style=flat-square)](LICENSE)
+[![CI Status](https://img.shields.io/github/actions/workflow/status/Thordata/thordata-python-sdk/ci.yml?branch=main&style=flat-square)](https://github.com/Thordata/thordata-python-sdk/actions)
+
+</div>
+
+---
+
+## 📖 Introduction
+
+This SDK provides a robust, high-performance interface to Thordata's AI data infrastructure. It is designed for high-concurrency scraping, reliable proxy tunneling, and seamless data extraction.
+
+**Key Features:**
+*   **🚀 Production Ready:** Built on `urllib3` connection pooling for low-latency proxy requests.
+*   **⚡ Async Support:** Native `aiohttp` client for high-concurrency SERP/Universal scraping.
+*   **🛡️ Robust:** Handles TLS-in-TLS tunneling, retries, and error parsing automatically.
+*   **✨ Developer Experience:** Fully typed (`mypy` compatible) with intuitive IDE autocomplete.
+*   **🧩 Lazy Validation:** Only validate credentials for the features you actually use.
+
+---
+
+## 📦 Installation
+
+```bash
+pip install thordata-sdk
+```
+
+---
+
+## 🔐 Configuration
+
+Set environment variables to avoid hardcoding credentials. You only need to set the variables for the features you use.
+
+```bash
+# [Required for SERP & Web Unlocker]
+export THORDATA_SCRAPER_TOKEN="your_token_here"
+
+# [Required for Proxy Network]
+export THORDATA_RESIDENTIAL_USERNAME="your_username"
+export THORDATA_RESIDENTIAL_PASSWORD="your_password"
+export THORDATA_PROXY_HOST="vpnXXXX.pr.thordata.net"
+
+# [Required for Task Management]
+export THORDATA_PUBLIC_TOKEN="public_token"
+export THORDATA_PUBLIC_KEY="public_key"
+```
+
+---
+
+## 🚀 Quick Start
+
+### 1. SERP Search (Google/Bing/Yandex)
+
+```python
+from thordata import ThordataClient, Engine
+
+client = ThordataClient()  # Loads THORDATA_SCRAPER_TOKEN from env
+
+# Simple Search
+print("Searching...")
+results = client.serp_search("latest AI trends", engine=Engine.GOOGLE_NEWS)
+
+for news in results.get("news_results", [])[:3]:
+    print(f"- {news['title']} ({news['source']})")
+```
+
+### 2. Universal Scrape (Web Unlocker)
+
+Bypass Cloudflare/Akamai and render JavaScript automatically.
+
+```python
+html = client.universal_scrape(
+    url="https://example.com/protected-page",
+    js_render=True,
+    wait_for=".content-loaded",
+    country="us"
+)
+print(f"Scraped {len(html)} bytes")
+```
+
+### 3. High-Performance Proxy
+
+Use Thordata's residential IPs with automatic connection pooling.
+
+```python
+from thordata import ProxyConfig, ProxyProduct
+
+# Config is optional if env vars are set, but allows granular control
+proxy = ProxyConfig(
+    product=ProxyProduct.RESIDENTIAL,
+    country="jp",
+    city="tokyo",
+    session_id="session-001",
+    session_duration=10  # Sticky IP for 10 mins
+)
+
+# Use the client to make requests (Reuses TCP connections)
+response = client.get("https://httpbin.org/ip", proxy_config=proxy)
+print(response.json())
+```
+
+---
+
+## ⚙️ Advanced Usage
+
+### Async Client (High Concurrency)
+
+For building AI agents or high-throughput spiders.
+
+```python
+import asyncio
+from thordata import AsyncThordataClient
+
+async def main():
+    async with AsyncThordataClient() as client:
+        # Fire off multiple requests in parallel
+        tasks = [
+            client.serp_search(f"query {i}") 
+            for i in range(5)
+        ]
+        results = await asyncio.gather(*tasks)
+        print(f"Completed {len(results)} searches")
+
+asyncio.run(main())
+```
+
+### Web Scraper API (Task Management)
+
+Create and manage large-scale scraping tasks asynchronously.
+
+```python
+# 1. Create a task
+task_id = client.create_scraper_task(
+    file_name="daily_scrape",
+    spider_id="universal",
+    spider_name="universal",
+    parameters={"url": "https://example.com"}
+)
+
+# 2. Wait for completion (Polling)
+status = client.wait_for_task(task_id)
+
+# 3. Get results
+if status == "ready":
+    url = client.get_task_result(task_id)
+    print(f"Download Data: {url}")
+```
+
+---
+
+## 📄 License
+
+MIT License. See [LICENSE](LICENSE) for details.

thordata_sdk-1.2.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+thordata/__init__.py,sha256=Ac7vBgJx4idvcw-Q5HFI0K9xkpSjtavQ_CRN5fZnBPk,3195
+thordata/_example_utils.py,sha256=T9QtVq9BHhubOShgtGp2GSusYYd-ZFUJFJAw7ubIsa4,2199
+thordata/_utils.py,sha256=Acr_6sHgdZXU7SQozd6FEYTZV6iHw__nlhpBTDwb66U,4917
+thordata/async_client.py,sha256=Cd94u7Si8WRClZMzVfPfUO0a9U1vEDOXysuBQMEjvjs,56075
+thordata/client.py,sha256=KRveHA62hF3NzbJHYQSQdIsEYeozhL7rdDHg--N7GaM,56159
+thordata/demo.py,sha256=HQzgaUM33bWD7mBQ6HEkK5K6zqFnSAHLvaam6BwPgFA,3762
+thordata/enums.py,sha256=MpZnS9_8sg2vtcFqM6UicB94cKZm5R1t83L3ejNSbLs,8502
+thordata/exceptions.py,sha256=P9czrxkFhT439DxW3LE5W-koS595ObH4-mAQOfaDM18,9976
+thordata/models.py,sha256=qtB7jE0v5zNEQfSpmOqdiacB5DgM2QfVR2PaYs-DisM,38206
+thordata/retry.py,sha256=5kRwULl3X68Nx8PlSzr9benfyCL0nRSpVQXrwjWr45M,11456
+thordata/serp_engines.py,sha256=iuMWncelcGOskCHXFzpcPMMTL5qfiLkazHB1uj3zpZo,5985
+thordata_sdk-1.2.0.dist-info/licenses/LICENSE,sha256=bAxpWgQIzb-5jl3nhLdOwOJ_vlbHLtSG7yev2B7vioY,1088
+thordata_sdk-1.2.0.dist-info/METADATA,sha256=fFDZQ8nh5_4RM8kUS7CAMylANPKROscmDusJId21ZjU,6600
+thordata_sdk-1.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thordata_sdk-1.2.0.dist-info/top_level.txt,sha256=Z8R_07m0lXCCSb1hapL9_nxMtyO3rf_9wOvq4n9u2Hg,9
+thordata_sdk-1.2.0.dist-info/RECORD,,

{thordata_sdk-0.2.4.dist-info → thordata_sdk-1.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

thordata_sdk-1.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Thordata · AI Proxy & Web Data
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

thordata_sdk-1.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+thordata

thordata_sdk/__init__.py

@@ -1,9 +0,0 @@
-# Expose main clients
-from .client import ThordataClient
-from .async_client import AsyncThordataClient
-from .enums import Engine, GoogleSearchType
-
-# Version of the thordata-sdk package
-__version__ = "0.2.4"
-
-__all__ = ["ThordataClient", "AsyncThordataClient"]

thordata_sdk/async_client.py

@@ -1,247 +0,0 @@
-import aiohttp
-import logging
-import json
-import base64
-from typing import Optional, Dict, Any, Union
-
-# 复用我们刚刚写好的逻辑和枚举
-from .enums import Engine
-from .parameters import normalize_serp_params
-
-logger = logging.getLogger(__name__)
-
-
-class AsyncThordataClient:
-    """
-    Thordata Asynchronous Client (built on aiohttp).
-    """
-
-    def __init__(
-        self,
-        scraper_token: str,
-        public_token: str,
-        public_key: str,
-        proxy_host: str = "gate.thordata.com",
-        proxy_port: int = 22225
-    ):
-        self.scraper_token = scraper_token
-        self.public_token = public_token
-        self.public_key = public_key
-
-        self.proxy_auth = aiohttp.BasicAuth(login=scraper_token, password='')
-        self.proxy_url = f"http://{proxy_host}:{proxy_port}"
-
-        self.base_url = "https://scraperapi.thordata.com"
-        self.universal_url = "https://universalapi.thordata.com"
-        self.api_url = "https://api.thordata.com/api/web-scraper-api"
-
-        self.SERP_API_URL = f"{self.base_url}/request"
-        self.UNIVERSAL_API_URL = f"{self.universal_url}/request"
-        self.SCRAPER_BUILDER_URL = f"{self.base_url}/builder"
-        self.SCRAPER_STATUS_URL = f"{self.api_url}/tasks-status"
-        self.SCRAPER_DOWNLOAD_URL = f"{self.api_url}/tasks-download"
-
-        self._session: Optional[aiohttp.ClientSession] = None
-
-    async def __aenter__(self):
-        if self._session is None or self._session.closed:
-            self._session = aiohttp.ClientSession(trust_env=True)
-        return self
-
-    async def __aexit__(self, exc_type, exc, tb):
-        await self.close()
-
-    async def close(self):
-        if self._session and not self._session.closed:
-            await self._session.close()
-            self._session = None
-
-    # --- Proxy (Unchanged) ---
-    async def get(self, url: str, **kwargs) -> aiohttp.ClientResponse:
-        if self._session is None:
-            raise RuntimeError("Client session not initialized.")
-        try:
-            return await self._session.get(
-                url,
-                proxy=self.proxy_url,
-                proxy_auth=self.proxy_auth,
-                **kwargs
-            )
-        except aiohttp.ClientError as e:
-            logger.error(f"Async Request failed: {e}")
-            raise
-
-    # --- SERP (Optimized) ---
-    async def serp_search(
-        self, 
-        query: str, 
-        engine: Union[Engine, str] = Engine.GOOGLE, 
-        num: int = 10, 
-        **kwargs
-    ) -> Dict[str, Any]:
-        """
-        Execute a real-time SERP search (Async).
-        """
-        if self._session is None:
-            raise RuntimeError("Client session not initialized.")
-
-        # 1. 转换枚举
-        engine_str = engine.value if isinstance(engine, Engine) else engine.lower()
-
-        # 2. 调用 parameters.py 复用逻辑 (Don't Repeat Yourself!)
-        payload = normalize_serp_params(engine_str, query, num=num, **kwargs)
-
-        headers = {
-            "Authorization": f"Bearer {self.scraper_token}",
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-
-        # 3. 发送请求
-        async with self._session.post(
-            self.SERP_API_URL, data=payload, headers=headers
-        ) as response:
-            response.raise_for_status()
-            data = await response.json()
-            if isinstance(data, str):
-                try:
-                    data = json.loads(data)
-                except Exception:
-                    pass
-            return data
-
-    # --- Universal (Unchanged) ---
-    async def universal_scrape(
-        self,
-        url: str,
-        js_render: bool = False,
-        output_format: str = "HTML",
-        country: str = None,
-        block_resources: bool = False
-    ) -> Union[str, bytes]:
-        if self._session is None:
-            raise RuntimeError("Client session not initialized.")
-
-        headers = {
-            "Authorization": f"Bearer {self.scraper_token}",
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-
-        payload = {
-            "url": url,
-            "js_render": "True" if js_render else "False",
-            "type": output_format.lower(),
-            "block_resources": "True" if block_resources else "False"
-        }
-        if country:
-            payload["country"] = country
-
-        async with self._session.post(
-            self.UNIVERSAL_API_URL, data=payload, headers=headers
-        ) as response:
-            response.raise_for_status()
-
-            try:
-                resp_json = await response.json()
-            except Exception:
-                if output_format.upper() == "PNG":
-                    return await response.read()
-                return await response.text()
-
-            if isinstance(resp_json, dict) and resp_json.get("code") \
-                    and resp_json.get("code") != 200:
-                raise Exception(f"Universal API Error: {resp_json}")
-
-            if "html" in resp_json:
-                return resp_json["html"]
-
-            if "png" in resp_json:
-                png_str = resp_json["png"]
-                if not png_str:
-                    raise Exception("API returned empty PNG data")
-
-                # 🛠️ FIX: 移除 Data URI Scheme 前缀
-                if "," in png_str:
-                    png_str = png_str.split(",", 1)[1]
-
-                png_str = png_str.replace("\n", "").replace("\r", "")
-                missing_padding = len(png_str) % 4
-                if missing_padding:
-                    png_str += '=' * (4 - missing_padding)
-                return base64.b64decode(png_str)
-
-            return str(resp_json)
-
-    # --- Web Scraper (Optimized) ---
-    async def create_scraper_task(
-        self,
-        file_name: str,
-        spider_id: str,
-        spider_name: str,
-        individual_params: Dict[str, Any],
-        universal_params: Dict[str, Any] = None
-    ) -> str:
-        """
-        Create an Asynchronous Web Scraper Task.
-        """
-        if self._session is None:
-            raise RuntimeError("Client session not initialized.")
-
-        headers = {
-            "Authorization": f"Bearer {self.scraper_token}",
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-
-        # 简化 Payload 构建，移除不必要的检查
-        payload = {
-            "file_name": file_name,
-            "spider_id": spider_id,
-            "spider_name": spider_name,
-            "spider_parameters": json.dumps([individual_params]),
-            "spider_errors": "true"
-        }
-        if universal_params:
-            payload["spider_universal"] = json.dumps(universal_params)
-
-        async with self._session.post(
-            self.SCRAPER_BUILDER_URL, data=payload, headers=headers
-        ) as response:
-            response.raise_for_status()
-            data = await response.json()
-            if data.get("code") != 200:
-                raise Exception(f"Creation failed: {data}")
-            return data["data"]["task_id"]
-
-    # --- Status & Result (Unchanged) ---
-    async def get_task_status(self, task_id: str) -> str:
-        headers = {
-            "token": self.public_token,
-            "key": self.public_key,
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-        payload = {"tasks_ids": task_id}
-
-        async with self._session.post(
-            self.SCRAPER_STATUS_URL, data=payload, headers=headers
-        ) as response:
-            data = await response.json()
-            if data.get("code") == 200 and data.get("data"):
-                for item in data["data"]:
-                    if str(item["task_id"]) == str(task_id):
-                        return item["status"]
-            return "Unknown"
-
-    async def get_task_result(self, task_id: str, file_type: str = "json") -> str:
-        headers = {
-            "token": self.public_token,
-            "key": self.public_key,
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-        payload = {"tasks_id": task_id, "type": "json"}
-
-        async with self._session.post(
-            self.SCRAPER_DOWNLOAD_URL, data=payload, headers=headers
-        ) as response:
-            data = await response.json()
-            if data.get("code") == 200:
-                return data["data"]["download"]
-            raise Exception(f"Result Error: {data}")