thordata-sdk 0.2.4__py3-none-any.whl → 0.3.1__py3-none-any.whl

thordata_sdk-0.3.1.dist-info/METADATA

@@ -0,0 +1,200 @@
+Metadata-Version: 2.4
+Name: thordata-sdk
+Version: 0.3.1
+Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
+Author-email: Thordata Developer Team <support@thordata.com>
+License: Apache-2.0
+Project-URL: Homepage, https://www.thordata.com
+Project-URL: Documentation, https://github.com/Thordata/thordata-python-sdk#readme
+Project-URL: Source, https://github.com/Thordata/thordata-python-sdk
+Project-URL: Tracker, https://github.com/Thordata/thordata-python-sdk/issues
+Keywords: web scraping,proxy,ai,llm,data-mining,serp,thordata
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.25.0
+Requires-Dist: aiohttp>=3.8.0
+Dynamic: license-file
+
+# Thordata Python SDK
+
+<h4 align="center">
+  Official Python client for Thordata's Proxy Network, SERP API, Universal Scraping API, and Web Scraper API.
+  <br>
+  <i>Async-ready, built for AI agents and large-scale data collection.</i>
+</h4>
+
+<p align="center">
+  <a href="https://github.com/Thordata/thordata-python-sdk/actions/workflows/ci.yml">
+    <img src="https://github.com/Thordata/thordata-python-sdk/actions/workflows/ci.yml/badge.svg" alt="CI">
+  </a>
+  <a href="https://pypi.org/project/thordata-sdk/">
+    <img src="https://img.shields.io/pypi/v/thordata-sdk?color=blue" alt="PyPI version">
+  </a>
+  <a href="https://github.com/Thordata/thordata-python-sdk/blob/main/LICENSE">
+    <img src="https://img.shields.io/badge/license-Apache%202.0-green" alt="License">
+  </a>
+  <a href="https://python.org">
+    <img src="https://img.shields.io/badge/python-3.8+-blue" alt="Python Versions">
+  </a>
+</p>
+
+---
+
+## Installation
+
+```bash
+pip install thordata-sdk
+```
+
+## Quick Start
+
+All examples below use the unified client:
+
+```python
+from thordata import ThordataClient, AsyncThordataClient
+```
+
+You can copy `examples/.env.example` to `.env` and fill in your tokens from the Thordata Dashboard.
+
+### 1. Proxy Network (Simple GET)
+
+```python
+import os
+from dotenv import load_dotenv
+from thordata import ThordataClient
+
+load_dotenv()
+
+client = ThordataClient(
+    scraper_token=os.getenv("THORDATA_SCRAPER_TOKEN"),
+    public_token=os.getenv("THORDATA_PUBLIC_TOKEN"),
+    public_key=os.getenv("THORDATA_PUBLIC_KEY"),
+)
+
+resp = client.get("http://httpbin.org/ip")
+print(resp.json())
+```
+
+### 2. SERP API (Google, Bing, Yandex, DuckDuckGo)
+
+```python
+from thordata import ThordataClient, Engine
+
+client = ThordataClient("SCRAPER_TOKEN", "PUBLIC_TOKEN", "PUBLIC_KEY")
+
+results = client.serp_search(
+    query="Thordata technology",
+    engine=Engine.GOOGLE,
+    num=10,
+    # Any engine-specific parameters are passed via **kwargs
+    # e.g. type="shopping", location="United States"
+)
+
+print(len(results.get("organic", [])))
+```
+
+### 3. Universal Scraping API
+
+```python
+from thordata import ThordataClient
+
+client = ThordataClient("SCRAPER_TOKEN", "PUBLIC_TOKEN", "PUBLIC_KEY")
+
+html = client.universal_scrape(
+    url="https://www.google.com",
+    js_render=True,
+    output_format="HTML",
+)
+print(html[:200])
+```
+
+### 4. Web Scraper API (Task-based)
+
+```python
+import time
+from thordata import ThordataClient
+
+client = ThordataClient("SCRAPER_TOKEN", "PUBLIC_TOKEN", "PUBLIC_KEY")
+
+task_id = client.create_scraper_task(
+    file_name="demo_youtube_data",
+    spider_id="youtube_video-post_by-url",
+    spider_name="youtube.com",
+    individual_params={
+        "url": "https://www.youtube.com/@stephcurry/videos",
+        "order_by": "",
+        "num_of_posts": ""
+    },
+)
+
+for _ in range(10):
+    status = client.get_task_status(task_id)
+    print("Status:", status)
+    if status in ["Ready", "Success"]:
+        break
+    if status == "Failed":
+        raise RuntimeError("Task failed")
+    time.sleep(3)
+
+download_url = client.get_task_result(task_id)
+print("Download URL:", download_url)
+```
+
+### 5. Asynchronous Usage (High Concurrency)
+
+```python
+import asyncio
+from thordata import AsyncThordataClient
+
+async def main():
+    async with AsyncThordataClient(
+        scraper_token="SCRAPER_TOKEN",
+        public_token="PUBLIC_TOKEN",
+        public_key="PUBLIC_KEY",
+    ) as client:
+        resp = await client.get("http://httpbin.org/ip")
+        print(await resp.json())
+
+asyncio.run(main())
+```
+
+More examples are available in the `examples/` directory.
+
+---
+
+## Features
+
+| Feature | Status | Description |
+|---------|--------|-------------|
+| Proxy Network | Stable | Residential, ISP, Mobile, Datacenter via HTTP/HTTPS gateway. |
+| SERP API | Stable | Google / Bing / Yandex / DuckDuckGo, flexible parameters. |
+| Universal Scraping API | Stable | JS rendering, HTML / PNG output, antibot bypass. |
+| Web Scraper API | Stable | Task-based scraping for complex sites (YouTube, E-commerce). |
+| Async Client | Stable | aiohttp-based client for high-concurrency workloads. |
+
+---
+
+## Development & Contributing
+
+See `CONTRIBUTING.md` for local development and contribution guidelines.
+
+## License
+
+This project is licensed under the Apache License 2.0.
+
+## Support
+
+For technical support, please contact support@thordata.com
+or verify your tokens and quotas in the Thordata Dashboard.

thordata_sdk-0.3.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+thordata/__init__.py,sha256=iv2luaDxUmcWTqScu08gGJfocUZr6pSwtzJs2akZ1Gg,365
+thordata/async_client.py,sha256=cpBtRIzr8oH6GuZs8gTh505tGYYV1aRFBUzbtmFOfEg,9717
+thordata/client.py,sha256=WVIvIZTACEpw9NaTbGtIkMGUlfliFL7kNGdCoTxxsUI,17193
+thordata/enums.py,sha256=PGUCQX3jw5a9mX8_JfhuyoR1WriWjWQpAgibVP_bpdM,679
+thordata/parameters.py,sha256=1lNx_BSS8ztBKEj_MXZMaIQQ9_W3EAlS-VFiBqSWb9E,1841
+thordata_sdk-0.3.1.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+thordata_sdk-0.3.1.dist-info/METADATA,sha256=NMq7Be240zn2q3MlUUg2Dmo4NFoQtDMgkRAGzjg_yjc,5901
+thordata_sdk-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thordata_sdk-0.3.1.dist-info/top_level.txt,sha256=Z8R_07m0lXCCSb1hapL9_nxMtyO3rf_9wOvq4n9u2Hg,9
+thordata_sdk-0.3.1.dist-info/RECORD,,

{thordata_sdk-0.2.4.dist-info → thordata_sdk-0.3.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

thordata_sdk-0.3.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+thordata

thordata_sdk/__init__.py

@@ -1,9 +0,0 @@
-# Expose main clients
-from .client import ThordataClient
-from .async_client import AsyncThordataClient
-from .enums import Engine, GoogleSearchType
-
-# Version of the thordata-sdk package
-__version__ = "0.2.4"
-
-__all__ = ["ThordataClient", "AsyncThordataClient"]

thordata_sdk/client.py

@@ -1,303 +0,0 @@
-import requests
-import logging
-import json
-import base64
-from typing import Dict, Any, Union, Optional
-
-from .enums import Engine
-from .parameters import normalize_serp_params
-
-# Configure a library-specific logger
-logger = logging.getLogger(__name__)
-
-
-class ThordataClient:
-    """
-    The official synchronous Python client for Thordata.
-
-    Handles authentication for:
-    1. Proxy Network (HTTP/HTTPS)
-    2. SERP API (Real-time Search)
-    3. Universal Scraping API (Single Page)
-    4. Web Scraper API (Async Task Management)
-    """
-
-    def __init__(
-        self,
-        scraper_token: str,
-        public_token: str,
-        public_key: str,
-        proxy_host: str = "gate.thordata.com",
-        proxy_port: int = 22225
-    ):
-        """
-        Initialize the Thordata Client.
-
-        Args:
-            scraper_token (str): Token from Dashboard bottom.
-            public_token (str): Token from Public API section.
-            public_key (str): Key from Public API section.
-            proxy_host (str): Proxy gateway host.
-            proxy_port (int): Proxy gateway port.
-        """
-        self.scraper_token = scraper_token
-        self.public_token = public_token
-        self.public_key = public_key
-
-        # Proxy Configuration
-        self.proxy_url = (
-            f"http://{self.scraper_token}:@{proxy_host}:{proxy_port}"
-        )
-
-        # API Endpoints
-        self.base_url = "https://scraperapi.thordata.com"
-        self.universal_url = "https://universalapi.thordata.com"
-        self.api_url = "https://api.thordata.com/api/web-scraper-api"
-
-        self.SERP_API_URL = f"{self.base_url}/request"
-        self.UNIVERSAL_API_URL = f"{self.universal_url}/request"
-        self.SCRAPER_BUILDER_URL = f"{self.base_url}/builder"
-        self.SCRAPER_STATUS_URL = f"{self.api_url}/tasks-status"
-        self.SCRAPER_DOWNLOAD_URL = f"{self.api_url}/tasks-download"
-
-        self.session = requests.Session()
-        self.session.proxies = {
-            "http": self.proxy_url,
-            "https": self.proxy_url,
-        }
-
-    def get(self, url: str, **kwargs) -> requests.Response:
-        """
-        Send a GET request through the Thordata Proxy Network.
-        """
-        logger.debug(f"Proxy Request: {url}")
-        kwargs.setdefault("timeout", 30)
-        return self.session.get(url, **kwargs)
-
-    def serp_search(
-        self, 
-        query: str, 
-        engine: Union[Engine, str] = Engine.GOOGLE, # 既可以是枚举，也可以是字符串
-        num: int = 10, 
-        **kwargs # 这里接收所有额外参数 (比如 type="maps")
-    ) -> Dict[str, Any]:
-        """
-        Execute a real-time SERP search.
-        
-        Args:
-            query: Keywords
-            engine: 'google', 'bing', 'yandex' etc.
-            num: Number of results (default 10)
-            **kwargs: Extra parameters (e.g., type="shopping", location="London")
-        """
-        # 兼容处理：如果用户传的是枚举对象，取它的值；如果是字符串，转小写
-        engine_str = engine.value if isinstance(engine, Engine) else engine.lower()
-
-        # 调用 parameters.py 里的逻辑
-        payload = normalize_serp_params(engine_str, query, num=num, **kwargs)
-
-        headers = {
-            "Authorization": f"Bearer {self.scraper_token}",
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-
-        logger.info(f"SERP Search: {engine_str} - {query}")
-        try:
-            response = self.session.post(
-                self.SERP_API_URL,
-                data=payload,
-                headers=headers,
-                timeout=60
-            )
-            response.raise_for_status()
-            
-            data = response.json()
-            if isinstance(data, str):
-                try: data = json.loads(data)
-                except: pass
-            return data
-        except Exception as e:
-            logger.error(f"SERP Request Failed: {e}")
-            raise
-
-
-    def universal_scrape(
-        self,
-        url: str,
-        js_render: bool = False,
-        output_format: str = "HTML",
-        country: str = None,
-        block_resources: bool = False
-    ) -> Union[str, bytes]:
-        """
-        Unlock target pages via the Universal Scraping API.
-        """
-        headers = {
-            "Authorization": f"Bearer {self.scraper_token}",
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-
-        payload = {
-            "url": url,
-            "js_render": "True" if js_render else "False",
-            "type": output_format.lower(),
-            "block_resources": "True" if block_resources else "False"
-        }
-        if country:
-            payload["country"] = country
-
-        logger.info(f"Universal Scrape: {url}")
-
-        try:
-            response = self.session.post(
-                self.UNIVERSAL_API_URL,
-                data=payload,
-                headers=headers,
-                timeout=60
-            )
-            response.raise_for_status()
-
-            # Parse JSON wrapper
-            try:
-                resp_json = response.json()
-            except json.JSONDecodeError:
-                # Fallback for raw response
-                if output_format.upper() == "PNG":
-                    return response.content
-                return response.text
-
-            # Check API errors
-            if isinstance(resp_json, dict) and resp_json.get("code") \
-                    and resp_json.get("code") != 200:
-                raise Exception(f"Universal API Error: {resp_json}")
-
-            # Extract HTML
-            if "html" in resp_json:
-                return resp_json["html"]
-
-            # Extract PNG
-            if "png" in resp_json:
-                png_str = resp_json["png"]
-                if not png_str:
-                    raise Exception("API returned empty PNG data")
-
-                # 🛠️ FIX: 移除 Data URI Scheme 前缀 (data:image/png;base64,)
-                if "," in png_str:
-                    png_str = png_str.split(",", 1)[1]
-
-                # Base64 解码 (处理 padding)
-                png_str = png_str.replace("\n", "").replace("\r", "")
-                missing_padding = len(png_str) % 4
-                if missing_padding:
-                    png_str += '=' * (4 - missing_padding)
-
-                return base64.b64decode(png_str)
-
-            return str(resp_json)
-
-        except Exception as e:
-            logger.error(f"Universal Scrape Failed: {e}")
-            raise
-
-    def create_scraper_task(
-        self,
-        file_name: str,
-        spider_id: str,     # 必须传，用户从仪表板获取
-        spider_name: str,   # 必须传，例如 "youtube.com"
-        individual_params: Dict[str, Any], # 用户把具体的参数打包在这个字典里传进来
-        universal_params: Dict[str, Any] = None
-    ) -> str:
-        """
-        Create a generic Web Scraper Task.
-        
-        Note: Check the Thordata Dashboard to get the correct 'spider_id' and 'spider_name'.
-        """
-        headers = {
-            "Authorization": f"Bearer {self.scraper_token}",
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-
-        # 直接打包发送，不替用户做太多复杂的校验，保证兼容性
-        payload = {
-            "spider_name": spider_name,
-            "spider_id": spider_id,
-            "spider_parameters": json.dumps([individual_params]),
-            "spider_errors": "true",
-            "file_name": file_name
-        }
-        if universal_params:
-            payload["spider_universal"] = json.dumps(universal_params)
-
-        logger.info(f"Creating Scraper Task: {spider_name} (ID: {spider_id})")
-        try:
-            response = self.session.post(
-                self.SCRAPER_BUILDER_URL,
-                data=payload,
-                headers=headers
-            )
-            response.raise_for_status()
-            data = response.json()
-
-            if data.get("code") != 200:
-                raise Exception(f"Creation failed: {data}")
-            return data["data"]["task_id"]
-        except Exception as e:
-            logger.error(f"Task Creation Failed: {e}")
-            raise
-
-    def get_task_status(self, task_id: str) -> str:
-        """
-        Check the status of a task.
-        """
-        headers = {
-            "token": self.public_token,
-            "key": self.public_key,
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-        payload = {"tasks_ids": task_id}
-
-        try:
-            response = self.session.post(
-                self.SCRAPER_STATUS_URL,
-                data=payload,
-                headers=headers
-            )
-            response.raise_for_status()
-            data = response.json()
-
-            if data.get("code") == 200 and data.get("data"):
-                for item in data["data"]:
-                    if str(item.get("task_id")) == str(task_id):
-                        return item["status"]
-            return "Unknown"
-        except Exception as e:
-            logger.error(f"Status Check Failed: {e}")
-            return "Error"
-
-    def get_task_result(self, task_id: str, file_type: str = "json") -> str:
-        """
-        Retrieve the download URL for a completed task.
-        """
-        headers = {
-            "token": self.public_token,
-            "key": self.public_key,
-            "Content-Type": "application/x-www-form-urlencoded"
-        }
-        payload = {"tasks_id": task_id, "type": file_type}
-
-        logger.info(f"Getting result URL: {task_id}")
-        try:
-            response = self.session.post(
-                self.SCRAPER_DOWNLOAD_URL,
-                data=payload,
-                headers=headers
-            )
-            response.raise_for_status()
-            data = response.json()
-
-            if data.get("code") == 200 and data.get("data"):
-                return data["data"]["download"]
-            raise Exception(f"API returned error: {data}")
-        except Exception as e:
-            logger.error(f"Get Result Failed: {e}")
-            raise

thordata_sdk/enums.py

@@ -1,20 +0,0 @@
-# thordata_sdk/enums.py
-from enum import Enum
-
-class Engine(str, Enum):
-    """SERP 核心支持的四大引擎"""
-    GOOGLE = "google"
-    BING = "bing"
-    YANDEX = "yandex"
-    DUCKDUCKGO = "duckduckgo"
-    BAIDU = "baidu"
-
-class GoogleSearchType(str, Enum):
-    """Google 搜索的常见子类型 (参考你的截图)"""
-    SEARCH = "search"      # 默认网页搜索
-    MAPS = "maps"          # 地图
-    SHOPPING = "shopping"  # 购物
-    NEWS = "news"          # 新闻
-    IMAGES = "images"      # 图片
-    VIDEOS = "videos"      # 视频
-    # 其他冷门的先不写，用户可以通过字符串传参

thordata_sdk/parameters.py

@@ -1,41 +0,0 @@
-# thordata_sdk/parameters.py
-from typing import Dict, Any
-
-def normalize_serp_params(engine: str, query: str, **kwargs) -> Dict[str, Any]:
-    """
-    统一不同搜索引擎的参数差异。
-    """
-    # 1. 基础参数
-    payload = {
-        "num": str(kwargs.get("num", 10)),
-        "json": "1",
-        "engine": engine,
-    }
-
-    # 2. 处理查询关键词 (Yandex 用 text，其他用 q)
-    if engine == "yandex":
-        payload["text"] = query
-        # 如果用户没传 url，给个默认的
-        if "url" not in kwargs:
-            payload["url"] = "yandex.com"
-    else:
-        payload["q"] = query
-        
-        # 3. 处理默认 URL (如果用户没传)
-        if "url" not in kwargs:
-            defaults = {
-                "google": "google.com",
-                "bing": "bing.com",
-                "duckduckgo": "duckduckgo.com",
-                "baidu": "baidu.com"
-            }
-            if engine in defaults:
-                payload["url"] = defaults[engine]
-
-    # 4. 把用户传入的其他所有参数（比如 type="shopping", google_domain="google.co.uk"）都透传进去
-    # 这样你就不用去定义那几十种类型了，用户传啥就是啥
-    for k, v in kwargs.items():
-        if k not in ["num", "engine", "q", "text"]: # 避免覆盖
-            payload[k] = v
-
-    return payload