thordata-sdk 1.0.0__tar.gz → 1.1.0__tar.gz

thordata_sdk-1.1.0/PKG-INFO

@@ -0,0 +1,271 @@
+Metadata-Version: 2.4
+Name: thordata-sdk
+Version: 1.1.0
+Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
+Author-email: Thordata Developer Team <support@thordata.com>
+License: MIT
+Project-URL: Homepage, https://www.thordata.com
+Project-URL: Documentation, https://github.com/Thordata/thordata-python-sdk#readme
+Project-URL: Source, https://github.com/Thordata/thordata-python-sdk
+Project-URL: Tracker, https://github.com/Thordata/thordata-python-sdk/issues
+Project-URL: Changelog, https://github.com/Thordata/thordata-python-sdk/blob/main/CHANGELOG.md
+Keywords: web scraping,proxy,residential proxy,datacenter proxy,ai,llm,data-mining,serp,thordata,web scraper,anti-bot bypass
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: Proxy Servers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.25.0
+Requires-Dist: aiohttp>=3.9.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-httpserver>=1.0.0; extra == "dev"
+Requires-Dist: python-dotenv>=1.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-requests>=2.28.0; extra == "dev"
+Requires-Dist: aioresponses>=0.7.6; extra == "dev"
+Dynamic: license-file
+
+# Thordata Python SDK
+
+<div align="center">
+
+**Official Python Client for Thordata APIs**
+
+*Proxy Network • SERP API • Web Unlocker • Web Scraper API*
+
+[![PyPI version](https://img.shields.io/pypi/v/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+</div>
+
+---
+
+## 📦 Installation
+
+```bash
+pip install thordata-sdk
+```
+
+Optional dependencies for Scraping Browser examples:
+```bash
+pip install playwright
+```
+
+## 🔐 Configuration
+
+Set the following environment variables (recommended):
+
+```bash
+# Required for SERP, Universal, and Proxy Network
+export THORDATA_SCRAPER_TOKEN="your_scraper_token"
+
+# Required for Web Scraper Tasks & Account Management
+export THORDATA_PUBLIC_TOKEN="your_public_token"
+export THORDATA_PUBLIC_KEY="your_public_key"
+
+# Optional: Default Proxy Credentials (for Proxy Network)
+export THORDATA_RESIDENTIAL_USERNAME="user"
+export THORDATA_RESIDENTIAL_PASSWORD="pass"
+```
+
+## 🚀 Quick Start
+
+```python
+from thordata import ThordataClient
+
+# Initialize (credentials loaded from env)
+client = ThordataClient(scraper_token="...") 
+
+# 1. SERP Search
+print("--- SERP Search ---")
+results = client.serp_search("python tutorial", engine="google")
+print(f"Title: {results['organic'][0]['title']}")
+
+# 2. Universal Scrape (Web Unlocker)
+print("\n--- Universal Scrape ---")
+html = client.universal_scrape("https://httpbin.org/html")
+print(f"HTML Length: {len(html)}")
+```
+
+## 📚 Core Features
+
+### 🌐 Proxy Network
+
+Easily generate proxy URLs with geo-targeting and sticky sessions. The SDK handles connection pooling automatically.
+
+```python
+from thordata import ProxyConfig, ProxyProduct
+
+# Create a proxy configuration
+proxy = ProxyConfig(
+    username="user",
+    password="pass",
+    product=ProxyProduct.RESIDENTIAL,
+    country="us",
+    city="new_york",
+    session_id="session123",
+    session_duration=10  # Sticky for 10 mins
+)
+
+# Use with the client (high performance)
+response = client.get("https://httpbin.org/ip", proxy_config=proxy)
+print(response.json())
+
+# Or get the URL string for other libs (requests, scrapy, etc.)
+proxy_url = proxy.build_proxy_url()
+print(f"Proxy URL: {proxy_url}")
+```
+
+### 🔍 SERP API
+
+Real-time search results from Google, Bing, Yandex, etc.
+
+```python
+from thordata import SerpRequest, Engine
+
+# Simple
+results = client.serp_search(
+    query="pizza near me",
+    engine=Engine.GOOGLE_MAPS,
+    country="us"
+)
+
+# Advanced (Strongly Typed)
+request = SerpRequest(
+    query="AI news",
+    engine="google_news",
+    num=50,
+    time_filter="week",
+    location="San Francisco",
+    render_js=True
+)
+results = client.serp_search_advanced(request)
+```
+
+### 🔓 Universal Scraping API (Web Unlocker)
+
+Bypass Cloudflare, CAPTCHAs, and antibot systems.
+
+```python
+html = client.universal_scrape(
+    url="https://example.com/protected",
+    js_render=True,
+    wait_for=".content",
+    country="gb",
+    output_format="html"
+)
+```
+
+### 🕷️ Web Scraper API (Async Tasks)
+
+Manage asynchronous scraping tasks for massive scale.
+
+```python
+# 1. Create Task
+task_id = client.create_scraper_task(
+    file_name="my_task",
+    spider_id="universal",
+    spider_name="universal",
+    parameters={"url": "https://example.com"}
+)
+print(f"Task Created: {task_id}")
+
+# 2. Wait for Completion
+status = client.wait_for_task(task_id, max_wait=600)
+
+# 3. Get Result
+if status == "ready":
+    download_url = client.get_task_result(task_id)
+    print(f"Result: {download_url}")
+```
+
+### 📹 Video/Audio Tasks
+
+Download content from YouTube and other supported platforms.
+
+```python
+from thordata import CommonSettings
+
+task_id = client.create_video_task(
+    file_name="video_{{VideoID}}",
+    spider_id="youtube_video_by-url",
+    spider_name="youtube.com",
+    parameters={"url": "https://youtube.com/watch?v=..."},
+    common_settings=CommonSettings(resolution="1080p")
+)
+```
+
+### 📊 Account Management
+
+Access usage statistics, manage sub-users, and whitelist IPs.
+
+```python
+# Get Usage Stats
+stats = client.get_usage_statistics("2024-01-01", "2024-01-31")
+print(f"Balance: {stats.balance_gb():.2f} GB")
+
+# List Proxy Users
+users = client.list_proxy_users()
+print(f"Active Sub-users: {users.user_count}")
+
+# Whitelist IP
+client.add_whitelist_ip("1.2.3.4")
+```
+
+## ⚙️ Advanced Usage
+
+### Async Client
+
+For high-concurrency applications, use `AsyncThordataClient`.
+
+```python
+import asyncio
+from thordata import AsyncThordataClient
+
+async def main():
+    async with AsyncThordataClient(scraper_token="...") as client:
+        # SERP
+        results = await client.serp_search("async python")
+        
+        # Universal
+        html = await client.universal_scrape("https://example.com")
+
+asyncio.run(main())
+```
+
+Note: `AsyncThordataClient` does not support HTTPS proxy tunneling (TLS-in-TLS) due to `aiohttp` limitations. For proxy network requests, use the sync client.
+
+### Custom Retry Configuration
+
+```python
+from thordata import RetryConfig
+
+retry = RetryConfig(
+    max_retries=5,
+    backoff_factor=1.5,
+    retry_on_status_codes={429, 500, 502, 503, 504}
+)
+
+client = ThordataClient(..., retry_config=retry)
+```
+
+## 📄 License
+
+MIT License

thordata_sdk-1.1.0/README.md

@@ -0,0 +1,228 @@
+# Thordata Python SDK
+
+<div align="center">
+
+**Official Python Client for Thordata APIs**
+
+*Proxy Network • SERP API • Web Unlocker • Web Scraper API*
+
+[![PyPI version](https://img.shields.io/pypi/v/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+</div>
+
+---
+
+## 📦 Installation
+
+```bash
+pip install thordata-sdk
+```
+
+Optional dependencies for Scraping Browser examples:
+```bash
+pip install playwright
+```
+
+## 🔐 Configuration
+
+Set the following environment variables (recommended):
+
+```bash
+# Required for SERP, Universal, and Proxy Network
+export THORDATA_SCRAPER_TOKEN="your_scraper_token"
+
+# Required for Web Scraper Tasks & Account Management
+export THORDATA_PUBLIC_TOKEN="your_public_token"
+export THORDATA_PUBLIC_KEY="your_public_key"
+
+# Optional: Default Proxy Credentials (for Proxy Network)
+export THORDATA_RESIDENTIAL_USERNAME="user"
+export THORDATA_RESIDENTIAL_PASSWORD="pass"
+```
+
+## 🚀 Quick Start
+
+```python
+from thordata import ThordataClient
+
+# Initialize (credentials loaded from env)
+client = ThordataClient(scraper_token="...") 
+
+# 1. SERP Search
+print("--- SERP Search ---")
+results = client.serp_search("python tutorial", engine="google")
+print(f"Title: {results['organic'][0]['title']}")
+
+# 2. Universal Scrape (Web Unlocker)
+print("\n--- Universal Scrape ---")
+html = client.universal_scrape("https://httpbin.org/html")
+print(f"HTML Length: {len(html)}")
+```
+
+## 📚 Core Features
+
+### 🌐 Proxy Network
+
+Easily generate proxy URLs with geo-targeting and sticky sessions. The SDK handles connection pooling automatically.
+
+```python
+from thordata import ProxyConfig, ProxyProduct
+
+# Create a proxy configuration
+proxy = ProxyConfig(
+    username="user",
+    password="pass",
+    product=ProxyProduct.RESIDENTIAL,
+    country="us",
+    city="new_york",
+    session_id="session123",
+    session_duration=10  # Sticky for 10 mins
+)
+
+# Use with the client (high performance)
+response = client.get("https://httpbin.org/ip", proxy_config=proxy)
+print(response.json())
+
+# Or get the URL string for other libs (requests, scrapy, etc.)
+proxy_url = proxy.build_proxy_url()
+print(f"Proxy URL: {proxy_url}")
+```
+
+### 🔍 SERP API
+
+Real-time search results from Google, Bing, Yandex, etc.
+
+```python
+from thordata import SerpRequest, Engine
+
+# Simple
+results = client.serp_search(
+    query="pizza near me",
+    engine=Engine.GOOGLE_MAPS,
+    country="us"
+)
+
+# Advanced (Strongly Typed)
+request = SerpRequest(
+    query="AI news",
+    engine="google_news",
+    num=50,
+    time_filter="week",
+    location="San Francisco",
+    render_js=True
+)
+results = client.serp_search_advanced(request)
+```
+
+### 🔓 Universal Scraping API (Web Unlocker)
+
+Bypass Cloudflare, CAPTCHAs, and antibot systems.
+
+```python
+html = client.universal_scrape(
+    url="https://example.com/protected",
+    js_render=True,
+    wait_for=".content",
+    country="gb",
+    output_format="html"
+)
+```
+
+### 🕷️ Web Scraper API (Async Tasks)
+
+Manage asynchronous scraping tasks for massive scale.
+
+```python
+# 1. Create Task
+task_id = client.create_scraper_task(
+    file_name="my_task",
+    spider_id="universal",
+    spider_name="universal",
+    parameters={"url": "https://example.com"}
+)
+print(f"Task Created: {task_id}")
+
+# 2. Wait for Completion
+status = client.wait_for_task(task_id, max_wait=600)
+
+# 3. Get Result
+if status == "ready":
+    download_url = client.get_task_result(task_id)
+    print(f"Result: {download_url}")
+```
+
+### 📹 Video/Audio Tasks
+
+Download content from YouTube and other supported platforms.
+
+```python
+from thordata import CommonSettings
+
+task_id = client.create_video_task(
+    file_name="video_{{VideoID}}",
+    spider_id="youtube_video_by-url",
+    spider_name="youtube.com",
+    parameters={"url": "https://youtube.com/watch?v=..."},
+    common_settings=CommonSettings(resolution="1080p")
+)
+```
+
+### 📊 Account Management
+
+Access usage statistics, manage sub-users, and whitelist IPs.
+
+```python
+# Get Usage Stats
+stats = client.get_usage_statistics("2024-01-01", "2024-01-31")
+print(f"Balance: {stats.balance_gb():.2f} GB")
+
+# List Proxy Users
+users = client.list_proxy_users()
+print(f"Active Sub-users: {users.user_count}")
+
+# Whitelist IP
+client.add_whitelist_ip("1.2.3.4")
+```
+
+## ⚙️ Advanced Usage
+
+### Async Client
+
+For high-concurrency applications, use `AsyncThordataClient`.
+
+```python
+import asyncio
+from thordata import AsyncThordataClient
+
+async def main():
+    async with AsyncThordataClient(scraper_token="...") as client:
+        # SERP
+        results = await client.serp_search("async python")
+        
+        # Universal
+        html = await client.universal_scrape("https://example.com")
+
+asyncio.run(main())
+```
+
+Note: `AsyncThordataClient` does not support HTTPS proxy tunneling (TLS-in-TLS) due to `aiohttp` limitations. For proxy network requests, use the sync client.
+
+### Custom Retry Configuration
+
+```python
+from thordata import RetryConfig
+
+retry = RetryConfig(
+    max_retries=5,
+    backoff_factor=1.5,
+    retry_on_status_codes={429, 500, 502, 503, 504}
+)
+
+client = ThordataClient(..., retry_config=retry)
+```
+
+## 📄 License
+
+MIT License

{thordata_sdk-1.0.0 → thordata_sdk-1.1.0}/pyproject.toml

@@ -1,10 +1,11 @@
+# thordata-python-sdk/pyproject.toml
 [build-system]
 requires = ["setuptools>=61.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "thordata-sdk"
-version = "1.0.0"
+version = "1.1.0"
 description = "The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -90,11 +91,12 @@ select = [
     "F",   # pyflakes
     "I",   # isort (import sorting)
     "B",   # flake8-bugbear
+    "UP",  # pyupgrade
+    "SIM", # flake8-simplify
 ]
 ignore = [
     "E501",  # line too long (handled by black)
     "E731",  # do not assign a lambda expression
-    "F401",  # imported but unused (we have some intentional re-exports)
 ]
 
 [tool.ruff.lint.isort]

{thordata_sdk-1.0.0 → thordata_sdk-1.1.0}/src/thordata/__init__.py

@@ -35,7 +35,7 @@ Async Usage:
     >>> asyncio.run(main())
 """
 
-__version__ = "1.0.0"
+__version__ = "1.1.0"
 __author__ = "Thordata Developer Team"
 __email__ = "support@thordata.com"
 

{thordata_sdk-1.0.0 → thordata_sdk-1.1.0}/src/thordata/_example_utils.py

@@ -2,8 +2,9 @@ from __future__ import annotations
 
 import json
 import os
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Any, Iterable, Optional
+from typing import Any
 
 try:
     from dotenv import load_dotenv
@@ -23,7 +24,7 @@ def env(name: str) -> str:
     return (os.getenv(name) or "").strip()
 
 
-def skip_if_missing(required: Iterable[str], *, tip: Optional[str] = None) -> bool:
+def skip_if_missing(required: Iterable[str], *, tip: str | None = None) -> bool:
     missing = [k for k in required if not env(k)]
     if not missing:
         return False

{thordata_sdk-1.0.0 → thordata_sdk-1.1.0}/src/thordata/_utils.py

@@ -9,7 +9,8 @@ from __future__ import annotations
 import base64
 import json
 import logging
-from typing import Any, Dict
+import platform
+from typing import Any
 
 logger = logging.getLogger(__name__)
 
@@ -70,7 +71,7 @@ def decode_base64_image(png_str: str) -> bytes:
         raise ValueError(f"Failed to decode base64 image: {e}") from e
 
 
-def build_auth_headers(token: str, mode: str = "bearer") -> Dict[str, str]:
+def build_auth_headers(token: str, mode: str = "bearer") -> dict[str, str]:
     """
     Build authorization headers for API requests.
 
@@ -104,7 +105,7 @@ def build_builder_headers(
     scraper_token: str,
     public_token: str,
     public_key: str,
-) -> Dict[str, str]:
+) -> dict[str, str]:
     """
     Build headers for Web Scraper builder API.
 
@@ -129,7 +130,7 @@ def build_builder_headers(
     }
 
 
-def build_public_api_headers(public_token: str, public_key: str) -> Dict[str, str]:
+def build_public_api_headers(public_token: str, public_key: str) -> dict[str, str]:
     """
     Build headers for public API requests (task status, locations, etc.)
 
@@ -171,17 +172,19 @@ def extract_error_message(payload: Any) -> str:
 
 def build_user_agent(sdk_version: str, http_client: str) -> str:
     """
-    Build a default User-Agent for the SDK.
-
-    Args:
-        sdk_version: SDK version string.
-        http_client: "requests" or "aiohttp" (or any identifier).
-
-    Returns:
-        A User-Agent string.
+    Build a standardized User-Agent for the SDK.
+    Format: thordata-python-sdk/{version} python/{py_ver} ({system}/{release}; {machine})
     """
-    import platform
-
-    py = platform.python_version()
-    system = platform.system()
-    return f"thordata-python-sdk/{sdk_version} (python {py}; {system}; {http_client})"
+    py_ver = platform.python_version()
+    system = platform.system() or "unknown"
+    release = platform.release() or "unknown"
+    machine = platform.machine() or "unknown"
+
+    # Clean up strings to avoid UA parsing issues (remove newlines, etc)
+    system = system.replace(";", "").strip()
+
+    return (
+        f"thordata-python-sdk/{sdk_version} "
+        f"python/{py_ver} "
+        f"({system}/{release}; {machine}; {http_client})"
+    )