thordata-sdk 0.3.1__tar.gz → 1.1.0__tar.gz

thordata_sdk-1.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Thordata · AI Proxy & Web Data
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

thordata_sdk-1.1.0/PKG-INFO

@@ -0,0 +1,271 @@
+Metadata-Version: 2.4
+Name: thordata-sdk
+Version: 1.1.0
+Summary: The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network.
+Author-email: Thordata Developer Team <support@thordata.com>
+License: MIT
+Project-URL: Homepage, https://www.thordata.com
+Project-URL: Documentation, https://github.com/Thordata/thordata-python-sdk#readme
+Project-URL: Source, https://github.com/Thordata/thordata-python-sdk
+Project-URL: Tracker, https://github.com/Thordata/thordata-python-sdk/issues
+Project-URL: Changelog, https://github.com/Thordata/thordata-python-sdk/blob/main/CHANGELOG.md
+Keywords: web scraping,proxy,residential proxy,datacenter proxy,ai,llm,data-mining,serp,thordata,web scraper,anti-bot bypass
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: Proxy Servers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.25.0
+Requires-Dist: aiohttp>=3.9.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-httpserver>=1.0.0; extra == "dev"
+Requires-Dist: python-dotenv>=1.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-requests>=2.28.0; extra == "dev"
+Requires-Dist: aioresponses>=0.7.6; extra == "dev"
+Dynamic: license-file
+
+# Thordata Python SDK
+
+<div align="center">
+
+**Official Python Client for Thordata APIs**
+
+*Proxy Network • SERP API • Web Unlocker • Web Scraper API*
+
+[![PyPI version](https://img.shields.io/pypi/v/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+</div>
+
+---
+
+## 📦 Installation
+
+```bash
+pip install thordata-sdk
+```
+
+Optional dependencies for Scraping Browser examples:
+```bash
+pip install playwright
+```
+
+## 🔐 Configuration
+
+Set the following environment variables (recommended):
+
+```bash
+# Required for SERP, Universal, and Proxy Network
+export THORDATA_SCRAPER_TOKEN="your_scraper_token"
+
+# Required for Web Scraper Tasks & Account Management
+export THORDATA_PUBLIC_TOKEN="your_public_token"
+export THORDATA_PUBLIC_KEY="your_public_key"
+
+# Optional: Default Proxy Credentials (for Proxy Network)
+export THORDATA_RESIDENTIAL_USERNAME="user"
+export THORDATA_RESIDENTIAL_PASSWORD="pass"
+```
+
+## 🚀 Quick Start
+
+```python
+from thordata import ThordataClient
+
+# Initialize (credentials loaded from env)
+client = ThordataClient(scraper_token="...") 
+
+# 1. SERP Search
+print("--- SERP Search ---")
+results = client.serp_search("python tutorial", engine="google")
+print(f"Title: {results['organic'][0]['title']}")
+
+# 2. Universal Scrape (Web Unlocker)
+print("\n--- Universal Scrape ---")
+html = client.universal_scrape("https://httpbin.org/html")
+print(f"HTML Length: {len(html)}")
+```
+
+## 📚 Core Features
+
+### 🌐 Proxy Network
+
+Easily generate proxy URLs with geo-targeting and sticky sessions. The SDK handles connection pooling automatically.
+
+```python
+from thordata import ProxyConfig, ProxyProduct
+
+# Create a proxy configuration
+proxy = ProxyConfig(
+    username="user",
+    password="pass",
+    product=ProxyProduct.RESIDENTIAL,
+    country="us",
+    city="new_york",
+    session_id="session123",
+    session_duration=10  # Sticky for 10 mins
+)
+
+# Use with the client (high performance)
+response = client.get("https://httpbin.org/ip", proxy_config=proxy)
+print(response.json())
+
+# Or get the URL string for other libs (requests, scrapy, etc.)
+proxy_url = proxy.build_proxy_url()
+print(f"Proxy URL: {proxy_url}")
+```
+
+### 🔍 SERP API
+
+Real-time search results from Google, Bing, Yandex, etc.
+
+```python
+from thordata import SerpRequest, Engine
+
+# Simple
+results = client.serp_search(
+    query="pizza near me",
+    engine=Engine.GOOGLE_MAPS,
+    country="us"
+)
+
+# Advanced (Strongly Typed)
+request = SerpRequest(
+    query="AI news",
+    engine="google_news",
+    num=50,
+    time_filter="week",
+    location="San Francisco",
+    render_js=True
+)
+results = client.serp_search_advanced(request)
+```
+
+### 🔓 Universal Scraping API (Web Unlocker)
+
+Bypass Cloudflare, CAPTCHAs, and antibot systems.
+
+```python
+html = client.universal_scrape(
+    url="https://example.com/protected",
+    js_render=True,
+    wait_for=".content",
+    country="gb",
+    output_format="html"
+)
+```
+
+### 🕷️ Web Scraper API (Async Tasks)
+
+Manage asynchronous scraping tasks for massive scale.
+
+```python
+# 1. Create Task
+task_id = client.create_scraper_task(
+    file_name="my_task",
+    spider_id="universal",
+    spider_name="universal",
+    parameters={"url": "https://example.com"}
+)
+print(f"Task Created: {task_id}")
+
+# 2. Wait for Completion
+status = client.wait_for_task(task_id, max_wait=600)
+
+# 3. Get Result
+if status == "ready":
+    download_url = client.get_task_result(task_id)
+    print(f"Result: {download_url}")
+```
+
+### 📹 Video/Audio Tasks
+
+Download content from YouTube and other supported platforms.
+
+```python
+from thordata import CommonSettings
+
+task_id = client.create_video_task(
+    file_name="video_{{VideoID}}",
+    spider_id="youtube_video_by-url",
+    spider_name="youtube.com",
+    parameters={"url": "https://youtube.com/watch?v=..."},
+    common_settings=CommonSettings(resolution="1080p")
+)
+```
+
+### 📊 Account Management
+
+Access usage statistics, manage sub-users, and whitelist IPs.
+
+```python
+# Get Usage Stats
+stats = client.get_usage_statistics("2024-01-01", "2024-01-31")
+print(f"Balance: {stats.balance_gb():.2f} GB")
+
+# List Proxy Users
+users = client.list_proxy_users()
+print(f"Active Sub-users: {users.user_count}")
+
+# Whitelist IP
+client.add_whitelist_ip("1.2.3.4")
+```
+
+## ⚙️ Advanced Usage
+
+### Async Client
+
+For high-concurrency applications, use `AsyncThordataClient`.
+
+```python
+import asyncio
+from thordata import AsyncThordataClient
+
+async def main():
+    async with AsyncThordataClient(scraper_token="...") as client:
+        # SERP
+        results = await client.serp_search("async python")
+        
+        # Universal
+        html = await client.universal_scrape("https://example.com")
+
+asyncio.run(main())
+```
+
+Note: `AsyncThordataClient` does not support HTTPS proxy tunneling (TLS-in-TLS) due to `aiohttp` limitations. For proxy network requests, use the sync client.
+
+### Custom Retry Configuration
+
+```python
+from thordata import RetryConfig
+
+retry = RetryConfig(
+    max_retries=5,
+    backoff_factor=1.5,
+    retry_on_status_codes={429, 500, 502, 503, 504}
+)
+
+client = ThordataClient(..., retry_config=retry)
+```
+
+## 📄 License
+
+MIT License

thordata_sdk-1.1.0/README.md

@@ -0,0 +1,228 @@
+# Thordata Python SDK
+
+<div align="center">
+
+**Official Python Client for Thordata APIs**
+
+*Proxy Network • SERP API • Web Unlocker • Web Scraper API*
+
+[![PyPI version](https://img.shields.io/pypi/v/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/thordata-sdk.svg)](https://pypi.org/project/thordata-sdk/)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+</div>
+
+---
+
+## 📦 Installation
+
+```bash
+pip install thordata-sdk
+```
+
+Optional dependencies for Scraping Browser examples:
+```bash
+pip install playwright
+```
+
+## 🔐 Configuration
+
+Set the following environment variables (recommended):
+
+```bash
+# Required for SERP, Universal, and Proxy Network
+export THORDATA_SCRAPER_TOKEN="your_scraper_token"
+
+# Required for Web Scraper Tasks & Account Management
+export THORDATA_PUBLIC_TOKEN="your_public_token"
+export THORDATA_PUBLIC_KEY="your_public_key"
+
+# Optional: Default Proxy Credentials (for Proxy Network)
+export THORDATA_RESIDENTIAL_USERNAME="user"
+export THORDATA_RESIDENTIAL_PASSWORD="pass"
+```
+
+## 🚀 Quick Start
+
+```python
+from thordata import ThordataClient
+
+# Initialize (credentials loaded from env)
+client = ThordataClient(scraper_token="...") 
+
+# 1. SERP Search
+print("--- SERP Search ---")
+results = client.serp_search("python tutorial", engine="google")
+print(f"Title: {results['organic'][0]['title']}")
+
+# 2. Universal Scrape (Web Unlocker)
+print("\n--- Universal Scrape ---")
+html = client.universal_scrape("https://httpbin.org/html")
+print(f"HTML Length: {len(html)}")
+```
+
+## 📚 Core Features
+
+### 🌐 Proxy Network
+
+Easily generate proxy URLs with geo-targeting and sticky sessions. The SDK handles connection pooling automatically.
+
+```python
+from thordata import ProxyConfig, ProxyProduct
+
+# Create a proxy configuration
+proxy = ProxyConfig(
+    username="user",
+    password="pass",
+    product=ProxyProduct.RESIDENTIAL,
+    country="us",
+    city="new_york",
+    session_id="session123",
+    session_duration=10  # Sticky for 10 mins
+)
+
+# Use with the client (high performance)
+response = client.get("https://httpbin.org/ip", proxy_config=proxy)
+print(response.json())
+
+# Or get the URL string for other libs (requests, scrapy, etc.)
+proxy_url = proxy.build_proxy_url()
+print(f"Proxy URL: {proxy_url}")
+```
+
+### 🔍 SERP API
+
+Real-time search results from Google, Bing, Yandex, etc.
+
+```python
+from thordata import SerpRequest, Engine
+
+# Simple
+results = client.serp_search(
+    query="pizza near me",
+    engine=Engine.GOOGLE_MAPS,
+    country="us"
+)
+
+# Advanced (Strongly Typed)
+request = SerpRequest(
+    query="AI news",
+    engine="google_news",
+    num=50,
+    time_filter="week",
+    location="San Francisco",
+    render_js=True
+)
+results = client.serp_search_advanced(request)
+```
+
+### 🔓 Universal Scraping API (Web Unlocker)
+
+Bypass Cloudflare, CAPTCHAs, and antibot systems.
+
+```python
+html = client.universal_scrape(
+    url="https://example.com/protected",
+    js_render=True,
+    wait_for=".content",
+    country="gb",
+    output_format="html"
+)
+```
+
+### 🕷️ Web Scraper API (Async Tasks)
+
+Manage asynchronous scraping tasks for massive scale.
+
+```python
+# 1. Create Task
+task_id = client.create_scraper_task(
+    file_name="my_task",
+    spider_id="universal",
+    spider_name="universal",
+    parameters={"url": "https://example.com"}
+)
+print(f"Task Created: {task_id}")
+
+# 2. Wait for Completion
+status = client.wait_for_task(task_id, max_wait=600)
+
+# 3. Get Result
+if status == "ready":
+    download_url = client.get_task_result(task_id)
+    print(f"Result: {download_url}")
+```
+
+### 📹 Video/Audio Tasks
+
+Download content from YouTube and other supported platforms.
+
+```python
+from thordata import CommonSettings
+
+task_id = client.create_video_task(
+    file_name="video_{{VideoID}}",
+    spider_id="youtube_video_by-url",
+    spider_name="youtube.com",
+    parameters={"url": "https://youtube.com/watch?v=..."},
+    common_settings=CommonSettings(resolution="1080p")
+)
+```
+
+### 📊 Account Management
+
+Access usage statistics, manage sub-users, and whitelist IPs.
+
+```python
+# Get Usage Stats
+stats = client.get_usage_statistics("2024-01-01", "2024-01-31")
+print(f"Balance: {stats.balance_gb():.2f} GB")
+
+# List Proxy Users
+users = client.list_proxy_users()
+print(f"Active Sub-users: {users.user_count}")
+
+# Whitelist IP
+client.add_whitelist_ip("1.2.3.4")
+```
+
+## ⚙️ Advanced Usage
+
+### Async Client
+
+For high-concurrency applications, use `AsyncThordataClient`.
+
+```python
+import asyncio
+from thordata import AsyncThordataClient
+
+async def main():
+    async with AsyncThordataClient(scraper_token="...") as client:
+        # SERP
+        results = await client.serp_search("async python")
+        
+        # Universal
+        html = await client.universal_scrape("https://example.com")
+
+asyncio.run(main())
+```
+
+Note: `AsyncThordataClient` does not support HTTPS proxy tunneling (TLS-in-TLS) due to `aiohttp` limitations. For proxy network requests, use the sync client.
+
+### Custom Retry Configuration
+
+```python
+from thordata import RetryConfig
+
+retry = RetryConfig(
+    max_retries=5,
+    backoff_factor=1.5,
+    retry_on_status_codes={429, 500, 502, 503, 504}
+)
+
+client = ThordataClient(..., retry_config=retry)
+```
+
+## 📄 License
+
+MIT License

thordata_sdk-1.1.0/pyproject.toml

@@ -0,0 +1,138 @@
+# thordata-python-sdk/pyproject.toml
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "thordata-sdk"
+version = "1.1.0"
+description = "The Official Python SDK for Thordata - AI Data Infrastructure & Proxy Network."
+readme = "README.md"
+requires-python = ">=3.9"
+license = {text = "MIT"}
+authors = [
+  {name = "Thordata Developer Team", email = "support@thordata.com"}
+]
+keywords = [
+    "web scraping", 
+    "proxy", 
+    "residential proxy",
+    "datacenter proxy",
+    "ai", 
+    "llm", 
+    "data-mining", 
+    "serp", 
+    "thordata",
+    "web scraper",
+    "anti-bot bypass"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Internet :: Proxy Servers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Typing :: Typed",
+]
+
+dependencies = [
+    "requests>=2.25.0",
+    "aiohttp>=3.9.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
+    "pytest-cov>=4.0.0",
+    "pytest-httpserver>=1.0.0",
+    "python-dotenv>=1.0.0",
+    "black>=23.0.0",
+    "ruff>=0.1.0",
+    "mypy>=1.0.0",
+    "types-requests>=2.28.0",
+    "aioresponses>=0.7.6",
+]
+
+[project.urls]
+"Homepage" = "https://www.thordata.com"
+"Documentation" = "https://github.com/Thordata/thordata-python-sdk#readme"
+"Source" = "https://github.com/Thordata/thordata-python-sdk"
+"Tracker" = "https://github.com/Thordata/thordata-python-sdk/issues"
+"Changelog" = "https://github.com/Thordata/thordata-python-sdk/blob/main/CHANGELOG.md"
+
+# Setuptools setup
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+thordata = ["py.typed"]
+
+[tool.black]
+line-length = 88
+target-version = ['py39', 'py310', 'py311', 'py312']
+include = '\.pyi?$'
+
+[tool.ruff]
+line-length = 88
+target-version = "py39"
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort (import sorting)
+    "B",   # flake8-bugbear
+    "UP",  # pyupgrade
+    "SIM", # flake8-simplify
+]
+ignore = [
+    "E501",  # line too long (handled by black)
+    "E731",  # do not assign a lambda expression
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["thordata"]
+
+# Mypy
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = false
+warn_unused_ignores = false
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+check_untyped_defs = false
+strict_optional = false
+show_error_codes = true
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = ["aiohttp.*", "requests.*"]
+ignore_missing_imports = true
+
+# Pytest setup
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+addopts = "-v --cov=thordata --cov-report=term-missing"
+
+# Coverage setup
+[tool.coverage.run]
+source = ["src/thordata"]
+branch = true
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise NotImplementedError",
+    "if TYPE_CHECKING:",
+]

{thordata_sdk-0.3.1 → thordata_sdk-1.1.0}/setup.cfg

@@ -1,4 +1,4 @@
-[egg_info]
-tag_build = 
-tag_date = 0
-
+[egg_info]
+tag_build = 
+tag_date = 0
+