thordata-sdk 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- thordata/__init__.py +15 -1
- thordata/_utils.py +66 -3
- thordata/async_client.py +787 -8
- thordata/client.py +851 -33
- thordata/enums.py +85 -16
- thordata/exceptions.py +16 -5
- thordata/models.py +294 -0
- thordata/retry.py +4 -1
- thordata_sdk-0.8.0.dist-info/METADATA +212 -0
- thordata_sdk-0.8.0.dist-info/RECORD +14 -0
- thordata/parameters.py +0 -53
- thordata_sdk-0.6.0.dist-info/METADATA +0 -1053
- thordata_sdk-0.6.0.dist-info/RECORD +0 -15
- {thordata_sdk-0.6.0.dist-info → thordata_sdk-0.8.0.dist-info}/WHEEL +0 -0
- {thordata_sdk-0.6.0.dist-info → thordata_sdk-0.8.0.dist-info}/licenses/LICENSE +0 -0
- {thordata_sdk-0.6.0.dist-info → thordata_sdk-0.8.0.dist-info}/top_level.txt +0 -0
thordata/async_client.py
CHANGED
|
@@ -25,6 +25,7 @@ from __future__ import annotations
|
|
|
25
25
|
import asyncio
|
|
26
26
|
import logging
|
|
27
27
|
import os
|
|
28
|
+
from datetime import date, datetime
|
|
28
29
|
from typing import Any, Dict, List, Optional, Union
|
|
29
30
|
|
|
30
31
|
import aiohttp
|
|
@@ -32,7 +33,9 @@ import aiohttp
|
|
|
32
33
|
from . import __version__ as _sdk_version
|
|
33
34
|
from ._utils import (
|
|
34
35
|
build_auth_headers,
|
|
36
|
+
build_builder_headers,
|
|
35
37
|
build_public_api_headers,
|
|
38
|
+
build_sign_headers,
|
|
36
39
|
build_user_agent,
|
|
37
40
|
decode_base64_image,
|
|
38
41
|
extract_error_message,
|
|
@@ -45,7 +48,18 @@ from .exceptions import (
|
|
|
45
48
|
ThordataTimeoutError,
|
|
46
49
|
raise_for_code,
|
|
47
50
|
)
|
|
48
|
-
from .models import
|
|
51
|
+
from .models import (
|
|
52
|
+
CommonSettings,
|
|
53
|
+
ProxyConfig,
|
|
54
|
+
ProxyServer,
|
|
55
|
+
ProxyUser,
|
|
56
|
+
ProxyUserList,
|
|
57
|
+
ScraperTaskConfig,
|
|
58
|
+
SerpRequest,
|
|
59
|
+
UniversalScrapeRequest,
|
|
60
|
+
UsageStatistics,
|
|
61
|
+
VideoTaskConfig,
|
|
62
|
+
)
|
|
49
63
|
from .retry import RetryConfig
|
|
50
64
|
|
|
51
65
|
logger = logging.getLogger(__name__)
|
|
@@ -78,18 +92,22 @@ class AsyncThordataClient:
|
|
|
78
92
|
# API Endpoints (same as sync client)
|
|
79
93
|
BASE_URL = "https://scraperapi.thordata.com"
|
|
80
94
|
UNIVERSAL_URL = "https://universalapi.thordata.com"
|
|
81
|
-
API_URL = "https://
|
|
82
|
-
LOCATIONS_URL = "https://
|
|
95
|
+
API_URL = "https://openapi.thordata.com/api/web-scraper-api"
|
|
96
|
+
LOCATIONS_URL = "https://openapi.thordata.com/api/locations"
|
|
83
97
|
|
|
84
98
|
def __init__(
|
|
85
99
|
self,
|
|
86
100
|
scraper_token: str,
|
|
87
101
|
public_token: Optional[str] = None,
|
|
88
102
|
public_key: Optional[str] = None,
|
|
103
|
+
sign: Optional[str] = None,
|
|
104
|
+
api_key: Optional[str] = None,
|
|
89
105
|
proxy_host: str = "pr.thordata.net",
|
|
90
106
|
proxy_port: int = 9999,
|
|
91
107
|
timeout: int = 30,
|
|
108
|
+
api_timeout: int = 60,
|
|
92
109
|
retry_config: Optional[RetryConfig] = None,
|
|
110
|
+
auth_mode: str = "bearer",
|
|
93
111
|
scraperapi_base_url: Optional[str] = None,
|
|
94
112
|
universalapi_base_url: Optional[str] = None,
|
|
95
113
|
web_scraper_api_base_url: Optional[str] = None,
|
|
@@ -103,14 +121,33 @@ class AsyncThordataClient:
|
|
|
103
121
|
self.public_token = public_token
|
|
104
122
|
self.public_key = public_key
|
|
105
123
|
|
|
124
|
+
# Automatic Fallback Logic: If sign/api_key is not provided, try using public_token/key
|
|
125
|
+
self.sign = sign or os.getenv("THORDATA_SIGN") or self.public_token
|
|
126
|
+
self.api_key = api_key or os.getenv("THORDATA_API_KEY") or self.public_key
|
|
127
|
+
|
|
128
|
+
# Public API authentication
|
|
129
|
+
self.sign = sign or os.getenv("THORDATA_SIGN")
|
|
130
|
+
self.api_key = api_key or os.getenv("THORDATA_API_KEY")
|
|
131
|
+
|
|
106
132
|
# Proxy configuration
|
|
107
133
|
self._proxy_host = proxy_host
|
|
108
134
|
self._proxy_port = proxy_port
|
|
109
135
|
self._default_timeout = aiohttp.ClientTimeout(total=timeout)
|
|
110
136
|
|
|
137
|
+
# Timeout configuration
|
|
138
|
+
self._default_timeout = aiohttp.ClientTimeout(total=timeout)
|
|
139
|
+
self._api_timeout = aiohttp.ClientTimeout(total=api_timeout)
|
|
140
|
+
|
|
111
141
|
# Retry configuration
|
|
112
142
|
self._retry_config = retry_config or RetryConfig()
|
|
113
143
|
|
|
144
|
+
# Authentication mode
|
|
145
|
+
self._auth_mode = auth_mode.lower()
|
|
146
|
+
if self._auth_mode not in ("bearer", "header_token"):
|
|
147
|
+
raise ThordataConfigError(
|
|
148
|
+
f"Invalid auth_mode: {auth_mode}. Must be 'bearer' or 'header_token'."
|
|
149
|
+
)
|
|
150
|
+
|
|
114
151
|
# Pre-calculate proxy auth
|
|
115
152
|
self._proxy_url = f"http://{proxy_host}:{proxy_port}"
|
|
116
153
|
self._proxy_auth = aiohttp.BasicAuth(
|
|
@@ -142,12 +179,39 @@ class AsyncThordataClient:
|
|
|
142
179
|
or self.LOCATIONS_URL
|
|
143
180
|
).rstrip("/")
|
|
144
181
|
|
|
182
|
+
gateway_base = os.getenv(
|
|
183
|
+
"THORDATA_GATEWAY_BASE_URL", "https://api.thordata.com/api/gateway"
|
|
184
|
+
)
|
|
185
|
+
child_base = os.getenv(
|
|
186
|
+
"THORDATA_CHILD_BASE_URL", "https://api.thordata.com/api/child"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
self._gateway_base_url = gateway_base
|
|
190
|
+
self._child_base_url = child_base
|
|
191
|
+
|
|
145
192
|
self._serp_url = f"{scraperapi_base}/request"
|
|
146
193
|
self._builder_url = f"{scraperapi_base}/builder"
|
|
194
|
+
self._video_builder_url = f"{scraperapi_base}/video_builder"
|
|
147
195
|
self._universal_url = f"{universalapi_base}/request"
|
|
148
196
|
self._status_url = f"{web_scraper_api_base}/tasks-status"
|
|
149
197
|
self._download_url = f"{web_scraper_api_base}/tasks-download"
|
|
150
198
|
self._locations_base_url = locations_base
|
|
199
|
+
self._usage_stats_url = (
|
|
200
|
+
f"{locations_base.replace('/locations', '')}/account/usage-statistics"
|
|
201
|
+
)
|
|
202
|
+
self._proxy_users_url = (
|
|
203
|
+
f"{locations_base.replace('/locations', '')}/proxy-users"
|
|
204
|
+
)
|
|
205
|
+
whitelist_base = os.getenv(
|
|
206
|
+
"THORDATA_WHITELIST_BASE_URL", "https://api.thordata.com/api"
|
|
207
|
+
)
|
|
208
|
+
self._whitelist_url = f"{whitelist_base}/whitelisted-ips"
|
|
209
|
+
proxy_api_base = os.getenv(
|
|
210
|
+
"THORDATA_PROXY_API_BASE_URL", "https://api.thordata.com/api"
|
|
211
|
+
)
|
|
212
|
+
self._proxy_list_url = f"{proxy_api_base}/proxy/proxy-list"
|
|
213
|
+
self._proxy_expiration_url = f"{proxy_api_base}/proxy/expiration-time"
|
|
214
|
+
self._list_url = f"{web_scraper_api_base}/tasks-list"
|
|
151
215
|
|
|
152
216
|
# Session initialized lazily
|
|
153
217
|
self._session: Optional[aiohttp.ClientSession] = None
|
|
@@ -156,7 +220,7 @@ class AsyncThordataClient:
|
|
|
156
220
|
"""Async context manager entry."""
|
|
157
221
|
if self._session is None or self._session.closed:
|
|
158
222
|
self._session = aiohttp.ClientSession(
|
|
159
|
-
timeout=self.
|
|
223
|
+
timeout=self._api_timeout,
|
|
160
224
|
trust_env=True,
|
|
161
225
|
headers={"User-Agent": build_user_agent(_sdk_version, "aiohttp")},
|
|
162
226
|
)
|
|
@@ -324,7 +388,7 @@ class AsyncThordataClient:
|
|
|
324
388
|
)
|
|
325
389
|
|
|
326
390
|
payload = request.to_payload()
|
|
327
|
-
headers = build_auth_headers(self.scraper_token)
|
|
391
|
+
headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
|
|
328
392
|
|
|
329
393
|
logger.info(f"Async SERP Search: {engine_str} - {query}")
|
|
330
394
|
|
|
@@ -372,7 +436,7 @@ class AsyncThordataClient:
|
|
|
372
436
|
session = self._get_session()
|
|
373
437
|
|
|
374
438
|
payload = request.to_payload()
|
|
375
|
-
headers = build_auth_headers(self.scraper_token)
|
|
439
|
+
headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
|
|
376
440
|
|
|
377
441
|
logger.info(f"Async SERP Advanced: {request.engine} - {request.query}")
|
|
378
442
|
|
|
@@ -466,7 +530,7 @@ class AsyncThordataClient:
|
|
|
466
530
|
session = self._get_session()
|
|
467
531
|
|
|
468
532
|
payload = request.to_payload()
|
|
469
|
-
headers = build_auth_headers(self.scraper_token)
|
|
533
|
+
headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
|
|
470
534
|
|
|
471
535
|
logger.info(f"Async Universal Scrape: {request.url}")
|
|
472
536
|
|
|
@@ -538,10 +602,16 @@ class AsyncThordataClient:
|
|
|
538
602
|
"""
|
|
539
603
|
Create a task using ScraperTaskConfig.
|
|
540
604
|
"""
|
|
605
|
+
self._require_public_credentials()
|
|
541
606
|
session = self._get_session()
|
|
542
607
|
|
|
543
608
|
payload = config.to_payload()
|
|
544
|
-
headers
|
|
609
|
+
# Builder needs 3 headers: token, key, Authorization Bearer
|
|
610
|
+
headers = build_builder_headers(
|
|
611
|
+
self.scraper_token,
|
|
612
|
+
self.public_token or "",
|
|
613
|
+
self.public_key or "",
|
|
614
|
+
)
|
|
545
615
|
|
|
546
616
|
logger.info(f"Async Task Creation: {config.spider_name}")
|
|
547
617
|
|
|
@@ -566,6 +636,75 @@ class AsyncThordataClient:
|
|
|
566
636
|
f"Task creation failed: {e}", original_error=e
|
|
567
637
|
) from e
|
|
568
638
|
|
|
639
|
+
async def create_video_task(
|
|
640
|
+
self,
|
|
641
|
+
file_name: str,
|
|
642
|
+
spider_id: str,
|
|
643
|
+
spider_name: str,
|
|
644
|
+
parameters: Dict[str, Any],
|
|
645
|
+
common_settings: CommonSettings,
|
|
646
|
+
) -> str:
|
|
647
|
+
"""
|
|
648
|
+
Create a YouTube video/audio download task.
|
|
649
|
+
"""
|
|
650
|
+
|
|
651
|
+
config = VideoTaskConfig(
|
|
652
|
+
file_name=file_name,
|
|
653
|
+
spider_id=spider_id,
|
|
654
|
+
spider_name=spider_name,
|
|
655
|
+
parameters=parameters,
|
|
656
|
+
common_settings=common_settings,
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
return await self.create_video_task_advanced(config)
|
|
660
|
+
|
|
661
|
+
async def create_video_task_advanced(self, config: VideoTaskConfig) -> str:
|
|
662
|
+
"""
|
|
663
|
+
Create a video task using VideoTaskConfig object.
|
|
664
|
+
"""
|
|
665
|
+
|
|
666
|
+
self._require_public_credentials()
|
|
667
|
+
session = self._get_session()
|
|
668
|
+
|
|
669
|
+
payload = config.to_payload()
|
|
670
|
+
headers = build_builder_headers(
|
|
671
|
+
self.scraper_token,
|
|
672
|
+
self.public_token or "",
|
|
673
|
+
self.public_key or "",
|
|
674
|
+
)
|
|
675
|
+
|
|
676
|
+
logger.info(
|
|
677
|
+
f"Async Video Task Creation: {config.spider_name} - {config.spider_id}"
|
|
678
|
+
)
|
|
679
|
+
|
|
680
|
+
try:
|
|
681
|
+
async with session.post(
|
|
682
|
+
self._video_builder_url,
|
|
683
|
+
data=payload,
|
|
684
|
+
headers=headers,
|
|
685
|
+
timeout=self._api_timeout,
|
|
686
|
+
) as response:
|
|
687
|
+
response.raise_for_status()
|
|
688
|
+
data = await response.json()
|
|
689
|
+
|
|
690
|
+
code = data.get("code")
|
|
691
|
+
if code != 200:
|
|
692
|
+
msg = extract_error_message(data)
|
|
693
|
+
raise_for_code(
|
|
694
|
+
f"Video task creation failed: {msg}", code=code, payload=data
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
return data["data"]["task_id"]
|
|
698
|
+
|
|
699
|
+
except asyncio.TimeoutError as e:
|
|
700
|
+
raise ThordataTimeoutError(
|
|
701
|
+
f"Video task creation timed out: {e}", original_error=e
|
|
702
|
+
) from e
|
|
703
|
+
except aiohttp.ClientError as e:
|
|
704
|
+
raise ThordataNetworkError(
|
|
705
|
+
f"Video task creation failed: {e}", original_error=e
|
|
706
|
+
) from e
|
|
707
|
+
|
|
569
708
|
async def get_task_status(self, task_id: str) -> str:
|
|
570
709
|
"""
|
|
571
710
|
Check async task status.
|
|
@@ -667,6 +806,61 @@ class AsyncThordataClient:
|
|
|
667
806
|
f"Get result failed: {e}", original_error=e
|
|
668
807
|
) from e
|
|
669
808
|
|
|
809
|
+
async def list_tasks(
|
|
810
|
+
self,
|
|
811
|
+
page: int = 1,
|
|
812
|
+
size: int = 20,
|
|
813
|
+
) -> Dict[str, Any]:
|
|
814
|
+
"""
|
|
815
|
+
List all Web Scraper tasks.
|
|
816
|
+
|
|
817
|
+
Args:
|
|
818
|
+
page: Page number (starts from 1).
|
|
819
|
+
size: Number of tasks per page.
|
|
820
|
+
|
|
821
|
+
Returns:
|
|
822
|
+
Dict containing 'count' and 'list' of tasks.
|
|
823
|
+
"""
|
|
824
|
+
self._require_public_credentials()
|
|
825
|
+
session = self._get_session()
|
|
826
|
+
|
|
827
|
+
headers = build_public_api_headers(
|
|
828
|
+
self.public_token or "", self.public_key or ""
|
|
829
|
+
)
|
|
830
|
+
payload: Dict[str, Any] = {}
|
|
831
|
+
if page:
|
|
832
|
+
payload["page"] = str(page)
|
|
833
|
+
if size:
|
|
834
|
+
payload["size"] = str(size)
|
|
835
|
+
|
|
836
|
+
logger.info(f"Async listing tasks: page={page}, size={size}")
|
|
837
|
+
|
|
838
|
+
try:
|
|
839
|
+
async with session.post(
|
|
840
|
+
self._list_url,
|
|
841
|
+
data=payload,
|
|
842
|
+
headers=headers,
|
|
843
|
+
timeout=self._api_timeout,
|
|
844
|
+
) as response:
|
|
845
|
+
response.raise_for_status()
|
|
846
|
+
data = await response.json()
|
|
847
|
+
|
|
848
|
+
code = data.get("code")
|
|
849
|
+
if code != 200:
|
|
850
|
+
msg = extract_error_message(data)
|
|
851
|
+
raise_for_code(f"List tasks failed: {msg}", code=code, payload=data)
|
|
852
|
+
|
|
853
|
+
return data.get("data", {"count": 0, "list": []})
|
|
854
|
+
|
|
855
|
+
except asyncio.TimeoutError as e:
|
|
856
|
+
raise ThordataTimeoutError(
|
|
857
|
+
f"List tasks timed out: {e}", original_error=e
|
|
858
|
+
) from e
|
|
859
|
+
except aiohttp.ClientError as e:
|
|
860
|
+
raise ThordataNetworkError(
|
|
861
|
+
f"List tasks failed: {e}", original_error=e
|
|
862
|
+
) from e
|
|
863
|
+
|
|
670
864
|
async def wait_for_task(
|
|
671
865
|
self,
|
|
672
866
|
task_id: str,
|
|
@@ -703,6 +897,591 @@ class AsyncThordataClient:
|
|
|
703
897
|
|
|
704
898
|
raise TimeoutError(f"Task {task_id} did not complete within {max_wait} seconds")
|
|
705
899
|
|
|
900
|
+
# =========================================================================
|
|
901
|
+
# Proxy Account Management Methods
|
|
902
|
+
# =========================================================================
|
|
903
|
+
|
|
904
|
+
async def get_usage_statistics(
|
|
905
|
+
self,
|
|
906
|
+
from_date: Union[str, date],
|
|
907
|
+
to_date: Union[str, date],
|
|
908
|
+
) -> UsageStatistics:
|
|
909
|
+
"""
|
|
910
|
+
Get account usage statistics for a date range.
|
|
911
|
+
|
|
912
|
+
Args:
|
|
913
|
+
from_date: Start date (YYYY-MM-DD string or date object).
|
|
914
|
+
to_date: End date (YYYY-MM-DD string or date object).
|
|
915
|
+
|
|
916
|
+
Returns:
|
|
917
|
+
UsageStatistics object with traffic data.
|
|
918
|
+
"""
|
|
919
|
+
|
|
920
|
+
self._require_public_credentials()
|
|
921
|
+
session = self._get_session()
|
|
922
|
+
|
|
923
|
+
# Convert dates to strings
|
|
924
|
+
if isinstance(from_date, date):
|
|
925
|
+
from_date = from_date.strftime("%Y-%m-%d")
|
|
926
|
+
if isinstance(to_date, date):
|
|
927
|
+
to_date = to_date.strftime("%Y-%m-%d")
|
|
928
|
+
|
|
929
|
+
params = {
|
|
930
|
+
"token": self.public_token,
|
|
931
|
+
"key": self.public_key,
|
|
932
|
+
"from_date": from_date,
|
|
933
|
+
"to_date": to_date,
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
logger.info(f"Async getting usage statistics: {from_date} to {to_date}")
|
|
937
|
+
|
|
938
|
+
try:
|
|
939
|
+
async with session.get(
|
|
940
|
+
self._usage_stats_url,
|
|
941
|
+
params=params,
|
|
942
|
+
timeout=self._api_timeout,
|
|
943
|
+
) as response:
|
|
944
|
+
response.raise_for_status()
|
|
945
|
+
data = await response.json()
|
|
946
|
+
|
|
947
|
+
if isinstance(data, dict):
|
|
948
|
+
code = data.get("code")
|
|
949
|
+
if code is not None and code != 200:
|
|
950
|
+
msg = extract_error_message(data)
|
|
951
|
+
raise_for_code(
|
|
952
|
+
f"Usage statistics error: {msg}",
|
|
953
|
+
code=code,
|
|
954
|
+
payload=data,
|
|
955
|
+
)
|
|
956
|
+
|
|
957
|
+
usage_data = data.get("data", data)
|
|
958
|
+
return UsageStatistics.from_dict(usage_data)
|
|
959
|
+
|
|
960
|
+
raise ThordataNetworkError(
|
|
961
|
+
f"Unexpected usage statistics response: {type(data).__name__}",
|
|
962
|
+
original_error=None,
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
except asyncio.TimeoutError as e:
|
|
966
|
+
raise ThordataTimeoutError(
|
|
967
|
+
f"Usage statistics timed out: {e}", original_error=e
|
|
968
|
+
) from e
|
|
969
|
+
except aiohttp.ClientError as e:
|
|
970
|
+
raise ThordataNetworkError(
|
|
971
|
+
f"Usage statistics failed: {e}", original_error=e
|
|
972
|
+
) from e
|
|
973
|
+
|
|
974
|
+
async def get_residential_balance(self) -> Dict[str, Any]:
|
|
975
|
+
"""
|
|
976
|
+
Get residential proxy balance (Public API NEW).
|
|
977
|
+
|
|
978
|
+
Requires sign and apiKey credentials.
|
|
979
|
+
|
|
980
|
+
Returns:
|
|
981
|
+
Dict with 'balance' (bytes) and 'expire_time' (timestamp).
|
|
982
|
+
"""
|
|
983
|
+
if not self.sign or not self.api_key:
|
|
984
|
+
raise ThordataConfigError(
|
|
985
|
+
"sign and api_key are required for Public API NEW. "
|
|
986
|
+
"Set THORDATA_SIGN and THORDATA_API_KEY environment variables."
|
|
987
|
+
)
|
|
988
|
+
|
|
989
|
+
session = self._get_session()
|
|
990
|
+
headers = build_sign_headers(self.sign, self.api_key)
|
|
991
|
+
|
|
992
|
+
logger.info("Async getting residential proxy balance (API NEW)")
|
|
993
|
+
|
|
994
|
+
try:
|
|
995
|
+
async with session.post(
|
|
996
|
+
f"{self._gateway_base_url}/getFlowBalance",
|
|
997
|
+
headers=headers,
|
|
998
|
+
data={},
|
|
999
|
+
timeout=self._api_timeout,
|
|
1000
|
+
) as response:
|
|
1001
|
+
response.raise_for_status()
|
|
1002
|
+
data = await response.json()
|
|
1003
|
+
|
|
1004
|
+
code = data.get("code")
|
|
1005
|
+
if code != 200:
|
|
1006
|
+
msg = extract_error_message(data)
|
|
1007
|
+
raise_for_code(
|
|
1008
|
+
f"Get balance failed: {msg}", code=code, payload=data
|
|
1009
|
+
)
|
|
1010
|
+
|
|
1011
|
+
return data.get("data", {})
|
|
1012
|
+
|
|
1013
|
+
except asyncio.TimeoutError as e:
|
|
1014
|
+
raise ThordataTimeoutError(
|
|
1015
|
+
f"Get balance timed out: {e}", original_error=e
|
|
1016
|
+
) from e
|
|
1017
|
+
except aiohttp.ClientError as e:
|
|
1018
|
+
raise ThordataNetworkError(
|
|
1019
|
+
f"Get balance failed: {e}", original_error=e
|
|
1020
|
+
) from e
|
|
1021
|
+
|
|
1022
|
+
async def get_residential_usage(
|
|
1023
|
+
self,
|
|
1024
|
+
start_time: Union[str, int],
|
|
1025
|
+
end_time: Union[str, int],
|
|
1026
|
+
) -> Dict[str, Any]:
|
|
1027
|
+
"""
|
|
1028
|
+
Get residential proxy usage records (Public API NEW).
|
|
1029
|
+
|
|
1030
|
+
Args:
|
|
1031
|
+
start_time: Start timestamp (Unix timestamp or YYYY-MM-DD HH:MM:SS).
|
|
1032
|
+
end_time: End timestamp (Unix timestamp or YYYY-MM-DD HH:MM:SS).
|
|
1033
|
+
|
|
1034
|
+
Returns:
|
|
1035
|
+
Dict with usage data including 'all_flow', 'all_used_flow', 'data' list.
|
|
1036
|
+
"""
|
|
1037
|
+
if not self.sign or not self.api_key:
|
|
1038
|
+
raise ThordataConfigError(
|
|
1039
|
+
"sign and api_key are required for Public API NEW."
|
|
1040
|
+
)
|
|
1041
|
+
|
|
1042
|
+
session = self._get_session()
|
|
1043
|
+
headers = build_sign_headers(self.sign, self.api_key)
|
|
1044
|
+
payload = {
|
|
1045
|
+
"start_time": str(start_time),
|
|
1046
|
+
"end_time": str(end_time),
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
logger.info(f"Async getting residential usage: {start_time} to {end_time}")
|
|
1050
|
+
|
|
1051
|
+
try:
|
|
1052
|
+
async with session.post(
|
|
1053
|
+
f"{self._gateway_base_url}/usageRecord",
|
|
1054
|
+
headers=headers,
|
|
1055
|
+
data=payload,
|
|
1056
|
+
timeout=self._api_timeout,
|
|
1057
|
+
) as response:
|
|
1058
|
+
response.raise_for_status()
|
|
1059
|
+
data = await response.json()
|
|
1060
|
+
|
|
1061
|
+
code = data.get("code")
|
|
1062
|
+
if code != 200:
|
|
1063
|
+
msg = extract_error_message(data)
|
|
1064
|
+
raise_for_code(f"Get usage failed: {msg}", code=code, payload=data)
|
|
1065
|
+
|
|
1066
|
+
return data.get("data", {})
|
|
1067
|
+
|
|
1068
|
+
except asyncio.TimeoutError as e:
|
|
1069
|
+
raise ThordataTimeoutError(
|
|
1070
|
+
f"Get usage timed out: {e}", original_error=e
|
|
1071
|
+
) from e
|
|
1072
|
+
except aiohttp.ClientError as e:
|
|
1073
|
+
raise ThordataNetworkError(
|
|
1074
|
+
f"Get usage failed: {e}", original_error=e
|
|
1075
|
+
) from e
|
|
1076
|
+
|
|
1077
|
+
async def list_proxy_users(
|
|
1078
|
+
self, proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL
|
|
1079
|
+
) -> ProxyUserList:
|
|
1080
|
+
"""List all proxy users (sub-accounts)."""
|
|
1081
|
+
|
|
1082
|
+
self._require_public_credentials()
|
|
1083
|
+
session = self._get_session()
|
|
1084
|
+
|
|
1085
|
+
params = {
|
|
1086
|
+
"token": self.public_token,
|
|
1087
|
+
"key": self.public_key,
|
|
1088
|
+
"proxy_type": str(
|
|
1089
|
+
int(proxy_type) if isinstance(proxy_type, ProxyType) else proxy_type
|
|
1090
|
+
),
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
logger.info(f"Async listing proxy users: type={params['proxy_type']}")
|
|
1094
|
+
|
|
1095
|
+
try:
|
|
1096
|
+
async with session.get(
|
|
1097
|
+
f"{self._proxy_users_url}/user-list",
|
|
1098
|
+
params=params,
|
|
1099
|
+
timeout=self._api_timeout,
|
|
1100
|
+
) as response:
|
|
1101
|
+
response.raise_for_status()
|
|
1102
|
+
data = await response.json()
|
|
1103
|
+
|
|
1104
|
+
if isinstance(data, dict):
|
|
1105
|
+
code = data.get("code")
|
|
1106
|
+
if code is not None and code != 200:
|
|
1107
|
+
msg = extract_error_message(data)
|
|
1108
|
+
raise_for_code(
|
|
1109
|
+
f"List proxy users error: {msg}", code=code, payload=data
|
|
1110
|
+
)
|
|
1111
|
+
|
|
1112
|
+
user_data = data.get("data", data)
|
|
1113
|
+
return ProxyUserList.from_dict(user_data)
|
|
1114
|
+
|
|
1115
|
+
raise ThordataNetworkError(
|
|
1116
|
+
f"Unexpected proxy users response: {type(data).__name__}",
|
|
1117
|
+
original_error=None,
|
|
1118
|
+
)
|
|
1119
|
+
|
|
1120
|
+
except asyncio.TimeoutError as e:
|
|
1121
|
+
raise ThordataTimeoutError(
|
|
1122
|
+
f"List users timed out: {e}", original_error=e
|
|
1123
|
+
) from e
|
|
1124
|
+
except aiohttp.ClientError as e:
|
|
1125
|
+
raise ThordataNetworkError(
|
|
1126
|
+
f"List users failed: {e}", original_error=e
|
|
1127
|
+
) from e
|
|
1128
|
+
|
|
1129
|
+
async def create_proxy_user(
|
|
1130
|
+
self,
|
|
1131
|
+
username: str,
|
|
1132
|
+
password: str,
|
|
1133
|
+
proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
|
|
1134
|
+
traffic_limit: int = 0,
|
|
1135
|
+
status: bool = True,
|
|
1136
|
+
) -> Dict[str, Any]:
|
|
1137
|
+
"""Create a new proxy user (sub-account)."""
|
|
1138
|
+
self._require_public_credentials()
|
|
1139
|
+
session = self._get_session()
|
|
1140
|
+
|
|
1141
|
+
headers = build_public_api_headers(
|
|
1142
|
+
self.public_token or "", self.public_key or ""
|
|
1143
|
+
)
|
|
1144
|
+
|
|
1145
|
+
payload = {
|
|
1146
|
+
"proxy_type": str(
|
|
1147
|
+
int(proxy_type) if isinstance(proxy_type, ProxyType) else proxy_type
|
|
1148
|
+
),
|
|
1149
|
+
"username": username,
|
|
1150
|
+
"password": password,
|
|
1151
|
+
"traffic_limit": str(traffic_limit),
|
|
1152
|
+
"status": "true" if status else "false",
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
logger.info(f"Async creating proxy user: {username}")
|
|
1156
|
+
|
|
1157
|
+
try:
|
|
1158
|
+
async with session.post(
|
|
1159
|
+
f"{self._proxy_users_url}/create-user",
|
|
1160
|
+
data=payload,
|
|
1161
|
+
headers=headers,
|
|
1162
|
+
timeout=self._api_timeout,
|
|
1163
|
+
) as response:
|
|
1164
|
+
response.raise_for_status()
|
|
1165
|
+
data = await response.json()
|
|
1166
|
+
|
|
1167
|
+
code = data.get("code")
|
|
1168
|
+
if code != 200:
|
|
1169
|
+
msg = extract_error_message(data)
|
|
1170
|
+
raise_for_code(
|
|
1171
|
+
f"Create proxy user failed: {msg}", code=code, payload=data
|
|
1172
|
+
)
|
|
1173
|
+
|
|
1174
|
+
return data.get("data", {})
|
|
1175
|
+
|
|
1176
|
+
except asyncio.TimeoutError as e:
|
|
1177
|
+
raise ThordataTimeoutError(
|
|
1178
|
+
f"Create user timed out: {e}", original_error=e
|
|
1179
|
+
) from e
|
|
1180
|
+
except aiohttp.ClientError as e:
|
|
1181
|
+
raise ThordataNetworkError(
|
|
1182
|
+
f"Create user failed: {e}", original_error=e
|
|
1183
|
+
) from e
|
|
1184
|
+
|
|
1185
|
+
async def add_whitelist_ip(
|
|
1186
|
+
self,
|
|
1187
|
+
ip: str,
|
|
1188
|
+
proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
|
|
1189
|
+
status: bool = True,
|
|
1190
|
+
) -> Dict[str, Any]:
|
|
1191
|
+
"""
|
|
1192
|
+
Add an IP to the whitelist for IP authentication.
|
|
1193
|
+
"""
|
|
1194
|
+
self._require_public_credentials()
|
|
1195
|
+
session = self._get_session()
|
|
1196
|
+
|
|
1197
|
+
headers = build_public_api_headers(
|
|
1198
|
+
self.public_token or "", self.public_key or ""
|
|
1199
|
+
)
|
|
1200
|
+
|
|
1201
|
+
proxy_type_int = (
|
|
1202
|
+
int(proxy_type) if isinstance(proxy_type, ProxyType) else proxy_type
|
|
1203
|
+
)
|
|
1204
|
+
|
|
1205
|
+
payload = {
|
|
1206
|
+
"proxy_type": str(proxy_type_int),
|
|
1207
|
+
"ip": ip,
|
|
1208
|
+
"status": "true" if status else "false",
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
logger.info(f"Async adding whitelist IP: {ip}")
|
|
1212
|
+
|
|
1213
|
+
try:
|
|
1214
|
+
async with session.post(
|
|
1215
|
+
f"{self._whitelist_url}/add-ip",
|
|
1216
|
+
data=payload,
|
|
1217
|
+
headers=headers,
|
|
1218
|
+
timeout=self._api_timeout,
|
|
1219
|
+
) as response:
|
|
1220
|
+
response.raise_for_status()
|
|
1221
|
+
data = await response.json()
|
|
1222
|
+
|
|
1223
|
+
code = data.get("code")
|
|
1224
|
+
if code != 200:
|
|
1225
|
+
msg = extract_error_message(data)
|
|
1226
|
+
raise_for_code(
|
|
1227
|
+
f"Add whitelist IP failed: {msg}", code=code, payload=data
|
|
1228
|
+
)
|
|
1229
|
+
|
|
1230
|
+
return data.get("data", {})
|
|
1231
|
+
|
|
1232
|
+
except asyncio.TimeoutError as e:
|
|
1233
|
+
raise ThordataTimeoutError(
|
|
1234
|
+
f"Add whitelist timed out: {e}", original_error=e
|
|
1235
|
+
) from e
|
|
1236
|
+
except aiohttp.ClientError as e:
|
|
1237
|
+
raise ThordataNetworkError(
|
|
1238
|
+
f"Add whitelist failed: {e}", original_error=e
|
|
1239
|
+
) from e
|
|
1240
|
+
|
|
1241
|
+
async def list_proxy_servers(
|
|
1242
|
+
self,
|
|
1243
|
+
proxy_type: int,
|
|
1244
|
+
) -> List[ProxyServer]:
|
|
1245
|
+
"""
|
|
1246
|
+
List ISP or Datacenter proxy servers.
|
|
1247
|
+
"""
|
|
1248
|
+
|
|
1249
|
+
self._require_public_credentials()
|
|
1250
|
+
session = self._get_session()
|
|
1251
|
+
|
|
1252
|
+
params = {
|
|
1253
|
+
"token": self.public_token,
|
|
1254
|
+
"key": self.public_key,
|
|
1255
|
+
"proxy_type": str(proxy_type),
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
logger.info(f"Async listing proxy servers: type={proxy_type}")
|
|
1259
|
+
|
|
1260
|
+
try:
|
|
1261
|
+
async with session.get(
|
|
1262
|
+
self._proxy_list_url,
|
|
1263
|
+
params=params,
|
|
1264
|
+
timeout=self._api_timeout,
|
|
1265
|
+
) as response:
|
|
1266
|
+
response.raise_for_status()
|
|
1267
|
+
data = await response.json()
|
|
1268
|
+
|
|
1269
|
+
if isinstance(data, dict):
|
|
1270
|
+
code = data.get("code")
|
|
1271
|
+
if code is not None and code != 200:
|
|
1272
|
+
msg = extract_error_message(data)
|
|
1273
|
+
raise_for_code(
|
|
1274
|
+
f"List proxy servers error: {msg}", code=code, payload=data
|
|
1275
|
+
)
|
|
1276
|
+
|
|
1277
|
+
server_list = data.get("data", data.get("list", []))
|
|
1278
|
+
elif isinstance(data, list):
|
|
1279
|
+
server_list = data
|
|
1280
|
+
else:
|
|
1281
|
+
raise ThordataNetworkError(
|
|
1282
|
+
f"Unexpected proxy list response: {type(data).__name__}",
|
|
1283
|
+
original_error=None,
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
return [ProxyServer.from_dict(s) for s in server_list]
|
|
1287
|
+
|
|
1288
|
+
except asyncio.TimeoutError as e:
|
|
1289
|
+
raise ThordataTimeoutError(
|
|
1290
|
+
f"List servers timed out: {e}", original_error=e
|
|
1291
|
+
) from e
|
|
1292
|
+
except aiohttp.ClientError as e:
|
|
1293
|
+
raise ThordataNetworkError(
|
|
1294
|
+
f"List servers failed: {e}", original_error=e
|
|
1295
|
+
) from e
|
|
1296
|
+
|
|
1297
|
+
async def get_isp_regions(self) -> List[Dict[str, Any]]:
|
|
1298
|
+
"""
|
|
1299
|
+
Get available ISP proxy regions (Public API NEW).
|
|
1300
|
+
|
|
1301
|
+
Returns:
|
|
1302
|
+
List of regions with id, continent, country, city, num, pricing.
|
|
1303
|
+
"""
|
|
1304
|
+
if not self.sign or not self.api_key:
|
|
1305
|
+
raise ThordataConfigError(
|
|
1306
|
+
"sign and api_key are required for Public API NEW."
|
|
1307
|
+
)
|
|
1308
|
+
|
|
1309
|
+
session = self._get_session()
|
|
1310
|
+
headers = build_sign_headers(self.sign, self.api_key)
|
|
1311
|
+
|
|
1312
|
+
logger.info("Async getting ISP regions (API NEW)")
|
|
1313
|
+
|
|
1314
|
+
try:
|
|
1315
|
+
async with session.post(
|
|
1316
|
+
f"{self._gateway_base_url}/getRegionIsp",
|
|
1317
|
+
headers=headers,
|
|
1318
|
+
data={},
|
|
1319
|
+
timeout=self._api_timeout,
|
|
1320
|
+
) as response:
|
|
1321
|
+
response.raise_for_status()
|
|
1322
|
+
data = await response.json()
|
|
1323
|
+
|
|
1324
|
+
code = data.get("code")
|
|
1325
|
+
if code != 200:
|
|
1326
|
+
msg = extract_error_message(data)
|
|
1327
|
+
raise_for_code(
|
|
1328
|
+
f"Get ISP regions failed: {msg}", code=code, payload=data
|
|
1329
|
+
)
|
|
1330
|
+
|
|
1331
|
+
return data.get("data", [])
|
|
1332
|
+
|
|
1333
|
+
except asyncio.TimeoutError as e:
|
|
1334
|
+
raise ThordataTimeoutError(
|
|
1335
|
+
f"Get ISP regions timed out: {e}", original_error=e
|
|
1336
|
+
) from e
|
|
1337
|
+
except aiohttp.ClientError as e:
|
|
1338
|
+
raise ThordataNetworkError(
|
|
1339
|
+
f"Get ISP regions failed: {e}", original_error=e
|
|
1340
|
+
) from e
|
|
1341
|
+
|
|
1342
|
+
async def list_isp_proxies(self) -> List[Dict[str, Any]]:
|
|
1343
|
+
"""
|
|
1344
|
+
List ISP proxies (Public API NEW).
|
|
1345
|
+
|
|
1346
|
+
Returns:
|
|
1347
|
+
List of ISP proxies with ip, port, user, pwd, startTime, expireTime.
|
|
1348
|
+
"""
|
|
1349
|
+
if not self.sign or not self.api_key:
|
|
1350
|
+
raise ThordataConfigError(
|
|
1351
|
+
"sign and api_key are required for Public API NEW."
|
|
1352
|
+
)
|
|
1353
|
+
|
|
1354
|
+
session = self._get_session()
|
|
1355
|
+
headers = build_sign_headers(self.sign, self.api_key)
|
|
1356
|
+
|
|
1357
|
+
logger.info("Async listing ISP proxies (API NEW)")
|
|
1358
|
+
|
|
1359
|
+
try:
|
|
1360
|
+
async with session.post(
|
|
1361
|
+
f"{self._gateway_base_url}/queryListIsp",
|
|
1362
|
+
headers=headers,
|
|
1363
|
+
data={},
|
|
1364
|
+
timeout=self._api_timeout,
|
|
1365
|
+
) as response:
|
|
1366
|
+
response.raise_for_status()
|
|
1367
|
+
data = await response.json()
|
|
1368
|
+
|
|
1369
|
+
code = data.get("code")
|
|
1370
|
+
if code != 200:
|
|
1371
|
+
msg = extract_error_message(data)
|
|
1372
|
+
raise_for_code(
|
|
1373
|
+
f"List ISP proxies failed: {msg}", code=code, payload=data
|
|
1374
|
+
)
|
|
1375
|
+
|
|
1376
|
+
return data.get("data", [])
|
|
1377
|
+
|
|
1378
|
+
except asyncio.TimeoutError as e:
|
|
1379
|
+
raise ThordataTimeoutError(
|
|
1380
|
+
f"List ISP proxies timed out: {e}", original_error=e
|
|
1381
|
+
) from e
|
|
1382
|
+
except aiohttp.ClientError as e:
|
|
1383
|
+
raise ThordataNetworkError(
|
|
1384
|
+
f"List ISP proxies failed: {e}", original_error=e
|
|
1385
|
+
) from e
|
|
1386
|
+
|
|
1387
|
+
async def get_wallet_balance(self) -> Dict[str, Any]:
|
|
1388
|
+
"""
|
|
1389
|
+
Get wallet balance for ISP proxies (Public API NEW).
|
|
1390
|
+
|
|
1391
|
+
Returns:
|
|
1392
|
+
Dict with 'walletBalance'.
|
|
1393
|
+
"""
|
|
1394
|
+
if not self.sign or not self.api_key:
|
|
1395
|
+
raise ThordataConfigError(
|
|
1396
|
+
"sign and api_key are required for Public API NEW."
|
|
1397
|
+
)
|
|
1398
|
+
|
|
1399
|
+
session = self._get_session()
|
|
1400
|
+
headers = build_sign_headers(self.sign, self.api_key)
|
|
1401
|
+
|
|
1402
|
+
logger.info("Async getting wallet balance (API NEW)")
|
|
1403
|
+
|
|
1404
|
+
try:
|
|
1405
|
+
async with session.post(
|
|
1406
|
+
f"{self._gateway_base_url}/getBalance",
|
|
1407
|
+
headers=headers,
|
|
1408
|
+
data={},
|
|
1409
|
+
timeout=self._api_timeout,
|
|
1410
|
+
) as response:
|
|
1411
|
+
response.raise_for_status()
|
|
1412
|
+
data = await response.json()
|
|
1413
|
+
|
|
1414
|
+
code = data.get("code")
|
|
1415
|
+
if code != 200:
|
|
1416
|
+
msg = extract_error_message(data)
|
|
1417
|
+
raise_for_code(
|
|
1418
|
+
f"Get wallet balance failed: {msg}", code=code, payload=data
|
|
1419
|
+
)
|
|
1420
|
+
|
|
1421
|
+
return data.get("data", {})
|
|
1422
|
+
|
|
1423
|
+
except asyncio.TimeoutError as e:
|
|
1424
|
+
raise ThordataTimeoutError(
|
|
1425
|
+
f"Get wallet balance timed out: {e}", original_error=e
|
|
1426
|
+
) from e
|
|
1427
|
+
except aiohttp.ClientError as e:
|
|
1428
|
+
raise ThordataNetworkError(
|
|
1429
|
+
f"Get wallet balance failed: {e}", original_error=e
|
|
1430
|
+
) from e
|
|
1431
|
+
|
|
1432
|
+
async def get_proxy_expiration(
|
|
1433
|
+
self,
|
|
1434
|
+
ips: Union[str, List[str]],
|
|
1435
|
+
proxy_type: int,
|
|
1436
|
+
) -> Dict[str, Any]:
|
|
1437
|
+
"""
|
|
1438
|
+
Get expiration time for specific proxy IPs.
|
|
1439
|
+
"""
|
|
1440
|
+
self._require_public_credentials()
|
|
1441
|
+
session = self._get_session()
|
|
1442
|
+
|
|
1443
|
+
if isinstance(ips, list):
|
|
1444
|
+
ips = ",".join(ips)
|
|
1445
|
+
|
|
1446
|
+
params = {
|
|
1447
|
+
"token": self.public_token,
|
|
1448
|
+
"key": self.public_key,
|
|
1449
|
+
"proxy_type": str(proxy_type),
|
|
1450
|
+
"ips": ips,
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
logger.info(f"Async getting proxy expiration: {ips}")
|
|
1454
|
+
|
|
1455
|
+
try:
|
|
1456
|
+
async with session.get(
|
|
1457
|
+
self._proxy_expiration_url,
|
|
1458
|
+
params=params,
|
|
1459
|
+
timeout=self._api_timeout,
|
|
1460
|
+
) as response:
|
|
1461
|
+
response.raise_for_status()
|
|
1462
|
+
data = await response.json()
|
|
1463
|
+
|
|
1464
|
+
if isinstance(data, dict):
|
|
1465
|
+
code = data.get("code")
|
|
1466
|
+
if code is not None and code != 200:
|
|
1467
|
+
msg = extract_error_message(data)
|
|
1468
|
+
raise_for_code(
|
|
1469
|
+
f"Get expiration error: {msg}", code=code, payload=data
|
|
1470
|
+
)
|
|
1471
|
+
|
|
1472
|
+
return data.get("data", data)
|
|
1473
|
+
|
|
1474
|
+
return data
|
|
1475
|
+
|
|
1476
|
+
except asyncio.TimeoutError as e:
|
|
1477
|
+
raise ThordataTimeoutError(
|
|
1478
|
+
f"Get expiration timed out: {e}", original_error=e
|
|
1479
|
+
) from e
|
|
1480
|
+
except aiohttp.ClientError as e:
|
|
1481
|
+
raise ThordataNetworkError(
|
|
1482
|
+
f"Get expiration failed: {e}", original_error=e
|
|
1483
|
+
) from e
|
|
1484
|
+
|
|
706
1485
|
# =========================================================================
|
|
707
1486
|
# Location API Methods
|
|
708
1487
|
# =========================================================================
|