thordata-sdk 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
thordata/async_client.py CHANGED
@@ -25,6 +25,7 @@ from __future__ import annotations
25
25
  import asyncio
26
26
  import logging
27
27
  import os
28
+ from datetime import date, datetime
28
29
  from typing import Any, Dict, List, Optional, Union
29
30
 
30
31
  import aiohttp
@@ -32,7 +33,9 @@ import aiohttp
32
33
  from . import __version__ as _sdk_version
33
34
  from ._utils import (
34
35
  build_auth_headers,
36
+ build_builder_headers,
35
37
  build_public_api_headers,
38
+ build_sign_headers,
36
39
  build_user_agent,
37
40
  decode_base64_image,
38
41
  extract_error_message,
@@ -45,7 +48,18 @@ from .exceptions import (
45
48
  ThordataTimeoutError,
46
49
  raise_for_code,
47
50
  )
48
- from .models import ProxyConfig, ScraperTaskConfig, SerpRequest, UniversalScrapeRequest
51
+ from .models import (
52
+ CommonSettings,
53
+ ProxyConfig,
54
+ ProxyServer,
55
+ ProxyUser,
56
+ ProxyUserList,
57
+ ScraperTaskConfig,
58
+ SerpRequest,
59
+ UniversalScrapeRequest,
60
+ UsageStatistics,
61
+ VideoTaskConfig,
62
+ )
49
63
  from .retry import RetryConfig
50
64
 
51
65
  logger = logging.getLogger(__name__)
@@ -78,18 +92,22 @@ class AsyncThordataClient:
78
92
  # API Endpoints (same as sync client)
79
93
  BASE_URL = "https://scraperapi.thordata.com"
80
94
  UNIVERSAL_URL = "https://universalapi.thordata.com"
81
- API_URL = "https://api.thordata.com/api/web-scraper-api"
82
- LOCATIONS_URL = "https://api.thordata.com/api/locations"
95
+ API_URL = "https://openapi.thordata.com/api/web-scraper-api"
96
+ LOCATIONS_URL = "https://openapi.thordata.com/api/locations"
83
97
 
84
98
  def __init__(
85
99
  self,
86
100
  scraper_token: str,
87
101
  public_token: Optional[str] = None,
88
102
  public_key: Optional[str] = None,
103
+ sign: Optional[str] = None,
104
+ api_key: Optional[str] = None,
89
105
  proxy_host: str = "pr.thordata.net",
90
106
  proxy_port: int = 9999,
91
107
  timeout: int = 30,
108
+ api_timeout: int = 60,
92
109
  retry_config: Optional[RetryConfig] = None,
110
+ auth_mode: str = "bearer",
93
111
  scraperapi_base_url: Optional[str] = None,
94
112
  universalapi_base_url: Optional[str] = None,
95
113
  web_scraper_api_base_url: Optional[str] = None,
@@ -103,14 +121,33 @@ class AsyncThordataClient:
103
121
  self.public_token = public_token
104
122
  self.public_key = public_key
105
123
 
124
+ # Automatic Fallback Logic: If sign/api_key is not provided, try using public_token/key
125
+ self.sign = sign or os.getenv("THORDATA_SIGN") or self.public_token
126
+ self.api_key = api_key or os.getenv("THORDATA_API_KEY") or self.public_key
127
+
128
+ # Public API authentication
129
+ self.sign = sign or os.getenv("THORDATA_SIGN")
130
+ self.api_key = api_key or os.getenv("THORDATA_API_KEY")
131
+
106
132
  # Proxy configuration
107
133
  self._proxy_host = proxy_host
108
134
  self._proxy_port = proxy_port
109
135
  self._default_timeout = aiohttp.ClientTimeout(total=timeout)
110
136
 
137
+ # Timeout configuration
138
+ self._default_timeout = aiohttp.ClientTimeout(total=timeout)
139
+ self._api_timeout = aiohttp.ClientTimeout(total=api_timeout)
140
+
111
141
  # Retry configuration
112
142
  self._retry_config = retry_config or RetryConfig()
113
143
 
144
+ # Authentication mode
145
+ self._auth_mode = auth_mode.lower()
146
+ if self._auth_mode not in ("bearer", "header_token"):
147
+ raise ThordataConfigError(
148
+ f"Invalid auth_mode: {auth_mode}. Must be 'bearer' or 'header_token'."
149
+ )
150
+
114
151
  # Pre-calculate proxy auth
115
152
  self._proxy_url = f"http://{proxy_host}:{proxy_port}"
116
153
  self._proxy_auth = aiohttp.BasicAuth(
@@ -142,12 +179,39 @@ class AsyncThordataClient:
142
179
  or self.LOCATIONS_URL
143
180
  ).rstrip("/")
144
181
 
182
+ gateway_base = os.getenv(
183
+ "THORDATA_GATEWAY_BASE_URL", "https://api.thordata.com/api/gateway"
184
+ )
185
+ child_base = os.getenv(
186
+ "THORDATA_CHILD_BASE_URL", "https://api.thordata.com/api/child"
187
+ )
188
+
189
+ self._gateway_base_url = gateway_base
190
+ self._child_base_url = child_base
191
+
145
192
  self._serp_url = f"{scraperapi_base}/request"
146
193
  self._builder_url = f"{scraperapi_base}/builder"
194
+ self._video_builder_url = f"{scraperapi_base}/video_builder"
147
195
  self._universal_url = f"{universalapi_base}/request"
148
196
  self._status_url = f"{web_scraper_api_base}/tasks-status"
149
197
  self._download_url = f"{web_scraper_api_base}/tasks-download"
150
198
  self._locations_base_url = locations_base
199
+ self._usage_stats_url = (
200
+ f"{locations_base.replace('/locations', '')}/account/usage-statistics"
201
+ )
202
+ self._proxy_users_url = (
203
+ f"{locations_base.replace('/locations', '')}/proxy-users"
204
+ )
205
+ whitelist_base = os.getenv(
206
+ "THORDATA_WHITELIST_BASE_URL", "https://api.thordata.com/api"
207
+ )
208
+ self._whitelist_url = f"{whitelist_base}/whitelisted-ips"
209
+ proxy_api_base = os.getenv(
210
+ "THORDATA_PROXY_API_BASE_URL", "https://api.thordata.com/api"
211
+ )
212
+ self._proxy_list_url = f"{proxy_api_base}/proxy/proxy-list"
213
+ self._proxy_expiration_url = f"{proxy_api_base}/proxy/expiration-time"
214
+ self._list_url = f"{web_scraper_api_base}/tasks-list"
151
215
 
152
216
  # Session initialized lazily
153
217
  self._session: Optional[aiohttp.ClientSession] = None
@@ -156,7 +220,7 @@ class AsyncThordataClient:
156
220
  """Async context manager entry."""
157
221
  if self._session is None or self._session.closed:
158
222
  self._session = aiohttp.ClientSession(
159
- timeout=self._default_timeout,
223
+ timeout=self._api_timeout,
160
224
  trust_env=True,
161
225
  headers={"User-Agent": build_user_agent(_sdk_version, "aiohttp")},
162
226
  )
@@ -324,7 +388,7 @@ class AsyncThordataClient:
324
388
  )
325
389
 
326
390
  payload = request.to_payload()
327
- headers = build_auth_headers(self.scraper_token)
391
+ headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
328
392
 
329
393
  logger.info(f"Async SERP Search: {engine_str} - {query}")
330
394
 
@@ -372,7 +436,7 @@ class AsyncThordataClient:
372
436
  session = self._get_session()
373
437
 
374
438
  payload = request.to_payload()
375
- headers = build_auth_headers(self.scraper_token)
439
+ headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
376
440
 
377
441
  logger.info(f"Async SERP Advanced: {request.engine} - {request.query}")
378
442
 
@@ -466,7 +530,7 @@ class AsyncThordataClient:
466
530
  session = self._get_session()
467
531
 
468
532
  payload = request.to_payload()
469
- headers = build_auth_headers(self.scraper_token)
533
+ headers = build_auth_headers(self.scraper_token, mode=self._auth_mode)
470
534
 
471
535
  logger.info(f"Async Universal Scrape: {request.url}")
472
536
 
@@ -538,10 +602,16 @@ class AsyncThordataClient:
538
602
  """
539
603
  Create a task using ScraperTaskConfig.
540
604
  """
605
+ self._require_public_credentials()
541
606
  session = self._get_session()
542
607
 
543
608
  payload = config.to_payload()
544
- headers = build_auth_headers(self.scraper_token)
609
+ # Builder needs 3 headers: token, key, Authorization Bearer
610
+ headers = build_builder_headers(
611
+ self.scraper_token,
612
+ self.public_token or "",
613
+ self.public_key or "",
614
+ )
545
615
 
546
616
  logger.info(f"Async Task Creation: {config.spider_name}")
547
617
 
@@ -566,6 +636,75 @@ class AsyncThordataClient:
566
636
  f"Task creation failed: {e}", original_error=e
567
637
  ) from e
568
638
 
639
+ async def create_video_task(
640
+ self,
641
+ file_name: str,
642
+ spider_id: str,
643
+ spider_name: str,
644
+ parameters: Dict[str, Any],
645
+ common_settings: CommonSettings,
646
+ ) -> str:
647
+ """
648
+ Create a YouTube video/audio download task.
649
+ """
650
+
651
+ config = VideoTaskConfig(
652
+ file_name=file_name,
653
+ spider_id=spider_id,
654
+ spider_name=spider_name,
655
+ parameters=parameters,
656
+ common_settings=common_settings,
657
+ )
658
+
659
+ return await self.create_video_task_advanced(config)
660
+
661
+ async def create_video_task_advanced(self, config: VideoTaskConfig) -> str:
662
+ """
663
+ Create a video task using VideoTaskConfig object.
664
+ """
665
+
666
+ self._require_public_credentials()
667
+ session = self._get_session()
668
+
669
+ payload = config.to_payload()
670
+ headers = build_builder_headers(
671
+ self.scraper_token,
672
+ self.public_token or "",
673
+ self.public_key or "",
674
+ )
675
+
676
+ logger.info(
677
+ f"Async Video Task Creation: {config.spider_name} - {config.spider_id}"
678
+ )
679
+
680
+ try:
681
+ async with session.post(
682
+ self._video_builder_url,
683
+ data=payload,
684
+ headers=headers,
685
+ timeout=self._api_timeout,
686
+ ) as response:
687
+ response.raise_for_status()
688
+ data = await response.json()
689
+
690
+ code = data.get("code")
691
+ if code != 200:
692
+ msg = extract_error_message(data)
693
+ raise_for_code(
694
+ f"Video task creation failed: {msg}", code=code, payload=data
695
+ )
696
+
697
+ return data["data"]["task_id"]
698
+
699
+ except asyncio.TimeoutError as e:
700
+ raise ThordataTimeoutError(
701
+ f"Video task creation timed out: {e}", original_error=e
702
+ ) from e
703
+ except aiohttp.ClientError as e:
704
+ raise ThordataNetworkError(
705
+ f"Video task creation failed: {e}", original_error=e
706
+ ) from e
707
+
569
708
  async def get_task_status(self, task_id: str) -> str:
570
709
  """
571
710
  Check async task status.
@@ -667,6 +806,61 @@ class AsyncThordataClient:
667
806
  f"Get result failed: {e}", original_error=e
668
807
  ) from e
669
808
 
809
+ async def list_tasks(
810
+ self,
811
+ page: int = 1,
812
+ size: int = 20,
813
+ ) -> Dict[str, Any]:
814
+ """
815
+ List all Web Scraper tasks.
816
+
817
+ Args:
818
+ page: Page number (starts from 1).
819
+ size: Number of tasks per page.
820
+
821
+ Returns:
822
+ Dict containing 'count' and 'list' of tasks.
823
+ """
824
+ self._require_public_credentials()
825
+ session = self._get_session()
826
+
827
+ headers = build_public_api_headers(
828
+ self.public_token or "", self.public_key or ""
829
+ )
830
+ payload: Dict[str, Any] = {}
831
+ if page:
832
+ payload["page"] = str(page)
833
+ if size:
834
+ payload["size"] = str(size)
835
+
836
+ logger.info(f"Async listing tasks: page={page}, size={size}")
837
+
838
+ try:
839
+ async with session.post(
840
+ self._list_url,
841
+ data=payload,
842
+ headers=headers,
843
+ timeout=self._api_timeout,
844
+ ) as response:
845
+ response.raise_for_status()
846
+ data = await response.json()
847
+
848
+ code = data.get("code")
849
+ if code != 200:
850
+ msg = extract_error_message(data)
851
+ raise_for_code(f"List tasks failed: {msg}", code=code, payload=data)
852
+
853
+ return data.get("data", {"count": 0, "list": []})
854
+
855
+ except asyncio.TimeoutError as e:
856
+ raise ThordataTimeoutError(
857
+ f"List tasks timed out: {e}", original_error=e
858
+ ) from e
859
+ except aiohttp.ClientError as e:
860
+ raise ThordataNetworkError(
861
+ f"List tasks failed: {e}", original_error=e
862
+ ) from e
863
+
670
864
  async def wait_for_task(
671
865
  self,
672
866
  task_id: str,
@@ -703,6 +897,591 @@ class AsyncThordataClient:
703
897
 
704
898
  raise TimeoutError(f"Task {task_id} did not complete within {max_wait} seconds")
705
899
 
900
+ # =========================================================================
901
+ # Proxy Account Management Methods
902
+ # =========================================================================
903
+
904
+ async def get_usage_statistics(
905
+ self,
906
+ from_date: Union[str, date],
907
+ to_date: Union[str, date],
908
+ ) -> UsageStatistics:
909
+ """
910
+ Get account usage statistics for a date range.
911
+
912
+ Args:
913
+ from_date: Start date (YYYY-MM-DD string or date object).
914
+ to_date: End date (YYYY-MM-DD string or date object).
915
+
916
+ Returns:
917
+ UsageStatistics object with traffic data.
918
+ """
919
+
920
+ self._require_public_credentials()
921
+ session = self._get_session()
922
+
923
+ # Convert dates to strings
924
+ if isinstance(from_date, date):
925
+ from_date = from_date.strftime("%Y-%m-%d")
926
+ if isinstance(to_date, date):
927
+ to_date = to_date.strftime("%Y-%m-%d")
928
+
929
+ params = {
930
+ "token": self.public_token,
931
+ "key": self.public_key,
932
+ "from_date": from_date,
933
+ "to_date": to_date,
934
+ }
935
+
936
+ logger.info(f"Async getting usage statistics: {from_date} to {to_date}")
937
+
938
+ try:
939
+ async with session.get(
940
+ self._usage_stats_url,
941
+ params=params,
942
+ timeout=self._api_timeout,
943
+ ) as response:
944
+ response.raise_for_status()
945
+ data = await response.json()
946
+
947
+ if isinstance(data, dict):
948
+ code = data.get("code")
949
+ if code is not None and code != 200:
950
+ msg = extract_error_message(data)
951
+ raise_for_code(
952
+ f"Usage statistics error: {msg}",
953
+ code=code,
954
+ payload=data,
955
+ )
956
+
957
+ usage_data = data.get("data", data)
958
+ return UsageStatistics.from_dict(usage_data)
959
+
960
+ raise ThordataNetworkError(
961
+ f"Unexpected usage statistics response: {type(data).__name__}",
962
+ original_error=None,
963
+ )
964
+
965
+ except asyncio.TimeoutError as e:
966
+ raise ThordataTimeoutError(
967
+ f"Usage statistics timed out: {e}", original_error=e
968
+ ) from e
969
+ except aiohttp.ClientError as e:
970
+ raise ThordataNetworkError(
971
+ f"Usage statistics failed: {e}", original_error=e
972
+ ) from e
973
+
974
+ async def get_residential_balance(self) -> Dict[str, Any]:
975
+ """
976
+ Get residential proxy balance (Public API NEW).
977
+
978
+ Requires sign and apiKey credentials.
979
+
980
+ Returns:
981
+ Dict with 'balance' (bytes) and 'expire_time' (timestamp).
982
+ """
983
+ if not self.sign or not self.api_key:
984
+ raise ThordataConfigError(
985
+ "sign and api_key are required for Public API NEW. "
986
+ "Set THORDATA_SIGN and THORDATA_API_KEY environment variables."
987
+ )
988
+
989
+ session = self._get_session()
990
+ headers = build_sign_headers(self.sign, self.api_key)
991
+
992
+ logger.info("Async getting residential proxy balance (API NEW)")
993
+
994
+ try:
995
+ async with session.post(
996
+ f"{self._gateway_base_url}/getFlowBalance",
997
+ headers=headers,
998
+ data={},
999
+ timeout=self._api_timeout,
1000
+ ) as response:
1001
+ response.raise_for_status()
1002
+ data = await response.json()
1003
+
1004
+ code = data.get("code")
1005
+ if code != 200:
1006
+ msg = extract_error_message(data)
1007
+ raise_for_code(
1008
+ f"Get balance failed: {msg}", code=code, payload=data
1009
+ )
1010
+
1011
+ return data.get("data", {})
1012
+
1013
+ except asyncio.TimeoutError as e:
1014
+ raise ThordataTimeoutError(
1015
+ f"Get balance timed out: {e}", original_error=e
1016
+ ) from e
1017
+ except aiohttp.ClientError as e:
1018
+ raise ThordataNetworkError(
1019
+ f"Get balance failed: {e}", original_error=e
1020
+ ) from e
1021
+
1022
+ async def get_residential_usage(
1023
+ self,
1024
+ start_time: Union[str, int],
1025
+ end_time: Union[str, int],
1026
+ ) -> Dict[str, Any]:
1027
+ """
1028
+ Get residential proxy usage records (Public API NEW).
1029
+
1030
+ Args:
1031
+ start_time: Start timestamp (Unix timestamp or YYYY-MM-DD HH:MM:SS).
1032
+ end_time: End timestamp (Unix timestamp or YYYY-MM-DD HH:MM:SS).
1033
+
1034
+ Returns:
1035
+ Dict with usage data including 'all_flow', 'all_used_flow', 'data' list.
1036
+ """
1037
+ if not self.sign or not self.api_key:
1038
+ raise ThordataConfigError(
1039
+ "sign and api_key are required for Public API NEW."
1040
+ )
1041
+
1042
+ session = self._get_session()
1043
+ headers = build_sign_headers(self.sign, self.api_key)
1044
+ payload = {
1045
+ "start_time": str(start_time),
1046
+ "end_time": str(end_time),
1047
+ }
1048
+
1049
+ logger.info(f"Async getting residential usage: {start_time} to {end_time}")
1050
+
1051
+ try:
1052
+ async with session.post(
1053
+ f"{self._gateway_base_url}/usageRecord",
1054
+ headers=headers,
1055
+ data=payload,
1056
+ timeout=self._api_timeout,
1057
+ ) as response:
1058
+ response.raise_for_status()
1059
+ data = await response.json()
1060
+
1061
+ code = data.get("code")
1062
+ if code != 200:
1063
+ msg = extract_error_message(data)
1064
+ raise_for_code(f"Get usage failed: {msg}", code=code, payload=data)
1065
+
1066
+ return data.get("data", {})
1067
+
1068
+ except asyncio.TimeoutError as e:
1069
+ raise ThordataTimeoutError(
1070
+ f"Get usage timed out: {e}", original_error=e
1071
+ ) from e
1072
+ except aiohttp.ClientError as e:
1073
+ raise ThordataNetworkError(
1074
+ f"Get usage failed: {e}", original_error=e
1075
+ ) from e
1076
+
1077
+ async def list_proxy_users(
1078
+ self, proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL
1079
+ ) -> ProxyUserList:
1080
+ """List all proxy users (sub-accounts)."""
1081
+
1082
+ self._require_public_credentials()
1083
+ session = self._get_session()
1084
+
1085
+ params = {
1086
+ "token": self.public_token,
1087
+ "key": self.public_key,
1088
+ "proxy_type": str(
1089
+ int(proxy_type) if isinstance(proxy_type, ProxyType) else proxy_type
1090
+ ),
1091
+ }
1092
+
1093
+ logger.info(f"Async listing proxy users: type={params['proxy_type']}")
1094
+
1095
+ try:
1096
+ async with session.get(
1097
+ f"{self._proxy_users_url}/user-list",
1098
+ params=params,
1099
+ timeout=self._api_timeout,
1100
+ ) as response:
1101
+ response.raise_for_status()
1102
+ data = await response.json()
1103
+
1104
+ if isinstance(data, dict):
1105
+ code = data.get("code")
1106
+ if code is not None and code != 200:
1107
+ msg = extract_error_message(data)
1108
+ raise_for_code(
1109
+ f"List proxy users error: {msg}", code=code, payload=data
1110
+ )
1111
+
1112
+ user_data = data.get("data", data)
1113
+ return ProxyUserList.from_dict(user_data)
1114
+
1115
+ raise ThordataNetworkError(
1116
+ f"Unexpected proxy users response: {type(data).__name__}",
1117
+ original_error=None,
1118
+ )
1119
+
1120
+ except asyncio.TimeoutError as e:
1121
+ raise ThordataTimeoutError(
1122
+ f"List users timed out: {e}", original_error=e
1123
+ ) from e
1124
+ except aiohttp.ClientError as e:
1125
+ raise ThordataNetworkError(
1126
+ f"List users failed: {e}", original_error=e
1127
+ ) from e
1128
+
1129
+ async def create_proxy_user(
1130
+ self,
1131
+ username: str,
1132
+ password: str,
1133
+ proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
1134
+ traffic_limit: int = 0,
1135
+ status: bool = True,
1136
+ ) -> Dict[str, Any]:
1137
+ """Create a new proxy user (sub-account)."""
1138
+ self._require_public_credentials()
1139
+ session = self._get_session()
1140
+
1141
+ headers = build_public_api_headers(
1142
+ self.public_token or "", self.public_key or ""
1143
+ )
1144
+
1145
+ payload = {
1146
+ "proxy_type": str(
1147
+ int(proxy_type) if isinstance(proxy_type, ProxyType) else proxy_type
1148
+ ),
1149
+ "username": username,
1150
+ "password": password,
1151
+ "traffic_limit": str(traffic_limit),
1152
+ "status": "true" if status else "false",
1153
+ }
1154
+
1155
+ logger.info(f"Async creating proxy user: {username}")
1156
+
1157
+ try:
1158
+ async with session.post(
1159
+ f"{self._proxy_users_url}/create-user",
1160
+ data=payload,
1161
+ headers=headers,
1162
+ timeout=self._api_timeout,
1163
+ ) as response:
1164
+ response.raise_for_status()
1165
+ data = await response.json()
1166
+
1167
+ code = data.get("code")
1168
+ if code != 200:
1169
+ msg = extract_error_message(data)
1170
+ raise_for_code(
1171
+ f"Create proxy user failed: {msg}", code=code, payload=data
1172
+ )
1173
+
1174
+ return data.get("data", {})
1175
+
1176
+ except asyncio.TimeoutError as e:
1177
+ raise ThordataTimeoutError(
1178
+ f"Create user timed out: {e}", original_error=e
1179
+ ) from e
1180
+ except aiohttp.ClientError as e:
1181
+ raise ThordataNetworkError(
1182
+ f"Create user failed: {e}", original_error=e
1183
+ ) from e
1184
+
1185
+ async def add_whitelist_ip(
1186
+ self,
1187
+ ip: str,
1188
+ proxy_type: Union[ProxyType, int] = ProxyType.RESIDENTIAL,
1189
+ status: bool = True,
1190
+ ) -> Dict[str, Any]:
1191
+ """
1192
+ Add an IP to the whitelist for IP authentication.
1193
+ """
1194
+ self._require_public_credentials()
1195
+ session = self._get_session()
1196
+
1197
+ headers = build_public_api_headers(
1198
+ self.public_token or "", self.public_key or ""
1199
+ )
1200
+
1201
+ proxy_type_int = (
1202
+ int(proxy_type) if isinstance(proxy_type, ProxyType) else proxy_type
1203
+ )
1204
+
1205
+ payload = {
1206
+ "proxy_type": str(proxy_type_int),
1207
+ "ip": ip,
1208
+ "status": "true" if status else "false",
1209
+ }
1210
+
1211
+ logger.info(f"Async adding whitelist IP: {ip}")
1212
+
1213
+ try:
1214
+ async with session.post(
1215
+ f"{self._whitelist_url}/add-ip",
1216
+ data=payload,
1217
+ headers=headers,
1218
+ timeout=self._api_timeout,
1219
+ ) as response:
1220
+ response.raise_for_status()
1221
+ data = await response.json()
1222
+
1223
+ code = data.get("code")
1224
+ if code != 200:
1225
+ msg = extract_error_message(data)
1226
+ raise_for_code(
1227
+ f"Add whitelist IP failed: {msg}", code=code, payload=data
1228
+ )
1229
+
1230
+ return data.get("data", {})
1231
+
1232
+ except asyncio.TimeoutError as e:
1233
+ raise ThordataTimeoutError(
1234
+ f"Add whitelist timed out: {e}", original_error=e
1235
+ ) from e
1236
+ except aiohttp.ClientError as e:
1237
+ raise ThordataNetworkError(
1238
+ f"Add whitelist failed: {e}", original_error=e
1239
+ ) from e
1240
+
1241
+ async def list_proxy_servers(
1242
+ self,
1243
+ proxy_type: int,
1244
+ ) -> List[ProxyServer]:
1245
+ """
1246
+ List ISP or Datacenter proxy servers.
1247
+ """
1248
+
1249
+ self._require_public_credentials()
1250
+ session = self._get_session()
1251
+
1252
+ params = {
1253
+ "token": self.public_token,
1254
+ "key": self.public_key,
1255
+ "proxy_type": str(proxy_type),
1256
+ }
1257
+
1258
+ logger.info(f"Async listing proxy servers: type={proxy_type}")
1259
+
1260
+ try:
1261
+ async with session.get(
1262
+ self._proxy_list_url,
1263
+ params=params,
1264
+ timeout=self._api_timeout,
1265
+ ) as response:
1266
+ response.raise_for_status()
1267
+ data = await response.json()
1268
+
1269
+ if isinstance(data, dict):
1270
+ code = data.get("code")
1271
+ if code is not None and code != 200:
1272
+ msg = extract_error_message(data)
1273
+ raise_for_code(
1274
+ f"List proxy servers error: {msg}", code=code, payload=data
1275
+ )
1276
+
1277
+ server_list = data.get("data", data.get("list", []))
1278
+ elif isinstance(data, list):
1279
+ server_list = data
1280
+ else:
1281
+ raise ThordataNetworkError(
1282
+ f"Unexpected proxy list response: {type(data).__name__}",
1283
+ original_error=None,
1284
+ )
1285
+
1286
+ return [ProxyServer.from_dict(s) for s in server_list]
1287
+
1288
+ except asyncio.TimeoutError as e:
1289
+ raise ThordataTimeoutError(
1290
+ f"List servers timed out: {e}", original_error=e
1291
+ ) from e
1292
+ except aiohttp.ClientError as e:
1293
+ raise ThordataNetworkError(
1294
+ f"List servers failed: {e}", original_error=e
1295
+ ) from e
1296
+
1297
+ async def get_isp_regions(self) -> List[Dict[str, Any]]:
1298
+ """
1299
+ Get available ISP proxy regions (Public API NEW).
1300
+
1301
+ Returns:
1302
+ List of regions with id, continent, country, city, num, pricing.
1303
+ """
1304
+ if not self.sign or not self.api_key:
1305
+ raise ThordataConfigError(
1306
+ "sign and api_key are required for Public API NEW."
1307
+ )
1308
+
1309
+ session = self._get_session()
1310
+ headers = build_sign_headers(self.sign, self.api_key)
1311
+
1312
+ logger.info("Async getting ISP regions (API NEW)")
1313
+
1314
+ try:
1315
+ async with session.post(
1316
+ f"{self._gateway_base_url}/getRegionIsp",
1317
+ headers=headers,
1318
+ data={},
1319
+ timeout=self._api_timeout,
1320
+ ) as response:
1321
+ response.raise_for_status()
1322
+ data = await response.json()
1323
+
1324
+ code = data.get("code")
1325
+ if code != 200:
1326
+ msg = extract_error_message(data)
1327
+ raise_for_code(
1328
+ f"Get ISP regions failed: {msg}", code=code, payload=data
1329
+ )
1330
+
1331
+ return data.get("data", [])
1332
+
1333
+ except asyncio.TimeoutError as e:
1334
+ raise ThordataTimeoutError(
1335
+ f"Get ISP regions timed out: {e}", original_error=e
1336
+ ) from e
1337
+ except aiohttp.ClientError as e:
1338
+ raise ThordataNetworkError(
1339
+ f"Get ISP regions failed: {e}", original_error=e
1340
+ ) from e
1341
+
1342
+ async def list_isp_proxies(self) -> List[Dict[str, Any]]:
1343
+ """
1344
+ List ISP proxies (Public API NEW).
1345
+
1346
+ Returns:
1347
+ List of ISP proxies with ip, port, user, pwd, startTime, expireTime.
1348
+ """
1349
+ if not self.sign or not self.api_key:
1350
+ raise ThordataConfigError(
1351
+ "sign and api_key are required for Public API NEW."
1352
+ )
1353
+
1354
+ session = self._get_session()
1355
+ headers = build_sign_headers(self.sign, self.api_key)
1356
+
1357
+ logger.info("Async listing ISP proxies (API NEW)")
1358
+
1359
+ try:
1360
+ async with session.post(
1361
+ f"{self._gateway_base_url}/queryListIsp",
1362
+ headers=headers,
1363
+ data={},
1364
+ timeout=self._api_timeout,
1365
+ ) as response:
1366
+ response.raise_for_status()
1367
+ data = await response.json()
1368
+
1369
+ code = data.get("code")
1370
+ if code != 200:
1371
+ msg = extract_error_message(data)
1372
+ raise_for_code(
1373
+ f"List ISP proxies failed: {msg}", code=code, payload=data
1374
+ )
1375
+
1376
+ return data.get("data", [])
1377
+
1378
+ except asyncio.TimeoutError as e:
1379
+ raise ThordataTimeoutError(
1380
+ f"List ISP proxies timed out: {e}", original_error=e
1381
+ ) from e
1382
+ except aiohttp.ClientError as e:
1383
+ raise ThordataNetworkError(
1384
+ f"List ISP proxies failed: {e}", original_error=e
1385
+ ) from e
1386
+
1387
+ async def get_wallet_balance(self) -> Dict[str, Any]:
1388
+ """
1389
+ Get wallet balance for ISP proxies (Public API NEW).
1390
+
1391
+ Returns:
1392
+ Dict with 'walletBalance'.
1393
+ """
1394
+ if not self.sign or not self.api_key:
1395
+ raise ThordataConfigError(
1396
+ "sign and api_key are required for Public API NEW."
1397
+ )
1398
+
1399
+ session = self._get_session()
1400
+ headers = build_sign_headers(self.sign, self.api_key)
1401
+
1402
+ logger.info("Async getting wallet balance (API NEW)")
1403
+
1404
+ try:
1405
+ async with session.post(
1406
+ f"{self._gateway_base_url}/getBalance",
1407
+ headers=headers,
1408
+ data={},
1409
+ timeout=self._api_timeout,
1410
+ ) as response:
1411
+ response.raise_for_status()
1412
+ data = await response.json()
1413
+
1414
+ code = data.get("code")
1415
+ if code != 200:
1416
+ msg = extract_error_message(data)
1417
+ raise_for_code(
1418
+ f"Get wallet balance failed: {msg}", code=code, payload=data
1419
+ )
1420
+
1421
+ return data.get("data", {})
1422
+
1423
+ except asyncio.TimeoutError as e:
1424
+ raise ThordataTimeoutError(
1425
+ f"Get wallet balance timed out: {e}", original_error=e
1426
+ ) from e
1427
+ except aiohttp.ClientError as e:
1428
+ raise ThordataNetworkError(
1429
+ f"Get wallet balance failed: {e}", original_error=e
1430
+ ) from e
1431
+
1432
+ async def get_proxy_expiration(
1433
+ self,
1434
+ ips: Union[str, List[str]],
1435
+ proxy_type: int,
1436
+ ) -> Dict[str, Any]:
1437
+ """
1438
+ Get expiration time for specific proxy IPs.
1439
+ """
1440
+ self._require_public_credentials()
1441
+ session = self._get_session()
1442
+
1443
+ if isinstance(ips, list):
1444
+ ips = ",".join(ips)
1445
+
1446
+ params = {
1447
+ "token": self.public_token,
1448
+ "key": self.public_key,
1449
+ "proxy_type": str(proxy_type),
1450
+ "ips": ips,
1451
+ }
1452
+
1453
+ logger.info(f"Async getting proxy expiration: {ips}")
1454
+
1455
+ try:
1456
+ async with session.get(
1457
+ self._proxy_expiration_url,
1458
+ params=params,
1459
+ timeout=self._api_timeout,
1460
+ ) as response:
1461
+ response.raise_for_status()
1462
+ data = await response.json()
1463
+
1464
+ if isinstance(data, dict):
1465
+ code = data.get("code")
1466
+ if code is not None and code != 200:
1467
+ msg = extract_error_message(data)
1468
+ raise_for_code(
1469
+ f"Get expiration error: {msg}", code=code, payload=data
1470
+ )
1471
+
1472
+ return data.get("data", data)
1473
+
1474
+ return data
1475
+
1476
+ except asyncio.TimeoutError as e:
1477
+ raise ThordataTimeoutError(
1478
+ f"Get expiration timed out: {e}", original_error=e
1479
+ ) from e
1480
+ except aiohttp.ClientError as e:
1481
+ raise ThordataNetworkError(
1482
+ f"Get expiration failed: {e}", original_error=e
1483
+ ) from e
1484
+
706
1485
  # =========================================================================
707
1486
  # Location API Methods
708
1487
  # =========================================================================