dataapp-resource-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataapp_resource_proxy/__init__.py +15 -0
- dataapp_resource_proxy/__main__.py +65 -0
- dataapp_resource_proxy/_local_config.py +286 -0
- dataapp_resource_proxy/client.py +1813 -0
- dataapp_resource_proxy/errors.py +16 -0
- dataapp_resource_proxy/handles.py +199 -0
- dataapp_resource_proxy/py.typed +0 -0
- dataapp_resource_proxy-0.1.0.dist-info/METADATA +6 -0
- dataapp_resource_proxy-0.1.0.dist-info/RECORD +10 -0
- dataapp_resource_proxy-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,1813 @@
|
|
|
1
|
+
"""Resource Proxy client with local and platform-provided transports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import copy
|
|
7
|
+
import hashlib
|
|
8
|
+
import inspect
|
|
9
|
+
import json as jsonlib
|
|
10
|
+
import logging
|
|
11
|
+
import math as mathlib
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import uuid
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
from datetime import datetime
|
|
17
|
+
from time import perf_counter
|
|
18
|
+
from typing import Any, Literal, Protocol
|
|
19
|
+
|
|
20
|
+
import httpx
|
|
21
|
+
|
|
22
|
+
from ._local_config import (
|
|
23
|
+
load_local_config,
|
|
24
|
+
load_project_config,
|
|
25
|
+
resolve_config_environment,
|
|
26
|
+
)
|
|
27
|
+
from .errors import ResourceProxyError
|
|
28
|
+
from .handles import (
|
|
29
|
+
BrowserResource,
|
|
30
|
+
CaptchaPluginGrant,
|
|
31
|
+
CaptchaTask,
|
|
32
|
+
ProxyResource,
|
|
33
|
+
_datetime,
|
|
34
|
+
_optional_text,
|
|
35
|
+
_required_text,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger(__name__)
|
|
39
|
+
_CAPTCHA_TASKS = "/v1/captcha/tasks"
|
|
40
|
+
_CAPTCHA_TASK_ID = re.compile(r"^cap_[a-f0-9]{32}$")
|
|
41
|
+
_CAPTCHA_POLL_SECONDS = 3
|
|
42
|
+
_MODEL_CHAT_COMPLETIONS = "/v1/model-proxy/chat/completions"
|
|
43
|
+
# The server permits up to 600 seconds and the TLS gateway allows another five.
|
|
44
|
+
_MODEL_READ_TIMEOUT_SECONDS = 605
|
|
45
|
+
_MIN_CAPTCHA_IMAGE_BASE64_CHARS = 136
|
|
46
|
+
_MAX_CAPTCHA_IMAGE_BASE64_CHARS = 66_668
|
|
47
|
+
_RESOURCE_PATH = re.compile(r"^(/v1/(?:ip-proxy/leases|aws-proxy/browser-sessions))/[^/]+")
|
|
48
|
+
_ACCESS_PROFILES = {
|
|
49
|
+
"bazhuayu_pre": ("bazhuayu_pre", None),
|
|
50
|
+
"bazhuayu_overseas_pre": ("bazhuayu_pre", "overseas"),
|
|
51
|
+
"bazhuayu": ("bazhuayu", None),
|
|
52
|
+
"bazhuayu_overseas": ("bazhuayu", "overseas"),
|
|
53
|
+
"octoparse_pre": ("octoparse_pre", None),
|
|
54
|
+
"octoparse": ("octoparse", None),
|
|
55
|
+
}
|
|
56
|
+
_VERIFIED_PRE_NODE_URLS = {
|
|
57
|
+
"bazhuayu_pre": {
|
|
58
|
+
"domestic": "https://pre-resource-proxy.bazhuayu.com:443",
|
|
59
|
+
"overseas": "https://pre-resource-proxy-overseas.bazhuayu.com:443",
|
|
60
|
+
},
|
|
61
|
+
"octoparse_pre": {
|
|
62
|
+
"domestic": "https://pre-resource-proxy.bazhuayu.com:443",
|
|
63
|
+
"overseas": "https://pre-resource-proxy-overseas.bazhuayu.com:443",
|
|
64
|
+
},
|
|
65
|
+
}
|
|
66
|
+
_API_KEYS_URLS = {
|
|
67
|
+
"bazhuayu_pre": "https://pre.bazhuayu.com/console/account-center/api-keys",
|
|
68
|
+
"bazhuayu": "https://www.bazhuayu.com/console/account-center/api-keys",
|
|
69
|
+
"octoparse_pre": "https://pre.octoparse.com/console/account-center/api-keys",
|
|
70
|
+
"octoparse": "https://www.octoparse.com/console/account-center/api-keys",
|
|
71
|
+
}
|
|
72
|
+
_BILLING_ROUTE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
|
73
|
+
_EXECUTION_ID = re.compile(r"^[A-Za-z0-9_-]{1,96}$")
|
|
74
|
+
_SAFE_ERROR_CODE = re.compile(r"^[a-z][a-z0-9-]{0,79}$")
|
|
75
|
+
_DEFAULT_REGIONAL_READ_TIMEOUT_SECONDS = 10.0
|
|
76
|
+
_MAX_REGIONAL_READ_TIMEOUT_SECONDS = 30.0
|
|
77
|
+
_CATALOG_CACHE_SECONDS = 30.0
|
|
78
|
+
_CATALOG_CACHE_MAX_ENTRIES = 64
|
|
79
|
+
_CAPTCHA_BILLING_TYPE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
|
|
80
|
+
|
|
81
|
+
# These are navigation hints for common crawler use cases. They are deliberately
|
|
82
|
+
# small and never substitute for the Resource Proxy's authoritative catalog.
|
|
83
|
+
_POPULAR_PROXY_COUNTRY_HINTS = (
|
|
84
|
+
{"code": "CN", "name": "China"},
|
|
85
|
+
{"code": "US", "name": "United States"},
|
|
86
|
+
{"code": "GB", "name": "United Kingdom"},
|
|
87
|
+
{"code": "JP", "name": "Japan"},
|
|
88
|
+
{"code": "SG", "name": "Singapore"},
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _captcha_directory(value: Mapping[str, Any]) -> dict[str, Any]:
|
|
93
|
+
"""Validate and defensively apply Billing's effective CAPTCHA allow-list."""
|
|
94
|
+
pricing_version = value.get("pricing_version")
|
|
95
|
+
raw_available = value.get("available_billing_types")
|
|
96
|
+
raw_types = value.get("types")
|
|
97
|
+
if (
|
|
98
|
+
not isinstance(value.get("version"), str)
|
|
99
|
+
or not value["version"].strip()
|
|
100
|
+
or not isinstance(pricing_version, str)
|
|
101
|
+
or not pricing_version.strip()
|
|
102
|
+
or not isinstance(raw_available, list)
|
|
103
|
+
or any(
|
|
104
|
+
not isinstance(item, str) or not _CAPTCHA_BILLING_TYPE.fullmatch(item)
|
|
105
|
+
for item in raw_available
|
|
106
|
+
)
|
|
107
|
+
or len(set(raw_available)) != len(raw_available)
|
|
108
|
+
or not isinstance(raw_types, list)
|
|
109
|
+
):
|
|
110
|
+
raise ResourceProxyError("resource service returned an invalid CAPTCHA directory")
|
|
111
|
+
|
|
112
|
+
available = set(raw_available)
|
|
113
|
+
filtered_types: list[dict[str, Any]] = []
|
|
114
|
+
for raw_entry in raw_types:
|
|
115
|
+
if (
|
|
116
|
+
not isinstance(raw_entry, dict)
|
|
117
|
+
or not isinstance(raw_entry.get("id"), str)
|
|
118
|
+
or not isinstance(raw_entry.get("variants"), list)
|
|
119
|
+
):
|
|
120
|
+
raise ResourceProxyError("resource service returned an invalid CAPTCHA directory")
|
|
121
|
+
entry = copy.deepcopy(raw_entry)
|
|
122
|
+
variants: list[dict[str, Any]] = []
|
|
123
|
+
for variant in entry["variants"]:
|
|
124
|
+
if not isinstance(variant, dict):
|
|
125
|
+
raise ResourceProxyError(
|
|
126
|
+
"resource service returned an invalid CAPTCHA directory"
|
|
127
|
+
)
|
|
128
|
+
billing_type = variant.get("billing_type")
|
|
129
|
+
overrides = variant.get("billing_type_overrides", [])
|
|
130
|
+
if (
|
|
131
|
+
not isinstance(billing_type, str)
|
|
132
|
+
or not _CAPTCHA_BILLING_TYPE.fullmatch(billing_type)
|
|
133
|
+
or not isinstance(overrides, list)
|
|
134
|
+
or any(
|
|
135
|
+
not isinstance(override, dict)
|
|
136
|
+
or not isinstance(override.get("billing_type"), str)
|
|
137
|
+
or not _CAPTCHA_BILLING_TYPE.fullmatch(override["billing_type"])
|
|
138
|
+
or not isinstance(override.get("parameter"), str)
|
|
139
|
+
or "equals" not in override
|
|
140
|
+
for override in overrides
|
|
141
|
+
)
|
|
142
|
+
):
|
|
143
|
+
raise ResourceProxyError(
|
|
144
|
+
"resource service returned an invalid CAPTCHA directory"
|
|
145
|
+
)
|
|
146
|
+
if billing_type not in available:
|
|
147
|
+
continue
|
|
148
|
+
unavailable = [
|
|
149
|
+
item for item in overrides if item["billing_type"] not in available
|
|
150
|
+
]
|
|
151
|
+
safe: dict[str, Any] = copy.deepcopy(variant)
|
|
152
|
+
supported = True
|
|
153
|
+
for override in unavailable:
|
|
154
|
+
parameter = override.get("parameter")
|
|
155
|
+
equals = override.get("equals")
|
|
156
|
+
properties = safe.get("properties")
|
|
157
|
+
required = safe.get("required", [])
|
|
158
|
+
one_of_required = safe.get("one_of_required", [])
|
|
159
|
+
if (
|
|
160
|
+
not isinstance(parameter, str)
|
|
161
|
+
or not isinstance(properties, dict)
|
|
162
|
+
or parameter not in properties
|
|
163
|
+
or not isinstance(required, list)
|
|
164
|
+
or parameter in required
|
|
165
|
+
or not isinstance(one_of_required, list)
|
|
166
|
+
or any(
|
|
167
|
+
not isinstance(group, list) or parameter in group
|
|
168
|
+
for group in one_of_required
|
|
169
|
+
)
|
|
170
|
+
or not isinstance(properties[parameter], dict)
|
|
171
|
+
):
|
|
172
|
+
supported = False
|
|
173
|
+
break
|
|
174
|
+
spec = properties[parameter]
|
|
175
|
+
enum = spec.get("enum")
|
|
176
|
+
if isinstance(enum, list):
|
|
177
|
+
allowed_values = [item for item in enum if item != equals]
|
|
178
|
+
if not allowed_values:
|
|
179
|
+
supported = False
|
|
180
|
+
break
|
|
181
|
+
spec["enum"] = allowed_values
|
|
182
|
+
elif spec.get("type") == "boolean" and isinstance(equals, bool):
|
|
183
|
+
spec["enum"] = [not equals]
|
|
184
|
+
elif any(
|
|
185
|
+
item.get("parameter") == parameter
|
|
186
|
+
and item["billing_type"] in available
|
|
187
|
+
for item in overrides
|
|
188
|
+
):
|
|
189
|
+
supported = False
|
|
190
|
+
break
|
|
191
|
+
else:
|
|
192
|
+
properties.pop(parameter)
|
|
193
|
+
if isinstance(safe.get("example"), dict):
|
|
194
|
+
safe["example"].pop(parameter, None)
|
|
195
|
+
if not supported:
|
|
196
|
+
continue
|
|
197
|
+
safe["billing_type_overrides"] = [
|
|
198
|
+
item for item in overrides if item["billing_type"] in available
|
|
199
|
+
]
|
|
200
|
+
if not safe["billing_type_overrides"]:
|
|
201
|
+
safe.pop("billing_type_overrides")
|
|
202
|
+
variants.append(safe)
|
|
203
|
+
if not variants:
|
|
204
|
+
continue
|
|
205
|
+
entry["variants"] = variants
|
|
206
|
+
default_variant = entry.get("default_variant")
|
|
207
|
+
if default_variant is not None and not any(
|
|
208
|
+
default_variant in {variant.get("id"), variant.get("variant_id")}
|
|
209
|
+
for variant in variants
|
|
210
|
+
):
|
|
211
|
+
entry.pop("default_variant", None)
|
|
212
|
+
filtered_types.append(entry)
|
|
213
|
+
|
|
214
|
+
result = copy.deepcopy(dict(value))
|
|
215
|
+
result["types"] = filtered_types
|
|
216
|
+
return result
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
class ResourceTransport(Protocol):
|
|
220
|
+
"""Authenticated transport used by platform and local clients."""
|
|
221
|
+
|
|
222
|
+
async def request(
|
|
223
|
+
self,
|
|
224
|
+
method: str,
|
|
225
|
+
path: str,
|
|
226
|
+
*,
|
|
227
|
+
json: Mapping[str, Any] | None = None,
|
|
228
|
+
headers: Mapping[str, str] | None = None,
|
|
229
|
+
) -> Mapping[str, Any]: ...
|
|
230
|
+
|
|
231
|
+
async def close(self) -> None: ...
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class _HttpTransport:
|
|
235
|
+
def __init__(
|
|
236
|
+
self,
|
|
237
|
+
base_url: str,
|
|
238
|
+
api_key: str,
|
|
239
|
+
*,
|
|
240
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
241
|
+
timeout: float = 30,
|
|
242
|
+
) -> None:
|
|
243
|
+
self._api_key = api_key
|
|
244
|
+
self._base_url = base_url.rstrip("/")
|
|
245
|
+
self._transport = transport
|
|
246
|
+
self._timeout = timeout
|
|
247
|
+
self._http = self._new_client()
|
|
248
|
+
|
|
249
|
+
def _new_client(self) -> httpx.AsyncClient:
|
|
250
|
+
return httpx.AsyncClient(
|
|
251
|
+
base_url=self._base_url, transport=self._transport, timeout=self._timeout
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
async def request(
|
|
255
|
+
self,
|
|
256
|
+
method: str,
|
|
257
|
+
path: str,
|
|
258
|
+
*,
|
|
259
|
+
json: Mapping[str, Any] | None = None,
|
|
260
|
+
headers: Mapping[str, str] | None = None,
|
|
261
|
+
) -> Mapping[str, Any]:
|
|
262
|
+
request_headers = {"Authorization": f"Bearer {self._api_key}"}
|
|
263
|
+
request_headers.update(headers or {})
|
|
264
|
+
timeout = self._http.timeout
|
|
265
|
+
if path == _MODEL_CHAT_COMPLETIONS:
|
|
266
|
+
timeout = httpx.Timeout(self._timeout, read=_MODEL_READ_TIMEOUT_SECONDS)
|
|
267
|
+
if self._http.is_closed:
|
|
268
|
+
# Receipt reads can outlive the execution and its HTTP connection pool.
|
|
269
|
+
async with self._new_client() as client:
|
|
270
|
+
response = await client.request(
|
|
271
|
+
method, path, json=json, headers=request_headers, timeout=timeout,
|
|
272
|
+
)
|
|
273
|
+
else:
|
|
274
|
+
response = await self._http.request(
|
|
275
|
+
method, path, json=json, headers=request_headers, timeout=timeout,
|
|
276
|
+
)
|
|
277
|
+
return _decode_response(response)
|
|
278
|
+
|
|
279
|
+
async def close(self) -> None:
|
|
280
|
+
await self._http.aclose()
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
class _ExecutionTransport:
|
|
284
|
+
def __init__(self, transport: ResourceTransport, token: str) -> None:
|
|
285
|
+
self._transport = transport
|
|
286
|
+
self._token = token
|
|
287
|
+
|
|
288
|
+
async def request(
|
|
289
|
+
self,
|
|
290
|
+
method: str,
|
|
291
|
+
path: str,
|
|
292
|
+
*,
|
|
293
|
+
json: Mapping[str, Any] | None = None,
|
|
294
|
+
headers: Mapping[str, str] | None = None,
|
|
295
|
+
) -> Mapping[str, Any]:
|
|
296
|
+
request_headers = {"Authorization": f"Bearer {self._token}"}
|
|
297
|
+
request_headers.update(headers or {})
|
|
298
|
+
return await self._transport.request(
|
|
299
|
+
method, path, json=json, headers=request_headers
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
async def close(self) -> None:
|
|
303
|
+
await self._transport.close()
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
class ResourceProxy:
|
|
307
|
+
"""Own managed resources within one local or platform execution."""
|
|
308
|
+
|
|
309
|
+
def __init__(
|
|
310
|
+
self,
|
|
311
|
+
transport: ResourceTransport,
|
|
312
|
+
*,
|
|
313
|
+
owns_transport: bool = False,
|
|
314
|
+
requires_execution_token: bool = False,
|
|
315
|
+
heartbeat_interval: float = 30,
|
|
316
|
+
app_id: str | None = None,
|
|
317
|
+
app_version: str | None = None,
|
|
318
|
+
environment: str | None = None,
|
|
319
|
+
network: str = "domestic",
|
|
320
|
+
regional_mode: bool = False,
|
|
321
|
+
regional_read_timeout: float = _DEFAULT_REGIONAL_READ_TIMEOUT_SECONDS,
|
|
322
|
+
) -> None:
|
|
323
|
+
if heartbeat_interval <= 0:
|
|
324
|
+
raise ValueError("heartbeat_interval must be positive")
|
|
325
|
+
self._regional_read_timeout = _validate_regional_read_timeout(
|
|
326
|
+
regional_read_timeout
|
|
327
|
+
)
|
|
328
|
+
self._transport = transport
|
|
329
|
+
self._owns_transport = owns_transport
|
|
330
|
+
self._requires_execution_token = requires_execution_token
|
|
331
|
+
self._heartbeat_interval = heartbeat_interval
|
|
332
|
+
self._app_id = app_id
|
|
333
|
+
self._app_version = app_version
|
|
334
|
+
self._environment = environment
|
|
335
|
+
self._network = network
|
|
336
|
+
self._regional_mode = regional_mode
|
|
337
|
+
self._billing_route_id: str | None = None
|
|
338
|
+
self._regional_clients: dict[str, ResourceProxy] = {}
|
|
339
|
+
self.execution_id: str | None = None
|
|
340
|
+
self.expires_at: datetime | None = None
|
|
341
|
+
self._resources: dict[int, Any] = {}
|
|
342
|
+
self._heartbeat_task: asyncio.Task[None] | None = None
|
|
343
|
+
self._operation_lock = asyncio.Lock()
|
|
344
|
+
self._start_lock = asyncio.Lock()
|
|
345
|
+
self._closed = False
|
|
346
|
+
self._local_transport: _HttpTransport | None = None
|
|
347
|
+
self._managed_billing_enabled = False
|
|
348
|
+
self._managed_billing_warned = False
|
|
349
|
+
self._catalog_cache: dict[tuple[Any, ...], tuple[float, dict[str, Any]]] = {}
|
|
350
|
+
self._captcha_task_networks: dict[str, str] = {}
|
|
351
|
+
|
|
352
|
+
def __repr__(self) -> str:
|
|
353
|
+
networks = tuple([self._network, *self._regional_clients])
|
|
354
|
+
return (
|
|
355
|
+
f"ResourceProxy(execution_id={self.execution_id!r}, "
|
|
356
|
+
f"environment={self._environment!r}, networks={networks!r})"
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
@classmethod
|
|
360
|
+
def from_env(
|
|
361
|
+
cls,
|
|
362
|
+
environ: Mapping[str, str] | None = None,
|
|
363
|
+
*,
|
|
364
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
365
|
+
heartbeat_interval: float = 30,
|
|
366
|
+
regional_read_timeout: float | None = None,
|
|
367
|
+
) -> ResourceProxy:
|
|
368
|
+
supplied = os.environ if environ is None else environ
|
|
369
|
+
project = load_project_config(supplied)
|
|
370
|
+
environment = resolve_config_environment(supplied, project)
|
|
371
|
+
has_process_key = bool(
|
|
372
|
+
supplied.get("DATAAPP_API_KEY") or supplied.get("DATAOS_API_KEY")
|
|
373
|
+
)
|
|
374
|
+
stored = (
|
|
375
|
+
{}
|
|
376
|
+
if has_process_key
|
|
377
|
+
else load_local_config(supplied, environment=environment)
|
|
378
|
+
)
|
|
379
|
+
env = {**stored, **supplied}
|
|
380
|
+
if supplied.get("DATAOS_API_KEY") and not supplied.get("DATAAPP_API_KEY"):
|
|
381
|
+
# An explicit process value outranks the per-user file even when it
|
|
382
|
+
# uses the compatibility name.
|
|
383
|
+
env["DATAAPP_API_KEY"] = supplied["DATAOS_API_KEY"]
|
|
384
|
+
env.pop("DATAOS_API_KEY", None)
|
|
385
|
+
configured_read_timeout = _configured_regional_read_timeout(
|
|
386
|
+
env, regional_read_timeout
|
|
387
|
+
)
|
|
388
|
+
legacy_url = env.get("DATAOS_RESOURCE_PROXY_URL") or None
|
|
389
|
+
domestic_url = env.get("DATAOS_RESOURCE_PROXY_DOMESTIC_URL") or None
|
|
390
|
+
overseas_url = env.get("DATAOS_RESOURCE_PROXY_OVERSEAS_URL") or None
|
|
391
|
+
new_environment = env.get("DATAAPP_ENVIRONMENT") or None
|
|
392
|
+
old_environment = env.get("DATAOS_ENVIRONMENT") or None
|
|
393
|
+
if new_environment and old_environment and new_environment != old_environment:
|
|
394
|
+
raise ResourceProxyError(
|
|
395
|
+
"DATAAPP_ENVIRONMENT conflicts with deprecated DATAOS_ENVIRONMENT",
|
|
396
|
+
code="configuration-error",
|
|
397
|
+
)
|
|
398
|
+
access_profile = new_environment or old_environment
|
|
399
|
+
if access_profile is not None and access_profile not in _ACCESS_PROFILES:
|
|
400
|
+
raise ResourceProxyError(
|
|
401
|
+
"DATAAPP_ENVIRONMENT must be bazhuayu_pre, bazhuayu_overseas_pre, "
|
|
402
|
+
"bazhuayu, bazhuayu_overseas, octoparse_pre, or octoparse",
|
|
403
|
+
code="configuration-error",
|
|
404
|
+
)
|
|
405
|
+
profile_environment, preferred_network = (
|
|
406
|
+
_ACCESS_PROFILES[access_profile]
|
|
407
|
+
if access_profile is not None
|
|
408
|
+
else (None, None)
|
|
409
|
+
)
|
|
410
|
+
if profile_environment is not None and profile_environment != environment:
|
|
411
|
+
raise ResourceProxyError(
|
|
412
|
+
"DATAAPP_ENVIRONMENT conflicts with the downloaded project environment",
|
|
413
|
+
code="configuration-error",
|
|
414
|
+
)
|
|
415
|
+
new_api_key = env.get("DATAAPP_API_KEY")
|
|
416
|
+
old_api_key = env.get("DATAOS_API_KEY")
|
|
417
|
+
if new_api_key and old_api_key and new_api_key.strip() != old_api_key.strip():
|
|
418
|
+
raise ResourceProxyError(
|
|
419
|
+
"DATAAPP_API_KEY conflicts with deprecated DATAOS_API_KEY",
|
|
420
|
+
code="configuration-error",
|
|
421
|
+
)
|
|
422
|
+
api_key = new_api_key or old_api_key
|
|
423
|
+
if project is not None:
|
|
424
|
+
expected_domestic = project.domestic_url.rstrip("/")
|
|
425
|
+
expected_overseas = project.overseas_url.rstrip("/")
|
|
426
|
+
if legacy_url and legacy_url.rstrip("/") != expected_domestic:
|
|
427
|
+
raise ResourceProxyError(
|
|
428
|
+
"DATAOS_RESOURCE_PROXY_URL conflicts with the downloaded project environment",
|
|
429
|
+
code="configuration-error",
|
|
430
|
+
)
|
|
431
|
+
if domestic_url and domestic_url.rstrip("/") != expected_domestic:
|
|
432
|
+
raise ResourceProxyError(
|
|
433
|
+
"DATAOS_RESOURCE_PROXY_DOMESTIC_URL conflicts with the downloaded "
|
|
434
|
+
"project environment",
|
|
435
|
+
code="configuration-error",
|
|
436
|
+
)
|
|
437
|
+
if overseas_url and overseas_url.rstrip("/") != expected_overseas:
|
|
438
|
+
raise ResourceProxyError(
|
|
439
|
+
"DATAOS_RESOURCE_PROXY_OVERSEAS_URL conflicts with the downloaded "
|
|
440
|
+
"project environment",
|
|
441
|
+
code="configuration-error",
|
|
442
|
+
)
|
|
443
|
+
domestic_url = project.domestic_url
|
|
444
|
+
overseas_url = project.overseas_url
|
|
445
|
+
if not legacy_url and not domestic_url and not overseas_url:
|
|
446
|
+
verified = _VERIFIED_PRE_NODE_URLS.get(environment or "")
|
|
447
|
+
if verified is not None:
|
|
448
|
+
if preferred_network == "overseas":
|
|
449
|
+
overseas_url = verified["overseas"]
|
|
450
|
+
else:
|
|
451
|
+
domestic_url = verified["domestic"]
|
|
452
|
+
overseas_url = verified["overseas"]
|
|
453
|
+
elif environment in {"bazhuayu", "octoparse"}:
|
|
454
|
+
raise ResourceProxyError(
|
|
455
|
+
"production local debug requires an explicit Resource Proxy URL",
|
|
456
|
+
code="configuration-error",
|
|
457
|
+
)
|
|
458
|
+
if (
|
|
459
|
+
legacy_url
|
|
460
|
+
and domestic_url
|
|
461
|
+
and legacy_url.rstrip("/") != domestic_url.rstrip("/")
|
|
462
|
+
):
|
|
463
|
+
raise ResourceProxyError(
|
|
464
|
+
"DATAOS_RESOURCE_PROXY_URL conflicts with DATAOS_RESOURCE_PROXY_DOMESTIC_URL",
|
|
465
|
+
code="configuration-error",
|
|
466
|
+
)
|
|
467
|
+
regional = bool(domestic_url or overseas_url)
|
|
468
|
+
base_url = domestic_url or legacy_url
|
|
469
|
+
base_network = preferred_network or "domestic"
|
|
470
|
+
if domestic_url is not None:
|
|
471
|
+
base_network = "domestic"
|
|
472
|
+
if base_url is None and overseas_url is not None:
|
|
473
|
+
base_url = overseas_url
|
|
474
|
+
base_network = "overseas"
|
|
475
|
+
if not api_key or not api_key.strip():
|
|
476
|
+
api_keys_url = (
|
|
477
|
+
project.api_keys_url
|
|
478
|
+
if project is not None
|
|
479
|
+
else _API_KEYS_URLS.get(environment or "")
|
|
480
|
+
)
|
|
481
|
+
account_center = (
|
|
482
|
+
f" Get a Key from {api_keys_url}." if api_keys_url is not None else ""
|
|
483
|
+
)
|
|
484
|
+
raise ResourceProxyError(
|
|
485
|
+
"DATAAPP_API_KEY is required for local managed resources. Managed IP "
|
|
486
|
+
"traffic and CAPTCHA solves accrue resource costs. After confirming use, "
|
|
487
|
+
"run `python -m dataapp_resource_proxy configure`; it securely prompts for "
|
|
488
|
+
f"the Key and stores it outside the project.{account_center} "
|
|
489
|
+
"Platform Runs use injected run identity and do not need this Key.",
|
|
490
|
+
code="configuration-error",
|
|
491
|
+
)
|
|
492
|
+
if not base_url:
|
|
493
|
+
raise ResourceProxyError(
|
|
494
|
+
"a Resource Proxy URL is required; use a downloaded project with "
|
|
495
|
+
".dataapp-sdk.json or set an explicit environment and URL. "
|
|
496
|
+
"No pre environment is selected implicitly.",
|
|
497
|
+
code="configuration-error",
|
|
498
|
+
)
|
|
499
|
+
local_transport = _HttpTransport(base_url, api_key, transport=transport)
|
|
500
|
+
client = cls(
|
|
501
|
+
local_transport,
|
|
502
|
+
owns_transport=True,
|
|
503
|
+
requires_execution_token=True,
|
|
504
|
+
heartbeat_interval=heartbeat_interval,
|
|
505
|
+
app_id=env.get("DATAOS_APP_ID") or None,
|
|
506
|
+
app_version=env.get("DATAOS_APP_VERSION") or None,
|
|
507
|
+
environment=environment,
|
|
508
|
+
network=base_network,
|
|
509
|
+
regional_mode=regional,
|
|
510
|
+
regional_read_timeout=configured_read_timeout,
|
|
511
|
+
)
|
|
512
|
+
client._local_transport = local_transport
|
|
513
|
+
if overseas_url and base_network != "overseas":
|
|
514
|
+
overseas_transport = _HttpTransport(overseas_url, api_key, transport=transport)
|
|
515
|
+
overseas = cls(
|
|
516
|
+
overseas_transport,
|
|
517
|
+
owns_transport=True,
|
|
518
|
+
requires_execution_token=True,
|
|
519
|
+
heartbeat_interval=heartbeat_interval,
|
|
520
|
+
app_id=env.get("DATAOS_APP_ID") or None,
|
|
521
|
+
app_version=env.get("DATAOS_APP_VERSION") or None,
|
|
522
|
+
environment=environment,
|
|
523
|
+
network="overseas",
|
|
524
|
+
regional_read_timeout=configured_read_timeout,
|
|
525
|
+
)
|
|
526
|
+
overseas._local_transport = overseas_transport
|
|
527
|
+
client._regional_clients["overseas"] = overseas
|
|
528
|
+
return client
|
|
529
|
+
|
|
530
|
+
async def __aenter__(self) -> ResourceProxy:
|
|
531
|
+
try:
|
|
532
|
+
# Regional clients create only the execution needed by the first
|
|
533
|
+
# operation. Explicit start() starts this client's configured node.
|
|
534
|
+
return self if self._regional_mode else await self.start()
|
|
535
|
+
except BaseException as error:
|
|
536
|
+
if self._owns_transport:
|
|
537
|
+
try:
|
|
538
|
+
await self._transport.close()
|
|
539
|
+
self._closed = True
|
|
540
|
+
except BaseException:
|
|
541
|
+
error.add_note("resource transport cleanup also failed")
|
|
542
|
+
raise
|
|
543
|
+
|
|
544
|
+
@property
|
|
545
|
+
def environment(self) -> str | None:
|
|
546
|
+
"""Account and Billing environment bound by the resource service."""
|
|
547
|
+
return self._environment
|
|
548
|
+
|
|
549
|
+
@property
|
|
550
|
+
def billing_route_id(self) -> str | None:
|
|
551
|
+
"""Server-owned Billing route identifier for the current execution."""
|
|
552
|
+
return self._billing_route_id
|
|
553
|
+
|
|
554
|
+
@property
|
|
555
|
+
def network(self) -> str:
|
|
556
|
+
"""Resource node network used by this execution."""
|
|
557
|
+
return self._network
|
|
558
|
+
|
|
559
|
+
@property
|
|
560
|
+
def regional_read_timeout(self) -> float:
|
|
561
|
+
"""Validated per-node deadline for regional receipt and usage reads."""
|
|
562
|
+
return self._regional_read_timeout
|
|
563
|
+
|
|
564
|
+
async def start(self) -> ResourceProxy:
|
|
565
|
+
"""Create the execution lazily and start its heartbeat."""
|
|
566
|
+
async with self._start_lock:
|
|
567
|
+
if self._closed:
|
|
568
|
+
raise RuntimeError("resource client is closed")
|
|
569
|
+
if self.execution_id is None:
|
|
570
|
+
value = await self._request(
|
|
571
|
+
"POST",
|
|
572
|
+
"/v1/executions",
|
|
573
|
+
json={
|
|
574
|
+
**({"app_id": self._app_id} if self._app_id is not None else {}),
|
|
575
|
+
**(
|
|
576
|
+
{"app_version": self._app_version}
|
|
577
|
+
if self._app_version is not None
|
|
578
|
+
else {}
|
|
579
|
+
),
|
|
580
|
+
**(
|
|
581
|
+
{"environment": self._environment}
|
|
582
|
+
if self._environment is not None
|
|
583
|
+
else {}
|
|
584
|
+
),
|
|
585
|
+
},
|
|
586
|
+
authenticated=False,
|
|
587
|
+
)
|
|
588
|
+
execution_id = _required_text(value, "execution_id")
|
|
589
|
+
expires_at = _datetime(value.get("expires_at"))
|
|
590
|
+
token = value.get("token")
|
|
591
|
+
if token is None and self._requires_execution_token:
|
|
592
|
+
raise ValueError("resource service returned no execution token")
|
|
593
|
+
if token is not None and (not isinstance(token, str) or not token):
|
|
594
|
+
raise ValueError("resource service returned invalid token")
|
|
595
|
+
execution_transport = (
|
|
596
|
+
_ExecutionTransport(self._transport, token)
|
|
597
|
+
if isinstance(token, str)
|
|
598
|
+
else None
|
|
599
|
+
)
|
|
600
|
+
returned_environment = value.get("environment")
|
|
601
|
+
returned_route = value.get("billing_route_id")
|
|
602
|
+
try:
|
|
603
|
+
if not _EXECUTION_ID.fullmatch(execution_id):
|
|
604
|
+
raise ResourceProxyError(
|
|
605
|
+
"resource service returned an invalid execution ID",
|
|
606
|
+
code="invalid-response",
|
|
607
|
+
)
|
|
608
|
+
if self._environment is not None:
|
|
609
|
+
if returned_environment != self._environment:
|
|
610
|
+
raise ResourceProxyError(
|
|
611
|
+
"resource service returned the wrong environment",
|
|
612
|
+
code="invalid-response",
|
|
613
|
+
)
|
|
614
|
+
if (
|
|
615
|
+
not isinstance(returned_route, str)
|
|
616
|
+
or not _BILLING_ROUTE_ID.fullmatch(returned_route)
|
|
617
|
+
):
|
|
618
|
+
raise ResourceProxyError(
|
|
619
|
+
"resource service returned an invalid billing route",
|
|
620
|
+
code="invalid-response",
|
|
621
|
+
)
|
|
622
|
+
except (ResourceProxyError, ValueError):
|
|
623
|
+
if execution_transport is not None and _EXECUTION_ID.fullmatch(execution_id):
|
|
624
|
+
await self._revoke_rejected_execution(execution_transport)
|
|
625
|
+
raise
|
|
626
|
+
if self._environment is not None:
|
|
627
|
+
assert isinstance(returned_route, str)
|
|
628
|
+
self._billing_route_id = returned_route
|
|
629
|
+
if execution_transport is not None:
|
|
630
|
+
self._transport = execution_transport
|
|
631
|
+
self.execution_id = execution_id
|
|
632
|
+
self.expires_at = expires_at
|
|
633
|
+
self._managed_billing_enabled = (
|
|
634
|
+
self._local_transport is not None
|
|
635
|
+
and value.get("managed_billing_enabled") is True
|
|
636
|
+
)
|
|
637
|
+
self._heartbeat_task = asyncio.create_task(
|
|
638
|
+
self._heartbeat_loop(), name="resource-proxy-heartbeat"
|
|
639
|
+
)
|
|
640
|
+
return self
|
|
641
|
+
|
|
642
|
+
async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
|
|
643
|
+
try:
|
|
644
|
+
await self.close()
|
|
645
|
+
except BaseException:
|
|
646
|
+
if exc is None:
|
|
647
|
+
raise
|
|
648
|
+
exc.add_note("managed resource cleanup also failed")
|
|
649
|
+
|
|
650
|
+
async def proxy(
|
|
651
|
+
self,
|
|
652
|
+
*,
|
|
653
|
+
country: str | None = None,
|
|
654
|
+
state: str | None = None,
|
|
655
|
+
city: str | None = None,
|
|
656
|
+
expire_time: str | None = None,
|
|
657
|
+
exit_policy: str = "renewable",
|
|
658
|
+
network_preference: str | None = None,
|
|
659
|
+
target_hint: str | None = None,
|
|
660
|
+
line_tier: str = "standard",
|
|
661
|
+
proxy_group: str | None = None,
|
|
662
|
+
idempotency_key: str | None = None,
|
|
663
|
+
) -> ProxyResource:
|
|
664
|
+
_validate_network_preference(network_preference)
|
|
665
|
+
_validate_target_hint(target_hint)
|
|
666
|
+
_validate_line_tier(line_tier)
|
|
667
|
+
_validate_proxy_group(proxy_group)
|
|
668
|
+
if proxy_group is not None and line_tier != "standard":
|
|
669
|
+
raise ValueError("proxy_group cannot be combined with a non-default line_tier")
|
|
670
|
+
group_network = _network_for_proxy_group(proxy_group)
|
|
671
|
+
if (
|
|
672
|
+
group_network is not None
|
|
673
|
+
and network_preference not in {None, "auto", group_network}
|
|
674
|
+
):
|
|
675
|
+
raise ValueError("proxy_group conflicts with network_preference")
|
|
676
|
+
selected_network = group_network or network_preference
|
|
677
|
+
selected = self._client_for_network(selected_network)
|
|
678
|
+
if selected is not self:
|
|
679
|
+
return await selected.proxy(
|
|
680
|
+
country=country,
|
|
681
|
+
state=state,
|
|
682
|
+
city=city,
|
|
683
|
+
expire_time=expire_time,
|
|
684
|
+
exit_policy=exit_policy,
|
|
685
|
+
network_preference=selected_network,
|
|
686
|
+
target_hint=target_hint,
|
|
687
|
+
line_tier=line_tier,
|
|
688
|
+
proxy_group=proxy_group,
|
|
689
|
+
idempotency_key=idempotency_key,
|
|
690
|
+
)
|
|
691
|
+
async with self._operation_lock:
|
|
692
|
+
await self.start()
|
|
693
|
+
body = {
|
|
694
|
+
"country": country,
|
|
695
|
+
"state": state,
|
|
696
|
+
"city": city,
|
|
697
|
+
"expire_time": expire_time,
|
|
698
|
+
"exit_policy": exit_policy,
|
|
699
|
+
"network_preference": (
|
|
700
|
+
None if selected_network == "auto" else selected_network
|
|
701
|
+
),
|
|
702
|
+
"target_hint": target_hint,
|
|
703
|
+
"line_tier": line_tier if line_tier != "standard" else None,
|
|
704
|
+
"proxy_group": proxy_group,
|
|
705
|
+
}
|
|
706
|
+
self._warn_managed_billing()
|
|
707
|
+
value = await self._request(
|
|
708
|
+
"POST",
|
|
709
|
+
"/v1/ip-proxy/leases",
|
|
710
|
+
json={key: item for key, item in body.items() if item is not None},
|
|
711
|
+
headers={"Idempotency-Key": idempotency_key or str(uuid.uuid4())},
|
|
712
|
+
)
|
|
713
|
+
lease_id = _required_text(value, "lease_id")
|
|
714
|
+
if any(
|
|
715
|
+
isinstance(existing, ProxyResource) and existing.lease_id == lease_id
|
|
716
|
+
for existing in self._resources.values()
|
|
717
|
+
):
|
|
718
|
+
raise ResourceProxyError(
|
|
719
|
+
"proxy lease already has a live handle",
|
|
720
|
+
code="duplicate-proxy-handle",
|
|
721
|
+
)
|
|
722
|
+
handle = ProxyResource(
|
|
723
|
+
client=self,
|
|
724
|
+
lease_id=lease_id,
|
|
725
|
+
proxy_url=_required_text(value, "proxy_url"),
|
|
726
|
+
expires_at=_datetime(value.get("expires_at")),
|
|
727
|
+
credential_generation=(
|
|
728
|
+
int(value["credential_generation"])
|
|
729
|
+
if value.get("credential_generation") is not None
|
|
730
|
+
else None
|
|
731
|
+
),
|
|
732
|
+
network=_optional_text(value.get("network")),
|
|
733
|
+
node_id=_optional_text(value.get("node_id")),
|
|
734
|
+
line_tier=_optional_text(value.get("line_tier")),
|
|
735
|
+
proxy_group=_optional_text(value.get("proxy_group")),
|
|
736
|
+
supplier_code=_optional_text(value.get("supplier_code")),
|
|
737
|
+
supplier_route_id=_optional_text(value.get("supplier_route_id")),
|
|
738
|
+
supplier_account_ref=_optional_text(value.get("supplier_account_ref")),
|
|
739
|
+
line_policy_version=_optional_text(value.get("line_policy_version")),
|
|
740
|
+
)
|
|
741
|
+
self._resources[id(handle)] = handle
|
|
742
|
+
return handle
|
|
743
|
+
|
|
744
|
+
@staticmethod
|
|
745
|
+
def proxy_location_hints() -> dict[str, Any]:
|
|
746
|
+
"""Return non-authoritative popular country hints for a selection UI or AI agent.
|
|
747
|
+
|
|
748
|
+
Call :meth:`proxy_locations` before presenting a location as currently
|
|
749
|
+
supported. Hints are not inventory and intentionally contain no cities.
|
|
750
|
+
"""
|
|
751
|
+
|
|
752
|
+
return {
|
|
753
|
+
"level": "country",
|
|
754
|
+
"items": copy.deepcopy(list(_POPULAR_PROXY_COUNTRY_HINTS)),
|
|
755
|
+
"authoritative": False,
|
|
756
|
+
"coverage_only": True,
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
async def proxy_groups(
|
|
760
|
+
self,
|
|
761
|
+
*,
|
|
762
|
+
network_preference: str | None = None,
|
|
763
|
+
refresh: bool = False,
|
|
764
|
+
) -> dict[str, Any]:
|
|
765
|
+
"""Return product groups exposed by the selected Resource Proxy node.
|
|
766
|
+
|
|
767
|
+
The result describes configured coverage, not live proxy inventory.
|
|
768
|
+
``refresh=True`` bypasses the SDK's short-lived response cache.
|
|
769
|
+
"""
|
|
770
|
+
|
|
771
|
+
_validate_network_preference(network_preference)
|
|
772
|
+
selected = self._client_for_network(network_preference)
|
|
773
|
+
if selected is not self:
|
|
774
|
+
return await selected.proxy_groups(
|
|
775
|
+
network_preference=network_preference, refresh=refresh
|
|
776
|
+
)
|
|
777
|
+
network = self._network if network_preference in {None, "auto"} else network_preference
|
|
778
|
+
body = {"network_preference": network}
|
|
779
|
+
return await self._catalog_query(
|
|
780
|
+
"/v1/ip-proxy/groups/query",
|
|
781
|
+
body,
|
|
782
|
+
cache_key=("groups", network),
|
|
783
|
+
refresh=refresh,
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
async def proxy_locations(
|
|
787
|
+
self,
|
|
788
|
+
*,
|
|
789
|
+
proxy_group: str = "auto",
|
|
790
|
+
country: str | None = None,
|
|
791
|
+
state: str | None = None,
|
|
792
|
+
expire_time: str | None = None,
|
|
793
|
+
network_preference: str | None = None,
|
|
794
|
+
offset: int = 0,
|
|
795
|
+
limit: int = 200,
|
|
796
|
+
refresh: bool = False,
|
|
797
|
+
) -> dict[str, Any]:
|
|
798
|
+
"""Query the authoritative location catalog for one proxy product group.
|
|
799
|
+
|
|
800
|
+
Catalog membership means that at least one configured route covers the
|
|
801
|
+
location. A later lease request can still fail when a supplier has no
|
|
802
|
+
temporary inventory.
|
|
803
|
+
"""
|
|
804
|
+
|
|
805
|
+
_validate_proxy_group(proxy_group)
|
|
806
|
+
_validate_network_preference(network_preference)
|
|
807
|
+
if state is not None and country is None:
|
|
808
|
+
raise ValueError("state requires country")
|
|
809
|
+
normalized_country = _normalize_country(country)
|
|
810
|
+
normalized_state = _optional_catalog_code(state, "state")
|
|
811
|
+
if expire_time is not None and (
|
|
812
|
+
not isinstance(expire_time, str) or not 1 <= len(expire_time) <= 24
|
|
813
|
+
):
|
|
814
|
+
raise ValueError("expire_time must contain 1-24 characters")
|
|
815
|
+
if (
|
|
816
|
+
isinstance(offset, bool)
|
|
817
|
+
or not isinstance(offset, int)
|
|
818
|
+
or not 0 <= offset <= 1_000_000
|
|
819
|
+
):
|
|
820
|
+
raise ValueError("offset must be between 0 and 1000000")
|
|
821
|
+
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 200:
|
|
822
|
+
raise ValueError("limit must be between 1 and 200")
|
|
823
|
+
group_network = _network_for_proxy_group(proxy_group)
|
|
824
|
+
if (
|
|
825
|
+
group_network is not None
|
|
826
|
+
and network_preference not in {None, "auto", group_network}
|
|
827
|
+
):
|
|
828
|
+
raise ValueError("proxy_group conflicts with network_preference")
|
|
829
|
+
selected_network = group_network or network_preference
|
|
830
|
+
selected = self._client_for_network(selected_network)
|
|
831
|
+
if selected is not self:
|
|
832
|
+
return await selected.proxy_locations(
|
|
833
|
+
proxy_group=proxy_group,
|
|
834
|
+
country=normalized_country,
|
|
835
|
+
state=normalized_state,
|
|
836
|
+
expire_time=expire_time,
|
|
837
|
+
network_preference=selected_network,
|
|
838
|
+
offset=offset,
|
|
839
|
+
limit=limit,
|
|
840
|
+
refresh=refresh,
|
|
841
|
+
)
|
|
842
|
+
network = self._network if selected_network in {None, "auto"} else selected_network
|
|
843
|
+
body = {
|
|
844
|
+
"proxy_group": proxy_group,
|
|
845
|
+
"network_preference": network,
|
|
846
|
+
"country": normalized_country,
|
|
847
|
+
"state": normalized_state,
|
|
848
|
+
"expire_time": expire_time,
|
|
849
|
+
"offset": offset,
|
|
850
|
+
"limit": limit,
|
|
851
|
+
}
|
|
852
|
+
clean_body = {key: value for key, value in body.items() if value is not None}
|
|
853
|
+
return await self._catalog_query(
|
|
854
|
+
"/v1/ip-proxy/locations/query",
|
|
855
|
+
clean_body,
|
|
856
|
+
cache_key=(
|
|
857
|
+
"locations", network, proxy_group, normalized_country, normalized_state,
|
|
858
|
+
expire_time, offset, limit,
|
|
859
|
+
),
|
|
860
|
+
refresh=refresh,
|
|
861
|
+
)
|
|
862
|
+
|
|
863
|
+
async def _catalog_query(
|
|
864
|
+
self,
|
|
865
|
+
path: str,
|
|
866
|
+
body: Mapping[str, Any],
|
|
867
|
+
*,
|
|
868
|
+
cache_key: tuple[Any, ...],
|
|
869
|
+
refresh: bool,
|
|
870
|
+
) -> dict[str, Any]:
|
|
871
|
+
now = perf_counter()
|
|
872
|
+
cached = self._catalog_cache.get(cache_key)
|
|
873
|
+
if not refresh and cached is not None and now - cached[0] <= _CATALOG_CACHE_SECONDS:
|
|
874
|
+
return copy.deepcopy(cached[1])
|
|
875
|
+
async with self._operation_lock:
|
|
876
|
+
now = perf_counter()
|
|
877
|
+
cached = self._catalog_cache.get(cache_key)
|
|
878
|
+
if not refresh and cached is not None and now - cached[0] <= _CATALOG_CACHE_SECONDS:
|
|
879
|
+
return copy.deepcopy(cached[1])
|
|
880
|
+
await self.start()
|
|
881
|
+
value = await self._request("POST", path, json=body)
|
|
882
|
+
if path == "/v1/captcha/types/query":
|
|
883
|
+
value = _captcha_directory(value)
|
|
884
|
+
else:
|
|
885
|
+
_validate_catalog_response(value, locations=path.endswith("/locations/query"))
|
|
886
|
+
if len(self._catalog_cache) >= _CATALOG_CACHE_MAX_ENTRIES:
|
|
887
|
+
oldest = min(self._catalog_cache, key=lambda key: self._catalog_cache[key][0])
|
|
888
|
+
self._catalog_cache.pop(oldest, None)
|
|
889
|
+
stored = copy.deepcopy(value)
|
|
890
|
+
self._catalog_cache[cache_key] = (perf_counter(), stored)
|
|
891
|
+
return copy.deepcopy(stored)
|
|
892
|
+
|
|
893
|
+
async def browser(
|
|
894
|
+
self,
|
|
895
|
+
*,
|
|
896
|
+
region: str | None = None,
|
|
897
|
+
browser_identifier: str | None = None,
|
|
898
|
+
session_timeout: int = 900,
|
|
899
|
+
viewport: tuple[int, int] | None = None,
|
|
900
|
+
slot_id: str | None = None,
|
|
901
|
+
network_preference: str | None = None,
|
|
902
|
+
target_hint: str | None = None,
|
|
903
|
+
idempotency_key: str | None = None,
|
|
904
|
+
) -> BrowserResource:
|
|
905
|
+
_validate_network_preference(network_preference)
|
|
906
|
+
_validate_target_hint(target_hint)
|
|
907
|
+
selected = self._client_for_network(network_preference)
|
|
908
|
+
if selected is not self:
|
|
909
|
+
return await selected.browser(
|
|
910
|
+
region=region,
|
|
911
|
+
browser_identifier=browser_identifier,
|
|
912
|
+
session_timeout=session_timeout,
|
|
913
|
+
viewport=viewport,
|
|
914
|
+
slot_id=slot_id,
|
|
915
|
+
network_preference=network_preference,
|
|
916
|
+
target_hint=target_hint,
|
|
917
|
+
idempotency_key=idempotency_key,
|
|
918
|
+
)
|
|
919
|
+
async with self._operation_lock:
|
|
920
|
+
await self.start()
|
|
921
|
+
create_key = idempotency_key or str(uuid.uuid4())
|
|
922
|
+
body: dict[str, Any] = {"session_timeout_seconds": session_timeout}
|
|
923
|
+
body["slot_id"] = slot_id or (
|
|
924
|
+
"slot_" + hashlib.sha256(create_key.encode()).hexdigest()[:32]
|
|
925
|
+
)
|
|
926
|
+
if region is not None:
|
|
927
|
+
body["region"] = region
|
|
928
|
+
if browser_identifier is not None:
|
|
929
|
+
body["browser_identifier"] = browser_identifier
|
|
930
|
+
if viewport is not None:
|
|
931
|
+
body["viewport"] = {"width": viewport[0], "height": viewport[1]}
|
|
932
|
+
if network_preference not in {None, "auto"}:
|
|
933
|
+
body["network_preference"] = network_preference
|
|
934
|
+
if target_hint is not None:
|
|
935
|
+
body["target_hint"] = target_hint
|
|
936
|
+
value = await self._request(
|
|
937
|
+
"POST",
|
|
938
|
+
"/v1/aws-proxy/browser-sessions",
|
|
939
|
+
json=body,
|
|
940
|
+
headers={"Idempotency-Key": create_key},
|
|
941
|
+
)
|
|
942
|
+
resource_id = _required_text(value, "resource_id")
|
|
943
|
+
if any(
|
|
944
|
+
isinstance(existing, BrowserResource)
|
|
945
|
+
and existing.resource_id == resource_id
|
|
946
|
+
for existing in self._resources.values()
|
|
947
|
+
):
|
|
948
|
+
raise ResourceProxyError(
|
|
949
|
+
"browser session already has a live handle",
|
|
950
|
+
code="duplicate-browser-handle",
|
|
951
|
+
)
|
|
952
|
+
status = str(value.get("status") or "ACTIVE")
|
|
953
|
+
if status.upper() not in {"ACTIVE", "READY"}:
|
|
954
|
+
raise ResourceProxyError(
|
|
955
|
+
"browser session is not active", code="browser-unavailable"
|
|
956
|
+
)
|
|
957
|
+
handle = BrowserResource(
|
|
958
|
+
client=self,
|
|
959
|
+
resource_id=resource_id,
|
|
960
|
+
status=status,
|
|
961
|
+
session_deadline=_datetime(
|
|
962
|
+
value.get("session_deadline") or value.get("expires_at")
|
|
963
|
+
),
|
|
964
|
+
network=_optional_text(value.get("network")),
|
|
965
|
+
node_id=_optional_text(value.get("node_id")),
|
|
966
|
+
)
|
|
967
|
+
self._resources[id(handle)] = handle
|
|
968
|
+
return handle
|
|
969
|
+
|
|
970
|
+
async def captcha_types(
|
|
971
|
+
self, *, type: str | None = None, network_preference: str | None = "overseas",
|
|
972
|
+
refresh: bool = False,
|
|
973
|
+
) -> dict[str, Any]:
|
|
974
|
+
"""Query supported task parameters without starting or charging a solver.
|
|
975
|
+
|
|
976
|
+
The directory contains only variants whose Billing classifications have
|
|
977
|
+
effective supplier-cost quotes at ``pricing_version``. Use the returned
|
|
978
|
+
parameters for ``solve_captcha``. Results are briefly cached guidance;
|
|
979
|
+
Resource Proxy rechecks availability when a task is submitted.
|
|
980
|
+
"""
|
|
981
|
+
_validate_network_preference(network_preference)
|
|
982
|
+
selected = self._client_for_network(network_preference)
|
|
983
|
+
if selected is not self:
|
|
984
|
+
return await selected.captcha_types(
|
|
985
|
+
type=type, network_preference=network_preference, refresh=refresh,
|
|
986
|
+
)
|
|
987
|
+
body = {"type": type} if type is not None else {}
|
|
988
|
+
return await self._catalog_query(
|
|
989
|
+
"/v1/captcha/types/query", body,
|
|
990
|
+
cache_key=("captcha_types", type, self._network), refresh=refresh,
|
|
991
|
+
)
|
|
992
|
+
|
|
993
|
+
async def create_captcha_task(
|
|
994
|
+
self,
|
|
995
|
+
*,
|
|
996
|
+
type: str,
|
|
997
|
+
parameters: dict[str, Any] | None = None,
|
|
998
|
+
task_variant: str | None = None,
|
|
999
|
+
website_url: str | None = None,
|
|
1000
|
+
website_key: str | None = None,
|
|
1001
|
+
action: str | None = None,
|
|
1002
|
+
data: str | None = None,
|
|
1003
|
+
page_data: str | None = None,
|
|
1004
|
+
user_agent: str | None = None,
|
|
1005
|
+
api_js_url: str | None = None,
|
|
1006
|
+
data_s: str | None = None,
|
|
1007
|
+
enterprise_s: str | None = None,
|
|
1008
|
+
image_body: str | None = None,
|
|
1009
|
+
case_sensitive: bool | None = None,
|
|
1010
|
+
numeric: Literal[0, 1] | None = None,
|
|
1011
|
+
math: bool | None = None,
|
|
1012
|
+
recognizing_threshold: int | None = None,
|
|
1013
|
+
capmonster_module: str | None = None,
|
|
1014
|
+
idempotency_key: str | None = None,
|
|
1015
|
+
network_preference: str | None = "overseas",
|
|
1016
|
+
) -> CaptchaTask:
|
|
1017
|
+
"""Submit one managed CAPTCHA using the platform's solver credentials.
|
|
1018
|
+
|
|
1019
|
+
Query :meth:`captcha_types` for generic ``type``, ``task_variant`` and
|
|
1020
|
+
vendor ``parameters``. Generic fields cannot be mixed with convenience
|
|
1021
|
+
fields. Structured answers are returned as ``CaptchaTask.data``; the
|
|
1022
|
+
developer applies them to the matching current browser challenge.
|
|
1023
|
+
``capmonster_module`` selects an ImageToText recognizer, not a detector.
|
|
1024
|
+
"""
|
|
1025
|
+
_validate_network_preference(network_preference)
|
|
1026
|
+
selected = self._client_for_network(network_preference)
|
|
1027
|
+
if selected is not self:
|
|
1028
|
+
task = await selected.create_captcha_task(
|
|
1029
|
+
type=type,
|
|
1030
|
+
parameters=parameters,
|
|
1031
|
+
task_variant=task_variant,
|
|
1032
|
+
website_url=website_url,
|
|
1033
|
+
website_key=website_key,
|
|
1034
|
+
action=action,
|
|
1035
|
+
data=data,
|
|
1036
|
+
page_data=page_data,
|
|
1037
|
+
user_agent=user_agent,
|
|
1038
|
+
api_js_url=api_js_url,
|
|
1039
|
+
data_s=data_s,
|
|
1040
|
+
enterprise_s=enterprise_s,
|
|
1041
|
+
image_body=image_body,
|
|
1042
|
+
case_sensitive=case_sensitive,
|
|
1043
|
+
numeric=numeric,
|
|
1044
|
+
math=math,
|
|
1045
|
+
recognizing_threshold=recognizing_threshold,
|
|
1046
|
+
capmonster_module=capmonster_module,
|
|
1047
|
+
idempotency_key=idempotency_key,
|
|
1048
|
+
network_preference=network_preference,
|
|
1049
|
+
)
|
|
1050
|
+
self._captcha_task_networks[task.task_id] = selected.network
|
|
1051
|
+
return task
|
|
1052
|
+
if capmonster_module is not None:
|
|
1053
|
+
if type != "image_to_text":
|
|
1054
|
+
raise ValueError("captcha module is only valid for image_to_text")
|
|
1055
|
+
if not isinstance(capmonster_module, str) or not capmonster_module.isprintable():
|
|
1056
|
+
raise ValueError("captcha module must contain printable characters only")
|
|
1057
|
+
capmonster_module = capmonster_module.strip()
|
|
1058
|
+
if not 1 <= len(capmonster_module) <= 128:
|
|
1059
|
+
raise ValueError("captcha module must contain 1-128 characters")
|
|
1060
|
+
if image_body is not None and not (
|
|
1061
|
+
_MIN_CAPTCHA_IMAGE_BASE64_CHARS
|
|
1062
|
+
<= len(image_body)
|
|
1063
|
+
<= _MAX_CAPTCHA_IMAGE_BASE64_CHARS
|
|
1064
|
+
):
|
|
1065
|
+
raise ValueError("captcha image Base64 length is outside the supported range")
|
|
1066
|
+
body = {
|
|
1067
|
+
"type": type,
|
|
1068
|
+
"parameters": parameters,
|
|
1069
|
+
"task_variant": task_variant,
|
|
1070
|
+
"website_url": website_url,
|
|
1071
|
+
"website_key": website_key,
|
|
1072
|
+
"action": action,
|
|
1073
|
+
"data": data,
|
|
1074
|
+
"page_data": page_data,
|
|
1075
|
+
"user_agent": user_agent,
|
|
1076
|
+
"api_js_url": api_js_url,
|
|
1077
|
+
"data_s": data_s,
|
|
1078
|
+
"enterprise_s": enterprise_s,
|
|
1079
|
+
"image_body": image_body,
|
|
1080
|
+
"case_sensitive": case_sensitive,
|
|
1081
|
+
"numeric": numeric,
|
|
1082
|
+
"math": math,
|
|
1083
|
+
"recognizing_threshold": recognizing_threshold,
|
|
1084
|
+
"capmonster_module": capmonster_module,
|
|
1085
|
+
}
|
|
1086
|
+
if not isinstance(type, str) or not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,63}", type):
|
|
1087
|
+
raise ValueError("invalid CAPTCHA task type")
|
|
1088
|
+
if parameters is not None:
|
|
1089
|
+
if not isinstance(parameters, dict):
|
|
1090
|
+
raise ValueError("CAPTCHA parameters must be an object")
|
|
1091
|
+
if any(value is not None for key, value in body.items()
|
|
1092
|
+
if key not in {"type", "parameters", "task_variant"}):
|
|
1093
|
+
raise ValueError("generic parameters cannot be mixed with convenience fields")
|
|
1094
|
+
try:
|
|
1095
|
+
encoded = jsonlib.dumps(parameters, ensure_ascii=False, allow_nan=False).encode()
|
|
1096
|
+
except (ValueError, TypeError, RecursionError):
|
|
1097
|
+
raise ValueError("CAPTCHA parameters must be finite JSON values") from None
|
|
1098
|
+
if len(encoded) > 262_144:
|
|
1099
|
+
raise ValueError("CAPTCHA parameters exceed the 256 KiB limit")
|
|
1100
|
+
elif task_variant is not None:
|
|
1101
|
+
raise ValueError("task_variant requires generic parameters")
|
|
1102
|
+
async with self._operation_lock:
|
|
1103
|
+
await self.start()
|
|
1104
|
+
self._warn_managed_billing()
|
|
1105
|
+
value = await self._request(
|
|
1106
|
+
"POST", _CAPTCHA_TASKS,
|
|
1107
|
+
json={key: item for key, item in body.items() if item is not None},
|
|
1108
|
+
headers={"Idempotency-Key": idempotency_key or str(uuid.uuid4())},
|
|
1109
|
+
)
|
|
1110
|
+
task = _captcha_task(value)
|
|
1111
|
+
self._captcha_task_networks[task.task_id] = self._network
|
|
1112
|
+
return task
|
|
1113
|
+
|
|
1114
|
+
async def get_captcha_task(
|
|
1115
|
+
self, task_id: str, *, network_preference: str | None = None
|
|
1116
|
+
) -> CaptchaTask:
|
|
1117
|
+
"""Read or poll a task owned by the current execution."""
|
|
1118
|
+
_validate_network_preference(network_preference)
|
|
1119
|
+
if not _CAPTCHA_TASK_ID.fullmatch(task_id):
|
|
1120
|
+
raise ValueError("invalid captcha task identifier")
|
|
1121
|
+
owner = self._captcha_task_networks.get(task_id)
|
|
1122
|
+
if owner is not None and network_preference not in {None, "auto", owner}:
|
|
1123
|
+
raise ValueError(
|
|
1124
|
+
f"captcha task belongs to the {owner} Resource Proxy node"
|
|
1125
|
+
)
|
|
1126
|
+
selected_network = owner or network_preference or "overseas"
|
|
1127
|
+
selected = self._client_for_network(selected_network)
|
|
1128
|
+
if selected is not self:
|
|
1129
|
+
task = await selected.get_captcha_task(
|
|
1130
|
+
task_id, network_preference=selected_network
|
|
1131
|
+
)
|
|
1132
|
+
return task
|
|
1133
|
+
async with self._operation_lock:
|
|
1134
|
+
await self.start()
|
|
1135
|
+
value = await self._request("GET", f"{_CAPTCHA_TASKS}/{task_id}")
|
|
1136
|
+
task = _captcha_task(value, expected_id=task_id)
|
|
1137
|
+
return task
|
|
1138
|
+
|
|
1139
|
+
async def solve_captcha(
|
|
1140
|
+
self,
|
|
1141
|
+
*,
|
|
1142
|
+
type: str,
|
|
1143
|
+
parameters: dict[str, Any] | None = None,
|
|
1144
|
+
task_variant: str | None = None,
|
|
1145
|
+
website_url: str | None = None,
|
|
1146
|
+
website_key: str | None = None,
|
|
1147
|
+
action: str | None = None,
|
|
1148
|
+
data: str | None = None,
|
|
1149
|
+
page_data: str | None = None,
|
|
1150
|
+
user_agent: str | None = None,
|
|
1151
|
+
api_js_url: str | None = None,
|
|
1152
|
+
data_s: str | None = None,
|
|
1153
|
+
enterprise_s: str | None = None,
|
|
1154
|
+
image_body: str | None = None,
|
|
1155
|
+
case_sensitive: bool | None = None,
|
|
1156
|
+
numeric: Literal[0, 1] | None = None,
|
|
1157
|
+
math: bool | None = None,
|
|
1158
|
+
recognizing_threshold: int | None = None,
|
|
1159
|
+
capmonster_module: str | None = None,
|
|
1160
|
+
idempotency_key: str | None = None,
|
|
1161
|
+
timeout: float = 120,
|
|
1162
|
+
network_preference: str | None = "overseas",
|
|
1163
|
+
) -> CaptchaTask:
|
|
1164
|
+
"""Wait for one solver result. The caller verifies it in the browser.
|
|
1165
|
+
|
|
1166
|
+
Optional ``capmonster_module`` selects recognition only for image tasks.
|
|
1167
|
+
"""
|
|
1168
|
+
_validate_network_preference(network_preference)
|
|
1169
|
+
selected = self._client_for_network(network_preference)
|
|
1170
|
+
if selected is not self:
|
|
1171
|
+
return await selected.solve_captcha(
|
|
1172
|
+
type=type,
|
|
1173
|
+
parameters=parameters,
|
|
1174
|
+
task_variant=task_variant,
|
|
1175
|
+
website_url=website_url,
|
|
1176
|
+
website_key=website_key,
|
|
1177
|
+
action=action,
|
|
1178
|
+
data=data,
|
|
1179
|
+
page_data=page_data,
|
|
1180
|
+
user_agent=user_agent,
|
|
1181
|
+
api_js_url=api_js_url,
|
|
1182
|
+
data_s=data_s,
|
|
1183
|
+
enterprise_s=enterprise_s,
|
|
1184
|
+
image_body=image_body,
|
|
1185
|
+
case_sensitive=case_sensitive,
|
|
1186
|
+
numeric=numeric,
|
|
1187
|
+
math=math,
|
|
1188
|
+
recognizing_threshold=recognizing_threshold,
|
|
1189
|
+
capmonster_module=capmonster_module,
|
|
1190
|
+
idempotency_key=idempotency_key,
|
|
1191
|
+
timeout=timeout,
|
|
1192
|
+
network_preference=network_preference,
|
|
1193
|
+
)
|
|
1194
|
+
if not mathlib.isfinite(timeout) or timeout <= 0:
|
|
1195
|
+
raise ValueError("captcha timeout must be finite and positive")
|
|
1196
|
+
try:
|
|
1197
|
+
async with asyncio.timeout(timeout):
|
|
1198
|
+
task = await self.create_captcha_task(
|
|
1199
|
+
type=type, parameters=parameters, task_variant=task_variant,
|
|
1200
|
+
website_url=website_url, website_key=website_key,
|
|
1201
|
+
action=action, data=data, page_data=page_data, user_agent=user_agent,
|
|
1202
|
+
api_js_url=api_js_url, data_s=data_s, enterprise_s=enterprise_s,
|
|
1203
|
+
image_body=image_body, case_sensitive=case_sensitive, numeric=numeric,
|
|
1204
|
+
math=math, recognizing_threshold=recognizing_threshold,
|
|
1205
|
+
capmonster_module=capmonster_module,
|
|
1206
|
+
idempotency_key=idempotency_key,
|
|
1207
|
+
network_preference=network_preference,
|
|
1208
|
+
)
|
|
1209
|
+
while task.status == "processing":
|
|
1210
|
+
await asyncio.sleep(_CAPTCHA_POLL_SECONDS)
|
|
1211
|
+
task = await self.get_captcha_task(task.task_id)
|
|
1212
|
+
except TimeoutError:
|
|
1213
|
+
raise ResourceProxyError(
|
|
1214
|
+
"captcha task exceeded its wait deadline", code="captcha-timeout",
|
|
1215
|
+
) from None
|
|
1216
|
+
if task.status == "failed":
|
|
1217
|
+
raise ResourceProxyError(
|
|
1218
|
+
"captcha task failed", code=task.error_code or "captcha-failed",
|
|
1219
|
+
)
|
|
1220
|
+
return task
|
|
1221
|
+
|
|
1222
|
+
async def captcha_plugin_grant(
|
|
1223
|
+
self, *, allowed_types: list[str], allowed_hosts: list[str] | None = None,
|
|
1224
|
+
ttl_seconds: int = 300,
|
|
1225
|
+
) -> CaptchaPluginGrant:
|
|
1226
|
+
"""Issue a narrow grant; original extensions need the managed adapter patch."""
|
|
1227
|
+
async with self._operation_lock:
|
|
1228
|
+
await self.start()
|
|
1229
|
+
self._warn_managed_billing()
|
|
1230
|
+
value = await self._request(
|
|
1231
|
+
"POST", "/v1/captcha/plugin-grants", json={
|
|
1232
|
+
"allowed_types": allowed_types,
|
|
1233
|
+
"allowed_hosts": allowed_hosts or [],
|
|
1234
|
+
"ttl_seconds": ttl_seconds,
|
|
1235
|
+
},
|
|
1236
|
+
)
|
|
1237
|
+
expires_at = _datetime(value.get("expires_at"))
|
|
1238
|
+
if expires_at is None:
|
|
1239
|
+
raise ResourceProxyError("resource service returned an invalid plugin grant")
|
|
1240
|
+
if value.get("protocol_version") != 1:
|
|
1241
|
+
raise ResourceProxyError("resource service returned an unsupported plugin protocol")
|
|
1242
|
+
return CaptchaPluginGrant(
|
|
1243
|
+
api_url=_required_text(value, "api_url"),
|
|
1244
|
+
client_key=_required_text(value, "client_key"),
|
|
1245
|
+
expires_at=expires_at,
|
|
1246
|
+
node_id=_required_text(value, "node_id"),
|
|
1247
|
+
network=_required_text(value, "network"),
|
|
1248
|
+
protocol_version=1,
|
|
1249
|
+
)
|
|
1250
|
+
|
|
1251
|
+
async def chat_completion(
|
|
1252
|
+
self, payload: Mapping[str, Any], *, idempotency_key: str | None = None,
|
|
1253
|
+
) -> Mapping[str, Any]:
|
|
1254
|
+
"""Send one non-streaming model request from a local or platform execution."""
|
|
1255
|
+
if payload.get("stream", False) is not False:
|
|
1256
|
+
raise ValueError("chat_completion requires a non-streaming request")
|
|
1257
|
+
async with self._operation_lock:
|
|
1258
|
+
await self.start()
|
|
1259
|
+
self._warn_managed_billing()
|
|
1260
|
+
return await self._request(
|
|
1261
|
+
"POST", _MODEL_CHAT_COMPLETIONS, json=dict(payload),
|
|
1262
|
+
headers={"Idempotency-Key": idempotency_key or str(uuid.uuid4())},
|
|
1263
|
+
enveloped=False,
|
|
1264
|
+
)
|
|
1265
|
+
|
|
1266
|
+
async def resource_charge(self) -> Mapping[str, Any] | None:
|
|
1267
|
+
"""Read this local execution's charge receipt, including after close.
|
|
1268
|
+
|
|
1269
|
+
Monetary amounts retain the server's decimal strings. A server or
|
|
1270
|
+
execution without managed billing returns None.
|
|
1271
|
+
"""
|
|
1272
|
+
async with self._operation_lock:
|
|
1273
|
+
if self._local_transport is None:
|
|
1274
|
+
raise RuntimeError("resource_charge is available for local resource executions")
|
|
1275
|
+
if self.execution_id is None:
|
|
1276
|
+
raise RuntimeError("start a local resource execution before reading its charge")
|
|
1277
|
+
if not self._managed_billing_enabled:
|
|
1278
|
+
return None
|
|
1279
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]+", self.execution_id):
|
|
1280
|
+
raise ResourceProxyError("resource service returned an invalid execution ID")
|
|
1281
|
+
return await self._request(
|
|
1282
|
+
"GET", f"/v1/executions/{self.execution_id}/resource-charge",
|
|
1283
|
+
authenticated=False, transport=self._local_transport,
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
async def resource_charges(self) -> tuple[Mapping[str, Any], ...]:
|
|
1287
|
+
"""Read charge receipts for every regional execution that was started."""
|
|
1288
|
+
clients = (self, *self._regional_clients.values())
|
|
1289
|
+
|
|
1290
|
+
async def read(client: ResourceProxy) -> Mapping[str, Any]:
|
|
1291
|
+
error_code: str | None = None
|
|
1292
|
+
error_type: str | None = None
|
|
1293
|
+
started = perf_counter()
|
|
1294
|
+
try:
|
|
1295
|
+
async with asyncio.timeout(client._regional_read_timeout):
|
|
1296
|
+
receipt = await client.resource_charge()
|
|
1297
|
+
except asyncio.CancelledError:
|
|
1298
|
+
raise
|
|
1299
|
+
except TimeoutError as error:
|
|
1300
|
+
receipt = None
|
|
1301
|
+
error_code = "resource-read-timeout"
|
|
1302
|
+
error_type = _safe_error_type(error)
|
|
1303
|
+
except Exception as error:
|
|
1304
|
+
receipt = None
|
|
1305
|
+
error_code = _safe_error_code(error, "receipt-unavailable")
|
|
1306
|
+
error_type = _safe_error_type(error)
|
|
1307
|
+
return {
|
|
1308
|
+
"network": client._network,
|
|
1309
|
+
"execution_id": client.execution_id,
|
|
1310
|
+
"environment": client._environment,
|
|
1311
|
+
"billing_route_id": client._billing_route_id,
|
|
1312
|
+
"receipt": receipt,
|
|
1313
|
+
"error_code": error_code,
|
|
1314
|
+
"error_type": error_type,
|
|
1315
|
+
"duration_ms": round((perf_counter() - started) * 1000, 2),
|
|
1316
|
+
"timeout_seconds": client._regional_read_timeout,
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
started = tuple(client for client in clients if client.execution_id is not None)
|
|
1320
|
+
return tuple(await asyncio.gather(*(read(client) for client in started)))
|
|
1321
|
+
|
|
1322
|
+
async def usage(self) -> Mapping[str, Any]:
|
|
1323
|
+
async with self._operation_lock:
|
|
1324
|
+
await self.start()
|
|
1325
|
+
return await self._request("GET", "/v1/usage")
|
|
1326
|
+
|
|
1327
|
+
async def usages(self) -> tuple[Mapping[str, Any], ...]:
|
|
1328
|
+
"""Read usage separately for each regional execution that was started."""
|
|
1329
|
+
clients = (self, *self._regional_clients.values())
|
|
1330
|
+
|
|
1331
|
+
async def read(client: ResourceProxy) -> Mapping[str, Any]:
|
|
1332
|
+
error_code: str | None = None
|
|
1333
|
+
error_type: str | None = None
|
|
1334
|
+
started = perf_counter()
|
|
1335
|
+
try:
|
|
1336
|
+
async with asyncio.timeout(client._regional_read_timeout):
|
|
1337
|
+
usage: Mapping[str, Any] | None = await client.usage()
|
|
1338
|
+
except asyncio.CancelledError:
|
|
1339
|
+
raise
|
|
1340
|
+
except TimeoutError as error:
|
|
1341
|
+
usage = None
|
|
1342
|
+
error_code = "resource-read-timeout"
|
|
1343
|
+
error_type = _safe_error_type(error)
|
|
1344
|
+
except Exception as error:
|
|
1345
|
+
usage = None
|
|
1346
|
+
error_code = _safe_error_code(error, "usage-unavailable")
|
|
1347
|
+
error_type = _safe_error_type(error)
|
|
1348
|
+
return {
|
|
1349
|
+
"network": client._network,
|
|
1350
|
+
"execution_id": client.execution_id,
|
|
1351
|
+
"environment": client._environment,
|
|
1352
|
+
"usage": usage,
|
|
1353
|
+
"error_code": error_code,
|
|
1354
|
+
"error_type": error_type,
|
|
1355
|
+
"duration_ms": round((perf_counter() - started) * 1000, 2),
|
|
1356
|
+
"timeout_seconds": client._regional_read_timeout,
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
started = tuple(client for client in clients if client.execution_id is not None)
|
|
1360
|
+
return tuple(await asyncio.gather(*(read(client) for client in started)))
|
|
1361
|
+
|
|
1362
|
+
async def close(self) -> None:
|
|
1363
|
+
async with self._operation_lock:
|
|
1364
|
+
async with self._start_lock:
|
|
1365
|
+
if self._closed:
|
|
1366
|
+
return
|
|
1367
|
+
heartbeat = self._heartbeat_task
|
|
1368
|
+
self._heartbeat_task = None
|
|
1369
|
+
if heartbeat is not None:
|
|
1370
|
+
heartbeat.cancel()
|
|
1371
|
+
await asyncio.gather(heartbeat, return_exceptions=True)
|
|
1372
|
+
first_error: BaseException | None = None
|
|
1373
|
+
for regional in self._regional_clients.values():
|
|
1374
|
+
try:
|
|
1375
|
+
await regional.close()
|
|
1376
|
+
except BaseException as error:
|
|
1377
|
+
first_error = first_error or error
|
|
1378
|
+
for resource in list(reversed(self._resources.values())):
|
|
1379
|
+
try:
|
|
1380
|
+
await resource.close()
|
|
1381
|
+
except BaseException as error:
|
|
1382
|
+
first_error = first_error or error
|
|
1383
|
+
self._resources.clear()
|
|
1384
|
+
self._catalog_cache.clear()
|
|
1385
|
+
self._captcha_task_networks.clear()
|
|
1386
|
+
if self.execution_id is not None:
|
|
1387
|
+
try:
|
|
1388
|
+
await self._request("DELETE", "/v1/executions/current")
|
|
1389
|
+
except ResourceProxyError as error:
|
|
1390
|
+
if error.status_code not in {401, 404}:
|
|
1391
|
+
first_error = first_error or error
|
|
1392
|
+
except BaseException as error:
|
|
1393
|
+
first_error = first_error or error
|
|
1394
|
+
self._closed = True
|
|
1395
|
+
if self._owns_transport:
|
|
1396
|
+
try:
|
|
1397
|
+
await self._transport.close()
|
|
1398
|
+
except BaseException as error:
|
|
1399
|
+
first_error = first_error or error
|
|
1400
|
+
if first_error is not None:
|
|
1401
|
+
if not isinstance(first_error, Exception):
|
|
1402
|
+
raise first_error
|
|
1403
|
+
raise ResourceProxyError("managed resource cleanup failed") from None
|
|
1404
|
+
|
|
1405
|
+
async def _heartbeat_loop(self) -> None:
|
|
1406
|
+
while True:
|
|
1407
|
+
await asyncio.sleep(self._heartbeat_interval)
|
|
1408
|
+
try:
|
|
1409
|
+
await self._request("POST", "/v1/executions/heartbeat", json={})
|
|
1410
|
+
except asyncio.CancelledError:
|
|
1411
|
+
raise
|
|
1412
|
+
except Exception:
|
|
1413
|
+
# Server-side expiry remains the authoritative cleanup fallback.
|
|
1414
|
+
continue
|
|
1415
|
+
|
|
1416
|
+
async def _revoke_rejected_execution(self, transport: ResourceTransport) -> None:
|
|
1417
|
+
"""Best-effort revoke using only the newly issued execution token."""
|
|
1418
|
+
try:
|
|
1419
|
+
await self._request(
|
|
1420
|
+
"DELETE",
|
|
1421
|
+
"/v1/executions/current",
|
|
1422
|
+
authenticated=False,
|
|
1423
|
+
transport=transport,
|
|
1424
|
+
)
|
|
1425
|
+
except asyncio.CancelledError:
|
|
1426
|
+
raise
|
|
1427
|
+
except Exception:
|
|
1428
|
+
logger.warning("rejected_resource_execution_cleanup_failed")
|
|
1429
|
+
|
|
1430
|
+
async def _renew_proxy(self, lease_id: str) -> dict[str, Any]:
|
|
1431
|
+
return await self._request("POST", f"/v1/ip-proxy/leases/{lease_id}/renew", json={})
|
|
1432
|
+
|
|
1433
|
+
async def _get_proxy(self, lease_id: str) -> dict[str, Any]:
|
|
1434
|
+
return await self._request("GET", f"/v1/ip-proxy/leases/{lease_id}")
|
|
1435
|
+
|
|
1436
|
+
async def _rotate_proxy(
|
|
1437
|
+
self, lease_id: str, *, credential_generation: int | None
|
|
1438
|
+
) -> dict[str, Any]:
|
|
1439
|
+
body = (
|
|
1440
|
+
{"credential_generation": credential_generation}
|
|
1441
|
+
if credential_generation is not None
|
|
1442
|
+
else {}
|
|
1443
|
+
)
|
|
1444
|
+
return await self._request(
|
|
1445
|
+
"POST",
|
|
1446
|
+
f"/v1/ip-proxy/leases/{lease_id}/credentials/rotate",
|
|
1447
|
+
json=body,
|
|
1448
|
+
headers={"Idempotency-Key": str(uuid.uuid4())},
|
|
1449
|
+
)
|
|
1450
|
+
|
|
1451
|
+
async def _release_proxy(self, lease_id: str) -> None:
|
|
1452
|
+
try:
|
|
1453
|
+
await self._request("DELETE", f"/v1/ip-proxy/leases/{lease_id}")
|
|
1454
|
+
except ResourceProxyError as error:
|
|
1455
|
+
if error.status_code not in {401, 404}:
|
|
1456
|
+
raise
|
|
1457
|
+
|
|
1458
|
+
async def _browser_connection(self, resource_id: str) -> dict[str, Any]:
|
|
1459
|
+
return await self._request(
|
|
1460
|
+
"POST", f"/v1/aws-proxy/browser-sessions/{resource_id}/connections", json={}
|
|
1461
|
+
)
|
|
1462
|
+
|
|
1463
|
+
async def _get_browser(self, resource_id: str) -> dict[str, Any]:
|
|
1464
|
+
return await self._request("GET", f"/v1/aws-proxy/browser-sessions/{resource_id}")
|
|
1465
|
+
|
|
1466
|
+
async def _release_browser(self, resource_id: str) -> None:
|
|
1467
|
+
try:
|
|
1468
|
+
await self._request("DELETE", f"/v1/aws-proxy/browser-sessions/{resource_id}")
|
|
1469
|
+
except ResourceProxyError as error:
|
|
1470
|
+
if error.status_code not in {401, 404}:
|
|
1471
|
+
raise
|
|
1472
|
+
|
|
1473
|
+
def _forget(self, resource: Any) -> None:
|
|
1474
|
+
self._resources.pop(id(resource), None)
|
|
1475
|
+
|
|
1476
|
+
def _warn_managed_billing(self) -> None:
|
|
1477
|
+
if self._managed_billing_enabled and not self._managed_billing_warned:
|
|
1478
|
+
self._managed_billing_warned = True
|
|
1479
|
+
logger.warning(
|
|
1480
|
+
"Managed IP traffic, CAPTCHA solves, and model requests during local debug "
|
|
1481
|
+
"accrue developer resource costs. These costs will be deducted from developer "
|
|
1482
|
+
"earnings at settlement; remaining earnings will be paid out after deduction. "
|
|
1483
|
+
"Use resource_charge() to check usage, prices, and the amount to be deducted. "
|
|
1484
|
+
"Pod cost is shown in cloud debug."
|
|
1485
|
+
)
|
|
1486
|
+
|
|
1487
|
+
def _client_for_network(self, network_preference: str | None) -> ResourceProxy:
|
|
1488
|
+
if not self._regional_mode:
|
|
1489
|
+
return self
|
|
1490
|
+
if network_preference in {None, "auto", "domestic"}:
|
|
1491
|
+
if self._network == "domestic":
|
|
1492
|
+
return self
|
|
1493
|
+
raise ResourceProxyError(
|
|
1494
|
+
"the domestic Resource Proxy node is not configured",
|
|
1495
|
+
code="configuration-error",
|
|
1496
|
+
)
|
|
1497
|
+
if self._network == "overseas":
|
|
1498
|
+
return self
|
|
1499
|
+
client = self._regional_clients.get("overseas")
|
|
1500
|
+
if client is None:
|
|
1501
|
+
raise ResourceProxyError(
|
|
1502
|
+
"the overseas Resource Proxy node is not configured",
|
|
1503
|
+
code="configuration-error",
|
|
1504
|
+
)
|
|
1505
|
+
return client
|
|
1506
|
+
|
|
1507
|
+
async def _request(
|
|
1508
|
+
self,
|
|
1509
|
+
method: str,
|
|
1510
|
+
path: str,
|
|
1511
|
+
*,
|
|
1512
|
+
json: Mapping[str, Any] | None = None,
|
|
1513
|
+
headers: Mapping[str, str] | None = None,
|
|
1514
|
+
authenticated: bool = True,
|
|
1515
|
+
transport: ResourceTransport | None = None,
|
|
1516
|
+
enveloped: bool = True,
|
|
1517
|
+
) -> dict[str, Any]:
|
|
1518
|
+
if authenticated:
|
|
1519
|
+
self._require_open_execution()
|
|
1520
|
+
request_id = uuid.uuid4().hex
|
|
1521
|
+
request_headers = {**(headers or {}), "X-Request-ID": request_id}
|
|
1522
|
+
started = perf_counter()
|
|
1523
|
+
outcome = "error"
|
|
1524
|
+
try:
|
|
1525
|
+
selected_transport = self._transport if transport is None else transport
|
|
1526
|
+
value = selected_transport.request(method, path, json=json, headers=request_headers)
|
|
1527
|
+
payload = await value if inspect.isawaitable(value) else value
|
|
1528
|
+
data = payload.get("data") if enveloped and isinstance(payload, Mapping) else payload
|
|
1529
|
+
if not isinstance(data, Mapping):
|
|
1530
|
+
raise ResourceProxyError("resource service returned an invalid response")
|
|
1531
|
+
outcome = "ok"
|
|
1532
|
+
return dict(data)
|
|
1533
|
+
except asyncio.CancelledError:
|
|
1534
|
+
outcome = "cancelled"
|
|
1535
|
+
raise
|
|
1536
|
+
except ResourceProxyError:
|
|
1537
|
+
raise
|
|
1538
|
+
except Exception as error:
|
|
1539
|
+
raise ResourceProxyError(
|
|
1540
|
+
f"resource service request failed ({type(error).__name__})",
|
|
1541
|
+
code="transport-error",
|
|
1542
|
+
) from None
|
|
1543
|
+
finally:
|
|
1544
|
+
route = (
|
|
1545
|
+
"/v1/executions/{id}/resource-charge" if path.endswith("/resource-charge")
|
|
1546
|
+
else _RESOURCE_PATH.sub(r"\1/{id}", path)
|
|
1547
|
+
)
|
|
1548
|
+
logger.debug(
|
|
1549
|
+
"resource_request method=%s route=%s request_id=%s outcome=%s duration_ms=%.2f",
|
|
1550
|
+
method, route, request_id, outcome, (perf_counter() - started) * 1000,
|
|
1551
|
+
)
|
|
1552
|
+
|
|
1553
|
+
def _require_open_execution(self) -> None:
|
|
1554
|
+
if self._closed:
|
|
1555
|
+
raise RuntimeError("resource client is closed")
|
|
1556
|
+
if self.execution_id is None:
|
|
1557
|
+
raise RuntimeError("use ResourceProxy inside an async context")
|
|
1558
|
+
|
|
1559
|
+
|
|
1560
|
+
def _captcha_task(value: dict[str, Any], *, expected_id: str | None = None) -> CaptchaTask:
|
|
1561
|
+
task_id = value.get("task_id")
|
|
1562
|
+
status = value.get("status")
|
|
1563
|
+
solution = value.get("solution")
|
|
1564
|
+
token = solution.get("token") if isinstance(solution, dict) else None
|
|
1565
|
+
text = solution.get("text") if isinstance(solution, dict) else None
|
|
1566
|
+
structured = solution.get("data") if isinstance(solution, dict) else None
|
|
1567
|
+
ready_solution = (
|
|
1568
|
+
status != "ready"
|
|
1569
|
+
or (isinstance(structured, dict) and bool(structured))
|
|
1570
|
+
or (
|
|
1571
|
+
isinstance(token, str)
|
|
1572
|
+
and bool(token)
|
|
1573
|
+
and text is None
|
|
1574
|
+
)
|
|
1575
|
+
or (
|
|
1576
|
+
isinstance(text, str)
|
|
1577
|
+
and bool(text)
|
|
1578
|
+
and token is None
|
|
1579
|
+
)
|
|
1580
|
+
)
|
|
1581
|
+
if (
|
|
1582
|
+
not isinstance(task_id, str)
|
|
1583
|
+
or not _CAPTCHA_TASK_ID.fullmatch(task_id)
|
|
1584
|
+
or (expected_id is not None and expected_id != task_id)
|
|
1585
|
+
or status not in ("processing", "ready", "failed")
|
|
1586
|
+
or not ready_solution
|
|
1587
|
+
):
|
|
1588
|
+
raise ResourceProxyError("resource service returned an invalid captcha task")
|
|
1589
|
+
return CaptchaTask(
|
|
1590
|
+
task_id=task_id,
|
|
1591
|
+
status=status,
|
|
1592
|
+
token=token if status == "ready" else None,
|
|
1593
|
+
text=text if status == "ready" else None,
|
|
1594
|
+
user_agent=_optional_text(solution.get("user_agent"))
|
|
1595
|
+
if status == "ready" and isinstance(solution, dict) else None,
|
|
1596
|
+
error_code=_optional_text(value.get("error_code")) if status == "failed" else None,
|
|
1597
|
+
node_id=_optional_text(value.get("node_id")),
|
|
1598
|
+
network=_optional_text(value.get("network")),
|
|
1599
|
+
data=(copy.deepcopy(structured)
|
|
1600
|
+
if status == "ready" and isinstance(structured, dict) else None),
|
|
1601
|
+
)
|
|
1602
|
+
|
|
1603
|
+
|
|
1604
|
+
def _decode_response(response: httpx.Response) -> Mapping[str, Any]:
|
|
1605
|
+
try:
|
|
1606
|
+
payload = response.json()
|
|
1607
|
+
except ValueError:
|
|
1608
|
+
payload = None
|
|
1609
|
+
if response.status_code >= 400:
|
|
1610
|
+
error = payload.get("error") if isinstance(payload, dict) else None
|
|
1611
|
+
code = error.get("code") if isinstance(error, dict) else "resource-service-error"
|
|
1612
|
+
message = error.get("message") if isinstance(error, dict) else None
|
|
1613
|
+
raise ResourceProxyError(
|
|
1614
|
+
str(message or f"resource service returned HTTP {response.status_code}"),
|
|
1615
|
+
code=str(code),
|
|
1616
|
+
status_code=response.status_code,
|
|
1617
|
+
)
|
|
1618
|
+
if not isinstance(payload, Mapping):
|
|
1619
|
+
raise ResourceProxyError("resource service returned an invalid response")
|
|
1620
|
+
return payload
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
def _validate_network_preference(value: str | None) -> None:
|
|
1624
|
+
if value is not None and (
|
|
1625
|
+
not isinstance(value, str) or value not in {"auto", "domestic", "overseas"}
|
|
1626
|
+
):
|
|
1627
|
+
raise ValueError("network_preference must be auto, domestic or overseas")
|
|
1628
|
+
|
|
1629
|
+
|
|
1630
|
+
def _normalize_country(value: str | None) -> str | None:
|
|
1631
|
+
if value is None:
|
|
1632
|
+
return None
|
|
1633
|
+
if not isinstance(value, str) or re.fullmatch(r"[A-Za-z]{2}", value) is None:
|
|
1634
|
+
raise ValueError("country must be an ISO alpha-2 code")
|
|
1635
|
+
return value.upper()
|
|
1636
|
+
|
|
1637
|
+
|
|
1638
|
+
def _optional_catalog_code(value: str | None, name: str) -> str | None:
|
|
1639
|
+
if value is None:
|
|
1640
|
+
return None
|
|
1641
|
+
if (
|
|
1642
|
+
not isinstance(value, str)
|
|
1643
|
+
or not 1 <= len(value) <= 64
|
|
1644
|
+
or any(ord(char) < 32 for char in value)
|
|
1645
|
+
):
|
|
1646
|
+
raise ValueError(f"{name} must contain 1-64 printable characters")
|
|
1647
|
+
return value
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def _validate_catalog_response(value: Mapping[str, Any], *, locations: bool) -> None:
|
|
1651
|
+
if value.get("network") not in {"domestic", "overseas"}:
|
|
1652
|
+
raise ResourceProxyError(
|
|
1653
|
+
"resource service returned an invalid proxy catalog",
|
|
1654
|
+
code="invalid-response",
|
|
1655
|
+
)
|
|
1656
|
+
if value.get("coverage_only") is not True:
|
|
1657
|
+
raise ResourceProxyError(
|
|
1658
|
+
"resource service returned an invalid proxy catalog",
|
|
1659
|
+
code="invalid-response",
|
|
1660
|
+
)
|
|
1661
|
+
key = "items" if locations else "groups"
|
|
1662
|
+
rows = value.get(key)
|
|
1663
|
+
if not isinstance(rows, list) or any(not isinstance(row, Mapping) for row in rows):
|
|
1664
|
+
raise ResourceProxyError(
|
|
1665
|
+
"resource service returned an invalid proxy catalog",
|
|
1666
|
+
code="invalid-response",
|
|
1667
|
+
)
|
|
1668
|
+
if locations:
|
|
1669
|
+
if value.get("level") not in {"country", "state", "city"}:
|
|
1670
|
+
raise ResourceProxyError(
|
|
1671
|
+
"resource service returned an invalid proxy catalog",
|
|
1672
|
+
code="invalid-response",
|
|
1673
|
+
)
|
|
1674
|
+
for row in rows:
|
|
1675
|
+
if not isinstance(row.get("code"), str) or not row["code"]:
|
|
1676
|
+
raise ResourceProxyError(
|
|
1677
|
+
"resource service returned an invalid proxy catalog",
|
|
1678
|
+
code="invalid-response",
|
|
1679
|
+
)
|
|
1680
|
+
_validate_location_availability(row)
|
|
1681
|
+
else:
|
|
1682
|
+
for row in rows:
|
|
1683
|
+
complete = row.get("duration_catalog_complete")
|
|
1684
|
+
if complete is not None and not isinstance(complete, bool):
|
|
1685
|
+
raise ResourceProxyError(
|
|
1686
|
+
"resource service returned an invalid proxy catalog",
|
|
1687
|
+
code="invalid-response",
|
|
1688
|
+
)
|
|
1689
|
+
|
|
1690
|
+
|
|
1691
|
+
def _validate_location_availability(row: Mapping[str, Any]) -> None:
|
|
1692
|
+
availability = row.get("availability")
|
|
1693
|
+
if availability is not None and availability not in {
|
|
1694
|
+
"unknown", "temporarily_unavailable",
|
|
1695
|
+
}:
|
|
1696
|
+
raise ResourceProxyError(
|
|
1697
|
+
"resource service returned an invalid proxy catalog",
|
|
1698
|
+
code="invalid-response",
|
|
1699
|
+
)
|
|
1700
|
+
eligible = row.get("eligible_routes")
|
|
1701
|
+
cooling = row.get("cooling_routes")
|
|
1702
|
+
for value in (eligible, cooling):
|
|
1703
|
+
if value is not None and (
|
|
1704
|
+
isinstance(value, bool) or not isinstance(value, int) or value < 0
|
|
1705
|
+
):
|
|
1706
|
+
raise ResourceProxyError(
|
|
1707
|
+
"resource service returned an invalid proxy catalog",
|
|
1708
|
+
code="invalid-response",
|
|
1709
|
+
)
|
|
1710
|
+
if isinstance(eligible, int) and isinstance(cooling, int) and cooling > eligible:
|
|
1711
|
+
raise ResourceProxyError(
|
|
1712
|
+
"resource service returned an invalid proxy catalog",
|
|
1713
|
+
code="invalid-response",
|
|
1714
|
+
)
|
|
1715
|
+
retry_at = row.get("earliest_retry_at")
|
|
1716
|
+
if retry_at is not None:
|
|
1717
|
+
if not isinstance(retry_at, str):
|
|
1718
|
+
raise ResourceProxyError(
|
|
1719
|
+
"resource service returned an invalid proxy catalog",
|
|
1720
|
+
code="invalid-response",
|
|
1721
|
+
)
|
|
1722
|
+
try:
|
|
1723
|
+
parsed = datetime.fromisoformat(retry_at.replace("Z", "+00:00"))
|
|
1724
|
+
except ValueError:
|
|
1725
|
+
parsed = None
|
|
1726
|
+
if parsed is None or parsed.tzinfo is None:
|
|
1727
|
+
raise ResourceProxyError(
|
|
1728
|
+
"resource service returned an invalid proxy catalog",
|
|
1729
|
+
code="invalid-response",
|
|
1730
|
+
)
|
|
1731
|
+
|
|
1732
|
+
|
|
1733
|
+
def _safe_error_code(error: Exception, fallback: str) -> str:
|
|
1734
|
+
code = error.code if isinstance(error, ResourceProxyError) else fallback
|
|
1735
|
+
return code if isinstance(code, str) and _SAFE_ERROR_CODE.fullmatch(code) else fallback
|
|
1736
|
+
|
|
1737
|
+
|
|
1738
|
+
def _safe_error_type(error: Exception) -> str:
|
|
1739
|
+
name = type(error).__name__
|
|
1740
|
+
return name if re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,79}", name) else "Exception"
|
|
1741
|
+
|
|
1742
|
+
|
|
1743
|
+
def _validate_regional_read_timeout(value: float) -> float:
|
|
1744
|
+
try:
|
|
1745
|
+
normalized = float(value)
|
|
1746
|
+
except (OverflowError, TypeError, ValueError):
|
|
1747
|
+
normalized = mathlib.nan
|
|
1748
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or (
|
|
1749
|
+
not mathlib.isfinite(normalized)
|
|
1750
|
+
or normalized <= 0
|
|
1751
|
+
or normalized > _MAX_REGIONAL_READ_TIMEOUT_SECONDS
|
|
1752
|
+
):
|
|
1753
|
+
raise ValueError(
|
|
1754
|
+
"regional_read_timeout must be finite, positive, and at most 30 seconds"
|
|
1755
|
+
)
|
|
1756
|
+
return normalized
|
|
1757
|
+
|
|
1758
|
+
|
|
1759
|
+
def _configured_regional_read_timeout(
|
|
1760
|
+
env: Mapping[str, str], override: float | None
|
|
1761
|
+
) -> float:
|
|
1762
|
+
try:
|
|
1763
|
+
if override is not None:
|
|
1764
|
+
return _validate_regional_read_timeout(override)
|
|
1765
|
+
raw = env.get("DATAOS_RESOURCE_READ_TIMEOUT_SECONDS")
|
|
1766
|
+
if raw is None:
|
|
1767
|
+
return _DEFAULT_REGIONAL_READ_TIMEOUT_SECONDS
|
|
1768
|
+
return _validate_regional_read_timeout(float(raw))
|
|
1769
|
+
except (TypeError, ValueError):
|
|
1770
|
+
raise ResourceProxyError(
|
|
1771
|
+
"resource read timeout must be a finite number greater than 0 and at most 30 seconds",
|
|
1772
|
+
code="configuration-error",
|
|
1773
|
+
) from None
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
def _validate_line_tier(value: str) -> None:
|
|
1777
|
+
if not isinstance(value, str) or value not in {"standard", "enhanced"}:
|
|
1778
|
+
raise ValueError("line_tier must be standard or enhanced")
|
|
1779
|
+
|
|
1780
|
+
|
|
1781
|
+
_PROXY_GROUPS = frozenset({
|
|
1782
|
+
"auto",
|
|
1783
|
+
"domestic_basic",
|
|
1784
|
+
"overseas_basic",
|
|
1785
|
+
"overseas_enhanced",
|
|
1786
|
+
})
|
|
1787
|
+
|
|
1788
|
+
|
|
1789
|
+
def _validate_proxy_group(value: str | None) -> None:
|
|
1790
|
+
if value is not None and (not isinstance(value, str) or value not in _PROXY_GROUPS):
|
|
1791
|
+
raise ValueError(
|
|
1792
|
+
"proxy_group must be auto, domestic_basic, overseas_basic, "
|
|
1793
|
+
"or overseas_enhanced"
|
|
1794
|
+
)
|
|
1795
|
+
|
|
1796
|
+
|
|
1797
|
+
def _network_for_proxy_group(value: str | None) -> str | None:
|
|
1798
|
+
if value == "domestic_basic":
|
|
1799
|
+
return "domestic"
|
|
1800
|
+
if value in {"overseas_basic", "overseas_enhanced"}:
|
|
1801
|
+
return "overseas"
|
|
1802
|
+
return None
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
def _validate_target_hint(value: str | None) -> None:
|
|
1806
|
+
if value is None:
|
|
1807
|
+
return
|
|
1808
|
+
if not isinstance(value, str) or len(value) > 253 or not re.fullmatch(
|
|
1809
|
+
r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
|
|
1810
|
+
r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*",
|
|
1811
|
+
value,
|
|
1812
|
+
):
|
|
1813
|
+
raise ValueError("target_hint must contain a host name only")
|