datafuel 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datafuel/__init__.py +165 -0
- datafuel/_client.py +1327 -0
- datafuel/_core.py +556 -0
- datafuel/errors.py +167 -0
- datafuel/models.py +659 -0
- datafuel/py.typed +0 -0
- datafuel-0.5.0.dist-info/METADATA +329 -0
- datafuel-0.5.0.dist-info/RECORD +10 -0
- datafuel-0.5.0.dist-info/WHEEL +4 -0
- datafuel-0.5.0.dist-info/licenses/LICENSE +21 -0
datafuel/_core.py
ADDED
|
@@ -0,0 +1,556 @@
|
|
|
1
|
+
"""Request building and answer classification. Pure functions, no I/O.
|
|
2
|
+
|
|
3
|
+
Every decision lives here — which envelope a module needs, where the sticky
|
|
4
|
+
session keys go, how ``AI`` expands, whether an answer is retryable — so the
|
|
5
|
+
sync and the async client cannot drift apart.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import random
|
|
12
|
+
import uuid
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from datetime import date
|
|
15
|
+
from importlib.metadata import PackageNotFoundError
|
|
16
|
+
from importlib.metadata import version as _installed_version
|
|
17
|
+
from typing import Any
|
|
18
|
+
from urllib.parse import quote
|
|
19
|
+
|
|
20
|
+
from .errors import APIError, Unavailable
|
|
21
|
+
from .models import AI, Proxy
|
|
22
|
+
|
|
23
|
+
DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1"
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
#: Read from the installed metadata, so pyproject.toml stays the one place
|
|
27
|
+
#: the version is written down.
|
|
28
|
+
VERSION = _installed_version("datafuel")
|
|
29
|
+
except PackageNotFoundError: # pragma: no cover - running from a source tree
|
|
30
|
+
VERSION = "0.0.0+dev"
|
|
31
|
+
|
|
32
|
+
#: How long a synchronous call waits for a result before giving up.
|
|
33
|
+
DEFAULT_TIMEOUT = 180.0
|
|
34
|
+
#: Gap between two attempts at a task the API is still processing.
|
|
35
|
+
STILL_PROCESSING_DELAY = 2.0
|
|
36
|
+
STILL_PROCESSING = "TASK_STILL_PROCESSING"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True, repr=False)
|
|
40
|
+
class Request:
|
|
41
|
+
"""One HTTP call, ready for either client to send."""
|
|
42
|
+
|
|
43
|
+
method: str
|
|
44
|
+
path: str
|
|
45
|
+
params: dict[str, Any] | None = None
|
|
46
|
+
body: Any = None
|
|
47
|
+
idempotency_key: str | None = None
|
|
48
|
+
#: A 503 that carries a health report is an answer, not an error.
|
|
49
|
+
degraded_ok: bool = False
|
|
50
|
+
#: A write the API treats as a no-op when repeated, e.g. a cancel.
|
|
51
|
+
idempotent: bool = False
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def retryable(self) -> bool:
|
|
55
|
+
"""GETs are safe by nature, writes when they carry an idempotency key
|
|
56
|
+
or repeating them changes nothing."""
|
|
57
|
+
return self.method == "GET" or self.idempotency_key is not None or self.idempotent
|
|
58
|
+
|
|
59
|
+
def __repr__(self) -> str:
|
|
60
|
+
"""Method and path only: the body can hold an LLM key or a cookie header."""
|
|
61
|
+
return f"<Request {self.method} {self.path}>"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def path_segment(value: str) -> str:
|
|
65
|
+
"""Escape an id before it becomes part of a URL path.
|
|
66
|
+
|
|
67
|
+
An id is caller data. Unescaped, a stray ``/`` or ``?`` in one would move
|
|
68
|
+
the request to a different endpoint.
|
|
69
|
+
"""
|
|
70
|
+
return quote(str(value), safe="")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def new_idempotency_key() -> str:
|
|
74
|
+
return str(uuid.uuid4())
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def headers(api_key: str, user_agent: str, request: Request) -> dict[str, str]:
|
|
78
|
+
"""The headers every call carries. The idempotency key sits beside the API key."""
|
|
79
|
+
out = {
|
|
80
|
+
"X-API-Key": api_key,
|
|
81
|
+
"Accept": "application/json",
|
|
82
|
+
"User-Agent": user_agent,
|
|
83
|
+
}
|
|
84
|
+
if request.body is not None:
|
|
85
|
+
out["Content-Type"] = "application/json"
|
|
86
|
+
if request.idempotency_key:
|
|
87
|
+
out["Idempotency-Key"] = request.idempotency_key
|
|
88
|
+
return out
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _selector(value: str | dict[str, str]) -> str:
|
|
92
|
+
"""The API takes multi-field extractors as a JSON object string."""
|
|
93
|
+
return value if isinstance(value, str) else json.dumps(value)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def scrape_attributes(
|
|
97
|
+
*,
|
|
98
|
+
format: str | None = None,
|
|
99
|
+
js_rendering: bool = False,
|
|
100
|
+
wait_for: str | None = None,
|
|
101
|
+
wait_for_timeout_ms: int | None = None,
|
|
102
|
+
js_instructions: list[dict[str, Any]] | dict[str, Any] | None = None,
|
|
103
|
+
block_resource: str | list[str] | None = None,
|
|
104
|
+
main_content_only: bool = False,
|
|
105
|
+
include_images: bool | None = None,
|
|
106
|
+
extract: str | dict[str, str] | None = None,
|
|
107
|
+
extract_regex: str | dict[str, str] | None = None,
|
|
108
|
+
template: str | None = None,
|
|
109
|
+
method: str | None = None,
|
|
110
|
+
body: str | None = None,
|
|
111
|
+
content_type: str | None = None,
|
|
112
|
+
headers: dict[str, str] | None = None,
|
|
113
|
+
header_order: list[str] | None = None,
|
|
114
|
+
cookies: str | None = None,
|
|
115
|
+
user_agent: str | None = None,
|
|
116
|
+
user_agent_type: str | None = None,
|
|
117
|
+
ai: AI | None = None,
|
|
118
|
+
) -> dict[str, Any]:
|
|
119
|
+
"""Flatten the page options into the API's ``attributes`` object.
|
|
120
|
+
|
|
121
|
+
Unset options are left out of the body entirely, never sent as null.
|
|
122
|
+
"""
|
|
123
|
+
attrs: dict[str, Any] = {}
|
|
124
|
+
if format:
|
|
125
|
+
attrs["result_format"] = str(getattr(format, "value", format))
|
|
126
|
+
if js_rendering:
|
|
127
|
+
attrs["js_rendering"] = True
|
|
128
|
+
if wait_for:
|
|
129
|
+
attrs["wait_for_selector"] = wait_for
|
|
130
|
+
if wait_for_timeout_ms:
|
|
131
|
+
attrs["wait_for_selector_timeout_ms"] = wait_for_timeout_ms
|
|
132
|
+
if js_instructions:
|
|
133
|
+
attrs["js_instructions"] = js_instructions
|
|
134
|
+
if block_resource:
|
|
135
|
+
attrs["block_resource"] = (
|
|
136
|
+
block_resource if isinstance(block_resource, str) else list(block_resource)
|
|
137
|
+
)
|
|
138
|
+
if main_content_only:
|
|
139
|
+
attrs["main_content_only"] = True
|
|
140
|
+
if include_images is not None:
|
|
141
|
+
attrs["include_images"] = include_images
|
|
142
|
+
if extract:
|
|
143
|
+
attrs["extract_selector"] = _selector(extract)
|
|
144
|
+
if extract_regex:
|
|
145
|
+
attrs["extract_regex"] = _selector(extract_regex)
|
|
146
|
+
if template:
|
|
147
|
+
attrs["result_template"] = template
|
|
148
|
+
if method:
|
|
149
|
+
attrs["method"] = method
|
|
150
|
+
if body:
|
|
151
|
+
attrs["body"] = body
|
|
152
|
+
if content_type:
|
|
153
|
+
attrs["content_type"] = content_type
|
|
154
|
+
if headers:
|
|
155
|
+
attrs["headers"] = headers
|
|
156
|
+
if header_order:
|
|
157
|
+
attrs["header_order"] = header_order
|
|
158
|
+
if cookies:
|
|
159
|
+
attrs["cookie_string"] = cookies
|
|
160
|
+
if user_agent:
|
|
161
|
+
attrs["user_agent"] = user_agent
|
|
162
|
+
if user_agent_type:
|
|
163
|
+
attrs["user_agent_type"] = user_agent_type
|
|
164
|
+
if ai is not None:
|
|
165
|
+
attrs.update(ai.attributes())
|
|
166
|
+
return attrs
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def envelope(
|
|
170
|
+
task_type: str,
|
|
171
|
+
attributes: dict[str, Any],
|
|
172
|
+
*,
|
|
173
|
+
proxy: Proxy | None = None,
|
|
174
|
+
multithreaded: bool | None = None,
|
|
175
|
+
) -> dict[str, Any]:
|
|
176
|
+
"""The body every write shares."""
|
|
177
|
+
body: dict[str, Any] = {"type": task_type}
|
|
178
|
+
if proxy is not None:
|
|
179
|
+
body.update(proxy.envelope())
|
|
180
|
+
if multithreaded is not None:
|
|
181
|
+
body["multithreaded"] = multithreaded
|
|
182
|
+
body["attributes"] = attributes
|
|
183
|
+
return body
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _page_attributes(
|
|
187
|
+
url_field: str,
|
|
188
|
+
target: Any,
|
|
189
|
+
proxy: Proxy | None,
|
|
190
|
+
options: dict[str, Any],
|
|
191
|
+
) -> dict[str, Any]:
|
|
192
|
+
attrs = scrape_attributes(**options)
|
|
193
|
+
attrs[url_field] = target
|
|
194
|
+
if proxy is not None:
|
|
195
|
+
attrs.update(proxy.session())
|
|
196
|
+
return attrs
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def build_scrape(
|
|
200
|
+
url: str, *, proxy: Proxy | None, options: dict[str, Any], idempotency_key: str
|
|
201
|
+
) -> Request:
|
|
202
|
+
attrs = _page_attributes("url", url, proxy, options)
|
|
203
|
+
return Request(
|
|
204
|
+
"POST",
|
|
205
|
+
"/task",
|
|
206
|
+
body=envelope("unlocker", attrs, proxy=proxy),
|
|
207
|
+
idempotency_key=idempotency_key,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def build_job(
|
|
212
|
+
urls: list[str],
|
|
213
|
+
*,
|
|
214
|
+
proxy: Proxy | None,
|
|
215
|
+
sequential: bool,
|
|
216
|
+
options: dict[str, Any],
|
|
217
|
+
idempotency_key: str,
|
|
218
|
+
) -> Request:
|
|
219
|
+
attrs = _page_attributes("urls", list(urls), proxy, options)
|
|
220
|
+
return Request(
|
|
221
|
+
"POST",
|
|
222
|
+
"/job",
|
|
223
|
+
body=envelope("unlocker", attrs, proxy=proxy, multithreaded=not sequential),
|
|
224
|
+
idempotency_key=idempotency_key,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def ask_attributes(
|
|
229
|
+
prompt_field: str,
|
|
230
|
+
prompt: Any,
|
|
231
|
+
*,
|
|
232
|
+
engine: str,
|
|
233
|
+
model: str | None = None,
|
|
234
|
+
websearch: bool = False,
|
|
235
|
+
follow_up: str | None = None,
|
|
236
|
+
country: str | None = None,
|
|
237
|
+
location: str | None = None,
|
|
238
|
+
format: str | None = None,
|
|
239
|
+
) -> dict[str, Any]:
|
|
240
|
+
"""Attributes for the llm_scraping module, which reads its proxy country here."""
|
|
241
|
+
attrs: dict[str, Any] = {prompt_field: prompt, "engine": str(getattr(engine, "value", engine))}
|
|
242
|
+
if model:
|
|
243
|
+
attrs["model"] = model
|
|
244
|
+
if websearch:
|
|
245
|
+
attrs["websearch"] = True
|
|
246
|
+
if follow_up:
|
|
247
|
+
attrs["follow_up_prompt"] = follow_up
|
|
248
|
+
if country:
|
|
249
|
+
attrs["proxy_country"] = country
|
|
250
|
+
if location:
|
|
251
|
+
attrs["location"] = location
|
|
252
|
+
if format:
|
|
253
|
+
attrs["result_format"] = str(getattr(format, "value", format))
|
|
254
|
+
return attrs
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def build_ask(prompt: str, *, options: dict[str, Any], idempotency_key: str) -> Request:
|
|
258
|
+
attrs = ask_attributes("prompt", prompt, **options)
|
|
259
|
+
return Request(
|
|
260
|
+
"POST", "/task", body=envelope("llm_scraping", attrs), idempotency_key=idempotency_key
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def build_ask_job(
|
|
265
|
+
prompts: list[str], *, sequential: bool, options: dict[str, Any], idempotency_key: str
|
|
266
|
+
) -> Request:
|
|
267
|
+
attrs = ask_attributes("prompts", list(prompts), **options)
|
|
268
|
+
return Request(
|
|
269
|
+
"POST",
|
|
270
|
+
"/job",
|
|
271
|
+
body=envelope("llm_scraping", attrs, multithreaded=not sequential),
|
|
272
|
+
idempotency_key=idempotency_key,
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def serp_attributes(
|
|
277
|
+
query_field: str,
|
|
278
|
+
query: Any,
|
|
279
|
+
*,
|
|
280
|
+
country: str | None = None,
|
|
281
|
+
language: str | None = None,
|
|
282
|
+
location: str | None = None,
|
|
283
|
+
page: int | None = None,
|
|
284
|
+
google_domain: str | None = None,
|
|
285
|
+
uule: str | None = None,
|
|
286
|
+
lat: float | None = None,
|
|
287
|
+
lon: float | None = None,
|
|
288
|
+
radius: int | None = None,
|
|
289
|
+
cr: str | None = None,
|
|
290
|
+
lr: str | None = None,
|
|
291
|
+
tbs: str | None = None,
|
|
292
|
+
safe: str | None = None,
|
|
293
|
+
nfpr: bool | None = None,
|
|
294
|
+
filter: bool | None = None,
|
|
295
|
+
uds: str | None = None,
|
|
296
|
+
kgmid: str | None = None,
|
|
297
|
+
si: str | None = None,
|
|
298
|
+
ludocid: str | None = None,
|
|
299
|
+
lsig: str | None = None,
|
|
300
|
+
ibp: str | None = None,
|
|
301
|
+
device: str | None = None,
|
|
302
|
+
ai_overview: bool = False,
|
|
303
|
+
proxy_country: str | None = None,
|
|
304
|
+
format: str | None = None,
|
|
305
|
+
) -> dict[str, Any]:
|
|
306
|
+
"""Attributes for the serp module. Unset options stay out of the body."""
|
|
307
|
+
attrs: dict[str, Any] = {query_field: query}
|
|
308
|
+
strings = {
|
|
309
|
+
"country": country,
|
|
310
|
+
"language": language,
|
|
311
|
+
"location": location,
|
|
312
|
+
"google_domain": google_domain,
|
|
313
|
+
"uule": uule,
|
|
314
|
+
"cr": cr,
|
|
315
|
+
"lr": lr,
|
|
316
|
+
"tbs": tbs,
|
|
317
|
+
"safe": safe,
|
|
318
|
+
"uds": uds,
|
|
319
|
+
"kgmid": kgmid,
|
|
320
|
+
"si": si,
|
|
321
|
+
"ludocid": ludocid,
|
|
322
|
+
"lsig": lsig,
|
|
323
|
+
"ibp": ibp,
|
|
324
|
+
"device": device,
|
|
325
|
+
"proxy_country": proxy_country,
|
|
326
|
+
}
|
|
327
|
+
attrs.update({name: value for name, value in strings.items() if value})
|
|
328
|
+
if page:
|
|
329
|
+
attrs["page"] = page
|
|
330
|
+
if radius:
|
|
331
|
+
attrs["radius"] = radius
|
|
332
|
+
for name, flag in (("lat", lat), ("lon", lon), ("nfpr", nfpr), ("filter", filter)):
|
|
333
|
+
if flag is not None:
|
|
334
|
+
attrs[name] = flag
|
|
335
|
+
if ai_overview:
|
|
336
|
+
attrs["ai_overview"] = True
|
|
337
|
+
if format:
|
|
338
|
+
attrs["result_format"] = str(getattr(format, "value", format))
|
|
339
|
+
return attrs
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def build_search(query: str, *, options: dict[str, Any], idempotency_key: str) -> Request:
|
|
343
|
+
attrs = serp_attributes("query", query, **options)
|
|
344
|
+
return Request("POST", "/task", body=envelope("serp", attrs), idempotency_key=idempotency_key)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def build_search_job(
|
|
348
|
+
queries: list[str], *, sequential: bool, options: dict[str, Any], idempotency_key: str
|
|
349
|
+
) -> Request:
|
|
350
|
+
attrs = serp_attributes("queries", list(queries), **options)
|
|
351
|
+
return Request(
|
|
352
|
+
"POST",
|
|
353
|
+
"/job",
|
|
354
|
+
body=envelope("serp", attrs, multithreaded=not sequential),
|
|
355
|
+
idempotency_key=idempotency_key,
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def build_map(
|
|
360
|
+
url: str,
|
|
361
|
+
*,
|
|
362
|
+
proxy: Proxy | None,
|
|
363
|
+
search: str | None = None,
|
|
364
|
+
sitemap: str | None = None,
|
|
365
|
+
limit: int | None = None,
|
|
366
|
+
include_subdomains: bool = False,
|
|
367
|
+
ignore_sitemap: bool = False,
|
|
368
|
+
sitemap_only: bool = False,
|
|
369
|
+
user_agent: str | None = None,
|
|
370
|
+
user_agent_type: str | None = None,
|
|
371
|
+
idempotency_key: str,
|
|
372
|
+
) -> Request:
|
|
373
|
+
attrs: dict[str, Any] = {"url": url}
|
|
374
|
+
if search:
|
|
375
|
+
attrs["search"] = search
|
|
376
|
+
if sitemap:
|
|
377
|
+
attrs["sitemap"] = sitemap
|
|
378
|
+
if limit:
|
|
379
|
+
attrs["limit"] = limit
|
|
380
|
+
if include_subdomains:
|
|
381
|
+
attrs["include_subdomains"] = True
|
|
382
|
+
if ignore_sitemap:
|
|
383
|
+
attrs["ignore_sitemap"] = True
|
|
384
|
+
if sitemap_only:
|
|
385
|
+
attrs["sitemap_only"] = True
|
|
386
|
+
if user_agent:
|
|
387
|
+
attrs["user_agent"] = user_agent
|
|
388
|
+
if user_agent_type:
|
|
389
|
+
attrs["user_agent_type"] = user_agent_type
|
|
390
|
+
if proxy is not None:
|
|
391
|
+
attrs.update(proxy.session())
|
|
392
|
+
return Request(
|
|
393
|
+
"POST", "/map", body=envelope("map", attrs, proxy=proxy), idempotency_key=idempotency_key
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def build_crawl(
|
|
398
|
+
url: str,
|
|
399
|
+
*,
|
|
400
|
+
proxy: Proxy | None,
|
|
401
|
+
max_pages: int | None = None,
|
|
402
|
+
max_depth: int | None = None,
|
|
403
|
+
include_paths: list[str] | None = None,
|
|
404
|
+
exclude_paths: list[str] | None = None,
|
|
405
|
+
include_subdomains: bool = False,
|
|
406
|
+
allow_backward_links: bool = False,
|
|
407
|
+
concurrency: int | None = None,
|
|
408
|
+
options: dict[str, Any],
|
|
409
|
+
idempotency_key: str,
|
|
410
|
+
) -> Request:
|
|
411
|
+
if options.get("ai") is not None:
|
|
412
|
+
raise ValueError("crawl does not support ai=: the API rejects result_use_ai on crawls")
|
|
413
|
+
attrs = _page_attributes("url", url, proxy, options)
|
|
414
|
+
if max_pages:
|
|
415
|
+
attrs["max_pages"] = max_pages
|
|
416
|
+
if max_depth:
|
|
417
|
+
attrs["max_depth"] = max_depth
|
|
418
|
+
if concurrency:
|
|
419
|
+
attrs["concurrency"] = concurrency
|
|
420
|
+
if include_paths:
|
|
421
|
+
attrs["include_paths"] = list(include_paths)
|
|
422
|
+
if exclude_paths:
|
|
423
|
+
attrs["exclude_paths"] = list(exclude_paths)
|
|
424
|
+
if include_subdomains:
|
|
425
|
+
attrs["include_subdomains"] = True
|
|
426
|
+
if allow_backward_links:
|
|
427
|
+
attrs["allow_backward_links"] = True
|
|
428
|
+
return Request(
|
|
429
|
+
"POST",
|
|
430
|
+
"/crawl",
|
|
431
|
+
body=envelope("crawl", attrs, proxy=proxy),
|
|
432
|
+
idempotency_key=idempotency_key,
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def crawl_results_request(crawl_id: str, cursor: str | None, limit: int | None) -> Request:
|
|
437
|
+
params: dict[str, Any] = {}
|
|
438
|
+
if cursor:
|
|
439
|
+
params["cursor"] = cursor
|
|
440
|
+
if limit:
|
|
441
|
+
params["limit"] = limit
|
|
442
|
+
return Request("GET", f"/crawl/{crawl_id}/results", params=params or None)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def proxy_request(path: str, proxy_type: str | None, *, country: str | None = None) -> Request:
|
|
446
|
+
params: dict[str, Any] = {}
|
|
447
|
+
if proxy_type:
|
|
448
|
+
params["proxy_type"] = str(getattr(proxy_type, "value", proxy_type))
|
|
449
|
+
if country:
|
|
450
|
+
params["country"] = country
|
|
451
|
+
return Request("GET", path, params=params or None)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def cancel_request(kind: str, job_id: str) -> Request:
|
|
455
|
+
"""Cancelling a cancelled job is a no-op, so a cancel is safe to retry."""
|
|
456
|
+
return Request("POST", f"/{kind}/{path_segment(job_id)}/cancel", idempotent=True)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def health_request(deep: bool) -> Request:
|
|
460
|
+
return Request("GET", "/healthz", params={"deep": "1"} if deep else None, degraded_ok=True)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def analytics_request(
|
|
464
|
+
start_date: date | str | None,
|
|
465
|
+
end_date: date | str | None,
|
|
466
|
+
interval: str | None,
|
|
467
|
+
module: str | None,
|
|
468
|
+
) -> Request:
|
|
469
|
+
params: dict[str, Any] = {}
|
|
470
|
+
for name, value in (("start_date", start_date), ("end_date", end_date)):
|
|
471
|
+
if value:
|
|
472
|
+
params[name] = value if isinstance(value, str) else value.strftime("%Y-%m-%d")
|
|
473
|
+
if interval:
|
|
474
|
+
params["interval"] = interval
|
|
475
|
+
if module:
|
|
476
|
+
params["module"] = str(getattr(module, "value", module))
|
|
477
|
+
return Request("GET", "/task/analytics/dashboard", params=params or None)
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _query(**values: Any) -> dict[str, Any] | None:
|
|
481
|
+
params: dict[str, Any] = {}
|
|
482
|
+
for name, value in values.items():
|
|
483
|
+
if value is None or value == "":
|
|
484
|
+
continue
|
|
485
|
+
if isinstance(value, date):
|
|
486
|
+
value = value.strftime("%Y-%m-%d")
|
|
487
|
+
params[name] = str(getattr(value, "value", value))
|
|
488
|
+
return params or None
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def list_request(
|
|
492
|
+
path: str,
|
|
493
|
+
*,
|
|
494
|
+
status: str | None = None,
|
|
495
|
+
type: str | None = None,
|
|
496
|
+
job_id: str | None = None,
|
|
497
|
+
start_date: date | str | None = None,
|
|
498
|
+
end_date: date | str | None = None,
|
|
499
|
+
limit: int | None = None,
|
|
500
|
+
cursor: str | None = None,
|
|
501
|
+
) -> Request:
|
|
502
|
+
params = _query(
|
|
503
|
+
status=status,
|
|
504
|
+
type=type,
|
|
505
|
+
job_id=job_id,
|
|
506
|
+
start_date=start_date,
|
|
507
|
+
end_date=end_date,
|
|
508
|
+
limit=limit,
|
|
509
|
+
cursor=cursor,
|
|
510
|
+
)
|
|
511
|
+
return Request("GET", path, params=params)
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def transactions_request(
|
|
515
|
+
operation: str | None,
|
|
516
|
+
start_date: date | str | None,
|
|
517
|
+
end_date: date | str | None,
|
|
518
|
+
page: int | None,
|
|
519
|
+
limit: int | None,
|
|
520
|
+
) -> Request:
|
|
521
|
+
params = _query(
|
|
522
|
+
operation=operation, start_date=start_date, end_date=end_date, page=page, limit=limit
|
|
523
|
+
)
|
|
524
|
+
return Request("GET", "/users/@me/transactions", params=params)
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def still_processing(body: Any) -> bool:
|
|
528
|
+
"""Whether the API answered 'the task is still running' instead of a result."""
|
|
529
|
+
return isinstance(body, dict) and body.get("code") == STILL_PROCESSING
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def should_retry(error: Exception) -> bool:
|
|
533
|
+
"""Go's retry policy: transport errors, 429, 502, 503, 504 — nothing else."""
|
|
534
|
+
if isinstance(error, Unavailable) and error.code in {
|
|
535
|
+
"MODULE_UNAVAILABLE",
|
|
536
|
+
"ENGINE_UNAVAILABLE",
|
|
537
|
+
}:
|
|
538
|
+
# A deliberate switch-off is a 503 too, but retrying it only adds load:
|
|
539
|
+
# it stays off until an operator turns it back on.
|
|
540
|
+
return False
|
|
541
|
+
if isinstance(error, APIError):
|
|
542
|
+
return error.status in {429, 502, 503, 504}
|
|
543
|
+
return True # transport error
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def backoff(attempt: int, retry_after: float = 0.0) -> float:
|
|
547
|
+
"""500ms, 1s, 2s, 4s, then 8s flat — halved and randomized. Retry-After wins."""
|
|
548
|
+
if retry_after > 0:
|
|
549
|
+
return min(retry_after, 30.0)
|
|
550
|
+
base = 0.5 * float(1 << min(attempt, 4))
|
|
551
|
+
return base / 2 + random.random() * base / 2
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def poll_delay(attempt: int) -> float:
|
|
555
|
+
"""Gap before re-sending a task the API is still processing."""
|
|
556
|
+
return max(STILL_PROCESSING_DELAY, backoff(attempt))
|