datafuel 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
datafuel/_core.py ADDED
@@ -0,0 +1,556 @@
1
+ """Request building and answer classification. Pure functions, no I/O.
2
+
3
+ Every decision lives here — which envelope a module needs, where the sticky
4
+ session keys go, how ``AI`` expands, whether an answer is retryable — so the
5
+ sync and the async client cannot drift apart.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import random
12
+ import uuid
13
+ from dataclasses import dataclass
14
+ from datetime import date
15
+ from importlib.metadata import PackageNotFoundError
16
+ from importlib.metadata import version as _installed_version
17
+ from typing import Any
18
+ from urllib.parse import quote
19
+
20
+ from .errors import APIError, Unavailable
21
+ from .models import AI, Proxy
22
+
23
+ DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1"
24
+
25
+ try:
26
+ #: Read from the installed metadata, so pyproject.toml stays the one place
27
+ #: the version is written down.
28
+ VERSION = _installed_version("datafuel")
29
+ except PackageNotFoundError: # pragma: no cover - running from a source tree
30
+ VERSION = "0.0.0+dev"
31
+
32
+ #: How long a synchronous call waits for a result before giving up.
33
+ DEFAULT_TIMEOUT = 180.0
34
+ #: Gap between two attempts at a task the API is still processing.
35
+ STILL_PROCESSING_DELAY = 2.0
36
+ STILL_PROCESSING = "TASK_STILL_PROCESSING"
37
+
38
+
39
+ @dataclass(frozen=True, repr=False)
40
+ class Request:
41
+ """One HTTP call, ready for either client to send."""
42
+
43
+ method: str
44
+ path: str
45
+ params: dict[str, Any] | None = None
46
+ body: Any = None
47
+ idempotency_key: str | None = None
48
+ #: A 503 that carries a health report is an answer, not an error.
49
+ degraded_ok: bool = False
50
+ #: A write the API treats as a no-op when repeated, e.g. a cancel.
51
+ idempotent: bool = False
52
+
53
+ @property
54
+ def retryable(self) -> bool:
55
+ """GETs are safe by nature, writes when they carry an idempotency key
56
+ or repeating them changes nothing."""
57
+ return self.method == "GET" or self.idempotency_key is not None or self.idempotent
58
+
59
+ def __repr__(self) -> str:
60
+ """Method and path only: the body can hold an LLM key or a cookie header."""
61
+ return f"<Request {self.method} {self.path}>"
62
+
63
+
64
+ def path_segment(value: str) -> str:
65
+ """Escape an id before it becomes part of a URL path.
66
+
67
+ An id is caller data. Unescaped, a stray ``/`` or ``?`` in one would move
68
+ the request to a different endpoint.
69
+ """
70
+ return quote(str(value), safe="")
71
+
72
+
73
+ def new_idempotency_key() -> str:
74
+ return str(uuid.uuid4())
75
+
76
+
77
+ def headers(api_key: str, user_agent: str, request: Request) -> dict[str, str]:
78
+ """The headers every call carries. The idempotency key sits beside the API key."""
79
+ out = {
80
+ "X-API-Key": api_key,
81
+ "Accept": "application/json",
82
+ "User-Agent": user_agent,
83
+ }
84
+ if request.body is not None:
85
+ out["Content-Type"] = "application/json"
86
+ if request.idempotency_key:
87
+ out["Idempotency-Key"] = request.idempotency_key
88
+ return out
89
+
90
+
91
+ def _selector(value: str | dict[str, str]) -> str:
92
+ """The API takes multi-field extractors as a JSON object string."""
93
+ return value if isinstance(value, str) else json.dumps(value)
94
+
95
+
96
+ def scrape_attributes(
97
+ *,
98
+ format: str | None = None,
99
+ js_rendering: bool = False,
100
+ wait_for: str | None = None,
101
+ wait_for_timeout_ms: int | None = None,
102
+ js_instructions: list[dict[str, Any]] | dict[str, Any] | None = None,
103
+ block_resource: str | list[str] | None = None,
104
+ main_content_only: bool = False,
105
+ include_images: bool | None = None,
106
+ extract: str | dict[str, str] | None = None,
107
+ extract_regex: str | dict[str, str] | None = None,
108
+ template: str | None = None,
109
+ method: str | None = None,
110
+ body: str | None = None,
111
+ content_type: str | None = None,
112
+ headers: dict[str, str] | None = None,
113
+ header_order: list[str] | None = None,
114
+ cookies: str | None = None,
115
+ user_agent: str | None = None,
116
+ user_agent_type: str | None = None,
117
+ ai: AI | None = None,
118
+ ) -> dict[str, Any]:
119
+ """Flatten the page options into the API's ``attributes`` object.
120
+
121
+ Unset options are left out of the body entirely, never sent as null.
122
+ """
123
+ attrs: dict[str, Any] = {}
124
+ if format:
125
+ attrs["result_format"] = str(getattr(format, "value", format))
126
+ if js_rendering:
127
+ attrs["js_rendering"] = True
128
+ if wait_for:
129
+ attrs["wait_for_selector"] = wait_for
130
+ if wait_for_timeout_ms:
131
+ attrs["wait_for_selector_timeout_ms"] = wait_for_timeout_ms
132
+ if js_instructions:
133
+ attrs["js_instructions"] = js_instructions
134
+ if block_resource:
135
+ attrs["block_resource"] = (
136
+ block_resource if isinstance(block_resource, str) else list(block_resource)
137
+ )
138
+ if main_content_only:
139
+ attrs["main_content_only"] = True
140
+ if include_images is not None:
141
+ attrs["include_images"] = include_images
142
+ if extract:
143
+ attrs["extract_selector"] = _selector(extract)
144
+ if extract_regex:
145
+ attrs["extract_regex"] = _selector(extract_regex)
146
+ if template:
147
+ attrs["result_template"] = template
148
+ if method:
149
+ attrs["method"] = method
150
+ if body:
151
+ attrs["body"] = body
152
+ if content_type:
153
+ attrs["content_type"] = content_type
154
+ if headers:
155
+ attrs["headers"] = headers
156
+ if header_order:
157
+ attrs["header_order"] = header_order
158
+ if cookies:
159
+ attrs["cookie_string"] = cookies
160
+ if user_agent:
161
+ attrs["user_agent"] = user_agent
162
+ if user_agent_type:
163
+ attrs["user_agent_type"] = user_agent_type
164
+ if ai is not None:
165
+ attrs.update(ai.attributes())
166
+ return attrs
167
+
168
+
169
+ def envelope(
170
+ task_type: str,
171
+ attributes: dict[str, Any],
172
+ *,
173
+ proxy: Proxy | None = None,
174
+ multithreaded: bool | None = None,
175
+ ) -> dict[str, Any]:
176
+ """The body every write shares."""
177
+ body: dict[str, Any] = {"type": task_type}
178
+ if proxy is not None:
179
+ body.update(proxy.envelope())
180
+ if multithreaded is not None:
181
+ body["multithreaded"] = multithreaded
182
+ body["attributes"] = attributes
183
+ return body
184
+
185
+
186
+ def _page_attributes(
187
+ url_field: str,
188
+ target: Any,
189
+ proxy: Proxy | None,
190
+ options: dict[str, Any],
191
+ ) -> dict[str, Any]:
192
+ attrs = scrape_attributes(**options)
193
+ attrs[url_field] = target
194
+ if proxy is not None:
195
+ attrs.update(proxy.session())
196
+ return attrs
197
+
198
+
199
+ def build_scrape(
200
+ url: str, *, proxy: Proxy | None, options: dict[str, Any], idempotency_key: str
201
+ ) -> Request:
202
+ attrs = _page_attributes("url", url, proxy, options)
203
+ return Request(
204
+ "POST",
205
+ "/task",
206
+ body=envelope("unlocker", attrs, proxy=proxy),
207
+ idempotency_key=idempotency_key,
208
+ )
209
+
210
+
211
+ def build_job(
212
+ urls: list[str],
213
+ *,
214
+ proxy: Proxy | None,
215
+ sequential: bool,
216
+ options: dict[str, Any],
217
+ idempotency_key: str,
218
+ ) -> Request:
219
+ attrs = _page_attributes("urls", list(urls), proxy, options)
220
+ return Request(
221
+ "POST",
222
+ "/job",
223
+ body=envelope("unlocker", attrs, proxy=proxy, multithreaded=not sequential),
224
+ idempotency_key=idempotency_key,
225
+ )
226
+
227
+
228
+ def ask_attributes(
229
+ prompt_field: str,
230
+ prompt: Any,
231
+ *,
232
+ engine: str,
233
+ model: str | None = None,
234
+ websearch: bool = False,
235
+ follow_up: str | None = None,
236
+ country: str | None = None,
237
+ location: str | None = None,
238
+ format: str | None = None,
239
+ ) -> dict[str, Any]:
240
+ """Attributes for the llm_scraping module, which reads its proxy country here."""
241
+ attrs: dict[str, Any] = {prompt_field: prompt, "engine": str(getattr(engine, "value", engine))}
242
+ if model:
243
+ attrs["model"] = model
244
+ if websearch:
245
+ attrs["websearch"] = True
246
+ if follow_up:
247
+ attrs["follow_up_prompt"] = follow_up
248
+ if country:
249
+ attrs["proxy_country"] = country
250
+ if location:
251
+ attrs["location"] = location
252
+ if format:
253
+ attrs["result_format"] = str(getattr(format, "value", format))
254
+ return attrs
255
+
256
+
257
+ def build_ask(prompt: str, *, options: dict[str, Any], idempotency_key: str) -> Request:
258
+ attrs = ask_attributes("prompt", prompt, **options)
259
+ return Request(
260
+ "POST", "/task", body=envelope("llm_scraping", attrs), idempotency_key=idempotency_key
261
+ )
262
+
263
+
264
+ def build_ask_job(
265
+ prompts: list[str], *, sequential: bool, options: dict[str, Any], idempotency_key: str
266
+ ) -> Request:
267
+ attrs = ask_attributes("prompts", list(prompts), **options)
268
+ return Request(
269
+ "POST",
270
+ "/job",
271
+ body=envelope("llm_scraping", attrs, multithreaded=not sequential),
272
+ idempotency_key=idempotency_key,
273
+ )
274
+
275
+
276
+ def serp_attributes(
277
+ query_field: str,
278
+ query: Any,
279
+ *,
280
+ country: str | None = None,
281
+ language: str | None = None,
282
+ location: str | None = None,
283
+ page: int | None = None,
284
+ google_domain: str | None = None,
285
+ uule: str | None = None,
286
+ lat: float | None = None,
287
+ lon: float | None = None,
288
+ radius: int | None = None,
289
+ cr: str | None = None,
290
+ lr: str | None = None,
291
+ tbs: str | None = None,
292
+ safe: str | None = None,
293
+ nfpr: bool | None = None,
294
+ filter: bool | None = None,
295
+ uds: str | None = None,
296
+ kgmid: str | None = None,
297
+ si: str | None = None,
298
+ ludocid: str | None = None,
299
+ lsig: str | None = None,
300
+ ibp: str | None = None,
301
+ device: str | None = None,
302
+ ai_overview: bool = False,
303
+ proxy_country: str | None = None,
304
+ format: str | None = None,
305
+ ) -> dict[str, Any]:
306
+ """Attributes for the serp module. Unset options stay out of the body."""
307
+ attrs: dict[str, Any] = {query_field: query}
308
+ strings = {
309
+ "country": country,
310
+ "language": language,
311
+ "location": location,
312
+ "google_domain": google_domain,
313
+ "uule": uule,
314
+ "cr": cr,
315
+ "lr": lr,
316
+ "tbs": tbs,
317
+ "safe": safe,
318
+ "uds": uds,
319
+ "kgmid": kgmid,
320
+ "si": si,
321
+ "ludocid": ludocid,
322
+ "lsig": lsig,
323
+ "ibp": ibp,
324
+ "device": device,
325
+ "proxy_country": proxy_country,
326
+ }
327
+ attrs.update({name: value for name, value in strings.items() if value})
328
+ if page:
329
+ attrs["page"] = page
330
+ if radius:
331
+ attrs["radius"] = radius
332
+ for name, flag in (("lat", lat), ("lon", lon), ("nfpr", nfpr), ("filter", filter)):
333
+ if flag is not None:
334
+ attrs[name] = flag
335
+ if ai_overview:
336
+ attrs["ai_overview"] = True
337
+ if format:
338
+ attrs["result_format"] = str(getattr(format, "value", format))
339
+ return attrs
340
+
341
+
342
+ def build_search(query: str, *, options: dict[str, Any], idempotency_key: str) -> Request:
343
+ attrs = serp_attributes("query", query, **options)
344
+ return Request("POST", "/task", body=envelope("serp", attrs), idempotency_key=idempotency_key)
345
+
346
+
347
+ def build_search_job(
348
+ queries: list[str], *, sequential: bool, options: dict[str, Any], idempotency_key: str
349
+ ) -> Request:
350
+ attrs = serp_attributes("queries", list(queries), **options)
351
+ return Request(
352
+ "POST",
353
+ "/job",
354
+ body=envelope("serp", attrs, multithreaded=not sequential),
355
+ idempotency_key=idempotency_key,
356
+ )
357
+
358
+
359
+ def build_map(
360
+ url: str,
361
+ *,
362
+ proxy: Proxy | None,
363
+ search: str | None = None,
364
+ sitemap: str | None = None,
365
+ limit: int | None = None,
366
+ include_subdomains: bool = False,
367
+ ignore_sitemap: bool = False,
368
+ sitemap_only: bool = False,
369
+ user_agent: str | None = None,
370
+ user_agent_type: str | None = None,
371
+ idempotency_key: str,
372
+ ) -> Request:
373
+ attrs: dict[str, Any] = {"url": url}
374
+ if search:
375
+ attrs["search"] = search
376
+ if sitemap:
377
+ attrs["sitemap"] = sitemap
378
+ if limit:
379
+ attrs["limit"] = limit
380
+ if include_subdomains:
381
+ attrs["include_subdomains"] = True
382
+ if ignore_sitemap:
383
+ attrs["ignore_sitemap"] = True
384
+ if sitemap_only:
385
+ attrs["sitemap_only"] = True
386
+ if user_agent:
387
+ attrs["user_agent"] = user_agent
388
+ if user_agent_type:
389
+ attrs["user_agent_type"] = user_agent_type
390
+ if proxy is not None:
391
+ attrs.update(proxy.session())
392
+ return Request(
393
+ "POST", "/map", body=envelope("map", attrs, proxy=proxy), idempotency_key=idempotency_key
394
+ )
395
+
396
+
397
+ def build_crawl(
398
+ url: str,
399
+ *,
400
+ proxy: Proxy | None,
401
+ max_pages: int | None = None,
402
+ max_depth: int | None = None,
403
+ include_paths: list[str] | None = None,
404
+ exclude_paths: list[str] | None = None,
405
+ include_subdomains: bool = False,
406
+ allow_backward_links: bool = False,
407
+ concurrency: int | None = None,
408
+ options: dict[str, Any],
409
+ idempotency_key: str,
410
+ ) -> Request:
411
+ if options.get("ai") is not None:
412
+ raise ValueError("crawl does not support ai=: the API rejects result_use_ai on crawls")
413
+ attrs = _page_attributes("url", url, proxy, options)
414
+ if max_pages:
415
+ attrs["max_pages"] = max_pages
416
+ if max_depth:
417
+ attrs["max_depth"] = max_depth
418
+ if concurrency:
419
+ attrs["concurrency"] = concurrency
420
+ if include_paths:
421
+ attrs["include_paths"] = list(include_paths)
422
+ if exclude_paths:
423
+ attrs["exclude_paths"] = list(exclude_paths)
424
+ if include_subdomains:
425
+ attrs["include_subdomains"] = True
426
+ if allow_backward_links:
427
+ attrs["allow_backward_links"] = True
428
+ return Request(
429
+ "POST",
430
+ "/crawl",
431
+ body=envelope("crawl", attrs, proxy=proxy),
432
+ idempotency_key=idempotency_key,
433
+ )
434
+
435
+
436
+ def crawl_results_request(crawl_id: str, cursor: str | None, limit: int | None) -> Request:
437
+ params: dict[str, Any] = {}
438
+ if cursor:
439
+ params["cursor"] = cursor
440
+ if limit:
441
+ params["limit"] = limit
442
+ return Request("GET", f"/crawl/{crawl_id}/results", params=params or None)
443
+
444
+
445
+ def proxy_request(path: str, proxy_type: str | None, *, country: str | None = None) -> Request:
446
+ params: dict[str, Any] = {}
447
+ if proxy_type:
448
+ params["proxy_type"] = str(getattr(proxy_type, "value", proxy_type))
449
+ if country:
450
+ params["country"] = country
451
+ return Request("GET", path, params=params or None)
452
+
453
+
454
+ def cancel_request(kind: str, job_id: str) -> Request:
455
+ """Cancelling a cancelled job is a no-op, so a cancel is safe to retry."""
456
+ return Request("POST", f"/{kind}/{path_segment(job_id)}/cancel", idempotent=True)
457
+
458
+
459
+ def health_request(deep: bool) -> Request:
460
+ return Request("GET", "/healthz", params={"deep": "1"} if deep else None, degraded_ok=True)
461
+
462
+
463
+ def analytics_request(
464
+ start_date: date | str | None,
465
+ end_date: date | str | None,
466
+ interval: str | None,
467
+ module: str | None,
468
+ ) -> Request:
469
+ params: dict[str, Any] = {}
470
+ for name, value in (("start_date", start_date), ("end_date", end_date)):
471
+ if value:
472
+ params[name] = value if isinstance(value, str) else value.strftime("%Y-%m-%d")
473
+ if interval:
474
+ params["interval"] = interval
475
+ if module:
476
+ params["module"] = str(getattr(module, "value", module))
477
+ return Request("GET", "/task/analytics/dashboard", params=params or None)
478
+
479
+
480
+ def _query(**values: Any) -> dict[str, Any] | None:
481
+ params: dict[str, Any] = {}
482
+ for name, value in values.items():
483
+ if value is None or value == "":
484
+ continue
485
+ if isinstance(value, date):
486
+ value = value.strftime("%Y-%m-%d")
487
+ params[name] = str(getattr(value, "value", value))
488
+ return params or None
489
+
490
+
491
+ def list_request(
492
+ path: str,
493
+ *,
494
+ status: str | None = None,
495
+ type: str | None = None,
496
+ job_id: str | None = None,
497
+ start_date: date | str | None = None,
498
+ end_date: date | str | None = None,
499
+ limit: int | None = None,
500
+ cursor: str | None = None,
501
+ ) -> Request:
502
+ params = _query(
503
+ status=status,
504
+ type=type,
505
+ job_id=job_id,
506
+ start_date=start_date,
507
+ end_date=end_date,
508
+ limit=limit,
509
+ cursor=cursor,
510
+ )
511
+ return Request("GET", path, params=params)
512
+
513
+
514
+ def transactions_request(
515
+ operation: str | None,
516
+ start_date: date | str | None,
517
+ end_date: date | str | None,
518
+ page: int | None,
519
+ limit: int | None,
520
+ ) -> Request:
521
+ params = _query(
522
+ operation=operation, start_date=start_date, end_date=end_date, page=page, limit=limit
523
+ )
524
+ return Request("GET", "/users/@me/transactions", params=params)
525
+
526
+
527
+ def still_processing(body: Any) -> bool:
528
+ """Whether the API answered 'the task is still running' instead of a result."""
529
+ return isinstance(body, dict) and body.get("code") == STILL_PROCESSING
530
+
531
+
532
+ def should_retry(error: Exception) -> bool:
533
+ """Go's retry policy: transport errors, 429, 502, 503, 504 — nothing else."""
534
+ if isinstance(error, Unavailable) and error.code in {
535
+ "MODULE_UNAVAILABLE",
536
+ "ENGINE_UNAVAILABLE",
537
+ }:
538
+ # A deliberate switch-off is a 503 too, but retrying it only adds load:
539
+ # it stays off until an operator turns it back on.
540
+ return False
541
+ if isinstance(error, APIError):
542
+ return error.status in {429, 502, 503, 504}
543
+ return True # transport error
544
+
545
+
546
+ def backoff(attempt: int, retry_after: float = 0.0) -> float:
547
+ """500ms, 1s, 2s, 4s, then 8s flat — halved and randomized. Retry-After wins."""
548
+ if retry_after > 0:
549
+ return min(retry_after, 30.0)
550
+ base = 0.5 * float(1 << min(attempt, 4))
551
+ return base / 2 + random.random() * base / 2
552
+
553
+
554
+ def poll_delay(attempt: int) -> float:
555
+ """Gap before re-sending a task the API is still processing."""
556
+ return max(STILL_PROCESSING_DELAY, backoff(attempt))