scrapely-client-python 3.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of scrapely-client-python might be problematic. Click here for more details.

Files changed (65) hide show
  1. scrapely_client/__init__.py +30 -0
  2. scrapely_client/_charging.py +186 -0
  3. scrapely_client/_client_registry.py +136 -0
  4. scrapely_client/_consts.py +96 -0
  5. scrapely_client/_docs.py +37 -0
  6. scrapely_client/_literals.py +539 -0
  7. scrapely_client/_logging.py +180 -0
  8. scrapely_client/_models.py +4172 -0
  9. scrapely_client/_pagination.py +220 -0
  10. scrapely_client/_resource_clients/__init__.py +84 -0
  11. scrapely_client/_resource_clients/_resource_client.py +567 -0
  12. scrapely_client/_resource_clients/actor.py +1042 -0
  13. scrapely_client/_resource_clients/actor_collection.py +377 -0
  14. scrapely_client/_resource_clients/actor_env_var.py +154 -0
  15. scrapely_client/_resource_clients/actor_env_var_collection.py +165 -0
  16. scrapely_client/_resource_clients/actor_version.py +254 -0
  17. scrapely_client/_resource_clients/actor_version_collection.py +232 -0
  18. scrapely_client/_resource_clients/build.py +244 -0
  19. scrapely_client/_resource_clients/build_collection.py +164 -0
  20. scrapely_client/_resource_clients/dataset.py +1151 -0
  21. scrapely_client/_resource_clients/dataset_collection.py +244 -0
  22. scrapely_client/_resource_clients/key_value_store.py +906 -0
  23. scrapely_client/_resource_clients/key_value_store_collection.py +243 -0
  24. scrapely_client/_resource_clients/log.py +225 -0
  25. scrapely_client/_resource_clients/request_queue.py +1167 -0
  26. scrapely_client/_resource_clients/request_queue_collection.py +240 -0
  27. scrapely_client/_resource_clients/run.py +875 -0
  28. scrapely_client/_resource_clients/run_collection.py +226 -0
  29. scrapely_client/_resource_clients/schedule.py +243 -0
  30. scrapely_client/_resource_clients/schedule_collection.py +260 -0
  31. scrapely_client/_resource_clients/store_collection.py +242 -0
  32. scrapely_client/_resource_clients/task.py +793 -0
  33. scrapely_client/_resource_clients/task_collection.py +322 -0
  34. scrapely_client/_resource_clients/user.py +262 -0
  35. scrapely_client/_resource_clients/webhook.py +292 -0
  36. scrapely_client/_resource_clients/webhook_collection.py +281 -0
  37. scrapely_client/_resource_clients/webhook_dispatch.py +84 -0
  38. scrapely_client/_resource_clients/webhook_dispatch_collection.py +158 -0
  39. scrapely_client/_scrapely_client.py +837 -0
  40. scrapely_client/_statistics.py +29 -0
  41. scrapely_client/_status_message_watcher.py +197 -0
  42. scrapely_client/_streamed_log.py +250 -0
  43. scrapely_client/_typeddicts.py +408 -0
  44. scrapely_client/_utils/__init__.py +0 -0
  45. scrapely_client/_utils/crypto.py +87 -0
  46. scrapely_client/_utils/encoding.py +133 -0
  47. scrapely_client/_utils/errors.py +33 -0
  48. scrapely_client/_utils/http.py +137 -0
  49. scrapely_client/_utils/time.py +32 -0
  50. scrapely_client/_utils/try_import.py +61 -0
  51. scrapely_client/errors.py +190 -0
  52. scrapely_client/http_clients/__init__.py +10 -0
  53. scrapely_client/http_clients/_base.py +430 -0
  54. scrapely_client/http_clients/_impit.py +598 -0
  55. scrapely_client/http_compressors/__init__.py +18 -0
  56. scrapely_client/http_compressors/_base.py +25 -0
  57. scrapely_client/http_compressors/_brotli.py +38 -0
  58. scrapely_client/http_compressors/_gzip.py +38 -0
  59. scrapely_client/http_compressors/_resolve.py +38 -0
  60. scrapely_client/py.typed +0 -0
  61. scrapely_client/types.py +52 -0
  62. scrapely_client_python-3.1.3.dist-info/METADATA +269 -0
  63. scrapely_client_python-3.1.3.dist-info/RECORD +65 -0
  64. scrapely_client_python-3.1.3.dist-info/WHEEL +4 -0
  65. scrapely_client_python-3.1.3.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,906 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from contextlib import asynccontextmanager, contextmanager
5
+ from http import HTTPStatus
6
+ from typing import TYPE_CHECKING, Any
7
+
8
+ from scrapely_client._docs import docs_group
9
+ from scrapely_client._models import (
10
+ KeyValueStore,
11
+ KeyValueStoreKey,
12
+ KeyValueStoreResponse,
13
+ ListOfKeys,
14
+ ListOfKeysResponse,
15
+ )
16
+ from scrapely_client._pagination import DEFAULT_CHUNK_SIZE, get_cursor_iterator, get_cursor_iterator_async
17
+ from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
18
+ from scrapely_client._utils.crypto import create_hmac_signature, create_storage_content_signature
19
+ from scrapely_client._utils.encoding import encode_key_value_store_record_value
20
+ from scrapely_client._utils.errors import catch_not_found_or_throw
21
+ from scrapely_client._utils.http import response_to_dict, to_path_segment
22
+ from scrapely_client.errors import ApifyApiError, InvalidResponseBodyError
23
+
24
+ if TYPE_CHECKING:
25
+ from collections.abc import AsyncIterator, Iterator
26
+ from datetime import timedelta
27
+
28
+ from scrapely_client._literals import GeneralAccess
29
+ from scrapely_client.http_clients import HttpResponse
30
+ from scrapely_client.types import Timeout
31
+
32
+
33
+ def _parse_get_record_response(response: HttpResponse) -> Any:
34
+ """Parse an HTTP response based on its content type.
35
+
36
+ Args:
37
+ response: The HTTP response to parse.
38
+
39
+ Returns:
40
+ Parsed response data (JSON dict/list, text string, or raw bytes).
41
+
42
+ Raises:
43
+ InvalidResponseBodyError: If the response body cannot be parsed.
44
+ """
45
+ if response.status_code == HTTPStatus.NO_CONTENT:
46
+ return None
47
+
48
+ content_type = ''
49
+ if 'content-type' in response.headers:
50
+ content_type = response.headers['content-type'].split(';')[0].strip()
51
+
52
+ try:
53
+ if re.search(r'^application/json', content_type, flags=re.IGNORECASE):
54
+ response_data = response.json()
55
+ elif re.search(r'^application/.*xml$', content_type, flags=re.IGNORECASE) or re.search(
56
+ r'^text/', content_type, flags=re.IGNORECASE
57
+ ):
58
+ response_data = response.text
59
+ else:
60
+ response_data = response.content
61
+ except ValueError as err:
62
+ raise InvalidResponseBodyError(response) from err
63
+ else:
64
+ return response_data
65
+
66
+
67
+ @docs_group('Resource clients')
68
+ class KeyValueStoreClient(ResourceClient):
69
+ """Sub-client for managing a specific key-value store.
70
+
71
+ Provides methods to manage a specific key-value store, e.g. get it, update it, or manage its records. Obtain an
72
+ instance via an appropriate method on the `ApifyClient` class.
73
+ """
74
+
75
+ def __init__(
76
+ self,
77
+ *,
78
+ resource_id: str | None = None,
79
+ resource_path: str = 'key-value-stores',
80
+ **kwargs: Any,
81
+ ) -> None:
82
+ super().__init__(
83
+ resource_id=resource_id,
84
+ resource_path=resource_path,
85
+ **kwargs,
86
+ )
87
+
88
+ def get(self, *, timeout: Timeout = 'short') -> KeyValueStore | None:
89
+ """Retrieve the key-value store.
90
+
91
+
92
+ Args:
93
+ timeout: Timeout for the API HTTP request.
94
+
95
+ Returns:
96
+ The retrieved key-value store, or None if it does not exist.
97
+ """
98
+ result = self._get(timeout=timeout)
99
+ if result is None:
100
+ return None
101
+ return KeyValueStoreResponse.model_validate(result).data
102
+
103
+ def update(
104
+ self,
105
+ *,
106
+ name: str | None = None,
107
+ general_access: GeneralAccess | None = None,
108
+ timeout: Timeout = 'long',
109
+ ) -> KeyValueStore:
110
+ """Update the key-value store with specified fields.
111
+
112
+
113
+ Args:
114
+ name: The new name for key-value store.
115
+ general_access: Determines how others can access the key-value store.
116
+ timeout: Timeout for the API HTTP request.
117
+
118
+ Returns:
119
+ The updated key-value store.
120
+ """
121
+ result = self._update(timeout=timeout, name=name, generalAccess=general_access)
122
+ return KeyValueStoreResponse.model_validate(result).data
123
+
124
+ def delete(self, *, timeout: Timeout = 'short') -> None:
125
+ """Delete the key-value store.
126
+
127
+
128
+ Args:
129
+ timeout: Timeout for the API HTTP request.
130
+ """
131
+ self._delete(timeout=timeout)
132
+
133
+ def list_keys(
134
+ self,
135
+ *,
136
+ limit: int | None = None,
137
+ exclusive_start_key: str | None = None,
138
+ collection: str | None = None,
139
+ prefix: str | None = None,
140
+ signature: str | None = None,
141
+ timeout: Timeout = 'medium',
142
+ ) -> ListOfKeys:
143
+ """List the keys in the key-value store.
144
+
145
+
146
+ Args:
147
+ limit: Number of keys to be returned. Maximum value is 1000.
148
+ exclusive_start_key: All keys up to this one (including) are skipped from the result.
149
+ collection: The name of the collection in store schema to list keys from.
150
+ prefix: The prefix of the keys to be listed.
151
+ signature: Signature used to access the items.
152
+ timeout: Timeout for the API HTTP request.
153
+
154
+ Returns:
155
+ The list of keys in the key-value store matching the given arguments.
156
+ """
157
+ request_params = self._build_params(
158
+ limit=limit,
159
+ exclusiveStartKey=exclusive_start_key,
160
+ collection=collection,
161
+ prefix=prefix,
162
+ signature=signature,
163
+ )
164
+
165
+ response = self._http_client.call(
166
+ url=self._build_url('keys'),
167
+ method='GET',
168
+ params=request_params,
169
+ timeout=timeout,
170
+ )
171
+
172
+ result = response_to_dict(response)
173
+ return ListOfKeysResponse.model_validate(result).data
174
+
175
+ def iterate_keys(
176
+ self,
177
+ *,
178
+ limit: int | None = None,
179
+ exclusive_start_key: str | None = None,
180
+ collection: str | None = None,
181
+ prefix: str | None = None,
182
+ signature: str | None = None,
183
+ chunk_size: int | None = None,
184
+ timeout: Timeout = 'long',
185
+ ) -> Iterator[KeyValueStoreKey]:
186
+ """Iterate over the keys in the key-value store.
187
+
188
+ Simple `list_keys` does only one API call, possibly not listing all items matching the criteria. This method
189
+ returns an iterator that is capable of making multiple API calls to retrieve all items matching the criteria.
190
+
191
+
192
+ Args:
193
+ limit: Maximum number of keys to return. By default there is no limit.
194
+ exclusive_start_key: All keys up to this one (including) are skipped from the result.
195
+ collection: The name of the collection in store schema to list keys from.
196
+ prefix: The prefix of the keys to be listed.
197
+ signature: Signature used to access the items.
198
+ chunk_size: Maximum number of keys requested per API call when iterating across pages.
199
+ timeout: Timeout for the API HTTP request.
200
+
201
+ Yields:
202
+ A key from the key-value store.
203
+ """
204
+
205
+ def _callback(*, cursor: str | None = None, limit: int | None = None) -> ListOfKeys:
206
+ return self.list_keys(
207
+ limit=limit,
208
+ exclusive_start_key=cursor,
209
+ collection=collection,
210
+ prefix=prefix,
211
+ signature=signature,
212
+ timeout=timeout,
213
+ )
214
+
215
+ return get_cursor_iterator(
216
+ _callback,
217
+ cursor=exclusive_start_key,
218
+ limit=limit,
219
+ chunk_size=chunk_size or DEFAULT_CHUNK_SIZE,
220
+ )
221
+
222
+ def get_record(self, key: str, *, signature: str | None = None, timeout: Timeout = 'long') -> dict | None:
223
+ """Retrieve the given record from the key-value store.
224
+
225
+
226
+ Args:
227
+ key: Key of the record to retrieve.
228
+ signature: Signature used to access the items.
229
+ timeout: Timeout for the API HTTP request.
230
+
231
+ Returns:
232
+ The requested record, or None, if the record does not exist.
233
+ """
234
+ try:
235
+ response = self._http_client.call(
236
+ url=self._build_url(f'records/{to_path_segment(key)}'),
237
+ method='GET',
238
+ params=self._build_params(signature=signature, attachment=True),
239
+ timeout=timeout,
240
+ )
241
+
242
+ return {
243
+ 'key': key,
244
+ 'value': _parse_get_record_response(response),
245
+ 'content_type': response.headers['content-type'],
246
+ }
247
+
248
+ except ApifyApiError as exc:
249
+ catch_not_found_or_throw(exc)
250
+
251
+ return None
252
+
253
+ def record_exists(self, key: str, *, timeout: Timeout = 'long') -> bool:
254
+ """Check if given record is present in the key-value store.
255
+
256
+
257
+ Args:
258
+ key: Key of the record to check.
259
+ timeout: Timeout for the API HTTP request.
260
+
261
+ Returns:
262
+ True if the record exists, False otherwise.
263
+ """
264
+ try:
265
+ response = self._http_client.call(
266
+ url=self._build_url(f'records/{to_path_segment(key)}'),
267
+ method='HEAD',
268
+ params=self._build_params(),
269
+ timeout=timeout,
270
+ )
271
+ except ApifyApiError as exc:
272
+ if exc.status_code == HTTPStatus.NOT_FOUND:
273
+ return False
274
+
275
+ raise
276
+
277
+ return response.status_code == HTTPStatus.OK
278
+
279
+ def get_record_as_bytes(self, key: str, *, signature: str | None = None, timeout: Timeout = 'long') -> dict | None:
280
+ """Retrieve the given record from the key-value store, without parsing it.
281
+
282
+
283
+ Args:
284
+ key: Key of the record to retrieve.
285
+ signature: Signature used to access the items.
286
+ timeout: Timeout for the API HTTP request.
287
+
288
+ Returns:
289
+ The requested record, or None, if the record does not exist.
290
+ """
291
+ try:
292
+ response = self._http_client.call(
293
+ url=self._build_url(f'records/{to_path_segment(key)}'),
294
+ method='GET',
295
+ params=self._build_params(signature=signature, attachment=True),
296
+ timeout=timeout,
297
+ )
298
+
299
+ return {
300
+ 'key': key,
301
+ 'value': response.content,
302
+ 'content_type': response.headers['content-type'],
303
+ }
304
+
305
+ except ApifyApiError as exc:
306
+ catch_not_found_or_throw(exc)
307
+
308
+ return None
309
+
310
+ @contextmanager
311
+ def stream_record(
312
+ self, key: str, *, signature: str | None = None, timeout: Timeout = 'long'
313
+ ) -> Iterator[dict | None]:
314
+ """Retrieve the given record from the key-value store, as a stream.
315
+
316
+
317
+ Args:
318
+ key: Key of the record to retrieve.
319
+ signature: Signature used to access the items.
320
+ timeout: Timeout for the API HTTP request.
321
+
322
+ Returns:
323
+ The requested record as a context-managed streaming Response, or None, if the record does not exist.
324
+ """
325
+ response = None
326
+ try:
327
+ response = self._http_client.call(
328
+ url=self._build_url(f'records/{to_path_segment(key)}'),
329
+ method='GET',
330
+ params=self._build_params(signature=signature, attachment=True),
331
+ stream=True,
332
+ timeout=timeout,
333
+ )
334
+
335
+ yield {
336
+ 'key': key,
337
+ 'value': response,
338
+ 'content_type': response.headers['content-type'],
339
+ }
340
+
341
+ except ApifyApiError as exc:
342
+ catch_not_found_or_throw(exc)
343
+ yield None
344
+ finally:
345
+ if response:
346
+ response.close()
347
+
348
+ def set_record(
349
+ self,
350
+ key: str,
351
+ value: Any,
352
+ *,
353
+ content_type: str | None = None,
354
+ content_encoding: str | None = None,
355
+ timeout: Timeout = 'long',
356
+ ) -> None:
357
+ """Set a value to the given record in the key-value store.
358
+
359
+
360
+ Args:
361
+ key: The key of the record to save the value to.
362
+ value: The value to save into the record.
363
+ content_type: The content type of the saved value.
364
+ content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it
365
+ to upload a pre-compressed value - the client then forwards the bytes as they are instead of
366
+ compressing them itself. The API accepts `gzip`, `br`, `deflate`, and `identity`, and stores the
367
+ record exactly as uploaded, so this also becomes the encoding the record is served with. Only a
368
+ bytes-like `value`, or a file-like one that reads into bytes, can carry a compression - anything
369
+ else raises `TypeError` instead of being stored under a header that misdescribes it.
370
+ timeout: Timeout for the API HTTP request.
371
+ """
372
+ value, content_type = encode_key_value_store_record_value(
373
+ value,
374
+ content_type=content_type,
375
+ content_encoding=content_encoding,
376
+ )
377
+
378
+ headers = {'content-type': content_type}
379
+ if content_encoding is not None:
380
+ headers['content-encoding'] = content_encoding
381
+
382
+ self._http_client.call(
383
+ url=self._build_url(f'records/{to_path_segment(key)}'),
384
+ method='PUT',
385
+ params=self._build_params(),
386
+ data=value,
387
+ headers=headers,
388
+ timeout=timeout,
389
+ )
390
+
391
+ def delete_record(self, key: str, *, timeout: Timeout = 'short') -> None:
392
+ """Delete the specified record from the key-value store.
393
+
394
+
395
+ Args:
396
+ key: The key of the record which to delete.
397
+ timeout: Timeout for the API HTTP request.
398
+ """
399
+ self._http_client.call(
400
+ url=self._build_url(f'records/{to_path_segment(key)}'),
401
+ method='DELETE',
402
+ params=self._build_params(),
403
+ timeout=timeout,
404
+ )
405
+
406
+ def get_record_public_url(self, key: str, *, timeout: Timeout = 'long') -> str:
407
+ """Generate a URL that can be used to access key-value store record.
408
+
409
+ If the client has permission to access the key-value store's URL signing key, the URL will include a signature
410
+ to verify its authenticity.
411
+
412
+ Args:
413
+ key: The key for which the URL should be generated.
414
+ timeout: Timeout for the API HTTP request.
415
+
416
+ Returns:
417
+ A public URL that can be used to access the value of the given key in the KVS.
418
+ """
419
+ if self._resource_id is None:
420
+ raise ValueError('resource_id cannot be None when generating a public URL')
421
+
422
+ # Encode the key first, so a key that cannot address a record is refused before spending a request.
423
+ record_path = f'records/{to_path_segment(key)}'
424
+
425
+ metadata = self.get(timeout=timeout)
426
+
427
+ request_params = self._build_params()
428
+
429
+ # The signature covers the raw key, which is what the API sees once it decodes the path segment.
430
+ if metadata and metadata.url_signing_secret_key:
431
+ request_params['signature'] = create_hmac_signature(metadata.url_signing_secret_key, key)
432
+
433
+ return self._build_public_url(record_path, request_params)
434
+
435
+ def create_keys_public_url(
436
+ self,
437
+ *,
438
+ limit: int | None = None,
439
+ exclusive_start_key: str | None = None,
440
+ collection: str | None = None,
441
+ prefix: str | None = None,
442
+ expires_in: timedelta | None = None,
443
+ timeout: Timeout = 'long',
444
+ ) -> str:
445
+ """Generate a URL that can be used to access key-value store keys.
446
+
447
+ If the client has permission to access the key-value store's URL signing key,
448
+ the URL will include a signature to verify its authenticity.
449
+
450
+ You can optionally control how long the signed URL should be valid using the `expires_in` option.
451
+ This value sets the expiration duration from the time the URL is generated.
452
+ If not provided, the URL will not expire.
453
+
454
+ Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
455
+
456
+ Args:
457
+ limit: Number of keys to be returned. Maximum value is 1000.
458
+ exclusive_start_key: All keys up to this one (including) are skipped from the result.
459
+ collection: The name of the collection in store schema to list keys from.
460
+ prefix: The prefix of the keys to be listed.
461
+ expires_in: How long the signed URL should be valid from the time it is generated.
462
+ timeout: Timeout for the API HTTP request.
463
+
464
+ Returns:
465
+ The public key-value store keys URL.
466
+ """
467
+ metadata = self.get(timeout=timeout)
468
+
469
+ request_params = self._build_params(
470
+ limit=limit,
471
+ exclusiveStartKey=exclusive_start_key,
472
+ collection=collection,
473
+ prefix=prefix,
474
+ )
475
+
476
+ if metadata and metadata.url_signing_secret_key:
477
+ signature = create_storage_content_signature(
478
+ metadata.id,
479
+ metadata.url_signing_secret_key,
480
+ expires_in=expires_in,
481
+ )
482
+ request_params['signature'] = signature
483
+
484
+ return self._build_public_url('keys', request_params)
485
+
486
+
487
+ @docs_group('Resource clients')
488
+ class KeyValueStoreClientAsync(ResourceClientAsync):
489
+ """Sub-client for managing a specific key-value store.
490
+
491
+ Provides methods to manage a specific key-value store, e.g. get it, update it, or manage its records. Obtain an
492
+ instance via an appropriate method on the `ApifyClientAsync` class.
493
+ """
494
+
495
+ def __init__(
496
+ self,
497
+ *,
498
+ resource_id: str | None = None,
499
+ resource_path: str = 'key-value-stores',
500
+ **kwargs: Any,
501
+ ) -> None:
502
+ super().__init__(
503
+ resource_id=resource_id,
504
+ resource_path=resource_path,
505
+ **kwargs,
506
+ )
507
+
508
+ async def get(self, *, timeout: Timeout = 'short') -> KeyValueStore | None:
509
+ """Retrieve the key-value store.
510
+
511
+
512
+ Args:
513
+ timeout: Timeout for the API HTTP request.
514
+
515
+ Returns:
516
+ The retrieved key-value store, or None if it does not exist.
517
+ """
518
+ result = await self._get(timeout=timeout)
519
+ if result is None:
520
+ return None
521
+ return KeyValueStoreResponse.model_validate(result).data
522
+
523
+ async def update(
524
+ self,
525
+ *,
526
+ name: str | None = None,
527
+ general_access: GeneralAccess | None = None,
528
+ timeout: Timeout = 'long',
529
+ ) -> KeyValueStore:
530
+ """Update the key-value store with specified fields.
531
+
532
+
533
+ Args:
534
+ name: The new name for key-value store.
535
+ general_access: Determines how others can access the key-value store.
536
+ timeout: Timeout for the API HTTP request.
537
+
538
+ Returns:
539
+ The updated key-value store.
540
+ """
541
+ result = await self._update(timeout=timeout, name=name, generalAccess=general_access)
542
+ return KeyValueStoreResponse.model_validate(result).data
543
+
544
+ async def delete(self, *, timeout: Timeout = 'short') -> None:
545
+ """Delete the key-value store.
546
+
547
+
548
+ Args:
549
+ timeout: Timeout for the API HTTP request.
550
+ """
551
+ await self._delete(timeout=timeout)
552
+
553
+ async def list_keys(
554
+ self,
555
+ *,
556
+ limit: int | None = None,
557
+ exclusive_start_key: str | None = None,
558
+ collection: str | None = None,
559
+ prefix: str | None = None,
560
+ signature: str | None = None,
561
+ timeout: Timeout = 'medium',
562
+ ) -> ListOfKeys:
563
+ """List the keys in the key-value store.
564
+
565
+
566
+ Args:
567
+ limit: Number of keys to be returned. Maximum value is 1000.
568
+ exclusive_start_key: All keys up to this one (including) are skipped from the result.
569
+ collection: The name of the collection in store schema to list keys from.
570
+ prefix: The prefix of the keys to be listed.
571
+ signature: Signature used to access the items.
572
+ timeout: Timeout for the API HTTP request.
573
+
574
+ Returns:
575
+ The list of keys in the key-value store matching the given arguments.
576
+ """
577
+ request_params = self._build_params(
578
+ limit=limit,
579
+ exclusiveStartKey=exclusive_start_key,
580
+ collection=collection,
581
+ prefix=prefix,
582
+ signature=signature,
583
+ )
584
+
585
+ response = await self._http_client.call(
586
+ url=self._build_url('keys'),
587
+ method='GET',
588
+ params=request_params,
589
+ timeout=timeout,
590
+ )
591
+
592
+ result = response_to_dict(response)
593
+ return ListOfKeysResponse.model_validate(result).data
594
+
595
+ def iterate_keys(
596
+ self,
597
+ *,
598
+ limit: int | None = None,
599
+ exclusive_start_key: str | None = None,
600
+ collection: str | None = None,
601
+ prefix: str | None = None,
602
+ signature: str | None = None,
603
+ chunk_size: int | None = None,
604
+ timeout: Timeout = 'long',
605
+ ) -> AsyncIterator[KeyValueStoreKey]:
606
+ """Iterate over the keys in the key-value store.
607
+
608
+ Simple `list_keys` does only one API call, possibly not listing all items matching the criteria. This method
609
+ returns an iterator that is capable of making multiple API calls to retrieve all items matching the criteria.
610
+
611
+
612
+ Args:
613
+ limit: Maximum number of keys to return. By default there is no limit.
614
+ exclusive_start_key: All keys up to this one (including) are skipped from the result.
615
+ collection: The name of the collection in store schema to list keys from.
616
+ prefix: The prefix of the keys to be listed.
617
+ signature: Signature used to access the items.
618
+ chunk_size: Maximum number of keys requested per API call when iterating across pages.
619
+ timeout: Timeout for the API HTTP request.
620
+
621
+ Yields:
622
+ A key from the key-value store.
623
+ """
624
+
625
+ async def _callback(*, cursor: str | None = None, limit: int | None = None) -> ListOfKeys:
626
+ return await self.list_keys(
627
+ limit=limit,
628
+ exclusive_start_key=cursor,
629
+ collection=collection,
630
+ prefix=prefix,
631
+ signature=signature,
632
+ timeout=timeout,
633
+ )
634
+
635
+ return get_cursor_iterator_async(
636
+ _callback,
637
+ cursor=exclusive_start_key,
638
+ limit=limit,
639
+ chunk_size=chunk_size or DEFAULT_CHUNK_SIZE,
640
+ )
641
+
642
+ async def get_record(self, key: str, *, signature: str | None = None, timeout: Timeout = 'long') -> dict | None:
643
+ """Retrieve the given record from the key-value store.
644
+
645
+
646
+ Args:
647
+ key: Key of the record to retrieve.
648
+ signature: Signature used to access the items.
649
+ timeout: Timeout for the API HTTP request.
650
+
651
+ Returns:
652
+ The requested record, or None, if the record does not exist.
653
+ """
654
+ try:
655
+ response = await self._http_client.call(
656
+ url=self._build_url(f'records/{to_path_segment(key)}'),
657
+ method='GET',
658
+ params=self._build_params(signature=signature, attachment=True),
659
+ timeout=timeout,
660
+ )
661
+
662
+ return {
663
+ 'key': key,
664
+ 'value': _parse_get_record_response(response),
665
+ 'content_type': response.headers['content-type'],
666
+ }
667
+
668
+ except ApifyApiError as exc:
669
+ catch_not_found_or_throw(exc)
670
+
671
+ return None
672
+
673
+ async def record_exists(self, key: str, *, timeout: Timeout = 'long') -> bool:
674
+ """Check if given record is present in the key-value store.
675
+
676
+
677
+ Args:
678
+ key: Key of the record to check.
679
+ timeout: Timeout for the API HTTP request.
680
+
681
+ Returns:
682
+ True if the record exists, False otherwise.
683
+ """
684
+ try:
685
+ response = await self._http_client.call(
686
+ url=self._build_url(f'records/{to_path_segment(key)}'),
687
+ method='HEAD',
688
+ params=self._build_params(),
689
+ timeout=timeout,
690
+ )
691
+ except ApifyApiError as exc:
692
+ if exc.status_code == HTTPStatus.NOT_FOUND:
693
+ return False
694
+
695
+ raise
696
+
697
+ return response.status_code == HTTPStatus.OK
698
+
699
+ async def get_record_as_bytes(
700
+ self, key: str, *, signature: str | None = None, timeout: Timeout = 'long'
701
+ ) -> dict | None:
702
+ """Retrieve the given record from the key-value store, without parsing it.
703
+
704
+
705
+ Args:
706
+ key: Key of the record to retrieve.
707
+ signature: Signature used to access the items.
708
+ timeout: Timeout for the API HTTP request.
709
+
710
+ Returns:
711
+ The requested record, or None, if the record does not exist.
712
+ """
713
+ try:
714
+ response = await self._http_client.call(
715
+ url=self._build_url(f'records/{to_path_segment(key)}'),
716
+ method='GET',
717
+ params=self._build_params(signature=signature, attachment=True),
718
+ timeout=timeout,
719
+ )
720
+
721
+ return {
722
+ 'key': key,
723
+ 'value': response.content,
724
+ 'content_type': response.headers['content-type'],
725
+ }
726
+
727
+ except ApifyApiError as exc:
728
+ catch_not_found_or_throw(exc)
729
+
730
+ return None
731
+
732
+ @asynccontextmanager
733
+ async def stream_record(
734
+ self, key: str, *, signature: str | None = None, timeout: Timeout = 'long'
735
+ ) -> AsyncIterator[dict | None]:
736
+ """Retrieve the given record from the key-value store, as a stream.
737
+
738
+
739
+ Args:
740
+ key: Key of the record to retrieve.
741
+ signature: Signature used to access the items.
742
+ timeout: Timeout for the API HTTP request.
743
+
744
+ Returns:
745
+ The requested record as a context-managed streaming Response, or None, if the record does not exist.
746
+ """
747
+ response = None
748
+ try:
749
+ response = await self._http_client.call(
750
+ url=self._build_url(f'records/{to_path_segment(key)}'),
751
+ method='GET',
752
+ params=self._build_params(signature=signature, attachment=True),
753
+ stream=True,
754
+ timeout=timeout,
755
+ )
756
+
757
+ yield {
758
+ 'key': key,
759
+ 'value': response,
760
+ 'content_type': response.headers['content-type'],
761
+ }
762
+
763
+ except ApifyApiError as exc:
764
+ catch_not_found_or_throw(exc)
765
+ yield None
766
+ finally:
767
+ if response:
768
+ await response.aclose()
769
+
770
+ async def set_record(
771
+ self,
772
+ key: str,
773
+ value: Any,
774
+ *,
775
+ content_type: str | None = None,
776
+ content_encoding: str | None = None,
777
+ timeout: Timeout = 'long',
778
+ ) -> None:
779
+ """Set a value to the given record in the key-value store.
780
+
781
+
782
+ Args:
783
+ key: The key of the record to save the value to.
784
+ value: The value to save into the record.
785
+ content_type: The content type of the saved value.
786
+ content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it
787
+ to upload a pre-compressed value - the client then forwards the bytes as they are instead of
788
+ compressing them itself. The API accepts `gzip`, `br`, `deflate`, and `identity`, and stores the
789
+ record exactly as uploaded, so this also becomes the encoding the record is served with. Only a
790
+ bytes-like `value`, or a file-like one that reads into bytes, can carry a compression - anything
791
+ else raises `TypeError` instead of being stored under a header that misdescribes it.
792
+ timeout: Timeout for the API HTTP request.
793
+ """
794
+ value, content_type = encode_key_value_store_record_value(
795
+ value,
796
+ content_type=content_type,
797
+ content_encoding=content_encoding,
798
+ )
799
+
800
+ headers = {'content-type': content_type}
801
+ if content_encoding is not None:
802
+ headers['content-encoding'] = content_encoding
803
+
804
+ await self._http_client.call(
805
+ url=self._build_url(f'records/{to_path_segment(key)}'),
806
+ method='PUT',
807
+ params=self._build_params(),
808
+ data=value,
809
+ headers=headers,
810
+ timeout=timeout,
811
+ )
812
+
813
+ async def delete_record(self, key: str, *, timeout: Timeout = 'short') -> None:
814
+ """Delete the specified record from the key-value store.
815
+
816
+
817
+ Args:
818
+ key: The key of the record which to delete.
819
+ timeout: Timeout for the API HTTP request.
820
+ """
821
+ await self._http_client.call(
822
+ url=self._build_url(f'records/{to_path_segment(key)}'),
823
+ method='DELETE',
824
+ params=self._build_params(),
825
+ timeout=timeout,
826
+ )
827
+
828
+ async def get_record_public_url(self, key: str, *, timeout: Timeout = 'long') -> str:
829
+ """Generate a URL that can be used to access key-value store record.
830
+
831
+ If the client has permission to access the key-value store's URL signing key, the URL will include a signature
832
+ to verify its authenticity.
833
+
834
+ Args:
835
+ key: The key for which the URL should be generated.
836
+ timeout: Timeout for the API HTTP request.
837
+
838
+ Returns:
839
+ A public URL that can be used to access the value of the given key in the KVS.
840
+ """
841
+ if self._resource_id is None:
842
+ raise ValueError('resource_id cannot be None when generating a public URL')
843
+
844
+ # Encode the key first, so a key that cannot address a record is refused before spending a request.
845
+ record_path = f'records/{to_path_segment(key)}'
846
+
847
+ metadata = await self.get(timeout=timeout)
848
+
849
+ request_params = self._build_params()
850
+
851
+ # The signature covers the raw key, which is what the API sees once it decodes the path segment.
852
+ if metadata and metadata.url_signing_secret_key:
853
+ request_params['signature'] = create_hmac_signature(metadata.url_signing_secret_key, key)
854
+
855
+ return self._build_public_url(record_path, request_params)
856
+
857
+ async def create_keys_public_url(
858
+ self,
859
+ *,
860
+ limit: int | None = None,
861
+ exclusive_start_key: str | None = None,
862
+ collection: str | None = None,
863
+ prefix: str | None = None,
864
+ expires_in: timedelta | None = None,
865
+ timeout: Timeout = 'long',
866
+ ) -> str:
867
+ """Generate a URL that can be used to access key-value store keys.
868
+
869
+ If the client has permission to access the key-value store's URL signing key,
870
+ the URL will include a signature to verify its authenticity.
871
+
872
+ You can optionally control how long the signed URL should be valid using the `expires_in` option.
873
+ This value sets the expiration duration from the time the URL is generated.
874
+ If not provided, the URL will not expire.
875
+
876
+ Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
877
+
878
+ Args:
879
+ limit: Number of keys to be returned. Maximum value is 1000.
880
+ exclusive_start_key: All keys up to this one (including) are skipped from the result.
881
+ collection: The name of the collection in store schema to list keys from.
882
+ prefix: The prefix of the keys to be listed.
883
+ expires_in: How long the signed URL should be valid from the time it is generated.
884
+ timeout: Timeout for the API HTTP request.
885
+
886
+ Returns:
887
+ The public key-value store keys URL.
888
+ """
889
+ metadata = await self.get(timeout=timeout)
890
+
891
+ request_params = self._build_params(
892
+ limit=limit,
893
+ exclusiveStartKey=exclusive_start_key,
894
+ collection=collection,
895
+ prefix=prefix,
896
+ )
897
+
898
+ if metadata and metadata.url_signing_secret_key:
899
+ signature = create_storage_content_signature(
900
+ metadata.id,
901
+ metadata.url_signing_secret_key,
902
+ expires_in=expires_in,
903
+ )
904
+ request_params['signature'] = signature
905
+
906
+ return self._build_public_url('keys', request_params)