scrapely-client-python 1.0.9__py3-none-any.whl → 1.0.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapely_client/_logging.py +9 -3
- scrapely_client/_models.py +1 -1
- scrapely_client/_resource_clients/actor.py +9 -2
- scrapely_client/_resource_clients/dataset.py +61 -29
- scrapely_client/_resource_clients/key_value_store.py +41 -21
- scrapely_client/_resource_clients/task.py +5 -2
- scrapely_client/_utils/crypto.py +87 -15
- scrapely_client/http_clients/_impit.py +89 -4
- {scrapely_client_python-1.0.9.dist-info → scrapely_client_python-1.0.10.dist-info}/METADATA +18 -18
- {scrapely_client_python-1.0.9.dist-info → scrapely_client_python-1.0.10.dist-info}/RECORD +13 -13
- {scrapely_client_python-1.0.9.dist-info → scrapely_client_python-1.0.10.dist-info}/WHEEL +1 -1
- {scrapely_client_python-1.0.9.dist-info → scrapely_client_python-1.0.10.dist-info}/licenses/LICENSE +0 -0
- {scrapely_client_python-1.0.9.dist-info → scrapely_client_python-1.0.10.dist-info}/licenses/NOTICE +0 -0
scrapely_client/_logging.py
CHANGED
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import functools
|
|
6
6
|
import inspect
|
|
7
7
|
import logging
|
|
8
|
+
import re
|
|
8
9
|
import threading
|
|
9
10
|
from contextvars import ContextVar
|
|
10
11
|
from typing import TYPE_CHECKING, Any, NamedTuple
|
|
@@ -25,6 +26,9 @@ logger_name = __name__.split('.')[0]
|
|
|
25
26
|
logger = logging.getLogger(logger_name)
|
|
26
27
|
"""Logger used throughout the library."""
|
|
27
28
|
|
|
29
|
+
# C0 controls except tab and newline, DEL, and C1 controls (incl. 8-bit CSI).
|
|
30
|
+
_CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\x9f]')
|
|
31
|
+
|
|
28
32
|
|
|
29
33
|
class LogContext(NamedTuple):
|
|
30
34
|
"""Request context details for logging (attempt, client method, HTTP method, resource ID, URL)."""
|
|
@@ -101,9 +105,11 @@ class RedirectLogFormatter(logging.Formatter):
|
|
|
101
105
|
Returns:
|
|
102
106
|
Formatted log message with colored logger name prefix.
|
|
103
107
|
"""
|
|
104
|
-
formatted_logger_name = f'{Fore.CYAN}[{record.name}]{Style.RESET_ALL}'
|
|
105
|
-
|
|
106
|
-
|
|
108
|
+
formatted_logger_name = f'{Fore.CYAN}[{_CONTROL_CHARS_RE.sub("", record.name)}]{Style.RESET_ALL}'
|
|
109
|
+
# Redirected messages come from another Actor's log, so escape sequences in it are removed
|
|
110
|
+
# and every line gets the prefix, so a line in it can't pass for one of our own.
|
|
111
|
+
formatted_message = _CONTROL_CHARS_RE.sub('', super().format(record).replace('\r\n', '\n').replace('\r', '\n'))
|
|
112
|
+
return '\n'.join(f'{formatted_logger_name} -> {msg_line}' for msg_line in formatted_message.split('\n'))
|
|
107
113
|
|
|
108
114
|
|
|
109
115
|
def create_redirect_logger(name: str) -> logging.Logger:
|
scrapely_client/_models.py
CHANGED
|
@@ -1246,7 +1246,7 @@ class EffectivePlatformFeature(BaseModel):
|
|
|
1246
1246
|
str | None,
|
|
1247
1247
|
Field(
|
|
1248
1248
|
examples=[
|
|
1249
|
-
'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact
|
|
1249
|
+
'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact hello@scrape.ly'
|
|
1250
1250
|
]
|
|
1251
1251
|
),
|
|
1252
1252
|
]
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import TYPE_CHECKING, Any, Literal
|
|
6
|
+
from uuid import uuid4
|
|
6
7
|
|
|
7
8
|
from pydantic import TypeAdapter
|
|
8
9
|
|
|
@@ -287,7 +288,8 @@ class ActorClient(ResourceClient):
|
|
|
287
288
|
response = self._http_client.call(
|
|
288
289
|
url=self._build_url('runs'),
|
|
289
290
|
method='POST',
|
|
290
|
-
|
|
291
|
+
# The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
|
|
292
|
+
headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
|
|
291
293
|
data=run_input,
|
|
292
294
|
params=request_params,
|
|
293
295
|
timeout=timeout,
|
|
@@ -418,6 +420,8 @@ class ActorClient(ResourceClient):
|
|
|
418
420
|
response = self._http_client.call(
|
|
419
421
|
url=self._build_url('builds'),
|
|
420
422
|
method='POST',
|
|
423
|
+
# One build per key (same contract as run starts), so a resent build start is safe.
|
|
424
|
+
headers={'idempotency-key': uuid4().hex},
|
|
421
425
|
params=request_params,
|
|
422
426
|
timeout=timeout,
|
|
423
427
|
)
|
|
@@ -783,7 +787,8 @@ class ActorClientAsync(ResourceClientAsync):
|
|
|
783
787
|
response = await self._http_client.call(
|
|
784
788
|
url=self._build_url('runs'),
|
|
785
789
|
method='POST',
|
|
786
|
-
|
|
790
|
+
# The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
|
|
791
|
+
headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
|
|
787
792
|
data=run_input,
|
|
788
793
|
params=request_params,
|
|
789
794
|
timeout=timeout,
|
|
@@ -918,6 +923,8 @@ class ActorClientAsync(ResourceClientAsync):
|
|
|
918
923
|
response = await self._http_client.call(
|
|
919
924
|
url=self._build_url('builds'),
|
|
920
925
|
method='POST',
|
|
926
|
+
# One build per key (same contract as run starts), so a resent build start is safe.
|
|
927
|
+
headers={'idempotency-key': uuid4().hex},
|
|
921
928
|
params=request_params,
|
|
922
929
|
timeout=timeout,
|
|
923
930
|
)
|
|
@@ -5,13 +5,15 @@ from __future__ import annotations
|
|
|
5
5
|
from contextlib import asynccontextmanager, contextmanager
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
from typing import TYPE_CHECKING, Any
|
|
8
|
+
from uuid import uuid4
|
|
8
9
|
|
|
9
10
|
from scrapely_client._docs import docs_group
|
|
10
11
|
from scrapely_client._models import Dataset, DatasetResponse, DatasetStatistics, DatasetStatisticsResponse
|
|
11
12
|
from scrapely_client._pagination import DEFAULT_CHUNK_SIZE, get_items_iterator, get_items_iterator_async
|
|
12
13
|
from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
|
|
13
|
-
from scrapely_client._utils.crypto import
|
|
14
|
+
from scrapely_client._utils.crypto import sign_storage_url
|
|
14
15
|
from scrapely_client._utils.http import response_to_dict, response_to_list
|
|
16
|
+
from scrapely_client.errors import ApifyApiError
|
|
15
17
|
|
|
16
18
|
if TYPE_CHECKING:
|
|
17
19
|
from collections.abc import AsyncIterator, Iterator
|
|
@@ -495,15 +497,26 @@ class DatasetClient(ResourceClient):
|
|
|
495
497
|
else:
|
|
496
498
|
json = items
|
|
497
499
|
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
500
|
+
for resend in (False, True):
|
|
501
|
+
try:
|
|
502
|
+
self._http_client.call(
|
|
503
|
+
url=self._build_url('items'),
|
|
504
|
+
method='POST',
|
|
505
|
+
# The server stores a push once per key, so a push resent after a 5xx or a dropped connection
|
|
506
|
+
# is safe.
|
|
507
|
+
headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
|
|
508
|
+
params=self._build_params(),
|
|
509
|
+
data=data,
|
|
510
|
+
json=json,
|
|
511
|
+
timeout=timeout,
|
|
512
|
+
)
|
|
513
|
+
except ApifyApiError as exc:
|
|
514
|
+
# The push failed mid-write and the server can't tell whether it was stored. Send it once more under
|
|
515
|
+
# a new key, as an unkeyed push would be resent: a second copy is possible, losing the items is not.
|
|
516
|
+
if resend or exc.type != 'push-outcome-unknown':
|
|
517
|
+
raise
|
|
518
|
+
else:
|
|
519
|
+
return
|
|
507
520
|
|
|
508
521
|
def get_statistics(self, *, timeout: Timeout = 'short') -> DatasetStatistics:
|
|
509
522
|
"""Get the dataset statistics.
|
|
@@ -551,7 +564,8 @@ class DatasetClient(ResourceClient):
|
|
|
551
564
|
|
|
552
565
|
You can optionally control how long the signed URL should be valid using the `expires_in` option.
|
|
553
566
|
This value sets the expiration duration from the time the URL is generated.
|
|
554
|
-
If not provided, the URL
|
|
567
|
+
If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
|
|
568
|
+
working if any of them is changed.
|
|
555
569
|
|
|
556
570
|
Any other options (like `limit` or `offset`) will be included as query parameters in the URL.
|
|
557
571
|
|
|
@@ -568,7 +582,7 @@ class DatasetClient(ResourceClient):
|
|
|
568
582
|
skip_hidden: If True, then hidden fields are skipped from the output.
|
|
569
583
|
flatten: A list of fields that should be flattened.
|
|
570
584
|
view: Name of the dataset view to be used.
|
|
571
|
-
expires_in: How long the signed URL should be valid.
|
|
585
|
+
expires_in: How long the signed URL should be valid. Defaults to 1 hour.
|
|
572
586
|
timeout: Timeout for the API HTTP request.
|
|
573
587
|
|
|
574
588
|
Returns:
|
|
@@ -590,15 +604,18 @@ class DatasetClient(ResourceClient):
|
|
|
590
604
|
view=view,
|
|
591
605
|
)
|
|
592
606
|
|
|
607
|
+
public_url = self._build_public_url('items', request_params)
|
|
608
|
+
|
|
609
|
+
# Signed last: the signature binds every query parameter above.
|
|
593
610
|
if dataset and dataset.url_signing_secret_key:
|
|
594
|
-
|
|
611
|
+
public_url = sign_storage_url(
|
|
612
|
+
public_url,
|
|
595
613
|
dataset.id,
|
|
596
614
|
dataset.url_signing_secret_key,
|
|
597
615
|
expires_in=expires_in,
|
|
598
616
|
)
|
|
599
|
-
request_params['signature'] = signature
|
|
600
617
|
|
|
601
|
-
return
|
|
618
|
+
return public_url
|
|
602
619
|
|
|
603
620
|
|
|
604
621
|
@docs_group('Resource clients')
|
|
@@ -1047,15 +1064,26 @@ class DatasetClientAsync(ResourceClientAsync):
|
|
|
1047
1064
|
else:
|
|
1048
1065
|
json = items
|
|
1049
1066
|
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1067
|
+
for resend in (False, True):
|
|
1068
|
+
try:
|
|
1069
|
+
await self._http_client.call(
|
|
1070
|
+
url=self._build_url('items'),
|
|
1071
|
+
method='POST',
|
|
1072
|
+
# The server stores a push once per key, so a push resent after a 5xx or a dropped connection
|
|
1073
|
+
# is safe.
|
|
1074
|
+
headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
|
|
1075
|
+
params=self._build_params(),
|
|
1076
|
+
data=data,
|
|
1077
|
+
json=json,
|
|
1078
|
+
timeout=timeout,
|
|
1079
|
+
)
|
|
1080
|
+
except ApifyApiError as exc:
|
|
1081
|
+
# The push failed mid-write and the server can't tell whether it was stored. Send it once more under
|
|
1082
|
+
# a new key, as an unkeyed push would be resent: a second copy is possible, losing the items is not.
|
|
1083
|
+
if resend or exc.type != 'push-outcome-unknown':
|
|
1084
|
+
raise
|
|
1085
|
+
else:
|
|
1086
|
+
return
|
|
1059
1087
|
|
|
1060
1088
|
async def get_statistics(self, *, timeout: Timeout = 'short') -> DatasetStatistics:
|
|
1061
1089
|
"""Get the dataset statistics.
|
|
@@ -1103,7 +1131,8 @@ class DatasetClientAsync(ResourceClientAsync):
|
|
|
1103
1131
|
|
|
1104
1132
|
You can optionally control how long the signed URL should be valid using the `expires_in` option.
|
|
1105
1133
|
This value sets the expiration duration from the time the URL is generated.
|
|
1106
|
-
If not provided, the URL
|
|
1134
|
+
If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
|
|
1135
|
+
working if any of them is changed.
|
|
1107
1136
|
|
|
1108
1137
|
Any other options (like `limit` or `offset`) will be included as query parameters in the URL.
|
|
1109
1138
|
|
|
@@ -1120,7 +1149,7 @@ class DatasetClientAsync(ResourceClientAsync):
|
|
|
1120
1149
|
skip_hidden: If True, then hidden fields are skipped from the output.
|
|
1121
1150
|
flatten: A list of fields that should be flattened.
|
|
1122
1151
|
view: Name of the dataset view to be used.
|
|
1123
|
-
expires_in: How long the signed URL should be valid.
|
|
1152
|
+
expires_in: How long the signed URL should be valid. Defaults to 1 hour.
|
|
1124
1153
|
timeout: Timeout for the API HTTP request.
|
|
1125
1154
|
|
|
1126
1155
|
Returns:
|
|
@@ -1142,12 +1171,15 @@ class DatasetClientAsync(ResourceClientAsync):
|
|
|
1142
1171
|
view=view,
|
|
1143
1172
|
)
|
|
1144
1173
|
|
|
1174
|
+
public_url = self._build_public_url('items', request_params)
|
|
1175
|
+
|
|
1176
|
+
# Signed last: the signature binds every query parameter above.
|
|
1145
1177
|
if dataset and dataset.url_signing_secret_key:
|
|
1146
|
-
|
|
1178
|
+
public_url = sign_storage_url(
|
|
1179
|
+
public_url,
|
|
1147
1180
|
dataset.id,
|
|
1148
1181
|
dataset.url_signing_secret_key,
|
|
1149
1182
|
expires_in=expires_in,
|
|
1150
1183
|
)
|
|
1151
|
-
request_params['signature'] = signature
|
|
1152
1184
|
|
|
1153
|
-
return
|
|
1185
|
+
return public_url
|
|
@@ -17,7 +17,7 @@ from scrapely_client._models import (
|
|
|
17
17
|
)
|
|
18
18
|
from scrapely_client._pagination import DEFAULT_CHUNK_SIZE, get_cursor_iterator, get_cursor_iterator_async
|
|
19
19
|
from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
|
|
20
|
-
from scrapely_client._utils.crypto import
|
|
20
|
+
from scrapely_client._utils.crypto import sign_storage_url
|
|
21
21
|
from scrapely_client._utils.encoding import encode_key_value_store_record_value
|
|
22
22
|
from scrapely_client._utils.errors import catch_not_found_or_throw
|
|
23
23
|
from scrapely_client._utils.http import response_to_dict, to_path_segment
|
|
@@ -409,7 +409,7 @@ class KeyValueStoreClient(ResourceClient):
|
|
|
409
409
|
"""Generate a URL that can be used to access key-value store record.
|
|
410
410
|
|
|
411
411
|
If the client has permission to access the key-value store's URL signing key, the URL will include a signature
|
|
412
|
-
to verify its authenticity.
|
|
412
|
+
to verify its authenticity. The signature expires in 1 hour.
|
|
413
413
|
|
|
414
414
|
Args:
|
|
415
415
|
key: The key for which the URL should be generated.
|
|
@@ -426,13 +426,19 @@ class KeyValueStoreClient(ResourceClient):
|
|
|
426
426
|
|
|
427
427
|
metadata = self.get(timeout=timeout)
|
|
428
428
|
|
|
429
|
-
|
|
429
|
+
public_url = self._build_public_url(record_path, self._build_params())
|
|
430
430
|
|
|
431
|
-
# The signature
|
|
431
|
+
# The signature binds the raw key, which is what the API sees once it decodes the path segment.
|
|
432
|
+
# It expires in 1 hour.
|
|
432
433
|
if metadata and metadata.url_signing_secret_key:
|
|
433
|
-
|
|
434
|
+
public_url = sign_storage_url(
|
|
435
|
+
public_url,
|
|
436
|
+
metadata.id,
|
|
437
|
+
metadata.url_signing_secret_key,
|
|
438
|
+
record_key=key,
|
|
439
|
+
)
|
|
434
440
|
|
|
435
|
-
return
|
|
441
|
+
return public_url
|
|
436
442
|
|
|
437
443
|
def create_keys_public_url(
|
|
438
444
|
self,
|
|
@@ -451,7 +457,8 @@ class KeyValueStoreClient(ResourceClient):
|
|
|
451
457
|
|
|
452
458
|
You can optionally control how long the signed URL should be valid using the `expires_in` option.
|
|
453
459
|
This value sets the expiration duration from the time the URL is generated.
|
|
454
|
-
If not provided, the URL
|
|
460
|
+
If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
|
|
461
|
+
working if any of them is changed.
|
|
455
462
|
|
|
456
463
|
Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
|
|
457
464
|
|
|
@@ -460,7 +467,7 @@ class KeyValueStoreClient(ResourceClient):
|
|
|
460
467
|
exclusive_start_key: All keys up to this one (including) are skipped from the result.
|
|
461
468
|
collection: The name of the collection in store schema to list keys from.
|
|
462
469
|
prefix: The prefix of the keys to be listed.
|
|
463
|
-
expires_in: How long the signed URL should be valid from the time it is generated.
|
|
470
|
+
expires_in: How long the signed URL should be valid from the time it is generated. Defaults to 1 hour.
|
|
464
471
|
timeout: Timeout for the API HTTP request.
|
|
465
472
|
|
|
466
473
|
Returns:
|
|
@@ -475,15 +482,18 @@ class KeyValueStoreClient(ResourceClient):
|
|
|
475
482
|
prefix=prefix,
|
|
476
483
|
)
|
|
477
484
|
|
|
485
|
+
public_url = self._build_public_url('keys', request_params)
|
|
486
|
+
|
|
487
|
+
# Signed last: the signature binds every query parameter above.
|
|
478
488
|
if metadata and metadata.url_signing_secret_key:
|
|
479
|
-
|
|
489
|
+
public_url = sign_storage_url(
|
|
490
|
+
public_url,
|
|
480
491
|
metadata.id,
|
|
481
492
|
metadata.url_signing_secret_key,
|
|
482
493
|
expires_in=expires_in,
|
|
483
494
|
)
|
|
484
|
-
request_params['signature'] = signature
|
|
485
495
|
|
|
486
|
-
return
|
|
496
|
+
return public_url
|
|
487
497
|
|
|
488
498
|
|
|
489
499
|
@docs_group('Resource clients')
|
|
@@ -831,7 +841,7 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
|
|
|
831
841
|
"""Generate a URL that can be used to access key-value store record.
|
|
832
842
|
|
|
833
843
|
If the client has permission to access the key-value store's URL signing key, the URL will include a signature
|
|
834
|
-
to verify its authenticity.
|
|
844
|
+
to verify its authenticity. The signature expires in 1 hour.
|
|
835
845
|
|
|
836
846
|
Args:
|
|
837
847
|
key: The key for which the URL should be generated.
|
|
@@ -848,13 +858,19 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
|
|
|
848
858
|
|
|
849
859
|
metadata = await self.get(timeout=timeout)
|
|
850
860
|
|
|
851
|
-
|
|
861
|
+
public_url = self._build_public_url(record_path, self._build_params())
|
|
852
862
|
|
|
853
|
-
# The signature
|
|
863
|
+
# The signature binds the raw key, which is what the API sees once it decodes the path segment.
|
|
864
|
+
# It expires in 1 hour.
|
|
854
865
|
if metadata and metadata.url_signing_secret_key:
|
|
855
|
-
|
|
866
|
+
public_url = sign_storage_url(
|
|
867
|
+
public_url,
|
|
868
|
+
metadata.id,
|
|
869
|
+
metadata.url_signing_secret_key,
|
|
870
|
+
record_key=key,
|
|
871
|
+
)
|
|
856
872
|
|
|
857
|
-
return
|
|
873
|
+
return public_url
|
|
858
874
|
|
|
859
875
|
async def create_keys_public_url(
|
|
860
876
|
self,
|
|
@@ -873,7 +889,8 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
|
|
|
873
889
|
|
|
874
890
|
You can optionally control how long the signed URL should be valid using the `expires_in` option.
|
|
875
891
|
This value sets the expiration duration from the time the URL is generated.
|
|
876
|
-
If not provided, the URL
|
|
892
|
+
If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
|
|
893
|
+
working if any of them is changed.
|
|
877
894
|
|
|
878
895
|
Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
|
|
879
896
|
|
|
@@ -882,7 +899,7 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
|
|
|
882
899
|
exclusive_start_key: All keys up to this one (including) are skipped from the result.
|
|
883
900
|
collection: The name of the collection in store schema to list keys from.
|
|
884
901
|
prefix: The prefix of the keys to be listed.
|
|
885
|
-
expires_in: How long the signed URL should be valid from the time it is generated.
|
|
902
|
+
expires_in: How long the signed URL should be valid from the time it is generated. Defaults to 1 hour.
|
|
886
903
|
timeout: Timeout for the API HTTP request.
|
|
887
904
|
|
|
888
905
|
Returns:
|
|
@@ -897,12 +914,15 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
|
|
|
897
914
|
prefix=prefix,
|
|
898
915
|
)
|
|
899
916
|
|
|
917
|
+
public_url = self._build_public_url('keys', request_params)
|
|
918
|
+
|
|
919
|
+
# Signed last: the signature binds every query parameter above.
|
|
900
920
|
if metadata and metadata.url_signing_secret_key:
|
|
901
|
-
|
|
921
|
+
public_url = sign_storage_url(
|
|
922
|
+
public_url,
|
|
902
923
|
metadata.id,
|
|
903
924
|
metadata.url_signing_secret_key,
|
|
904
925
|
expires_in=expires_in,
|
|
905
926
|
)
|
|
906
|
-
request_params['signature'] = signature
|
|
907
927
|
|
|
908
|
-
return
|
|
928
|
+
return public_url
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import TYPE_CHECKING, Any
|
|
6
|
+
from uuid import uuid4
|
|
6
7
|
|
|
7
8
|
from scrapely_client._docs import docs_group
|
|
8
9
|
from scrapely_client._models import (
|
|
@@ -272,7 +273,8 @@ class TaskClient(ResourceClient):
|
|
|
272
273
|
response = self._http_client.call(
|
|
273
274
|
url=self._build_url('runs'),
|
|
274
275
|
method='POST',
|
|
275
|
-
|
|
276
|
+
# The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
|
|
277
|
+
headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
|
|
276
278
|
json=task_input.model_dump() if task_input is not None else None,
|
|
277
279
|
params=request_params,
|
|
278
280
|
timeout=timeout,
|
|
@@ -652,7 +654,8 @@ class TaskClientAsync(ResourceClientAsync):
|
|
|
652
654
|
response = await self._http_client.call(
|
|
653
655
|
url=self._build_url('runs'),
|
|
654
656
|
method='POST',
|
|
655
|
-
|
|
657
|
+
# The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
|
|
658
|
+
headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
|
|
656
659
|
json=task_input.model_dump() if task_input is not None else None,
|
|
657
660
|
params=request_params,
|
|
658
661
|
timeout=timeout,
|
scrapely_client/_utils/crypto.py
CHANGED
|
@@ -7,15 +7,19 @@ import hmac
|
|
|
7
7
|
import string
|
|
8
8
|
import time
|
|
9
9
|
from base64 import urlsafe_b64encode
|
|
10
|
-
from
|
|
10
|
+
from datetime import timedelta
|
|
11
|
+
from urllib.parse import parse_qsl, quote, urlencode, urlsplit, urlunsplit
|
|
11
12
|
|
|
12
13
|
from scrapely_client._utils.time import to_seconds
|
|
13
14
|
|
|
14
|
-
if TYPE_CHECKING:
|
|
15
|
-
from datetime import timedelta
|
|
16
|
-
|
|
17
15
|
_BASE62_CHARSET = string.digits + string.ascii_letters
|
|
18
16
|
|
|
17
|
+
STORAGE_SIGNATURE_VERSION = 2
|
|
18
|
+
"""Version of newly minted storage signatures (binds method, path, record key and query string)."""
|
|
19
|
+
|
|
20
|
+
DEFAULT_STORAGE_URL_EXPIRES_IN = timedelta(hours=1)
|
|
21
|
+
"""Validity of a storage signature when the caller does not set `expires_in`."""
|
|
22
|
+
|
|
19
23
|
|
|
20
24
|
def encode_base62(num: int) -> str:
|
|
21
25
|
"""Encode an integer to a base62 string.
|
|
@@ -58,32 +62,100 @@ def create_hmac_signature(secret_key: str, message: str) -> str:
|
|
|
58
62
|
return encode_base62(decimal_signature)
|
|
59
63
|
|
|
60
64
|
|
|
65
|
+
def storage_signature_query(url: str) -> str:
|
|
66
|
+
"""Return the canonical query string a version 2 storage signature binds.
|
|
67
|
+
|
|
68
|
+
The raw query of `url` is parsed like `URLSearchParams` (`+` is a space), `signature` is dropped, each key and
|
|
69
|
+
value is percent-encoded (RFC 3986: only `A-Z a-z 0-9 - . _ ~` stay as is), and the pairs are sorted by encoded
|
|
70
|
+
key, then encoded value, and joined as `k=v&k=v`. The Scrapely API and the JS client build the same string.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
url: The URL (or path with a query string) to canonicalize.
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
The canonical query string, empty when the URL has no other parameters.
|
|
77
|
+
"""
|
|
78
|
+
pairs = sorted(
|
|
79
|
+
(quote(key, safe=''), quote(value, safe=''))
|
|
80
|
+
for key, value in parse_qsl(urlsplit(url).query, keep_blank_values=True)
|
|
81
|
+
if key != 'signature'
|
|
82
|
+
)
|
|
83
|
+
return '&'.join(f'{key}={value}' for key, value in pairs)
|
|
84
|
+
|
|
85
|
+
|
|
61
86
|
def create_storage_content_signature(
|
|
62
87
|
resource_id: str,
|
|
63
88
|
url_signing_secret_key: str,
|
|
64
89
|
*,
|
|
90
|
+
method: str,
|
|
91
|
+
path: str,
|
|
92
|
+
record_key: str | None = None,
|
|
93
|
+
query: str = '',
|
|
65
94
|
expires_in: timedelta | None = None,
|
|
66
|
-
version: int = 0,
|
|
67
95
|
) -> str:
|
|
68
|
-
"""Create a
|
|
96
|
+
"""Create a signature for a storage resource URL like dataset items or a key-value store record.
|
|
69
97
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
and
|
|
98
|
+
The signature binds the HTTP method, the URL path, the canonical query string (`storage_signature_query`) and,
|
|
99
|
+
for a single record, the record key, so the API refuses it for any other request. The signed message is
|
|
100
|
+
`{version}.{expires_at}.{resource_id}.{METHOD}.{path}.{record_key}.{query}`, the same as the Scrapely API and the
|
|
101
|
+
JS client.
|
|
73
102
|
|
|
74
103
|
Args:
|
|
75
104
|
resource_id: The unique identifier of the storage resource.
|
|
76
105
|
url_signing_secret_key: The secret key for signing the URL.
|
|
77
|
-
|
|
78
|
-
|
|
106
|
+
method: The HTTP method the signature authorizes, e.g. `GET`.
|
|
107
|
+
path: The URL path (no query string), beginning with `/`.
|
|
108
|
+
record_key: The record key when the signature authorizes exactly one record.
|
|
109
|
+
query: The canonical query string from `storage_signature_query`.
|
|
110
|
+
expires_in: How long the signature is valid. Defaults to 1 hour; it cannot be non-expiring.
|
|
79
111
|
|
|
80
112
|
Returns:
|
|
81
113
|
The base64url-encoded signature string.
|
|
82
114
|
"""
|
|
83
|
-
|
|
115
|
+
expires_in_secs = to_seconds(expires_in if expires_in is not None else DEFAULT_STORAGE_URL_EXPIRES_IN)
|
|
116
|
+
if expires_in_secs <= 0:
|
|
117
|
+
raise ValueError('expires_in must be a positive duration')
|
|
118
|
+
expires_at = int(time.time() * 1000) + round(expires_in_secs * 1000)
|
|
119
|
+
version = STORAGE_SIGNATURE_VERSION
|
|
84
120
|
|
|
85
|
-
message_to_sign = f'{version}.{expires_at}.{resource_id}'
|
|
121
|
+
message_to_sign = f'{version}.{expires_at}.{resource_id}.{method.upper()}.{path}.{record_key or ""}.{query}'
|
|
86
122
|
hmac_sig = create_hmac_signature(url_signing_secret_key, message_to_sign)
|
|
87
123
|
|
|
88
|
-
|
|
89
|
-
return
|
|
124
|
+
# Unpadded, like the JS client and Node's base64url.
|
|
125
|
+
return urlsafe_b64encode(f'{version}.{expires_at}.{hmac_sig}'.encode()).decode('utf-8').rstrip('=')
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def sign_storage_url(
|
|
129
|
+
url: str,
|
|
130
|
+
resource_id: str,
|
|
131
|
+
url_signing_secret_key: str,
|
|
132
|
+
*,
|
|
133
|
+
method: str = 'GET',
|
|
134
|
+
record_key: str | None = None,
|
|
135
|
+
expires_in: timedelta | None = None,
|
|
136
|
+
) -> str:
|
|
137
|
+
"""Append a `signature` parameter that binds the URL's path and query string.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
url: The finished URL, with every other query parameter already set.
|
|
141
|
+
resource_id: The unique identifier of the storage resource.
|
|
142
|
+
url_signing_secret_key: The secret key for signing the URL.
|
|
143
|
+
method: The HTTP method the signature authorizes.
|
|
144
|
+
record_key: The record key when the URL is for exactly one record.
|
|
145
|
+
expires_in: How long the signature is valid. Defaults to 1 hour.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
The URL with the signature appended.
|
|
149
|
+
"""
|
|
150
|
+
parts = urlsplit(url)
|
|
151
|
+
signature = create_storage_content_signature(
|
|
152
|
+
resource_id,
|
|
153
|
+
url_signing_secret_key,
|
|
154
|
+
method=method,
|
|
155
|
+
path=parts.path,
|
|
156
|
+
record_key=record_key,
|
|
157
|
+
query=storage_signature_query(url),
|
|
158
|
+
expires_in=expires_in,
|
|
159
|
+
)
|
|
160
|
+
signature_param = urlencode({'signature': signature})
|
|
161
|
+
return urlunsplit(parts._replace(query=f'{parts.query}&{signature_param}' if parts.query else signature_param))
|
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import asyncio
|
|
6
6
|
import logging
|
|
7
7
|
import random
|
|
8
|
+
import re
|
|
8
9
|
import time
|
|
9
10
|
from contextlib import suppress
|
|
10
11
|
from datetime import timedelta
|
|
@@ -67,6 +68,50 @@ def _is_retryable_error(exc: Exception) -> bool:
|
|
|
67
68
|
return isinstance(exc, impit.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS)
|
|
68
69
|
|
|
69
70
|
|
|
71
|
+
# Failures that happen before the request reaches the server, so resending can't apply it twice.
|
|
72
|
+
_NOT_SENT_ERRORS = (impit.ConnectError, impit.ConnectTimeout, impit.PoolTimeout)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# POSTs that are safe to resend without a key: request queue adds (the queue keeps one request per uniqueKey),
|
|
76
|
+
# unlocks and aborts (repeating them changes nothing) and head locks (a repeat only locks more requests until the
|
|
77
|
+
# lock expires). A run's default queue is also reached as /actor-runs/<id>/request-queue.
|
|
78
|
+
_SAFE_TO_RESEND_POST_PATH_RE = re.compile(
|
|
79
|
+
r'/(?:request-queues/[^/]+|actor-runs/[^/]+/request-queue)/(?:requests(?:/batch|/unlock)?|head/lock)/?$|/abort/?$'
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# Get-or-create of a storage by name: a resend finds the storage the first request made.
|
|
83
|
+
_STORAGE_COLLECTION_PATH_RE = re.compile(r'/(?:datasets|key-value-stores|request-queues)/?$')
|
|
84
|
+
|
|
85
|
+
# Answers from the proxy in front of the API when the API did not take the request (e.g. restarting during a deploy),
|
|
86
|
+
# so resending can't apply it twice.
|
|
87
|
+
_NOT_APPLIED_STATUS_CODES = frozenset({HTTPStatus.BAD_GATEWAY, HTTPStatus.SERVICE_UNAVAILABLE})
|
|
88
|
+
|
|
89
|
+
# Keyed requests: the first request with the key is still being applied, so a resend gets its answer shortly.
|
|
90
|
+
_KEY_IN_PROGRESS_ERROR_TYPE = 'idempotency-key-in-progress'
|
|
91
|
+
# Keyed requests: the server can't tell whether the first request was applied, and no resend changes that.
|
|
92
|
+
_OUTCOME_UNKNOWN_ERROR_TYPE = 'push-outcome-unknown'
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _is_unsafe_to_resend(method: str, url: str, headers: dict[str, str], params: dict[str, Any] | None = None) -> bool:
|
|
96
|
+
"""Check if resending the request after an ambiguous failure could apply it twice.
|
|
97
|
+
|
|
98
|
+
A POST or PATCH without an `Idempotency-Key` header may already have been applied when the server answers 5xx or
|
|
99
|
+
the connection drops mid-request, e.g. a dataset push stored twice or a second run started. Request queue adds,
|
|
100
|
+
unlocks, head locks and aborts are always safe (see `_SAFE_TO_RESEND_POST_PATH_RE`), and so is a named storage's
|
|
101
|
+
get-or-create (see `_STORAGE_COLLECTION_PATH_RE`).
|
|
102
|
+
"""
|
|
103
|
+
path = url.split('?', maxsplit=1)[0]
|
|
104
|
+
if method.upper() not in ('POST', 'PATCH') or _SAFE_TO_RESEND_POST_PATH_RE.search(path):
|
|
105
|
+
return False
|
|
106
|
+
if method.upper() == 'POST' and _STORAGE_COLLECTION_PATH_RE.search(path) and (params or {}).get('name'):
|
|
107
|
+
return False
|
|
108
|
+
return not _has_idempotency_key(headers)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _has_idempotency_key(headers: dict[str, str]) -> bool:
|
|
112
|
+
return any(name.lower() == 'idempotency-key' for name in headers)
|
|
113
|
+
|
|
114
|
+
|
|
70
115
|
@docs_group('HTTP clients')
|
|
71
116
|
class ImpitHttpClient(HttpClient):
|
|
72
117
|
"""Synchronous HTTP client for the Scrapely API built on top of Impit.
|
|
@@ -249,16 +294,32 @@ class ImpitHttpClient(HttpClient):
|
|
|
249
294
|
if not _is_retryable_error(exc):
|
|
250
295
|
logger.debug('Exception is not retryable', exc_info=exc)
|
|
251
296
|
stop_retrying()
|
|
297
|
+
elif _is_unsafe_to_resend(method, url, headers, params) and not isinstance(exc, _NOT_SENT_ERRORS):
|
|
298
|
+
logger.debug('Request may have been applied; not resending without an idempotency key', exc_info=exc)
|
|
299
|
+
stop_retrying()
|
|
252
300
|
raise
|
|
253
301
|
|
|
254
|
-
# Retry only server errors (5xx)
|
|
302
|
+
# Retry only server errors (502/503, other 5xx when resending is safe), rate limits (429) and a keyed
|
|
303
|
+
# request's 409 while the first request with that key is still being applied (decided by the error type below).
|
|
255
304
|
logger.debug('Request unsuccessful', extra={'status_code': response.status_code})
|
|
305
|
+
keyed_conflict = response.status_code == HTTPStatus.CONFLICT and _has_idempotency_key(headers)
|
|
256
306
|
if (
|
|
257
307
|
response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR
|
|
258
308
|
and response.status_code != HTTPStatus.TOO_MANY_REQUESTS
|
|
309
|
+
and not keyed_conflict
|
|
259
310
|
):
|
|
260
311
|
logger.debug('Status code is not retryable', extra={'status_code': response.status_code})
|
|
261
312
|
stop_retrying()
|
|
313
|
+
elif (
|
|
314
|
+
response.status_code != HTTPStatus.TOO_MANY_REQUESTS
|
|
315
|
+
and response.status_code not in _NOT_APPLIED_STATUS_CODES
|
|
316
|
+
and _is_unsafe_to_resend(method, url, headers, params)
|
|
317
|
+
):
|
|
318
|
+
logger.debug(
|
|
319
|
+
'Request may have been applied; not resending without an idempotency key',
|
|
320
|
+
extra={'status_code': response.status_code},
|
|
321
|
+
)
|
|
322
|
+
stop_retrying()
|
|
262
323
|
|
|
263
324
|
# Read the response in case it is a stream, so we can raise the error properly. A failed read goes through
|
|
264
325
|
# the same classification as a failed send.
|
|
@@ -273,7 +334,11 @@ class ImpitHttpClient(HttpClient):
|
|
|
273
334
|
stop_retrying()
|
|
274
335
|
raise
|
|
275
336
|
|
|
276
|
-
|
|
337
|
+
error = ApifyApiError(response, attempt, method=method)
|
|
338
|
+
if (keyed_conflict and error.type != _KEY_IN_PROGRESS_ERROR_TYPE) or error.type == _OUTCOME_UNKNOWN_ERROR_TYPE:
|
|
339
|
+
logger.debug('Error type is not retryable', extra={'status_code': response.status_code})
|
|
340
|
+
stop_retrying()
|
|
341
|
+
raise error
|
|
277
342
|
|
|
278
343
|
@staticmethod
|
|
279
344
|
def _retry_with_exp_backoff(
|
|
@@ -522,16 +587,32 @@ class ImpitHttpClientAsync(HttpClientAsync):
|
|
|
522
587
|
if not _is_retryable_error(exc):
|
|
523
588
|
logger.debug('Exception is not retryable', exc_info=exc)
|
|
524
589
|
stop_retrying()
|
|
590
|
+
elif _is_unsafe_to_resend(method, url, headers, params) and not isinstance(exc, _NOT_SENT_ERRORS):
|
|
591
|
+
logger.debug('Request may have been applied; not resending without an idempotency key', exc_info=exc)
|
|
592
|
+
stop_retrying()
|
|
525
593
|
raise
|
|
526
594
|
|
|
527
|
-
# Retry only server errors (5xx)
|
|
595
|
+
# Retry only server errors (502/503, other 5xx when resending is safe), rate limits (429) and a keyed
|
|
596
|
+
# request's 409 while the first request with that key is still being applied (decided by the error type below).
|
|
528
597
|
logger.debug('Request unsuccessful', extra={'status_code': response.status_code})
|
|
598
|
+
keyed_conflict = response.status_code == HTTPStatus.CONFLICT and _has_idempotency_key(headers)
|
|
529
599
|
if (
|
|
530
600
|
response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR
|
|
531
601
|
and response.status_code != HTTPStatus.TOO_MANY_REQUESTS
|
|
602
|
+
and not keyed_conflict
|
|
532
603
|
):
|
|
533
604
|
logger.debug('Status code is not retryable', extra={'status_code': response.status_code})
|
|
534
605
|
stop_retrying()
|
|
606
|
+
elif (
|
|
607
|
+
response.status_code != HTTPStatus.TOO_MANY_REQUESTS
|
|
608
|
+
and response.status_code not in _NOT_APPLIED_STATUS_CODES
|
|
609
|
+
and _is_unsafe_to_resend(method, url, headers, params)
|
|
610
|
+
):
|
|
611
|
+
logger.debug(
|
|
612
|
+
'Request may have been applied; not resending without an idempotency key',
|
|
613
|
+
extra={'status_code': response.status_code},
|
|
614
|
+
)
|
|
615
|
+
stop_retrying()
|
|
535
616
|
|
|
536
617
|
# Read the response in case it is a stream, so we can raise the error properly. A failed read goes through
|
|
537
618
|
# the same classification as a failed send.
|
|
@@ -546,7 +627,11 @@ class ImpitHttpClientAsync(HttpClientAsync):
|
|
|
546
627
|
stop_retrying()
|
|
547
628
|
raise
|
|
548
629
|
|
|
549
|
-
|
|
630
|
+
error = ApifyApiError(response, attempt, method=method)
|
|
631
|
+
if (keyed_conflict and error.type != _KEY_IN_PROGRESS_ERROR_TYPE) or error.type == _OUTCOME_UNKNOWN_ERROR_TYPE:
|
|
632
|
+
logger.debug('Error type is not retryable', extra={'status_code': response.status_code})
|
|
633
|
+
stop_retrying()
|
|
634
|
+
raise error
|
|
550
635
|
|
|
551
636
|
@staticmethod
|
|
552
637
|
async def _retry_with_exp_backoff(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapely-client-python
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.10
|
|
4
4
|
Summary: Scrapely API client for Python
|
|
5
5
|
Keywords: scrapely,api,client,automation,crawling,scraping
|
|
6
6
|
Author: TeraShift GmbH (Scrape.ly)
|
|
@@ -36,7 +36,7 @@ Description-Content-Type: text/markdown
|
|
|
36
36
|
<h1 align="center">Scrapely API client for Python</h1>
|
|
37
37
|
|
|
38
38
|
<p align="center">
|
|
39
|
-
<strong>The official Python client for the <a href="https://
|
|
39
|
+
<strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
|
|
40
40
|
</p>
|
|
41
41
|
|
|
42
42
|
<p align="center">
|
|
@@ -46,9 +46,9 @@ Description-Content-Type: text/markdown
|
|
|
46
46
|
<a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
|
|
47
47
|
</p>
|
|
48
48
|
|
|
49
|
-
`scrapely-client-python` lets you talk to the [Scrapely platform](https://
|
|
49
|
+
`scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
|
|
50
50
|
|
|
51
|
-
> If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
|
|
51
|
+
> If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
|
|
52
52
|
|
|
53
53
|
## Table of contents
|
|
54
54
|
|
|
@@ -92,7 +92,7 @@ Description-Content-Type: text/markdown
|
|
|
92
92
|
|
|
93
93
|
## Quick start
|
|
94
94
|
|
|
95
|
-
You'll need a Scrapely API token —
|
|
95
|
+
You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
|
|
96
96
|
|
|
97
97
|
The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
|
|
98
98
|
|
|
@@ -222,33 +222,33 @@ client = ScrapelyClient(
|
|
|
222
222
|
)
|
|
223
223
|
```
|
|
224
224
|
|
|
225
|
-
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the
|
|
225
|
+
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
|
|
226
226
|
|
|
227
227
|
## Documentation
|
|
228
228
|
|
|
229
|
-
The
|
|
229
|
+
The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
|
|
230
|
+
|
|
231
|
+
The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
|
|
230
232
|
|
|
231
233
|
| Section | What you'll find |
|
|
232
234
|
| --- | --- |
|
|
233
|
-
| Introduction | Overview,
|
|
234
|
-
|
|
|
235
|
-
|
|
|
236
|
-
|
|
|
237
|
-
| API reference | Generated reference for every class, method, and model. |
|
|
238
|
-
| Changelog | Release history and breaking changes. |
|
|
235
|
+
| [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
|
|
236
|
+
| [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
|
|
237
|
+
| [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
|
|
238
|
+
| [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
|
|
239
239
|
|
|
240
240
|
## Related projects
|
|
241
241
|
|
|
242
|
-
- **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
242
|
+
- **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
243
243
|
- **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
|
|
244
|
-
- **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
|
|
245
|
-
- **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
|
|
244
|
+
- **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
|
|
245
|
+
- **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
|
|
246
246
|
- **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
|
|
247
|
-
- **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
247
|
+
- **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
248
248
|
|
|
249
249
|
## Support and community
|
|
250
250
|
|
|
251
|
-
- **GitHub issues** — report a bug or request a feature in the
|
|
251
|
+
- **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
|
|
252
252
|
|
|
253
253
|
## Contributing
|
|
254
254
|
|
|
@@ -4,12 +4,12 @@ scrapely_client/_client_registry.py,sha256=FXqW7wYdHaFUjrtugXkh9IORzazzxGQssHOgt
|
|
|
4
4
|
scrapely_client/_consts.py,sha256=zfGGi17JdkNR23W8hQinwbm81wXg_H_Pw1C_mpMMhWc,3668
|
|
5
5
|
scrapely_client/_docs.py,sha256=MtrntqA7S0BDKzJlZzBAfqeI2bUtybwcb85JfP9vaDA,1117
|
|
6
6
|
scrapely_client/_literals.py,sha256=qSc0qIjhcI7qa42fNQ-IGPm_7cpCI9_OaQ6oYGIOdy4,17006
|
|
7
|
-
scrapely_client/_logging.py,sha256=
|
|
8
|
-
scrapely_client/_models.py,sha256=
|
|
7
|
+
scrapely_client/_logging.py,sha256=F-lwPd2tkDh9vomswuVzo1I3LliHBshgn-65Q29_3gg,7266
|
|
8
|
+
scrapely_client/_models.py,sha256=gX_Gw5_0R3cScpOxgpRDNJliBiOqcU0Ot208H4aaW_w,142907
|
|
9
9
|
scrapely_client/_pagination.py,sha256=by-2FLNUQquXfxgr2mJVCgOqBBZRYMJpmTjfy4aWfpE,8579
|
|
10
10
|
scrapely_client/_resource_clients/__init__.py,sha256=cy_gBIqs8m7UpnKToffNad2zX2Su8We5B_q5IDM879M,4016
|
|
11
11
|
scrapely_client/_resource_clients/_resource_client.py,sha256=b22QuV7osKBor62RsFWMOMKoyevUAY9PY2JrirxrWA8,21732
|
|
12
|
-
scrapely_client/_resource_clients/actor.py,sha256=
|
|
12
|
+
scrapely_client/_resource_clients/actor.py,sha256=oh9uc0m84fo1RouHjkbGpaetzWMqBMXXac9duPc-0CQ,46764
|
|
13
13
|
scrapely_client/_resource_clients/actor_collection.py,sha256=LynOa8R-sTogMVN6jVl2mDW4xmw4a1IZ35Hv8gHX71c,16966
|
|
14
14
|
scrapely_client/_resource_clients/actor_env_var.py,sha256=dWKE_5xqhjQwCiNPNRXiyCw6g0GcUq1DxUR6Nd1JrRs,4810
|
|
15
15
|
scrapely_client/_resource_clients/actor_env_var_collection.py,sha256=1Ueh_QYHbIS1Ztl7X9ZpIMglFND1U8Nmw08u0Z4ERrs,5675
|
|
@@ -19,9 +19,9 @@ scrapely_client/_resource_clients/automation.py,sha256=5dzATC-ZNsbfYUgvBxPadDtID
|
|
|
19
19
|
scrapely_client/_resource_clients/automation_collection.py,sha256=0AOWLE3wuAlD1oWywEAfXBsnClLHs_IQ3l4Hi28ibac,5592
|
|
20
20
|
scrapely_client/_resource_clients/build.py,sha256=Bp2OTcil31OunorPKnw6CIc9K0ZlnYLStaNENtPkq1Y,7805
|
|
21
21
|
scrapely_client/_resource_clients/build_collection.py,sha256=TAuG4H46n6HIIntjg4tWiKqiDOgIUao5LX95xWjExwc,5766
|
|
22
|
-
scrapely_client/_resource_clients/dataset.py,sha256
|
|
22
|
+
scrapely_client/_resource_clients/dataset.py,sha256=96bk1iRtS51LEWyccUU93LiZueV4QAnvOb8DhKsLPRU,54008
|
|
23
23
|
scrapely_client/_resource_clients/dataset_collection.py,sha256=3IGH7B12agw1kHPhzO_1-HzXHyUCtPnFmSxT35_N1Bw,8859
|
|
24
|
-
scrapely_client/_resource_clients/key_value_store.py,sha256=
|
|
24
|
+
scrapely_client/_resource_clients/key_value_store.py,sha256=PbJQ2ecY9j4qTnR9s3fdDnE8Mqq91vGexy8nufwTd1g,33314
|
|
25
25
|
scrapely_client/_resource_clients/key_value_store_collection.py,sha256=D9XPMI7DVdAB1rPK-VKbeY1_JNxSajyPxJIhTTcCT0g,9279
|
|
26
26
|
scrapely_client/_resource_clients/log.py,sha256=XBuJRpUvBKfVhFypMFE0Wowpv5VOg2fs57PDkJEVaEQ,7491
|
|
27
27
|
scrapely_client/_resource_clients/request_queue.py,sha256=ajPrm2uAjjMiAUlsie4D6o9rLb6J1H9fhOdKxXiLJCQ,42518
|
|
@@ -31,7 +31,7 @@ scrapely_client/_resource_clients/run_collection.py,sha256=tU-UX1W_mYD7tRfL9S2YH
|
|
|
31
31
|
scrapely_client/_resource_clients/schedule.py,sha256=jY1ppDtVBRT7qMD-0XKXx_mXdNr0nNLjv5vD2YV8UYI,8094
|
|
32
32
|
scrapely_client/_resource_clients/schedule_collection.py,sha256=NheBq6TfWLn6mip0toHB4lPhOgFaVFpicFJ5XscSf2o,9514
|
|
33
33
|
scrapely_client/_resource_clients/store_collection.py,sha256=TOoxgyWy8JLjzmZ8m4gv52s6CZF5TV_ovpArFHQeneU,9230
|
|
34
|
-
scrapely_client/_resource_clients/task.py,sha256=
|
|
34
|
+
scrapely_client/_resource_clients/task.py,sha256=pOpn_n3V02jX5xvgJlJx-q7bQy4JH51Q20t1m7klgCU,32908
|
|
35
35
|
scrapely_client/_resource_clients/task_collection.py,sha256=Lygg-I_tvdIbW0zo9yS4vqTpmofx9qFtr8KhhG7jBbI,13327
|
|
36
36
|
scrapely_client/_resource_clients/user.py,sha256=-xxfBt71rDdEMZq4qGd7DkYQ4fMJnr_epqrU57nM804,8769
|
|
37
37
|
scrapely_client/_resource_clients/webhook.py,sha256=bZNla1cNM508EQL8zPe-7vgvfsCffQp5A3qnbfxuxe0,10484
|
|
@@ -44,7 +44,7 @@ scrapely_client/_status_message_watcher.py,sha256=9Tstosbk_bUoq5K9xkhRQ5PBqsFaJF
|
|
|
44
44
|
scrapely_client/_streamed_log.py,sha256=OaxB5-ghZX1DYuZmjNCgVeHAH0fYPReqjQlWaa8-dVA,11659
|
|
45
45
|
scrapely_client/_typeddicts.py,sha256=oFrRIATgElDDaDay53l64WeS0Ctet8ztZ9Akhzd-SnA,12425
|
|
46
46
|
scrapely_client/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
47
|
-
scrapely_client/_utils/crypto.py,sha256=
|
|
47
|
+
scrapely_client/_utils/crypto.py,sha256=Ney70TRfBVfttn6XMUfSnJzDZHacT9mtrH6SSqvGaCM,6038
|
|
48
48
|
scrapely_client/_utils/encoding.py,sha256=w2J8CWXB_yHCLkilWGwIddkOJbpJK7nOSa24JiwZdhs,6460
|
|
49
49
|
scrapely_client/_utils/errors.py,sha256=NyuYpfqG_Scq-yYECJisnmG1r3h0vCXYItdDX8DNKaU,1234
|
|
50
50
|
scrapely_client/_utils/http.py,sha256=o0GETJsVjdcOJUuFle7XFsToss7B6k5NtV62kE2UISQ,4959
|
|
@@ -53,7 +53,7 @@ scrapely_client/_utils/try_import.py,sha256=T1m3vcn6V6crHwm4Syw8X_TwcKfhfv_shWZD
|
|
|
53
53
|
scrapely_client/errors.py,sha256=sNKvkA7KUjLfBPrCkhrFu6Owt9cBKfCOL5Olw0IhjnQ,6955
|
|
54
54
|
scrapely_client/http_clients/__init__.py,sha256=s6d7wVMJKLWodAfGR3lPAXfy4aEF1RWQgHKB1OMR9tY,396
|
|
55
55
|
scrapely_client/http_clients/_base.py,sha256=5babUECMpMiU-OMy7J_-LNqQxQ2wERERw8YuRuy9Vvg,19011
|
|
56
|
-
scrapely_client/http_clients/_impit.py,sha256=
|
|
56
|
+
scrapely_client/http_clients/_impit.py,sha256=L2XnwJJHdtvkvaeVgf4kKVt1zyAwNPdFe-p9T7tvPyA,29257
|
|
57
57
|
scrapely_client/http_compressors/__init__.py,sha256=2n7Ym-q5axJda90TjsdDwkniB_KQzPwjrCjqTwk7nW8,1104
|
|
58
58
|
scrapely_client/http_compressors/_base.py,sha256=25W0zDO54qdSnhYLvYyF4EbYM2mh5gxwlaaQ5yKAuzw,672
|
|
59
59
|
scrapely_client/http_compressors/_brotli.py,sha256=ItLG3DgHqH3dbyAyQUcRrTPaJgxwCk-C6OL7exGWb0s,1294
|
|
@@ -61,8 +61,8 @@ scrapely_client/http_compressors/_gzip.py,sha256=IJHZ87B5L1nrnXNFf1PjsYf-1MIIp9G
|
|
|
61
61
|
scrapely_client/http_compressors/_resolve.py,sha256=AfIUzwX-ExczMgMQNAhfPd4CKAVKhIGq697bUm9i62k,1653
|
|
62
62
|
scrapely_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
63
63
|
scrapely_client/types.py,sha256=Wnb9-GBN6OcQGfiDk18TJjIzvHXb5MHrjJaIce-2UzY,2215
|
|
64
|
-
scrapely_client_python-1.0.
|
|
65
|
-
scrapely_client_python-1.0.
|
|
66
|
-
scrapely_client_python-1.0.
|
|
67
|
-
scrapely_client_python-1.0.
|
|
68
|
-
scrapely_client_python-1.0.
|
|
64
|
+
scrapely_client_python-1.0.10.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
|
|
65
|
+
scrapely_client_python-1.0.10.dist-info/licenses/NOTICE,sha256=0Seguz6wGdTod6Rld--Q4nHcmVqZF56Q_Atbh4YFEts,337
|
|
66
|
+
scrapely_client_python-1.0.10.dist-info/WHEEL,sha256=MWx7psoF7KNST9G72pGtiBYuaXRXQXBk9pw5R6FzYko,80
|
|
67
|
+
scrapely_client_python-1.0.10.dist-info/METADATA,sha256=biT4HxozDqRCqbDk_FNauqD-hZseGMPPArhWhvyIUpM,12419
|
|
68
|
+
scrapely_client_python-1.0.10.dist-info/RECORD,,
|
{scrapely_client_python-1.0.9.dist-info → scrapely_client_python-1.0.10.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{scrapely_client_python-1.0.9.dist-info → scrapely_client_python-1.0.10.dist-info}/licenses/NOTICE
RENAMED
|
File without changes
|