scrapely-client-python 1.0.8__py3-none-any.whl → 1.0.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,9 @@ from datetime import timedelta
7
7
  DEFAULT_API_URL = 'https://api.scrape.ly'
8
8
  """Default base URL for the Scrapely API."""
9
9
 
10
+ API_URL_ENV_VAR = 'SCRAPELY_API_URL'
11
+ """Environment variable that sets the API URL when `api_url` is not passed to the client."""
12
+
10
13
  API_VERSION = 'v2'
11
14
  """Current Scrapely API version."""
12
15
 
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import functools
6
6
  import inspect
7
7
  import logging
8
+ import re
8
9
  import threading
9
10
  from contextvars import ContextVar
10
11
  from typing import TYPE_CHECKING, Any, NamedTuple
@@ -25,6 +26,9 @@ logger_name = __name__.split('.')[0]
25
26
  logger = logging.getLogger(logger_name)
26
27
  """Logger used throughout the library."""
27
28
 
29
+ # C0 controls except tab and newline, DEL, and C1 controls (incl. 8-bit CSI).
30
+ _CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\x9f]')
31
+
28
32
 
29
33
  class LogContext(NamedTuple):
30
34
  """Request context details for logging (attempt, client method, HTTP method, resource ID, URL)."""
@@ -101,9 +105,11 @@ class RedirectLogFormatter(logging.Formatter):
101
105
  Returns:
102
106
  Formatted log message with colored logger name prefix.
103
107
  """
104
- formatted_logger_name = f'{Fore.CYAN}[{record.name}]{Style.RESET_ALL}'
105
- formatted_message = super().format(record)
106
- return f'{formatted_logger_name} -> {formatted_message}'
108
+ formatted_logger_name = f'{Fore.CYAN}[{_CONTROL_CHARS_RE.sub("", record.name)}]{Style.RESET_ALL}'
109
+ # Redirected messages come from another Actor's log, so escape sequences in it are removed
110
+ # and every line gets the prefix, so a line in it can't pass for one of our own.
111
+ formatted_message = _CONTROL_CHARS_RE.sub('', super().format(record).replace('\r\n', '\n').replace('\r', '\n'))
112
+ return '\n'.join(f'{formatted_logger_name} -> {msg_line}' for msg_line in formatted_message.split('\n'))
107
113
 
108
114
 
109
115
  def create_redirect_logger(name: str) -> logging.Logger:
@@ -1246,7 +1246,7 @@ class EffectivePlatformFeature(BaseModel):
1246
1246
  str | None,
1247
1247
  Field(
1248
1248
  examples=[
1249
- 'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact support@scrape.ly'
1249
+ 'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact hello@scrape.ly'
1250
1250
  ]
1251
1251
  ),
1252
1252
  ]
@@ -3,6 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  from typing import TYPE_CHECKING, Any, Literal
6
+ from uuid import uuid4
6
7
 
7
8
  from pydantic import TypeAdapter
8
9
 
@@ -287,7 +288,8 @@ class ActorClient(ResourceClient):
287
288
  response = self._http_client.call(
288
289
  url=self._build_url('runs'),
289
290
  method='POST',
290
- headers={'content-type': content_type},
291
+ # The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
292
+ headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
291
293
  data=run_input,
292
294
  params=request_params,
293
295
  timeout=timeout,
@@ -418,6 +420,8 @@ class ActorClient(ResourceClient):
418
420
  response = self._http_client.call(
419
421
  url=self._build_url('builds'),
420
422
  method='POST',
423
+ # One build per key (same contract as run starts), so a resent build start is safe.
424
+ headers={'idempotency-key': uuid4().hex},
421
425
  params=request_params,
422
426
  timeout=timeout,
423
427
  )
@@ -783,7 +787,8 @@ class ActorClientAsync(ResourceClientAsync):
783
787
  response = await self._http_client.call(
784
788
  url=self._build_url('runs'),
785
789
  method='POST',
786
- headers={'content-type': content_type},
790
+ # The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
791
+ headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
787
792
  data=run_input,
788
793
  params=request_params,
789
794
  timeout=timeout,
@@ -918,6 +923,8 @@ class ActorClientAsync(ResourceClientAsync):
918
923
  response = await self._http_client.call(
919
924
  url=self._build_url('builds'),
920
925
  method='POST',
926
+ # One build per key (same contract as run starts), so a resent build start is safe.
927
+ headers={'idempotency-key': uuid4().hex},
921
928
  params=request_params,
922
929
  timeout=timeout,
923
930
  )
@@ -5,13 +5,15 @@ from __future__ import annotations
5
5
  from contextlib import asynccontextmanager, contextmanager
6
6
  from dataclasses import dataclass
7
7
  from typing import TYPE_CHECKING, Any
8
+ from uuid import uuid4
8
9
 
9
10
  from scrapely_client._docs import docs_group
10
11
  from scrapely_client._models import Dataset, DatasetResponse, DatasetStatistics, DatasetStatisticsResponse
11
12
  from scrapely_client._pagination import DEFAULT_CHUNK_SIZE, get_items_iterator, get_items_iterator_async
12
13
  from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
13
- from scrapely_client._utils.crypto import create_storage_content_signature
14
+ from scrapely_client._utils.crypto import sign_storage_url
14
15
  from scrapely_client._utils.http import response_to_dict, response_to_list
16
+ from scrapely_client.errors import ApifyApiError
15
17
 
16
18
  if TYPE_CHECKING:
17
19
  from collections.abc import AsyncIterator, Iterator
@@ -495,15 +497,26 @@ class DatasetClient(ResourceClient):
495
497
  else:
496
498
  json = items
497
499
 
498
- self._http_client.call(
499
- url=self._build_url('items'),
500
- method='POST',
501
- headers={'content-type': 'application/json; charset=utf-8'},
502
- params=self._build_params(),
503
- data=data,
504
- json=json,
505
- timeout=timeout,
506
- )
500
+ for resend in (False, True):
501
+ try:
502
+ self._http_client.call(
503
+ url=self._build_url('items'),
504
+ method='POST',
505
+ # The server stores a push once per key, so a push resent after a 5xx or a dropped connection
506
+ # is safe.
507
+ headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
508
+ params=self._build_params(),
509
+ data=data,
510
+ json=json,
511
+ timeout=timeout,
512
+ )
513
+ except ApifyApiError as exc:
514
+ # The push failed mid-write and the server can't tell whether it was stored. Send it once more under
515
+ # a new key, as an unkeyed push would be resent: a second copy is possible, losing the items is not.
516
+ if resend or exc.type != 'push-outcome-unknown':
517
+ raise
518
+ else:
519
+ return
507
520
 
508
521
  def get_statistics(self, *, timeout: Timeout = 'short') -> DatasetStatistics:
509
522
  """Get the dataset statistics.
@@ -551,7 +564,8 @@ class DatasetClient(ResourceClient):
551
564
 
552
565
  You can optionally control how long the signed URL should be valid using the `expires_in` option.
553
566
  This value sets the expiration duration from the time the URL is generated.
554
- If not provided, the URL will not expire.
567
+ If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
568
+ working if any of them is changed.
555
569
 
556
570
  Any other options (like `limit` or `offset`) will be included as query parameters in the URL.
557
571
 
@@ -568,7 +582,7 @@ class DatasetClient(ResourceClient):
568
582
  skip_hidden: If True, then hidden fields are skipped from the output.
569
583
  flatten: A list of fields that should be flattened.
570
584
  view: Name of the dataset view to be used.
571
- expires_in: How long the signed URL should be valid.
585
+ expires_in: How long the signed URL should be valid. Defaults to 1 hour.
572
586
  timeout: Timeout for the API HTTP request.
573
587
 
574
588
  Returns:
@@ -590,15 +604,18 @@ class DatasetClient(ResourceClient):
590
604
  view=view,
591
605
  )
592
606
 
607
+ public_url = self._build_public_url('items', request_params)
608
+
609
+ # Signed last: the signature binds every query parameter above.
593
610
  if dataset and dataset.url_signing_secret_key:
594
- signature = create_storage_content_signature(
611
+ public_url = sign_storage_url(
612
+ public_url,
595
613
  dataset.id,
596
614
  dataset.url_signing_secret_key,
597
615
  expires_in=expires_in,
598
616
  )
599
- request_params['signature'] = signature
600
617
 
601
- return self._build_public_url('items', request_params)
618
+ return public_url
602
619
 
603
620
 
604
621
  @docs_group('Resource clients')
@@ -1047,15 +1064,26 @@ class DatasetClientAsync(ResourceClientAsync):
1047
1064
  else:
1048
1065
  json = items
1049
1066
 
1050
- await self._http_client.call(
1051
- url=self._build_url('items'),
1052
- method='POST',
1053
- headers={'content-type': 'application/json; charset=utf-8'},
1054
- params=self._build_params(),
1055
- data=data,
1056
- json=json,
1057
- timeout=timeout,
1058
- )
1067
+ for resend in (False, True):
1068
+ try:
1069
+ await self._http_client.call(
1070
+ url=self._build_url('items'),
1071
+ method='POST',
1072
+ # The server stores a push once per key, so a push resent after a 5xx or a dropped connection
1073
+ # is safe.
1074
+ headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
1075
+ params=self._build_params(),
1076
+ data=data,
1077
+ json=json,
1078
+ timeout=timeout,
1079
+ )
1080
+ except ApifyApiError as exc:
1081
+ # The push failed mid-write and the server can't tell whether it was stored. Send it once more under
1082
+ # a new key, as an unkeyed push would be resent: a second copy is possible, losing the items is not.
1083
+ if resend or exc.type != 'push-outcome-unknown':
1084
+ raise
1085
+ else:
1086
+ return
1059
1087
 
1060
1088
  async def get_statistics(self, *, timeout: Timeout = 'short') -> DatasetStatistics:
1061
1089
  """Get the dataset statistics.
@@ -1103,7 +1131,8 @@ class DatasetClientAsync(ResourceClientAsync):
1103
1131
 
1104
1132
  You can optionally control how long the signed URL should be valid using the `expires_in` option.
1105
1133
  This value sets the expiration duration from the time the URL is generated.
1106
- If not provided, the URL will not expire.
1134
+ If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
1135
+ working if any of them is changed.
1107
1136
 
1108
1137
  Any other options (like `limit` or `offset`) will be included as query parameters in the URL.
1109
1138
 
@@ -1120,7 +1149,7 @@ class DatasetClientAsync(ResourceClientAsync):
1120
1149
  skip_hidden: If True, then hidden fields are skipped from the output.
1121
1150
  flatten: A list of fields that should be flattened.
1122
1151
  view: Name of the dataset view to be used.
1123
- expires_in: How long the signed URL should be valid.
1152
+ expires_in: How long the signed URL should be valid. Defaults to 1 hour.
1124
1153
  timeout: Timeout for the API HTTP request.
1125
1154
 
1126
1155
  Returns:
@@ -1142,12 +1171,15 @@ class DatasetClientAsync(ResourceClientAsync):
1142
1171
  view=view,
1143
1172
  )
1144
1173
 
1174
+ public_url = self._build_public_url('items', request_params)
1175
+
1176
+ # Signed last: the signature binds every query parameter above.
1145
1177
  if dataset and dataset.url_signing_secret_key:
1146
- signature = create_storage_content_signature(
1178
+ public_url = sign_storage_url(
1179
+ public_url,
1147
1180
  dataset.id,
1148
1181
  dataset.url_signing_secret_key,
1149
1182
  expires_in=expires_in,
1150
1183
  )
1151
- request_params['signature'] = signature
1152
1184
 
1153
- return self._build_public_url('items', request_params)
1185
+ return public_url
@@ -17,7 +17,7 @@ from scrapely_client._models import (
17
17
  )
18
18
  from scrapely_client._pagination import DEFAULT_CHUNK_SIZE, get_cursor_iterator, get_cursor_iterator_async
19
19
  from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
20
- from scrapely_client._utils.crypto import create_hmac_signature, create_storage_content_signature
20
+ from scrapely_client._utils.crypto import sign_storage_url
21
21
  from scrapely_client._utils.encoding import encode_key_value_store_record_value
22
22
  from scrapely_client._utils.errors import catch_not_found_or_throw
23
23
  from scrapely_client._utils.http import response_to_dict, to_path_segment
@@ -409,7 +409,7 @@ class KeyValueStoreClient(ResourceClient):
409
409
  """Generate a URL that can be used to access key-value store record.
410
410
 
411
411
  If the client has permission to access the key-value store's URL signing key, the URL will include a signature
412
- to verify its authenticity.
412
+ to verify its authenticity. The signature expires in 1 hour.
413
413
 
414
414
  Args:
415
415
  key: The key for which the URL should be generated.
@@ -426,13 +426,19 @@ class KeyValueStoreClient(ResourceClient):
426
426
 
427
427
  metadata = self.get(timeout=timeout)
428
428
 
429
- request_params = self._build_params()
429
+ public_url = self._build_public_url(record_path, self._build_params())
430
430
 
431
- # The signature covers the raw key, which is what the API sees once it decodes the path segment.
431
+ # The signature binds the raw key, which is what the API sees once it decodes the path segment.
432
+ # It expires in 1 hour.
432
433
  if metadata and metadata.url_signing_secret_key:
433
- request_params['signature'] = create_hmac_signature(metadata.url_signing_secret_key, key)
434
+ public_url = sign_storage_url(
435
+ public_url,
436
+ metadata.id,
437
+ metadata.url_signing_secret_key,
438
+ record_key=key,
439
+ )
434
440
 
435
- return self._build_public_url(record_path, request_params)
441
+ return public_url
436
442
 
437
443
  def create_keys_public_url(
438
444
  self,
@@ -451,7 +457,8 @@ class KeyValueStoreClient(ResourceClient):
451
457
 
452
458
  You can optionally control how long the signed URL should be valid using the `expires_in` option.
453
459
  This value sets the expiration duration from the time the URL is generated.
454
- If not provided, the URL will not expire.
460
+ If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
461
+ working if any of them is changed.
455
462
 
456
463
  Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
457
464
 
@@ -460,7 +467,7 @@ class KeyValueStoreClient(ResourceClient):
460
467
  exclusive_start_key: All keys up to this one (including) are skipped from the result.
461
468
  collection: The name of the collection in store schema to list keys from.
462
469
  prefix: The prefix of the keys to be listed.
463
- expires_in: How long the signed URL should be valid from the time it is generated.
470
+ expires_in: How long the signed URL should be valid from the time it is generated. Defaults to 1 hour.
464
471
  timeout: Timeout for the API HTTP request.
465
472
 
466
473
  Returns:
@@ -475,15 +482,18 @@ class KeyValueStoreClient(ResourceClient):
475
482
  prefix=prefix,
476
483
  )
477
484
 
485
+ public_url = self._build_public_url('keys', request_params)
486
+
487
+ # Signed last: the signature binds every query parameter above.
478
488
  if metadata and metadata.url_signing_secret_key:
479
- signature = create_storage_content_signature(
489
+ public_url = sign_storage_url(
490
+ public_url,
480
491
  metadata.id,
481
492
  metadata.url_signing_secret_key,
482
493
  expires_in=expires_in,
483
494
  )
484
- request_params['signature'] = signature
485
495
 
486
- return self._build_public_url('keys', request_params)
496
+ return public_url
487
497
 
488
498
 
489
499
  @docs_group('Resource clients')
@@ -831,7 +841,7 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
831
841
  """Generate a URL that can be used to access key-value store record.
832
842
 
833
843
  If the client has permission to access the key-value store's URL signing key, the URL will include a signature
834
- to verify its authenticity.
844
+ to verify its authenticity. The signature expires in 1 hour.
835
845
 
836
846
  Args:
837
847
  key: The key for which the URL should be generated.
@@ -848,13 +858,19 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
848
858
 
849
859
  metadata = await self.get(timeout=timeout)
850
860
 
851
- request_params = self._build_params()
861
+ public_url = self._build_public_url(record_path, self._build_params())
852
862
 
853
- # The signature covers the raw key, which is what the API sees once it decodes the path segment.
863
+ # The signature binds the raw key, which is what the API sees once it decodes the path segment.
864
+ # It expires in 1 hour.
854
865
  if metadata and metadata.url_signing_secret_key:
855
- request_params['signature'] = create_hmac_signature(metadata.url_signing_secret_key, key)
866
+ public_url = sign_storage_url(
867
+ public_url,
868
+ metadata.id,
869
+ metadata.url_signing_secret_key,
870
+ record_key=key,
871
+ )
856
872
 
857
- return self._build_public_url(record_path, request_params)
873
+ return public_url
858
874
 
859
875
  async def create_keys_public_url(
860
876
  self,
@@ -873,7 +889,8 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
873
889
 
874
890
  You can optionally control how long the signed URL should be valid using the `expires_in` option.
875
891
  This value sets the expiration duration from the time the URL is generated.
876
- If not provided, the URL will not expire.
892
+ If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
893
+ working if any of them is changed.
877
894
 
878
895
  Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
879
896
 
@@ -882,7 +899,7 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
882
899
  exclusive_start_key: All keys up to this one (including) are skipped from the result.
883
900
  collection: The name of the collection in store schema to list keys from.
884
901
  prefix: The prefix of the keys to be listed.
885
- expires_in: How long the signed URL should be valid from the time it is generated.
902
+ expires_in: How long the signed URL should be valid from the time it is generated. Defaults to 1 hour.
886
903
  timeout: Timeout for the API HTTP request.
887
904
 
888
905
  Returns:
@@ -897,12 +914,15 @@ class KeyValueStoreClientAsync(ResourceClientAsync):
897
914
  prefix=prefix,
898
915
  )
899
916
 
917
+ public_url = self._build_public_url('keys', request_params)
918
+
919
+ # Signed last: the signature binds every query parameter above.
900
920
  if metadata and metadata.url_signing_secret_key:
901
- signature = create_storage_content_signature(
921
+ public_url = sign_storage_url(
922
+ public_url,
902
923
  metadata.id,
903
924
  metadata.url_signing_secret_key,
904
925
  expires_in=expires_in,
905
926
  )
906
- request_params['signature'] = signature
907
927
 
908
- return self._build_public_url('keys', request_params)
928
+ return public_url
@@ -3,6 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  from typing import TYPE_CHECKING, Any
6
+ from uuid import uuid4
6
7
 
7
8
  from scrapely_client._docs import docs_group
8
9
  from scrapely_client._models import (
@@ -272,7 +273,8 @@ class TaskClient(ResourceClient):
272
273
  response = self._http_client.call(
273
274
  url=self._build_url('runs'),
274
275
  method='POST',
275
- headers={'content-type': 'application/json; charset=utf-8'},
276
+ # The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
277
+ headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
276
278
  json=task_input.model_dump() if task_input is not None else None,
277
279
  params=request_params,
278
280
  timeout=timeout,
@@ -652,7 +654,8 @@ class TaskClientAsync(ResourceClientAsync):
652
654
  response = await self._http_client.call(
653
655
  url=self._build_url('runs'),
654
656
  method='POST',
655
- headers={'content-type': 'application/json; charset=utf-8'},
657
+ # The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
658
+ headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
656
659
  json=task_input.model_dump() if task_input is not None else None,
657
660
  params=request_params,
658
661
  timeout=timeout,
@@ -2,12 +2,14 @@
2
2
  # Modified by TeraShift GmbH. See the LICENSE and NOTICE files.
3
3
  from __future__ import annotations
4
4
 
5
+ import os
5
6
  from functools import cached_property
6
7
  from typing import TYPE_CHECKING
7
8
 
8
9
  from scrapely_client._charging import ChargingHooks, wrap_dataset_client_with_charging
9
10
  from scrapely_client._client_registry import ClientRegistry, ClientRegistryAsync
10
11
  from scrapely_client._consts import (
12
+ API_URL_ENV_VAR,
11
13
  API_VERSION,
12
14
  DEFAULT_API_URL,
13
15
  DEFAULT_MAX_RETRIES,
@@ -90,6 +92,13 @@ if TYPE_CHECKING:
90
92
  from scrapely_client.types import HttpCompressionAlgorithm
91
93
 
92
94
 
95
+ def _resolve_api_url(api_url: str | None) -> str:
96
+ """Return `api_url` if given, else the `SCRAPELY_API_URL` environment variable, else the production API."""
97
+ if api_url is not None:
98
+ return api_url
99
+ return os.environ.get(API_URL_ENV_VAR, '').strip() or DEFAULT_API_URL
100
+
101
+
93
102
  @docs_group('Scrapely API clients')
94
103
  class ApifyClient:
95
104
  """Synchronous client for the Scrapely API.
@@ -124,7 +133,7 @@ class ApifyClient:
124
133
  self,
125
134
  token: str | None = None,
126
135
  *,
127
- api_url: str = DEFAULT_API_URL,
136
+ api_url: str | None = None,
128
137
  api_public_url: str | None = DEFAULT_API_URL,
129
138
  max_retries: int = DEFAULT_MAX_RETRIES,
130
139
  min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
@@ -144,9 +153,9 @@ class ApifyClient:
144
153
  Args:
145
154
  token: The Scrapely API token. You can find your token on the
146
155
  Integrations page in the Scrapely dashboard.
147
- api_url: The URL of the Scrapely API server to connect to. Defaults to https://api.scrape.ly. It can
148
- be an internal URL that is not globally accessible, in which case `api_public_url` should be set
149
- as well.
156
+ api_url: The URL of the Scrapely API server to connect to. When not given, the `SCRAPELY_API_URL`
157
+ environment variable is used, then https://api.scrape.ly. It can be an internal URL that is not
158
+ globally accessible, in which case `api_public_url` should be set as well.
150
159
  api_public_url: The globally accessible URL of the Scrapely API server. Should be set only if `api_url`
151
160
  is an internal URL that is not globally accessible. Defaults to https://api.scrape.ly.
152
161
  max_retries: How many times to retry a failed request at most.
@@ -165,7 +174,7 @@ class ApifyClient:
165
174
  default_dataset_id: ID of the actor run's default dataset, used to gate charging interception.
166
175
  """
167
176
  # We need to do this because of mocking in tests and default mutable arguments.
168
- api_url = DEFAULT_API_URL if api_url is None else api_url
177
+ api_url = _resolve_api_url(api_url)
169
178
  api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
170
179
 
171
180
  if headers:
@@ -234,7 +243,7 @@ class ApifyClient:
234
243
  cls,
235
244
  token: str | None = None,
236
245
  *,
237
- api_url: str = DEFAULT_API_URL,
246
+ api_url: str | None = None,
238
247
  api_public_url: str | None = DEFAULT_API_URL,
239
248
  http_client: HttpClient,
240
249
  ) -> ApifyClient:
@@ -263,7 +272,8 @@ class ApifyClient:
263
272
  Args:
264
273
  token: The Scrapely API token. It is set as the `Authorization` header on the custom client,
265
274
  unless the client already has one configured.
266
- api_url: The URL of the Scrapely API server to connect to. Defaults to https://api.scrape.ly.
275
+ api_url: The URL of the Scrapely API server to connect to. When not given, the `SCRAPELY_API_URL`
276
+ environment variable is used, then https://api.scrape.ly.
267
277
  api_public_url: The globally accessible URL of the Scrapely API server. Defaults to https://api.scrape.ly.
268
278
  http_client: A custom HTTP client instance extending `HttpClient`.
269
279
  """
@@ -513,7 +523,7 @@ class ApifyClientAsync:
513
523
  self,
514
524
  token: str | None = None,
515
525
  *,
516
- api_url: str = DEFAULT_API_URL,
526
+ api_url: str | None = None,
517
527
  api_public_url: str | None = DEFAULT_API_URL,
518
528
  max_retries: int = DEFAULT_MAX_RETRIES,
519
529
  min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
@@ -533,9 +543,9 @@ class ApifyClientAsync:
533
543
  Args:
534
544
  token: The Scrapely API token. You can find your token on the
535
545
  Integrations page in the Scrapely dashboard.
536
- api_url: The URL of the Scrapely API server to connect to. Defaults to https://api.scrape.ly. It can
537
- be an internal URL that is not globally accessible, in which case `api_public_url` should be set
538
- as well.
546
+ api_url: The URL of the Scrapely API server to connect to. When not given, the `SCRAPELY_API_URL`
547
+ environment variable is used, then https://api.scrape.ly. It can be an internal URL that is not
548
+ globally accessible, in which case `api_public_url` should be set as well.
539
549
  api_public_url: The globally accessible URL of the Scrapely API server. Should be set only if `api_url`
540
550
  is an internal URL that is not globally accessible. Defaults to https://api.scrape.ly.
541
551
  max_retries: How many times to retry a failed request at most.
@@ -554,7 +564,7 @@ class ApifyClientAsync:
554
564
  default_dataset_id: ID of the actor run's default dataset, used to gate charging interception.
555
565
  """
556
566
  # We need to do this because of mocking in tests and default mutable arguments.
557
- api_url = DEFAULT_API_URL if api_url is None else api_url
567
+ api_url = _resolve_api_url(api_url)
558
568
  api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
559
569
 
560
570
  if headers:
@@ -623,7 +633,7 @@ class ApifyClientAsync:
623
633
  cls,
624
634
  token: str | None = None,
625
635
  *,
626
- api_url: str = DEFAULT_API_URL,
636
+ api_url: str | None = None,
627
637
  api_public_url: str | None = DEFAULT_API_URL,
628
638
  http_client: HttpClientAsync,
629
639
  ) -> ApifyClientAsync:
@@ -652,7 +662,8 @@ class ApifyClientAsync:
652
662
  Args:
653
663
  token: The Scrapely API token. It is set as the `Authorization` header on the custom client,
654
664
  unless the client already has one configured.
655
- api_url: The URL of the Scrapely API server to connect to. Defaults to https://api.scrape.ly.
665
+ api_url: The URL of the Scrapely API server to connect to. When not given, the `SCRAPELY_API_URL`
666
+ environment variable is used, then https://api.scrape.ly.
656
667
  api_public_url: The globally accessible URL of the Scrapely API server. Defaults to https://api.scrape.ly.
657
668
  http_client: A custom HTTP client instance extending `HttpClientAsync`.
658
669
  """
@@ -7,15 +7,19 @@ import hmac
7
7
  import string
8
8
  import time
9
9
  from base64 import urlsafe_b64encode
10
- from typing import TYPE_CHECKING
10
+ from datetime import timedelta
11
+ from urllib.parse import parse_qsl, quote, urlencode, urlsplit, urlunsplit
11
12
 
12
13
  from scrapely_client._utils.time import to_seconds
13
14
 
14
- if TYPE_CHECKING:
15
- from datetime import timedelta
16
-
17
15
  _BASE62_CHARSET = string.digits + string.ascii_letters
18
16
 
17
+ STORAGE_SIGNATURE_VERSION = 2
18
+ """Version of newly minted storage signatures (binds method, path, record key and query string)."""
19
+
20
+ DEFAULT_STORAGE_URL_EXPIRES_IN = timedelta(hours=1)
21
+ """Validity of a storage signature when the caller does not set `expires_in`."""
22
+
19
23
 
20
24
  def encode_base62(num: int) -> str:
21
25
  """Encode an integer to a base62 string.
@@ -58,32 +62,100 @@ def create_hmac_signature(secret_key: str, message: str) -> str:
58
62
  return encode_base62(decimal_signature)
59
63
 
60
64
 
65
+ def storage_signature_query(url: str) -> str:
66
+ """Return the canonical query string a version 2 storage signature binds.
67
+
68
+ The raw query of `url` is parsed like `URLSearchParams` (`+` is a space), `signature` is dropped, each key and
69
+ value is percent-encoded (RFC 3986: only `A-Z a-z 0-9 - . _ ~` stay as is), and the pairs are sorted by encoded
70
+ key, then encoded value, and joined as `k=v&k=v`. The Scrapely API and the JS client build the same string.
71
+
72
+ Args:
73
+ url: The URL (or path with a query string) to canonicalize.
74
+
75
+ Returns:
76
+ The canonical query string, empty when the URL has no other parameters.
77
+ """
78
+ pairs = sorted(
79
+ (quote(key, safe=''), quote(value, safe=''))
80
+ for key, value in parse_qsl(urlsplit(url).query, keep_blank_values=True)
81
+ if key != 'signature'
82
+ )
83
+ return '&'.join(f'{key}={value}' for key, value in pairs)
84
+
85
+
61
86
  def create_storage_content_signature(
62
87
  resource_id: str,
63
88
  url_signing_secret_key: str,
64
89
  *,
90
+ method: str,
91
+ path: str,
92
+ record_key: str | None = None,
93
+ query: str = '',
65
94
  expires_in: timedelta | None = None,
66
- version: int = 0,
67
95
  ) -> str:
68
- """Create a secure signature for a storage resource like a dataset or key-value store.
96
+ """Create a signature for a storage resource URL like dataset items or a key-value store record.
69
97
 
70
- This signature is used to generate a signed URL for authenticated access, which can be expiring or permanent.
71
- The signature is created using HMAC with the provided secret key and includes the resource ID, expiration time,
72
- and version.
98
+ The signature binds the HTTP method, the URL path, the canonical query string (`storage_signature_query`) and,
99
+ for a single record, the record key, so the API refuses it for any other request. The signed message is
100
+ `{version}.{expires_at}.{resource_id}.{METHOD}.{path}.{record_key}.{query}`, the same as the Scrapely API and the
101
+ JS client.
73
102
 
74
103
  Args:
75
104
  resource_id: The unique identifier of the storage resource.
76
105
  url_signing_secret_key: The secret key for signing the URL.
77
- expires_in: Optional expiration duration; if None, the signature never expires.
78
- version: The signature version number (default: 0).
106
+ method: The HTTP method the signature authorizes, e.g. `GET`.
107
+ path: The URL path (no query string), beginning with `/`.
108
+ record_key: The record key when the signature authorizes exactly one record.
109
+ query: The canonical query string from `storage_signature_query`.
110
+ expires_in: How long the signature is valid. Defaults to 1 hour; it cannot be non-expiring.
79
111
 
80
112
  Returns:
81
113
  The base64url-encoded signature string.
82
114
  """
83
- expires_at = int(time.time() * 1000) + int(to_seconds(expires_in) * 1000) if expires_in is not None else 0
115
+ expires_in_secs = to_seconds(expires_in if expires_in is not None else DEFAULT_STORAGE_URL_EXPIRES_IN)
116
+ if expires_in_secs <= 0:
117
+ raise ValueError('expires_in must be a positive duration')
118
+ expires_at = int(time.time() * 1000) + round(expires_in_secs * 1000)
119
+ version = STORAGE_SIGNATURE_VERSION
84
120
 
85
- message_to_sign = f'{version}.{expires_at}.{resource_id}'
121
+ message_to_sign = f'{version}.{expires_at}.{resource_id}.{method.upper()}.{path}.{record_key or ""}.{query}'
86
122
  hmac_sig = create_hmac_signature(url_signing_secret_key, message_to_sign)
87
123
 
88
- base64url_encoded_payload = urlsafe_b64encode(f'{version}.{expires_at}.{hmac_sig}'.encode())
89
- return base64url_encoded_payload.decode('utf-8')
124
+ # Unpadded, like the JS client and Node's base64url.
125
+ return urlsafe_b64encode(f'{version}.{expires_at}.{hmac_sig}'.encode()).decode('utf-8').rstrip('=')
126
+
127
+
128
+ def sign_storage_url(
129
+ url: str,
130
+ resource_id: str,
131
+ url_signing_secret_key: str,
132
+ *,
133
+ method: str = 'GET',
134
+ record_key: str | None = None,
135
+ expires_in: timedelta | None = None,
136
+ ) -> str:
137
+ """Append a `signature` parameter that binds the URL's path and query string.
138
+
139
+ Args:
140
+ url: The finished URL, with every other query parameter already set.
141
+ resource_id: The unique identifier of the storage resource.
142
+ url_signing_secret_key: The secret key for signing the URL.
143
+ method: The HTTP method the signature authorizes.
144
+ record_key: The record key when the URL is for exactly one record.
145
+ expires_in: How long the signature is valid. Defaults to 1 hour.
146
+
147
+ Returns:
148
+ The URL with the signature appended.
149
+ """
150
+ parts = urlsplit(url)
151
+ signature = create_storage_content_signature(
152
+ resource_id,
153
+ url_signing_secret_key,
154
+ method=method,
155
+ path=parts.path,
156
+ record_key=record_key,
157
+ query=storage_signature_query(url),
158
+ expires_in=expires_in,
159
+ )
160
+ signature_param = urlencode({'signature': signature})
161
+ return urlunsplit(parts._replace(query=f'{parts.query}&{signature_param}' if parts.query else signature_param))
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import asyncio
6
6
  import logging
7
7
  import random
8
+ import re
8
9
  import time
9
10
  from contextlib import suppress
10
11
  from datetime import timedelta
@@ -67,6 +68,50 @@ def _is_retryable_error(exc: Exception) -> bool:
67
68
  return isinstance(exc, impit.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS)
68
69
 
69
70
 
71
+ # Failures that happen before the request reaches the server, so resending can't apply it twice.
72
+ _NOT_SENT_ERRORS = (impit.ConnectError, impit.ConnectTimeout, impit.PoolTimeout)
73
+
74
+
75
+ # POSTs that are safe to resend without a key: request queue adds (the queue keeps one request per uniqueKey),
76
+ # unlocks and aborts (repeating them changes nothing) and head locks (a repeat only locks more requests until the
77
+ # lock expires). A run's default queue is also reached as /actor-runs/<id>/request-queue.
78
+ _SAFE_TO_RESEND_POST_PATH_RE = re.compile(
79
+ r'/(?:request-queues/[^/]+|actor-runs/[^/]+/request-queue)/(?:requests(?:/batch|/unlock)?|head/lock)/?$|/abort/?$'
80
+ )
81
+
82
+ # Get-or-create of a storage by name: a resend finds the storage the first request made.
83
+ _STORAGE_COLLECTION_PATH_RE = re.compile(r'/(?:datasets|key-value-stores|request-queues)/?$')
84
+
85
+ # Answers from the proxy in front of the API when the API did not take the request (e.g. restarting during a deploy),
86
+ # so resending can't apply it twice.
87
+ _NOT_APPLIED_STATUS_CODES = frozenset({HTTPStatus.BAD_GATEWAY, HTTPStatus.SERVICE_UNAVAILABLE})
88
+
89
+ # Keyed requests: the first request with the key is still being applied, so a resend gets its answer shortly.
90
+ _KEY_IN_PROGRESS_ERROR_TYPE = 'idempotency-key-in-progress'
91
+ # Keyed requests: the server can't tell whether the first request was applied, and no resend changes that.
92
+ _OUTCOME_UNKNOWN_ERROR_TYPE = 'push-outcome-unknown'
93
+
94
+
95
+ def _is_unsafe_to_resend(method: str, url: str, headers: dict[str, str], params: dict[str, Any] | None = None) -> bool:
96
+ """Check if resending the request after an ambiguous failure could apply it twice.
97
+
98
+ A POST or PATCH without an `Idempotency-Key` header may already have been applied when the server answers 5xx or
99
+ the connection drops mid-request, e.g. a dataset push stored twice or a second run started. Request queue adds,
100
+ unlocks, head locks and aborts are always safe (see `_SAFE_TO_RESEND_POST_PATH_RE`), and so is a named storage's
101
+ get-or-create (see `_STORAGE_COLLECTION_PATH_RE`).
102
+ """
103
+ path = url.split('?', maxsplit=1)[0]
104
+ if method.upper() not in ('POST', 'PATCH') or _SAFE_TO_RESEND_POST_PATH_RE.search(path):
105
+ return False
106
+ if method.upper() == 'POST' and _STORAGE_COLLECTION_PATH_RE.search(path) and (params or {}).get('name'):
107
+ return False
108
+ return not _has_idempotency_key(headers)
109
+
110
+
111
+ def _has_idempotency_key(headers: dict[str, str]) -> bool:
112
+ return any(name.lower() == 'idempotency-key' for name in headers)
113
+
114
+
70
115
  @docs_group('HTTP clients')
71
116
  class ImpitHttpClient(HttpClient):
72
117
  """Synchronous HTTP client for the Scrapely API built on top of Impit.
@@ -249,16 +294,32 @@ class ImpitHttpClient(HttpClient):
249
294
  if not _is_retryable_error(exc):
250
295
  logger.debug('Exception is not retryable', exc_info=exc)
251
296
  stop_retrying()
297
+ elif _is_unsafe_to_resend(method, url, headers, params) and not isinstance(exc, _NOT_SENT_ERRORS):
298
+ logger.debug('Request may have been applied; not resending without an idempotency key', exc_info=exc)
299
+ stop_retrying()
252
300
  raise
253
301
 
254
- # Retry only server errors (5xx) and rate limits (429).
302
+ # Retry only server errors (502/503, other 5xx when resending is safe), rate limits (429) and a keyed
303
+ # request's 409 while the first request with that key is still being applied (decided by the error type below).
255
304
  logger.debug('Request unsuccessful', extra={'status_code': response.status_code})
305
+ keyed_conflict = response.status_code == HTTPStatus.CONFLICT and _has_idempotency_key(headers)
256
306
  if (
257
307
  response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR
258
308
  and response.status_code != HTTPStatus.TOO_MANY_REQUESTS
309
+ and not keyed_conflict
259
310
  ):
260
311
  logger.debug('Status code is not retryable', extra={'status_code': response.status_code})
261
312
  stop_retrying()
313
+ elif (
314
+ response.status_code != HTTPStatus.TOO_MANY_REQUESTS
315
+ and response.status_code not in _NOT_APPLIED_STATUS_CODES
316
+ and _is_unsafe_to_resend(method, url, headers, params)
317
+ ):
318
+ logger.debug(
319
+ 'Request may have been applied; not resending without an idempotency key',
320
+ extra={'status_code': response.status_code},
321
+ )
322
+ stop_retrying()
262
323
 
263
324
  # Read the response in case it is a stream, so we can raise the error properly. A failed read goes through
264
325
  # the same classification as a failed send.
@@ -273,7 +334,11 @@ class ImpitHttpClient(HttpClient):
273
334
  stop_retrying()
274
335
  raise
275
336
 
276
- raise ApifyApiError(response, attempt, method=method)
337
+ error = ApifyApiError(response, attempt, method=method)
338
+ if (keyed_conflict and error.type != _KEY_IN_PROGRESS_ERROR_TYPE) or error.type == _OUTCOME_UNKNOWN_ERROR_TYPE:
339
+ logger.debug('Error type is not retryable', extra={'status_code': response.status_code})
340
+ stop_retrying()
341
+ raise error
277
342
 
278
343
  @staticmethod
279
344
  def _retry_with_exp_backoff(
@@ -522,16 +587,32 @@ class ImpitHttpClientAsync(HttpClientAsync):
522
587
  if not _is_retryable_error(exc):
523
588
  logger.debug('Exception is not retryable', exc_info=exc)
524
589
  stop_retrying()
590
+ elif _is_unsafe_to_resend(method, url, headers, params) and not isinstance(exc, _NOT_SENT_ERRORS):
591
+ logger.debug('Request may have been applied; not resending without an idempotency key', exc_info=exc)
592
+ stop_retrying()
525
593
  raise
526
594
 
527
- # Retry only server errors (5xx) and rate limits (429).
595
+ # Retry only server errors (502/503, other 5xx when resending is safe), rate limits (429) and a keyed
596
+ # request's 409 while the first request with that key is still being applied (decided by the error type below).
528
597
  logger.debug('Request unsuccessful', extra={'status_code': response.status_code})
598
+ keyed_conflict = response.status_code == HTTPStatus.CONFLICT and _has_idempotency_key(headers)
529
599
  if (
530
600
  response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR
531
601
  and response.status_code != HTTPStatus.TOO_MANY_REQUESTS
602
+ and not keyed_conflict
532
603
  ):
533
604
  logger.debug('Status code is not retryable', extra={'status_code': response.status_code})
534
605
  stop_retrying()
606
+ elif (
607
+ response.status_code != HTTPStatus.TOO_MANY_REQUESTS
608
+ and response.status_code not in _NOT_APPLIED_STATUS_CODES
609
+ and _is_unsafe_to_resend(method, url, headers, params)
610
+ ):
611
+ logger.debug(
612
+ 'Request may have been applied; not resending without an idempotency key',
613
+ extra={'status_code': response.status_code},
614
+ )
615
+ stop_retrying()
535
616
 
536
617
  # Read the response in case it is a stream, so we can raise the error properly. A failed read goes through
537
618
  # the same classification as a failed send.
@@ -546,7 +627,11 @@ class ImpitHttpClientAsync(HttpClientAsync):
546
627
  stop_retrying()
547
628
  raise
548
629
 
549
- raise ApifyApiError(response, attempt, method=method)
630
+ error = ApifyApiError(response, attempt, method=method)
631
+ if (keyed_conflict and error.type != _KEY_IN_PROGRESS_ERROR_TYPE) or error.type == _OUTCOME_UNKNOWN_ERROR_TYPE:
632
+ logger.debug('Error type is not retryable', extra={'status_code': response.status_code})
633
+ stop_retrying()
634
+ raise error
550
635
 
551
636
  @staticmethod
552
637
  async def _retry_with_exp_backoff(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-client-python
3
- Version: 1.0.8
3
+ Version: 1.0.10
4
4
  Summary: Scrapely API client for Python
5
5
  Keywords: scrapely,api,client,automation,crawling,scraping
6
6
  Author: TeraShift GmbH (Scrape.ly)
@@ -36,7 +36,7 @@ Description-Content-Type: text/markdown
36
36
  <h1 align="center">Scrapely API client for Python</h1>
37
37
 
38
38
  <p align="center">
39
- <strong>The official Python client for the <a href="https://api.scrape.ly">Scrapely REST API</a>.</strong>
39
+ <strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
40
40
  </p>
41
41
 
42
42
  <p align="center">
@@ -46,9 +46,9 @@ Description-Content-Type: text/markdown
46
46
  <a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
47
47
  </p>
48
48
 
49
- `scrapely-client-python` lets you talk to the [Scrapely platform](https://api.scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
49
+ `scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
50
50
 
51
- > If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
51
+ > If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
52
52
 
53
53
  ## Table of contents
54
54
 
@@ -92,7 +92,9 @@ Description-Content-Type: text/markdown
92
92
 
93
93
  ## Quick start
94
94
 
95
- You'll need a Scrapely API token — find yours in the integrations section of the Scrapely dashboard. Pass it to the client and you're ready to go.
95
+ You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
96
+
97
+ The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
96
98
 
97
99
  ### Synchronous client
98
100
 
@@ -190,6 +192,21 @@ for item in client.dataset('DATASET-ID').iterate_items(fields=['title', 'url']):
190
192
  process(item)
191
193
  ```
192
194
 
195
+ ### Create and run an automation
196
+
197
+ ```python
198
+ automation = client.automations().create({
199
+ 'title': 'Daily dentists',
200
+ 'trigger': {'kind': 'schedule', 'cron': '0 9 * * *', 'timezone': 'Europe/Berlin'},
201
+ 'steps': [{'kind': 'savedInput', 'savedInputId': 'SAVED-INPUT-ID'}],
202
+ 'ai': {'mode': 'summarize', 'prompt': 'List the new practices with their phone numbers.'},
203
+ 'deliveries': [{'channel': 'email'}],
204
+ })
205
+ run = client.automation(automation['id']).run()
206
+ ```
207
+
208
+ Automations take and return the API's camelCase dictionaries (see the automations section of the API reference).
209
+
193
210
  ### Tune retries and timeouts
194
211
 
195
212
  ```python
@@ -205,33 +222,33 @@ client = ScrapelyClient(
205
222
  )
206
223
  ```
207
224
 
208
- For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the Guides.
225
+ For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
209
226
 
210
227
  ## Documentation
211
228
 
212
- The full documentation lives at **https://api.scrape.ly**.
229
+ The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
230
+
231
+ The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
213
232
 
214
233
  | Section | What you'll find |
215
234
  | --- | --- |
216
- | Introduction | Overview, prerequisites, and a tour of the client. |
217
- | Quick start | Authenticate, run an Actor, and fetch its results step by step. |
218
- | Concepts | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. |
219
- | Guides | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
220
- | API reference | Generated reference for every class, method, and model. |
221
- | Changelog | Release history and breaking changes. |
235
+ | [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
236
+ | [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
237
+ | [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
238
+ | [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
222
239
 
223
240
  ## Related projects
224
241
 
225
- - **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
242
+ - **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
226
243
  - **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
227
- - **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
228
- - **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
244
+ - **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
245
+ - **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
229
246
  - **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
230
- - **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
247
+ - **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
231
248
 
232
249
  ## Support and community
233
250
 
234
- - **GitHub issues** — report a bug or request a feature in the repository's issue tracker.
251
+ - **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
235
252
 
236
253
  ## Contributing
237
254
 
@@ -1,15 +1,15 @@
1
1
  scrapely_client/__init__.py,sha256=Fu2-Un-vv1wGjqG3QyxDkbxMOmg6kfwdDBQ0yPGFm2M,879
2
2
  scrapely_client/_charging.py,sha256=L7vvpX9bj0UwHc-83_zFpfcGyrMlbzoB_EpE9iSH2zM,6807
3
3
  scrapely_client/_client_registry.py,sha256=FXqW7wYdHaFUjrtugXkh9IORzazzxGQssHOgtudj8Jc,5588
4
- scrapely_client/_consts.py,sha256=Eh9KdIKY1yeFjT0GGMg8AZASa6ed9VjbzVtpgrNU1Ak,3537
4
+ scrapely_client/_consts.py,sha256=zfGGi17JdkNR23W8hQinwbm81wXg_H_Pw1C_mpMMhWc,3668
5
5
  scrapely_client/_docs.py,sha256=MtrntqA7S0BDKzJlZzBAfqeI2bUtybwcb85JfP9vaDA,1117
6
6
  scrapely_client/_literals.py,sha256=qSc0qIjhcI7qa42fNQ-IGPm_7cpCI9_OaQ6oYGIOdy4,17006
7
- scrapely_client/_logging.py,sha256=7yZ2GE8zv9M5r5IdmPB_jiAPy-fJN_kYSWQ2UwXu6p8,6780
8
- scrapely_client/_models.py,sha256=fQ7WQQpXp5f-8_RiXG7Un2JVDBdO4_rQFUIP5-DewpM,142909
7
+ scrapely_client/_logging.py,sha256=F-lwPd2tkDh9vomswuVzo1I3LliHBshgn-65Q29_3gg,7266
8
+ scrapely_client/_models.py,sha256=gX_Gw5_0R3cScpOxgpRDNJliBiOqcU0Ot208H4aaW_w,142907
9
9
  scrapely_client/_pagination.py,sha256=by-2FLNUQquXfxgr2mJVCgOqBBZRYMJpmTjfy4aWfpE,8579
10
10
  scrapely_client/_resource_clients/__init__.py,sha256=cy_gBIqs8m7UpnKToffNad2zX2Su8We5B_q5IDM879M,4016
11
11
  scrapely_client/_resource_clients/_resource_client.py,sha256=b22QuV7osKBor62RsFWMOMKoyevUAY9PY2JrirxrWA8,21732
12
- scrapely_client/_resource_clients/actor.py,sha256=9aJipoTI8BBYg1NZcHKuDQASatHe3XsUqKwtmNGSHkI,46153
12
+ scrapely_client/_resource_clients/actor.py,sha256=oh9uc0m84fo1RouHjkbGpaetzWMqBMXXac9duPc-0CQ,46764
13
13
  scrapely_client/_resource_clients/actor_collection.py,sha256=LynOa8R-sTogMVN6jVl2mDW4xmw4a1IZ35Hv8gHX71c,16966
14
14
  scrapely_client/_resource_clients/actor_env_var.py,sha256=dWKE_5xqhjQwCiNPNRXiyCw6g0GcUq1DxUR6Nd1JrRs,4810
15
15
  scrapely_client/_resource_clients/actor_env_var_collection.py,sha256=1Ueh_QYHbIS1Ztl7X9ZpIMglFND1U8Nmw08u0Z4ERrs,5675
@@ -19,9 +19,9 @@ scrapely_client/_resource_clients/automation.py,sha256=5dzATC-ZNsbfYUgvBxPadDtID
19
19
  scrapely_client/_resource_clients/automation_collection.py,sha256=0AOWLE3wuAlD1oWywEAfXBsnClLHs_IQ3l4Hi28ibac,5592
20
20
  scrapely_client/_resource_clients/build.py,sha256=Bp2OTcil31OunorPKnw6CIc9K0ZlnYLStaNENtPkq1Y,7805
21
21
  scrapely_client/_resource_clients/build_collection.py,sha256=TAuG4H46n6HIIntjg4tWiKqiDOgIUao5LX95xWjExwc,5766
22
- scrapely_client/_resource_clients/dataset.py,sha256=-4TFeqr9hVLEs_QbbkjP1jlD-AXPdPQC4Nl_FqlxPI0,52150
22
+ scrapely_client/_resource_clients/dataset.py,sha256=96bk1iRtS51LEWyccUU93LiZueV4QAnvOb8DhKsLPRU,54008
23
23
  scrapely_client/_resource_clients/dataset_collection.py,sha256=3IGH7B12agw1kHPhzO_1-HzXHyUCtPnFmSxT35_N1Bw,8859
24
- scrapely_client/_resource_clients/key_value_store.py,sha256=Fthku9RWqCGjBwvuvdUiiT6YpvlOLNQDYsu4VMMEmSU,32673
24
+ scrapely_client/_resource_clients/key_value_store.py,sha256=PbJQ2ecY9j4qTnR9s3fdDnE8Mqq91vGexy8nufwTd1g,33314
25
25
  scrapely_client/_resource_clients/key_value_store_collection.py,sha256=D9XPMI7DVdAB1rPK-VKbeY1_JNxSajyPxJIhTTcCT0g,9279
26
26
  scrapely_client/_resource_clients/log.py,sha256=XBuJRpUvBKfVhFypMFE0Wowpv5VOg2fs57PDkJEVaEQ,7491
27
27
  scrapely_client/_resource_clients/request_queue.py,sha256=ajPrm2uAjjMiAUlsie4D6o9rLb6J1H9fhOdKxXiLJCQ,42518
@@ -31,20 +31,20 @@ scrapely_client/_resource_clients/run_collection.py,sha256=tU-UX1W_mYD7tRfL9S2YH
31
31
  scrapely_client/_resource_clients/schedule.py,sha256=jY1ppDtVBRT7qMD-0XKXx_mXdNr0nNLjv5vD2YV8UYI,8094
32
32
  scrapely_client/_resource_clients/schedule_collection.py,sha256=NheBq6TfWLn6mip0toHB4lPhOgFaVFpicFJ5XscSf2o,9514
33
33
  scrapely_client/_resource_clients/store_collection.py,sha256=TOoxgyWy8JLjzmZ8m4gv52s6CZF5TV_ovpArFHQeneU,9230
34
- scrapely_client/_resource_clients/task.py,sha256=EXDcUmEmpkQWL4cMQUL7hmOpk9ACG_5wZvkKxAnIK9A,32597
34
+ scrapely_client/_resource_clients/task.py,sha256=pOpn_n3V02jX5xvgJlJx-q7bQy4JH51Q20t1m7klgCU,32908
35
35
  scrapely_client/_resource_clients/task_collection.py,sha256=Lygg-I_tvdIbW0zo9yS4vqTpmofx9qFtr8KhhG7jBbI,13327
36
36
  scrapely_client/_resource_clients/user.py,sha256=-xxfBt71rDdEMZq4qGd7DkYQ4fMJnr_epqrU57nM804,8769
37
37
  scrapely_client/_resource_clients/webhook.py,sha256=bZNla1cNM508EQL8zPe-7vgvfsCffQp5A3qnbfxuxe0,10484
38
38
  scrapely_client/_resource_clients/webhook_collection.py,sha256=-1LYc4upqSonEJdLVEldMvaV3Wi38EaL-Lzd9cmlAbQ,11351
39
39
  scrapely_client/_resource_clients/webhook_dispatch.py,sha256=ylrPUHHH-0UlF6htefV2tceHGB3w3WdrZpjKLD1oyec,2660
40
40
  scrapely_client/_resource_clients/webhook_dispatch_collection.py,sha256=RZXnczm6UKxnqDWmDYBvwyusp1_h1ONJMB7uCsBHg6E,5949
41
- scrapely_client/_scrapely_client.py,sha256=N0JNmFq279IzP57uWmzIUikpsDeqOYgALFcDAf2wjSs,38026
41
+ scrapely_client/_scrapely_client.py,sha256=1h03mj75oLpQGw_gIZeZrNk7_8PwQTTxzAPaQr4RxWo,38566
42
42
  scrapely_client/_statistics.py,sha256=mTsUFVByduYFPoHocZULv53k5dYg77Bl03wamQD6Jyc,903
43
43
  scrapely_client/_status_message_watcher.py,sha256=9Tstosbk_bUoq5K9xkhRQ5PBqsFaJFtvsGsAU0OvLjY,7893
44
44
  scrapely_client/_streamed_log.py,sha256=OaxB5-ghZX1DYuZmjNCgVeHAH0fYPReqjQlWaa8-dVA,11659
45
45
  scrapely_client/_typeddicts.py,sha256=oFrRIATgElDDaDay53l64WeS0Ctet8ztZ9Akhzd-SnA,12425
46
46
  scrapely_client/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
47
- scrapely_client/_utils/crypto.py,sha256=dPBSGHxS-Bh7eRbwDvekPtepL6w11Pp9R0JlmBsgoC8,2899
47
+ scrapely_client/_utils/crypto.py,sha256=Ney70TRfBVfttn6XMUfSnJzDZHacT9mtrH6SSqvGaCM,6038
48
48
  scrapely_client/_utils/encoding.py,sha256=w2J8CWXB_yHCLkilWGwIddkOJbpJK7nOSa24JiwZdhs,6460
49
49
  scrapely_client/_utils/errors.py,sha256=NyuYpfqG_Scq-yYECJisnmG1r3h0vCXYItdDX8DNKaU,1234
50
50
  scrapely_client/_utils/http.py,sha256=o0GETJsVjdcOJUuFle7XFsToss7B6k5NtV62kE2UISQ,4959
@@ -53,7 +53,7 @@ scrapely_client/_utils/try_import.py,sha256=T1m3vcn6V6crHwm4Syw8X_TwcKfhfv_shWZD
53
53
  scrapely_client/errors.py,sha256=sNKvkA7KUjLfBPrCkhrFu6Owt9cBKfCOL5Olw0IhjnQ,6955
54
54
  scrapely_client/http_clients/__init__.py,sha256=s6d7wVMJKLWodAfGR3lPAXfy4aEF1RWQgHKB1OMR9tY,396
55
55
  scrapely_client/http_clients/_base.py,sha256=5babUECMpMiU-OMy7J_-LNqQxQ2wERERw8YuRuy9Vvg,19011
56
- scrapely_client/http_clients/_impit.py,sha256=LmBdDlRBQfVcfWyQb3bJE9nJbm-Eawweci4BndR_jkk,24167
56
+ scrapely_client/http_clients/_impit.py,sha256=L2XnwJJHdtvkvaeVgf4kKVt1zyAwNPdFe-p9T7tvPyA,29257
57
57
  scrapely_client/http_compressors/__init__.py,sha256=2n7Ym-q5axJda90TjsdDwkniB_KQzPwjrCjqTwk7nW8,1104
58
58
  scrapely_client/http_compressors/_base.py,sha256=25W0zDO54qdSnhYLvYyF4EbYM2mh5gxwlaaQ5yKAuzw,672
59
59
  scrapely_client/http_compressors/_brotli.py,sha256=ItLG3DgHqH3dbyAyQUcRrTPaJgxwCk-C6OL7exGWb0s,1294
@@ -61,8 +61,8 @@ scrapely_client/http_compressors/_gzip.py,sha256=IJHZ87B5L1nrnXNFf1PjsYf-1MIIp9G
61
61
  scrapely_client/http_compressors/_resolve.py,sha256=AfIUzwX-ExczMgMQNAhfPd4CKAVKhIGq697bUm9i62k,1653
62
62
  scrapely_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
63
63
  scrapely_client/types.py,sha256=Wnb9-GBN6OcQGfiDk18TJjIzvHXb5MHrjJaIce-2UzY,2215
64
- scrapely_client_python-1.0.8.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
65
- scrapely_client_python-1.0.8.dist-info/licenses/NOTICE,sha256=0Seguz6wGdTod6Rld--Q4nHcmVqZF56Q_Atbh4YFEts,337
66
- scrapely_client_python-1.0.8.dist-info/WHEEL,sha256=1dD_GoEqEIxQ7VnrUExgdj_eExY4b4UgfR0iBlGKVac,80
67
- scrapely_client_python-1.0.8.dist-info/METADATA,sha256=Jw66r2HD-Za0sJCWMhWcyE6MMOC6khab84ypgwZdhHI,10554
68
- scrapely_client_python-1.0.8.dist-info/RECORD,,
64
+ scrapely_client_python-1.0.10.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
65
+ scrapely_client_python-1.0.10.dist-info/licenses/NOTICE,sha256=0Seguz6wGdTod6Rld--Q4nHcmVqZF56Q_Atbh4YFEts,337
66
+ scrapely_client_python-1.0.10.dist-info/WHEEL,sha256=MWx7psoF7KNST9G72pGtiBYuaXRXQXBk9pw5R6FzYko,80
67
+ scrapely_client_python-1.0.10.dist-info/METADATA,sha256=biT4HxozDqRCqbDk_FNauqD-hZseGMPPArhWhvyIUpM,12419
68
+ scrapely_client_python-1.0.10.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: uv 0.13.4
2
+ Generator: uv 0.13.7
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any