scrapely-client-python 1.0.7__py3-none-any.whl → 1.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapely_client/_consts.py +3 -0
- scrapely_client/_models.py +40 -19
- scrapely_client/_resource_clients/_resource_client.py +3 -2
- scrapely_client/_resource_clients/automation.py +3 -4
- scrapely_client/_resource_clients/request_queue.py +40 -11
- scrapely_client/_scrapely_client.py +25 -14
- {scrapely_client_python-1.0.7.dist-info → scrapely_client_python-1.0.9.dist-info}/METADATA +18 -1
- {scrapely_client_python-1.0.7.dist-info → scrapely_client_python-1.0.9.dist-info}/RECORD +11 -11
- {scrapely_client_python-1.0.7.dist-info → scrapely_client_python-1.0.9.dist-info}/WHEEL +1 -1
- {scrapely_client_python-1.0.7.dist-info → scrapely_client_python-1.0.9.dist-info}/licenses/LICENSE +0 -0
- {scrapely_client_python-1.0.7.dist-info → scrapely_client_python-1.0.9.dist-info}/licenses/NOTICE +0 -0
scrapely_client/_consts.py
CHANGED
|
@@ -7,6 +7,9 @@ from datetime import timedelta
|
|
|
7
7
|
DEFAULT_API_URL = 'https://api.scrape.ly'
|
|
8
8
|
"""Default base URL for the Scrapely API."""
|
|
9
9
|
|
|
10
|
+
API_URL_ENV_VAR = 'SCRAPELY_API_URL'
|
|
11
|
+
"""Environment variable that sets the API URL when `api_url` is not passed to the client."""
|
|
12
|
+
|
|
10
13
|
API_VERSION = 'v2'
|
|
11
14
|
"""Current Scrapely API version."""
|
|
12
15
|
|
scrapely_client/_models.py
CHANGED
|
@@ -51,9 +51,7 @@ class ActVersion(BaseModel):
|
|
|
51
51
|
version_number: Annotated[
|
|
52
52
|
str | None, Field(examples=['0.0'], pattern='^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[0-9]+)?$')
|
|
53
53
|
] = None
|
|
54
|
-
git_repo_url: Annotated[
|
|
55
|
-
str | None, Field(examples=['https://github.com/example/my-actor.git#main:src'])
|
|
56
|
-
] = None
|
|
54
|
+
git_repo_url: Annotated[str | None, Field(examples=['https://github.com/example/my-actor.git#main:src'])] = None
|
|
57
55
|
"""
|
|
58
56
|
URL of the git repository, present when sourceType is GIT_REPO.
|
|
59
57
|
"""
|
|
@@ -151,9 +149,7 @@ class Actor(BaseModel):
|
|
|
151
149
|
"""
|
|
152
150
|
Description of the Actor to display by search engines such as Google.
|
|
153
151
|
"""
|
|
154
|
-
picture_url: Annotated[
|
|
155
|
-
str | None, Field(examples=['https://images.scrape.ly/.../actor-picture.png'])
|
|
156
|
-
] = None
|
|
152
|
+
picture_url: Annotated[str | None, Field(examples=['https://images.scrape.ly/.../actor-picture.png'])] = None
|
|
157
153
|
"""
|
|
158
154
|
URL of the Actor's icon, displayed on the Actor's page in Scrapely Store and dashboard.
|
|
159
155
|
"""
|
|
@@ -224,7 +220,7 @@ class ActorChargeEvent(BaseModel):
|
|
|
224
220
|
|
|
225
221
|
@docs_group('Models')
|
|
226
222
|
class ActorDefinition(BaseModel):
|
|
227
|
-
"""The definition of the Actor,
|
|
223
|
+
"""The definition of the Actor, as declared in its Actor configuration file (for example `.actor/actor.json`)."""
|
|
228
224
|
|
|
229
225
|
model_config = ConfigDict(
|
|
230
226
|
extra='allow',
|
|
@@ -265,7 +261,7 @@ class ActorDefinition(BaseModel):
|
|
|
265
261
|
"""
|
|
266
262
|
input: dict[str, Any] | None = None
|
|
267
263
|
"""
|
|
268
|
-
The input schema object
|
|
264
|
+
The input schema object.
|
|
269
265
|
"""
|
|
270
266
|
changelog: str | None = None
|
|
271
267
|
"""
|
|
@@ -988,7 +984,7 @@ class Dataset(BaseModel):
|
|
|
988
984
|
),
|
|
989
985
|
] = None
|
|
990
986
|
"""
|
|
991
|
-
Defines the schema of items in your dataset
|
|
987
|
+
Defines the schema of items in your dataset.
|
|
992
988
|
"""
|
|
993
989
|
console_url: Annotated[AnyUrl, Field(examples=['https://api.scrape.ly/storage/datasets/27TmTznX9YPeAYhkC'])]
|
|
994
990
|
items_public_url: Annotated[
|
|
@@ -1290,7 +1286,11 @@ class EncodeAndSignData(BaseModel):
|
|
|
1290
1286
|
|
|
1291
1287
|
@docs_group('Models')
|
|
1292
1288
|
class DecodeAndVerifyRequest(EncodeAndSignData):
|
|
1293
|
-
|
|
1289
|
+
model_config = ConfigDict(
|
|
1290
|
+
extra='allow',
|
|
1291
|
+
populate_by_name=True,
|
|
1292
|
+
alias_generator=to_camel,
|
|
1293
|
+
)
|
|
1294
1294
|
|
|
1295
1295
|
|
|
1296
1296
|
@docs_group('Models')
|
|
@@ -1978,8 +1978,10 @@ class MonthlyUsage(BaseModel):
|
|
|
1978
1978
|
alias_generator=to_camel,
|
|
1979
1979
|
)
|
|
1980
1980
|
usage_cycle: UsageCycle
|
|
1981
|
-
# Scrapely emits raw byte counts (int) for the `DATA_TRANSFER_*_BYTES` keys
|
|
1982
1981
|
monthly_service_usage: dict[str, UsageItem | int]
|
|
1982
|
+
"""
|
|
1983
|
+
Scrapely emits raw byte counts (integers) for the `DATA_TRANSFER_*_BYTES` keys.
|
|
1984
|
+
"""
|
|
1983
1985
|
daily_service_usages: list[DailyServiceUsages]
|
|
1984
1986
|
total_usage_credits_usd_before_volume_discount: Annotated[float, Field(examples=[0.786143673840067])]
|
|
1985
1987
|
total_usage_credits_usd_after_volume_discount: Annotated[float, Field(examples=[0.786143673840067])]
|
|
@@ -2133,9 +2135,14 @@ class ListOfStoreActors(PaginationResponse):
|
|
|
2133
2135
|
alias_generator=to_camel,
|
|
2134
2136
|
)
|
|
2135
2137
|
items: list[StoreListActor]
|
|
2136
|
-
# Scrapely's store envelope omits `desc` and `count`; tolerate their absence.
|
|
2137
2138
|
desc: Annotated[bool, Field(examples=[False])] = False
|
|
2139
|
+
"""
|
|
2140
|
+
Whether the results are sorted in descending order. The store listing may omit it.
|
|
2141
|
+
"""
|
|
2138
2142
|
count: Annotated[int, Field(examples=[1], ge=0)] = 0
|
|
2143
|
+
"""
|
|
2144
|
+
The number of items returned in this response. The store listing may omit it.
|
|
2145
|
+
"""
|
|
2139
2146
|
|
|
2140
2147
|
|
|
2141
2148
|
@docs_group('Models')
|
|
@@ -2285,7 +2292,9 @@ class Profile(BaseModel):
|
|
|
2285
2292
|
)
|
|
2286
2293
|
bio: Annotated[str | None, Field(examples=['I started web scraping in 1985 using Altair BASIC.'])] = None
|
|
2287
2294
|
name: Annotated[str | None, Field(examples=['Jane Doe'])] = None
|
|
2288
|
-
picture_url: Annotated[AnyUrl | None, Field(examples=['https://api.scrape.ly/img/anonymous_user_picture.png'])] =
|
|
2295
|
+
picture_url: Annotated[AnyUrl | None, Field(examples=['https://api.scrape.ly/img/anonymous_user_picture.png'])] = (
|
|
2296
|
+
None
|
|
2297
|
+
)
|
|
2289
2298
|
github_username: Annotated[str | None, Field(examples=['torvalds.'])] = None
|
|
2290
2299
|
website_url: Annotated[AnyUrl | None, Field(examples=['http://www.example.com'])] = None
|
|
2291
2300
|
twitter_username: Annotated[str | None, Field(examples=['@BillGates'])] = None
|
|
@@ -3414,7 +3423,7 @@ class Storages(BaseModel):
|
|
|
3414
3423
|
)
|
|
3415
3424
|
dataset: dict[str, Any] | None = None
|
|
3416
3425
|
"""
|
|
3417
|
-
Defines the schema of items in your dataset
|
|
3426
|
+
Defines the schema of items in your dataset.
|
|
3418
3427
|
"""
|
|
3419
3428
|
|
|
3420
3429
|
|
|
@@ -3426,8 +3435,10 @@ class StoreListActor(BaseModel):
|
|
|
3426
3435
|
alias_generator=to_camel,
|
|
3427
3436
|
)
|
|
3428
3437
|
id: Annotated[str, Field(examples=['zdc3Pyhyz3m8vjDeM'])]
|
|
3429
|
-
# Scrapely can return null titles; tolerate their absence.
|
|
3430
3438
|
title: Annotated[str | None, Field(examples=['My Public Actor'])] = None
|
|
3439
|
+
"""
|
|
3440
|
+
Scrapely can return a `null` title.
|
|
3441
|
+
"""
|
|
3431
3442
|
name: Annotated[str, Field(examples=['my-public-actor'])]
|
|
3432
3443
|
username: Annotated[str, Field(examples=['jane35'])]
|
|
3433
3444
|
user_full_name: Annotated[str | None, Field(examples=['Jane H. Doe'])] = None
|
|
@@ -3752,7 +3763,7 @@ class UpdateLimitsRequest(BaseModel):
|
|
|
3752
3763
|
"""
|
|
3753
3764
|
data_retention_days: Annotated[int | None, Field(examples=[90])] = None
|
|
3754
3765
|
"""
|
|
3755
|
-
How many days unnamed storages (datasets, key-value stores and request queues without a name) are kept: Free 2 days (48 hours), Starter 31, Scale 60, Business 90. Runs
|
|
3766
|
+
How many days unnamed storages (datasets, key-value stores and request queues without a name) are kept: Free 2 days (48 hours), Starter 31, Scale 60, Business 90. Runs and their logs are kept for the same time as their results; on the Free plan your latest 10 runs are kept with their results until you delete them. This value is set by your plan and cannot be changed through the API.
|
|
3756
3767
|
|
|
3757
3768
|
"""
|
|
3758
3769
|
|
|
@@ -3800,7 +3811,11 @@ class UpdateRunRequest(BaseModel):
|
|
|
3800
3811
|
|
|
3801
3812
|
@docs_group('Models')
|
|
3802
3813
|
class UpdateStoreRequest(UpdateDatasetRequest):
|
|
3803
|
-
|
|
3814
|
+
model_config = ConfigDict(
|
|
3815
|
+
extra='allow',
|
|
3816
|
+
populate_by_name=True,
|
|
3817
|
+
alias_generator=to_camel,
|
|
3818
|
+
)
|
|
3804
3819
|
|
|
3805
3820
|
|
|
3806
3821
|
@docs_group('Models')
|
|
@@ -3909,7 +3924,9 @@ class Version(BaseModel):
|
|
|
3909
3924
|
populate_by_name=True,
|
|
3910
3925
|
alias_generator=to_camel,
|
|
3911
3926
|
)
|
|
3912
|
-
version_number: Annotated[
|
|
3927
|
+
version_number: Annotated[
|
|
3928
|
+
str, Field(examples=['0.0'], pattern='^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[0-9]+)?$')
|
|
3929
|
+
]
|
|
3913
3930
|
"""
|
|
3914
3931
|
The version number of the Actor. `MAJOR.MINOR` or `MAJOR.MINOR.PATCH` (Scrapely staging uses triplets like `0.0.1`).
|
|
3915
3932
|
"""
|
|
@@ -4037,7 +4054,11 @@ class WebhookDispatch(BaseModel):
|
|
|
4037
4054
|
|
|
4038
4055
|
@docs_group('Models')
|
|
4039
4056
|
class WebhookDispatchResponse(TestWebhookResponse):
|
|
4040
|
-
|
|
4057
|
+
model_config = ConfigDict(
|
|
4058
|
+
extra='allow',
|
|
4059
|
+
populate_by_name=True,
|
|
4060
|
+
alias_generator=to_camel,
|
|
4061
|
+
)
|
|
4041
4062
|
|
|
4042
4063
|
|
|
4043
4064
|
@docs_group('Models')
|
|
@@ -13,7 +13,7 @@ from scrapely_client._consts import DEFAULT_WAIT_FOR_FINISH, DEFAULT_WAIT_WHEN_J
|
|
|
13
13
|
from scrapely_client._docs import docs_group
|
|
14
14
|
from scrapely_client._logging import WithLogDetailsClient
|
|
15
15
|
from scrapely_client._utils.errors import catch_not_found_for_resource_or_throw, catch_not_found_or_throw
|
|
16
|
-
from scrapely_client._utils.http import response_to_dict, to_safe_id
|
|
16
|
+
from scrapely_client._utils.http import response_to_dict, to_path_segment, to_safe_id
|
|
17
17
|
from scrapely_client._utils.time import to_seconds
|
|
18
18
|
from scrapely_client.errors import ApifyApiError
|
|
19
19
|
|
|
@@ -77,7 +77,8 @@ class ResourceClientBase(metaclass=WithLogDetailsClient):
|
|
|
77
77
|
"""Build the full resource URL from base URL, path, and optional ID."""
|
|
78
78
|
url = f'{self._base_url}/{self._resource_path}'
|
|
79
79
|
if self._resource_id is not None:
|
|
80
|
-
|
|
80
|
+
# One path segment whatever the id holds: `..`, `?` or `#` must not retarget the call.
|
|
81
|
+
url = f'{url}/{to_path_segment(to_safe_id(self._resource_id))}'
|
|
81
82
|
return url
|
|
82
83
|
|
|
83
84
|
@cached_property
|
|
@@ -3,12 +3,11 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import TYPE_CHECKING, Any
|
|
6
|
-
from urllib.parse import quote
|
|
7
6
|
|
|
8
7
|
from scrapely_client._docs import docs_group
|
|
9
8
|
from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
|
|
10
9
|
from scrapely_client._utils.errors import catch_not_found_or_throw
|
|
11
|
-
from scrapely_client._utils.http import response_to_dict
|
|
10
|
+
from scrapely_client._utils.http import response_to_dict, to_path_segment
|
|
12
11
|
from scrapely_client.errors import ApifyApiError
|
|
13
12
|
|
|
14
13
|
if TYPE_CHECKING:
|
|
@@ -134,7 +133,7 @@ class AutomationClient(ResourceClient):
|
|
|
134
133
|
"""
|
|
135
134
|
try:
|
|
136
135
|
response = self._http_client.call(
|
|
137
|
-
url=self._build_url(f'runs/{
|
|
136
|
+
url=self._build_url(f'runs/{to_path_segment(run_id)}'),
|
|
138
137
|
method='GET',
|
|
139
138
|
params=self._build_params(),
|
|
140
139
|
timeout=timeout,
|
|
@@ -290,7 +289,7 @@ class AutomationClientAsync(ResourceClientAsync):
|
|
|
290
289
|
"""
|
|
291
290
|
try:
|
|
292
291
|
response = await self._http_client.call(
|
|
293
|
-
url=self._build_url(f'runs/{
|
|
292
|
+
url=self._build_url(f'runs/{to_path_segment(run_id)}'),
|
|
294
293
|
method='GET',
|
|
295
294
|
params=self._build_params(),
|
|
296
295
|
timeout=timeout,
|
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import asyncio
|
|
6
6
|
import json
|
|
7
7
|
import math
|
|
8
|
+
import re
|
|
8
9
|
from collections.abc import Iterable
|
|
9
10
|
from queue import Queue
|
|
10
11
|
from typing import TYPE_CHECKING, Any, Literal
|
|
@@ -40,7 +41,7 @@ from scrapely_client._models import (
|
|
|
40
41
|
from scrapely_client._pagination import DEFAULT_CHUNK_SIZE, get_cursor_iterator, get_cursor_iterator_async
|
|
41
42
|
from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
|
|
42
43
|
from scrapely_client._utils.errors import catch_not_found_or_throw
|
|
43
|
-
from scrapely_client._utils.http import response_to_dict
|
|
44
|
+
from scrapely_client._utils.http import response_to_dict, to_path_segment
|
|
44
45
|
from scrapely_client._utils.time import to_seconds
|
|
45
46
|
from scrapely_client.errors import ApifyApiError
|
|
46
47
|
|
|
@@ -68,6 +69,32 @@ _MAX_PAYLOAD_SIZE_BYTES = 9 * 1024 * 1024
|
|
|
68
69
|
_SAFETY_BUFFER_PERCENT = 0.01 / 100
|
|
69
70
|
"""Safety margin (0.01%) deducted from the maximum payload size when splitting requests into batches."""
|
|
70
71
|
|
|
72
|
+
_REQUEST_ID_PATTERN = re.compile(r'[A-Za-z0-9_~-]{1,128}')
|
|
73
|
+
"""Request ids the API accepts: letters, digits, `-`, `_` and `~`, at most 128 characters."""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _request_path(request_id: str, suffix: str = '') -> str:
|
|
77
|
+
"""Build the path of one request, with the id percent-encoded as a single path segment.
|
|
78
|
+
|
|
79
|
+
Request ids can come from queue contents written by someone else. Unencoded, an id such as `..`,
|
|
80
|
+
`../../key-value-stores/<id>/records/INPUT` or `a?b` would point the authenticated call at another resource.
|
|
81
|
+
"""
|
|
82
|
+
if not isinstance(request_id, str):
|
|
83
|
+
raise TypeError(f'Request id must be a string, not {type(request_id).__name__}.')
|
|
84
|
+
return f'requests/{to_path_segment(request_id)}{suffix}'
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _stored_request_id(request: Request) -> str:
|
|
88
|
+
"""Return the id of a request passed to `update_request`, after checking it against the API's id grammar."""
|
|
89
|
+
request_id = request.id
|
|
90
|
+
if request_id is None:
|
|
91
|
+
raise ValueError('The request has no id. update_request needs the id of a request stored in the queue.')
|
|
92
|
+
if not isinstance(request_id, str) or not _REQUEST_ID_PATTERN.fullmatch(request_id):
|
|
93
|
+
raise ValueError(
|
|
94
|
+
f'Invalid request id {request_id!r}: use 1-128 letters, digits, "-", "_" or "~", as returned by the API.'
|
|
95
|
+
)
|
|
96
|
+
return request_id
|
|
97
|
+
|
|
71
98
|
|
|
72
99
|
def _serialize_requests(
|
|
73
100
|
requests: list[RequestDraft] | list[RequestDraftDict] | list[RequestDraftCamelDict],
|
|
@@ -270,7 +297,7 @@ class RequestQueueClient(ResourceClient):
|
|
|
270
297
|
"""
|
|
271
298
|
try:
|
|
272
299
|
response = self._http_client.call(
|
|
273
|
-
url=self._build_url(
|
|
300
|
+
url=self._build_url(_request_path(request_id)),
|
|
274
301
|
method='GET',
|
|
275
302
|
params=self._build_params(),
|
|
276
303
|
timeout=timeout,
|
|
@@ -303,11 +330,12 @@ class RequestQueueClient(ResourceClient):
|
|
|
303
330
|
"""
|
|
304
331
|
if not isinstance(request, Request):
|
|
305
332
|
request = Request.model_validate(request)
|
|
333
|
+
request_id = _stored_request_id(request)
|
|
306
334
|
|
|
307
335
|
request_params = self._build_params(forefront=forefront, clientKey=self.client_key)
|
|
308
336
|
|
|
309
337
|
response = self._http_client.call(
|
|
310
|
-
url=self._build_url(
|
|
338
|
+
url=self._build_url(_request_path(request_id)),
|
|
311
339
|
method='PUT',
|
|
312
340
|
json=request.model_dump(by_alias=True, exclude_none=True),
|
|
313
341
|
params=request_params,
|
|
@@ -330,7 +358,7 @@ class RequestQueueClient(ResourceClient):
|
|
|
330
358
|
)
|
|
331
359
|
|
|
332
360
|
self._http_client.call(
|
|
333
|
-
url=self._build_url(
|
|
361
|
+
url=self._build_url(_request_path(request_id)),
|
|
334
362
|
method='DELETE',
|
|
335
363
|
params=request_params,
|
|
336
364
|
timeout=timeout,
|
|
@@ -360,7 +388,7 @@ class RequestQueueClient(ResourceClient):
|
|
|
360
388
|
)
|
|
361
389
|
|
|
362
390
|
response = self._http_client.call(
|
|
363
|
-
url=self._build_url(
|
|
391
|
+
url=self._build_url(_request_path(request_id, '/lock')),
|
|
364
392
|
method='PUT',
|
|
365
393
|
params=request_params,
|
|
366
394
|
timeout=timeout,
|
|
@@ -387,7 +415,7 @@ class RequestQueueClient(ResourceClient):
|
|
|
387
415
|
request_params = self._build_params(clientKey=self.client_key, forefront=forefront)
|
|
388
416
|
|
|
389
417
|
self._http_client.call(
|
|
390
|
-
url=self._build_url(
|
|
418
|
+
url=self._build_url(_request_path(request_id, '/lock')),
|
|
391
419
|
method='DELETE',
|
|
392
420
|
params=request_params,
|
|
393
421
|
timeout=timeout,
|
|
@@ -781,7 +809,7 @@ class RequestQueueClientAsync(ResourceClientAsync):
|
|
|
781
809
|
"""
|
|
782
810
|
try:
|
|
783
811
|
response = await self._http_client.call(
|
|
784
|
-
url=self._build_url(
|
|
812
|
+
url=self._build_url(_request_path(request_id)),
|
|
785
813
|
method='GET',
|
|
786
814
|
params=self._build_params(),
|
|
787
815
|
timeout=timeout,
|
|
@@ -812,11 +840,12 @@ class RequestQueueClientAsync(ResourceClientAsync):
|
|
|
812
840
|
"""
|
|
813
841
|
if not isinstance(request, Request):
|
|
814
842
|
request = Request.model_validate(request)
|
|
843
|
+
request_id = _stored_request_id(request)
|
|
815
844
|
|
|
816
845
|
request_params = self._build_params(forefront=forefront, clientKey=self.client_key)
|
|
817
846
|
|
|
818
847
|
response = await self._http_client.call(
|
|
819
|
-
url=self._build_url(
|
|
848
|
+
url=self._build_url(_request_path(request_id)),
|
|
820
849
|
method='PUT',
|
|
821
850
|
json=request.model_dump(by_alias=True, exclude_none=True),
|
|
822
851
|
params=request_params,
|
|
@@ -837,7 +866,7 @@ class RequestQueueClientAsync(ResourceClientAsync):
|
|
|
837
866
|
request_params = self._build_params(clientKey=self.client_key)
|
|
838
867
|
|
|
839
868
|
await self._http_client.call(
|
|
840
|
-
url=self._build_url(
|
|
869
|
+
url=self._build_url(_request_path(request_id)),
|
|
841
870
|
method='DELETE',
|
|
842
871
|
params=request_params,
|
|
843
872
|
timeout=timeout,
|
|
@@ -867,7 +896,7 @@ class RequestQueueClientAsync(ResourceClientAsync):
|
|
|
867
896
|
)
|
|
868
897
|
|
|
869
898
|
response = await self._http_client.call(
|
|
870
|
-
url=self._build_url(
|
|
899
|
+
url=self._build_url(_request_path(request_id, '/lock')),
|
|
871
900
|
method='PUT',
|
|
872
901
|
params=request_params,
|
|
873
902
|
timeout=timeout,
|
|
@@ -894,7 +923,7 @@ class RequestQueueClientAsync(ResourceClientAsync):
|
|
|
894
923
|
request_params = self._build_params(clientKey=self.client_key, forefront=forefront)
|
|
895
924
|
|
|
896
925
|
await self._http_client.call(
|
|
897
|
-
url=self._build_url(
|
|
926
|
+
url=self._build_url(_request_path(request_id, '/lock')),
|
|
898
927
|
method='DELETE',
|
|
899
928
|
params=request_params,
|
|
900
929
|
timeout=timeout,
|
|
@@ -2,12 +2,14 @@
|
|
|
2
2
|
# Modified by TeraShift GmbH. See the LICENSE and NOTICE files.
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import os
|
|
5
6
|
from functools import cached_property
|
|
6
7
|
from typing import TYPE_CHECKING
|
|
7
8
|
|
|
8
9
|
from scrapely_client._charging import ChargingHooks, wrap_dataset_client_with_charging
|
|
9
10
|
from scrapely_client._client_registry import ClientRegistry, ClientRegistryAsync
|
|
10
11
|
from scrapely_client._consts import (
|
|
12
|
+
API_URL_ENV_VAR,
|
|
11
13
|
API_VERSION,
|
|
12
14
|
DEFAULT_API_URL,
|
|
13
15
|
DEFAULT_MAX_RETRIES,
|
|
@@ -90,6 +92,13 @@ if TYPE_CHECKING:
|
|
|
90
92
|
from scrapely_client.types import HttpCompressionAlgorithm
|
|
91
93
|
|
|
92
94
|
|
|
95
|
+
def _resolve_api_url(api_url: str | None) -> str:
|
|
96
|
+
"""Return `api_url` if given, else the `SCRAPELY_API_URL` environment variable, else the production API."""
|
|
97
|
+
if api_url is not None:
|
|
98
|
+
return api_url
|
|
99
|
+
return os.environ.get(API_URL_ENV_VAR, '').strip() or DEFAULT_API_URL
|
|
100
|
+
|
|
101
|
+
|
|
93
102
|
@docs_group('Scrapely API clients')
|
|
94
103
|
class ApifyClient:
|
|
95
104
|
"""Synchronous client for the Scrapely API.
|
|
@@ -124,7 +133,7 @@ class ApifyClient:
|
|
|
124
133
|
self,
|
|
125
134
|
token: str | None = None,
|
|
126
135
|
*,
|
|
127
|
-
api_url: str =
|
|
136
|
+
api_url: str | None = None,
|
|
128
137
|
api_public_url: str | None = DEFAULT_API_URL,
|
|
129
138
|
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
130
139
|
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
|
|
@@ -144,9 +153,9 @@ class ApifyClient:
|
|
|
144
153
|
Args:
|
|
145
154
|
token: The Scrapely API token. You can find your token on the
|
|
146
155
|
Integrations page in the Scrapely dashboard.
|
|
147
|
-
api_url: The URL of the Scrapely API server to connect to.
|
|
148
|
-
|
|
149
|
-
as well.
|
|
156
|
+
api_url: The URL of the Scrapely API server to connect to. When not given, the `SCRAPELY_API_URL`
|
|
157
|
+
environment variable is used, then https://api.scrape.ly. It can be an internal URL that is not
|
|
158
|
+
globally accessible, in which case `api_public_url` should be set as well.
|
|
150
159
|
api_public_url: The globally accessible URL of the Scrapely API server. Should be set only if `api_url`
|
|
151
160
|
is an internal URL that is not globally accessible. Defaults to https://api.scrape.ly.
|
|
152
161
|
max_retries: How many times to retry a failed request at most.
|
|
@@ -165,7 +174,7 @@ class ApifyClient:
|
|
|
165
174
|
default_dataset_id: ID of the actor run's default dataset, used to gate charging interception.
|
|
166
175
|
"""
|
|
167
176
|
# We need to do this because of mocking in tests and default mutable arguments.
|
|
168
|
-
api_url =
|
|
177
|
+
api_url = _resolve_api_url(api_url)
|
|
169
178
|
api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
|
|
170
179
|
|
|
171
180
|
if headers:
|
|
@@ -234,7 +243,7 @@ class ApifyClient:
|
|
|
234
243
|
cls,
|
|
235
244
|
token: str | None = None,
|
|
236
245
|
*,
|
|
237
|
-
api_url: str =
|
|
246
|
+
api_url: str | None = None,
|
|
238
247
|
api_public_url: str | None = DEFAULT_API_URL,
|
|
239
248
|
http_client: HttpClient,
|
|
240
249
|
) -> ApifyClient:
|
|
@@ -263,7 +272,8 @@ class ApifyClient:
|
|
|
263
272
|
Args:
|
|
264
273
|
token: The Scrapely API token. It is set as the `Authorization` header on the custom client,
|
|
265
274
|
unless the client already has one configured.
|
|
266
|
-
api_url: The URL of the Scrapely API server to connect to.
|
|
275
|
+
api_url: The URL of the Scrapely API server to connect to. When not given, the `SCRAPELY_API_URL`
|
|
276
|
+
environment variable is used, then https://api.scrape.ly.
|
|
267
277
|
api_public_url: The globally accessible URL of the Scrapely API server. Defaults to https://api.scrape.ly.
|
|
268
278
|
http_client: A custom HTTP client instance extending `HttpClient`.
|
|
269
279
|
"""
|
|
@@ -513,7 +523,7 @@ class ApifyClientAsync:
|
|
|
513
523
|
self,
|
|
514
524
|
token: str | None = None,
|
|
515
525
|
*,
|
|
516
|
-
api_url: str =
|
|
526
|
+
api_url: str | None = None,
|
|
517
527
|
api_public_url: str | None = DEFAULT_API_URL,
|
|
518
528
|
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
519
529
|
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
|
|
@@ -533,9 +543,9 @@ class ApifyClientAsync:
|
|
|
533
543
|
Args:
|
|
534
544
|
token: The Scrapely API token. You can find your token on the
|
|
535
545
|
Integrations page in the Scrapely dashboard.
|
|
536
|
-
api_url: The URL of the Scrapely API server to connect to.
|
|
537
|
-
|
|
538
|
-
as well.
|
|
546
|
+
api_url: The URL of the Scrapely API server to connect to. When not given, the `SCRAPELY_API_URL`
|
|
547
|
+
environment variable is used, then https://api.scrape.ly. It can be an internal URL that is not
|
|
548
|
+
globally accessible, in which case `api_public_url` should be set as well.
|
|
539
549
|
api_public_url: The globally accessible URL of the Scrapely API server. Should be set only if `api_url`
|
|
540
550
|
is an internal URL that is not globally accessible. Defaults to https://api.scrape.ly.
|
|
541
551
|
max_retries: How many times to retry a failed request at most.
|
|
@@ -554,7 +564,7 @@ class ApifyClientAsync:
|
|
|
554
564
|
default_dataset_id: ID of the actor run's default dataset, used to gate charging interception.
|
|
555
565
|
"""
|
|
556
566
|
# We need to do this because of mocking in tests and default mutable arguments.
|
|
557
|
-
api_url =
|
|
567
|
+
api_url = _resolve_api_url(api_url)
|
|
558
568
|
api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
|
|
559
569
|
|
|
560
570
|
if headers:
|
|
@@ -623,7 +633,7 @@ class ApifyClientAsync:
|
|
|
623
633
|
cls,
|
|
624
634
|
token: str | None = None,
|
|
625
635
|
*,
|
|
626
|
-
api_url: str =
|
|
636
|
+
api_url: str | None = None,
|
|
627
637
|
api_public_url: str | None = DEFAULT_API_URL,
|
|
628
638
|
http_client: HttpClientAsync,
|
|
629
639
|
) -> ApifyClientAsync:
|
|
@@ -652,7 +662,8 @@ class ApifyClientAsync:
|
|
|
652
662
|
Args:
|
|
653
663
|
token: The Scrapely API token. It is set as the `Authorization` header on the custom client,
|
|
654
664
|
unless the client already has one configured.
|
|
655
|
-
api_url: The URL of the Scrapely API server to connect to.
|
|
665
|
+
api_url: The URL of the Scrapely API server to connect to. When not given, the `SCRAPELY_API_URL`
|
|
666
|
+
environment variable is used, then https://api.scrape.ly.
|
|
656
667
|
api_public_url: The globally accessible URL of the Scrapely API server. Defaults to https://api.scrape.ly.
|
|
657
668
|
http_client: A custom HTTP client instance extending `HttpClientAsync`.
|
|
658
669
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapely-client-python
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.9
|
|
4
4
|
Summary: Scrapely API client for Python
|
|
5
5
|
Keywords: scrapely,api,client,automation,crawling,scraping
|
|
6
6
|
Author: TeraShift GmbH (Scrape.ly)
|
|
@@ -94,6 +94,8 @@ Description-Content-Type: text/markdown
|
|
|
94
94
|
|
|
95
95
|
You'll need a Scrapely API token — find yours in the integrations section of the Scrapely dashboard. Pass it to the client and you're ready to go.
|
|
96
96
|
|
|
97
|
+
The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
|
|
98
|
+
|
|
97
99
|
### Synchronous client
|
|
98
100
|
|
|
99
101
|
```python
|
|
@@ -190,6 +192,21 @@ for item in client.dataset('DATASET-ID').iterate_items(fields=['title', 'url']):
|
|
|
190
192
|
process(item)
|
|
191
193
|
```
|
|
192
194
|
|
|
195
|
+
### Create and run an automation
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
automation = client.automations().create({
|
|
199
|
+
'title': 'Daily dentists',
|
|
200
|
+
'trigger': {'kind': 'schedule', 'cron': '0 9 * * *', 'timezone': 'Europe/Berlin'},
|
|
201
|
+
'steps': [{'kind': 'savedInput', 'savedInputId': 'SAVED-INPUT-ID'}],
|
|
202
|
+
'ai': {'mode': 'summarize', 'prompt': 'List the new practices with their phone numbers.'},
|
|
203
|
+
'deliveries': [{'channel': 'email'}],
|
|
204
|
+
})
|
|
205
|
+
run = client.automation(automation['id']).run()
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Automations take and return the API's camelCase dictionaries (see the automations section of the API reference).
|
|
209
|
+
|
|
193
210
|
### Tune retries and timeouts
|
|
194
211
|
|
|
195
212
|
```python
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
scrapely_client/__init__.py,sha256=Fu2-Un-vv1wGjqG3QyxDkbxMOmg6kfwdDBQ0yPGFm2M,879
|
|
2
2
|
scrapely_client/_charging.py,sha256=L7vvpX9bj0UwHc-83_zFpfcGyrMlbzoB_EpE9iSH2zM,6807
|
|
3
3
|
scrapely_client/_client_registry.py,sha256=FXqW7wYdHaFUjrtugXkh9IORzazzxGQssHOgtudj8Jc,5588
|
|
4
|
-
scrapely_client/_consts.py,sha256=
|
|
4
|
+
scrapely_client/_consts.py,sha256=zfGGi17JdkNR23W8hQinwbm81wXg_H_Pw1C_mpMMhWc,3668
|
|
5
5
|
scrapely_client/_docs.py,sha256=MtrntqA7S0BDKzJlZzBAfqeI2bUtybwcb85JfP9vaDA,1117
|
|
6
6
|
scrapely_client/_literals.py,sha256=qSc0qIjhcI7qa42fNQ-IGPm_7cpCI9_OaQ6oYGIOdy4,17006
|
|
7
7
|
scrapely_client/_logging.py,sha256=7yZ2GE8zv9M5r5IdmPB_jiAPy-fJN_kYSWQ2UwXu6p8,6780
|
|
8
|
-
scrapely_client/_models.py,sha256
|
|
8
|
+
scrapely_client/_models.py,sha256=fQ7WQQpXp5f-8_RiXG7Un2JVDBdO4_rQFUIP5-DewpM,142909
|
|
9
9
|
scrapely_client/_pagination.py,sha256=by-2FLNUQquXfxgr2mJVCgOqBBZRYMJpmTjfy4aWfpE,8579
|
|
10
10
|
scrapely_client/_resource_clients/__init__.py,sha256=cy_gBIqs8m7UpnKToffNad2zX2Su8We5B_q5IDM879M,4016
|
|
11
|
-
scrapely_client/_resource_clients/_resource_client.py,sha256=
|
|
11
|
+
scrapely_client/_resource_clients/_resource_client.py,sha256=b22QuV7osKBor62RsFWMOMKoyevUAY9PY2JrirxrWA8,21732
|
|
12
12
|
scrapely_client/_resource_clients/actor.py,sha256=9aJipoTI8BBYg1NZcHKuDQASatHe3XsUqKwtmNGSHkI,46153
|
|
13
13
|
scrapely_client/_resource_clients/actor_collection.py,sha256=LynOa8R-sTogMVN6jVl2mDW4xmw4a1IZ35Hv8gHX71c,16966
|
|
14
14
|
scrapely_client/_resource_clients/actor_env_var.py,sha256=dWKE_5xqhjQwCiNPNRXiyCw6g0GcUq1DxUR6Nd1JrRs,4810
|
|
15
15
|
scrapely_client/_resource_clients/actor_env_var_collection.py,sha256=1Ueh_QYHbIS1Ztl7X9ZpIMglFND1U8Nmw08u0Z4ERrs,5675
|
|
16
16
|
scrapely_client/_resource_clients/actor_version.py,sha256=Qo8ztYWuKg6nPBKiOiCXuIBbaiTTrH5DfLEDZjQj-mE,10232
|
|
17
17
|
scrapely_client/_resource_clients/actor_version_collection.py,sha256=GtiB_UV_D7E3bpBlLhiudIRZ31lN78GoYpzTn4HjOo8,9712
|
|
18
|
-
scrapely_client/_resource_clients/automation.py,sha256=
|
|
18
|
+
scrapely_client/_resource_clients/automation.py,sha256=5dzATC-ZNsbfYUgvBxPadDtIDVNOuDc5yaI1AIRdPP0,11301
|
|
19
19
|
scrapely_client/_resource_clients/automation_collection.py,sha256=0AOWLE3wuAlD1oWywEAfXBsnClLHs_IQ3l4Hi28ibac,5592
|
|
20
20
|
scrapely_client/_resource_clients/build.py,sha256=Bp2OTcil31OunorPKnw6CIc9K0ZlnYLStaNENtPkq1Y,7805
|
|
21
21
|
scrapely_client/_resource_clients/build_collection.py,sha256=TAuG4H46n6HIIntjg4tWiKqiDOgIUao5LX95xWjExwc,5766
|
|
@@ -24,7 +24,7 @@ scrapely_client/_resource_clients/dataset_collection.py,sha256=3IGH7B12agw1kHPhz
|
|
|
24
24
|
scrapely_client/_resource_clients/key_value_store.py,sha256=Fthku9RWqCGjBwvuvdUiiT6YpvlOLNQDYsu4VMMEmSU,32673
|
|
25
25
|
scrapely_client/_resource_clients/key_value_store_collection.py,sha256=D9XPMI7DVdAB1rPK-VKbeY1_JNxSajyPxJIhTTcCT0g,9279
|
|
26
26
|
scrapely_client/_resource_clients/log.py,sha256=XBuJRpUvBKfVhFypMFE0Wowpv5VOg2fs57PDkJEVaEQ,7491
|
|
27
|
-
scrapely_client/_resource_clients/request_queue.py,sha256=
|
|
27
|
+
scrapely_client/_resource_clients/request_queue.py,sha256=ajPrm2uAjjMiAUlsie4D6o9rLb6J1H9fhOdKxXiLJCQ,42518
|
|
28
28
|
scrapely_client/_resource_clients/request_queue_collection.py,sha256=wAX3bODdo5ozDXkeZ-HHUmTJdPxAj-xr3H6PKBNjkPE,8983
|
|
29
29
|
scrapely_client/_resource_clients/run.py,sha256=Xu-Xldw4wqDeg8pGAn1VIEOyQahaA9spZUykCdsZcW8,32219
|
|
30
30
|
scrapely_client/_resource_clients/run_collection.py,sha256=tU-UX1W_mYD7tRfL9S2YHU4oxZXUSZ6Z9mNwuX-sc64,8441
|
|
@@ -38,7 +38,7 @@ scrapely_client/_resource_clients/webhook.py,sha256=bZNla1cNM508EQL8zPe-7vgvfsCf
|
|
|
38
38
|
scrapely_client/_resource_clients/webhook_collection.py,sha256=-1LYc4upqSonEJdLVEldMvaV3Wi38EaL-Lzd9cmlAbQ,11351
|
|
39
39
|
scrapely_client/_resource_clients/webhook_dispatch.py,sha256=ylrPUHHH-0UlF6htefV2tceHGB3w3WdrZpjKLD1oyec,2660
|
|
40
40
|
scrapely_client/_resource_clients/webhook_dispatch_collection.py,sha256=RZXnczm6UKxnqDWmDYBvwyusp1_h1ONJMB7uCsBHg6E,5949
|
|
41
|
-
scrapely_client/_scrapely_client.py,sha256=
|
|
41
|
+
scrapely_client/_scrapely_client.py,sha256=1h03mj75oLpQGw_gIZeZrNk7_8PwQTTxzAPaQr4RxWo,38566
|
|
42
42
|
scrapely_client/_statistics.py,sha256=mTsUFVByduYFPoHocZULv53k5dYg77Bl03wamQD6Jyc,903
|
|
43
43
|
scrapely_client/_status_message_watcher.py,sha256=9Tstosbk_bUoq5K9xkhRQ5PBqsFaJFtvsGsAU0OvLjY,7893
|
|
44
44
|
scrapely_client/_streamed_log.py,sha256=OaxB5-ghZX1DYuZmjNCgVeHAH0fYPReqjQlWaa8-dVA,11659
|
|
@@ -61,8 +61,8 @@ scrapely_client/http_compressors/_gzip.py,sha256=IJHZ87B5L1nrnXNFf1PjsYf-1MIIp9G
|
|
|
61
61
|
scrapely_client/http_compressors/_resolve.py,sha256=AfIUzwX-ExczMgMQNAhfPd4CKAVKhIGq697bUm9i62k,1653
|
|
62
62
|
scrapely_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
63
63
|
scrapely_client/types.py,sha256=Wnb9-GBN6OcQGfiDk18TJjIzvHXb5MHrjJaIce-2UzY,2215
|
|
64
|
-
scrapely_client_python-1.0.
|
|
65
|
-
scrapely_client_python-1.0.
|
|
66
|
-
scrapely_client_python-1.0.
|
|
67
|
-
scrapely_client_python-1.0.
|
|
68
|
-
scrapely_client_python-1.0.
|
|
64
|
+
scrapely_client_python-1.0.9.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
|
|
65
|
+
scrapely_client_python-1.0.9.dist-info/licenses/NOTICE,sha256=0Seguz6wGdTod6Rld--Q4nHcmVqZF56Q_Atbh4YFEts,337
|
|
66
|
+
scrapely_client_python-1.0.9.dist-info/WHEEL,sha256=si2p8vW03RnhyBnhWl4tj-ObJa78SpE-c9HsEtblGR0,80
|
|
67
|
+
scrapely_client_python-1.0.9.dist-info/METADATA,sha256=RHU05F0ObBU5UuLh3jNzgKAE5_NY8bENCfejeNuW0fI,11529
|
|
68
|
+
scrapely_client_python-1.0.9.dist-info/RECORD,,
|
{scrapely_client_python-1.0.7.dist-info → scrapely_client_python-1.0.9.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{scrapely_client_python-1.0.7.dist-info → scrapely_client_python-1.0.9.dist-info}/licenses/NOTICE
RENAMED
|
File without changes
|