scrapely-client-python 1.0.2__py3-none-any.whl → 1.0.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapely_client/__init__.py +2 -0
- scrapely_client/_charging.py +3 -4
- scrapely_client/_client_registry.py +2 -0
- scrapely_client/_consts.py +3 -1
- scrapely_client/_docs.py +2 -0
- scrapely_client/_literals.py +2 -0
- scrapely_client/_logging.py +3 -1
- scrapely_client/_models.py +39 -43
- scrapely_client/_pagination.py +2 -0
- scrapely_client/_resource_clients/__init__.py +6 -0
- scrapely_client/_resource_clients/_resource_client.py +4 -2
- scrapely_client/_resource_clients/actor.py +4 -2
- scrapely_client/_resource_clients/actor_collection.py +4 -2
- scrapely_client/_resource_clients/actor_env_var.py +2 -0
- scrapely_client/_resource_clients/actor_env_var_collection.py +2 -0
- scrapely_client/_resource_clients/actor_version.py +2 -0
- scrapely_client/_resource_clients/actor_version_collection.py +2 -0
- scrapely_client/_resource_clients/automation.py +330 -0
- scrapely_client/_resource_clients/automation_collection.py +169 -0
- scrapely_client/_resource_clients/build.py +4 -2
- scrapely_client/_resource_clients/build_collection.py +4 -2
- scrapely_client/_resource_clients/dataset.py +4 -2
- scrapely_client/_resource_clients/dataset_collection.py +4 -2
- scrapely_client/_resource_clients/key_value_store.py +4 -2
- scrapely_client/_resource_clients/key_value_store_collection.py +4 -2
- scrapely_client/_resource_clients/log.py +4 -2
- scrapely_client/_resource_clients/request_queue.py +4 -2
- scrapely_client/_resource_clients/request_queue_collection.py +4 -2
- scrapely_client/_resource_clients/run.py +4 -2
- scrapely_client/_resource_clients/run_collection.py +4 -2
- scrapely_client/_resource_clients/schedule.py +4 -2
- scrapely_client/_resource_clients/schedule_collection.py +4 -2
- scrapely_client/_resource_clients/store_collection.py +4 -2
- scrapely_client/_resource_clients/task.py +4 -2
- scrapely_client/_resource_clients/task_collection.py +4 -2
- scrapely_client/_resource_clients/user.py +4 -2
- scrapely_client/_resource_clients/webhook.py +4 -2
- scrapely_client/_resource_clients/webhook_collection.py +4 -2
- scrapely_client/_resource_clients/webhook_dispatch.py +4 -2
- scrapely_client/_resource_clients/webhook_dispatch_collection.py +4 -2
- scrapely_client/_scrapely_client.py +56 -26
- scrapely_client/_status_message_watcher.py +2 -0
- scrapely_client/_streamed_log.py +2 -0
- scrapely_client/_typeddicts.py +2 -0
- scrapely_client/_utils/crypto.py +2 -0
- scrapely_client/_utils/encoding.py +2 -0
- scrapely_client/_utils/errors.py +3 -1
- scrapely_client/_utils/http.py +2 -0
- scrapely_client/_utils/try_import.py +2 -0
- scrapely_client/errors.py +5 -3
- scrapely_client/http_clients/__init__.py +2 -0
- scrapely_client/http_clients/_base.py +7 -5
- scrapely_client/http_clients/_impit.py +6 -4
- scrapely_client/http_compressors/__init__.py +2 -0
- scrapely_client/http_compressors/_brotli.py +3 -1
- scrapely_client/http_compressors/_gzip.py +2 -0
- scrapely_client/http_compressors/_resolve.py +2 -0
- scrapely_client/types.py +3 -1
- {scrapely_client_python-1.0.2.dist-info → scrapely_client_python-1.0.6.dist-info}/METADATA +10 -9
- scrapely_client_python-1.0.6.dist-info/RECORD +68 -0
- {scrapely_client_python-1.0.2.dist-info → scrapely_client_python-1.0.6.dist-info}/WHEEL +1 -1
- scrapely_client_python-1.0.6.dist-info/licenses/NOTICE +7 -0
- scrapely_client_python-1.0.2.dist-info/RECORD +0 -65
- {scrapely_client_python-1.0.2.dist-info → scrapely_client_python-1.0.6.dist-info}/licenses/LICENSE +0 -0
scrapely_client/__init__.py
CHANGED
scrapely_client/_charging.py
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
"""Opt-in pay-per-event charging interception for dataset pushes.
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Charging interception for dataset pushes, implemented in the client
|
|
4
4
|
itself, with the charging logic supplied by the host SDK via the
|
|
5
5
|
:class:`ChargingHooks` protocol (no SDK import — the SDK adapts its
|
|
6
6
|
``ChargingManager`` to these hooks later).
|
|
7
7
|
|
|
8
|
-
Without hooks (the default),
|
|
9
|
-
``apify-client``.
|
|
8
|
+
Without hooks (the default), dataset pushes are not intercepted.
|
|
10
9
|
"""
|
|
11
10
|
|
|
12
11
|
from __future__ import annotations
|
|
@@ -126,7 +125,7 @@ def _merge_context_charge_result(context: PpeAwarePushDataContext, hooks: Chargi
|
|
|
126
125
|
def wrap_dataset_client_with_charging(client: Any, hooks: ChargingHooks, *, is_async: bool) -> Any:
|
|
127
126
|
"""Wrap a dataset client's ``push_items`` with charging interception (method swap by design).
|
|
128
127
|
|
|
129
|
-
|
|
128
|
+
Normalize items →
|
|
130
129
|
``push_and_charge`` with a ``push_fn`` that pushes the accepted subset via the
|
|
131
130
|
original ``push_items``, and accumulate the charge result into the ambient
|
|
132
131
|
``push_data_charging_context`` when present.
|
scrapely_client/_consts.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
# Derived from apify-client-python (Copyright Apify Technologies s.r.o., Apache-2.0).
|
|
2
|
+
# Modified by TeraShift GmbH. See the LICENSE and NOTICE files.
|
|
1
3
|
from __future__ import annotations
|
|
2
4
|
|
|
3
5
|
from datetime import timedelta
|
|
4
6
|
|
|
5
|
-
DEFAULT_API_URL = 'https://api
|
|
7
|
+
DEFAULT_API_URL = 'https://api.scrape.ly'
|
|
6
8
|
"""Default base URL for the Scrapely API."""
|
|
7
9
|
|
|
8
10
|
API_VERSION = 'v2'
|
scrapely_client/_docs.py
CHANGED
scrapely_client/_literals.py
CHANGED
scrapely_client/_logging.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# Derived from apify-client-python (Copyright Apify Technologies s.r.o., Apache-2.0).
|
|
2
|
+
# Modified by TeraShift GmbH. See the LICENSE and NOTICE files.
|
|
1
3
|
from __future__ import annotations
|
|
2
4
|
|
|
3
5
|
import functools
|
|
@@ -108,7 +110,7 @@ def create_redirect_logger(name: str) -> logging.Logger:
|
|
|
108
110
|
"""Create a logger for redirecting logs from another Actor.
|
|
109
111
|
|
|
110
112
|
Args:
|
|
111
|
-
name: Logger name. Use dot notation for hierarchy (e.g., "
|
|
113
|
+
name: Logger name. Use dot notation for hierarchy (e.g., "scrapely.xyz" creates "xyz" under "scrapely").
|
|
112
114
|
|
|
113
115
|
Returns:
|
|
114
116
|
Configured logger with RedirectLogFormatter.
|
scrapely_client/_models.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# Derived from apify-client-python (Copyright Apify Technologies s.r.o., Apache-2.0).
|
|
2
|
+
# Modified by TeraShift GmbH. See the LICENSE and NOTICE files.
|
|
1
3
|
# generated by datamodel-codegen
|
|
2
4
|
|
|
3
5
|
from __future__ import annotations
|
|
@@ -86,7 +88,7 @@ class Actor(BaseModel):
|
|
|
86
88
|
"""
|
|
87
89
|
description: Annotated[str | None, Field(examples=['Extract data from hundreds of places fast.'])] = None
|
|
88
90
|
"""
|
|
89
|
-
Short description of the Actor, displayed in Scrapely Store and
|
|
91
|
+
Short description of the Actor, displayed in Scrapely Store and dashboard.
|
|
90
92
|
"""
|
|
91
93
|
restart_on_error: Annotated[bool | None, Field(deprecated=True, examples=[False])] = None
|
|
92
94
|
is_public: Annotated[bool, Field(examples=[False])]
|
|
@@ -131,7 +133,7 @@ class Actor(BaseModel):
|
|
|
131
133
|
"""
|
|
132
134
|
title: Annotated[str | None, Field(examples=['Google Search Extractor'])] = None
|
|
133
135
|
"""
|
|
134
|
-
Human-readable name of the Actor, displayed in Scrapely Store and
|
|
136
|
+
Human-readable name of the Actor, displayed in Scrapely Store and dashboard.
|
|
135
137
|
"""
|
|
136
138
|
tagged_builds: dict[str, TaggedBuildInfo | None] | None = None
|
|
137
139
|
actor_standby: ActorStandby | None = None
|
|
@@ -141,7 +143,7 @@ class Actor(BaseModel):
|
|
|
141
143
|
"""
|
|
142
144
|
seo_title: Annotated[str | None, Field(examples=['Web Scraper'])] = None
|
|
143
145
|
"""
|
|
144
|
-
Name of the Actor to display by search engines such as Google. Can be different from the Actor's name displayed in Scrapely Store and
|
|
146
|
+
Name of the Actor to display by search engines such as Google. Can be different from the Actor's name displayed in Scrapely Store and dashboard.
|
|
145
147
|
"""
|
|
146
148
|
seo_description: Annotated[
|
|
147
149
|
str | None, Field(examples=['Crawls websites using Chrome and extracts data from pages using JavaScript.'])
|
|
@@ -150,12 +152,12 @@ class Actor(BaseModel):
|
|
|
150
152
|
Description of the Actor to display by search engines such as Google.
|
|
151
153
|
"""
|
|
152
154
|
picture_url: Annotated[
|
|
153
|
-
str | None, Field(examples=['https://
|
|
155
|
+
str | None, Field(examples=['https://images.scrape.ly/.../actor-picture.png'])
|
|
154
156
|
] = None
|
|
155
157
|
"""
|
|
156
|
-
URL of the Actor's icon, displayed on the Actor's page in Scrapely Store and
|
|
158
|
+
URL of the Actor's icon, displayed on the Actor's page in Scrapely Store and dashboard.
|
|
157
159
|
"""
|
|
158
|
-
standby_url: Annotated[str | None, Field(examples=['https://jane35--my-actor.
|
|
160
|
+
standby_url: Annotated[str | None, Field(examples=['https://jane35--my-actor.scrape.ly'])] = None
|
|
159
161
|
"""
|
|
160
162
|
URL for sending requests to the Actor in Standby mode.
|
|
161
163
|
`null` if the Standby mode isn't enabled.
|
|
@@ -290,7 +292,7 @@ class ActorDefinition(BaseModel):
|
|
|
290
292
|
|
|
291
293
|
@docs_group('Models')
|
|
292
294
|
class ActorNotice(Enum):
|
|
293
|
-
"""A warning displayed on the Actor's page in Scrapely Store and
|
|
295
|
+
"""A warning displayed on the Actor's page in Scrapely Store and dashboard. Can be set by the Actor's developer or automatically by Scrapely's quality checks."""
|
|
294
296
|
|
|
295
297
|
NONE = 'NONE'
|
|
296
298
|
RESIDENTIAL_PROXY_REQUIRED = 'RESIDENTIAL_PROXY_REQUIRED'
|
|
@@ -452,7 +454,7 @@ class AddedRequest(BaseModel):
|
|
|
452
454
|
"""
|
|
453
455
|
A unique identifier assigned to the request.
|
|
454
456
|
"""
|
|
455
|
-
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
457
|
+
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])]
|
|
456
458
|
"""
|
|
457
459
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
458
460
|
"""
|
|
@@ -735,11 +737,6 @@ class CommonActorPricingInfo(BaseModel):
|
|
|
735
737
|
populate_by_name=True,
|
|
736
738
|
alias_generator=to_camel,
|
|
737
739
|
)
|
|
738
|
-
apify_margin_percentage: float | None = None
|
|
739
|
-
"""
|
|
740
|
-
In [0, 1], fraction of pricePerUnitUsd that goes to Scrapely.
|
|
741
|
-
Optional: Scrapely staging PAY_PER_EVENT payloads often omit this.
|
|
742
|
-
"""
|
|
743
740
|
created_at: AwareDatetime | None = None
|
|
744
741
|
"""
|
|
745
742
|
When this pricing info record has been created.
|
|
@@ -772,11 +769,11 @@ class CreateActorRequest(BaseModel):
|
|
|
772
769
|
None
|
|
773
770
|
)
|
|
774
771
|
"""
|
|
775
|
-
Short description of the Actor, displayed in Scrapely Store and
|
|
772
|
+
Short description of the Actor, displayed in Scrapely Store and dashboard.
|
|
776
773
|
"""
|
|
777
774
|
title: Annotated[str | None, Field(examples=['Instagram scraper'])] = None
|
|
778
775
|
"""
|
|
779
|
-
Human-readable name of the Actor, displayed in Scrapely Store and
|
|
776
|
+
Human-readable name of the Actor, displayed in Scrapely Store and dashboard. Can contain spaces and capital letters. Recommended length is 40-50 characters. You can change this title without affecting the Actor's URL or SEO.
|
|
780
777
|
"""
|
|
781
778
|
is_public: Annotated[bool | None, Field(examples=[False])] = None
|
|
782
779
|
"""
|
|
@@ -784,7 +781,7 @@ class CreateActorRequest(BaseModel):
|
|
|
784
781
|
"""
|
|
785
782
|
seo_title: Annotated[str | None, Field(examples=['Free Instagram scraper'])] = None
|
|
786
783
|
"""
|
|
787
|
-
Name of the Actor to display by search engines such as Google. Can be different from the Actor's name displayed in Scrapely Store and
|
|
784
|
+
Name of the Actor to display by search engines such as Google. Can be different from the Actor's name displayed in Scrapely Store and dashboard. Recommended length is 40-50 characters.
|
|
788
785
|
"""
|
|
789
786
|
seo_description: Annotated[str | None, Field(examples=['The best scraper for Instagram'])] = None
|
|
790
787
|
"""
|
|
@@ -918,7 +915,6 @@ class CurrentPricingInfo(BaseModel):
|
|
|
918
915
|
alias_generator=to_camel,
|
|
919
916
|
)
|
|
920
917
|
pricing_model: Annotated[str, Field(examples=['FREE'])]
|
|
921
|
-
apify_margin_percentage: Annotated[float | None, Field(examples=[0.2])] = None
|
|
922
918
|
created_at: Annotated[AwareDatetime | None, Field(examples=['2023-01-01T00:00:00.000Z'])] = None
|
|
923
919
|
started_at: Annotated[AwareDatetime | None, Field(examples=['2023-01-01T00:00:00.000Z'])] = None
|
|
924
920
|
notified_about_change_at: Annotated[AwareDatetime | None, Field(examples=[None])] = None
|
|
@@ -996,7 +992,7 @@ class Dataset(BaseModel):
|
|
|
996
992
|
"""
|
|
997
993
|
console_url: Annotated[AnyUrl, Field(examples=['https://api.scrape.ly/storage/datasets/27TmTznX9YPeAYhkC'])]
|
|
998
994
|
items_public_url: Annotated[
|
|
999
|
-
AnyUrl | None, Field(examples=['https://api
|
|
995
|
+
AnyUrl | None, Field(examples=['https://api.scrape.ly/v2/datasets/WkzbQMuFYuamGv3YF/items?signature=abc123'])
|
|
1000
996
|
] = None
|
|
1001
997
|
"""
|
|
1002
998
|
A public link to access the dataset items directly.
|
|
@@ -1213,7 +1209,7 @@ class DeletedRequestById(BaseModel):
|
|
|
1213
1209
|
populate_by_name=True,
|
|
1214
1210
|
alias_generator=to_camel,
|
|
1215
1211
|
)
|
|
1216
|
-
unique_key: Annotated[str | None, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
1212
|
+
unique_key: Annotated[str | None, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])] = None
|
|
1217
1213
|
"""
|
|
1218
1214
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
1219
1215
|
"""
|
|
@@ -1232,7 +1228,7 @@ class DeletedRequestByUniqueKey(BaseModel):
|
|
|
1232
1228
|
populate_by_name=True,
|
|
1233
1229
|
alias_generator=to_camel,
|
|
1234
1230
|
)
|
|
1235
|
-
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
1231
|
+
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])]
|
|
1236
1232
|
"""
|
|
1237
1233
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
1238
1234
|
"""
|
|
@@ -1470,11 +1466,11 @@ class HeadRequest(BaseModel):
|
|
|
1470
1466
|
"""
|
|
1471
1467
|
A unique identifier assigned to the request.
|
|
1472
1468
|
"""
|
|
1473
|
-
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
1469
|
+
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])]
|
|
1474
1470
|
"""
|
|
1475
1471
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
1476
1472
|
"""
|
|
1477
|
-
url: Annotated[str, Field(examples=['https://api
|
|
1473
|
+
url: Annotated[str, Field(examples=['https://api.scrape.ly'])]
|
|
1478
1474
|
"""
|
|
1479
1475
|
The URL of the request.
|
|
1480
1476
|
"""
|
|
@@ -1535,13 +1531,13 @@ class KeyValueStore(BaseModel):
|
|
|
1535
1531
|
] = None
|
|
1536
1532
|
keys_public_url: Annotated[
|
|
1537
1533
|
AnyUrl | None,
|
|
1538
|
-
Field(examples=['https://api
|
|
1534
|
+
Field(examples=['https://api.scrape.ly/v2/key-value-stores/WkzbQMuFYuamGv3YF/keys?signature=abc123']),
|
|
1539
1535
|
] = None
|
|
1540
1536
|
"""
|
|
1541
1537
|
A public link to access keys of the key-value store directly.
|
|
1542
1538
|
"""
|
|
1543
1539
|
records_public_url: Annotated[
|
|
1544
|
-
AnyUrl | None, Field(examples=['https://api
|
|
1540
|
+
AnyUrl | None, Field(examples=['https://api.scrape.ly/v2/key-value-stores/WkzbQMuFYuamGv3YF/records'])
|
|
1545
1541
|
] = None
|
|
1546
1542
|
"""
|
|
1547
1543
|
A public link to access records of the key-value store directly.
|
|
@@ -1570,7 +1566,7 @@ class KeyValueStoreKey(BaseModel):
|
|
|
1570
1566
|
record_public_url: Annotated[
|
|
1571
1567
|
AnyUrl,
|
|
1572
1568
|
Field(
|
|
1573
|
-
examples=['https://api
|
|
1569
|
+
examples=['https://api.scrape.ly/v2/key-value-stores/WkzbQMuFYuamGv3YF/records/some-key?signature=abc123']
|
|
1574
1570
|
),
|
|
1575
1571
|
]
|
|
1576
1572
|
"""
|
|
@@ -1889,11 +1885,11 @@ class LockedHeadRequest(BaseModel):
|
|
|
1889
1885
|
"""
|
|
1890
1886
|
A unique identifier assigned to the request.
|
|
1891
1887
|
"""
|
|
1892
|
-
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
1888
|
+
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])]
|
|
1893
1889
|
"""
|
|
1894
1890
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
1895
1891
|
"""
|
|
1896
|
-
url: Annotated[str, Field(examples=['https://api
|
|
1892
|
+
url: Annotated[str, Field(examples=['https://api.scrape.ly'])]
|
|
1897
1893
|
"""
|
|
1898
1894
|
The URL of the request.
|
|
1899
1895
|
"""
|
|
@@ -2210,7 +2206,7 @@ class Plan(BaseModel):
|
|
|
2210
2206
|
max_monthly_external_data_transfer_gbytes: Annotated[float | None, Field(examples=[1000])] = None
|
|
2211
2207
|
max_actor_count: Annotated[int | None, Field(examples=[100])] = None
|
|
2212
2208
|
max_actor_task_count: Annotated[int | None, Field(examples=[1000])] = None
|
|
2213
|
-
data_retention_days: Annotated[int | None, Field(examples=[
|
|
2209
|
+
data_retention_days: Annotated[int | None, Field(examples=[90])] = None
|
|
2214
2210
|
available_proxy_groups: dict[str, int]
|
|
2215
2211
|
team_account_seat_count: Annotated[int | None, Field(examples=[1])] = None
|
|
2216
2212
|
support_level: Annotated[str | None, Field(examples=['COMMUNITY'])] = None
|
|
@@ -2289,7 +2285,7 @@ class Profile(BaseModel):
|
|
|
2289
2285
|
)
|
|
2290
2286
|
bio: Annotated[str | None, Field(examples=['I started web scraping in 1985 using Altair BASIC.'])] = None
|
|
2291
2287
|
name: Annotated[str | None, Field(examples=['Jane Doe'])] = None
|
|
2292
|
-
picture_url: Annotated[AnyUrl | None, Field(examples=['https://api
|
|
2288
|
+
picture_url: Annotated[AnyUrl | None, Field(examples=['https://api.scrape.ly/img/anonymous_user_picture.png'])] = None
|
|
2293
2289
|
github_username: Annotated[str | None, Field(examples=['torvalds.'])] = None
|
|
2294
2290
|
website_url: Annotated[AnyUrl | None, Field(examples=['http://www.example.com'])] = None
|
|
2295
2291
|
twitter_username: Annotated[str | None, Field(examples=['@BillGates'])] = None
|
|
@@ -2433,11 +2429,11 @@ class RequestBase(BaseModel):
|
|
|
2433
2429
|
populate_by_name=True,
|
|
2434
2430
|
alias_generator=to_camel,
|
|
2435
2431
|
)
|
|
2436
|
-
unique_key: Annotated[str | None, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
2432
|
+
unique_key: Annotated[str | None, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])] = None
|
|
2437
2433
|
"""
|
|
2438
2434
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
2439
2435
|
"""
|
|
2440
|
-
url: Annotated[str | None, Field(examples=['https://api
|
|
2436
|
+
url: Annotated[str | None, Field(examples=['https://api.scrape.ly'])] = None
|
|
2441
2437
|
"""
|
|
2442
2438
|
The URL of the request.
|
|
2443
2439
|
"""
|
|
@@ -2446,7 +2442,7 @@ class RequestBase(BaseModel):
|
|
|
2446
2442
|
"""
|
|
2447
2443
|
The number of times this request has been retried.
|
|
2448
2444
|
"""
|
|
2449
|
-
loaded_url: Annotated[str | None, Field(examples=['https://api
|
|
2445
|
+
loaded_url: Annotated[str | None, Field(examples=['https://api.scrape.ly/jobs'])] = None
|
|
2450
2446
|
"""
|
|
2451
2447
|
The final URL that was loaded, after redirects (if any).
|
|
2452
2448
|
"""
|
|
@@ -2501,11 +2497,11 @@ class RequestDraft(BaseModel):
|
|
|
2501
2497
|
"""
|
|
2502
2498
|
A unique identifier assigned to the request.
|
|
2503
2499
|
"""
|
|
2504
|
-
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
2500
|
+
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])]
|
|
2505
2501
|
"""
|
|
2506
2502
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
2507
2503
|
"""
|
|
2508
|
-
url: Annotated[str, Field(examples=['https://api
|
|
2504
|
+
url: Annotated[str, Field(examples=['https://api.scrape.ly'])]
|
|
2509
2505
|
"""
|
|
2510
2506
|
The URL of the request.
|
|
2511
2507
|
"""
|
|
@@ -2525,7 +2521,7 @@ class RequestDraftDeleteById(BaseModel):
|
|
|
2525
2521
|
"""
|
|
2526
2522
|
A unique identifier assigned to the request.
|
|
2527
2523
|
"""
|
|
2528
|
-
unique_key: Annotated[str | None, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
2524
|
+
unique_key: Annotated[str | None, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])] = None
|
|
2529
2525
|
"""
|
|
2530
2526
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
2531
2527
|
"""
|
|
@@ -2544,7 +2540,7 @@ class RequestDraftDeleteByUniqueKey(BaseModel):
|
|
|
2544
2540
|
"""
|
|
2545
2541
|
A unique identifier assigned to the request.
|
|
2546
2542
|
"""
|
|
2547
|
-
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api
|
|
2543
|
+
unique_key: Annotated[str, Field(examples=['GET|60d83e70|e3b0c442|https://api.scrape.ly'])]
|
|
2548
2544
|
"""
|
|
2549
2545
|
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
|
|
2550
2546
|
"""
|
|
@@ -2630,9 +2626,9 @@ class RequestQueue(BaseModel):
|
|
|
2630
2626
|
"""
|
|
2631
2627
|
Whether the request queue has been accessed by multiple different clients.
|
|
2632
2628
|
"""
|
|
2633
|
-
console_url: Annotated[AnyUrl, Field(examples=['https://api
|
|
2629
|
+
console_url: Annotated[AnyUrl, Field(examples=['https://api.scrape.ly/v2/request-queues/27TmTznX9YPeAYhkC'])]
|
|
2634
2630
|
"""
|
|
2635
|
-
The URL to view the request queue in the Scrapely
|
|
2631
|
+
The URL to view the request queue in the Scrapely dashboard.
|
|
2636
2632
|
"""
|
|
2637
2633
|
stats: RequestQueueStats | None = None
|
|
2638
2634
|
general_access: GeneralAccess | None = None
|
|
@@ -2950,7 +2946,7 @@ class Run(BaseModel):
|
|
|
2950
2946
|
"""
|
|
2951
2947
|
Build number of the Actor build used for this run.
|
|
2952
2948
|
"""
|
|
2953
|
-
container_url: Annotated[AnyUrl | None, Field(examples=['https://g8kd8kbc5ge8.runs.
|
|
2949
|
+
container_url: Annotated[AnyUrl | None, Field(examples=['https://g8kd8kbc5ge8.runs.scrape.ly'])] = None
|
|
2954
2950
|
"""
|
|
2955
2951
|
URL of the container running the Actor.
|
|
2956
2952
|
"""
|
|
@@ -3667,7 +3663,7 @@ class UpdateActorRequest(BaseModel):
|
|
|
3667
3663
|
None
|
|
3668
3664
|
)
|
|
3669
3665
|
"""
|
|
3670
|
-
Short description of the Actor, displayed in Scrapely Store and
|
|
3666
|
+
Short description of the Actor, displayed in Scrapely Store and dashboard.
|
|
3671
3667
|
"""
|
|
3672
3668
|
is_public: Annotated[bool | None, Field(examples=[False])] = None
|
|
3673
3669
|
"""
|
|
@@ -3676,7 +3672,7 @@ class UpdateActorRequest(BaseModel):
|
|
|
3676
3672
|
actor_permission_level: ActorPermissionLevel | None = None
|
|
3677
3673
|
seo_title: Annotated[str | None, Field(examples=['Free Instagram scraper'])] = None
|
|
3678
3674
|
"""
|
|
3679
|
-
Name of the Actor to display by search engines such as Google. Can be different from the Actor's name displayed in Scrapely Store and
|
|
3675
|
+
Name of the Actor to display by search engines such as Google. Can be different from the Actor's name displayed in Scrapely Store and dashboard. Recommended length is 40-50 characters.
|
|
3680
3676
|
"""
|
|
3681
3677
|
seo_description: Annotated[str | None, Field(examples=['The best scraper for Instagram'])] = None
|
|
3682
3678
|
"""
|
|
@@ -3684,7 +3680,7 @@ class UpdateActorRequest(BaseModel):
|
|
|
3684
3680
|
"""
|
|
3685
3681
|
title: Annotated[str | None, Field(examples=['Instagram scraper'])] = None
|
|
3686
3682
|
"""
|
|
3687
|
-
Human-readable name of the Actor, displayed in Scrapely Store and
|
|
3683
|
+
Human-readable name of the Actor, displayed in Scrapely Store and dashboard. Can contain spaces and capital letters. Recommended length is 40-50 characters. You can change this title without affecting the Actor's URL or SEO.
|
|
3688
3684
|
"""
|
|
3689
3685
|
restart_on_error: Annotated[bool | None, Field(deprecated=True, examples=[False])] = None
|
|
3690
3686
|
versions: list[CreateOrUpdateVersionRequest] | None = None
|
|
@@ -3756,7 +3752,7 @@ class UpdateLimitsRequest(BaseModel):
|
|
|
3756
3752
|
"""
|
|
3757
3753
|
data_retention_days: Annotated[int | None, Field(examples=[90])] = None
|
|
3758
3754
|
"""
|
|
3759
|
-
|
|
3755
|
+
How many days unnamed storages (datasets, key-value stores and request queues without a name) are kept: Free 2 days (48 hours), Starter 31, Scale 60, Business 90. Runs follow their own retention: on the Free plan the latest 10 runs per Actor are kept and older ones are deleted after 7 days; on paid plans runs are deleted after 31 days. This value is set by your plan and cannot be changed through the API.
|
|
3760
3756
|
|
|
3761
3757
|
"""
|
|
3762
3758
|
|
scrapely_client/_pagination.py
CHANGED
|
@@ -4,6 +4,8 @@ from .actor_env_var import ActorEnvVarClient, ActorEnvVarClientAsync
|
|
|
4
4
|
from .actor_env_var_collection import ActorEnvVarCollectionClient, ActorEnvVarCollectionClientAsync
|
|
5
5
|
from .actor_version import ActorVersionClient, ActorVersionClientAsync
|
|
6
6
|
from .actor_version_collection import ActorVersionCollectionClient, ActorVersionCollectionClientAsync
|
|
7
|
+
from .automation import AutomationClient, AutomationClientAsync
|
|
8
|
+
from .automation_collection import AutomationCollectionClient, AutomationCollectionClientAsync
|
|
7
9
|
from .build import BuildClient, BuildClientAsync
|
|
8
10
|
from .build_collection import BuildCollectionClient, BuildCollectionClientAsync
|
|
9
11
|
from .dataset import DatasetClient, DatasetClientAsync
|
|
@@ -39,6 +41,10 @@ __all__ = [
|
|
|
39
41
|
'ActorVersionClientAsync',
|
|
40
42
|
'ActorVersionCollectionClient',
|
|
41
43
|
'ActorVersionCollectionClientAsync',
|
|
44
|
+
'AutomationClient',
|
|
45
|
+
'AutomationClientAsync',
|
|
46
|
+
'AutomationCollectionClient',
|
|
47
|
+
'AutomationCollectionClientAsync',
|
|
42
48
|
'BuildClient',
|
|
43
49
|
'BuildClientAsync',
|
|
44
50
|
'BuildCollectionClient',
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# Derived from apify-client-python (Copyright Apify Technologies s.r.o., Apache-2.0).
|
|
2
|
+
# Modified by TeraShift GmbH. See the LICENSE and NOTICE files.
|
|
1
3
|
from __future__ import annotations
|
|
2
4
|
|
|
3
5
|
import asyncio
|
|
@@ -326,7 +328,7 @@ class ResourceClient(ResourceClientBase):
|
|
|
326
328
|
DEFAULT_WAIT_WHEN_JOB_NOT_EXIST seconds.
|
|
327
329
|
|
|
328
330
|
Raises:
|
|
329
|
-
|
|
331
|
+
ScrapelyApiError: If API returns errors other than 404.
|
|
330
332
|
"""
|
|
331
333
|
deadline = (datetime.now(UTC) + wait_duration) if wait_duration is not None else None
|
|
332
334
|
not_found_deadline: datetime | None = None
|
|
@@ -518,7 +520,7 @@ class ResourceClientAsync(ResourceClientBase):
|
|
|
518
520
|
DEFAULT_WAIT_WHEN_JOB_NOT_EXIST seconds.
|
|
519
521
|
|
|
520
522
|
Raises:
|
|
521
|
-
|
|
523
|
+
ScrapelyApiError: If API returns errors other than 404.
|
|
522
524
|
"""
|
|
523
525
|
deadline = (datetime.now(UTC) + wait_duration) if wait_duration is not None else None
|
|
524
526
|
not_found_deadline: datetime | None = None
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# Derived from apify-client-python (Copyright Apify Technologies s.r.o., Apache-2.0).
|
|
2
|
+
# Modified by TeraShift GmbH. See the LICENSE and NOTICE files.
|
|
1
3
|
from __future__ import annotations
|
|
2
4
|
|
|
3
5
|
from typing import TYPE_CHECKING, Any, Literal
|
|
@@ -65,7 +67,7 @@ class ActorClient(ResourceClient):
|
|
|
65
67
|
"""Sub-client for managing a specific Actor.
|
|
66
68
|
|
|
67
69
|
Provides methods to manage a specific Actor, e.g. update it, delete it, build it, or start runs. Obtain an instance
|
|
68
|
-
via an appropriate method on the `
|
|
70
|
+
via an appropriate method on the `ScrapelyClient` class.
|
|
69
71
|
"""
|
|
70
72
|
|
|
71
73
|
def __init__(
|
|
@@ -561,7 +563,7 @@ class ActorClientAsync(ResourceClientAsync):
|
|
|
561
563
|
"""Sub-client for managing a specific Actor.
|
|
562
564
|
|
|
563
565
|
Provides methods to manage a specific Actor, e.g. update it, delete it, build it, or start runs. Obtain an instance
|
|
564
|
-
via an appropriate method on the `
|
|
566
|
+
via an appropriate method on the `ScrapelyClientAsync` class.
|
|
565
567
|
"""
|
|
566
568
|
|
|
567
569
|
def __init__(
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# Derived from apify-client-python (Copyright Apify Technologies s.r.o., Apache-2.0).
|
|
2
|
+
# Modified by TeraShift GmbH. See the LICENSE and NOTICE files.
|
|
1
3
|
from __future__ import annotations
|
|
2
4
|
|
|
3
5
|
from typing import TYPE_CHECKING, Any, Literal
|
|
@@ -30,7 +32,7 @@ class ActorCollectionClient(ResourceClient):
|
|
|
30
32
|
"""Sub-client for the Actor collection.
|
|
31
33
|
|
|
32
34
|
Provides methods to manage the Actor collection, e.g. list or create Actors. Obtain an instance via an appropriate
|
|
33
|
-
method on the `
|
|
35
|
+
method on the `ScrapelyClient` class.
|
|
34
36
|
"""
|
|
35
37
|
|
|
36
38
|
def __init__(
|
|
@@ -206,7 +208,7 @@ class ActorCollectionClientAsync(ResourceClientAsync):
|
|
|
206
208
|
"""Sub-client for the Actor collection.
|
|
207
209
|
|
|
208
210
|
Provides methods to manage the Actor collection, e.g. list or create Actors. Obtain an instance via an appropriate
|
|
209
|
-
method on the `
|
|
211
|
+
method on the `ScrapelyClientAsync` class.
|
|
210
212
|
"""
|
|
211
213
|
|
|
212
214
|
def __init__(
|