scrapely-client-python 3.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of scrapely-client-python might be problematic. Click here for more details.
- scrapely_client/__init__.py +30 -0
- scrapely_client/_charging.py +186 -0
- scrapely_client/_client_registry.py +136 -0
- scrapely_client/_consts.py +96 -0
- scrapely_client/_docs.py +37 -0
- scrapely_client/_literals.py +539 -0
- scrapely_client/_logging.py +180 -0
- scrapely_client/_models.py +4172 -0
- scrapely_client/_pagination.py +220 -0
- scrapely_client/_resource_clients/__init__.py +84 -0
- scrapely_client/_resource_clients/_resource_client.py +567 -0
- scrapely_client/_resource_clients/actor.py +1042 -0
- scrapely_client/_resource_clients/actor_collection.py +377 -0
- scrapely_client/_resource_clients/actor_env_var.py +154 -0
- scrapely_client/_resource_clients/actor_env_var_collection.py +165 -0
- scrapely_client/_resource_clients/actor_version.py +254 -0
- scrapely_client/_resource_clients/actor_version_collection.py +232 -0
- scrapely_client/_resource_clients/build.py +244 -0
- scrapely_client/_resource_clients/build_collection.py +164 -0
- scrapely_client/_resource_clients/dataset.py +1151 -0
- scrapely_client/_resource_clients/dataset_collection.py +244 -0
- scrapely_client/_resource_clients/key_value_store.py +906 -0
- scrapely_client/_resource_clients/key_value_store_collection.py +243 -0
- scrapely_client/_resource_clients/log.py +225 -0
- scrapely_client/_resource_clients/request_queue.py +1167 -0
- scrapely_client/_resource_clients/request_queue_collection.py +240 -0
- scrapely_client/_resource_clients/run.py +875 -0
- scrapely_client/_resource_clients/run_collection.py +226 -0
- scrapely_client/_resource_clients/schedule.py +243 -0
- scrapely_client/_resource_clients/schedule_collection.py +260 -0
- scrapely_client/_resource_clients/store_collection.py +242 -0
- scrapely_client/_resource_clients/task.py +793 -0
- scrapely_client/_resource_clients/task_collection.py +322 -0
- scrapely_client/_resource_clients/user.py +262 -0
- scrapely_client/_resource_clients/webhook.py +292 -0
- scrapely_client/_resource_clients/webhook_collection.py +281 -0
- scrapely_client/_resource_clients/webhook_dispatch.py +84 -0
- scrapely_client/_resource_clients/webhook_dispatch_collection.py +158 -0
- scrapely_client/_scrapely_client.py +837 -0
- scrapely_client/_statistics.py +29 -0
- scrapely_client/_status_message_watcher.py +197 -0
- scrapely_client/_streamed_log.py +250 -0
- scrapely_client/_typeddicts.py +408 -0
- scrapely_client/_utils/__init__.py +0 -0
- scrapely_client/_utils/crypto.py +87 -0
- scrapely_client/_utils/encoding.py +133 -0
- scrapely_client/_utils/errors.py +33 -0
- scrapely_client/_utils/http.py +137 -0
- scrapely_client/_utils/time.py +32 -0
- scrapely_client/_utils/try_import.py +61 -0
- scrapely_client/errors.py +190 -0
- scrapely_client/http_clients/__init__.py +10 -0
- scrapely_client/http_clients/_base.py +430 -0
- scrapely_client/http_clients/_impit.py +598 -0
- scrapely_client/http_compressors/__init__.py +18 -0
- scrapely_client/http_compressors/_base.py +25 -0
- scrapely_client/http_compressors/_brotli.py +38 -0
- scrapely_client/http_compressors/_gzip.py +38 -0
- scrapely_client/http_compressors/_resolve.py +38 -0
- scrapely_client/py.typed +0 -0
- scrapely_client/types.py +52 -0
- scrapely_client_python-3.1.3.dist-info/METADATA +269 -0
- scrapely_client_python-3.1.3.dist-info/RECORD +65 -0
- scrapely_client_python-3.1.3.dist-info/WHEEL +4 -0
- scrapely_client_python-3.1.3.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import gzip
|
|
4
|
+
|
|
5
|
+
from scrapely_client.http_compressors._base import HttpCompressor
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class GzipHttpCompressor(HttpCompressor):
|
|
9
|
+
"""Compresses request bodies using gzip.
|
|
10
|
+
|
|
11
|
+
Uses the standard library `gzip` module. No extra dependencies required.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
content_encoding = 'gzip'
|
|
15
|
+
|
|
16
|
+
_min_quality = 1
|
|
17
|
+
"""Lowest valid quality (fastest, least compression)."""
|
|
18
|
+
|
|
19
|
+
_max_quality = 9
|
|
20
|
+
"""Highest valid quality (slowest, best compression)."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, *, quality: int = _max_quality) -> None:
|
|
23
|
+
"""Initialize the gzip compressor.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
quality: Compression level, from the fastest to the best compression.
|
|
27
|
+
|
|
28
|
+
Raises:
|
|
29
|
+
ValueError: If `quality` is out of the valid range.
|
|
30
|
+
"""
|
|
31
|
+
if not self._min_quality <= quality <= self._max_quality:
|
|
32
|
+
raise ValueError(
|
|
33
|
+
f'gzip quality must be between {self._min_quality} and {self._max_quality}, got {quality}.'
|
|
34
|
+
)
|
|
35
|
+
self._quality = quality
|
|
36
|
+
|
|
37
|
+
def compress(self, data: bytes) -> bytes:
|
|
38
|
+
return gzip.compress(data, compresslevel=self._quality)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from scrapely_client.http_compressors._base import HttpCompressor
|
|
6
|
+
from scrapely_client.http_compressors._gzip import GzipHttpCompressor
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from scrapely_client.types import HttpCompressionAlgorithm
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def resolve_compressor(compression: HttpCompressionAlgorithm | HttpCompressor) -> HttpCompressor:
|
|
13
|
+
"""Convert a compression string or `HttpCompressor` instance into a concrete `HttpCompressor`.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
compression: A string literal naming an HTTP compression algorithm, or an `HttpCompressor` instance.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
A ready-to-use `HttpCompressor`.
|
|
20
|
+
|
|
21
|
+
Raises:
|
|
22
|
+
ImportError: If the requested algorithm needs an optional extra that is not installed.
|
|
23
|
+
ValueError: If `compression` is not a recognized compression algorithm.
|
|
24
|
+
"""
|
|
25
|
+
if isinstance(compression, HttpCompressor):
|
|
26
|
+
return compression
|
|
27
|
+
if compression == 'gzip':
|
|
28
|
+
return GzipHttpCompressor()
|
|
29
|
+
if compression == 'brotli':
|
|
30
|
+
# The import is here so the ImportError is raised at call time,
|
|
31
|
+
# not at module import time, giving users a clear message.
|
|
32
|
+
from scrapely_client.http_compressors import BrotliHttpCompressor # noqa: PLC0415
|
|
33
|
+
|
|
34
|
+
return BrotliHttpCompressor()
|
|
35
|
+
|
|
36
|
+
# The backend supports also `deflate` and `identity` (no compression). One can build
|
|
37
|
+
# a custom compressor if needed.
|
|
38
|
+
raise ValueError(f'Unsupported compression algorithm: {compression!r}')
|
scrapely_client/py.typed
ADDED
|
File without changes
|
scrapely_client/types.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import timedelta
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
from scrapely_client._models import WebhookCreate, WebhookRepresentation
|
|
7
|
+
from scrapely_client._typeddicts import (
|
|
8
|
+
WebhookCreateCamelDict,
|
|
9
|
+
WebhookCreateDict,
|
|
10
|
+
WebhookRepresentationCamelDict,
|
|
11
|
+
WebhookRepresentationDict,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
HttpCompressionAlgorithm = Literal['brotli', 'gzip']
|
|
15
|
+
"""Accepted string literals for the `compression` parameter on `ApifyClient` and `ApifyClientAsync`."""
|
|
16
|
+
|
|
17
|
+
Timeout = timedelta | Literal['no_timeout', 'short', 'medium', 'long']
|
|
18
|
+
"""Type for the `timeout` parameter on resource client methods.
|
|
19
|
+
|
|
20
|
+
`'short'`, `'medium'`, and `'long'` are tier literals resolved by the HTTP client to configured values.
|
|
21
|
+
A `timedelta` overrides the timeout for this call, and `'no_timeout'` disables the timeout entirely.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
WebhooksList = (
|
|
25
|
+
list[WebhookCreate]
|
|
26
|
+
| list[WebhookCreateDict]
|
|
27
|
+
| list[WebhookCreateCamelDict]
|
|
28
|
+
| list[WebhookRepresentation]
|
|
29
|
+
| list[WebhookRepresentationDict]
|
|
30
|
+
| list[WebhookRepresentationCamelDict]
|
|
31
|
+
)
|
|
32
|
+
"""Type for the `webhooks` parameter on resource-client `start`/`call` methods and `from_webhooks`.
|
|
33
|
+
|
|
34
|
+
`WebhookRepresentation` / `WebhookRepresentationDict` / `WebhookRepresentationCamelDict` are the minimal ad-hoc
|
|
35
|
+
webhook shape (only `event_types` and `request_url` required). `WebhookCreate` / `WebhookCreateDict` /
|
|
36
|
+
`WebhookCreateCamelDict` are accepted so a persistent-webhook definition can be reused; their fields not relevant
|
|
37
|
+
to ad-hoc webhooks (e.g. `condition`) are ignored at runtime. The `*CamelDict` variants accept camelCase keys
|
|
38
|
+
matching the Scrapely API spelling.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
JsonSerializable = dict[str, 'JsonSerializable'] | list['JsonSerializable'] | str | int | float | bool | None
|
|
42
|
+
"""Recursive type for JSON-serializable values - primitives plus objects and arrays with JSON-serializable contents.
|
|
43
|
+
|
|
44
|
+
Based on the definition discussed in https://github.com/python/typing/issues/182.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
'HttpCompressionAlgorithm',
|
|
49
|
+
'JsonSerializable',
|
|
50
|
+
'Timeout',
|
|
51
|
+
'WebhooksList',
|
|
52
|
+
]
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scrapely-client-python
|
|
3
|
+
Version: 3.1.3
|
|
4
|
+
Summary: Scrapely API client for Python - a drop-in replacement for apify-client
|
|
5
|
+
Keywords: scrapely,apify,api,client,automation,crawling,scraping
|
|
6
|
+
Author: Scrapely
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
18
|
+
Requires-Dist: colorama>=0.4.0
|
|
19
|
+
Requires-Dist: impit~=0.13.0
|
|
20
|
+
Requires-Dist: more-itertools>=10.0.0
|
|
21
|
+
Requires-Dist: pydantic[email]>=2.11.0
|
|
22
|
+
Requires-Dist: typing-extensions>=4.6.0
|
|
23
|
+
Requires-Dist: brotli>=1.0.9 ; extra == 'brotli'
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Project-URL: Homepage, https://api.scrape.ly
|
|
26
|
+
Project-URL: Changelog, https://github.com/SandeepDev1/scrapely/blob/main/scrapely-client-python/CHANGELOG.md
|
|
27
|
+
Project-URL: Documentation, https://api.scrape.ly
|
|
28
|
+
Project-URL: Issue Tracker, https://github.com/SandeepDev1/scrapely/issues
|
|
29
|
+
Project-URL: Source Code, https://github.com/SandeepDev1/scrapely
|
|
30
|
+
Provides-Extra: brotli
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
<h1 align="center">Scrapely API client for Python</h1>
|
|
34
|
+
|
|
35
|
+
<p align="center">
|
|
36
|
+
<strong>The official Python client for the <a href="https://api.scrape.ly">Scrapely REST API</a>, and a drop-in replacement for <code>apify-client</code>.</strong>
|
|
37
|
+
</p>
|
|
38
|
+
|
|
39
|
+
<p align="center">
|
|
40
|
+
<a href="https://pypi.org/project/scrapely-client-python/"><img src="https://badge.fury.io/py/scrapely-client-python.svg" alt="PyPI version"></a>
|
|
41
|
+
<a href="https://pypi.org/project/scrapely-client-python/"><img src="https://img.shields.io/pypi/dm/scrapely-client-python" alt="PyPI downloads"></a>
|
|
42
|
+
<a href="https://pypi.org/project/scrapely-client-python/"><img src="https://img.shields.io/badge/python-3.11%2B-blue" alt="Python versions"></a>
|
|
43
|
+
<a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
`scrapely-client-python` lets you talk to the [Scrapely platform](https://api.scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
|
|
47
|
+
|
|
48
|
+
> **Migrating from `apify-client`?** Just swap the import — the class names (`ApifyClient`, `ApifyClientAsync`), constructor arguments, methods and return types are identical. See [Migrating from apify-client](#migrating-from-apify-client) below.
|
|
49
|
+
>
|
|
50
|
+
> If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
|
|
51
|
+
|
|
52
|
+
## Table of contents
|
|
53
|
+
|
|
54
|
+
- [Installation](#installation)
|
|
55
|
+
- [Quick start](#quick-start)
|
|
56
|
+
- [Migrating from apify-client](#migrating-from-apify-client)
|
|
57
|
+
- [Features](#features)
|
|
58
|
+
- [Usage examples](#usage-examples)
|
|
59
|
+
- [Documentation](#documentation)
|
|
60
|
+
- [Related projects](#related-projects)
|
|
61
|
+
- [Support and community](#support-and-community)
|
|
62
|
+
- [Contributing](#contributing)
|
|
63
|
+
- [License](#license)
|
|
64
|
+
|
|
65
|
+
## Installation
|
|
66
|
+
|
|
67
|
+
`scrapely-client-python` requires **Python 3.11 or higher** and is published on [PyPI](https://pypi.org/project/scrapely-client-python/).
|
|
68
|
+
|
|
69
|
+
- From [PyPI](https://pypi.org/project/scrapely-client-python/), it can be installed for example with [pip](https://pip.pypa.io/):
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pip install scrapely-client-python
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
or with [uv](https://docs.astral.sh/uv/):
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
uv add scrapely-client-python
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
or any other Python package manager that consumes PyPI.
|
|
82
|
+
|
|
83
|
+
The client compresses request bodies with `gzip` by default (no extra dependencies required). To opt in to
|
|
84
|
+
`brotli` (better compression ratio), install the optional extra and pass `compression='brotli'`:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install "scrapely-client-python[brotli]"
|
|
88
|
+
# or
|
|
89
|
+
uv add "scrapely-client-python[brotli]"
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
## Quick start
|
|
94
|
+
|
|
95
|
+
You'll need a Scrapely API token — find yours in the integrations section of the Scrapely Console. Pass it to the client and you're ready to go.
|
|
96
|
+
|
|
97
|
+
### Synchronous client
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from scrapely_client import ApifyClient
|
|
101
|
+
|
|
102
|
+
client = ApifyClient('MY-SCRAPELY-TOKEN')
|
|
103
|
+
|
|
104
|
+
# Start an Actor and wait for it to finish.
|
|
105
|
+
run = client.actor('apify/hello-world').call(
|
|
106
|
+
run_input={'message': 'Hello, Scrapely!'},
|
|
107
|
+
)
|
|
108
|
+
if run is None:
|
|
109
|
+
raise RuntimeError('Actor run was not found.')
|
|
110
|
+
|
|
111
|
+
# Iterate items from the run's default dataset.
|
|
112
|
+
for item in client.dataset(run.default_dataset_id).iterate_items():
|
|
113
|
+
print(item)
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### Asynchronous client
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
import asyncio
|
|
120
|
+
|
|
121
|
+
from scrapely_client import ApifyClientAsync
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
async def main() -> None:
|
|
125
|
+
client = ApifyClientAsync('MY-SCRAPELY-TOKEN')
|
|
126
|
+
|
|
127
|
+
run = await client.actor('apify/hello-world').call(
|
|
128
|
+
run_input={'message': 'Hello, Scrapely!'},
|
|
129
|
+
)
|
|
130
|
+
if run is None:
|
|
131
|
+
raise RuntimeError('Actor run was not found.')
|
|
132
|
+
|
|
133
|
+
# Iterate items from the run's default dataset.
|
|
134
|
+
async for item in client.dataset(run.default_dataset_id).iterate_items():
|
|
135
|
+
print(item)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
asyncio.run(main())
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
> **Keep your token secret.** It authorizes requests on your behalf and can incur usage costs. Never commit it to source control or expose it to client-side code.
|
|
142
|
+
|
|
143
|
+
## Migrating from apify-client
|
|
144
|
+
|
|
145
|
+
`scrapely-client-python` is a drop-in replacement for `apify-client` — swap the import and nothing else in
|
|
146
|
+
your code needs to change:
|
|
147
|
+
|
|
148
|
+
```diff
|
|
149
|
+
- from apify_client import ApifyClient, ApifyClientAsync
|
|
150
|
+
+ from scrapely_client import ApifyClient, ApifyClientAsync
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
- The class names (`ApifyClient`, `ApifyClientAsync` and all resource clients), constructor
|
|
154
|
+
arguments, methods and return types are identical to `apify-client` (aliases `ScrapelyClient` /
|
|
155
|
+
`ScrapelyClientAsync` are also exported).
|
|
156
|
+
- The client talks to `https://api.scrape.ly` by default; override with the `api_url` /
|
|
157
|
+
`api_public_url` arguments if you need a different endpoint.
|
|
158
|
+
- Scrapely-specific extensions (e.g. pay-per-event `charging_hooks`, `task().publish()` /
|
|
159
|
+
`task().unpublish()`, `run().charge()`) are strictly additive — code written for `apify-client`
|
|
160
|
+
runs unchanged.
|
|
161
|
+
|
|
162
|
+
For a guided walkthrough — authenticating, running an Actor, and reading its results — see the Quick start guide.
|
|
163
|
+
|
|
164
|
+
## Features
|
|
165
|
+
|
|
166
|
+
- **Synchronous and asynchronous clients** — pick `ApifyClient` or `ApifyClientAsync` to match your codebase; both expose the same API (Asyncio support).
|
|
167
|
+
- **Fully typed responses** — every method returns a [Pydantic](https://docs.pydantic.dev/) model generated from the Apify OpenAPI spec, with IDE autocomplete and runtime validation (Typed models).
|
|
168
|
+
- **Automatic retries** — exponential backoff for network errors, HTTP 429, and 5xx responses, configurable per client (Retries).
|
|
169
|
+
- **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call (Timeouts).
|
|
170
|
+
- **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering (Pagination, Streaming).
|
|
171
|
+
- **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks (Convenience methods).
|
|
172
|
+
- **Pluggable HTTP layer** — swap the default Impit-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation (Custom HTTP clients).
|
|
173
|
+
- **Structured errors** — every API error surfaces as an `ApifyApiError` with HTTP-specific subclasses for precise handling (Error handling).
|
|
174
|
+
- **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more (Logging).
|
|
175
|
+
|
|
176
|
+
## Usage examples
|
|
177
|
+
|
|
178
|
+
The client mirrors the platform's resource model. Each entry point returns either a **single-resource client** for an individual item or a **collection client** for listing and creating items (Single and collection clients).
|
|
179
|
+
|
|
180
|
+
### List Actors and create one
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
actors = client.actors()
|
|
184
|
+
print(actors.list(limit=10).items)
|
|
185
|
+
|
|
186
|
+
new_actor = actors.create(name='my-actor')
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### Stream live logs while a run is in progress
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
run = client.actor('apify/web-scraper').start(run_input={...})
|
|
193
|
+
|
|
194
|
+
with client.run(run.id).log().stream() as log_stream:
|
|
195
|
+
for chunk in log_stream.iter_bytes():
|
|
196
|
+
print(chunk.decode(), end='')
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
### Read and write key-value store records
|
|
200
|
+
|
|
201
|
+
```python
|
|
202
|
+
store = client.key_value_store('STORE-ID')
|
|
203
|
+
store.set_record('greeting', {'message': 'Hello!'})
|
|
204
|
+
record = store.get_record('greeting')
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### Iterate dataset items with automatic pagination
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
for item in client.dataset('DATASET-ID').iterate_items(fields=['title', 'url']):
|
|
211
|
+
process(item)
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### Tune retries and timeouts
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
from datetime import timedelta
|
|
218
|
+
|
|
219
|
+
from scrapely_client import ApifyClient
|
|
220
|
+
|
|
221
|
+
client = ApifyClient(
|
|
222
|
+
token='MY-SCRAPELY-TOKEN',
|
|
223
|
+
max_retries=8,
|
|
224
|
+
min_delay_between_retries=timedelta(milliseconds=500),
|
|
225
|
+
timeout_long=timedelta(minutes=10),
|
|
226
|
+
)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the Guides.
|
|
230
|
+
|
|
231
|
+
## Documentation
|
|
232
|
+
|
|
233
|
+
The full documentation lives at **https://api.scrape.ly**.
|
|
234
|
+
|
|
235
|
+
| Section | What you'll find |
|
|
236
|
+
| --- | --- |
|
|
237
|
+
| Introduction | Overview, prerequisites, and a tour of the client. |
|
|
238
|
+
| Quick start | Authenticate, run an Actor, and fetch its results step by step. |
|
|
239
|
+
| Concepts | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. |
|
|
240
|
+
| Guides | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
|
|
241
|
+
| Upgrading | Migrating between major versions. |
|
|
242
|
+
| API reference | Generated reference for every class, method, and model. |
|
|
243
|
+
| Changelog | Release history and breaking changes. |
|
|
244
|
+
|
|
245
|
+
## Related projects
|
|
246
|
+
|
|
247
|
+
- **Apify SDK for Python** — toolkit for **building** Apify Actors in Python (this client is bundled with it).
|
|
248
|
+
- **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
|
|
249
|
+
- **Apify API client for JavaScript / TypeScript** — equivalent Apify API client for Node.js.
|
|
250
|
+
- **Apify SDK for JavaScript / TypeScript** — equivalent Apify SDK for Node.js.
|
|
251
|
+
- **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — original Node.js implementation of the Crawlee framework.
|
|
252
|
+
- **Apify CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
253
|
+
|
|
254
|
+
## Support and community
|
|
255
|
+
|
|
256
|
+
- **GitHub issues** — report a bug or request a feature in the repository's [issue tracker](https://github.com/apify/apify-client-python/issues).
|
|
257
|
+
|
|
258
|
+
## Contributing
|
|
259
|
+
|
|
260
|
+
Bug reports, fixes, and improvements are welcome! See [CONTRIBUTING.md](./CONTRIBUTING.md) for the development setup, coding standards, testing, and the release process. The repo uses [uv](https://docs.astral.sh/uv/) for project management and [Poe the Poet](https://poethepoet.natn.io/) as a task runner; the typical loop is:
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
uv run poe install-dev # install dev deps and git hooks
|
|
264
|
+
uv run poe check-code # lint, type-check, unit tests, docstring check
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
## License
|
|
268
|
+
|
|
269
|
+
Released under the [Apache License 2.0](./LICENSE).
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
scrapely_client/__init__.py,sha256=F2qrB4pNh5zgKsGFQkmOYbbqs3fU8GSzkqH1deuwgPQ,729
|
|
2
|
+
scrapely_client/_charging.py,sha256=wvOgbk85xkoiJ0kU8_5WN9u7ZqFPZki-GsAwnKt8emI,6903
|
|
3
|
+
scrapely_client/_client_registry.py,sha256=TaCXlXmE6vTLfoiEStJlP07sMu-O0iomnoSxhM4NHyQ,5438
|
|
4
|
+
scrapely_client/_consts.py,sha256=Bb3EkGfvFFAM_fpETffmNWWAEtpgUfkURWoL8pCjw0M,3395
|
|
5
|
+
scrapely_client/_docs.py,sha256=2HBk31suQhWYwME8h5m7lZkKBh8kh1DAT-5fVtF_m3Q,967
|
|
6
|
+
scrapely_client/_literals.py,sha256=vZFD2akQV-32CY_fktYGzLA5DQS3_rewfAjnrTz6jT0,16856
|
|
7
|
+
scrapely_client/_logging.py,sha256=3hGBNL6iMb-rvlHJvKYOZTap0QLDAMENxmw0FmvGMQY,6624
|
|
8
|
+
scrapely_client/_models.py,sha256=td6-tHVttRLXAGAsTWYqqRmuIklDAzAs0LmUSmKTHTE,142785
|
|
9
|
+
scrapely_client/_pagination.py,sha256=BZwPPF7vwf-e9SPFqpyvlw1ZvqWopMtUi7MecQAsK4w,8429
|
|
10
|
+
scrapely_client/_resource_clients/__init__.py,sha256=fLcGFozm7zSt-ZY7KWdL01dph4adJiuVOg7uovmGgyQ,3581
|
|
11
|
+
scrapely_client/_resource_clients/_resource_client.py,sha256=NJNgU3WZ_irz7rtLXhYBcGDUyW3JwdwdiFOAggl8IvI,21443
|
|
12
|
+
scrapely_client/_resource_clients/actor.py,sha256=crSsFhFVgOsV0Qs0LG6KJYpT7OhFmv4JTJ-Odyt8pHQ,44829
|
|
13
|
+
scrapely_client/_resource_clients/actor_collection.py,sha256=RYM6SFGP7bOO8HIqCgIZvIf27pbK1Hfy4KXBvwKPruk,16810
|
|
14
|
+
scrapely_client/_resource_clients/actor_env_var.py,sha256=YsSXdoJ6bJ6lXWkMQ0Le0BUUHEfHwsFLiYgoNqkDOmE,4660
|
|
15
|
+
scrapely_client/_resource_clients/actor_env_var_collection.py,sha256=rFTdOaqhsVgIIY9pSYF7LCj5FazLf84Md42uiRKJc9s,5525
|
|
16
|
+
scrapely_client/_resource_clients/actor_version.py,sha256=hhPgpa2igb9YtRcB75eoJKAV-DtrEtq6ikgzqaSVT0E,10082
|
|
17
|
+
scrapely_client/_resource_clients/actor_version_collection.py,sha256=XsWmHLdsb1ghZnwcsp0lw4w2speFndXYehhdt2fzWZw,9562
|
|
18
|
+
scrapely_client/_resource_clients/build.py,sha256=MA_C6RblCXiR1rys33zhtRneuS5AVPeskBb-Aa_H2eA,7649
|
|
19
|
+
scrapely_client/_resource_clients/build_collection.py,sha256=iNg0XfhDboBbkFLCGoi3gsve-YdXv9dWCCRbsVVzH8I,5610
|
|
20
|
+
scrapely_client/_resource_clients/dataset.py,sha256=jJdjshHnKMuzSl26z7LnT3fK1jrlbUxfLzkRCiEMo40,51974
|
|
21
|
+
scrapely_client/_resource_clients/dataset_collection.py,sha256=Ao0p1M0M7D8iMqEwT5f0YsPRm52LmKDh72YbIr3vpKA,8703
|
|
22
|
+
scrapely_client/_resource_clients/key_value_store.py,sha256=eDFa66XhKoNkO0ftTa1frv0750TESLMY99JJ6TMSjFc,32517
|
|
23
|
+
scrapely_client/_resource_clients/key_value_store_collection.py,sha256=yiQmPrkmCiuHkFFh8MM8yi5xHELWapv0grQSgYCXWGk,9123
|
|
24
|
+
scrapely_client/_resource_clients/log.py,sha256=ntZY0rNkHlvz-WT74uZDestv6Yy5Og-pQ1NX4kz7WnE,7335
|
|
25
|
+
scrapely_client/_resource_clients/request_queue.py,sha256=XPCCeMSgvsBpKfQZL2x4yYWmzuaRZZeQ2LoRpzBF-2c,40891
|
|
26
|
+
scrapely_client/_resource_clients/request_queue_collection.py,sha256=JcuKvZUAK47I9u41uVS7OLnlgFPiRCOAtpO5NdyNFDk,8827
|
|
27
|
+
scrapely_client/_resource_clients/run.py,sha256=NUoZ0pDIEXUfB4z4RB_LDyZBUGieFStyxnBh3L6yUL4,32063
|
|
28
|
+
scrapely_client/_resource_clients/run_collection.py,sha256=qQkOZ6lBeRmNT1AscHkVp8Aa4OKNh-iPHzmXmEV1IcI,8285
|
|
29
|
+
scrapely_client/_resource_clients/schedule.py,sha256=JGSuRXA7FeaTZTNuMrfYin_BmmhQpdqBtG5FxRr2lGg,7938
|
|
30
|
+
scrapely_client/_resource_clients/schedule_collection.py,sha256=I9FSFO2t40X3XyZaVjhqQcS1J5PhxiXJw5A7L2SHjQ4,9358
|
|
31
|
+
scrapely_client/_resource_clients/store_collection.py,sha256=ulwi7D0jGBH7gc-voWIpEL04WqRMo63UA_kMinv4kUM,9074
|
|
32
|
+
scrapely_client/_resource_clients/task.py,sha256=CxU-x3yug3scAGhoxpZViN5M99T4eR00gNA9NyncXks,32441
|
|
33
|
+
scrapely_client/_resource_clients/task_collection.py,sha256=0gMs9JTahKVPQcY9Th4JHQxuKQMfj98uFyY9kpAYGbg,13171
|
|
34
|
+
scrapely_client/_resource_clients/user.py,sha256=ySaBDn2_gbb3-qVylX9oylTL7GzC0JOBwPsHt6Gr9VY,8613
|
|
35
|
+
scrapely_client/_resource_clients/webhook.py,sha256=QljYqPYleWaRk2msdu1GaB_7AVA2hocG4nSpFr-BFU4,10328
|
|
36
|
+
scrapely_client/_resource_clients/webhook_collection.py,sha256=__e5uGx_VnSZULC0M-o65xugPve3gR2KY6wUAZA1H3M,11195
|
|
37
|
+
scrapely_client/_resource_clients/webhook_dispatch.py,sha256=uyJItKA1W_EXtEByAAXHzit2pXzjAG2C2YYbR753Yww,2504
|
|
38
|
+
scrapely_client/_resource_clients/webhook_dispatch_collection.py,sha256=vj5oJuJAoo6X2YTl3Dsof1u14oA-YSTRYEKUaEWHtyo,5793
|
|
39
|
+
scrapely_client/_scrapely_client.py,sha256=lned1hgp2EGCPCjI-OnI62rpb4hfvLqZZQfW0Omsz1w,36672
|
|
40
|
+
scrapely_client/_statistics.py,sha256=mTsUFVByduYFPoHocZULv53k5dYg77Bl03wamQD6Jyc,903
|
|
41
|
+
scrapely_client/_status_message_watcher.py,sha256=8C0--UW6Ao4sy_PQ2BeNiNIXjyJu81b3Uc0BNUilciU,7743
|
|
42
|
+
scrapely_client/_streamed_log.py,sha256=zAN5X8Mj1iFTrgIDC20skSi0nkHTjZB2106Zy2KP7Ts,11509
|
|
43
|
+
scrapely_client/_typeddicts.py,sha256=YGNpxx4bQglWYhVYBUQltXjLHrhR62WyTdFSeezKKus,12275
|
|
44
|
+
scrapely_client/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
|
+
scrapely_client/_utils/crypto.py,sha256=CgO2iVj27GsfpSP-BJ3fdfvBdbidoKXqzjd8zYuCbAg,2749
|
|
46
|
+
scrapely_client/_utils/encoding.py,sha256=gptg6fB2zaIKYpMTQlEmTwcsbYhIYb4x0WV0uwOIroA,6310
|
|
47
|
+
scrapely_client/_utils/errors.py,sha256=wvLqWQvySQK2FbLZ7ExsTCGUXTANyrTQbC-7fg0R884,1081
|
|
48
|
+
scrapely_client/_utils/http.py,sha256=2HMiHFuPGet4LHahCNXnlLnL6_AdSlk3wjJF6P9SNBQ,4809
|
|
49
|
+
scrapely_client/_utils/time.py,sha256=cYnigcaMMX_H-b2aqMvPmj5p9i1v6cBYa90d8lhet3c,908
|
|
50
|
+
scrapely_client/_utils/try_import.py,sha256=sVqkJyWP4uqeBY29xivxxly7BjVXc3JahsAfSkRmU68,1799
|
|
51
|
+
scrapely_client/errors.py,sha256=K8pbTz_S1SoOsqtyZCKXh6LQWr8qubuNRB2VSkxifnc,6833
|
|
52
|
+
scrapely_client/http_clients/__init__.py,sha256=iVimn73xwMtmCeofeBwSLnHOEmSEpATGf-gtjbx7nWg,246
|
|
53
|
+
scrapely_client/http_clients/_base.py,sha256=aW9enxg4dG30z5E1pX7r7ICAQbZXjLLxLeYIhHlkaCQ,18842
|
|
54
|
+
scrapely_client/http_clients/_impit.py,sha256=OHDhdpAaQS_VX8XH8pwqCiNvNA_96xKuV2mKOGqPwK0,24157
|
|
55
|
+
scrapely_client/http_compressors/__init__.py,sha256=yz7ocf3DQO22K83UDHTmqwHDzRkTUXVyET8DFtbXg2c,954
|
|
56
|
+
scrapely_client/http_compressors/_base.py,sha256=25W0zDO54qdSnhYLvYyF4EbYM2mh5gxwlaaQ5yKAuzw,672
|
|
57
|
+
scrapely_client/http_compressors/_brotli.py,sha256=-naU_0qq1kDomTuA36EhvSxvbRuEAmpT81DPvZsSRow,1134
|
|
58
|
+
scrapely_client/http_compressors/_gzip.py,sha256=lQvws73pj6QDX9z9FeFZyK-8jEfSQUyx9M69aQNTnpY,1146
|
|
59
|
+
scrapely_client/http_compressors/_resolve.py,sha256=1w5xWGSjoMQ9gPifIPdJ3vKJ84LxoGW6-OKEZ6hCtV0,1503
|
|
60
|
+
scrapely_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
61
|
+
scrapely_client/types.py,sha256=gjngoXB5LnY8kkodTG4T1JEypNExrjxqpGMFsPHX6YM,2059
|
|
62
|
+
scrapely_client_python-3.1.3.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
|
|
63
|
+
scrapely_client_python-3.1.3.dist-info/WHEEL,sha256=Kot-FOXwz2no6rGIg4et79Z7m6vcKPAXr2Y6fTFpZuI,81
|
|
64
|
+
scrapely_client_python-3.1.3.dist-info/METADATA,sha256=MkyD9NlCEMkfFBopQVnfpMsnjIFgS-yI8YgRxOq7sAE,11802
|
|
65
|
+
scrapely_client_python-3.1.3.dist-info/RECORD,,
|