scrapely-client-python 1.0.8__tar.gz → 1.0.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/CHANGELOG.md +11 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/CONTRIBUTING.md +17 -36
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/PKG-INFO +35 -18
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/README.md +34 -17
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/pyproject.toml +7 -2
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/pyproject.toml.orig +2 -2
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_consts.py +3 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_logging.py +9 -3
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_models.py +1 -1
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor.py +9 -2
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/dataset.py +61 -29
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/key_value_store.py +41 -21
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/task.py +5 -2
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_scrapely_client.py +25 -14
- scrapely_client_python-1.0.10/src/scrapely_client/_utils/crypto.py +161 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/_impit.py +89 -4
- scrapely_client_python-1.0.8/src/scrapely_client/_utils/crypto.py +0 -89
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/LICENSE +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/NOTICE +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/__init__.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_charging.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_client_registry.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_docs.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_literals.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_pagination.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/__init__.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/_resource_client.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_env_var.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_env_var_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_version.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_version_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/automation.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/automation_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/build.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/build_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/dataset_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/key_value_store_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/log.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/request_queue.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/request_queue_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/run.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/run_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/schedule.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/schedule_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/store_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/task_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/user.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_dispatch.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_dispatch_collection.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_statistics.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_status_message_watcher.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_streamed_log.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_typeddicts.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/__init__.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/encoding.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/errors.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/http.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/time.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/try_import.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/errors.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/__init__.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/_base.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/__init__.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_base.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_brotli.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_gzip.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_resolve.py +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/py.typed +0 -0
- {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/types.py +0 -0
|
@@ -2,6 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [1.0.10] - 2026-11-02
|
|
6
|
+
|
|
7
|
+
- Run starts (`ActorClient.start` / `call`, `TaskClient.start` / `call`), `ActorClient.build` and `DatasetClient.push_items` send an `Idempotency-Key` header, so a request resent after a 5xx or a dropped connection starts one run or build and stores the items once.
|
|
8
|
+
- Any other POST or PATCH without an `Idempotency-Key` is no longer resent after a 5xx or a connection that dropped mid-request, since it may already have been applied. Rate limits (429), failures before the request was sent, request queue adds (de-duplicated by `uniqueKey`), request unlocks, head locks, run aborts and get-or-create of a storage by name are still retried.
|
|
9
|
+
- A keyed request answered with 409 `idempotency-key-in-progress` (the first request with that key is still being applied) is resent until it gets that request's answer; a 503 `push-outcome-unknown` is not resent, since no resend can change it.
|
|
10
|
+
|
|
11
|
+
## [1.0.9] - 2026-10-24
|
|
12
|
+
|
|
13
|
+
- The `SCRAPELY_API_URL` environment variable sets the API URL when `api_url` is not passed to `ScrapelyClient`, `ScrapelyClientAsync` or `with_custom_http_client`; `api_url` still wins, and without either the client uses `https://api.scrape.ly`. An empty value is ignored. `api_public_url` is unchanged.
|
|
14
|
+
- Version 1.0.3 defaulted to `https://api-staging.scrape.ly`; upgrade to 1.0.6 or later.
|
|
15
|
+
|
|
5
16
|
## [1.0.8] - 2026-10-20
|
|
6
17
|
|
|
7
18
|
- Request ids are percent-encoded as one URL path segment in `get_request`, `update_request`, `delete_request`, `prolong_request_lock` and `delete_request_lock` (sync and async); `''`, `.` and `..` raise `ValueError`. `update_request` checks the id against the API's grammar (1-128 letters, digits, `-`, `_`, `~`) and refuses a request without an id instead of sending `requests/None`. Resource ids (`client.dataset(id)`, versions, env vars) and automation run ids are encoded the same way. An id such as `..` used to send the call to another resource.
|
|
@@ -18,7 +18,7 @@ All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be ru
|
|
|
18
18
|
| Task | Description |
|
|
19
19
|
| ---- | ----------- |
|
|
20
20
|
| `install-dev` | Install development dependencies |
|
|
21
|
-
| `check-code` | Run lint, type-check,
|
|
21
|
+
| `check-code` | Run lint, type-check, docstring check, and unit tests |
|
|
22
22
|
| `lint` | Run linter |
|
|
23
23
|
| `format` | Fix lint issues and format code |
|
|
24
24
|
| `type-check` | Run type checker |
|
|
@@ -28,8 +28,10 @@ All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be ru
|
|
|
28
28
|
| `integration-tests-cov` | Run integration tests with coverage |
|
|
29
29
|
| `check-docstrings` | Check async client docstrings |
|
|
30
30
|
| `fix-docstrings` | Fix async client docstrings |
|
|
31
|
-
| `build-docs` | Build documentation website |
|
|
32
|
-
| `run-docs` | Run documentation website locally |
|
|
31
|
+
| `build-docs` | Build documentation website (does not work yet, see [Documentation](#documentation)) |
|
|
32
|
+
| `run-docs` | Run documentation website locally (does not work yet, see [Documentation](#documentation)) |
|
|
33
|
+
| `generate-models` | Regenerate the models from `openapi/scrapely-api.json` |
|
|
34
|
+
| `accept-openapi-spec` | Record the hash of a reviewed change to `openapi/scrapely-api.json` |
|
|
33
35
|
| `build` | Build package |
|
|
34
36
|
| `clean` | Remove build artifacts and clean caches |
|
|
35
37
|
|
|
@@ -103,7 +105,7 @@ Prerequisites:
|
|
|
103
105
|
|
|
104
106
|
- Set `SCRAPELY_TEST_USER_API_TOKEN` to your Scrapely API token
|
|
105
107
|
- Set `SCRAPELY_TEST_USER_2_API_TOKEN` for storage access tests (requires two accounts)
|
|
106
|
-
- Optionally set `SCRAPELY_INTEGRATION_TESTS_API_URL` to use a different Scrapely API environment
|
|
108
|
+
- Optionally set `SCRAPELY_INTEGRATION_TESTS_API_URL` to use a different Scrapely API environment. It defaults to `https://api.scrape.ly`; for test accounts on staging, set it to `https://api-staging.scrape.ly`
|
|
107
109
|
|
|
108
110
|
To run integration tests:
|
|
109
111
|
|
|
@@ -115,19 +117,15 @@ uv run poe integration-tests
|
|
|
115
117
|
|
|
116
118
|
We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/).
|
|
117
119
|
|
|
118
|
-
|
|
120
|
+
The user guide for this client is published at https://scrape.ly/docs/client-python. The Markdown files in the `docs/` folder are copied to the public repository [scrapely-app/scrapely-client-python](https://github.com/scrapely-app/scrapely-client-python) with each release and are read there.
|
|
119
121
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
```sh
|
|
123
|
-
uv run poe run-docs
|
|
124
|
-
```
|
|
122
|
+
The `website/` folder holds a [Docusaurus](https://docusaurus.io/) setup that would render `docs/` together with an API reference generated from the docstrings by [pydoc-markdown](https://pypi.org/project/pydoc-markdown/). It is not published anywhere, and `uv run poe run-docs` / `build-docs` do not work at the moment: the site depends on the `@scrapely/docs-theme` npm package, which has not been published.
|
|
125
123
|
|
|
126
124
|
### Linting the docs and website
|
|
127
125
|
|
|
128
126
|
Markdown content (this guide, `README.md`, and the `docs/` folder) is checked with
|
|
129
127
|
[markdownlint](https://github.com/DavidAnson/markdownlint). The Docusaurus website code is linted
|
|
130
|
-
with [oxlint](https://oxc.rs/) and formatted with [oxfmt](https://oxc.rs/).
|
|
128
|
+
with [oxlint](https://oxc.rs/) and formatted with [oxfmt](https://oxc.rs/). None of them run in CI at the moment.
|
|
131
129
|
To run them locally (requires Node.js 22.12 or newer and pnpm), from the `website/` directory:
|
|
132
130
|
|
|
133
131
|
```sh
|
|
@@ -141,7 +139,7 @@ Doc images are committed as optimized `.webp`. To convert a new image, run
|
|
|
141
139
|
|
|
142
140
|
## Commits
|
|
143
141
|
|
|
144
|
-
We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages.
|
|
142
|
+
We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages.
|
|
145
143
|
|
|
146
144
|
### Available commit types
|
|
147
145
|
|
|
@@ -161,31 +159,14 @@ We use [Conventional Commits](https://www.conventionalcommits.org/) format for c
|
|
|
161
159
|
|
|
162
160
|
## Release process
|
|
163
161
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
- **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI.
|
|
167
|
-
- **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overridden using the `custom` release type.
|
|
168
|
-
|
|
169
|
-
### Publishing to PyPI manually
|
|
170
|
-
|
|
171
|
-
1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `release` workflow instead.
|
|
172
|
-
2. **Make sure you know what you're doing.**
|
|
173
|
-
3. Update the version number by modifying the `version` field under `project` in `pyproject.toml`:
|
|
174
|
-
|
|
175
|
-
```toml
|
|
176
|
-
[project]
|
|
177
|
-
name = "scrapely-client-python"
|
|
178
|
-
version = "x.z.y"
|
|
179
|
-
```
|
|
162
|
+
This package lives in the `scrapely-client-python/` folder of the Scrapely monorepo. Changes go to the `staging` branch first and reach `main` when staging is promoted.
|
|
180
163
|
|
|
181
|
-
|
|
164
|
+
Publishing to [PyPI](https://pypi.org/project/scrapely-client-python/) is automated by the monorepo's `publish-packages.yml` GitHub Actions workflow. When a push to `main` changes this folder and the `version` in `pyproject.toml` is newer than the version on PyPI, the workflow builds the package with `uv build`, publishes it, and copies `src/`, `docs/`, `pyproject.toml`, `uv.lock`, `.editorconfig`, `README.md`, `CHANGELOG.md`, `LICENSE` and `NOTICE` to [scrapely-app/scrapely-client-python](https://github.com/scrapely-app/scrapely-client-python). There are no beta releases.
|
|
182
165
|
|
|
183
|
-
|
|
184
|
-
uv run poe build
|
|
185
|
-
```
|
|
166
|
+
To release a new version:
|
|
186
167
|
|
|
187
|
-
|
|
168
|
+
1. Bump `version` under `[project]` in `pyproject.toml`.
|
|
169
|
+
2. Add an entry for the new version at the top of `CHANGELOG.md`.
|
|
170
|
+
3. Merge the change to `staging`. It is published when it reaches `main`.
|
|
188
171
|
|
|
189
|
-
|
|
190
|
-
uv publish --token YOUR_API_TOKEN
|
|
191
|
-
```
|
|
172
|
+
The README on PyPI is the one shipped with the latest published version, so README and docs changes reach PyPI and the public repository only with the next version bump.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapely-client-python
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.10
|
|
4
4
|
Summary: Scrapely API client for Python
|
|
5
5
|
Keywords: scrapely,api,client,automation,crawling,scraping
|
|
6
6
|
Author: TeraShift GmbH (Scrape.ly)
|
|
@@ -36,7 +36,7 @@ Description-Content-Type: text/markdown
|
|
|
36
36
|
<h1 align="center">Scrapely API client for Python</h1>
|
|
37
37
|
|
|
38
38
|
<p align="center">
|
|
39
|
-
<strong>The official Python client for the <a href="https://
|
|
39
|
+
<strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
|
|
40
40
|
</p>
|
|
41
41
|
|
|
42
42
|
<p align="center">
|
|
@@ -46,9 +46,9 @@ Description-Content-Type: text/markdown
|
|
|
46
46
|
<a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
|
|
47
47
|
</p>
|
|
48
48
|
|
|
49
|
-
`scrapely-client-python` lets you talk to the [Scrapely platform](https://
|
|
49
|
+
`scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
|
|
50
50
|
|
|
51
|
-
> If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
|
|
51
|
+
> If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
|
|
52
52
|
|
|
53
53
|
## Table of contents
|
|
54
54
|
|
|
@@ -92,7 +92,9 @@ Description-Content-Type: text/markdown
|
|
|
92
92
|
|
|
93
93
|
## Quick start
|
|
94
94
|
|
|
95
|
-
You'll need a Scrapely API token —
|
|
95
|
+
You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
|
|
96
|
+
|
|
97
|
+
The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
|
|
96
98
|
|
|
97
99
|
### Synchronous client
|
|
98
100
|
|
|
@@ -190,6 +192,21 @@ for item in client.dataset('DATASET-ID').iterate_items(fields=['title', 'url']):
|
|
|
190
192
|
process(item)
|
|
191
193
|
```
|
|
192
194
|
|
|
195
|
+
### Create and run an automation
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
automation = client.automations().create({
|
|
199
|
+
'title': 'Daily dentists',
|
|
200
|
+
'trigger': {'kind': 'schedule', 'cron': '0 9 * * *', 'timezone': 'Europe/Berlin'},
|
|
201
|
+
'steps': [{'kind': 'savedInput', 'savedInputId': 'SAVED-INPUT-ID'}],
|
|
202
|
+
'ai': {'mode': 'summarize', 'prompt': 'List the new practices with their phone numbers.'},
|
|
203
|
+
'deliveries': [{'channel': 'email'}],
|
|
204
|
+
})
|
|
205
|
+
run = client.automation(automation['id']).run()
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Automations take and return the API's camelCase dictionaries (see the automations section of the API reference).
|
|
209
|
+
|
|
193
210
|
### Tune retries and timeouts
|
|
194
211
|
|
|
195
212
|
```python
|
|
@@ -205,33 +222,33 @@ client = ScrapelyClient(
|
|
|
205
222
|
)
|
|
206
223
|
```
|
|
207
224
|
|
|
208
|
-
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the
|
|
225
|
+
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
|
|
209
226
|
|
|
210
227
|
## Documentation
|
|
211
228
|
|
|
212
|
-
The
|
|
229
|
+
The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
|
|
230
|
+
|
|
231
|
+
The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
|
|
213
232
|
|
|
214
233
|
| Section | What you'll find |
|
|
215
234
|
| --- | --- |
|
|
216
|
-
| Introduction | Overview,
|
|
217
|
-
|
|
|
218
|
-
|
|
|
219
|
-
|
|
|
220
|
-
| API reference | Generated reference for every class, method, and model. |
|
|
221
|
-
| Changelog | Release history and breaking changes. |
|
|
235
|
+
| [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
|
|
236
|
+
| [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
|
|
237
|
+
| [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
|
|
238
|
+
| [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
|
|
222
239
|
|
|
223
240
|
## Related projects
|
|
224
241
|
|
|
225
|
-
- **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
242
|
+
- **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
226
243
|
- **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
|
|
227
|
-
- **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
|
|
228
|
-
- **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
|
|
244
|
+
- **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
|
|
245
|
+
- **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
|
|
229
246
|
- **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
|
|
230
|
-
- **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
247
|
+
- **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
231
248
|
|
|
232
249
|
## Support and community
|
|
233
250
|
|
|
234
|
-
- **GitHub issues** — report a bug or request a feature in the
|
|
251
|
+
- **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
|
|
235
252
|
|
|
236
253
|
## Contributing
|
|
237
254
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
<h1 align="center">Scrapely API client for Python</h1>
|
|
4
4
|
|
|
5
5
|
<p align="center">
|
|
6
|
-
<strong>The official Python client for the <a href="https://
|
|
6
|
+
<strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
|
|
7
7
|
</p>
|
|
8
8
|
|
|
9
9
|
<p align="center">
|
|
@@ -13,9 +13,9 @@
|
|
|
13
13
|
<a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
`scrapely-client-python` lets you talk to the [Scrapely platform](https://
|
|
16
|
+
`scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
|
|
17
17
|
|
|
18
|
-
> If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
|
|
18
|
+
> If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
|
|
19
19
|
|
|
20
20
|
## Table of contents
|
|
21
21
|
|
|
@@ -59,7 +59,9 @@
|
|
|
59
59
|
|
|
60
60
|
## Quick start
|
|
61
61
|
|
|
62
|
-
You'll need a Scrapely API token —
|
|
62
|
+
You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
|
|
63
|
+
|
|
64
|
+
The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
|
|
63
65
|
|
|
64
66
|
### Synchronous client
|
|
65
67
|
|
|
@@ -157,6 +159,21 @@ for item in client.dataset('DATASET-ID').iterate_items(fields=['title', 'url']):
|
|
|
157
159
|
process(item)
|
|
158
160
|
```
|
|
159
161
|
|
|
162
|
+
### Create and run an automation
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
automation = client.automations().create({
|
|
166
|
+
'title': 'Daily dentists',
|
|
167
|
+
'trigger': {'kind': 'schedule', 'cron': '0 9 * * *', 'timezone': 'Europe/Berlin'},
|
|
168
|
+
'steps': [{'kind': 'savedInput', 'savedInputId': 'SAVED-INPUT-ID'}],
|
|
169
|
+
'ai': {'mode': 'summarize', 'prompt': 'List the new practices with their phone numbers.'},
|
|
170
|
+
'deliveries': [{'channel': 'email'}],
|
|
171
|
+
})
|
|
172
|
+
run = client.automation(automation['id']).run()
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Automations take and return the API's camelCase dictionaries (see the automations section of the API reference).
|
|
176
|
+
|
|
160
177
|
### Tune retries and timeouts
|
|
161
178
|
|
|
162
179
|
```python
|
|
@@ -172,33 +189,33 @@ client = ScrapelyClient(
|
|
|
172
189
|
)
|
|
173
190
|
```
|
|
174
191
|
|
|
175
|
-
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the
|
|
192
|
+
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
|
|
176
193
|
|
|
177
194
|
## Documentation
|
|
178
195
|
|
|
179
|
-
The
|
|
196
|
+
The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
|
|
197
|
+
|
|
198
|
+
The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
|
|
180
199
|
|
|
181
200
|
| Section | What you'll find |
|
|
182
201
|
| --- | --- |
|
|
183
|
-
| Introduction | Overview,
|
|
184
|
-
|
|
|
185
|
-
|
|
|
186
|
-
|
|
|
187
|
-
| API reference | Generated reference for every class, method, and model. |
|
|
188
|
-
| Changelog | Release history and breaking changes. |
|
|
202
|
+
| [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
|
|
203
|
+
| [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
|
|
204
|
+
| [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
|
|
205
|
+
| [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
|
|
189
206
|
|
|
190
207
|
## Related projects
|
|
191
208
|
|
|
192
|
-
- **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
209
|
+
- **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
193
210
|
- **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
|
|
194
|
-
- **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
|
|
195
|
-
- **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
|
|
211
|
+
- **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
|
|
212
|
+
- **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
|
|
196
213
|
- **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
|
|
197
|
-
- **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
214
|
+
- **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
198
215
|
|
|
199
216
|
## Support and community
|
|
200
217
|
|
|
201
|
-
- **GitHub issues** — report a bug or request a feature in the
|
|
218
|
+
- **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
|
|
202
219
|
|
|
203
220
|
## Contributing
|
|
204
221
|
|
|
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapely-client-python"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.10"
|
|
8
8
|
description = "Scrapely API client for Python"
|
|
9
9
|
license = "Apache-2.0"
|
|
10
10
|
license-files = [
|
|
@@ -282,7 +282,6 @@ crawlee = false
|
|
|
282
282
|
clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
|
|
283
283
|
install-sync = "uv sync --all-extras"
|
|
284
284
|
build = "uv build"
|
|
285
|
-
publish-to-pypi = "uv publish --token ${SCRAPELY_PYPI_TOKEN}"
|
|
286
285
|
type-check = "uv run ty check"
|
|
287
286
|
unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit"
|
|
288
287
|
unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/scrapely_client --cov-report=xml:coverage-unit.xml tests/unit"
|
|
@@ -297,6 +296,12 @@ check-code = [
|
|
|
297
296
|
"unit-tests",
|
|
298
297
|
]
|
|
299
298
|
|
|
299
|
+
[tool.poe.tasks.publish-to-pypi]
|
|
300
|
+
cmd = "uv publish"
|
|
301
|
+
|
|
302
|
+
[tool.poe.tasks.publish-to-pypi.env]
|
|
303
|
+
UV_PUBLISH_TOKEN = "${SCRAPELY_PYPI_TOKEN}"
|
|
304
|
+
|
|
300
305
|
[tool.poe.tasks.install-dev]
|
|
301
306
|
shell = "uv sync --all-extras && uv run pre-commit install"
|
|
302
307
|
|
|
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapely-client-python"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.10"
|
|
8
8
|
description = "Scrapely API client for Python"
|
|
9
9
|
authors = [{ name = "TeraShift GmbH (Scrape.ly)" }]
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -267,7 +267,7 @@ crawlee = false
|
|
|
267
267
|
clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
|
|
268
268
|
install-sync = "uv sync --all-extras"
|
|
269
269
|
build = "uv build"
|
|
270
|
-
publish-to-pypi = "uv publish
|
|
270
|
+
publish-to-pypi = { cmd = "uv publish", env = { UV_PUBLISH_TOKEN = "${SCRAPELY_PYPI_TOKEN}" } }
|
|
271
271
|
type-check = "uv run ty check"
|
|
272
272
|
unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit"
|
|
273
273
|
unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/scrapely_client --cov-report=xml:coverage-unit.xml tests/unit"
|
{scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_consts.py
RENAMED
|
@@ -7,6 +7,9 @@ from datetime import timedelta
|
|
|
7
7
|
DEFAULT_API_URL = 'https://api.scrape.ly'
|
|
8
8
|
"""Default base URL for the Scrapely API."""
|
|
9
9
|
|
|
10
|
+
API_URL_ENV_VAR = 'SCRAPELY_API_URL'
|
|
11
|
+
"""Environment variable that sets the API URL when `api_url` is not passed to the client."""
|
|
12
|
+
|
|
10
13
|
API_VERSION = 'v2'
|
|
11
14
|
"""Current Scrapely API version."""
|
|
12
15
|
|
{scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_logging.py
RENAMED
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import functools
|
|
6
6
|
import inspect
|
|
7
7
|
import logging
|
|
8
|
+
import re
|
|
8
9
|
import threading
|
|
9
10
|
from contextvars import ContextVar
|
|
10
11
|
from typing import TYPE_CHECKING, Any, NamedTuple
|
|
@@ -25,6 +26,9 @@ logger_name = __name__.split('.')[0]
|
|
|
25
26
|
logger = logging.getLogger(logger_name)
|
|
26
27
|
"""Logger used throughout the library."""
|
|
27
28
|
|
|
29
|
+
# C0 controls except tab and newline, DEL, and C1 controls (incl. 8-bit CSI).
|
|
30
|
+
_CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\x9f]')
|
|
31
|
+
|
|
28
32
|
|
|
29
33
|
class LogContext(NamedTuple):
|
|
30
34
|
"""Request context details for logging (attempt, client method, HTTP method, resource ID, URL)."""
|
|
@@ -101,9 +105,11 @@ class RedirectLogFormatter(logging.Formatter):
|
|
|
101
105
|
Returns:
|
|
102
106
|
Formatted log message with colored logger name prefix.
|
|
103
107
|
"""
|
|
104
|
-
formatted_logger_name = f'{Fore.CYAN}[{record.name}]{Style.RESET_ALL}'
|
|
105
|
-
|
|
106
|
-
|
|
108
|
+
formatted_logger_name = f'{Fore.CYAN}[{_CONTROL_CHARS_RE.sub("", record.name)}]{Style.RESET_ALL}'
|
|
109
|
+
# Redirected messages come from another Actor's log, so escape sequences in it are removed
|
|
110
|
+
# and every line gets the prefix, so a line in it can't pass for one of our own.
|
|
111
|
+
formatted_message = _CONTROL_CHARS_RE.sub('', super().format(record).replace('\r\n', '\n').replace('\r', '\n'))
|
|
112
|
+
return '\n'.join(f'{formatted_logger_name} -> {msg_line}' for msg_line in formatted_message.split('\n'))
|
|
107
113
|
|
|
108
114
|
|
|
109
115
|
def create_redirect_logger(name: str) -> logging.Logger:
|
{scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_models.py
RENAMED
|
@@ -1246,7 +1246,7 @@ class EffectivePlatformFeature(BaseModel):
|
|
|
1246
1246
|
str | None,
|
|
1247
1247
|
Field(
|
|
1248
1248
|
examples=[
|
|
1249
|
-
'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact
|
|
1249
|
+
'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact hello@scrape.ly'
|
|
1250
1250
|
]
|
|
1251
1251
|
),
|
|
1252
1252
|
]
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import TYPE_CHECKING, Any, Literal
|
|
6
|
+
from uuid import uuid4
|
|
6
7
|
|
|
7
8
|
from pydantic import TypeAdapter
|
|
8
9
|
|
|
@@ -287,7 +288,8 @@ class ActorClient(ResourceClient):
|
|
|
287
288
|
response = self._http_client.call(
|
|
288
289
|
url=self._build_url('runs'),
|
|
289
290
|
method='POST',
|
|
290
|
-
|
|
291
|
+
# The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
|
|
292
|
+
headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
|
|
291
293
|
data=run_input,
|
|
292
294
|
params=request_params,
|
|
293
295
|
timeout=timeout,
|
|
@@ -418,6 +420,8 @@ class ActorClient(ResourceClient):
|
|
|
418
420
|
response = self._http_client.call(
|
|
419
421
|
url=self._build_url('builds'),
|
|
420
422
|
method='POST',
|
|
423
|
+
# One build per key (same contract as run starts), so a resent build start is safe.
|
|
424
|
+
headers={'idempotency-key': uuid4().hex},
|
|
421
425
|
params=request_params,
|
|
422
426
|
timeout=timeout,
|
|
423
427
|
)
|
|
@@ -783,7 +787,8 @@ class ActorClientAsync(ResourceClientAsync):
|
|
|
783
787
|
response = await self._http_client.call(
|
|
784
788
|
url=self._build_url('runs'),
|
|
785
789
|
method='POST',
|
|
786
|
-
|
|
790
|
+
# The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
|
|
791
|
+
headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
|
|
787
792
|
data=run_input,
|
|
788
793
|
params=request_params,
|
|
789
794
|
timeout=timeout,
|
|
@@ -918,6 +923,8 @@ class ActorClientAsync(ResourceClientAsync):
|
|
|
918
923
|
response = await self._http_client.call(
|
|
919
924
|
url=self._build_url('builds'),
|
|
920
925
|
method='POST',
|
|
926
|
+
# One build per key (same contract as run starts), so a resent build start is safe.
|
|
927
|
+
headers={'idempotency-key': uuid4().hex},
|
|
921
928
|
params=request_params,
|
|
922
929
|
timeout=timeout,
|
|
923
930
|
)
|