scrapely-client-python 1.0.8__tar.gz → 1.0.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/CHANGELOG.md +11 -0
  2. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/CONTRIBUTING.md +17 -36
  3. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/PKG-INFO +35 -18
  4. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/README.md +34 -17
  5. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/pyproject.toml +7 -2
  6. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/pyproject.toml.orig +2 -2
  7. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_consts.py +3 -0
  8. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_logging.py +9 -3
  9. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_models.py +1 -1
  10. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor.py +9 -2
  11. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/dataset.py +61 -29
  12. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/key_value_store.py +41 -21
  13. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/task.py +5 -2
  14. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_scrapely_client.py +25 -14
  15. scrapely_client_python-1.0.10/src/scrapely_client/_utils/crypto.py +161 -0
  16. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/_impit.py +89 -4
  17. scrapely_client_python-1.0.8/src/scrapely_client/_utils/crypto.py +0 -89
  18. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/LICENSE +0 -0
  19. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/NOTICE +0 -0
  20. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/__init__.py +0 -0
  21. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_charging.py +0 -0
  22. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_client_registry.py +0 -0
  23. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_docs.py +0 -0
  24. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_literals.py +0 -0
  25. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_pagination.py +0 -0
  26. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/__init__.py +0 -0
  27. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/_resource_client.py +0 -0
  28. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_collection.py +0 -0
  29. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_env_var.py +0 -0
  30. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_env_var_collection.py +0 -0
  31. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_version.py +0 -0
  32. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_version_collection.py +0 -0
  33. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/automation.py +0 -0
  34. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/automation_collection.py +0 -0
  35. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/build.py +0 -0
  36. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/build_collection.py +0 -0
  37. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/dataset_collection.py +0 -0
  38. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/key_value_store_collection.py +0 -0
  39. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/log.py +0 -0
  40. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/request_queue.py +0 -0
  41. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/request_queue_collection.py +0 -0
  42. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/run.py +0 -0
  43. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/run_collection.py +0 -0
  44. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/schedule.py +0 -0
  45. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/schedule_collection.py +0 -0
  46. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/store_collection.py +0 -0
  47. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/task_collection.py +0 -0
  48. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/user.py +0 -0
  49. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook.py +0 -0
  50. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_collection.py +0 -0
  51. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_dispatch.py +0 -0
  52. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_dispatch_collection.py +0 -0
  53. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_statistics.py +0 -0
  54. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_status_message_watcher.py +0 -0
  55. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_streamed_log.py +0 -0
  56. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_typeddicts.py +0 -0
  57. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/__init__.py +0 -0
  58. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/encoding.py +0 -0
  59. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/errors.py +0 -0
  60. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/http.py +0 -0
  61. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/time.py +0 -0
  62. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/try_import.py +0 -0
  63. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/errors.py +0 -0
  64. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/__init__.py +0 -0
  65. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/_base.py +0 -0
  66. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/__init__.py +0 -0
  67. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_base.py +0 -0
  68. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_brotli.py +0 -0
  69. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_gzip.py +0 -0
  70. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_resolve.py +0 -0
  71. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/py.typed +0 -0
  72. {scrapely_client_python-1.0.8 → scrapely_client_python-1.0.10}/src/scrapely_client/types.py +0 -0
@@ -2,6 +2,17 @@
2
2
 
3
3
  All notable changes to this project will be documented in this file.
4
4
 
5
+ ## [1.0.10] - 2026-11-02
6
+
7
+ - Run starts (`ActorClient.start` / `call`, `TaskClient.start` / `call`), `ActorClient.build` and `DatasetClient.push_items` send an `Idempotency-Key` header, so a request resent after a 5xx or a dropped connection starts one run or build and stores the items once.
8
+ - Any other POST or PATCH without an `Idempotency-Key` is no longer resent after a 5xx or a connection that dropped mid-request, since it may already have been applied. Rate limits (429), failures before the request was sent, request queue adds (de-duplicated by `uniqueKey`), request unlocks, head locks, run aborts and get-or-create of a storage by name are still retried.
9
+ - A keyed request answered with 409 `idempotency-key-in-progress` (the first request with that key is still being applied) is resent until it gets that request's answer; a 503 `push-outcome-unknown` is not resent, since no resend can change it.
10
+
11
+ ## [1.0.9] - 2026-10-24
12
+
13
+ - The `SCRAPELY_API_URL` environment variable sets the API URL when `api_url` is not passed to `ScrapelyClient`, `ScrapelyClientAsync` or `with_custom_http_client`; `api_url` still wins, and without either the client uses `https://api.scrape.ly`. An empty value is ignored. `api_public_url` is unchanged.
14
+ - Version 1.0.3 defaulted to `https://api-staging.scrape.ly`; upgrade to 1.0.6 or later.
15
+
5
16
  ## [1.0.8] - 2026-10-20
6
17
 
7
18
  - Request ids are percent-encoded as one URL path segment in `get_request`, `update_request`, `delete_request`, `prolong_request_lock` and `delete_request_lock` (sync and async); `''`, `.` and `..` raise `ValueError`. `update_request` checks the id against the API's grammar (1-128 letters, digits, `-`, `_`, `~`) and refuses a request without an id instead of sending `requests/None`. Resource ids (`client.dataset(id)`, versions, env vars) and automation run ids are encoded the same way. An id such as `..` used to send the call to another resource.
@@ -18,7 +18,7 @@ All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be ru
18
18
  | Task | Description |
19
19
  | ---- | ----------- |
20
20
  | `install-dev` | Install development dependencies |
21
- | `check-code` | Run lint, type-check, unit-tests, and docstring check |
21
+ | `check-code` | Run lint, type-check, docstring check, and unit tests |
22
22
  | `lint` | Run linter |
23
23
  | `format` | Fix lint issues and format code |
24
24
  | `type-check` | Run type checker |
@@ -28,8 +28,10 @@ All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be ru
28
28
  | `integration-tests-cov` | Run integration tests with coverage |
29
29
  | `check-docstrings` | Check async client docstrings |
30
30
  | `fix-docstrings` | Fix async client docstrings |
31
- | `build-docs` | Build documentation website |
32
- | `run-docs` | Run documentation website locally |
31
+ | `build-docs` | Build documentation website (does not work yet, see [Documentation](#documentation)) |
32
+ | `run-docs` | Run documentation website locally (does not work yet, see [Documentation](#documentation)) |
33
+ | `generate-models` | Regenerate the models from `openapi/scrapely-api.json` |
34
+ | `accept-openapi-spec` | Record the hash of a reviewed change to `openapi/scrapely-api.json` |
33
35
  | `build` | Build package |
34
36
  | `clean` | Remove build artifacts and clean caches |
35
37
 
@@ -103,7 +105,7 @@ Prerequisites:
103
105
 
104
106
  - Set `SCRAPELY_TEST_USER_API_TOKEN` to your Scrapely API token
105
107
  - Set `SCRAPELY_TEST_USER_2_API_TOKEN` for storage access tests (requires two accounts)
106
- - Optionally set `SCRAPELY_INTEGRATION_TESTS_API_URL` to use a different Scrapely API environment
108
+ - Optionally set `SCRAPELY_INTEGRATION_TESTS_API_URL` to use a different Scrapely API environment. It defaults to `https://api.scrape.ly`; for test accounts on staging, set it to `https://api-staging.scrape.ly`
107
109
 
108
110
  To run integration tests:
109
111
 
@@ -115,19 +117,15 @@ uv run poe integration-tests
115
117
 
116
118
  We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/).
117
119
 
118
- Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages.
120
+ The user guide for this client is published at https://scrape.ly/docs/client-python. The Markdown files in the `docs/` folder are copied to the public repository [scrapely-app/scrapely-client-python](https://github.com/scrapely-app/scrapely-client-python) with each release and are read there.
119
121
 
120
- To run the documentation locally (requires Node.js):
121
-
122
- ```sh
123
- uv run poe run-docs
124
- ```
122
+ The `website/` folder holds a [Docusaurus](https://docusaurus.io/) setup that would render `docs/` together with an API reference generated from the docstrings by [pydoc-markdown](https://pypi.org/project/pydoc-markdown/). It is not published anywhere, and `uv run poe run-docs` / `build-docs` do not work at the moment: the site depends on the `@scrapely/docs-theme` npm package, which has not been published.
125
123
 
126
124
  ### Linting the docs and website
127
125
 
128
126
  Markdown content (this guide, `README.md`, and the `docs/` folder) is checked with
129
127
  [markdownlint](https://github.com/DavidAnson/markdownlint). The Docusaurus website code is linted
130
- with [oxlint](https://oxc.rs/) and formatted with [oxfmt](https://oxc.rs/). All of them run in CI.
128
+ with [oxlint](https://oxc.rs/) and formatted with [oxfmt](https://oxc.rs/). None of them run in CI at the moment.
131
129
  To run them locally (requires Node.js 22.12 or newer and pnpm), from the `website/` directory:
132
130
 
133
131
  ```sh
@@ -141,7 +139,7 @@ Doc images are committed as optimized `.webp`. To convert a new image, run
141
139
 
142
140
  ## Commits
143
141
 
144
- We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages. This convention is used to automatically determine version bumps during the release process.
142
+ We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages.
145
143
 
146
144
  ### Available commit types
147
145
 
@@ -161,31 +159,14 @@ We use [Conventional Commits](https://www.conventionalcommits.org/) format for c
161
159
 
162
160
  ## Release process
163
161
 
164
- Publishing new versions to [PyPI](https://pypi.org/project/scrapely-client-python/) is automated through GitHub Actions.
165
-
166
- - **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI.
167
- - **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overridden using the `custom` release type.
168
-
169
- ### Publishing to PyPI manually
170
-
171
- 1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `release` workflow instead.
172
- 2. **Make sure you know what you're doing.**
173
- 3. Update the version number by modifying the `version` field under `project` in `pyproject.toml`:
174
-
175
- ```toml
176
- [project]
177
- name = "scrapely-client-python"
178
- version = "x.z.y"
179
- ```
162
+ This package lives in the `scrapely-client-python/` folder of the Scrapely monorepo. Changes go to the `staging` branch first and reach `main` when staging is promoted.
180
163
 
181
- 4. Build the package:
164
+ Publishing to [PyPI](https://pypi.org/project/scrapely-client-python/) is automated by the monorepo's `publish-packages.yml` GitHub Actions workflow. When a push to `main` changes this folder and the `version` in `pyproject.toml` is newer than the version on PyPI, the workflow builds the package with `uv build`, publishes it, and copies `src/`, `docs/`, `pyproject.toml`, `uv.lock`, `.editorconfig`, `README.md`, `CHANGELOG.md`, `LICENSE` and `NOTICE` to [scrapely-app/scrapely-client-python](https://github.com/scrapely-app/scrapely-client-python). There are no beta releases.
182
165
 
183
- ```sh
184
- uv run poe build
185
- ```
166
+ To release a new version:
186
167
 
187
- 5. Upload to PyPI:
168
+ 1. Bump `version` under `[project]` in `pyproject.toml`.
169
+ 2. Add an entry for the new version at the top of `CHANGELOG.md`.
170
+ 3. Merge the change to `staging`. It is published when it reaches `main`.
188
171
 
189
- ```sh
190
- uv publish --token YOUR_API_TOKEN
191
- ```
172
+ The README on PyPI is the one shipped with the latest published version, so README and docs changes reach PyPI and the public repository only with the next version bump.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-client-python
3
- Version: 1.0.8
3
+ Version: 1.0.10
4
4
  Summary: Scrapely API client for Python
5
5
  Keywords: scrapely,api,client,automation,crawling,scraping
6
6
  Author: TeraShift GmbH (Scrape.ly)
@@ -36,7 +36,7 @@ Description-Content-Type: text/markdown
36
36
  <h1 align="center">Scrapely API client for Python</h1>
37
37
 
38
38
  <p align="center">
39
- <strong>The official Python client for the <a href="https://api.scrape.ly">Scrapely REST API</a>.</strong>
39
+ <strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
40
40
  </p>
41
41
 
42
42
  <p align="center">
@@ -46,9 +46,9 @@ Description-Content-Type: text/markdown
46
46
  <a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
47
47
  </p>
48
48
 
49
- `scrapely-client-python` lets you talk to the [Scrapely platform](https://api.scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
49
+ `scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
50
50
 
51
- > If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
51
+ > If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
52
52
 
53
53
  ## Table of contents
54
54
 
@@ -92,7 +92,9 @@ Description-Content-Type: text/markdown
92
92
 
93
93
  ## Quick start
94
94
 
95
- You'll need a Scrapely API token — find yours in the integrations section of the Scrapely dashboard. Pass it to the client and you're ready to go.
95
+ You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
96
+
97
+ The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
96
98
 
97
99
  ### Synchronous client
98
100
 
@@ -190,6 +192,21 @@ for item in client.dataset('DATASET-ID').iterate_items(fields=['title', 'url']):
190
192
  process(item)
191
193
  ```
192
194
 
195
+ ### Create and run an automation
196
+
197
+ ```python
198
+ automation = client.automations().create({
199
+ 'title': 'Daily dentists',
200
+ 'trigger': {'kind': 'schedule', 'cron': '0 9 * * *', 'timezone': 'Europe/Berlin'},
201
+ 'steps': [{'kind': 'savedInput', 'savedInputId': 'SAVED-INPUT-ID'}],
202
+ 'ai': {'mode': 'summarize', 'prompt': 'List the new practices with their phone numbers.'},
203
+ 'deliveries': [{'channel': 'email'}],
204
+ })
205
+ run = client.automation(automation['id']).run()
206
+ ```
207
+
208
+ Automations take and return the API's camelCase dictionaries (see the automations section of the API reference).
209
+
193
210
  ### Tune retries and timeouts
194
211
 
195
212
  ```python
@@ -205,33 +222,33 @@ client = ScrapelyClient(
205
222
  )
206
223
  ```
207
224
 
208
- For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the Guides.
225
+ For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
209
226
 
210
227
  ## Documentation
211
228
 
212
- The full documentation lives at **https://api.scrape.ly**.
229
+ The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
230
+
231
+ The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
213
232
 
214
233
  | Section | What you'll find |
215
234
  | --- | --- |
216
- | Introduction | Overview, prerequisites, and a tour of the client. |
217
- | Quick start | Authenticate, run an Actor, and fetch its results step by step. |
218
- | Concepts | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. |
219
- | Guides | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
220
- | API reference | Generated reference for every class, method, and model. |
221
- | Changelog | Release history and breaking changes. |
235
+ | [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
236
+ | [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
237
+ | [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
238
+ | [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
222
239
 
223
240
  ## Related projects
224
241
 
225
- - **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
242
+ - **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
226
243
  - **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
227
- - **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
228
- - **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
244
+ - **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
245
+ - **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
229
246
  - **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
230
- - **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
247
+ - **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
231
248
 
232
249
  ## Support and community
233
250
 
234
- - **GitHub issues** — report a bug or request a feature in the repository's issue tracker.
251
+ - **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
235
252
 
236
253
  ## Contributing
237
254
 
@@ -3,7 +3,7 @@
3
3
  <h1 align="center">Scrapely API client for Python</h1>
4
4
 
5
5
  <p align="center">
6
- <strong>The official Python client for the <a href="https://api.scrape.ly">Scrapely REST API</a>.</strong>
6
+ <strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
7
7
  </p>
8
8
 
9
9
  <p align="center">
@@ -13,9 +13,9 @@
13
13
  <a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
14
14
  </p>
15
15
 
16
- `scrapely-client-python` lets you talk to the [Scrapely platform](https://api.scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
16
+ `scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
17
17
 
18
- > If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
18
+ > If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
19
19
 
20
20
  ## Table of contents
21
21
 
@@ -59,7 +59,9 @@
59
59
 
60
60
  ## Quick start
61
61
 
62
- You'll need a Scrapely API token — find yours in the integrations section of the Scrapely dashboard. Pass it to the client and you're ready to go.
62
+ You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
63
+
64
+ The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
63
65
 
64
66
  ### Synchronous client
65
67
 
@@ -157,6 +159,21 @@ for item in client.dataset('DATASET-ID').iterate_items(fields=['title', 'url']):
157
159
  process(item)
158
160
  ```
159
161
 
162
+ ### Create and run an automation
163
+
164
+ ```python
165
+ automation = client.automations().create({
166
+ 'title': 'Daily dentists',
167
+ 'trigger': {'kind': 'schedule', 'cron': '0 9 * * *', 'timezone': 'Europe/Berlin'},
168
+ 'steps': [{'kind': 'savedInput', 'savedInputId': 'SAVED-INPUT-ID'}],
169
+ 'ai': {'mode': 'summarize', 'prompt': 'List the new practices with their phone numbers.'},
170
+ 'deliveries': [{'channel': 'email'}],
171
+ })
172
+ run = client.automation(automation['id']).run()
173
+ ```
174
+
175
+ Automations take and return the API's camelCase dictionaries (see the automations section of the API reference).
176
+
160
177
  ### Tune retries and timeouts
161
178
 
162
179
  ```python
@@ -172,33 +189,33 @@ client = ScrapelyClient(
172
189
  )
173
190
  ```
174
191
 
175
- For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the Guides.
192
+ For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
176
193
 
177
194
  ## Documentation
178
195
 
179
- The full documentation lives at **https://api.scrape.ly**.
196
+ The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
197
+
198
+ The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
180
199
 
181
200
  | Section | What you'll find |
182
201
  | --- | --- |
183
- | Introduction | Overview, prerequisites, and a tour of the client. |
184
- | Quick start | Authenticate, run an Actor, and fetch its results step by step. |
185
- | Concepts | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. |
186
- | Guides | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
187
- | API reference | Generated reference for every class, method, and model. |
188
- | Changelog | Release history and breaking changes. |
202
+ | [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
203
+ | [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
204
+ | [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
205
+ | [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
189
206
 
190
207
  ## Related projects
191
208
 
192
- - **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
209
+ - **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
193
210
  - **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
194
- - **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
195
- - **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
211
+ - **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
212
+ - **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
196
213
  - **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
197
- - **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
214
+ - **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
198
215
 
199
216
  ## Support and community
200
217
 
201
- - **GitHub issues** — report a bug or request a feature in the repository's issue tracker.
218
+ - **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
202
219
 
203
220
  ## Contributing
204
221
 
@@ -4,7 +4,7 @@ build-backend = "uv_build"
4
4
 
5
5
  [project]
6
6
  name = "scrapely-client-python"
7
- version = "1.0.8"
7
+ version = "1.0.10"
8
8
  description = "Scrapely API client for Python"
9
9
  license = "Apache-2.0"
10
10
  license-files = [
@@ -282,7 +282,6 @@ crawlee = false
282
282
  clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
283
283
  install-sync = "uv sync --all-extras"
284
284
  build = "uv build"
285
- publish-to-pypi = "uv publish --token ${SCRAPELY_PYPI_TOKEN}"
286
285
  type-check = "uv run ty check"
287
286
  unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit"
288
287
  unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/scrapely_client --cov-report=xml:coverage-unit.xml tests/unit"
@@ -297,6 +296,12 @@ check-code = [
297
296
  "unit-tests",
298
297
  ]
299
298
 
299
+ [tool.poe.tasks.publish-to-pypi]
300
+ cmd = "uv publish"
301
+
302
+ [tool.poe.tasks.publish-to-pypi.env]
303
+ UV_PUBLISH_TOKEN = "${SCRAPELY_PYPI_TOKEN}"
304
+
300
305
  [tool.poe.tasks.install-dev]
301
306
  shell = "uv sync --all-extras && uv run pre-commit install"
302
307
 
@@ -4,7 +4,7 @@ build-backend = "uv_build"
4
4
 
5
5
  [project]
6
6
  name = "scrapely-client-python"
7
- version = "1.0.8"
7
+ version = "1.0.10"
8
8
  description = "Scrapely API client for Python"
9
9
  authors = [{ name = "TeraShift GmbH (Scrape.ly)" }]
10
10
  license = "Apache-2.0"
@@ -267,7 +267,7 @@ crawlee = false
267
267
  clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
268
268
  install-sync = "uv sync --all-extras"
269
269
  build = "uv build"
270
- publish-to-pypi = "uv publish --token ${SCRAPELY_PYPI_TOKEN}"
270
+ publish-to-pypi = { cmd = "uv publish", env = { UV_PUBLISH_TOKEN = "${SCRAPELY_PYPI_TOKEN}" } }
271
271
  type-check = "uv run ty check"
272
272
  unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit"
273
273
  unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/scrapely_client --cov-report=xml:coverage-unit.xml tests/unit"
@@ -7,6 +7,9 @@ from datetime import timedelta
7
7
  DEFAULT_API_URL = 'https://api.scrape.ly'
8
8
  """Default base URL for the Scrapely API."""
9
9
 
10
+ API_URL_ENV_VAR = 'SCRAPELY_API_URL'
11
+ """Environment variable that sets the API URL when `api_url` is not passed to the client."""
12
+
10
13
  API_VERSION = 'v2'
11
14
  """Current Scrapely API version."""
12
15
 
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import functools
6
6
  import inspect
7
7
  import logging
8
+ import re
8
9
  import threading
9
10
  from contextvars import ContextVar
10
11
  from typing import TYPE_CHECKING, Any, NamedTuple
@@ -25,6 +26,9 @@ logger_name = __name__.split('.')[0]
25
26
  logger = logging.getLogger(logger_name)
26
27
  """Logger used throughout the library."""
27
28
 
29
+ # C0 controls except tab and newline, DEL, and C1 controls (incl. 8-bit CSI).
30
+ _CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\x9f]')
31
+
28
32
 
29
33
  class LogContext(NamedTuple):
30
34
  """Request context details for logging (attempt, client method, HTTP method, resource ID, URL)."""
@@ -101,9 +105,11 @@ class RedirectLogFormatter(logging.Formatter):
101
105
  Returns:
102
106
  Formatted log message with colored logger name prefix.
103
107
  """
104
- formatted_logger_name = f'{Fore.CYAN}[{record.name}]{Style.RESET_ALL}'
105
- formatted_message = super().format(record)
106
- return f'{formatted_logger_name} -> {formatted_message}'
108
+ formatted_logger_name = f'{Fore.CYAN}[{_CONTROL_CHARS_RE.sub("", record.name)}]{Style.RESET_ALL}'
109
+ # Redirected messages come from another Actor's log, so escape sequences in it are removed
110
+ # and every line gets the prefix, so a line in it can't pass for one of our own.
111
+ formatted_message = _CONTROL_CHARS_RE.sub('', super().format(record).replace('\r\n', '\n').replace('\r', '\n'))
112
+ return '\n'.join(f'{formatted_logger_name} -> {msg_line}' for msg_line in formatted_message.split('\n'))
107
113
 
108
114
 
109
115
  def create_redirect_logger(name: str) -> logging.Logger:
@@ -1246,7 +1246,7 @@ class EffectivePlatformFeature(BaseModel):
1246
1246
  str | None,
1247
1247
  Field(
1248
1248
  examples=[
1249
- 'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact support@scrape.ly'
1249
+ 'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact hello@scrape.ly'
1250
1250
  ]
1251
1251
  ),
1252
1252
  ]
@@ -3,6 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  from typing import TYPE_CHECKING, Any, Literal
6
+ from uuid import uuid4
6
7
 
7
8
  from pydantic import TypeAdapter
8
9
 
@@ -287,7 +288,8 @@ class ActorClient(ResourceClient):
287
288
  response = self._http_client.call(
288
289
  url=self._build_url('runs'),
289
290
  method='POST',
290
- headers={'content-type': content_type},
291
+ # The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
292
+ headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
291
293
  data=run_input,
292
294
  params=request_params,
293
295
  timeout=timeout,
@@ -418,6 +420,8 @@ class ActorClient(ResourceClient):
418
420
  response = self._http_client.call(
419
421
  url=self._build_url('builds'),
420
422
  method='POST',
423
+ # One build per key (same contract as run starts), so a resent build start is safe.
424
+ headers={'idempotency-key': uuid4().hex},
421
425
  params=request_params,
422
426
  timeout=timeout,
423
427
  )
@@ -783,7 +787,8 @@ class ActorClientAsync(ResourceClientAsync):
783
787
  response = await self._http_client.call(
784
788
  url=self._build_url('runs'),
785
789
  method='POST',
786
- headers={'content-type': content_type},
790
+ # The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
791
+ headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
787
792
  data=run_input,
788
793
  params=request_params,
789
794
  timeout=timeout,
@@ -918,6 +923,8 @@ class ActorClientAsync(ResourceClientAsync):
918
923
  response = await self._http_client.call(
919
924
  url=self._build_url('builds'),
920
925
  method='POST',
926
+ # One build per key (same contract as run starts), so a resent build start is safe.
927
+ headers={'idempotency-key': uuid4().hex},
921
928
  params=request_params,
922
929
  timeout=timeout,
923
930
  )