scrapely-client-python 1.0.9__tar.gz → 1.0.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/CHANGELOG.md +6 -0
  2. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/CONTRIBUTING.md +17 -36
  3. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/PKG-INFO +18 -18
  4. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/README.md +17 -17
  5. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/pyproject.toml +7 -2
  6. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/pyproject.toml.orig +2 -2
  7. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_logging.py +9 -3
  8. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_models.py +1 -1
  9. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor.py +9 -2
  10. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/dataset.py +61 -29
  11. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/key_value_store.py +41 -21
  12. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/task.py +5 -2
  13. scrapely_client_python-1.0.10/src/scrapely_client/_utils/crypto.py +161 -0
  14. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/_impit.py +89 -4
  15. scrapely_client_python-1.0.9/src/scrapely_client/_utils/crypto.py +0 -89
  16. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/LICENSE +0 -0
  17. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/NOTICE +0 -0
  18. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/__init__.py +0 -0
  19. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_charging.py +0 -0
  20. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_client_registry.py +0 -0
  21. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_consts.py +0 -0
  22. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_docs.py +0 -0
  23. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_literals.py +0 -0
  24. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_pagination.py +0 -0
  25. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/__init__.py +0 -0
  26. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/_resource_client.py +0 -0
  27. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_collection.py +0 -0
  28. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_env_var.py +0 -0
  29. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_env_var_collection.py +0 -0
  30. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_version.py +0 -0
  31. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_version_collection.py +0 -0
  32. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/automation.py +0 -0
  33. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/automation_collection.py +0 -0
  34. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/build.py +0 -0
  35. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/build_collection.py +0 -0
  36. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/dataset_collection.py +0 -0
  37. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/key_value_store_collection.py +0 -0
  38. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/log.py +0 -0
  39. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/request_queue.py +0 -0
  40. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/request_queue_collection.py +0 -0
  41. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/run.py +0 -0
  42. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/run_collection.py +0 -0
  43. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/schedule.py +0 -0
  44. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/schedule_collection.py +0 -0
  45. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/store_collection.py +0 -0
  46. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/task_collection.py +0 -0
  47. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/user.py +0 -0
  48. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook.py +0 -0
  49. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_collection.py +0 -0
  50. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_dispatch.py +0 -0
  51. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_dispatch_collection.py +0 -0
  52. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_scrapely_client.py +0 -0
  53. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_statistics.py +0 -0
  54. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_status_message_watcher.py +0 -0
  55. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_streamed_log.py +0 -0
  56. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_typeddicts.py +0 -0
  57. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/__init__.py +0 -0
  58. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/encoding.py +0 -0
  59. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/errors.py +0 -0
  60. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/http.py +0 -0
  61. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/time.py +0 -0
  62. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/try_import.py +0 -0
  63. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/errors.py +0 -0
  64. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/__init__.py +0 -0
  65. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/_base.py +0 -0
  66. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/__init__.py +0 -0
  67. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_base.py +0 -0
  68. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_brotli.py +0 -0
  69. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_gzip.py +0 -0
  70. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_resolve.py +0 -0
  71. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/py.typed +0 -0
  72. {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/types.py +0 -0
@@ -2,6 +2,12 @@
2
2
 
3
3
  All notable changes to this project will be documented in this file.
4
4
 
5
+ ## [1.0.10] - 2026-11-02
6
+
7
+ - Run starts (`ActorClient.start` / `call`, `TaskClient.start` / `call`), `ActorClient.build` and `DatasetClient.push_items` send an `Idempotency-Key` header, so a request resent after a 5xx or a dropped connection starts one run or build and stores the items once.
8
+ - Any other POST or PATCH without an `Idempotency-Key` is no longer resent after a 5xx or a connection that dropped mid-request, since it may already have been applied. Rate limits (429), failures before the request was sent, request queue adds (de-duplicated by `uniqueKey`), request unlocks, head locks, run aborts and get-or-create of a storage by name are still retried.
9
+ - A keyed request answered with 409 `idempotency-key-in-progress` (the first request with that key is still being applied) is resent until it gets that request's answer; a 503 `push-outcome-unknown` is not resent, since no resend can change it.
10
+
5
11
  ## [1.0.9] - 2026-10-24
6
12
 
7
13
  - The `SCRAPELY_API_URL` environment variable sets the API URL when `api_url` is not passed to `ScrapelyClient`, `ScrapelyClientAsync` or `with_custom_http_client`; `api_url` still wins, and without either the client uses `https://api.scrape.ly`. An empty value is ignored. `api_public_url` is unchanged.
@@ -18,7 +18,7 @@ All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be ru
18
18
  | Task | Description |
19
19
  | ---- | ----------- |
20
20
  | `install-dev` | Install development dependencies |
21
- | `check-code` | Run lint, type-check, unit-tests, and docstring check |
21
+ | `check-code` | Run lint, type-check, docstring check, and unit tests |
22
22
  | `lint` | Run linter |
23
23
  | `format` | Fix lint issues and format code |
24
24
  | `type-check` | Run type checker |
@@ -28,8 +28,10 @@ All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be ru
28
28
  | `integration-tests-cov` | Run integration tests with coverage |
29
29
  | `check-docstrings` | Check async client docstrings |
30
30
  | `fix-docstrings` | Fix async client docstrings |
31
- | `build-docs` | Build documentation website |
32
- | `run-docs` | Run documentation website locally |
31
+ | `build-docs` | Build documentation website (does not work yet, see [Documentation](#documentation)) |
32
+ | `run-docs` | Run documentation website locally (does not work yet, see [Documentation](#documentation)) |
33
+ | `generate-models` | Regenerate the models from `openapi/scrapely-api.json` |
34
+ | `accept-openapi-spec` | Record the hash of a reviewed change to `openapi/scrapely-api.json` |
33
35
  | `build` | Build package |
34
36
  | `clean` | Remove build artifacts and clean caches |
35
37
 
@@ -103,7 +105,7 @@ Prerequisites:
103
105
 
104
106
  - Set `SCRAPELY_TEST_USER_API_TOKEN` to your Scrapely API token
105
107
  - Set `SCRAPELY_TEST_USER_2_API_TOKEN` for storage access tests (requires two accounts)
106
- - Optionally set `SCRAPELY_INTEGRATION_TESTS_API_URL` to use a different Scrapely API environment
108
+ - Optionally set `SCRAPELY_INTEGRATION_TESTS_API_URL` to use a different Scrapely API environment. It defaults to `https://api.scrape.ly`; for test accounts on staging, set it to `https://api-staging.scrape.ly`
107
109
 
108
110
  To run integration tests:
109
111
 
@@ -115,19 +117,15 @@ uv run poe integration-tests
115
117
 
116
118
  We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/).
117
119
 
118
- Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages.
120
+ The user guide for this client is published at https://scrape.ly/docs/client-python. The Markdown files in the `docs/` folder are copied to the public repository [scrapely-app/scrapely-client-python](https://github.com/scrapely-app/scrapely-client-python) with each release and are read there.
119
121
 
120
- To run the documentation locally (requires Node.js):
121
-
122
- ```sh
123
- uv run poe run-docs
124
- ```
122
+ The `website/` folder holds a [Docusaurus](https://docusaurus.io/) setup that would render `docs/` together with an API reference generated from the docstrings by [pydoc-markdown](https://pypi.org/project/pydoc-markdown/). It is not published anywhere, and `uv run poe run-docs` / `build-docs` do not work at the moment: the site depends on the `@scrapely/docs-theme` npm package, which has not been published.
125
123
 
126
124
  ### Linting the docs and website
127
125
 
128
126
  Markdown content (this guide, `README.md`, and the `docs/` folder) is checked with
129
127
  [markdownlint](https://github.com/DavidAnson/markdownlint). The Docusaurus website code is linted
130
- with [oxlint](https://oxc.rs/) and formatted with [oxfmt](https://oxc.rs/). All of them run in CI.
128
+ with [oxlint](https://oxc.rs/) and formatted with [oxfmt](https://oxc.rs/). None of them run in CI at the moment.
131
129
  To run them locally (requires Node.js 22.12 or newer and pnpm), from the `website/` directory:
132
130
 
133
131
  ```sh
@@ -141,7 +139,7 @@ Doc images are committed as optimized `.webp`. To convert a new image, run
141
139
 
142
140
  ## Commits
143
141
 
144
- We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages. This convention is used to automatically determine version bumps during the release process.
142
+ We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages.
145
143
 
146
144
  ### Available commit types
147
145
 
@@ -161,31 +159,14 @@ We use [Conventional Commits](https://www.conventionalcommits.org/) format for c
161
159
 
162
160
  ## Release process
163
161
 
164
- Publishing new versions to [PyPI](https://pypi.org/project/scrapely-client-python/) is automated through GitHub Actions.
165
-
166
- - **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI.
167
- - **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overridden using the `custom` release type.
168
-
169
- ### Publishing to PyPI manually
170
-
171
- 1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `release` workflow instead.
172
- 2. **Make sure you know what you're doing.**
173
- 3. Update the version number by modifying the `version` field under `project` in `pyproject.toml`:
174
-
175
- ```toml
176
- [project]
177
- name = "scrapely-client-python"
178
- version = "x.z.y"
179
- ```
162
+ This package lives in the `scrapely-client-python/` folder of the Scrapely monorepo. Changes go to the `staging` branch first and reach `main` when staging is promoted.
180
163
 
181
- 4. Build the package:
164
+ Publishing to [PyPI](https://pypi.org/project/scrapely-client-python/) is automated by the monorepo's `publish-packages.yml` GitHub Actions workflow. When a push to `main` changes this folder and the `version` in `pyproject.toml` is newer than the version on PyPI, the workflow builds the package with `uv build`, publishes it, and copies `src/`, `docs/`, `pyproject.toml`, `uv.lock`, `.editorconfig`, `README.md`, `CHANGELOG.md`, `LICENSE` and `NOTICE` to [scrapely-app/scrapely-client-python](https://github.com/scrapely-app/scrapely-client-python). There are no beta releases.
182
165
 
183
- ```sh
184
- uv run poe build
185
- ```
166
+ To release a new version:
186
167
 
187
- 5. Upload to PyPI:
168
+ 1. Bump `version` under `[project]` in `pyproject.toml`.
169
+ 2. Add an entry for the new version at the top of `CHANGELOG.md`.
170
+ 3. Merge the change to `staging`. It is published when it reaches `main`.
188
171
 
189
- ```sh
190
- uv publish --token YOUR_API_TOKEN
191
- ```
172
+ The README on PyPI is the one shipped with the latest published version, so README and docs changes reach PyPI and the public repository only with the next version bump.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-client-python
3
- Version: 1.0.9
3
+ Version: 1.0.10
4
4
  Summary: Scrapely API client for Python
5
5
  Keywords: scrapely,api,client,automation,crawling,scraping
6
6
  Author: TeraShift GmbH (Scrape.ly)
@@ -36,7 +36,7 @@ Description-Content-Type: text/markdown
36
36
  <h1 align="center">Scrapely API client for Python</h1>
37
37
 
38
38
  <p align="center">
39
- <strong>The official Python client for the <a href="https://api.scrape.ly">Scrapely REST API</a>.</strong>
39
+ <strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
40
40
  </p>
41
41
 
42
42
  <p align="center">
@@ -46,9 +46,9 @@ Description-Content-Type: text/markdown
46
46
  <a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
47
47
  </p>
48
48
 
49
- `scrapely-client-python` lets you talk to the [Scrapely platform](https://api.scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
49
+ `scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
50
50
 
51
- > If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
51
+ > If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
52
52
 
53
53
  ## Table of contents
54
54
 
@@ -92,7 +92,7 @@ Description-Content-Type: text/markdown
92
92
 
93
93
  ## Quick start
94
94
 
95
- You'll need a Scrapely API token — find yours in the integrations section of the Scrapely dashboard. Pass it to the client and you're ready to go.
95
+ You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
96
96
 
97
97
  The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
98
98
 
@@ -222,33 +222,33 @@ client = ScrapelyClient(
222
222
  )
223
223
  ```
224
224
 
225
- For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the Guides.
225
+ For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
226
226
 
227
227
  ## Documentation
228
228
 
229
- The full documentation lives at **https://api.scrape.ly**.
229
+ The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
230
+
231
+ The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
230
232
 
231
233
  | Section | What you'll find |
232
234
  | --- | --- |
233
- | Introduction | Overview, prerequisites, and a tour of the client. |
234
- | Quick start | Authenticate, run an Actor, and fetch its results step by step. |
235
- | Concepts | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. |
236
- | Guides | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
237
- | API reference | Generated reference for every class, method, and model. |
238
- | Changelog | Release history and breaking changes. |
235
+ | [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
236
+ | [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
237
+ | [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
238
+ | [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
239
239
 
240
240
  ## Related projects
241
241
 
242
- - **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
242
+ - **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
243
243
  - **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
244
- - **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
245
- - **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
244
+ - **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
245
+ - **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
246
246
  - **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
247
- - **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
247
+ - **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
248
248
 
249
249
  ## Support and community
250
250
 
251
- - **GitHub issues** — report a bug or request a feature in the repository's issue tracker.
251
+ - **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
252
252
 
253
253
  ## Contributing
254
254
 
@@ -3,7 +3,7 @@
3
3
  <h1 align="center">Scrapely API client for Python</h1>
4
4
 
5
5
  <p align="center">
6
- <strong>The official Python client for the <a href="https://api.scrape.ly">Scrapely REST API</a>.</strong>
6
+ <strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
7
7
  </p>
8
8
 
9
9
  <p align="center">
@@ -13,9 +13,9 @@
13
13
  <a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
14
14
  </p>
15
15
 
16
- `scrapely-client-python` lets you talk to the [Scrapely platform](https://api.scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
16
+ `scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
17
17
 
18
- > If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
18
+ > If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
19
19
 
20
20
  ## Table of contents
21
21
 
@@ -59,7 +59,7 @@
59
59
 
60
60
  ## Quick start
61
61
 
62
- You'll need a Scrapely API token — find yours in the integrations section of the Scrapely dashboard. Pass it to the client and you're ready to go.
62
+ You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
63
63
 
64
64
  The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
65
65
 
@@ -189,33 +189,33 @@ client = ScrapelyClient(
189
189
  )
190
190
  ```
191
191
 
192
- For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the Guides.
192
+ For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
193
193
 
194
194
  ## Documentation
195
195
 
196
- The full documentation lives at **https://api.scrape.ly**.
196
+ The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
197
+
198
+ The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
197
199
 
198
200
  | Section | What you'll find |
199
201
  | --- | --- |
200
- | Introduction | Overview, prerequisites, and a tour of the client. |
201
- | Quick start | Authenticate, run an Actor, and fetch its results step by step. |
202
- | Concepts | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. |
203
- | Guides | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
204
- | API reference | Generated reference for every class, method, and model. |
205
- | Changelog | Release history and breaking changes. |
202
+ | [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
203
+ | [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
204
+ | [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
205
+ | [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
206
206
 
207
207
  ## Related projects
208
208
 
209
- - **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
209
+ - **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
210
210
  - **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
211
- - **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
212
- - **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
211
+ - **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
212
+ - **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
213
213
  - **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
214
- - **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
214
+ - **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
215
215
 
216
216
  ## Support and community
217
217
 
218
- - **GitHub issues** — report a bug or request a feature in the repository's issue tracker.
218
+ - **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
219
219
 
220
220
  ## Contributing
221
221
 
@@ -4,7 +4,7 @@ build-backend = "uv_build"
4
4
 
5
5
  [project]
6
6
  name = "scrapely-client-python"
7
- version = "1.0.9"
7
+ version = "1.0.10"
8
8
  description = "Scrapely API client for Python"
9
9
  license = "Apache-2.0"
10
10
  license-files = [
@@ -282,7 +282,6 @@ crawlee = false
282
282
  clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
283
283
  install-sync = "uv sync --all-extras"
284
284
  build = "uv build"
285
- publish-to-pypi = "uv publish --token ${SCRAPELY_PYPI_TOKEN}"
286
285
  type-check = "uv run ty check"
287
286
  unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit"
288
287
  unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/scrapely_client --cov-report=xml:coverage-unit.xml tests/unit"
@@ -297,6 +296,12 @@ check-code = [
297
296
  "unit-tests",
298
297
  ]
299
298
 
299
+ [tool.poe.tasks.publish-to-pypi]
300
+ cmd = "uv publish"
301
+
302
+ [tool.poe.tasks.publish-to-pypi.env]
303
+ UV_PUBLISH_TOKEN = "${SCRAPELY_PYPI_TOKEN}"
304
+
300
305
  [tool.poe.tasks.install-dev]
301
306
  shell = "uv sync --all-extras && uv run pre-commit install"
302
307
 
@@ -4,7 +4,7 @@ build-backend = "uv_build"
4
4
 
5
5
  [project]
6
6
  name = "scrapely-client-python"
7
- version = "1.0.9"
7
+ version = "1.0.10"
8
8
  description = "Scrapely API client for Python"
9
9
  authors = [{ name = "TeraShift GmbH (Scrape.ly)" }]
10
10
  license = "Apache-2.0"
@@ -267,7 +267,7 @@ crawlee = false
267
267
  clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
268
268
  install-sync = "uv sync --all-extras"
269
269
  build = "uv build"
270
- publish-to-pypi = "uv publish --token ${SCRAPELY_PYPI_TOKEN}"
270
+ publish-to-pypi = { cmd = "uv publish", env = { UV_PUBLISH_TOKEN = "${SCRAPELY_PYPI_TOKEN}" } }
271
271
  type-check = "uv run ty check"
272
272
  unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit"
273
273
  unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/scrapely_client --cov-report=xml:coverage-unit.xml tests/unit"
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import functools
6
6
  import inspect
7
7
  import logging
8
+ import re
8
9
  import threading
9
10
  from contextvars import ContextVar
10
11
  from typing import TYPE_CHECKING, Any, NamedTuple
@@ -25,6 +26,9 @@ logger_name = __name__.split('.')[0]
25
26
  logger = logging.getLogger(logger_name)
26
27
  """Logger used throughout the library."""
27
28
 
29
+ # C0 controls except tab and newline, DEL, and C1 controls (incl. 8-bit CSI).
30
+ _CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\x9f]')
31
+
28
32
 
29
33
  class LogContext(NamedTuple):
30
34
  """Request context details for logging (attempt, client method, HTTP method, resource ID, URL)."""
@@ -101,9 +105,11 @@ class RedirectLogFormatter(logging.Formatter):
101
105
  Returns:
102
106
  Formatted log message with colored logger name prefix.
103
107
  """
104
- formatted_logger_name = f'{Fore.CYAN}[{record.name}]{Style.RESET_ALL}'
105
- formatted_message = super().format(record)
106
- return f'{formatted_logger_name} -> {formatted_message}'
108
+ formatted_logger_name = f'{Fore.CYAN}[{_CONTROL_CHARS_RE.sub("", record.name)}]{Style.RESET_ALL}'
109
+ # Redirected messages come from another Actor's log, so escape sequences in it are removed
110
+ # and every line gets the prefix, so a line in it can't pass for one of our own.
111
+ formatted_message = _CONTROL_CHARS_RE.sub('', super().format(record).replace('\r\n', '\n').replace('\r', '\n'))
112
+ return '\n'.join(f'{formatted_logger_name} -> {msg_line}' for msg_line in formatted_message.split('\n'))
107
113
 
108
114
 
109
115
  def create_redirect_logger(name: str) -> logging.Logger:
@@ -1246,7 +1246,7 @@ class EffectivePlatformFeature(BaseModel):
1246
1246
  str | None,
1247
1247
  Field(
1248
1248
  examples=[
1249
- 'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact support@scrape.ly'
1249
+ 'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact hello@scrape.ly'
1250
1250
  ]
1251
1251
  ),
1252
1252
  ]
@@ -3,6 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  from typing import TYPE_CHECKING, Any, Literal
6
+ from uuid import uuid4
6
7
 
7
8
  from pydantic import TypeAdapter
8
9
 
@@ -287,7 +288,8 @@ class ActorClient(ResourceClient):
287
288
  response = self._http_client.call(
288
289
  url=self._build_url('runs'),
289
290
  method='POST',
290
- headers={'content-type': content_type},
291
+ # The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
292
+ headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
291
293
  data=run_input,
292
294
  params=request_params,
293
295
  timeout=timeout,
@@ -418,6 +420,8 @@ class ActorClient(ResourceClient):
418
420
  response = self._http_client.call(
419
421
  url=self._build_url('builds'),
420
422
  method='POST',
423
+ # One build per key (same contract as run starts), so a resent build start is safe.
424
+ headers={'idempotency-key': uuid4().hex},
421
425
  params=request_params,
422
426
  timeout=timeout,
423
427
  )
@@ -783,7 +787,8 @@ class ActorClientAsync(ResourceClientAsync):
783
787
  response = await self._http_client.call(
784
788
  url=self._build_url('runs'),
785
789
  method='POST',
786
- headers={'content-type': content_type},
790
+ # The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
791
+ headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
787
792
  data=run_input,
788
793
  params=request_params,
789
794
  timeout=timeout,
@@ -918,6 +923,8 @@ class ActorClientAsync(ResourceClientAsync):
918
923
  response = await self._http_client.call(
919
924
  url=self._build_url('builds'),
920
925
  method='POST',
926
+ # One build per key (same contract as run starts), so a resent build start is safe.
927
+ headers={'idempotency-key': uuid4().hex},
921
928
  params=request_params,
922
929
  timeout=timeout,
923
930
  )
@@ -5,13 +5,15 @@ from __future__ import annotations
5
5
  from contextlib import asynccontextmanager, contextmanager
6
6
  from dataclasses import dataclass
7
7
  from typing import TYPE_CHECKING, Any
8
+ from uuid import uuid4
8
9
 
9
10
  from scrapely_client._docs import docs_group
10
11
  from scrapely_client._models import Dataset, DatasetResponse, DatasetStatistics, DatasetStatisticsResponse
11
12
  from scrapely_client._pagination import DEFAULT_CHUNK_SIZE, get_items_iterator, get_items_iterator_async
12
13
  from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
13
- from scrapely_client._utils.crypto import create_storage_content_signature
14
+ from scrapely_client._utils.crypto import sign_storage_url
14
15
  from scrapely_client._utils.http import response_to_dict, response_to_list
16
+ from scrapely_client.errors import ApifyApiError
15
17
 
16
18
  if TYPE_CHECKING:
17
19
  from collections.abc import AsyncIterator, Iterator
@@ -495,15 +497,26 @@ class DatasetClient(ResourceClient):
495
497
  else:
496
498
  json = items
497
499
 
498
- self._http_client.call(
499
- url=self._build_url('items'),
500
- method='POST',
501
- headers={'content-type': 'application/json; charset=utf-8'},
502
- params=self._build_params(),
503
- data=data,
504
- json=json,
505
- timeout=timeout,
506
- )
500
+ for resend in (False, True):
501
+ try:
502
+ self._http_client.call(
503
+ url=self._build_url('items'),
504
+ method='POST',
505
+ # The server stores a push once per key, so a push resent after a 5xx or a dropped connection
506
+ # is safe.
507
+ headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
508
+ params=self._build_params(),
509
+ data=data,
510
+ json=json,
511
+ timeout=timeout,
512
+ )
513
+ except ApifyApiError as exc:
514
+ # The push failed mid-write and the server can't tell whether it was stored. Send it once more under
515
+ # a new key, as an unkeyed push would be resent: a second copy is possible, losing the items is not.
516
+ if resend or exc.type != 'push-outcome-unknown':
517
+ raise
518
+ else:
519
+ return
507
520
 
508
521
  def get_statistics(self, *, timeout: Timeout = 'short') -> DatasetStatistics:
509
522
  """Get the dataset statistics.
@@ -551,7 +564,8 @@ class DatasetClient(ResourceClient):
551
564
 
552
565
  You can optionally control how long the signed URL should be valid using the `expires_in` option.
553
566
  This value sets the expiration duration from the time the URL is generated.
554
- If not provided, the URL will not expire.
567
+ If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
568
+ working if any of them is changed.
555
569
 
556
570
  Any other options (like `limit` or `offset`) will be included as query parameters in the URL.
557
571
 
@@ -568,7 +582,7 @@ class DatasetClient(ResourceClient):
568
582
  skip_hidden: If True, then hidden fields are skipped from the output.
569
583
  flatten: A list of fields that should be flattened.
570
584
  view: Name of the dataset view to be used.
571
- expires_in: How long the signed URL should be valid.
585
+ expires_in: How long the signed URL should be valid. Defaults to 1 hour.
572
586
  timeout: Timeout for the API HTTP request.
573
587
 
574
588
  Returns:
@@ -590,15 +604,18 @@ class DatasetClient(ResourceClient):
590
604
  view=view,
591
605
  )
592
606
 
607
+ public_url = self._build_public_url('items', request_params)
608
+
609
+ # Signed last: the signature binds every query parameter above.
593
610
  if dataset and dataset.url_signing_secret_key:
594
- signature = create_storage_content_signature(
611
+ public_url = sign_storage_url(
612
+ public_url,
595
613
  dataset.id,
596
614
  dataset.url_signing_secret_key,
597
615
  expires_in=expires_in,
598
616
  )
599
- request_params['signature'] = signature
600
617
 
601
- return self._build_public_url('items', request_params)
618
+ return public_url
602
619
 
603
620
 
604
621
  @docs_group('Resource clients')
@@ -1047,15 +1064,26 @@ class DatasetClientAsync(ResourceClientAsync):
1047
1064
  else:
1048
1065
  json = items
1049
1066
 
1050
- await self._http_client.call(
1051
- url=self._build_url('items'),
1052
- method='POST',
1053
- headers={'content-type': 'application/json; charset=utf-8'},
1054
- params=self._build_params(),
1055
- data=data,
1056
- json=json,
1057
- timeout=timeout,
1058
- )
1067
+ for resend in (False, True):
1068
+ try:
1069
+ await self._http_client.call(
1070
+ url=self._build_url('items'),
1071
+ method='POST',
1072
+ # The server stores a push once per key, so a push resent after a 5xx or a dropped connection
1073
+ # is safe.
1074
+ headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
1075
+ params=self._build_params(),
1076
+ data=data,
1077
+ json=json,
1078
+ timeout=timeout,
1079
+ )
1080
+ except ApifyApiError as exc:
1081
+ # The push failed mid-write and the server can't tell whether it was stored. Send it once more under
1082
+ # a new key, as an unkeyed push would be resent: a second copy is possible, losing the items is not.
1083
+ if resend or exc.type != 'push-outcome-unknown':
1084
+ raise
1085
+ else:
1086
+ return
1059
1087
 
1060
1088
  async def get_statistics(self, *, timeout: Timeout = 'short') -> DatasetStatistics:
1061
1089
  """Get the dataset statistics.
@@ -1103,7 +1131,8 @@ class DatasetClientAsync(ResourceClientAsync):
1103
1131
 
1104
1132
  You can optionally control how long the signed URL should be valid using the `expires_in` option.
1105
1133
  This value sets the expiration duration from the time the URL is generated.
1106
- If not provided, the URL will not expire.
1134
+ If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
1135
+ working if any of them is changed.
1107
1136
 
1108
1137
  Any other options (like `limit` or `offset`) will be included as query parameters in the URL.
1109
1138
 
@@ -1120,7 +1149,7 @@ class DatasetClientAsync(ResourceClientAsync):
1120
1149
  skip_hidden: If True, then hidden fields are skipped from the output.
1121
1150
  flatten: A list of fields that should be flattened.
1122
1151
  view: Name of the dataset view to be used.
1123
- expires_in: How long the signed URL should be valid.
1152
+ expires_in: How long the signed URL should be valid. Defaults to 1 hour.
1124
1153
  timeout: Timeout for the API HTTP request.
1125
1154
 
1126
1155
  Returns:
@@ -1142,12 +1171,15 @@ class DatasetClientAsync(ResourceClientAsync):
1142
1171
  view=view,
1143
1172
  )
1144
1173
 
1174
+ public_url = self._build_public_url('items', request_params)
1175
+
1176
+ # Signed last: the signature binds every query parameter above.
1145
1177
  if dataset and dataset.url_signing_secret_key:
1146
- signature = create_storage_content_signature(
1178
+ public_url = sign_storage_url(
1179
+ public_url,
1147
1180
  dataset.id,
1148
1181
  dataset.url_signing_secret_key,
1149
1182
  expires_in=expires_in,
1150
1183
  )
1151
- request_params['signature'] = signature
1152
1184
 
1153
- return self._build_public_url('items', request_params)
1185
+ return public_url