scrapely-client-python 1.0.9__tar.gz → 1.0.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/CHANGELOG.md +6 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/CONTRIBUTING.md +17 -36
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/PKG-INFO +18 -18
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/README.md +17 -17
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/pyproject.toml +7 -2
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/pyproject.toml.orig +2 -2
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_logging.py +9 -3
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_models.py +1 -1
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor.py +9 -2
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/dataset.py +61 -29
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/key_value_store.py +41 -21
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/task.py +5 -2
- scrapely_client_python-1.0.10/src/scrapely_client/_utils/crypto.py +161 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/_impit.py +89 -4
- scrapely_client_python-1.0.9/src/scrapely_client/_utils/crypto.py +0 -89
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/LICENSE +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/NOTICE +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/__init__.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_charging.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_client_registry.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_consts.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_docs.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_literals.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_pagination.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/__init__.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/_resource_client.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_env_var.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_env_var_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_version.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/actor_version_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/automation.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/automation_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/build.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/build_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/dataset_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/key_value_store_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/log.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/request_queue.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/request_queue_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/run.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/run_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/schedule.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/schedule_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/store_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/task_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/user.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_dispatch.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_resource_clients/webhook_dispatch_collection.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_scrapely_client.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_statistics.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_status_message_watcher.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_streamed_log.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_typeddicts.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/__init__.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/encoding.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/errors.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/http.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/time.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_utils/try_import.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/errors.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/__init__.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_clients/_base.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/__init__.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_base.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_brotli.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_gzip.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/http_compressors/_resolve.py +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/py.typed +0 -0
- {scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/types.py +0 -0
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [1.0.10] - 2026-11-02
|
|
6
|
+
|
|
7
|
+
- Run starts (`ActorClient.start` / `call`, `TaskClient.start` / `call`), `ActorClient.build` and `DatasetClient.push_items` send an `Idempotency-Key` header, so a request resent after a 5xx or a dropped connection starts one run or build and stores the items once.
|
|
8
|
+
- Any other POST or PATCH without an `Idempotency-Key` is no longer resent after a 5xx or a connection that dropped mid-request, since it may already have been applied. Rate limits (429), failures before the request was sent, request queue adds (de-duplicated by `uniqueKey`), request unlocks, head locks, run aborts and get-or-create of a storage by name are still retried.
|
|
9
|
+
- A keyed request answered with 409 `idempotency-key-in-progress` (the first request with that key is still being applied) is resent until it gets that request's answer; a 503 `push-outcome-unknown` is not resent, since no resend can change it.
|
|
10
|
+
|
|
5
11
|
## [1.0.9] - 2026-10-24
|
|
6
12
|
|
|
7
13
|
- The `SCRAPELY_API_URL` environment variable sets the API URL when `api_url` is not passed to `ScrapelyClient`, `ScrapelyClientAsync` or `with_custom_http_client`; `api_url` still wins, and without either the client uses `https://api.scrape.ly`. An empty value is ignored. `api_public_url` is unchanged.
|
|
@@ -18,7 +18,7 @@ All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be ru
|
|
|
18
18
|
| Task | Description |
|
|
19
19
|
| ---- | ----------- |
|
|
20
20
|
| `install-dev` | Install development dependencies |
|
|
21
|
-
| `check-code` | Run lint, type-check,
|
|
21
|
+
| `check-code` | Run lint, type-check, docstring check, and unit tests |
|
|
22
22
|
| `lint` | Run linter |
|
|
23
23
|
| `format` | Fix lint issues and format code |
|
|
24
24
|
| `type-check` | Run type checker |
|
|
@@ -28,8 +28,10 @@ All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be ru
|
|
|
28
28
|
| `integration-tests-cov` | Run integration tests with coverage |
|
|
29
29
|
| `check-docstrings` | Check async client docstrings |
|
|
30
30
|
| `fix-docstrings` | Fix async client docstrings |
|
|
31
|
-
| `build-docs` | Build documentation website |
|
|
32
|
-
| `run-docs` | Run documentation website locally |
|
|
31
|
+
| `build-docs` | Build documentation website (does not work yet, see [Documentation](#documentation)) |
|
|
32
|
+
| `run-docs` | Run documentation website locally (does not work yet, see [Documentation](#documentation)) |
|
|
33
|
+
| `generate-models` | Regenerate the models from `openapi/scrapely-api.json` |
|
|
34
|
+
| `accept-openapi-spec` | Record the hash of a reviewed change to `openapi/scrapely-api.json` |
|
|
33
35
|
| `build` | Build package |
|
|
34
36
|
| `clean` | Remove build artifacts and clean caches |
|
|
35
37
|
|
|
@@ -103,7 +105,7 @@ Prerequisites:
|
|
|
103
105
|
|
|
104
106
|
- Set `SCRAPELY_TEST_USER_API_TOKEN` to your Scrapely API token
|
|
105
107
|
- Set `SCRAPELY_TEST_USER_2_API_TOKEN` for storage access tests (requires two accounts)
|
|
106
|
-
- Optionally set `SCRAPELY_INTEGRATION_TESTS_API_URL` to use a different Scrapely API environment
|
|
108
|
+
- Optionally set `SCRAPELY_INTEGRATION_TESTS_API_URL` to use a different Scrapely API environment. It defaults to `https://api.scrape.ly`; for test accounts on staging, set it to `https://api-staging.scrape.ly`
|
|
107
109
|
|
|
108
110
|
To run integration tests:
|
|
109
111
|
|
|
@@ -115,19 +117,15 @@ uv run poe integration-tests
|
|
|
115
117
|
|
|
116
118
|
We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/).
|
|
117
119
|
|
|
118
|
-
|
|
120
|
+
The user guide for this client is published at https://scrape.ly/docs/client-python. The Markdown files in the `docs/` folder are copied to the public repository [scrapely-app/scrapely-client-python](https://github.com/scrapely-app/scrapely-client-python) with each release and are read there.
|
|
119
121
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
```sh
|
|
123
|
-
uv run poe run-docs
|
|
124
|
-
```
|
|
122
|
+
The `website/` folder holds a [Docusaurus](https://docusaurus.io/) setup that would render `docs/` together with an API reference generated from the docstrings by [pydoc-markdown](https://pypi.org/project/pydoc-markdown/). It is not published anywhere, and `uv run poe run-docs` / `build-docs` do not work at the moment: the site depends on the `@scrapely/docs-theme` npm package, which has not been published.
|
|
125
123
|
|
|
126
124
|
### Linting the docs and website
|
|
127
125
|
|
|
128
126
|
Markdown content (this guide, `README.md`, and the `docs/` folder) is checked with
|
|
129
127
|
[markdownlint](https://github.com/DavidAnson/markdownlint). The Docusaurus website code is linted
|
|
130
|
-
with [oxlint](https://oxc.rs/) and formatted with [oxfmt](https://oxc.rs/).
|
|
128
|
+
with [oxlint](https://oxc.rs/) and formatted with [oxfmt](https://oxc.rs/). None of them run in CI at the moment.
|
|
131
129
|
To run them locally (requires Node.js 22.12 or newer and pnpm), from the `website/` directory:
|
|
132
130
|
|
|
133
131
|
```sh
|
|
@@ -141,7 +139,7 @@ Doc images are committed as optimized `.webp`. To convert a new image, run
|
|
|
141
139
|
|
|
142
140
|
## Commits
|
|
143
141
|
|
|
144
|
-
We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages.
|
|
142
|
+
We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages.
|
|
145
143
|
|
|
146
144
|
### Available commit types
|
|
147
145
|
|
|
@@ -161,31 +159,14 @@ We use [Conventional Commits](https://www.conventionalcommits.org/) format for c
|
|
|
161
159
|
|
|
162
160
|
## Release process
|
|
163
161
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
- **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI.
|
|
167
|
-
- **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overridden using the `custom` release type.
|
|
168
|
-
|
|
169
|
-
### Publishing to PyPI manually
|
|
170
|
-
|
|
171
|
-
1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `release` workflow instead.
|
|
172
|
-
2. **Make sure you know what you're doing.**
|
|
173
|
-
3. Update the version number by modifying the `version` field under `project` in `pyproject.toml`:
|
|
174
|
-
|
|
175
|
-
```toml
|
|
176
|
-
[project]
|
|
177
|
-
name = "scrapely-client-python"
|
|
178
|
-
version = "x.z.y"
|
|
179
|
-
```
|
|
162
|
+
This package lives in the `scrapely-client-python/` folder of the Scrapely monorepo. Changes go to the `staging` branch first and reach `main` when staging is promoted.
|
|
180
163
|
|
|
181
|
-
|
|
164
|
+
Publishing to [PyPI](https://pypi.org/project/scrapely-client-python/) is automated by the monorepo's `publish-packages.yml` GitHub Actions workflow. When a push to `main` changes this folder and the `version` in `pyproject.toml` is newer than the version on PyPI, the workflow builds the package with `uv build`, publishes it, and copies `src/`, `docs/`, `pyproject.toml`, `uv.lock`, `.editorconfig`, `README.md`, `CHANGELOG.md`, `LICENSE` and `NOTICE` to [scrapely-app/scrapely-client-python](https://github.com/scrapely-app/scrapely-client-python). There are no beta releases.
|
|
182
165
|
|
|
183
|
-
|
|
184
|
-
uv run poe build
|
|
185
|
-
```
|
|
166
|
+
To release a new version:
|
|
186
167
|
|
|
187
|
-
|
|
168
|
+
1. Bump `version` under `[project]` in `pyproject.toml`.
|
|
169
|
+
2. Add an entry for the new version at the top of `CHANGELOG.md`.
|
|
170
|
+
3. Merge the change to `staging`. It is published when it reaches `main`.
|
|
188
171
|
|
|
189
|
-
|
|
190
|
-
uv publish --token YOUR_API_TOKEN
|
|
191
|
-
```
|
|
172
|
+
The README on PyPI is the one shipped with the latest published version, so README and docs changes reach PyPI and the public repository only with the next version bump.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapely-client-python
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.10
|
|
4
4
|
Summary: Scrapely API client for Python
|
|
5
5
|
Keywords: scrapely,api,client,automation,crawling,scraping
|
|
6
6
|
Author: TeraShift GmbH (Scrape.ly)
|
|
@@ -36,7 +36,7 @@ Description-Content-Type: text/markdown
|
|
|
36
36
|
<h1 align="center">Scrapely API client for Python</h1>
|
|
37
37
|
|
|
38
38
|
<p align="center">
|
|
39
|
-
<strong>The official Python client for the <a href="https://
|
|
39
|
+
<strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
|
|
40
40
|
</p>
|
|
41
41
|
|
|
42
42
|
<p align="center">
|
|
@@ -46,9 +46,9 @@ Description-Content-Type: text/markdown
|
|
|
46
46
|
<a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
|
|
47
47
|
</p>
|
|
48
48
|
|
|
49
|
-
`scrapely-client-python` lets you talk to the [Scrapely platform](https://
|
|
49
|
+
`scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
|
|
50
50
|
|
|
51
|
-
> If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
|
|
51
|
+
> If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
|
|
52
52
|
|
|
53
53
|
## Table of contents
|
|
54
54
|
|
|
@@ -92,7 +92,7 @@ Description-Content-Type: text/markdown
|
|
|
92
92
|
|
|
93
93
|
## Quick start
|
|
94
94
|
|
|
95
|
-
You'll need a Scrapely API token —
|
|
95
|
+
You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
|
|
96
96
|
|
|
97
97
|
The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
|
|
98
98
|
|
|
@@ -222,33 +222,33 @@ client = ScrapelyClient(
|
|
|
222
222
|
)
|
|
223
223
|
```
|
|
224
224
|
|
|
225
|
-
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the
|
|
225
|
+
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
|
|
226
226
|
|
|
227
227
|
## Documentation
|
|
228
228
|
|
|
229
|
-
The
|
|
229
|
+
The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
|
|
230
|
+
|
|
231
|
+
The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
|
|
230
232
|
|
|
231
233
|
| Section | What you'll find |
|
|
232
234
|
| --- | --- |
|
|
233
|
-
| Introduction | Overview,
|
|
234
|
-
|
|
|
235
|
-
|
|
|
236
|
-
|
|
|
237
|
-
| API reference | Generated reference for every class, method, and model. |
|
|
238
|
-
| Changelog | Release history and breaking changes. |
|
|
235
|
+
| [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
|
|
236
|
+
| [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
|
|
237
|
+
| [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
|
|
238
|
+
| [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
|
|
239
239
|
|
|
240
240
|
## Related projects
|
|
241
241
|
|
|
242
|
-
- **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
242
|
+
- **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
243
243
|
- **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
|
|
244
|
-
- **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
|
|
245
|
-
- **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
|
|
244
|
+
- **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
|
|
245
|
+
- **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
|
|
246
246
|
- **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
|
|
247
|
-
- **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
247
|
+
- **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
248
248
|
|
|
249
249
|
## Support and community
|
|
250
250
|
|
|
251
|
-
- **GitHub issues** — report a bug or request a feature in the
|
|
251
|
+
- **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
|
|
252
252
|
|
|
253
253
|
## Contributing
|
|
254
254
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
<h1 align="center">Scrapely API client for Python</h1>
|
|
4
4
|
|
|
5
5
|
<p align="center">
|
|
6
|
-
<strong>The official Python client for the <a href="https://
|
|
6
|
+
<strong>The official Python client for the <a href="https://scrape.ly/docs/api">Scrapely REST API</a>.</strong>
|
|
7
7
|
</p>
|
|
8
8
|
|
|
9
9
|
<p align="center">
|
|
@@ -13,9 +13,9 @@
|
|
|
13
13
|
<a href="./LICENSE"><img src="https://img.shields.io/pypi/l/scrapely-client-python" alt="License"></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
`scrapely-client-python` lets you talk to the [Scrapely platform](https://
|
|
16
|
+
`scrapely-client-python` lets you talk to the [Scrapely platform](https://scrape.ly) from Python — run Actors, manage storages (datasets, key-value stores, request queues), schedule tasks, configure webhooks, and use everything else exposed by the Scrapely API. It ships both synchronous and asynchronous clients, fully typed responses, automatic retries with exponential backoff, tiered timeouts, pagination helpers, streaming, and a pluggable HTTP layer.
|
|
17
17
|
|
|
18
|
-
> If you want to **build** Actors in Python rather than consume the API, use the Scrapely SDK for Python instead — it bundles this client and adds Actor-side primitives.
|
|
18
|
+
> If you want to **build** Actors in Python rather than consume the API, use the [Scrapely SDK for Python](https://scrape.ly/docs/sdk-python) (`scrapely-python`) instead — it bundles this client and adds Actor-side primitives.
|
|
19
19
|
|
|
20
20
|
## Table of contents
|
|
21
21
|
|
|
@@ -59,7 +59,7 @@
|
|
|
59
59
|
|
|
60
60
|
## Quick start
|
|
61
61
|
|
|
62
|
-
You'll need a Scrapely API token —
|
|
62
|
+
You'll need a Scrapely API token — create one in the dashboard under **Settings → API & Integrations** (see [API keys](https://scrape.ly/docs/api-keys)). Pass it to the client and you're ready to go.
|
|
63
63
|
|
|
64
64
|
The client sends requests to `https://api.scrape.ly` unless you pass another API origin as `api_url`. When `api_url` is not given, the `SCRAPELY_API_URL` environment variable sets it (for example `SCRAPELY_API_URL=https://api-staging.scrape.ly`). `api_public_url`, used for public storage links, keeps its own argument and default. The client does not read the token from the environment.
|
|
65
65
|
|
|
@@ -189,33 +189,33 @@ client = ScrapelyClient(
|
|
|
189
189
|
)
|
|
190
190
|
```
|
|
191
191
|
|
|
192
|
-
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the
|
|
192
|
+
For end-to-end recipes — passing input, managing tasks for reusable input, retrieving and merging Actor data, integrating with Pandas, plugging in a custom HTTP client — see the [guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides).
|
|
193
193
|
|
|
194
194
|
## Documentation
|
|
195
195
|
|
|
196
|
-
The
|
|
196
|
+
The client guide lives at **https://scrape.ly/docs/client-python**, and the REST API it wraps is described in the [API reference](https://scrape.ly/docs/api).
|
|
197
|
+
|
|
198
|
+
The [`docs/`](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs) folder of the repository has more detail:
|
|
197
199
|
|
|
198
200
|
| Section | What you'll find |
|
|
199
201
|
| --- | --- |
|
|
200
|
-
| Introduction | Overview,
|
|
201
|
-
|
|
|
202
|
-
|
|
|
203
|
-
|
|
|
204
|
-
| API reference | Generated reference for every class, method, and model. |
|
|
205
|
-
| Changelog | Release history and breaking changes. |
|
|
202
|
+
| [Introduction](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/01_introduction) | Overview, installation, and a quick start: authenticate, run an Actor, and fetch its results. |
|
|
203
|
+
| [Concepts](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/02_concepts) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts, typed models, HTTP compression. |
|
|
204
|
+
| [Guides](https://github.com/scrapely-app/scrapely-client-python/tree/main/docs/03_guides) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. |
|
|
205
|
+
| [Changelog](https://github.com/scrapely-app/scrapely-client-python/blob/main/CHANGELOG.md) | Release history and breaking changes. |
|
|
206
206
|
|
|
207
207
|
## Related projects
|
|
208
208
|
|
|
209
|
-
- **Scrapely SDK for Python** — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
209
|
+
- **[Scrapely SDK for Python](https://scrape.ly/docs/sdk-python)** (`scrapely-python`) — toolkit for **building** Actors in Python (this client is bundled with it).
|
|
210
210
|
- **[Crawlee for Python](https://crawlee.dev/python)** — high-level web scraping and browser automation framework that powers many Actors.
|
|
211
|
-
- **Scrapely API client for JavaScript / TypeScript** — equivalent Scrapely API client for Node.js.
|
|
212
|
-
- **Scrapely SDK for JavaScript / TypeScript** — equivalent Scrapely SDK for Node.js.
|
|
211
|
+
- **[Scrapely API client for JavaScript / TypeScript](https://scrape.ly/docs/client-js)** — equivalent Scrapely API client for Node.js.
|
|
212
|
+
- **[Scrapely SDK for JavaScript / TypeScript](https://scrape.ly/docs/sdk-node)** — equivalent Scrapely SDK for Node.js.
|
|
213
213
|
- **[Crawlee for JavaScript / TypeScript](https://crawlee.dev)** — the Node.js implementation of the Crawlee framework.
|
|
214
|
-
- **Scrapely CLI** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
214
|
+
- **[Scrapely CLI](https://scrape.ly/docs/cli)** — command-line tool for interacting with the Scrapely platform: managing Actors, runs, storages, local development, and deployment.
|
|
215
215
|
|
|
216
216
|
## Support and community
|
|
217
217
|
|
|
218
|
-
- **GitHub issues** — report a bug or request a feature in the
|
|
218
|
+
- **GitHub issues** — report a bug or request a feature in the [issue tracker](https://github.com/scrapely-app/scrapely-client-python/issues).
|
|
219
219
|
|
|
220
220
|
## Contributing
|
|
221
221
|
|
|
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapely-client-python"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.10"
|
|
8
8
|
description = "Scrapely API client for Python"
|
|
9
9
|
license = "Apache-2.0"
|
|
10
10
|
license-files = [
|
|
@@ -282,7 +282,6 @@ crawlee = false
|
|
|
282
282
|
clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
|
|
283
283
|
install-sync = "uv sync --all-extras"
|
|
284
284
|
build = "uv build"
|
|
285
|
-
publish-to-pypi = "uv publish --token ${SCRAPELY_PYPI_TOKEN}"
|
|
286
285
|
type-check = "uv run ty check"
|
|
287
286
|
unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit"
|
|
288
287
|
unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/scrapely_client --cov-report=xml:coverage-unit.xml tests/unit"
|
|
@@ -297,6 +296,12 @@ check-code = [
|
|
|
297
296
|
"unit-tests",
|
|
298
297
|
]
|
|
299
298
|
|
|
299
|
+
[tool.poe.tasks.publish-to-pypi]
|
|
300
|
+
cmd = "uv publish"
|
|
301
|
+
|
|
302
|
+
[tool.poe.tasks.publish-to-pypi.env]
|
|
303
|
+
UV_PUBLISH_TOKEN = "${SCRAPELY_PYPI_TOKEN}"
|
|
304
|
+
|
|
300
305
|
[tool.poe.tasks.install-dev]
|
|
301
306
|
shell = "uv sync --all-extras && uv run pre-commit install"
|
|
302
307
|
|
|
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapely-client-python"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.10"
|
|
8
8
|
description = "Scrapely API client for Python"
|
|
9
9
|
authors = [{ name = "TeraShift GmbH (Scrape.ly)" }]
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -267,7 +267,7 @@ crawlee = false
|
|
|
267
267
|
clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
|
|
268
268
|
install-sync = "uv sync --all-extras"
|
|
269
269
|
build = "uv build"
|
|
270
|
-
publish-to-pypi = "uv publish
|
|
270
|
+
publish-to-pypi = { cmd = "uv publish", env = { UV_PUBLISH_TOKEN = "${SCRAPELY_PYPI_TOKEN}" } }
|
|
271
271
|
type-check = "uv run ty check"
|
|
272
272
|
unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit"
|
|
273
273
|
unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/scrapely_client --cov-report=xml:coverage-unit.xml tests/unit"
|
{scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_logging.py
RENAMED
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import functools
|
|
6
6
|
import inspect
|
|
7
7
|
import logging
|
|
8
|
+
import re
|
|
8
9
|
import threading
|
|
9
10
|
from contextvars import ContextVar
|
|
10
11
|
from typing import TYPE_CHECKING, Any, NamedTuple
|
|
@@ -25,6 +26,9 @@ logger_name = __name__.split('.')[0]
|
|
|
25
26
|
logger = logging.getLogger(logger_name)
|
|
26
27
|
"""Logger used throughout the library."""
|
|
27
28
|
|
|
29
|
+
# C0 controls except tab and newline, DEL, and C1 controls (incl. 8-bit CSI).
|
|
30
|
+
_CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\x9f]')
|
|
31
|
+
|
|
28
32
|
|
|
29
33
|
class LogContext(NamedTuple):
|
|
30
34
|
"""Request context details for logging (attempt, client method, HTTP method, resource ID, URL)."""
|
|
@@ -101,9 +105,11 @@ class RedirectLogFormatter(logging.Formatter):
|
|
|
101
105
|
Returns:
|
|
102
106
|
Formatted log message with colored logger name prefix.
|
|
103
107
|
"""
|
|
104
|
-
formatted_logger_name = f'{Fore.CYAN}[{record.name}]{Style.RESET_ALL}'
|
|
105
|
-
|
|
106
|
-
|
|
108
|
+
formatted_logger_name = f'{Fore.CYAN}[{_CONTROL_CHARS_RE.sub("", record.name)}]{Style.RESET_ALL}'
|
|
109
|
+
# Redirected messages come from another Actor's log, so escape sequences in it are removed
|
|
110
|
+
# and every line gets the prefix, so a line in it can't pass for one of our own.
|
|
111
|
+
formatted_message = _CONTROL_CHARS_RE.sub('', super().format(record).replace('\r\n', '\n').replace('\r', '\n'))
|
|
112
|
+
return '\n'.join(f'{formatted_logger_name} -> {msg_line}' for msg_line in formatted_message.split('\n'))
|
|
107
113
|
|
|
108
114
|
|
|
109
115
|
def create_redirect_logger(name: str) -> logging.Logger:
|
{scrapely_client_python-1.0.9 → scrapely_client_python-1.0.10}/src/scrapely_client/_models.py
RENAMED
|
@@ -1246,7 +1246,7 @@ class EffectivePlatformFeature(BaseModel):
|
|
|
1246
1246
|
str | None,
|
|
1247
1247
|
Field(
|
|
1248
1248
|
examples=[
|
|
1249
|
-
'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact
|
|
1249
|
+
'The "Selected public Actors for developers" feature is not enabled for your account. Please upgrade your plan or contact hello@scrape.ly'
|
|
1250
1250
|
]
|
|
1251
1251
|
),
|
|
1252
1252
|
]
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import TYPE_CHECKING, Any, Literal
|
|
6
|
+
from uuid import uuid4
|
|
6
7
|
|
|
7
8
|
from pydantic import TypeAdapter
|
|
8
9
|
|
|
@@ -287,7 +288,8 @@ class ActorClient(ResourceClient):
|
|
|
287
288
|
response = self._http_client.call(
|
|
288
289
|
url=self._build_url('runs'),
|
|
289
290
|
method='POST',
|
|
290
|
-
|
|
291
|
+
# The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
|
|
292
|
+
headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
|
|
291
293
|
data=run_input,
|
|
292
294
|
params=request_params,
|
|
293
295
|
timeout=timeout,
|
|
@@ -418,6 +420,8 @@ class ActorClient(ResourceClient):
|
|
|
418
420
|
response = self._http_client.call(
|
|
419
421
|
url=self._build_url('builds'),
|
|
420
422
|
method='POST',
|
|
423
|
+
# One build per key (same contract as run starts), so a resent build start is safe.
|
|
424
|
+
headers={'idempotency-key': uuid4().hex},
|
|
421
425
|
params=request_params,
|
|
422
426
|
timeout=timeout,
|
|
423
427
|
)
|
|
@@ -783,7 +787,8 @@ class ActorClientAsync(ResourceClientAsync):
|
|
|
783
787
|
response = await self._http_client.call(
|
|
784
788
|
url=self._build_url('runs'),
|
|
785
789
|
method='POST',
|
|
786
|
-
|
|
790
|
+
# The server starts one run per key, so a start resent after a 5xx or a dropped connection is safe.
|
|
791
|
+
headers={'content-type': content_type, 'idempotency-key': uuid4().hex},
|
|
787
792
|
data=run_input,
|
|
788
793
|
params=request_params,
|
|
789
794
|
timeout=timeout,
|
|
@@ -918,6 +923,8 @@ class ActorClientAsync(ResourceClientAsync):
|
|
|
918
923
|
response = await self._http_client.call(
|
|
919
924
|
url=self._build_url('builds'),
|
|
920
925
|
method='POST',
|
|
926
|
+
# One build per key (same contract as run starts), so a resent build start is safe.
|
|
927
|
+
headers={'idempotency-key': uuid4().hex},
|
|
921
928
|
params=request_params,
|
|
922
929
|
timeout=timeout,
|
|
923
930
|
)
|
|
@@ -5,13 +5,15 @@ from __future__ import annotations
|
|
|
5
5
|
from contextlib import asynccontextmanager, contextmanager
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
from typing import TYPE_CHECKING, Any
|
|
8
|
+
from uuid import uuid4
|
|
8
9
|
|
|
9
10
|
from scrapely_client._docs import docs_group
|
|
10
11
|
from scrapely_client._models import Dataset, DatasetResponse, DatasetStatistics, DatasetStatisticsResponse
|
|
11
12
|
from scrapely_client._pagination import DEFAULT_CHUNK_SIZE, get_items_iterator, get_items_iterator_async
|
|
12
13
|
from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
|
|
13
|
-
from scrapely_client._utils.crypto import
|
|
14
|
+
from scrapely_client._utils.crypto import sign_storage_url
|
|
14
15
|
from scrapely_client._utils.http import response_to_dict, response_to_list
|
|
16
|
+
from scrapely_client.errors import ApifyApiError
|
|
15
17
|
|
|
16
18
|
if TYPE_CHECKING:
|
|
17
19
|
from collections.abc import AsyncIterator, Iterator
|
|
@@ -495,15 +497,26 @@ class DatasetClient(ResourceClient):
|
|
|
495
497
|
else:
|
|
496
498
|
json = items
|
|
497
499
|
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
500
|
+
for resend in (False, True):
|
|
501
|
+
try:
|
|
502
|
+
self._http_client.call(
|
|
503
|
+
url=self._build_url('items'),
|
|
504
|
+
method='POST',
|
|
505
|
+
# The server stores a push once per key, so a push resent after a 5xx or a dropped connection
|
|
506
|
+
# is safe.
|
|
507
|
+
headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
|
|
508
|
+
params=self._build_params(),
|
|
509
|
+
data=data,
|
|
510
|
+
json=json,
|
|
511
|
+
timeout=timeout,
|
|
512
|
+
)
|
|
513
|
+
except ApifyApiError as exc:
|
|
514
|
+
# The push failed mid-write and the server can't tell whether it was stored. Send it once more under
|
|
515
|
+
# a new key, as an unkeyed push would be resent: a second copy is possible, losing the items is not.
|
|
516
|
+
if resend or exc.type != 'push-outcome-unknown':
|
|
517
|
+
raise
|
|
518
|
+
else:
|
|
519
|
+
return
|
|
507
520
|
|
|
508
521
|
def get_statistics(self, *, timeout: Timeout = 'short') -> DatasetStatistics:
|
|
509
522
|
"""Get the dataset statistics.
|
|
@@ -551,7 +564,8 @@ class DatasetClient(ResourceClient):
|
|
|
551
564
|
|
|
552
565
|
You can optionally control how long the signed URL should be valid using the `expires_in` option.
|
|
553
566
|
This value sets the expiration duration from the time the URL is generated.
|
|
554
|
-
If not provided, the URL
|
|
567
|
+
If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
|
|
568
|
+
working if any of them is changed.
|
|
555
569
|
|
|
556
570
|
Any other options (like `limit` or `offset`) will be included as query parameters in the URL.
|
|
557
571
|
|
|
@@ -568,7 +582,7 @@ class DatasetClient(ResourceClient):
|
|
|
568
582
|
skip_hidden: If True, then hidden fields are skipped from the output.
|
|
569
583
|
flatten: A list of fields that should be flattened.
|
|
570
584
|
view: Name of the dataset view to be used.
|
|
571
|
-
expires_in: How long the signed URL should be valid.
|
|
585
|
+
expires_in: How long the signed URL should be valid. Defaults to 1 hour.
|
|
572
586
|
timeout: Timeout for the API HTTP request.
|
|
573
587
|
|
|
574
588
|
Returns:
|
|
@@ -590,15 +604,18 @@ class DatasetClient(ResourceClient):
|
|
|
590
604
|
view=view,
|
|
591
605
|
)
|
|
592
606
|
|
|
607
|
+
public_url = self._build_public_url('items', request_params)
|
|
608
|
+
|
|
609
|
+
# Signed last: the signature binds every query parameter above.
|
|
593
610
|
if dataset and dataset.url_signing_secret_key:
|
|
594
|
-
|
|
611
|
+
public_url = sign_storage_url(
|
|
612
|
+
public_url,
|
|
595
613
|
dataset.id,
|
|
596
614
|
dataset.url_signing_secret_key,
|
|
597
615
|
expires_in=expires_in,
|
|
598
616
|
)
|
|
599
|
-
request_params['signature'] = signature
|
|
600
617
|
|
|
601
|
-
return
|
|
618
|
+
return public_url
|
|
602
619
|
|
|
603
620
|
|
|
604
621
|
@docs_group('Resource clients')
|
|
@@ -1047,15 +1064,26 @@ class DatasetClientAsync(ResourceClientAsync):
|
|
|
1047
1064
|
else:
|
|
1048
1065
|
json = items
|
|
1049
1066
|
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1067
|
+
for resend in (False, True):
|
|
1068
|
+
try:
|
|
1069
|
+
await self._http_client.call(
|
|
1070
|
+
url=self._build_url('items'),
|
|
1071
|
+
method='POST',
|
|
1072
|
+
# The server stores a push once per key, so a push resent after a 5xx or a dropped connection
|
|
1073
|
+
# is safe.
|
|
1074
|
+
headers={'content-type': 'application/json; charset=utf-8', 'idempotency-key': uuid4().hex},
|
|
1075
|
+
params=self._build_params(),
|
|
1076
|
+
data=data,
|
|
1077
|
+
json=json,
|
|
1078
|
+
timeout=timeout,
|
|
1079
|
+
)
|
|
1080
|
+
except ApifyApiError as exc:
|
|
1081
|
+
# The push failed mid-write and the server can't tell whether it was stored. Send it once more under
|
|
1082
|
+
# a new key, as an unkeyed push would be resent: a second copy is possible, losing the items is not.
|
|
1083
|
+
if resend or exc.type != 'push-outcome-unknown':
|
|
1084
|
+
raise
|
|
1085
|
+
else:
|
|
1086
|
+
return
|
|
1059
1087
|
|
|
1060
1088
|
async def get_statistics(self, *, timeout: Timeout = 'short') -> DatasetStatistics:
|
|
1061
1089
|
"""Get the dataset statistics.
|
|
@@ -1103,7 +1131,8 @@ class DatasetClientAsync(ResourceClientAsync):
|
|
|
1103
1131
|
|
|
1104
1132
|
You can optionally control how long the signed URL should be valid using the `expires_in` option.
|
|
1105
1133
|
This value sets the expiration duration from the time the URL is generated.
|
|
1106
|
-
If not provided, the URL
|
|
1134
|
+
If not provided, the URL expires in 1 hour. The signature covers the query parameters, so the URL stops
|
|
1135
|
+
working if any of them is changed.
|
|
1107
1136
|
|
|
1108
1137
|
Any other options (like `limit` or `offset`) will be included as query parameters in the URL.
|
|
1109
1138
|
|
|
@@ -1120,7 +1149,7 @@ class DatasetClientAsync(ResourceClientAsync):
|
|
|
1120
1149
|
skip_hidden: If True, then hidden fields are skipped from the output.
|
|
1121
1150
|
flatten: A list of fields that should be flattened.
|
|
1122
1151
|
view: Name of the dataset view to be used.
|
|
1123
|
-
expires_in: How long the signed URL should be valid.
|
|
1152
|
+
expires_in: How long the signed URL should be valid. Defaults to 1 hour.
|
|
1124
1153
|
timeout: Timeout for the API HTTP request.
|
|
1125
1154
|
|
|
1126
1155
|
Returns:
|
|
@@ -1142,12 +1171,15 @@ class DatasetClientAsync(ResourceClientAsync):
|
|
|
1142
1171
|
view=view,
|
|
1143
1172
|
)
|
|
1144
1173
|
|
|
1174
|
+
public_url = self._build_public_url('items', request_params)
|
|
1175
|
+
|
|
1176
|
+
# Signed last: the signature binds every query parameter above.
|
|
1145
1177
|
if dataset and dataset.url_signing_secret_key:
|
|
1146
|
-
|
|
1178
|
+
public_url = sign_storage_url(
|
|
1179
|
+
public_url,
|
|
1147
1180
|
dataset.id,
|
|
1148
1181
|
dataset.url_signing_secret_key,
|
|
1149
1182
|
expires_in=expires_in,
|
|
1150
1183
|
)
|
|
1151
|
-
request_params['signature'] = signature
|
|
1152
1184
|
|
|
1153
|
-
return
|
|
1185
|
+
return public_url
|