wherobots-python-sdk 0.2.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/PKG-INFO +132 -18
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/README.md +130 -10
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/pyproject.toml +42 -23
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_api.py +6 -6
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_base_client.py +118 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_client.py +196 -8
- wherobots_python_sdk-0.4.0/tests/test_client_header.py +216 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_config.py +14 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_exceptions.py +2 -3
- wherobots_python_sdk-0.4.0/tests/test_files_api.py +735 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_init_exports.py +8 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_integration.py +1 -1
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_models.py +128 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_security.py +4 -4
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/__init__.py +4 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/__version__.py +1 -1
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/api/base.py +17 -10
- wherobots_python_sdk-0.4.0/wherobots/api/files.py +580 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/api/runs.py +8 -8
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/client.py +121 -38
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/config.py +15 -4
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/exceptions.py +3 -3
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/models.py +149 -18
- wherobots_python_sdk-0.4.0/wherobots/utils/client_header.py +205 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/utils/validation.py +1 -1
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots_python_sdk.egg-info/PKG-INFO +132 -18
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots_python_sdk.egg-info/SOURCES.txt +2 -0
- wherobots_python_sdk-0.4.0/wherobots_python_sdk.egg-info/requires.txt +2 -0
- wherobots_python_sdk-0.2.1/tests/test_files_api.py +0 -622
- wherobots_python_sdk-0.2.1/wherobots/api/files.py +0 -386
- wherobots_python_sdk-0.2.1/wherobots_python_sdk.egg-info/requires.txt +0 -9
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/LICENSE +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/setup.cfg +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_enums.py +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_logger.py +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_regressions.py +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/tests/test_utils.py +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/api/__init__.py +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/enums.py +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/py.typed +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/utils/__init__.py +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots/utils/logger.py +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots_python_sdk.egg-info/dependency_links.txt +0 -0
- {wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.4.0}/wherobots_python_sdk.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wherobots-python-sdk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Python SDK for Wherobots (currently covers the Jobs REST API)
|
|
5
5
|
Author-email: Wherobots <support@wherobots.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -22,13 +22,7 @@ Requires-Python: >=3.10
|
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
23
23
|
License-File: LICENSE
|
|
24
24
|
Requires-Dist: requests>=2.33.0
|
|
25
|
-
|
|
26
|
-
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
27
|
-
Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
28
|
-
Requires-Dist: pytest-mock>=3.11.1; extra == "dev"
|
|
29
|
-
Requires-Dist: ruff>=0.6.0; extra == "dev"
|
|
30
|
-
Requires-Dist: mypy>=1.5.0; extra == "dev"
|
|
31
|
-
Requires-Dist: types-requests>=2.31; extra == "dev"
|
|
25
|
+
Requires-Dist: boto3>=1.34.0
|
|
32
26
|
Dynamic: license-file
|
|
33
27
|
|
|
34
28
|
# Wherobots Python SDK
|
|
@@ -66,11 +60,11 @@ The simplest way to run a local script — only an API key is needed:
|
|
|
66
60
|
from wherobots import WherobotsJob
|
|
67
61
|
|
|
68
62
|
job = WherobotsJob(
|
|
69
|
-
script="my_analysis.py",
|
|
63
|
+
script="my_analysis.py", # local file — auto-uploaded via presigned URL
|
|
70
64
|
name="analysis-job-001",
|
|
71
65
|
runtime="tiny",
|
|
72
66
|
region="aws-us-west-2",
|
|
73
|
-
api_key="wbk_...",
|
|
67
|
+
api_key="wbk_...", # or set WHEROBOTS_API_KEY env var
|
|
74
68
|
)
|
|
75
69
|
|
|
76
70
|
run_id = job.submit()
|
|
@@ -134,6 +128,7 @@ The API key can also be set via the `WHEROBOTS_API_KEY` environment variable.
|
|
|
134
128
|
| `WHEROBOTS_API_BASE_URL` | API base URL | `https://api.cloud.wherobots.com` |
|
|
135
129
|
| `WHEROBOTS_VERSION` | Wherobots version | `latest` |
|
|
136
130
|
| `WHEROBOTS_REQUEST_TIMEOUT_SECONDS`| HTTP request timeout (seconds) | `30` |
|
|
131
|
+
| `WHEROBOTS_CLIENT_CHAIN` | Upstream client-attribution chain (see below) | *(none)* |
|
|
137
132
|
|
|
138
133
|
### WherobotsConfig
|
|
139
134
|
|
|
@@ -163,6 +158,35 @@ config = WherobotsConfig(
|
|
|
163
158
|
| `base_url` | `str` | `"https://api.cloud.wherobots.com"` |
|
|
164
159
|
| `version` | `str` | `"latest"` |
|
|
165
160
|
| `request_timeout_seconds` | `int` | `30` |
|
|
161
|
+
| `client_chain` | `str \| None` | `None` |
|
|
162
|
+
|
|
163
|
+
### Client attribution
|
|
164
|
+
|
|
165
|
+
Every request the SDK sends carries an advisory `X-Wherobots-Client` header so
|
|
166
|
+
Wherobots can attribute workloads to the client that started them:
|
|
167
|
+
|
|
168
|
+
```
|
|
169
|
+
X-Wherobots-Client: client=python-sdk;ver=0.3.0;plat=darwin
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
The header is an ordered, comma-separated chain of hops — leftmost is the
|
|
173
|
+
origin client, and each component appends its own hop on the right. If your
|
|
174
|
+
tool embeds this SDK, pass your own hop as `client_chain` and the SDK appends
|
|
175
|
+
itself after it:
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
job = WherobotsJob(
|
|
179
|
+
script="job.py",
|
|
180
|
+
name="my-job-name",
|
|
181
|
+
client_chain="client=airflow;ver=1.2.0",
|
|
182
|
+
)
|
|
183
|
+
# X-Wherobots-Client: client=airflow;ver=1.2.0, client=python-sdk;ver=0.3.0;plat=linux
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
`client_chain` also works on `WherobotsJob.from_run_id(...)` and
|
|
187
|
+
`WherobotsConfig`, or can be set once via `WHEROBOTS_CLIENT_CHAIN`. The header
|
|
188
|
+
is advisory only: it is used for attribution and analytics and never affects
|
|
189
|
+
authentication or authorization.
|
|
166
190
|
|
|
167
191
|
## SDK Reference
|
|
168
192
|
|
|
@@ -187,10 +211,11 @@ WherobotsJob(
|
|
|
187
211
|
spark_driver_disk_gb: int | None = None, # Driver disk size (GB)
|
|
188
212
|
spark_executor_disk_gb: int | None = None, # Executor disk size (GB)
|
|
189
213
|
jar_main_class: str | None = None, # Main class (required for JARs)
|
|
190
|
-
auto_upload: bool = True, # Upload local files to S3
|
|
214
|
+
auto_upload: bool = True, # Upload local files to S3 (managed storage)
|
|
191
215
|
base_url: str | None = None, # Override API URL
|
|
192
216
|
request_timeout_seconds: int | None = None, # HTTP timeout
|
|
193
217
|
config: WherobotsConfig | None = None, # Full config object
|
|
218
|
+
client_chain: str | None = None, # Upstream X-Wherobots-Client chain
|
|
194
219
|
)
|
|
195
220
|
```
|
|
196
221
|
|
|
@@ -235,7 +260,7 @@ with WherobotsJob(script="s3://bucket/script.py", name="my-job") as job:
|
|
|
235
260
|
| `from_run_id(run_id, api_key=None, ...)` | `WherobotsJob` | Attach to an existing run for read-only log/metric access. No script required. `submit()` is disabled on the returned instance. |
|
|
236
261
|
| `list_runs(...)` | `RunListPage` | List runs with optional filters. No instance required. |
|
|
237
262
|
| `add_pypi_dependency(name, version)` | `dict` | Create a PyPI dependency dict for the `dependencies` parameter. |
|
|
238
|
-
| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). |
|
|
263
|
+
| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). Accepts an `s3://` URI or a local path — local paths are auto-uploaded to **managed storage** at `submit()` time. For non-managed destinations, upload via `FilesAPI` first and pass the resulting `s3://` URI. |
|
|
239
264
|
|
|
240
265
|
#### Attaching to an Existing Run
|
|
241
266
|
|
|
@@ -307,6 +332,80 @@ job = WherobotsJob(
|
|
|
307
332
|
)
|
|
308
333
|
```
|
|
309
334
|
|
|
335
|
+
##### Local Dependency Files
|
|
336
|
+
|
|
337
|
+
`add_file_dependency` also accepts a **local path** to a `.whl`, `.jar`, `.zip`,
|
|
338
|
+
or `.json`. The SDK uploads the file just before the job is submitted, fetching
|
|
339
|
+
short-lived AWS STS credentials from the Wherobots API (`POST /storage/credentials`)
|
|
340
|
+
and using boto3 multipart under the hood — no AWS credentials need to leave your
|
|
341
|
+
environment.
|
|
342
|
+
|
|
343
|
+
```python
|
|
344
|
+
job = WherobotsJob(
|
|
345
|
+
script="my_job.py", # local script auto-upload still works
|
|
346
|
+
name="job-with-local-wheel",
|
|
347
|
+
dependencies=[
|
|
348
|
+
# Local wheel — uploaded to managed storage at submit() time
|
|
349
|
+
WherobotsJob.add_file_dependency("dist/mypkg-0.1.0-py3-none-any.whl"),
|
|
350
|
+
# Already-uploaded artifact — used as-is
|
|
351
|
+
WherobotsJob.add_file_dependency("s3://bucket/shared-libs/lib-2.0.jar"),
|
|
352
|
+
WherobotsJob.add_pypi_dependency("pandas", "2.0.3"),
|
|
353
|
+
],
|
|
354
|
+
)
|
|
355
|
+
job.submit()
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
Auto-upload always targets the org's **managed storage**:
|
|
359
|
+
|
|
360
|
+
| Artifact | Destination |
|
|
361
|
+
|---|---|
|
|
362
|
+
| Script (when `script` is a local path) | `s3://<managed.path>/scripts/<filename>` |
|
|
363
|
+
| Dependency (when path is a local file) | `s3://<managed.path>/dependencies/<filename>` |
|
|
364
|
+
|
|
365
|
+
Local dependency uploads honor the same `auto_upload` flag as the script —
|
|
366
|
+
if `auto_upload=False`, attempting to submit a job with a local dependency
|
|
367
|
+
path raises `WherobotsValidationError`. The 5 GiB cap on a single upload
|
|
368
|
+
applies; for larger artifacts, pre-upload and pass an `s3://` URI.
|
|
369
|
+
|
|
370
|
+
### Uploading to a Storage Integration
|
|
371
|
+
|
|
372
|
+
`WherobotsJob` stays focused on job orchestration — it doesn't take any
|
|
373
|
+
"where files go" kwargs. To upload to a [Storage Integration](https://docs.wherobots.com/latest/develop/storage-management/storage/)
|
|
374
|
+
or a custom subdirectory, use `FilesAPI` directly and pass the resulting
|
|
375
|
+
`s3://` URI to `WherobotsJob`:
|
|
376
|
+
|
|
377
|
+
```python
|
|
378
|
+
from wherobots import FilesAPI, WherobotsJob
|
|
379
|
+
from wherobots.config import WherobotsConfig
|
|
380
|
+
|
|
381
|
+
with FilesAPI.from_config(WherobotsConfig.from_env()) as files:
|
|
382
|
+
si = files.resolve_integration("customer-bucket")
|
|
383
|
+
# Build explicit destinations; upload_file is the "upload this file
|
|
384
|
+
# to this exact s3 path" primitive.
|
|
385
|
+
script_uri = files.upload_file(
|
|
386
|
+
"my_job.py",
|
|
387
|
+
files.dest_uri_for(si, "my_job.py", "job-scripts/sdk-e2e"),
|
|
388
|
+
)
|
|
389
|
+
wheel_uri = files.upload_file(
|
|
390
|
+
"dist/internal-lib-1.0.whl",
|
|
391
|
+
files.dest_uri_for(si, "internal-lib-1.0.whl", "job-scripts/sdk-e2e"),
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
job = WherobotsJob(
|
|
395
|
+
script=script_uri, # already-uploaded URI
|
|
396
|
+
name="job-in-integration",
|
|
397
|
+
dependencies=[WherobotsJob.add_file_dependency(wheel_uri)],
|
|
398
|
+
)
|
|
399
|
+
job.submit()
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
Both artifacts land under `s3://<integration.path>/job-scripts/sdk-e2e/`.
|
|
403
|
+
|
|
404
|
+
`FilesAPI` is also where to look for low-level controls (custom managed
|
|
405
|
+
subdirs, raw `s3://` destinations, STS credential fetch for BYO upload
|
|
406
|
+
tooling). See `FilesAPI.upload_file`, `upload_managed_file`,
|
|
407
|
+
`upload_dependency`, `dest_uri_for`, and `get_storage_credentials`.
|
|
408
|
+
|
|
310
409
|
#### JAR Jobs
|
|
311
410
|
|
|
312
411
|
```python
|
|
@@ -553,10 +652,8 @@ Base URL: `https://api.cloud.wherobots.com`
|
|
|
553
652
|
```bash
|
|
554
653
|
git clone https://github.com/wherobots/wherobots-python-sdk.git
|
|
555
654
|
cd wherobots-python-sdk
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
pip install -e ".[dev]"
|
|
559
|
-
pre-commit install
|
|
655
|
+
uv sync --group dev
|
|
656
|
+
uv run pre-commit install
|
|
560
657
|
```
|
|
561
658
|
|
|
562
659
|
### Quality checks
|
|
@@ -564,10 +661,27 @@ pre-commit install
|
|
|
564
661
|
```bash
|
|
565
662
|
make lint # ruff check + ruff format --check
|
|
566
663
|
make format # auto-fix lint issues and reformat
|
|
567
|
-
make typecheck #
|
|
568
|
-
make
|
|
664
|
+
make typecheck # pyrefly (strict config)
|
|
665
|
+
make rails # import-linter + no-suppression + ast-grep
|
|
666
|
+
make check # lint + typecheck + rails + tests + build
|
|
569
667
|
```
|
|
570
668
|
|
|
669
|
+
### Suppression baseline
|
|
670
|
+
|
|
671
|
+
Pre-existing debt is grandfathered in `.slop-suppressions.json` (CODEOWNED): the inline
|
|
672
|
+
suppressions already in the tree, plus the `# noqa: CODE` / `# pyrefly: ignore` comments that
|
|
673
|
+
`ruff check --add-noqa` and `pyrefly check --suppress-errors` wrote for pre-existing violations.
|
|
674
|
+
The suppressions are functional at the tool level; the keystone
|
|
675
|
+
(`.slop-repellant/check_no_suppression.py`) fails any net-new one.
|
|
676
|
+
|
|
677
|
+
- **Prune** after fixing baselined code (drops stale entries):
|
|
678
|
+
- ruff's `RUF100` auto-removes stale noqas via the pre-commit hook's `--fix`
|
|
679
|
+
- `uv run pyrefly suppress --remove-unused`
|
|
680
|
+
- then `uv run python .slop-repellant/check_no_suppression.py --update`
|
|
681
|
+
- **Reset** (re-record from scratch): delete `.slop-suppressions.json`, then run the same `--update`.
|
|
682
|
+
- Shrinking the baseline is always welcome; **growing it requires CODEOWNER sign-off** — fix the
|
|
683
|
+
code instead of widening the baseline.
|
|
684
|
+
|
|
571
685
|
### Releasing
|
|
572
686
|
|
|
573
687
|
See [`RELEASING.md`](RELEASING.md) for the release flow (version
|
|
@@ -33,11 +33,11 @@ The simplest way to run a local script — only an API key is needed:
|
|
|
33
33
|
from wherobots import WherobotsJob
|
|
34
34
|
|
|
35
35
|
job = WherobotsJob(
|
|
36
|
-
script="my_analysis.py",
|
|
36
|
+
script="my_analysis.py", # local file — auto-uploaded via presigned URL
|
|
37
37
|
name="analysis-job-001",
|
|
38
38
|
runtime="tiny",
|
|
39
39
|
region="aws-us-west-2",
|
|
40
|
-
api_key="wbk_...",
|
|
40
|
+
api_key="wbk_...", # or set WHEROBOTS_API_KEY env var
|
|
41
41
|
)
|
|
42
42
|
|
|
43
43
|
run_id = job.submit()
|
|
@@ -101,6 +101,7 @@ The API key can also be set via the `WHEROBOTS_API_KEY` environment variable.
|
|
|
101
101
|
| `WHEROBOTS_API_BASE_URL` | API base URL | `https://api.cloud.wherobots.com` |
|
|
102
102
|
| `WHEROBOTS_VERSION` | Wherobots version | `latest` |
|
|
103
103
|
| `WHEROBOTS_REQUEST_TIMEOUT_SECONDS`| HTTP request timeout (seconds) | `30` |
|
|
104
|
+
| `WHEROBOTS_CLIENT_CHAIN` | Upstream client-attribution chain (see below) | *(none)* |
|
|
104
105
|
|
|
105
106
|
### WherobotsConfig
|
|
106
107
|
|
|
@@ -130,6 +131,35 @@ config = WherobotsConfig(
|
|
|
130
131
|
| `base_url` | `str` | `"https://api.cloud.wherobots.com"` |
|
|
131
132
|
| `version` | `str` | `"latest"` |
|
|
132
133
|
| `request_timeout_seconds` | `int` | `30` |
|
|
134
|
+
| `client_chain` | `str \| None` | `None` |
|
|
135
|
+
|
|
136
|
+
### Client attribution
|
|
137
|
+
|
|
138
|
+
Every request the SDK sends carries an advisory `X-Wherobots-Client` header so
|
|
139
|
+
Wherobots can attribute workloads to the client that started them:
|
|
140
|
+
|
|
141
|
+
```
|
|
142
|
+
X-Wherobots-Client: client=python-sdk;ver=0.3.0;plat=darwin
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
The header is an ordered, comma-separated chain of hops — leftmost is the
|
|
146
|
+
origin client, and each component appends its own hop on the right. If your
|
|
147
|
+
tool embeds this SDK, pass your own hop as `client_chain` and the SDK appends
|
|
148
|
+
itself after it:
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
job = WherobotsJob(
|
|
152
|
+
script="job.py",
|
|
153
|
+
name="my-job-name",
|
|
154
|
+
client_chain="client=airflow;ver=1.2.0",
|
|
155
|
+
)
|
|
156
|
+
# X-Wherobots-Client: client=airflow;ver=1.2.0, client=python-sdk;ver=0.3.0;plat=linux
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
`client_chain` also works on `WherobotsJob.from_run_id(...)` and
|
|
160
|
+
`WherobotsConfig`, or can be set once via `WHEROBOTS_CLIENT_CHAIN`. The header
|
|
161
|
+
is advisory only: it is used for attribution and analytics and never affects
|
|
162
|
+
authentication or authorization.
|
|
133
163
|
|
|
134
164
|
## SDK Reference
|
|
135
165
|
|
|
@@ -154,10 +184,11 @@ WherobotsJob(
|
|
|
154
184
|
spark_driver_disk_gb: int | None = None, # Driver disk size (GB)
|
|
155
185
|
spark_executor_disk_gb: int | None = None, # Executor disk size (GB)
|
|
156
186
|
jar_main_class: str | None = None, # Main class (required for JARs)
|
|
157
|
-
auto_upload: bool = True, # Upload local files to S3
|
|
187
|
+
auto_upload: bool = True, # Upload local files to S3 (managed storage)
|
|
158
188
|
base_url: str | None = None, # Override API URL
|
|
159
189
|
request_timeout_seconds: int | None = None, # HTTP timeout
|
|
160
190
|
config: WherobotsConfig | None = None, # Full config object
|
|
191
|
+
client_chain: str | None = None, # Upstream X-Wherobots-Client chain
|
|
161
192
|
)
|
|
162
193
|
```
|
|
163
194
|
|
|
@@ -202,7 +233,7 @@ with WherobotsJob(script="s3://bucket/script.py", name="my-job") as job:
|
|
|
202
233
|
| `from_run_id(run_id, api_key=None, ...)` | `WherobotsJob` | Attach to an existing run for read-only log/metric access. No script required. `submit()` is disabled on the returned instance. |
|
|
203
234
|
| `list_runs(...)` | `RunListPage` | List runs with optional filters. No instance required. |
|
|
204
235
|
| `add_pypi_dependency(name, version)` | `dict` | Create a PyPI dependency dict for the `dependencies` parameter. |
|
|
205
|
-
| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). |
|
|
236
|
+
| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). Accepts an `s3://` URI or a local path — local paths are auto-uploaded to **managed storage** at `submit()` time. For non-managed destinations, upload via `FilesAPI` first and pass the resulting `s3://` URI. |
|
|
206
237
|
|
|
207
238
|
#### Attaching to an Existing Run
|
|
208
239
|
|
|
@@ -274,6 +305,80 @@ job = WherobotsJob(
|
|
|
274
305
|
)
|
|
275
306
|
```
|
|
276
307
|
|
|
308
|
+
##### Local Dependency Files
|
|
309
|
+
|
|
310
|
+
`add_file_dependency` also accepts a **local path** to a `.whl`, `.jar`, `.zip`,
|
|
311
|
+
or `.json`. The SDK uploads the file just before the job is submitted, fetching
|
|
312
|
+
short-lived AWS STS credentials from the Wherobots API (`POST /storage/credentials`)
|
|
313
|
+
and using boto3 multipart under the hood — no AWS credentials need to leave your
|
|
314
|
+
environment.
|
|
315
|
+
|
|
316
|
+
```python
|
|
317
|
+
job = WherobotsJob(
|
|
318
|
+
script="my_job.py", # local script auto-upload still works
|
|
319
|
+
name="job-with-local-wheel",
|
|
320
|
+
dependencies=[
|
|
321
|
+
# Local wheel — uploaded to managed storage at submit() time
|
|
322
|
+
WherobotsJob.add_file_dependency("dist/mypkg-0.1.0-py3-none-any.whl"),
|
|
323
|
+
# Already-uploaded artifact — used as-is
|
|
324
|
+
WherobotsJob.add_file_dependency("s3://bucket/shared-libs/lib-2.0.jar"),
|
|
325
|
+
WherobotsJob.add_pypi_dependency("pandas", "2.0.3"),
|
|
326
|
+
],
|
|
327
|
+
)
|
|
328
|
+
job.submit()
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
Auto-upload always targets the org's **managed storage**:
|
|
332
|
+
|
|
333
|
+
| Artifact | Destination |
|
|
334
|
+
|---|---|
|
|
335
|
+
| Script (when `script` is a local path) | `s3://<managed.path>/scripts/<filename>` |
|
|
336
|
+
| Dependency (when path is a local file) | `s3://<managed.path>/dependencies/<filename>` |
|
|
337
|
+
|
|
338
|
+
Local dependency uploads honor the same `auto_upload` flag as the script —
|
|
339
|
+
if `auto_upload=False`, attempting to submit a job with a local dependency
|
|
340
|
+
path raises `WherobotsValidationError`. The 5 GiB cap on a single upload
|
|
341
|
+
applies; for larger artifacts, pre-upload and pass an `s3://` URI.
|
|
342
|
+
|
|
343
|
+
### Uploading to a Storage Integration
|
|
344
|
+
|
|
345
|
+
`WherobotsJob` stays focused on job orchestration — it doesn't take any
|
|
346
|
+
"where files go" kwargs. To upload to a [Storage Integration](https://docs.wherobots.com/latest/develop/storage-management/storage/)
|
|
347
|
+
or a custom subdirectory, use `FilesAPI` directly and pass the resulting
|
|
348
|
+
`s3://` URI to `WherobotsJob`:
|
|
349
|
+
|
|
350
|
+
```python
|
|
351
|
+
from wherobots import FilesAPI, WherobotsJob
|
|
352
|
+
from wherobots.config import WherobotsConfig
|
|
353
|
+
|
|
354
|
+
with FilesAPI.from_config(WherobotsConfig.from_env()) as files:
|
|
355
|
+
si = files.resolve_integration("customer-bucket")
|
|
356
|
+
# Build explicit destinations; upload_file is the "upload this file
|
|
357
|
+
# to this exact s3 path" primitive.
|
|
358
|
+
script_uri = files.upload_file(
|
|
359
|
+
"my_job.py",
|
|
360
|
+
files.dest_uri_for(si, "my_job.py", "job-scripts/sdk-e2e"),
|
|
361
|
+
)
|
|
362
|
+
wheel_uri = files.upload_file(
|
|
363
|
+
"dist/internal-lib-1.0.whl",
|
|
364
|
+
files.dest_uri_for(si, "internal-lib-1.0.whl", "job-scripts/sdk-e2e"),
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
job = WherobotsJob(
|
|
368
|
+
script=script_uri, # already-uploaded URI
|
|
369
|
+
name="job-in-integration",
|
|
370
|
+
dependencies=[WherobotsJob.add_file_dependency(wheel_uri)],
|
|
371
|
+
)
|
|
372
|
+
job.submit()
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
Both artifacts land under `s3://<integration.path>/job-scripts/sdk-e2e/`.
|
|
376
|
+
|
|
377
|
+
`FilesAPI` is also where to look for low-level controls (custom managed
|
|
378
|
+
subdirs, raw `s3://` destinations, STS credential fetch for BYO upload
|
|
379
|
+
tooling). See `FilesAPI.upload_file`, `upload_managed_file`,
|
|
380
|
+
`upload_dependency`, `dest_uri_for`, and `get_storage_credentials`.
|
|
381
|
+
|
|
277
382
|
#### JAR Jobs
|
|
278
383
|
|
|
279
384
|
```python
|
|
@@ -520,10 +625,8 @@ Base URL: `https://api.cloud.wherobots.com`
|
|
|
520
625
|
```bash
|
|
521
626
|
git clone https://github.com/wherobots/wherobots-python-sdk.git
|
|
522
627
|
cd wherobots-python-sdk
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
pip install -e ".[dev]"
|
|
526
|
-
pre-commit install
|
|
628
|
+
uv sync --group dev
|
|
629
|
+
uv run pre-commit install
|
|
527
630
|
```
|
|
528
631
|
|
|
529
632
|
### Quality checks
|
|
@@ -531,10 +634,27 @@ pre-commit install
|
|
|
531
634
|
```bash
|
|
532
635
|
make lint # ruff check + ruff format --check
|
|
533
636
|
make format # auto-fix lint issues and reformat
|
|
534
|
-
make typecheck #
|
|
535
|
-
make
|
|
637
|
+
make typecheck # pyrefly (strict config)
|
|
638
|
+
make rails # import-linter + no-suppression + ast-grep
|
|
639
|
+
make check # lint + typecheck + rails + tests + build
|
|
536
640
|
```
|
|
537
641
|
|
|
642
|
+
### Suppression baseline
|
|
643
|
+
|
|
644
|
+
Pre-existing debt is grandfathered in `.slop-suppressions.json` (CODEOWNED): the inline
|
|
645
|
+
suppressions already in the tree, plus the `# noqa: CODE` / `# pyrefly: ignore` comments that
|
|
646
|
+
`ruff check --add-noqa` and `pyrefly check --suppress-errors` wrote for pre-existing violations.
|
|
647
|
+
The suppressions are functional at the tool level; the keystone
|
|
648
|
+
(`.slop-repellant/check_no_suppression.py`) fails any net-new one.
|
|
649
|
+
|
|
650
|
+
- **Prune** after fixing baselined code (drops stale entries):
|
|
651
|
+
- ruff's `RUF100` auto-removes stale noqas via the pre-commit hook's `--fix`
|
|
652
|
+
- `uv run pyrefly suppress --remove-unused`
|
|
653
|
+
- then `uv run python .slop-repellant/check_no_suppression.py --update`
|
|
654
|
+
- **Reset** (re-record from scratch): delete `.slop-suppressions.json`, then run the same `--update`.
|
|
655
|
+
- Shrinking the baseline is always welcome; **growing it requires CODEOWNER sign-off** — fix the
|
|
656
|
+
code instead of widening the baseline.
|
|
657
|
+
|
|
538
658
|
### Releasing
|
|
539
659
|
|
|
540
660
|
See [`RELEASING.md`](RELEASING.md) for the release flow (version
|
|
@@ -26,6 +26,7 @@ classifiers = [
|
|
|
26
26
|
keywords = ["wherobots", "spark", "sedona", "geospatial", "jobs", "sdk", "rest-api"]
|
|
27
27
|
dependencies = [
|
|
28
28
|
"requests>=2.33.0",
|
|
29
|
+
"boto3>=1.34.0",
|
|
29
30
|
]
|
|
30
31
|
|
|
31
32
|
[project.urls]
|
|
@@ -34,13 +35,19 @@ Repository = "https://github.com/wherobots/wherobots-python-sdk"
|
|
|
34
35
|
Issues = "https://github.com/wherobots/wherobots-python-sdk/issues"
|
|
35
36
|
Documentation = "https://docs.wherobots.com"
|
|
36
37
|
|
|
37
|
-
|
|
38
|
+
# PEP 735 dev dependencies — installed with `uv sync --group dev`.
|
|
39
|
+
[dependency-groups]
|
|
38
40
|
dev = [
|
|
39
|
-
"pytest>=
|
|
41
|
+
"pytest>=8.0",
|
|
40
42
|
"pytest-cov>=4.1.0",
|
|
41
43
|
"pytest-mock>=3.11.1",
|
|
42
|
-
"ruff>=0.
|
|
43
|
-
"
|
|
44
|
+
"ruff>=0.15,<0.17",
|
|
45
|
+
"pyrefly>=0.16",
|
|
46
|
+
"import-linter>=2.1",
|
|
47
|
+
"pre-commit>=3.8",
|
|
48
|
+
"hypothesis>=6.100",
|
|
49
|
+
"cosmic-ray>=8.4",
|
|
50
|
+
"ast-grep-cli>=0.39",
|
|
44
51
|
"types-requests>=2.31",
|
|
45
52
|
]
|
|
46
53
|
|
|
@@ -53,37 +60,49 @@ wherobots = ["py.typed"]
|
|
|
53
60
|
[tool.setuptools.dynamic]
|
|
54
61
|
version = {attr = "wherobots.__version__.__version__"}
|
|
55
62
|
|
|
63
|
+
# SLOP REPELLANT ruff rail: line-length 100, py310 target, select = ["ALL"]
|
|
64
|
+
# plus the typing.Any/typing.cast ban all come from the base config.
|
|
65
|
+
# NOTE: `lint.select`/`lint.ignore` would REPLACE the base's — tune only via
|
|
66
|
+
# the extend-* keys.
|
|
56
67
|
[tool.ruff]
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
68
|
+
extend = ".slop-repellant/ruff-base.toml"
|
|
69
|
+
# .slop-repellant/ is vendored, kit-MANAGED tooling (re-rendered on
|
|
70
|
+
# slop-repellant-update) — linting it locally would fork the managed files.
|
|
71
|
+
extend-exclude = ["env", ".venv", "venv", "build", "dist", ".slop-repellant"]
|
|
60
72
|
|
|
61
73
|
[tool.ruff.lint]
|
|
62
|
-
select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
|
|
63
74
|
# SIM117 merges nested with-statements — often hurts readability when
|
|
64
75
|
# the context managers are logically distinct. Disable.
|
|
65
|
-
|
|
76
|
+
# EM101/EM102/TRY003 (exception-message-as-literal trio): style position,
|
|
77
|
+
# not debt — we keep messages inline at the raise site. Disable globally.
|
|
78
|
+
# PLR0917 (too-many-positional-arguments): newly enabled in ruff 0.16's ALL
|
|
79
|
+
# set. It is the positional-only variant of PLR0913 (too-many-arguments),
|
|
80
|
+
# which the few wide signatures here already carry an accepted `# noqa:
|
|
81
|
+
# PLR0913` for — so PLR0917 is redundant with a complaint the codebase has
|
|
82
|
+
# already signed off on. Ignored globally rather than making those args
|
|
83
|
+
# keyword-only, which would be a semver-breaking public-API change.
|
|
84
|
+
extend-ignore = ["SIM117", "EM101", "EM102", "TRY003", "PLR0917"]
|
|
66
85
|
|
|
67
|
-
[tool.ruff.lint.per-file-ignores]
|
|
86
|
+
[tool.ruff.lint.extend-per-file-ignores]
|
|
68
87
|
# Tests may assert freely, re-raise without chaining, shadow builtins
|
|
69
88
|
# for fixtures, and use mock call patterns that look like unused
|
|
70
|
-
# locals.
|
|
71
|
-
|
|
89
|
+
# locals. N803/N806 fire on @patch-injected MockSession-style names and
|
|
90
|
+
# PLC0415 on deliberate deferred imports — idiomatic in tests, not debt.
|
|
91
|
+
# (Extends the base's tests/** ignore set.)
|
|
92
|
+
"tests/**/*.py" = ["B011", "B904", "F841", "RUF059", "RUF013", "N803", "N806", "PLC0415"]
|
|
93
|
+
# e2e/ holds standalone CLI scripts: printing is their job (T201), and
|
|
94
|
+
# they are intentionally not a package (INP001) with exec bits (EXE001).
|
|
95
|
+
"e2e/**/*.py" = ["T201", "INP001", "EXE001"]
|
|
72
96
|
# Package __init__ groups exports by category with comment headers;
|
|
73
97
|
# don't let ruff re-sort them alphabetically.
|
|
74
98
|
"wherobots/__init__.py" = ["RUF022"]
|
|
75
99
|
|
|
76
|
-
[tool.
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
[[tool.mypy.overrides]]
|
|
84
|
-
# Test code is not held to strict typing — productionize incrementally.
|
|
85
|
-
module = "tests.*"
|
|
86
|
-
ignore_errors = true
|
|
100
|
+
[tool.pyrefly]
|
|
101
|
+
project-includes = ["wherobots"]
|
|
102
|
+
check-unannotated-defs = true
|
|
103
|
+
strict-callable-subtyping = true
|
|
104
|
+
permissive-ignores = false
|
|
105
|
+
ignore-errors-in-generated-code = false
|
|
87
106
|
|
|
88
107
|
[tool.pytest.ini_options]
|
|
89
108
|
testpaths = ["tests"]
|
|
@@ -49,7 +49,7 @@ def _make_response(json_data, status_code=200):
|
|
|
49
49
|
|
|
50
50
|
|
|
51
51
|
# =========================================================================== #
|
|
52
|
-
# list()
|
|
52
|
+
# list() # noqa: ERA001
|
|
53
53
|
# =========================================================================== #
|
|
54
54
|
|
|
55
55
|
|
|
@@ -120,7 +120,7 @@ class TestList:
|
|
|
120
120
|
|
|
121
121
|
|
|
122
122
|
# =========================================================================== #
|
|
123
|
-
# create()
|
|
123
|
+
# create() # noqa: ERA001
|
|
124
124
|
# =========================================================================== #
|
|
125
125
|
|
|
126
126
|
|
|
@@ -177,7 +177,7 @@ class TestCreate:
|
|
|
177
177
|
|
|
178
178
|
|
|
179
179
|
# =========================================================================== #
|
|
180
|
-
# get()
|
|
180
|
+
# get() # noqa: ERA001
|
|
181
181
|
# =========================================================================== #
|
|
182
182
|
|
|
183
183
|
|
|
@@ -197,7 +197,7 @@ class TestGet:
|
|
|
197
197
|
|
|
198
198
|
|
|
199
199
|
# =========================================================================== #
|
|
200
|
-
# cancel()
|
|
200
|
+
# cancel() # noqa: ERA001
|
|
201
201
|
# =========================================================================== #
|
|
202
202
|
|
|
203
203
|
|
|
@@ -213,7 +213,7 @@ class TestCancel:
|
|
|
213
213
|
|
|
214
214
|
|
|
215
215
|
# =========================================================================== #
|
|
216
|
-
# get_logs()
|
|
216
|
+
# get_logs() # noqa: ERA001
|
|
217
217
|
# =========================================================================== #
|
|
218
218
|
|
|
219
219
|
|
|
@@ -241,7 +241,7 @@ class TestGetLogs:
|
|
|
241
241
|
|
|
242
242
|
|
|
243
243
|
# =========================================================================== #
|
|
244
|
-
# get_metrics()
|
|
244
|
+
# get_metrics() # noqa: ERA001
|
|
245
245
|
# =========================================================================== #
|
|
246
246
|
|
|
247
247
|
|