wherobots-python-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wherobots/__init__.py +87 -0
- wherobots/__version__.py +5 -0
- wherobots/api/__init__.py +7 -0
- wherobots/api/base.py +256 -0
- wherobots/api/files.py +386 -0
- wherobots/api/runs.py +255 -0
- wherobots/client.py +640 -0
- wherobots/config.py +165 -0
- wherobots/enums.py +140 -0
- wherobots/exceptions.py +47 -0
- wherobots/models.py +1080 -0
- wherobots/py.typed +0 -0
- wherobots/utils/__init__.py +6 -0
- wherobots/utils/logger.py +34 -0
- wherobots/utils/validation.py +31 -0
- wherobots_python_sdk-0.1.0.dist-info/METADATA +580 -0
- wherobots_python_sdk-0.1.0.dist-info/RECORD +20 -0
- wherobots_python_sdk-0.1.0.dist-info/WHEEL +5 -0
- wherobots_python_sdk-0.1.0.dist-info/licenses/LICENSE +191 -0
- wherobots_python_sdk-0.1.0.dist-info/top_level.txt +1 -0
wherobots/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Library-safe logging for the Wherobots Jobs SDK.
|
|
2
|
+
|
|
3
|
+
Uses a ``NullHandler`` so the library never forces log output on
|
|
4
|
+
applications that embed it. Consumers who *do* want to see SDK logs
|
|
5
|
+
can attach their own handler to the ``"wherobots"`` logger::
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
logging.getLogger("wherobots").addHandler(logging.StreamHandler())
|
|
9
|
+
logging.getLogger("wherobots").setLevel(logging.DEBUG)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_logger(name: str | None = None) -> logging.Logger:
|
|
18
|
+
"""Return a child logger under the ``wherobots`` namespace.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
name: Optional suffix appended to ``"wherobots"``.
|
|
22
|
+
If *None*, the root ``"wherobots"`` logger is returned.
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
A ``logging.Logger`` instance for the requested namespace.
|
|
26
|
+
"""
|
|
27
|
+
base = "wherobots"
|
|
28
|
+
logger_name = f"{base}.{name}" if name else base
|
|
29
|
+
return logging.getLogger(logger_name)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# Attach the NullHandler once at import time so that library consumers
|
|
33
|
+
# never see "No handler found" warnings.
|
|
34
|
+
logging.getLogger("wherobots").addHandler(logging.NullHandler())
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Validation utilities for Wherobots Jobs client."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from wherobots.exceptions import WherobotsValidationError
|
|
6
|
+
|
|
7
|
+
NAME_PATTERN = re.compile(r"^[a-zA-Z0-9_.-]+$")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def validate_name(name: str) -> str:
|
|
11
|
+
"""Validate that a job name meets Wherobots API requirements.
|
|
12
|
+
|
|
13
|
+
Names must be 8-255 characters and contain only alphanumeric
|
|
14
|
+
characters, underscores, hyphens, and periods.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
name: The job name to validate.
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
The validated name (unchanged).
|
|
21
|
+
|
|
22
|
+
Raises:
|
|
23
|
+
WherobotsValidationError: If the name is too short, too long,
|
|
24
|
+
or contains invalid characters.
|
|
25
|
+
"""
|
|
26
|
+
if not 8 <= len(name) <= 255:
|
|
27
|
+
raise WherobotsValidationError("Name must be 8-255 characters")
|
|
28
|
+
|
|
29
|
+
if not NAME_PATTERN.match(name):
|
|
30
|
+
raise WherobotsValidationError("Name must contain only letters, numbers, _, -, and .")
|
|
31
|
+
return name
|
|
@@ -0,0 +1,580 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wherobots-python-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK for Wherobots (currently covers the Jobs REST API)
|
|
5
|
+
Author-email: Wherobots <support@wherobots.com>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://www.wherobots.com
|
|
8
|
+
Project-URL: Repository, https://github.com/wherobots/wherobots-python-sdk
|
|
9
|
+
Project-URL: Issues, https://github.com/wherobots/wherobots-python-sdk/issues
|
|
10
|
+
Project-URL: Documentation, https://docs.wherobots.com
|
|
11
|
+
Keywords: wherobots,spark,sedona,geospatial,jobs,sdk,rest-api
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: requests>=2.33.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-mock>=3.11.1; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff>=0.6.0; extra == "dev"
|
|
30
|
+
Requires-Dist: mypy>=1.5.0; extra == "dev"
|
|
31
|
+
Requires-Dist: types-requests>=2.31; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# Wherobots Python SDK
|
|
35
|
+
|
|
36
|
+
[](https://github.com/wherobots/wherobots-python-sdk/actions/workflows/ci.yml)
|
|
37
|
+
[](https://www.python.org/downloads/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
[](https://pypi.org/project/wherobots-python-sdk/)
|
|
40
|
+
|
|
41
|
+
The official Python SDK for Wherobots. Currently wraps the
|
|
42
|
+
[Wherobots Jobs REST API](https://docs.wherobots.com/reference/runs) —
|
|
43
|
+
submit, monitor, cancel, and inspect Spark/Sedona job runs from Python.
|
|
44
|
+
Additional Wherobots surfaces will be added over time.
|
|
45
|
+
|
|
46
|
+
## Features
|
|
47
|
+
|
|
48
|
+
- Submit Python scripts or JARs as Wherobots jobs
|
|
49
|
+
- Auto-upload local scripts via presigned URLs (only needs an API key)
|
|
50
|
+
- Stream or iterate logs with pagination
|
|
51
|
+
- List runs with status, name, and date filters
|
|
52
|
+
- Typed dataclass models for all API responses (no pydantic dependency)
|
|
53
|
+
- Environment-based configuration
|
|
54
|
+
|
|
55
|
+
## Installation
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install wherobots-python-sdk
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Quickstart
|
|
62
|
+
|
|
63
|
+
The simplest way to run a local script — only an API key is needed:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from wherobots import WherobotsJob
|
|
67
|
+
|
|
68
|
+
job = WherobotsJob(
|
|
69
|
+
script="my_analysis.py", # local file — auto-uploaded via presigned URL
|
|
70
|
+
name="analysis-job-001",
|
|
71
|
+
runtime="tiny",
|
|
72
|
+
region="aws-us-west-2",
|
|
73
|
+
api_key="wbk_...", # or set WHEROBOTS_API_KEY env var
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
run_id = job.submit()
|
|
77
|
+
print("Run ID:", run_id)
|
|
78
|
+
|
|
79
|
+
# Block until done, streaming logs to stdout
|
|
80
|
+
job.wait_for_completion(stream_logs=True)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
You can also pass an S3 URI directly if the script is already uploaded:
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
job = WherobotsJob(
|
|
87
|
+
script="s3://my-bucket/scripts/analysis.py",
|
|
88
|
+
name="analysis-job-001",
|
|
89
|
+
runtime="tiny",
|
|
90
|
+
)
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Script Upload Behavior
|
|
94
|
+
|
|
95
|
+
When `script` is a local file path (not an `s3://` URI) and `auto_upload=True`
|
|
96
|
+
(the default), the SDK uploads your script via a presigned S3 URL to Wherobots
|
|
97
|
+
managed storage. Only needs a valid API key — no AWS credentials required.
|
|
98
|
+
|
|
99
|
+
You can also reference scripts already in a
|
|
100
|
+
[Storage Integration](https://docs.wherobots.com/latest/develop/storage-management/storage/)
|
|
101
|
+
bucket by passing the S3 URI directly:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
job = WherobotsJob(
|
|
105
|
+
script="s3://my-integration-bucket/scripts/my_analysis.py",
|
|
106
|
+
name="analysis-job-001",
|
|
107
|
+
runtime="tiny",
|
|
108
|
+
auto_upload=False,
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Authentication
|
|
113
|
+
|
|
114
|
+
The SDK uses API key authentication via the `X-API-Key` header.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
job = WherobotsJob(
|
|
118
|
+
script="s3://bucket/script.py",
|
|
119
|
+
name="my-job-001",
|
|
120
|
+
api_key="wbk_...",
|
|
121
|
+
)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The API key can also be set via the `WHEROBOTS_API_KEY` environment variable.
|
|
125
|
+
|
|
126
|
+
## Configuration
|
|
127
|
+
|
|
128
|
+
### Environment Variables
|
|
129
|
+
|
|
130
|
+
| Variable | Description | Default |
|
|
131
|
+
|------------------------------------|---------------------------------|--------------------------------------|
|
|
132
|
+
| `WHEROBOTS_API_KEY` | API key | *(required)* |
|
|
133
|
+
| `WHEROBOTS_REGION` | Default AWS region | `aws-us-west-2` |
|
|
134
|
+
| `WHEROBOTS_API_BASE_URL` | API base URL | `https://api.cloud.wherobots.com` |
|
|
135
|
+
| `WHEROBOTS_VERSION` | Wherobots version | `latest` |
|
|
136
|
+
| `WHEROBOTS_REQUEST_TIMEOUT_SECONDS`| HTTP request timeout (seconds) | `30` |
|
|
137
|
+
|
|
138
|
+
### WherobotsConfig
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
from wherobots.config import WherobotsConfig
|
|
142
|
+
|
|
143
|
+
# Build from env vars with optional overrides
|
|
144
|
+
config = WherobotsConfig.from_env(
|
|
145
|
+
api_key="wbk_...",
|
|
146
|
+
region="aws-us-west-2",
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
# Or construct directly
|
|
150
|
+
config = WherobotsConfig(
|
|
151
|
+
api_key="wbk_...",
|
|
152
|
+
region="aws-us-west-2",
|
|
153
|
+
base_url="https://api.cloud.wherobots.com",
|
|
154
|
+
request_timeout_seconds=30,
|
|
155
|
+
version="latest",
|
|
156
|
+
)
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
| Parameter | Type | Default |
|
|
160
|
+
|----------------------------|----------------|-------------------------------------|
|
|
161
|
+
| `api_key` | `str \| None` | `None` (read from env) |
|
|
162
|
+
| `region` | `str \| None` | `None` |
|
|
163
|
+
| `base_url` | `str` | `"https://api.cloud.wherobots.com"` |
|
|
164
|
+
| `version` | `str` | `"latest"` |
|
|
165
|
+
| `request_timeout_seconds` | `int` | `30` |
|
|
166
|
+
|
|
167
|
+
## SDK Reference
|
|
168
|
+
|
|
169
|
+
### WherobotsJob
|
|
170
|
+
|
|
171
|
+
The primary class for managing job runs.
|
|
172
|
+
|
|
173
|
+
#### Constructor
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
WherobotsJob(
|
|
177
|
+
script: str, # Path or S3 URI to .py or .jar
|
|
178
|
+
name: str, # Job name (8-255 chars, [a-zA-Z0-9_\-.]+)
|
|
179
|
+
runtime: str | Runtime = "tiny", # Compute size
|
|
180
|
+
region: str | Region | None = None, # AWS region
|
|
181
|
+
api_key: str | None = None, # API key (or env var)
|
|
182
|
+
version: str | None = None, # "latest" | "preview"
|
|
183
|
+
timeout_seconds: int = 3600, # Job timeout
|
|
184
|
+
args: list[str] | None = None, # Script arguments
|
|
185
|
+
spark_configs: dict[str, str] | None = None, # Spark config overrides
|
|
186
|
+
dependencies: list[dict] | None = None, # PyPI or file dependencies
|
|
187
|
+
spark_driver_disk_gb: int | None = None, # Driver disk size (GB)
|
|
188
|
+
spark_executor_disk_gb: int | None = None, # Executor disk size (GB)
|
|
189
|
+
jar_main_class: str | None = None, # Main class (required for JARs)
|
|
190
|
+
auto_upload: bool = True, # Upload local files to S3
|
|
191
|
+
base_url: str | None = None, # Override API URL
|
|
192
|
+
request_timeout_seconds: int | None = None, # HTTP timeout
|
|
193
|
+
config: WherobotsConfig | None = None, # Full config object
|
|
194
|
+
)
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
#### Instance Methods
|
|
198
|
+
|
|
199
|
+
| Method | Returns | Description |
|
|
200
|
+
|--------|---------|-------------|
|
|
201
|
+
| `submit()` | `str` | Submit the job. Returns the run ID. Idempotent (returns existing ID if already submitted). |
|
|
202
|
+
| `get_status()` | `RunView` | Get current job status and full details. |
|
|
203
|
+
| `get_logs(cursor=0, size=100)` | `LogsResponse` | Fetch a page of log entries. |
|
|
204
|
+
| `get_metrics()` | `RunMetricsResponse` | Fetch CPU/memory metrics for the run. |
|
|
205
|
+
| `iter_logs(cursor=0, size=100)` | `Iterator[dict]` | Iterate over all log entries, handling pagination automatically. |
|
|
206
|
+
| `poll_for_logs(follow=True, interval=2.0, log_handler=None, max_errors=10)` | `None` | Poll and print logs. If `follow=True`, continues until job completes. `max_errors` sets the max consecutive transient errors before giving up. |
|
|
207
|
+
| `cancel()` | `bool` | Request cancellation. Returns `True` on success. |
|
|
208
|
+
| `wait_for_completion(poll_interval=5.0, stream_logs=True, log_interval=2.0, max_wait_seconds=None)` | `JobStatus` | Block until the job reaches a terminal state. Set `max_wait_seconds` to limit wait time (raises `WherobotsTimeoutError`). |
|
|
209
|
+
| `close()` | `None` | Close the underlying HTTP session and release resources. |
|
|
210
|
+
|
|
211
|
+
`WherobotsJob` supports the context manager protocol for automatic cleanup:
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
with WherobotsJob(script="s3://bucket/script.py", name="my-job") as job:
|
|
215
|
+
run_id = job.submit()
|
|
216
|
+
job.wait_for_completion()
|
|
217
|
+
# session is automatically closed
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
> **Alias:** `Job` is a convenience alias for `WherobotsJob`:
|
|
221
|
+
> ```python
|
|
222
|
+
> from wherobots import Job
|
|
223
|
+
> job = Job(script="s3://bucket/script.py", name="my-job")
|
|
224
|
+
> ```
|
|
225
|
+
|
|
226
|
+
#### Class Methods
|
|
227
|
+
|
|
228
|
+
| Method | Returns | Description |
|
|
229
|
+
|--------|---------|-------------|
|
|
230
|
+
| `list_runs(...)` | `RunListPage` | List runs with optional filters. No instance required. |
|
|
231
|
+
| `add_pypi_dependency(name, version)` | `dict` | Create a PyPI dependency dict for the `dependencies` parameter. |
|
|
232
|
+
| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). |
|
|
233
|
+
|
|
234
|
+
#### Listing Runs
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
from wherobots import WherobotsJob, JobStatus
|
|
238
|
+
|
|
239
|
+
page = WherobotsJob.list_runs(
|
|
240
|
+
api_key="wbk_...",
|
|
241
|
+
region="aws-us-west-2",
|
|
242
|
+
status=[JobStatus.RUNNING, JobStatus.PENDING],
|
|
243
|
+
name_pattern="etl-*",
|
|
244
|
+
size=20,
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
for run in page.items:
|
|
248
|
+
print(f"{run.id} {run.name} {run.status.value}")
|
|
249
|
+
|
|
250
|
+
# Paginate
|
|
251
|
+
if page.next_page:
|
|
252
|
+
next_page = WherobotsJob.list_runs(cursor=page.next_page)
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
#### Dependencies
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
from wherobots import WherobotsJob
|
|
259
|
+
|
|
260
|
+
job = WherobotsJob(
|
|
261
|
+
script="s3://bucket/script.py",
|
|
262
|
+
name="dep-test-job",
|
|
263
|
+
dependencies=[
|
|
264
|
+
WherobotsJob.add_pypi_dependency("pandas", "2.0.0"),
|
|
265
|
+
WherobotsJob.add_pypi_dependency("numpy", "1.24.0"),
|
|
266
|
+
WherobotsJob.add_file_dependency("s3://bucket/libs/my_lib-1.0-py3-none-any.whl"),
|
|
267
|
+
],
|
|
268
|
+
spark_configs={
|
|
269
|
+
"spark.executor.memory": "4g",
|
|
270
|
+
"spark.executor.cores": "2",
|
|
271
|
+
},
|
|
272
|
+
)
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
#### JAR Jobs
|
|
276
|
+
|
|
277
|
+
```python
|
|
278
|
+
job = WherobotsJob(
|
|
279
|
+
script="s3://bucket/jars/my-app.jar",
|
|
280
|
+
name="jar-job-001",
|
|
281
|
+
jar_main_class="com.example.Main",
|
|
282
|
+
args=["--input", "s3://data/input", "--output", "s3://data/output"],
|
|
283
|
+
runtime="medium",
|
|
284
|
+
)
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
## Enums
|
|
288
|
+
|
|
289
|
+
### JobStatus
|
|
290
|
+
|
|
291
|
+
| Value | Terminal | Description |
|
|
292
|
+
|--------------|----------|-----------------|
|
|
293
|
+
| `PENDING` | No | Queued |
|
|
294
|
+
| `RUNNING` | No | Executing |
|
|
295
|
+
| `COMPLETED` | Yes | Finished OK |
|
|
296
|
+
| `FAILED` | Yes | Errored out |
|
|
297
|
+
| `CANCELLED` | Yes | User cancelled |
|
|
298
|
+
|
|
299
|
+
```python
|
|
300
|
+
from wherobots import JobStatus
|
|
301
|
+
|
|
302
|
+
status = JobStatus.RUNNING
|
|
303
|
+
status.is_terminal # False
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
### Runtime
|
|
307
|
+
|
|
308
|
+
Available compute sizes (API string values):
|
|
309
|
+
|
|
310
|
+
| Enum | Value |
|
|
311
|
+
|-----------------------------|------------------------|
|
|
312
|
+
| `Runtime.MICRO` | `micro` |
|
|
313
|
+
| `Runtime.TINY` | `tiny` |
|
|
314
|
+
| `Runtime.SMALL` | `small` |
|
|
315
|
+
| `Runtime.MEDIUM` | `medium` |
|
|
316
|
+
| `Runtime.LARGE` | `large` |
|
|
317
|
+
| `Runtime.X_LARGE` | `x-large` |
|
|
318
|
+
| `Runtime.DOUBLE_X_LARGE` | `2x-large` |
|
|
319
|
+
| `Runtime.QUAD_X_LARGE` | `4x-large` |
|
|
320
|
+
| `Runtime.MEDIUM_HIMEM` | `medium-himem` |
|
|
321
|
+
| `Runtime.LARGE_HIMEM` | `large-himem` |
|
|
322
|
+
| `Runtime.X_LARGE_HIMEM` | `x-large-himem` |
|
|
323
|
+
| `Runtime.DOUBLE_X_LARGE_HIMEM` | `2x-large-himem` |
|
|
324
|
+
| `Runtime.QUAD_X_LARGE_HIMEM`| `4x-large-himem` |
|
|
325
|
+
| `Runtime.X_LARGE_HICPU` | `x-large-hicpu` |
|
|
326
|
+
| `Runtime.DOUBLE_X_LARGE_HICPU` | `2x-large-hicpu` |
|
|
327
|
+
| `Runtime.X_LARGE_MATCHER` | `x-large-matcher` |
|
|
328
|
+
| `Runtime.DOUBLE_X_LARGE_MATCHER` | `2x-large-matcher` |
|
|
329
|
+
| `Runtime.MICRO_A10_GPU` | `micro-a10-gpu` |
|
|
330
|
+
| `Runtime.TINY_A10_GPU` | `tiny-a10-gpu` |
|
|
331
|
+
| `Runtime.SMALL_A10_GPU` | `small-a10-gpu` |
|
|
332
|
+
| `Runtime.MEDIUM_A10_GPU` | `medium-a10-gpu` |
|
|
333
|
+
| `Runtime.LARGE_A10_GPU` | `large-a10-gpu` |
|
|
334
|
+
| `Runtime.X_LARGE_A10_GPU` | `x-large-a10-gpu` |
|
|
335
|
+
|
|
336
|
+
```python
|
|
337
|
+
from wherobots import Runtime
|
|
338
|
+
|
|
339
|
+
job = WherobotsJob(
|
|
340
|
+
script="s3://bucket/script.py",
|
|
341
|
+
name="gpu-job-001",
|
|
342
|
+
runtime=Runtime.SMALL_A10_GPU,
|
|
343
|
+
)
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
### Region
|
|
347
|
+
|
|
348
|
+
| Enum | Value |
|
|
349
|
+
|--------------------|-------------------|
|
|
350
|
+
| `Region.US_WEST_2` | `aws-us-west-2` |
|
|
351
|
+
| `Region.US_EAST_1` | `aws-us-east-1` |
|
|
352
|
+
| `Region.EU_WEST_1` | `aws-eu-west-1` |
|
|
353
|
+
| `Region.AP_SOUTH_1`| `aws-ap-south-1` |
|
|
354
|
+
|
|
355
|
+
### DependencyType
|
|
356
|
+
|
|
357
|
+
| Enum | Value |
|
|
358
|
+
|-----------------------|--------|
|
|
359
|
+
| `DependencyType.PYPI` | `PYPI` |
|
|
360
|
+
| `DependencyType.FILE` | `FILE` |
|
|
361
|
+
|
|
362
|
+
### DependencyFileType
|
|
363
|
+
|
|
364
|
+
| Enum | Value |
|
|
365
|
+
|-------------------------------|----------------|
|
|
366
|
+
| `DependencyFileType.JAR` | `JAR` |
|
|
367
|
+
| `DependencyFileType.PYTHON_WHEEL` | `PYTHON_WHEEL` |
|
|
368
|
+
| `DependencyFileType.ZIP` | `ZIP` |
|
|
369
|
+
| `DependencyFileType.OTHER` | `OTHER` |
|
|
370
|
+
|
|
371
|
+
## Models
|
|
372
|
+
|
|
373
|
+
All models are Python dataclasses with `from_dict()` and `to_dict()` methods
|
|
374
|
+
for JSON round-tripping.
|
|
375
|
+
|
|
376
|
+
### RunView
|
|
377
|
+
|
|
378
|
+
Returned by `get_status()` and `list_runs()`. Represents a single job run.
|
|
379
|
+
|
|
380
|
+
| Field | Type | Description |
|
|
381
|
+
|--------------------|---------------------------|--------------------------------|
|
|
382
|
+
| `id` | `str` | Run ID |
|
|
383
|
+
| `name` | `str` | Job name |
|
|
384
|
+
| `status` | `JobStatus \| None` | Current status |
|
|
385
|
+
| `runtime` | `str \| None` | Compute runtime size |
|
|
386
|
+
| `region` | `str \| None` | AWS region |
|
|
387
|
+
| `version` | `str \| None` | Wherobots version |
|
|
388
|
+
| `timeout_seconds` | `int \| None` | Job timeout |
|
|
389
|
+
| `create_time` | `str \| None` | ISO-8601 creation timestamp |
|
|
390
|
+
| `update_time` | `str \| None` | ISO-8601 last update timestamp |
|
|
391
|
+
| `start_time` | `str \| None` | ISO-8601 start timestamp |
|
|
392
|
+
| `complete_time` | `str \| None` | ISO-8601 completion timestamp |
|
|
393
|
+
| `run_python` | `RunPythonPayload \| None`| Python script payload |
|
|
394
|
+
| `run_jar` | `RunJarPayload \| None` | JAR payload |
|
|
395
|
+
| `environment` | `RunEnvironment \| None` | Spark config & dependencies |
|
|
396
|
+
| `triggered_by` | `dict \| None` | User who triggered the run |
|
|
397
|
+
| `kube_app` | `RunKubeApp \| None` | Kubernetes app details |
|
|
398
|
+
| `app_meta` | `RunAppMeta \| None` | Spark/Sedona UI URLs |
|
|
399
|
+
| `organization` | `OrganizationCustomer \| None` | Org info |
|
|
400
|
+
| `extra` | `dict` | Any additional API fields |
|
|
401
|
+
|
|
402
|
+
### LogsResponse
|
|
403
|
+
|
|
404
|
+
Returned by `get_logs()`.
|
|
405
|
+
|
|
406
|
+
| Field | Type | Description |
|
|
407
|
+
|----------------|------------------------|------------------------------|
|
|
408
|
+
| `items` | `list[LogItem]` | Log entries |
|
|
409
|
+
| `next_page` | `int \| str \| None` | Next page cursor |
|
|
410
|
+
| `current_page` | `int \| str \| None` | Current page cursor |
|
|
411
|
+
|
|
412
|
+
### LogItem
|
|
413
|
+
|
|
414
|
+
| Field | Type | Description |
|
|
415
|
+
|-------------|------------------------|---------------------------------|
|
|
416
|
+
| `raw` | `str` | Raw log line text |
|
|
417
|
+
| `timestamp` | `int \| str \| None` | Epoch int or ISO-8601 string |
|
|
418
|
+
| `level` | `str \| None` | Log level (INFO, ERROR, etc.) |
|
|
419
|
+
| `message` | `str \| None` | Parsed message |
|
|
420
|
+
| `extra` | `dict` | Additional fields |
|
|
421
|
+
|
|
422
|
+
### RunListPage
|
|
423
|
+
|
|
424
|
+
Returned by `list_runs()`.
|
|
425
|
+
|
|
426
|
+
| Field | Type | Description |
|
|
427
|
+
|--------------------------|-------------------|--------------------------------|
|
|
428
|
+
| `items` | `list[RunView]` | Runs on this page |
|
|
429
|
+
| `total` | `int` | Total matching runs |
|
|
430
|
+
| `next_page` | `str \| None` | Next page cursor |
|
|
431
|
+
| `previous_page` | `str \| None` | Previous page cursor |
|
|
432
|
+
| `current_page` | `str \| None` | Current page cursor |
|
|
433
|
+
| `current_page_backwards` | `str \| None` | Backward pagination cursor |
|
|
434
|
+
|
|
435
|
+
### RunMetricsResponse
|
|
436
|
+
|
|
437
|
+
Returned by `get_metrics()`.
|
|
438
|
+
|
|
439
|
+
| Field | Type | Description |
|
|
440
|
+
|-------------------|--------|--------------------------|
|
|
441
|
+
| `series_metrics` | `dict` | Time-series metrics |
|
|
442
|
+
| `instant_metrics` | `dict` | Point-in-time metrics |
|
|
443
|
+
| `extra` | `dict` | Additional fields |
|
|
444
|
+
|
|
445
|
+
## Exceptions
|
|
446
|
+
|
|
447
|
+
All exceptions inherit from `WherobotsJobError`.
|
|
448
|
+
|
|
449
|
+
| Exception | Description |
|
|
450
|
+
|----------------------------|------------------------------------------------------|
|
|
451
|
+
| `WherobotsJobError` | Base exception for all SDK errors. |
|
|
452
|
+
| `WherobotsAPIError` | API returned an error response. Has `status_code`, `response`, `request_id` attributes. |
|
|
453
|
+
| `WherobotsValidationError` | Client-side validation failed (e.g., invalid name). |
|
|
454
|
+
| `WherobotsS3Error` | S3 upload or access operation failed. Deprecated — presigned uploads raise `WherobotsAPIError` instead. |
|
|
455
|
+
| `WherobotsConfigError` | Configuration is missing or invalid. |
|
|
456
|
+
| `WherobotsTimeoutError` | Job exceeded the specified timeout. |
|
|
457
|
+
|
|
458
|
+
```python
|
|
459
|
+
from wherobots import WherobotsAPIError, WherobotsJobError
|
|
460
|
+
|
|
461
|
+
try:
|
|
462
|
+
job.submit()
|
|
463
|
+
except WherobotsAPIError as e:
|
|
464
|
+
print(f"HTTP {e.status_code}: {e}")
|
|
465
|
+
print(f"Request ID: {e.request_id}")
|
|
466
|
+
except WherobotsJobError as e:
|
|
467
|
+
print(f"Error: {e}")
|
|
468
|
+
```
|
|
469
|
+
|
|
470
|
+
## Storage Integrations
|
|
471
|
+
|
|
472
|
+
The SDK can run scripts stored in
|
|
473
|
+
[Storage Integrations](https://docs.wherobots.com/latest/develop/storage-management/storage/) —
|
|
474
|
+
user-configured S3 buckets registered with your Wherobots organization. Pass the
|
|
475
|
+
S3 URI directly as the `script` parameter:
|
|
476
|
+
|
|
477
|
+
```python
|
|
478
|
+
job = WherobotsJob(
|
|
479
|
+
script="s3://my-integration-bucket/scripts/pipeline.py",
|
|
480
|
+
name="pipeline-job-001",
|
|
481
|
+
runtime="small",
|
|
482
|
+
auto_upload=False,
|
|
483
|
+
)
|
|
484
|
+
```
|
|
485
|
+
|
|
486
|
+
To discover your integration paths programmatically:
|
|
487
|
+
|
|
488
|
+
```python
|
|
489
|
+
from wherobots.api.files import FilesAPI
|
|
490
|
+
from wherobots.config import WherobotsConfig
|
|
491
|
+
|
|
492
|
+
config = WherobotsConfig.from_env()
|
|
493
|
+
with FilesAPI.from_config(config) as files_api:
|
|
494
|
+
for si in files_api.list_integrations():
|
|
495
|
+
print(f"{si.name}: {si.path} ({si.region})")
|
|
496
|
+
```
|
|
497
|
+
|
|
498
|
+
To set up a Storage Integration, follow the
|
|
499
|
+
[S3 Storage Integration guide](https://docs.wherobots.com/latest/develop/storage-management/s3-storage-integration/).
|
|
500
|
+
|
|
501
|
+
## API Endpoints
|
|
502
|
+
|
|
503
|
+
| Method | Endpoint | Description |
|
|
504
|
+
|--------|------------------------|--------------------|
|
|
505
|
+
| POST | `/runs?region=<region>`| Create a run |
|
|
506
|
+
| GET | `/runs/{id}` | Get run details |
|
|
507
|
+
| POST | `/runs/{id}/cancel` | Cancel a run |
|
|
508
|
+
| GET | `/runs/{id}/logs` | Get run logs |
|
|
509
|
+
| GET | `/runs/{id}/metrics` | Get run metrics |
|
|
510
|
+
| GET | `/runs` | List runs |
|
|
511
|
+
|
|
512
|
+
Base URL: `https://api.cloud.wherobots.com`
|
|
513
|
+
|
|
514
|
+
## Development
|
|
515
|
+
|
|
516
|
+
### Setup
|
|
517
|
+
|
|
518
|
+
```bash
|
|
519
|
+
git clone https://github.com/wherobots/wherobots-python-sdk.git
|
|
520
|
+
cd wherobots-python-sdk
|
|
521
|
+
python -m venv .venv
|
|
522
|
+
source .venv/bin/activate
|
|
523
|
+
pip install -e ".[dev]"
|
|
524
|
+
pre-commit install
|
|
525
|
+
```
|
|
526
|
+
|
|
527
|
+
### Quality checks
|
|
528
|
+
|
|
529
|
+
```bash
|
|
530
|
+
make lint # ruff check + ruff format --check
|
|
531
|
+
make format # auto-fix lint issues and reformat
|
|
532
|
+
make typecheck # mypy (strict)
|
|
533
|
+
make check # lint + typecheck + tests + build
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
### Releasing
|
|
537
|
+
|
|
538
|
+
See [`RELEASING.md`](RELEASING.md) for the release flow (version
|
|
539
|
+
bump → GitHub release → automated PyPI publish).
|
|
540
|
+
|
|
541
|
+
### Running Tests
|
|
542
|
+
|
|
543
|
+
```bash
|
|
544
|
+
make test # unit tests (integration tests skipped by default)
|
|
545
|
+
|
|
546
|
+
# Or directly:
|
|
547
|
+
pytest -o "addopts=" # override pyproject.toml --cov flags if pytest-cov missing
|
|
548
|
+
pytest -v # verbose
|
|
549
|
+
pytest --cov=wherobots --cov-report=term-missing # with coverage
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
#### Integration Tests
|
|
553
|
+
|
|
554
|
+
Integration tests run against the live Wherobots API and are **skipped by
|
|
555
|
+
default**. To run them:
|
|
556
|
+
|
|
557
|
+
```bash
|
|
558
|
+
export WHEROBOTS_API_KEY="your-key"
|
|
559
|
+
make test-integration
|
|
560
|
+
|
|
561
|
+
# Or pass the key via CLI option:
|
|
562
|
+
pytest -m integration -v -o "addopts=" --wherobots-api-key="your-key"
|
|
563
|
+
```
|
|
564
|
+
|
|
565
|
+
### Linting & Formatting
|
|
566
|
+
|
|
567
|
+
```bash
|
|
568
|
+
make format # auto-format with ruff
|
|
569
|
+
make lint # ruff check + ruff format --check (same as CI)
|
|
570
|
+
```
|
|
571
|
+
|
|
572
|
+
### Before You Push
|
|
573
|
+
|
|
574
|
+
```bash
|
|
575
|
+
make check # lint + test + build — mirrors the CI pipeline
|
|
576
|
+
```
|
|
577
|
+
|
|
578
|
+
## License
|
|
579
|
+
|
|
580
|
+
Apache-2.0. See [LICENSE](LICENSE) for details.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
wherobots/__init__.py,sha256=UGvPiw86dJwrZiA-7u-Nx4thyZj7QmdOio343-vscHg,1754
|
|
2
|
+
wherobots/__version__.py,sha256=gdQx5awJ980s8gFK_3u0b4RogasTty0-FJ6cROVHjZo,111
|
|
3
|
+
wherobots/client.py,sha256=go2-usZMWm74zUXupiy5k-M3Faprn_WJaH1wK0Sr0po,23467
|
|
4
|
+
wherobots/config.py,sha256=Ib20Y5mGsE4aP_g99ZKEyZvV8q-aOXq7RVpYP8i0lpo,6679
|
|
5
|
+
wherobots/enums.py,sha256=YsxXN3ZcY99aoEsUtjYs4otHDevsSPizAgCp-vShds8,4154
|
|
6
|
+
wherobots/exceptions.py,sha256=ERoTU7Edf-dFCE4-ScyuEcC-2O2d5JRa8issQWc-PNA,1296
|
|
7
|
+
wherobots/models.py,sha256=RKmxMpHTLzhOdb3FcvxWLgrxjpSCBpuc9l-BacbqLl4,35673
|
|
8
|
+
wherobots/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
wherobots/api/__init__.py,sha256=jUb587jazP_FA-sH_6yCYfvGxsNiUOyIYAPb3CHOYtQ,218
|
|
10
|
+
wherobots/api/base.py,sha256=IiT3k17G19bT0rM2zYZs9Tpob8pgTrSLAWdKlLUVM1o,8769
|
|
11
|
+
wherobots/api/files.py,sha256=c2yGQ-yQKzvp1O-47JrFp1H6Uj7FDCBRiyJ0ym8g-TE,14040
|
|
12
|
+
wherobots/api/runs.py,sha256=25Hz0evE0gfu7YCQma77AcxAbrk89iKgzdoE0XFJr7U,8442
|
|
13
|
+
wherobots/utils/__init__.py,sha256=cTM2QNJDauQWyGBTJPE5yH4ULc4Ucg_4KFzemjM3HaA,193
|
|
14
|
+
wherobots/utils/logger.py,sha256=eclhD9Vf2LsSm4engRzT3jE2v0wdHJH01OfeI3ml4HM,1103
|
|
15
|
+
wherobots/utils/validation.py,sha256=R0iG4OqzER4prdP8-TOQV_pQFJ3emEQyWYYuc3aCefU,890
|
|
16
|
+
wherobots_python_sdk-0.1.0.dist-info/licenses/LICENSE,sha256=U1EH466XmvxbqaWOY0cax49w0R87_6w6yJLyV14v4bc,10767
|
|
17
|
+
wherobots_python_sdk-0.1.0.dist-info/METADATA,sha256=EH_LfjKliEImWBAi3a2t2a5HvbvokyzpIsqV0p-jqWI,21512
|
|
18
|
+
wherobots_python_sdk-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
19
|
+
wherobots_python_sdk-0.1.0.dist-info/top_level.txt,sha256=oL-n-9GIG0rmUeukuytUqdN6yfROIsVh8bNrgZsOdxw,10
|
|
20
|
+
wherobots_python_sdk-0.1.0.dist-info/RECORD,,
|