spooled 1.2.2__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {spooled-1.2.2 → spooled-1.4.0}/PKG-INFO +91 -13
- {spooled-1.2.2 → spooled-1.4.0}/README.md +90 -12
- {spooled-1.2.2 → spooled-1.4.0}/proto/spooled.proto +19 -1
- {spooled-1.2.2 → spooled-1.4.0}/pyproject.toml +1 -1
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/__init__.py +1 -1
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/_version.py +1 -1
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/async_client.py +84 -59
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/client.py +68 -57
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/config.py +37 -2
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/errors.py +136 -14
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/grpc/__init__.py +1 -1
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/grpc/client.py +105 -48
- spooled-1.4.0/src/spooled/grpc/stubs/spooled_pb2.py +104 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/grpc/stubs/spooled_pb2.pyi +4 -2
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/realtime/sse.py +195 -61
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/realtime/unified.py +77 -3
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/realtime/websocket.py +2 -2
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/jobs.py +106 -36
- spooled-1.4.0/src/spooled/resources/metrics.py +38 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/organizations.py +24 -4
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/queues.py +6 -2
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/webhooks.py +30 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/workers.py +12 -2
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/workflows.py +0 -1
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/__init__.py +16 -0
- spooled-1.4.0/src/spooled/types/common.py +137 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/jobs.py +147 -15
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/organizations.py +5 -3
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/schedules.py +4 -4
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/webhooks.py +74 -4
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/workers.py +14 -2
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/workflows.py +17 -8
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/utils/async_http.py +57 -21
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/utils/http.py +58 -16
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/utils/retry.py +6 -4
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/worker/async_worker.py +259 -141
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/worker/worker.py +259 -113
- spooled-1.2.2/src/spooled/grpc/stubs/spooled_pb2.py +0 -104
- spooled-1.2.2/src/spooled/resources/metrics.py +0 -25
- spooled-1.2.2/src/spooled/types/common.py +0 -24
- {spooled-1.2.2 → spooled-1.4.0}/.gitignore +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/LICENSE +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/grpc/stubs/__init__.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/grpc/stubs/spooled_pb2_grpc.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/py.typed +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/realtime/__init__.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/realtime/events.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/__init__.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/admin.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/api_keys.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/auth.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/base.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/billing.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/dashboard.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/health.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/ingest.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/resources/schedules.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/admin.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/api_keys.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/auth.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/billing.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/dashboard.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/health.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/types/queues.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/utils/__init__.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/utils/casing.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/utils/circuit_breaker.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/utils/jwt.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/worker/__init__.py +0 -0
- {spooled-1.2.2 → spooled-1.4.0}/src/spooled/worker/types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: spooled
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Official Python SDK for Spooled Cloud job queue service
|
|
5
5
|
Project-URL: Homepage, https://spooled.cloud
|
|
6
6
|
Project-URL: Documentation, https://spooled.cloud/docs/
|
|
@@ -140,9 +140,43 @@ def handle_job(ctx):
|
|
|
140
140
|
def on_completed(event):
|
|
141
141
|
print(f"Job {event.job_id} completed!")
|
|
142
142
|
|
|
143
|
-
worker.start() # Blocking
|
|
143
|
+
worker.start() # Blocking; returns after worker.stop() has drained running jobs
|
|
144
144
|
```
|
|
145
145
|
|
|
146
|
+
`worker.stop()` (call it from a SIGTERM/SIGINT handler) stops claiming jobs, lets running
|
|
147
|
+
handlers finish for up to `shutdown_timeout` seconds and still acks their results, then
|
|
148
|
+
deregisters. A handler is only signalled (`ctx.signal`) and failed once it outlives that
|
|
149
|
+
timeout or the job's own `timeout_seconds`. See [docs/WORKER.md](docs/WORKER.md).
|
|
150
|
+
|
|
151
|
+
If you drive the worker endpoints yourself (`client.jobs.claim/complete/fail/heartbeat`),
|
|
152
|
+
send the `lease_id` from the claimed job on every `complete`, `fail` and `heartbeat`: the API
|
|
153
|
+
requires it (422 without it) and applies the call only to the lease you hold. `SpooledWorker`
|
|
154
|
+
does this for you.
|
|
155
|
+
|
|
156
|
+
### List Jobs, History and Paging
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
from datetime import datetime, timezone
|
|
160
|
+
|
|
161
|
+
page = client.jobs.list({
|
|
162
|
+
"status": ["failed", "deadletter"], # one status, a list, or "failed,deadletter"
|
|
163
|
+
"q": "invoice", # id prefix, queue name or payload.job_type
|
|
164
|
+
"job_type": "send_invoice", # exact payload.job_type
|
|
165
|
+
"created_after": datetime(2026, 10, 1, tzinfo=timezone.utc),
|
|
166
|
+
"limit": 50,
|
|
167
|
+
})
|
|
168
|
+
for job in page:
|
|
169
|
+
print(job.id, job.status)
|
|
170
|
+
if page.has_more: # from the X-Has-More response header
|
|
171
|
+
next_page = client.jobs.list({"status": ["failed", "deadletter"], "offset": 50})
|
|
172
|
+
|
|
173
|
+
for event in client.jobs.history(job.id): # oldest first; {"order_dir": "desc"} for newest
|
|
174
|
+
print(event.created_at, event.event_type, event.worker_id)
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
`jobs.list()`, `jobs.dlq.list()` and `jobs.history()` return a normal `list` with a
|
|
178
|
+
`has_more` attribute (`None` when the server sends no `X-Has-More`).
|
|
179
|
+
|
|
146
180
|
## Real-world examples (beginner friendly)
|
|
147
181
|
|
|
148
182
|
If you want 5 copy/paste “real life” setups (Stripe → jobs, GitHub Actions → jobs, cron schedules, CSV import, website signup), see:
|
|
@@ -229,7 +263,8 @@ The SDK includes a high-performance gRPC client for high-throughput worker scena
|
|
|
229
263
|
```python
|
|
230
264
|
from spooled.grpc import SpooledGrpcClient
|
|
231
265
|
|
|
232
|
-
# Connect to Spooled Cloud gRPC (TLS required for Cloudflare Tunnel)
|
|
266
|
+
# Connect to Spooled Cloud gRPC (TLS required for Cloudflare Tunnel).
|
|
267
|
+
# `SpooledClient(api_key=...).grpc` does this for you and honours `grpc_address=`.
|
|
233
268
|
client = SpooledGrpcClient(
|
|
234
269
|
address="grpc.spooled.cloud:443",
|
|
235
270
|
api_key="sp_live_...",
|
|
@@ -250,6 +285,29 @@ print(f"Pending jobs: {stats.pending}")
|
|
|
250
285
|
client.close()
|
|
251
286
|
```
|
|
252
287
|
|
|
288
|
+
Leases and streams (server behaviour):
|
|
289
|
+
|
|
290
|
+
- `queue.dequeue(...)` hands out up to `batch_size` jobs, each with a lease of
|
|
291
|
+
`lease_duration_secs` (5-3600 s; when it is omitted or `0` the server default applies, 300 s on
|
|
292
|
+
Spooled Cloud). A job that can run longer than its lease needs a longer lease, or
|
|
293
|
+
`queue.renew_lease(job_id, worker_id, extension_secs, lease_id=...)` before it expires.
|
|
294
|
+
- `queue.stream_jobs(...)` delivers **one job at a time per stream**: the next job arrives only after
|
|
295
|
+
the previous one on that stream is completed, failed or released. Settle every job, and open one
|
|
296
|
+
stream per job slot to run jobs in parallel (at most 100 open streams per organization).
|
|
297
|
+
- `queue.complete`, `queue.fail` and `queue.renew_lease` need the dequeued job's `lease_id`
|
|
298
|
+
(`INVALID_ARGUMENT` without it). `queue.fail(..., retry=False, lease_id=...)` dead-letters the
|
|
299
|
+
job now; the default (`retry=True`) retries it until `max_retries` is used up.
|
|
300
|
+
- `workers.register(queue_name, hostname, worker_id="pod-7")` takes an optional stable
|
|
301
|
+
`worker_id`: registering the same id again after a restart updates that worker instead of
|
|
302
|
+
adding one.
|
|
303
|
+
|
|
304
|
+
Payloads over gRPC are `google.protobuf.Struct`, which has two limits that REST does not:
|
|
305
|
+
|
|
306
|
+
- `payload` and `result` must be JSON **objects**. A job created over REST with a string,
|
|
307
|
+
number or array payload arrives over gRPC as an empty dict.
|
|
308
|
+
- Struct numbers are doubles, so integers of 2^53 (9 007 199 254 740 992) and above lose
|
|
309
|
+
precision over gRPC. REST keeps them exact; send large ids as strings, or use REST.
|
|
310
|
+
|
|
253
311
|
### When to use gRPC?
|
|
254
312
|
|
|
255
313
|
- **High Throughput**: Persistent HTTP/2 and Protobuf can reduce overhead; benchmark your workload and network path.
|
|
@@ -305,6 +363,7 @@ from spooled.errors import (
|
|
|
305
363
|
NotFoundError,
|
|
306
364
|
ValidationError,
|
|
307
365
|
RateLimitError,
|
|
366
|
+
QuotaExceededError,
|
|
308
367
|
)
|
|
309
368
|
|
|
310
369
|
client = SpooledClient(api_key="sp_live_...")
|
|
@@ -314,16 +373,30 @@ try:
|
|
|
314
373
|
except NotFoundError:
|
|
315
374
|
print("Job not found")
|
|
316
375
|
except RateLimitError as e:
|
|
317
|
-
# 429 covers both
|
|
318
|
-
#
|
|
319
|
-
|
|
320
|
-
|
|
376
|
+
# 429 covers both request-rate limiting and plan-quota errors. A rate-limited 429
|
|
377
|
+
# carries Retry-After; e.limit, e.remaining, e.reset (aware UTC datetime) and e.scope
|
|
378
|
+
# come from the X-RateLimit-* headers. The plan limit is per organization in
|
|
379
|
+
# requests/second, an API key's own rate_limit is requests/minute.
|
|
380
|
+
# A plan quota is a QuotaExceededError (code "QUOTA_EXCEEDED", with
|
|
381
|
+
# resource/current/limit/plan): it is never retried, because it will not clear in seconds.
|
|
382
|
+
if isinstance(e, QuotaExceededError):
|
|
383
|
+
print(f"Plan quota exceeded: {e.message} ({e.current}/{e.limit} on {e.plan})")
|
|
321
384
|
else:
|
|
322
385
|
print(f"Rate limited. Retry after {e.retry_after}s")
|
|
323
386
|
except SpooledError as e:
|
|
324
387
|
print(f"API error: {e.code} - {e.message}")
|
|
325
388
|
```
|
|
326
389
|
|
|
390
|
+
The API sends `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset` (Unix
|
|
391
|
+
seconds) and `X-RateLimit-Scope` on **every** response. The latest values are on
|
|
392
|
+
`client.rate_limit` (a `RateLimitInfo`, `None` before the first response):
|
|
393
|
+
|
|
394
|
+
```python
|
|
395
|
+
client.jobs.get_stats()
|
|
396
|
+
info = client.rate_limit
|
|
397
|
+
print(info.remaining, "of", info.limit, "left until", info.reset, f"({info.scope})")
|
|
398
|
+
```
|
|
399
|
+
|
|
327
400
|
## API Reference
|
|
328
401
|
|
|
329
402
|
### Client Resources
|
|
@@ -335,7 +408,8 @@ except SpooledError as e:
|
|
|
335
408
|
| `client.workers` | Worker registration, heartbeat |
|
|
336
409
|
| `client.schedules` | Cron job scheduling |
|
|
337
410
|
| `client.workflows` | Multi-job workflows with dependencies |
|
|
338
|
-
| `client.webhooks` | Outgoing webhook
|
|
411
|
+
| `client.webhooks` | Outgoing webhooks (events, `json`/`slack`/`discord`/`teams` format), deliveries, inbound webhook log |
|
|
412
|
+
| `client.ingest` | Inbound custom webhooks (`POST /webhooks/{org_id}/custom`) |
|
|
339
413
|
| `client.api_keys` | API key management |
|
|
340
414
|
| `client.organizations` | Organization settings, usage |
|
|
341
415
|
| `client.billing` | Billing status, portal |
|
|
@@ -357,10 +431,14 @@ except SpooledError as e:
|
|
|
357
431
|
|
|
358
432
|
## Environment Variables
|
|
359
433
|
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
434
|
+
The SDK does **not** read environment variables: pass the API key (and a non-default base
|
|
435
|
+
URL) to `SpooledClient` yourself. The examples and the integration tests in this repository
|
|
436
|
+
use these names by convention:
|
|
437
|
+
|
|
438
|
+
| Variable | Used for |
|
|
439
|
+
|----------|----------|
|
|
440
|
+
| `SPOOLED_API_KEY` | `SpooledClient(api_key=os.environ["SPOOLED_API_KEY"])` |
|
|
441
|
+
| `SPOOLED_BASE_URL` | `base_url=` (the SDK default is `https://api.spooled.cloud`) |
|
|
364
442
|
|
|
365
443
|
## Requirements
|
|
366
444
|
|
|
@@ -369,7 +447,7 @@ except SpooledError as e:
|
|
|
369
447
|
- gRPC extra: `grpcio>=1.80.0`, `grpcio-tools>=1.80.0`, `protobuf>=6.33.5`
|
|
370
448
|
- Realtime extra: `websockets>=12.0`, `sseclient-py>=1.8.0`
|
|
371
449
|
|
|
372
|
-
The package version is defined in `pyproject.toml`. At runtime, `spooled.__version__`, the default REST `User-Agent`, worker options, and REST/gRPC worker registration read installed package metadata; source-only imports fall back to `1.
|
|
450
|
+
The package version is defined in `pyproject.toml`. At runtime, `spooled.__version__`, the default REST `User-Agent`, worker options, and REST/gRPC worker registration read installed package metadata; source-only imports fall back to `1.4.0`.
|
|
373
451
|
|
|
374
452
|
Release maintainers should follow the [publishing checklist](docs/PUBLISHING.md) before creating a tag or publishing to PyPI.
|
|
375
453
|
|
|
@@ -91,9 +91,43 @@ def handle_job(ctx):
|
|
|
91
91
|
def on_completed(event):
|
|
92
92
|
print(f"Job {event.job_id} completed!")
|
|
93
93
|
|
|
94
|
-
worker.start() # Blocking
|
|
94
|
+
worker.start() # Blocking; returns after worker.stop() has drained running jobs
|
|
95
95
|
```
|
|
96
96
|
|
|
97
|
+
`worker.stop()` (call it from a SIGTERM/SIGINT handler) stops claiming jobs, lets running
|
|
98
|
+
handlers finish for up to `shutdown_timeout` seconds and still acks their results, then
|
|
99
|
+
deregisters. A handler is only signalled (`ctx.signal`) and failed once it outlives that
|
|
100
|
+
timeout or the job's own `timeout_seconds`. See [docs/WORKER.md](docs/WORKER.md).
|
|
101
|
+
|
|
102
|
+
If you drive the worker endpoints yourself (`client.jobs.claim/complete/fail/heartbeat`),
|
|
103
|
+
send the `lease_id` from the claimed job on every `complete`, `fail` and `heartbeat`: the API
|
|
104
|
+
requires it (422 without it) and applies the call only to the lease you hold. `SpooledWorker`
|
|
105
|
+
does this for you.
|
|
106
|
+
|
|
107
|
+
### List Jobs, History and Paging
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from datetime import datetime, timezone
|
|
111
|
+
|
|
112
|
+
page = client.jobs.list({
|
|
113
|
+
"status": ["failed", "deadletter"], # one status, a list, or "failed,deadletter"
|
|
114
|
+
"q": "invoice", # id prefix, queue name or payload.job_type
|
|
115
|
+
"job_type": "send_invoice", # exact payload.job_type
|
|
116
|
+
"created_after": datetime(2026, 10, 1, tzinfo=timezone.utc),
|
|
117
|
+
"limit": 50,
|
|
118
|
+
})
|
|
119
|
+
for job in page:
|
|
120
|
+
print(job.id, job.status)
|
|
121
|
+
if page.has_more: # from the X-Has-More response header
|
|
122
|
+
next_page = client.jobs.list({"status": ["failed", "deadletter"], "offset": 50})
|
|
123
|
+
|
|
124
|
+
for event in client.jobs.history(job.id): # oldest first; {"order_dir": "desc"} for newest
|
|
125
|
+
print(event.created_at, event.event_type, event.worker_id)
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
`jobs.list()`, `jobs.dlq.list()` and `jobs.history()` return a normal `list` with a
|
|
129
|
+
`has_more` attribute (`None` when the server sends no `X-Has-More`).
|
|
130
|
+
|
|
97
131
|
## Real-world examples (beginner friendly)
|
|
98
132
|
|
|
99
133
|
If you want 5 copy/paste “real life” setups (Stripe → jobs, GitHub Actions → jobs, cron schedules, CSV import, website signup), see:
|
|
@@ -180,7 +214,8 @@ The SDK includes a high-performance gRPC client for high-throughput worker scena
|
|
|
180
214
|
```python
|
|
181
215
|
from spooled.grpc import SpooledGrpcClient
|
|
182
216
|
|
|
183
|
-
# Connect to Spooled Cloud gRPC (TLS required for Cloudflare Tunnel)
|
|
217
|
+
# Connect to Spooled Cloud gRPC (TLS required for Cloudflare Tunnel).
|
|
218
|
+
# `SpooledClient(api_key=...).grpc` does this for you and honours `grpc_address=`.
|
|
184
219
|
client = SpooledGrpcClient(
|
|
185
220
|
address="grpc.spooled.cloud:443",
|
|
186
221
|
api_key="sp_live_...",
|
|
@@ -201,6 +236,29 @@ print(f"Pending jobs: {stats.pending}")
|
|
|
201
236
|
client.close()
|
|
202
237
|
```
|
|
203
238
|
|
|
239
|
+
Leases and streams (server behaviour):
|
|
240
|
+
|
|
241
|
+
- `queue.dequeue(...)` hands out up to `batch_size` jobs, each with a lease of
|
|
242
|
+
`lease_duration_secs` (5-3600 s; when it is omitted or `0` the server default applies, 300 s on
|
|
243
|
+
Spooled Cloud). A job that can run longer than its lease needs a longer lease, or
|
|
244
|
+
`queue.renew_lease(job_id, worker_id, extension_secs, lease_id=...)` before it expires.
|
|
245
|
+
- `queue.stream_jobs(...)` delivers **one job at a time per stream**: the next job arrives only after
|
|
246
|
+
the previous one on that stream is completed, failed or released. Settle every job, and open one
|
|
247
|
+
stream per job slot to run jobs in parallel (at most 100 open streams per organization).
|
|
248
|
+
- `queue.complete`, `queue.fail` and `queue.renew_lease` need the dequeued job's `lease_id`
|
|
249
|
+
(`INVALID_ARGUMENT` without it). `queue.fail(..., retry=False, lease_id=...)` dead-letters the
|
|
250
|
+
job now; the default (`retry=True`) retries it until `max_retries` is used up.
|
|
251
|
+
- `workers.register(queue_name, hostname, worker_id="pod-7")` takes an optional stable
|
|
252
|
+
`worker_id`: registering the same id again after a restart updates that worker instead of
|
|
253
|
+
adding one.
|
|
254
|
+
|
|
255
|
+
Payloads over gRPC are `google.protobuf.Struct`, which has two limits that REST does not:
|
|
256
|
+
|
|
257
|
+
- `payload` and `result` must be JSON **objects**. A job created over REST with a string,
|
|
258
|
+
number or array payload arrives over gRPC as an empty dict.
|
|
259
|
+
- Struct numbers are doubles, so integers of 2^53 (9 007 199 254 740 992) and above lose
|
|
260
|
+
precision over gRPC. REST keeps them exact; send large ids as strings, or use REST.
|
|
261
|
+
|
|
204
262
|
### When to use gRPC?
|
|
205
263
|
|
|
206
264
|
- **High Throughput**: Persistent HTTP/2 and Protobuf can reduce overhead; benchmark your workload and network path.
|
|
@@ -256,6 +314,7 @@ from spooled.errors import (
|
|
|
256
314
|
NotFoundError,
|
|
257
315
|
ValidationError,
|
|
258
316
|
RateLimitError,
|
|
317
|
+
QuotaExceededError,
|
|
259
318
|
)
|
|
260
319
|
|
|
261
320
|
client = SpooledClient(api_key="sp_live_...")
|
|
@@ -265,16 +324,30 @@ try:
|
|
|
265
324
|
except NotFoundError:
|
|
266
325
|
print("Job not found")
|
|
267
326
|
except RateLimitError as e:
|
|
268
|
-
# 429 covers both
|
|
269
|
-
#
|
|
270
|
-
|
|
271
|
-
|
|
327
|
+
# 429 covers both request-rate limiting and plan-quota errors. A rate-limited 429
|
|
328
|
+
# carries Retry-After; e.limit, e.remaining, e.reset (aware UTC datetime) and e.scope
|
|
329
|
+
# come from the X-RateLimit-* headers. The plan limit is per organization in
|
|
330
|
+
# requests/second, an API key's own rate_limit is requests/minute.
|
|
331
|
+
# A plan quota is a QuotaExceededError (code "QUOTA_EXCEEDED", with
|
|
332
|
+
# resource/current/limit/plan): it is never retried, because it will not clear in seconds.
|
|
333
|
+
if isinstance(e, QuotaExceededError):
|
|
334
|
+
print(f"Plan quota exceeded: {e.message} ({e.current}/{e.limit} on {e.plan})")
|
|
272
335
|
else:
|
|
273
336
|
print(f"Rate limited. Retry after {e.retry_after}s")
|
|
274
337
|
except SpooledError as e:
|
|
275
338
|
print(f"API error: {e.code} - {e.message}")
|
|
276
339
|
```
|
|
277
340
|
|
|
341
|
+
The API sends `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset` (Unix
|
|
342
|
+
seconds) and `X-RateLimit-Scope` on **every** response. The latest values are on
|
|
343
|
+
`client.rate_limit` (a `RateLimitInfo`, `None` before the first response):
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
client.jobs.get_stats()
|
|
347
|
+
info = client.rate_limit
|
|
348
|
+
print(info.remaining, "of", info.limit, "left until", info.reset, f"({info.scope})")
|
|
349
|
+
```
|
|
350
|
+
|
|
278
351
|
## API Reference
|
|
279
352
|
|
|
280
353
|
### Client Resources
|
|
@@ -286,7 +359,8 @@ except SpooledError as e:
|
|
|
286
359
|
| `client.workers` | Worker registration, heartbeat |
|
|
287
360
|
| `client.schedules` | Cron job scheduling |
|
|
288
361
|
| `client.workflows` | Multi-job workflows with dependencies |
|
|
289
|
-
| `client.webhooks` | Outgoing webhook
|
|
362
|
+
| `client.webhooks` | Outgoing webhooks (events, `json`/`slack`/`discord`/`teams` format), deliveries, inbound webhook log |
|
|
363
|
+
| `client.ingest` | Inbound custom webhooks (`POST /webhooks/{org_id}/custom`) |
|
|
290
364
|
| `client.api_keys` | API key management |
|
|
291
365
|
| `client.organizations` | Organization settings, usage |
|
|
292
366
|
| `client.billing` | Billing status, portal |
|
|
@@ -308,10 +382,14 @@ except SpooledError as e:
|
|
|
308
382
|
|
|
309
383
|
## Environment Variables
|
|
310
384
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
385
|
+
The SDK does **not** read environment variables: pass the API key (and a non-default base
|
|
386
|
+
URL) to `SpooledClient` yourself. The examples and the integration tests in this repository
|
|
387
|
+
use these names by convention:
|
|
388
|
+
|
|
389
|
+
| Variable | Used for |
|
|
390
|
+
|----------|----------|
|
|
391
|
+
| `SPOOLED_API_KEY` | `SpooledClient(api_key=os.environ["SPOOLED_API_KEY"])` |
|
|
392
|
+
| `SPOOLED_BASE_URL` | `base_url=` (the SDK default is `https://api.spooled.cloud`) |
|
|
315
393
|
|
|
316
394
|
## Requirements
|
|
317
395
|
|
|
@@ -320,7 +398,7 @@ except SpooledError as e:
|
|
|
320
398
|
- gRPC extra: `grpcio>=1.80.0`, `grpcio-tools>=1.80.0`, `protobuf>=6.33.5`
|
|
321
399
|
- Realtime extra: `websockets>=12.0`, `sseclient-py>=1.8.0`
|
|
322
400
|
|
|
323
|
-
The package version is defined in `pyproject.toml`. At runtime, `spooled.__version__`, the default REST `User-Agent`, worker options, and REST/gRPC worker registration read installed package metadata; source-only imports fall back to `1.
|
|
401
|
+
The package version is defined in `pyproject.toml`. At runtime, `spooled.__version__`, the default REST `User-Agent`, worker options, and REST/gRPC worker registration read installed package metadata; source-only imports fall back to `1.4.0`.
|
|
324
402
|
|
|
325
403
|
Release maintainers should follow the [publishing checklist](docs/PUBLISHING.md) before creating a tag or publishing to PyPI.
|
|
326
404
|
|
|
@@ -143,7 +143,10 @@ message FailRequest {
|
|
|
143
143
|
string job_id = 1;
|
|
144
144
|
string worker_id = 2;
|
|
145
145
|
string error = 3;
|
|
146
|
-
|
|
146
|
+
// Omitted: retry according to the job's max_retries (same as REST).
|
|
147
|
+
// false: move to the dead-letter queue now. `optional` gives presence, so
|
|
148
|
+
// clients that never set the field no longer dead-letter on first failure.
|
|
149
|
+
optional bool retry = 4;
|
|
147
150
|
// Lease fencing token from the dequeued Job. When set, the failure
|
|
148
151
|
// succeeds only if it matches the job's current lease.
|
|
149
152
|
string lease_id = 5;
|
|
@@ -238,10 +241,20 @@ message ErrorResponse {
|
|
|
238
241
|
message RegisterWorkerRequest {
|
|
239
242
|
string queue_name = 1;
|
|
240
243
|
string hostname = 2;
|
|
244
|
+
// At most 50 characters.
|
|
241
245
|
string worker_type = 3;
|
|
246
|
+
// 1-100; larger values are lowered to 100. 0 means the default (5).
|
|
242
247
|
int32 max_concurrency = 4;
|
|
248
|
+
// At most 50 characters.
|
|
243
249
|
string version = 5;
|
|
250
|
+
// At most 64 KB as JSON.
|
|
244
251
|
map<string, string> metadata = 6;
|
|
252
|
+
// Stable worker id (optional). Send the same id after a restart and the
|
|
253
|
+
// server updates that worker instead of adding one, so a restarting worker
|
|
254
|
+
// does not count twice against the plan's worker limit. Omitted or empty:
|
|
255
|
+
// the server assigns a new id. 1-128 characters: letters, digits, `-`,
|
|
256
|
+
// `_`, `.`. An id another organization already uses returns ALREADY_EXISTS.
|
|
257
|
+
optional string worker_id = 7;
|
|
245
258
|
}
|
|
246
259
|
|
|
247
260
|
// Worker Registration Response
|
|
@@ -255,7 +268,12 @@ message RegisterWorkerResponse {
|
|
|
255
268
|
message HeartbeatRequest {
|
|
256
269
|
string worker_id = 1;
|
|
257
270
|
int32 current_jobs = 2;
|
|
271
|
+
// healthy, degraded, draining or offline. Empty keeps the current status,
|
|
272
|
+
// except that a worker marked offline comes back as healthy. Draining
|
|
273
|
+
// stays until the worker registers again.
|
|
258
274
|
string status = 3;
|
|
275
|
+
// Replaces the stored metadata (at most 64 KB as JSON). Empty keeps what
|
|
276
|
+
// is stored.
|
|
259
277
|
map<string, string> metadata = 4;
|
|
260
278
|
}
|
|
261
279
|
|
|
@@ -5,7 +5,7 @@ Official Python SDK for the Spooled Cloud job queue service.
|
|
|
5
5
|
|
|
6
6
|
Example:
|
|
7
7
|
>>> from spooled import SpooledClient
|
|
8
|
-
>>> client = SpooledClient(api_key="
|
|
8
|
+
>>> client = SpooledClient(api_key="sp_live_...")
|
|
9
9
|
>>> result = client.jobs.create(
|
|
10
10
|
... queue_name="emails",
|
|
11
11
|
... payload={"to": "user@example.com"}
|