zerodb-celery 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zerodb_celery/__init__.py +16 -0
- zerodb_celery/backend.py +172 -0
- zerodb_celery/broker.py +214 -0
- zerodb_celery/provision.py +132 -0
- zerodb_celery-0.1.0.dist-info/METADATA +155 -0
- zerodb_celery-0.1.0.dist-info/RECORD +8 -0
- zerodb_celery-0.1.0.dist-info/WHEEL +4 -0
- zerodb_celery-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""zerodb-celery: Celery broker + result backend powered by ZeroDB.
|
|
2
|
+
|
|
3
|
+
Replace Redis or RabbitMQ with ZeroDB in one line:
|
|
4
|
+
|
|
5
|
+
app = Celery('tasks')
|
|
6
|
+
app.config_from_object({
|
|
7
|
+
'broker_url': 'zerodb://auto',
|
|
8
|
+
'result_backend': 'zerodb://auto',
|
|
9
|
+
})
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from zerodb_celery.backend import ZeroDBBackend
|
|
13
|
+
from zerodb_celery.broker import ZeroDBBroker
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
__all__ = ["ZeroDBBackend", "ZeroDBBroker"]
|
zerodb_celery/backend.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""ZeroDB Celery Result Backend — stores task results in a ZeroDB table.
|
|
2
|
+
|
|
3
|
+
Uses the ZeroDB tables API:
|
|
4
|
+
- POST /api/v1/zerodb/tables/{table}/rows (store result)
|
|
5
|
+
- GET /api/v1/zerodb/tables/{table}/rows/{id} (get result)
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import time
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
|
|
12
|
+
from celery.backends.base import BaseKeyValueStoreBackend
|
|
13
|
+
from celery.exceptions import ImproperlyConfigured
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
from zerodb_celery.provision import (
|
|
18
|
+
ZERODB_API,
|
|
19
|
+
auto_provision,
|
|
20
|
+
ensure_results_table,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ZeroDBBackend(BaseKeyValueStoreBackend):
|
|
25
|
+
"""Celery result backend using ZeroDB tables.
|
|
26
|
+
|
|
27
|
+
Configuration via Celery app:
|
|
28
|
+
app.conf.update(
|
|
29
|
+
result_backend='zerodb://auto',
|
|
30
|
+
zerodb_api_key='...', # optional, auto-provisions
|
|
31
|
+
zerodb_project_id='...', # optional, auto-provisions
|
|
32
|
+
zerodb_base_url='...', # optional
|
|
33
|
+
zerodb_results_table='celery_results', # optional
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
Or use the helper:
|
|
37
|
+
ZeroDBBackend.configure(app)
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
# Celery uses this to find the backend class from URL scheme
|
|
41
|
+
# e.g., result_backend = 'zerodb://auto'
|
|
42
|
+
|
|
43
|
+
def __init__(self, app=None, url=None, **kwargs):
|
|
44
|
+
super().__init__(app=app, url=url, **kwargs)
|
|
45
|
+
self._api_key = None
|
|
46
|
+
self._project_id = None
|
|
47
|
+
self._base_url = ZERODB_API
|
|
48
|
+
self._table = "celery_results"
|
|
49
|
+
self._provisioned = False
|
|
50
|
+
self._resolve_config()
|
|
51
|
+
|
|
52
|
+
def _resolve_config(self):
|
|
53
|
+
"""Resolve ZeroDB credentials from Celery config or auto-provision."""
|
|
54
|
+
conf = self.app.conf if self.app else None
|
|
55
|
+
|
|
56
|
+
if conf:
|
|
57
|
+
self._api_key = getattr(conf, "zerodb_api_key", None)
|
|
58
|
+
self._project_id = getattr(conf, "zerodb_project_id", None)
|
|
59
|
+
base = getattr(conf, "zerodb_base_url", None)
|
|
60
|
+
if base:
|
|
61
|
+
self._base_url = base
|
|
62
|
+
table = getattr(conf, "zerodb_results_table", None)
|
|
63
|
+
if table:
|
|
64
|
+
self._table = table
|
|
65
|
+
|
|
66
|
+
if not self._api_key or not self._project_id:
|
|
67
|
+
self._api_key, self._project_id = auto_provision(self._base_url)
|
|
68
|
+
|
|
69
|
+
def _ensure_table(self):
|
|
70
|
+
"""Lazily ensure the results table exists (once per process)."""
|
|
71
|
+
if not self._provisioned:
|
|
72
|
+
ensure_results_table(self._api_key, self._project_id, self._base_url)
|
|
73
|
+
self._provisioned = True
|
|
74
|
+
|
|
75
|
+
def _headers(self):
|
|
76
|
+
return {
|
|
77
|
+
"Authorization": f"Bearer {self._api_key}",
|
|
78
|
+
"X-Project-ID": self._project_id,
|
|
79
|
+
"Content-Type": "application/json",
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
def get(self, key):
|
|
83
|
+
"""Retrieve a task result by key (task_id)."""
|
|
84
|
+
self._ensure_table()
|
|
85
|
+
try:
|
|
86
|
+
resp = requests.get(
|
|
87
|
+
f"{self._base_url}/api/v1/zerodb/tables/{self._table}/rows/{key}",
|
|
88
|
+
headers=self._headers(),
|
|
89
|
+
timeout=15,
|
|
90
|
+
)
|
|
91
|
+
if resp.status_code == 404:
|
|
92
|
+
return None
|
|
93
|
+
resp.raise_for_status()
|
|
94
|
+
row = resp.json()
|
|
95
|
+
# Return the raw stored value (JSON string)
|
|
96
|
+
result_data = row.get("result") or row.get("data", {}).get("result")
|
|
97
|
+
if isinstance(result_data, str):
|
|
98
|
+
return result_data.encode("utf-8")
|
|
99
|
+
return result_data
|
|
100
|
+
except requests.RequestException:
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
def set(self, key, value):
|
|
104
|
+
"""Store a task result."""
|
|
105
|
+
self._ensure_table()
|
|
106
|
+
# value comes as bytes from Celery
|
|
107
|
+
if isinstance(value, bytes):
|
|
108
|
+
value = value.decode("utf-8")
|
|
109
|
+
|
|
110
|
+
try:
|
|
111
|
+
resp = requests.post(
|
|
112
|
+
f"{self._base_url}/api/v1/zerodb/tables/{self._table}/rows",
|
|
113
|
+
headers=self._headers(),
|
|
114
|
+
json={
|
|
115
|
+
"id": key,
|
|
116
|
+
"data": {
|
|
117
|
+
"task_id": key,
|
|
118
|
+
"result": value,
|
|
119
|
+
"date_done": datetime.now(timezone.utc).isoformat(),
|
|
120
|
+
},
|
|
121
|
+
},
|
|
122
|
+
timeout=15,
|
|
123
|
+
)
|
|
124
|
+
resp.raise_for_status()
|
|
125
|
+
except requests.RequestException as e:
|
|
126
|
+
raise BackendError(f"Failed to store result in ZeroDB: {e}") from e
|
|
127
|
+
|
|
128
|
+
def mget(self, keys):
|
|
129
|
+
"""Retrieve multiple results."""
|
|
130
|
+
return [self.get(key) for key in keys]
|
|
131
|
+
|
|
132
|
+
def delete(self, key):
|
|
133
|
+
"""Delete a task result."""
|
|
134
|
+
try:
|
|
135
|
+
requests.delete(
|
|
136
|
+
f"{self._base_url}/api/v1/zerodb/tables/{self._table}/rows/{key}",
|
|
137
|
+
headers=self._headers(),
|
|
138
|
+
timeout=15,
|
|
139
|
+
)
|
|
140
|
+
except requests.RequestException:
|
|
141
|
+
pass
|
|
142
|
+
|
|
143
|
+
@classmethod
|
|
144
|
+
def configure(cls, app, api_key=None, project_id=None, base_url=None,
|
|
145
|
+
results_table=None):
|
|
146
|
+
"""Configure a Celery app to use ZeroDB as result backend.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
app: Celery application instance.
|
|
150
|
+
api_key: ZeroDB API key (auto-provisions if not set).
|
|
151
|
+
project_id: ZeroDB project ID (auto-provisions if not set).
|
|
152
|
+
base_url: ZeroDB API base URL.
|
|
153
|
+
results_table: Name of the results table (default: celery_results).
|
|
154
|
+
"""
|
|
155
|
+
conf = {
|
|
156
|
+
"result_backend": "zerodb_celery.backend:ZeroDBBackend",
|
|
157
|
+
}
|
|
158
|
+
if api_key:
|
|
159
|
+
conf["zerodb_api_key"] = api_key
|
|
160
|
+
if project_id:
|
|
161
|
+
conf["zerodb_project_id"] = project_id
|
|
162
|
+
if base_url:
|
|
163
|
+
conf["zerodb_base_url"] = base_url
|
|
164
|
+
if results_table:
|
|
165
|
+
conf["zerodb_results_table"] = results_table
|
|
166
|
+
|
|
167
|
+
app.config_from_object(conf)
|
|
168
|
+
return app
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class BackendError(Exception):
|
|
172
|
+
"""Raised when backend operations fail."""
|
zerodb_celery/broker.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""ZeroDB Celery Broker — uses ZeroDB event stream as a message queue.
|
|
2
|
+
|
|
3
|
+
Publishes tasks via POST /api/v1/zerodb/events and consumes them
|
|
4
|
+
via GET /api/v1/zerodb/events?topic={queue}.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
from kombu.transport import virtual
|
|
11
|
+
from kombu.utils.encoding import bytes_to_str
|
|
12
|
+
|
|
13
|
+
from zerodb_celery.provision import ZERODB_API, auto_provision
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Channel(virtual.Channel):
|
|
19
|
+
"""A Kombu channel backed by ZeroDB event stream."""
|
|
20
|
+
|
|
21
|
+
# Prefix for ZeroDB event topics so they don't collide
|
|
22
|
+
_topic_prefix = "celery:"
|
|
23
|
+
|
|
24
|
+
def __init__(self, *args, **kwargs):
|
|
25
|
+
super().__init__(*args, **kwargs)
|
|
26
|
+
self._api_key = None
|
|
27
|
+
self._project_id = None
|
|
28
|
+
self._base_url = ZERODB_API
|
|
29
|
+
self._provision()
|
|
30
|
+
|
|
31
|
+
def _provision(self):
|
|
32
|
+
"""Resolve credentials from the broker URL or auto-provision."""
|
|
33
|
+
url = self.connection.client.hostname or ""
|
|
34
|
+
# Parse zerodb://key:project@host or zerodb://auto
|
|
35
|
+
if url and url not in ("auto", "localhost"):
|
|
36
|
+
self._base_url = f"https://{url}"
|
|
37
|
+
|
|
38
|
+
transport_opts = self.connection.client.transport_options or {}
|
|
39
|
+
self._api_key = transport_opts.get("api_key")
|
|
40
|
+
self._project_id = transport_opts.get("project_id")
|
|
41
|
+
base = transport_opts.get("base_url")
|
|
42
|
+
if base:
|
|
43
|
+
self._base_url = base
|
|
44
|
+
|
|
45
|
+
if not self._api_key or not self._project_id:
|
|
46
|
+
self._api_key, self._project_id = auto_provision(self._base_url)
|
|
47
|
+
|
|
48
|
+
def _headers(self):
|
|
49
|
+
return {
|
|
50
|
+
"Authorization": f"Bearer {self._api_key}",
|
|
51
|
+
"X-Project-ID": self._project_id,
|
|
52
|
+
"Content-Type": "application/json",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
def _put(self, queue, message, **kwargs):
|
|
56
|
+
"""Publish a message (task) to the queue via ZeroDB events."""
|
|
57
|
+
topic = f"{self._topic_prefix}{queue}"
|
|
58
|
+
payload = message if isinstance(message, str) else json.dumps(message)
|
|
59
|
+
try:
|
|
60
|
+
resp = requests.post(
|
|
61
|
+
f"{self._base_url}/api/v1/zerodb/events",
|
|
62
|
+
headers=self._headers(),
|
|
63
|
+
json={
|
|
64
|
+
"topic": topic,
|
|
65
|
+
"data": payload,
|
|
66
|
+
},
|
|
67
|
+
timeout=15,
|
|
68
|
+
)
|
|
69
|
+
resp.raise_for_status()
|
|
70
|
+
except requests.RequestException as e:
|
|
71
|
+
raise ChannelError(f"Failed to publish task to ZeroDB: {e}") from e
|
|
72
|
+
|
|
73
|
+
def _get(self, queue, timeout=None):
|
|
74
|
+
"""Consume the next message from the queue via ZeroDB events."""
|
|
75
|
+
topic = f"{self._topic_prefix}{queue}"
|
|
76
|
+
try:
|
|
77
|
+
resp = requests.get(
|
|
78
|
+
f"{self._base_url}/api/v1/zerodb/events",
|
|
79
|
+
headers=self._headers(),
|
|
80
|
+
params={
|
|
81
|
+
"topic": topic,
|
|
82
|
+
"limit": 1,
|
|
83
|
+
"consume": "true",
|
|
84
|
+
},
|
|
85
|
+
timeout=timeout or 15,
|
|
86
|
+
)
|
|
87
|
+
if resp.status_code == 404:
|
|
88
|
+
raise virtual.Empty()
|
|
89
|
+
resp.raise_for_status()
|
|
90
|
+
data = resp.json()
|
|
91
|
+
except requests.RequestException:
|
|
92
|
+
raise virtual.Empty()
|
|
93
|
+
|
|
94
|
+
events = data if isinstance(data, list) else data.get("events", [])
|
|
95
|
+
if not events:
|
|
96
|
+
raise virtual.Empty()
|
|
97
|
+
|
|
98
|
+
event = events[0]
|
|
99
|
+
raw = event.get("data", event)
|
|
100
|
+
if isinstance(raw, str):
|
|
101
|
+
try:
|
|
102
|
+
raw = json.loads(raw)
|
|
103
|
+
except (json.JSONDecodeError, TypeError):
|
|
104
|
+
pass
|
|
105
|
+
return raw
|
|
106
|
+
|
|
107
|
+
def _purge(self, queue):
|
|
108
|
+
"""Purge all messages from a queue."""
|
|
109
|
+
topic = f"{self._topic_prefix}{queue}"
|
|
110
|
+
try:
|
|
111
|
+
resp = requests.delete(
|
|
112
|
+
f"{self._base_url}/api/v1/zerodb/events",
|
|
113
|
+
headers=self._headers(),
|
|
114
|
+
params={"topic": topic},
|
|
115
|
+
timeout=15,
|
|
116
|
+
)
|
|
117
|
+
if resp.ok:
|
|
118
|
+
return resp.json().get("deleted", 0)
|
|
119
|
+
except requests.RequestException:
|
|
120
|
+
pass
|
|
121
|
+
return 0
|
|
122
|
+
|
|
123
|
+
def _size(self, queue):
|
|
124
|
+
"""Return approximate queue size."""
|
|
125
|
+
topic = f"{self._topic_prefix}{queue}"
|
|
126
|
+
try:
|
|
127
|
+
resp = requests.get(
|
|
128
|
+
f"{self._base_url}/api/v1/zerodb/events",
|
|
129
|
+
headers=self._headers(),
|
|
130
|
+
params={"topic": topic, "count_only": "true"},
|
|
131
|
+
timeout=10,
|
|
132
|
+
)
|
|
133
|
+
if resp.ok:
|
|
134
|
+
data = resp.json()
|
|
135
|
+
return data.get("count", 0)
|
|
136
|
+
except requests.RequestException:
|
|
137
|
+
pass
|
|
138
|
+
return 0
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class ChannelError(Exception):
|
|
142
|
+
"""Raised when broker operations fail."""
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class Transport(virtual.Transport):
|
|
146
|
+
"""Kombu transport that uses ZeroDB as the message broker."""
|
|
147
|
+
|
|
148
|
+
Channel = Channel
|
|
149
|
+
|
|
150
|
+
driver_type = "zerodb"
|
|
151
|
+
driver_name = "zerodb"
|
|
152
|
+
|
|
153
|
+
# Connection defaults
|
|
154
|
+
default_port = 443
|
|
155
|
+
connection_errors = (
|
|
156
|
+
virtual.Transport.connection_errors + (
|
|
157
|
+
requests.ConnectionError,
|
|
158
|
+
requests.Timeout,
|
|
159
|
+
)
|
|
160
|
+
)
|
|
161
|
+
channel_errors = (
|
|
162
|
+
virtual.Transport.channel_errors + (
|
|
163
|
+
ChannelError,
|
|
164
|
+
)
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
# Polling interval (seconds) when no messages are available
|
|
168
|
+
polling_interval = 1.0
|
|
169
|
+
|
|
170
|
+
def driver_version(self):
|
|
171
|
+
from zerodb_celery import __version__
|
|
172
|
+
return __version__
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class ZeroDBBroker:
|
|
176
|
+
"""Convenience wrapper for configuring Celery with ZeroDB broker.
|
|
177
|
+
|
|
178
|
+
Usage:
|
|
179
|
+
app = Celery('tasks')
|
|
180
|
+
app.config_from_object({
|
|
181
|
+
'broker_url': 'zerodb://auto',
|
|
182
|
+
'broker_transport': 'zerodb_celery.broker:Transport',
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
Or use the helper:
|
|
186
|
+
ZeroDBBroker.configure(app)
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
TRANSPORT = "zerodb_celery.broker:Transport"
|
|
190
|
+
|
|
191
|
+
@classmethod
|
|
192
|
+
def configure(cls, app, api_key=None, project_id=None, base_url=None):
|
|
193
|
+
"""Configure a Celery app to use ZeroDB as broker.
|
|
194
|
+
|
|
195
|
+
Args:
|
|
196
|
+
app: Celery application instance.
|
|
197
|
+
api_key: ZeroDB API key (auto-provisions if not set).
|
|
198
|
+
project_id: ZeroDB project ID (auto-provisions if not set).
|
|
199
|
+
base_url: ZeroDB API base URL.
|
|
200
|
+
"""
|
|
201
|
+
transport_options = {}
|
|
202
|
+
if api_key:
|
|
203
|
+
transport_options["api_key"] = api_key
|
|
204
|
+
if project_id:
|
|
205
|
+
transport_options["project_id"] = project_id
|
|
206
|
+
if base_url:
|
|
207
|
+
transport_options["base_url"] = base_url
|
|
208
|
+
|
|
209
|
+
app.conf.update(
|
|
210
|
+
broker_url="zerodb://auto",
|
|
211
|
+
broker_transport=cls.TRANSPORT,
|
|
212
|
+
broker_transport_options=transport_options,
|
|
213
|
+
)
|
|
214
|
+
return app
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Auto-provisioning for ZeroDB projects.
|
|
2
|
+
|
|
3
|
+
Creates a free ZeroDB project on first use — no signup, no credit card.
|
|
4
|
+
Credentials are cached in ~/.zerodb/credentials.json for reuse.
|
|
5
|
+
Also ensures the celery_results table exists.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import requests
|
|
14
|
+
|
|
15
|
+
ZERODB_API = "https://api.ainative.studio"
|
|
16
|
+
CREDENTIALS_PATH = Path.home() / ".zerodb" / "credentials.json"
|
|
17
|
+
|
|
18
|
+
CELERY_RESULTS_TABLE = "celery_results"
|
|
19
|
+
CELERY_RESULTS_COLUMNS = {
|
|
20
|
+
"task_id": "string",
|
|
21
|
+
"status": "string",
|
|
22
|
+
"result": "string",
|
|
23
|
+
"traceback": "string",
|
|
24
|
+
"date_done": "string",
|
|
25
|
+
"meta": "string",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _load_cached_credentials():
|
|
30
|
+
"""Load cached credentials from disk."""
|
|
31
|
+
if CREDENTIALS_PATH.exists():
|
|
32
|
+
try:
|
|
33
|
+
data = json.loads(CREDENTIALS_PATH.read_text())
|
|
34
|
+
if data.get("api_key") and data.get("project_id"):
|
|
35
|
+
return data["api_key"], data["project_id"]
|
|
36
|
+
except (json.JSONDecodeError, KeyError):
|
|
37
|
+
pass
|
|
38
|
+
return None, None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _save_credentials(api_key: str, project_id: str):
|
|
42
|
+
"""Cache credentials to disk for reuse."""
|
|
43
|
+
CREDENTIALS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
44
|
+
CREDENTIALS_PATH.write_text(json.dumps({
|
|
45
|
+
"api_key": api_key,
|
|
46
|
+
"project_id": project_id,
|
|
47
|
+
}, indent=2))
|
|
48
|
+
try:
|
|
49
|
+
CREDENTIALS_PATH.chmod(0o600)
|
|
50
|
+
except OSError:
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def auto_provision(base_url: str = ZERODB_API) -> tuple:
|
|
55
|
+
"""Auto-provision a ZeroDB project.
|
|
56
|
+
|
|
57
|
+
Resolution order:
|
|
58
|
+
1. ZERODB_API_KEY + ZERODB_PROJECT_ID env vars
|
|
59
|
+
2. Cached credentials in ~/.zerodb/credentials.json
|
|
60
|
+
3. Provision a new free project via API
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
tuple: (api_key, project_id)
|
|
64
|
+
|
|
65
|
+
Raises:
|
|
66
|
+
RuntimeError: If provisioning fails.
|
|
67
|
+
"""
|
|
68
|
+
# 1. Environment variables
|
|
69
|
+
api_key = os.environ.get("ZERODB_API_KEY")
|
|
70
|
+
project_id = os.environ.get("ZERODB_PROJECT_ID")
|
|
71
|
+
if api_key and project_id:
|
|
72
|
+
return api_key, project_id
|
|
73
|
+
|
|
74
|
+
# 2. Cached credentials
|
|
75
|
+
api_key, project_id = _load_cached_credentials()
|
|
76
|
+
if api_key and project_id:
|
|
77
|
+
return api_key, project_id
|
|
78
|
+
|
|
79
|
+
# 3. Auto-provision
|
|
80
|
+
print("[zerodb-celery] Auto-provisioning free ZeroDB project...", file=sys.stderr)
|
|
81
|
+
try:
|
|
82
|
+
resp = requests.post(
|
|
83
|
+
f"{base_url}/api/v1/zerodb/projects/provision",
|
|
84
|
+
json={"source": "zerodb-celery"},
|
|
85
|
+
timeout=30,
|
|
86
|
+
)
|
|
87
|
+
resp.raise_for_status()
|
|
88
|
+
data = resp.json()
|
|
89
|
+
api_key = data["api_key"]
|
|
90
|
+
project_id = data["project_id"]
|
|
91
|
+
except requests.RequestException as e:
|
|
92
|
+
raise RuntimeError(
|
|
93
|
+
f"Failed to auto-provision ZeroDB project: {e}\n"
|
|
94
|
+
"Set ZERODB_API_KEY and ZERODB_PROJECT_ID manually, or visit "
|
|
95
|
+
"https://ainative.studio to create a free account."
|
|
96
|
+
) from e
|
|
97
|
+
|
|
98
|
+
_save_credentials(api_key, project_id)
|
|
99
|
+
|
|
100
|
+
claim_url = data.get("claim_url", "https://ainative.studio/claim")
|
|
101
|
+
print(
|
|
102
|
+
f"[zerodb-celery] Project provisioned! Claim it at: {claim_url}",
|
|
103
|
+
file=sys.stderr,
|
|
104
|
+
)
|
|
105
|
+
return api_key, project_id
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def ensure_results_table(api_key: str, project_id: str, base_url: str = ZERODB_API):
|
|
109
|
+
"""Create the celery_results table if it does not exist.
|
|
110
|
+
|
|
111
|
+
This is idempotent — calling it multiple times is safe.
|
|
112
|
+
"""
|
|
113
|
+
try:
|
|
114
|
+
resp = requests.post(
|
|
115
|
+
f"{base_url}/api/v1/zerodb/tables",
|
|
116
|
+
headers={
|
|
117
|
+
"Authorization": f"Bearer {api_key}",
|
|
118
|
+
"X-Project-ID": project_id,
|
|
119
|
+
},
|
|
120
|
+
json={
|
|
121
|
+
"name": CELERY_RESULTS_TABLE,
|
|
122
|
+
"columns": CELERY_RESULTS_COLUMNS,
|
|
123
|
+
},
|
|
124
|
+
timeout=15,
|
|
125
|
+
)
|
|
126
|
+
# 409 = table already exists — that's fine
|
|
127
|
+
if resp.status_code not in (200, 201, 409):
|
|
128
|
+
resp.raise_for_status()
|
|
129
|
+
except requests.RequestException:
|
|
130
|
+
# Non-fatal: the table might already exist or will be created
|
|
131
|
+
# lazily on first write by ZeroDB
|
|
132
|
+
pass
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: zerodb-celery
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Celery broker + result backend powered by ZeroDB. Replace Redis/RabbitMQ with one line — zero infrastructure, auto-provisioned.
|
|
5
|
+
Project-URL: Homepage, https://github.com/AINative-Studio/zerodb-celery
|
|
6
|
+
Project-URL: Documentation, https://docs.ainative.studio
|
|
7
|
+
Project-URL: Repository, https://github.com/AINative-Studio/zerodb-celery
|
|
8
|
+
Project-URL: Issues, https://github.com/AINative-Studio/zerodb-celery/issues
|
|
9
|
+
Author-email: AINative Studio <hello@ainative.studio>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: ai-pipeline,ainative,auto-provisioning,background-jobs,celery,celery-backend,celery-broker,claude,cursor,event-driven,ml-pipeline,python,rabbitmq-alternative,redis-alternative,task-queue,zerodb
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Framework :: Celery
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
26
|
+
Requires-Python: >=3.8
|
|
27
|
+
Requires-Dist: celery>=5.0
|
|
28
|
+
Requires-Dist: requests>=2.28
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# zerodb-celery
|
|
32
|
+
|
|
33
|
+
**Celery broker + result backend powered by ZeroDB. Replace Redis/RabbitMQ with one line.**
|
|
34
|
+
|
|
35
|
+
[](https://pypi.org/project/zerodb-celery/)
|
|
36
|
+
[](LICENSE)
|
|
37
|
+
|
|
38
|
+
## Why?
|
|
39
|
+
|
|
40
|
+
Celery requires Redis or RabbitMQ for its broker and result backend. That means provisioning, configuring, and paying for infrastructure you shouldn't need.
|
|
41
|
+
|
|
42
|
+
`zerodb-celery` replaces both with [ZeroDB](https://ainative.studio) — a cloud database that auto-provisions on first use. No signup, no credit card, no infrastructure.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install zerodb-celery
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Quick Start
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from celery import Celery
|
|
54
|
+
from zerodb_celery import ZeroDBBroker, ZeroDBBackend
|
|
55
|
+
|
|
56
|
+
app = Celery('tasks')
|
|
57
|
+
|
|
58
|
+
# Configure both broker and backend
|
|
59
|
+
ZeroDBBroker.configure(app)
|
|
60
|
+
ZeroDBBackend.configure(app)
|
|
61
|
+
|
|
62
|
+
@app.task
|
|
63
|
+
def add(x, y):
|
|
64
|
+
return x + y
|
|
65
|
+
|
|
66
|
+
# Trigger a task
|
|
67
|
+
result = add.delay(4, 6)
|
|
68
|
+
print(result.get(timeout=30)) # 10
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Configuration
|
|
72
|
+
|
|
73
|
+
### Auto-provisioning (default)
|
|
74
|
+
|
|
75
|
+
Just use `zerodb://auto` — a free ZeroDB project is created on first use:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
app.config_from_object({
|
|
79
|
+
'broker_url': 'zerodb://auto',
|
|
80
|
+
'broker_transport': 'zerodb_celery.broker:Transport',
|
|
81
|
+
'result_backend': 'zerodb_celery.backend:ZeroDBBackend',
|
|
82
|
+
})
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### Explicit credentials
|
|
86
|
+
|
|
87
|
+
Set environment variables:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
export ZERODB_API_KEY=zdb_your_key_here
|
|
91
|
+
export ZERODB_PROJECT_ID=proj_your_id_here
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Or pass them directly:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
ZeroDBBroker.configure(app, api_key='zdb_...', project_id='proj_...')
|
|
98
|
+
ZeroDBBackend.configure(app, api_key='zdb_...', project_id='proj_...')
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### Credential resolution order
|
|
102
|
+
|
|
103
|
+
1. Explicit `api_key` / `project_id` parameters
|
|
104
|
+
2. `ZERODB_API_KEY` / `ZERODB_PROJECT_ID` environment variables
|
|
105
|
+
3. Cached credentials in `~/.zerodb/credentials.json`
|
|
106
|
+
4. Auto-provision a new free project via API
|
|
107
|
+
|
|
108
|
+
## How It Works
|
|
109
|
+
|
|
110
|
+
### Broker (message queue)
|
|
111
|
+
|
|
112
|
+
The broker uses ZeroDB's event stream as a message queue:
|
|
113
|
+
|
|
114
|
+
- **Publish task**: `POST /api/v1/zerodb/events` with topic `celery:{queue_name}`
|
|
115
|
+
- **Consume task**: `GET /api/v1/zerodb/events?topic=celery:{queue_name}&consume=true`
|
|
116
|
+
|
|
117
|
+
### Result backend (task results)
|
|
118
|
+
|
|
119
|
+
The backend stores results in a ZeroDB table called `celery_results`:
|
|
120
|
+
|
|
121
|
+
- **Store result**: `POST /api/v1/zerodb/tables/celery_results/rows`
|
|
122
|
+
- **Get result**: `GET /api/v1/zerodb/tables/celery_results/rows/{task_id}`
|
|
123
|
+
|
|
124
|
+
The table is auto-created on first use.
|
|
125
|
+
|
|
126
|
+
## Use Cases
|
|
127
|
+
|
|
128
|
+
- **ML pipelines**: Queue training jobs, store results — no Redis needed
|
|
129
|
+
- **AI agent workflows**: Background task processing for agent swarms
|
|
130
|
+
- **Serverless apps**: No infrastructure to manage alongside your Celery workers
|
|
131
|
+
- **Prototyping**: Get Celery running in 30 seconds without Docker
|
|
132
|
+
|
|
133
|
+
## Comparison
|
|
134
|
+
|
|
135
|
+
| Feature | Redis | RabbitMQ | zerodb-celery |
|
|
136
|
+
|---------|-------|----------|---------------|
|
|
137
|
+
| Setup time | 5-30 min | 10-60 min | 0 min (auto) |
|
|
138
|
+
| Infrastructure | Self-hosted or managed | Self-hosted or managed | None |
|
|
139
|
+
| Cost | $15-100+/mo | $15-100+/mo | Free tier |
|
|
140
|
+
| Persistence | Optional (AOF/RDB) | Yes | Yes |
|
|
141
|
+
| Auto-provisioning | No | No | Yes |
|
|
142
|
+
|
|
143
|
+
## Development
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
# Install dev dependencies
|
|
147
|
+
pip install -e ".[dev]"
|
|
148
|
+
|
|
149
|
+
# Run tests
|
|
150
|
+
pytest tests/ -v
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## License
|
|
154
|
+
|
|
155
|
+
MIT
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
zerodb_celery/__init__.py,sha256=Lc7g-H6cxH4t4N8qRcSQdzmYSWjKMbgKusuba-GgfDI,431
|
|
2
|
+
zerodb_celery/backend.py,sha256=OWQXV-O7ZgB6Zt9JGnxWkVIMU5JL5ig9lubs4XVcQbA,5785
|
|
3
|
+
zerodb_celery/broker.py,sha256=l8upQGFxj2h8I4Ot8lPduSRYKJSAdralezVcDTw3b88,6612
|
|
4
|
+
zerodb_celery/provision.py,sha256=wBxPTH1gMiOvVjJ2AR5wMGe05FIaHXG_k1rRt9uYF3E,4022
|
|
5
|
+
zerodb_celery-0.1.0.dist-info/METADATA,sha256=wC32qJof1gEz0Mi-w1UQGqcyx6A6HIA1Vi_nhD9txMg,4851
|
|
6
|
+
zerodb_celery-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
7
|
+
zerodb_celery-0.1.0.dist-info/licenses/LICENSE,sha256=-3M2h1U80S6mPyiuvRG25A0l1xZGpa7eO43lysBIzBY,1072
|
|
8
|
+
zerodb_celery-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 AINative Studio
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|