soniq 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- elephantq/__init__.py +376 -0
- elephantq/app.py +651 -0
- elephantq/backends/__init__.py +247 -0
- elephantq/backends/memory.py +417 -0
- elephantq/backends/postgres.py +799 -0
- elephantq/backends/sqlite.py +502 -0
- elephantq/cli/__init__.py +0 -0
- elephantq/cli/colors.py +396 -0
- elephantq/cli/commands/__init__.py +3 -0
- elephantq/cli/commands/core.py +705 -0
- elephantq/cli/commands/extended.py +359 -0
- elephantq/cli/main.py +71 -0
- elephantq/cli/registry.py +167 -0
- elephantq/core/__init__.py +0 -0
- elephantq/core/heartbeat.py +144 -0
- elephantq/core/leadership.py +61 -0
- elephantq/core/processor.py +266 -0
- elephantq/core/queue.py +70 -0
- elephantq/core/registry.py +206 -0
- elephantq/core/retry.py +86 -0
- elephantq/dashboard/__init__.py +36 -0
- elephantq/dashboard/app.py +468 -0
- elephantq/dashboard/fastapi_app.py +898 -0
- elephantq/db/__init__.py +0 -0
- elephantq/db/connection.py +83 -0
- elephantq/db/context.py +190 -0
- elephantq/db/helpers.py +11 -0
- elephantq/db/migrations/001_core_jobs.sql +53 -0
- elephantq/db/migrations/002_workers.sql +42 -0
- elephantq/db/migrations/003_scheduling.sql +40 -0
- elephantq/db/migrations/004_features.sql +97 -0
- elephantq/db/migrations.py +256 -0
- elephantq/discovery.py +93 -0
- elephantq/errors.py +73 -0
- elephantq/features/__init__.py +26 -0
- elephantq/features/dead_letter.py +747 -0
- elephantq/features/flags.py +11 -0
- elephantq/features/logging.py +699 -0
- elephantq/features/managers.py +268 -0
- elephantq/features/metrics.py +575 -0
- elephantq/features/recurring.py +1011 -0
- elephantq/features/scheduling.py +404 -0
- elephantq/features/signing.py +197 -0
- elephantq/features/timeout_processor.py +660 -0
- elephantq/features/webhooks.py +775 -0
- elephantq/job.py +61 -0
- elephantq/py.typed +0 -0
- elephantq/settings.py +402 -0
- elephantq/utils/__init__.py +1 -0
- elephantq/utils/hashing.py +51 -0
- elephantq/utils/signals.py +156 -0
- elephantq/worker.py +288 -0
- soniq-0.0.1.dist-info/METADATA +192 -0
- soniq-0.0.1.dist-info/RECORD +58 -0
- soniq-0.0.1.dist-info/WHEEL +5 -0
- soniq-0.0.1.dist-info/entry_points.txt +2 -0
- soniq-0.0.1.dist-info/licenses/LICENSE +21 -0
- soniq-0.0.1.dist-info/top_level.txt +1 -0
elephantq/__init__.py
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ElephantQ: Async Job Queue for Python (Backed by PostgreSQL)
|
|
3
|
+
|
|
4
|
+
Simple global usage:
|
|
5
|
+
|
|
6
|
+
import elephantq
|
|
7
|
+
|
|
8
|
+
@elephantq.job()
|
|
9
|
+
async def my_job(message: str):
|
|
10
|
+
print(f"Processing: {message}")
|
|
11
|
+
|
|
12
|
+
await elephantq.enqueue(my_job, message="Hello World")
|
|
13
|
+
await elephantq.run_worker()
|
|
14
|
+
|
|
15
|
+
Instance-based usage for advanced scenarios:
|
|
16
|
+
|
|
17
|
+
app = ElephantQ(database_url="postgresql://localhost/myapp")
|
|
18
|
+
|
|
19
|
+
@app.job()
|
|
20
|
+
async def my_job(message: str):
|
|
21
|
+
print(f"Processing: {message}")
|
|
22
|
+
|
|
23
|
+
await app.enqueue(my_job, message="Hello World")
|
|
24
|
+
await app.run_worker()
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from datetime import datetime, timedelta, timezone
|
|
28
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
29
|
+
from typing import Optional, Union
|
|
30
|
+
|
|
31
|
+
from .app import ElephantQ
|
|
32
|
+
from .job import JobContext, JobStatus, Snooze
|
|
33
|
+
from .settings import configure as settings_configure
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
__version__ = version("elephantq")
|
|
37
|
+
except PackageNotFoundError:
|
|
38
|
+
__version__ = "0.0.0"
|
|
39
|
+
|
|
40
|
+
# Global ElephantQ instance for convenience API
|
|
41
|
+
_global_app: Optional[ElephantQ] = None
|
|
42
|
+
|
|
43
|
+
# Global job registry to survive instance recreation
|
|
44
|
+
_global_job_registry: list[tuple] = []
|
|
45
|
+
|
|
46
|
+
__all__ = [
|
|
47
|
+
"ElephantQ",
|
|
48
|
+
"job",
|
|
49
|
+
"enqueue",
|
|
50
|
+
"schedule",
|
|
51
|
+
"run_worker",
|
|
52
|
+
"_setup",
|
|
53
|
+
"_reset",
|
|
54
|
+
"configure",
|
|
55
|
+
"get_job_status",
|
|
56
|
+
"cancel_job",
|
|
57
|
+
"retry_job",
|
|
58
|
+
"delete_job",
|
|
59
|
+
"list_jobs",
|
|
60
|
+
"get_queue_stats",
|
|
61
|
+
"periodic",
|
|
62
|
+
"JobContext",
|
|
63
|
+
"JobStatus",
|
|
64
|
+
"Snooze",
|
|
65
|
+
"every",
|
|
66
|
+
"cron",
|
|
67
|
+
"features",
|
|
68
|
+
"DASHBOARD_AVAILABLE",
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
# Dashboard availability flag (for CLI checks)
|
|
72
|
+
try:
|
|
73
|
+
from .dashboard.fastapi_app import FASTAPI_AVAILABLE as DASHBOARD_AVAILABLE
|
|
74
|
+
except Exception:
|
|
75
|
+
DASHBOARD_AVAILABLE = False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _get_global_app() -> ElephantQ:
|
|
79
|
+
"""
|
|
80
|
+
Get or create the global ElephantQ application instance.
|
|
81
|
+
|
|
82
|
+
This enables a global convenience API.
|
|
83
|
+
The global app is created lazily on first use with default settings.
|
|
84
|
+
If the existing global app is closed, a new one is created automatically.
|
|
85
|
+
"""
|
|
86
|
+
global _global_app
|
|
87
|
+
|
|
88
|
+
if _global_app is None or _global_app._is_closed:
|
|
89
|
+
_global_app = ElephantQ()
|
|
90
|
+
|
|
91
|
+
# Re-register all global jobs with the new instance
|
|
92
|
+
for job_func, job_kwargs in _global_job_registry:
|
|
93
|
+
_global_app.job(**job_kwargs)(job_func)
|
|
94
|
+
|
|
95
|
+
return _global_app
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def configure(
|
|
99
|
+
*,
|
|
100
|
+
database_url: Optional[str] = None,
|
|
101
|
+
concurrency: Optional[int] = None,
|
|
102
|
+
max_retries: Optional[int] = None,
|
|
103
|
+
queues: Optional[list] = None,
|
|
104
|
+
pool_min_size: Optional[int] = None,
|
|
105
|
+
pool_max_size: Optional[int] = None,
|
|
106
|
+
result_ttl: Optional[int] = None,
|
|
107
|
+
debug: Optional[bool] = None,
|
|
108
|
+
environment: Optional[str] = None,
|
|
109
|
+
**extra,
|
|
110
|
+
):
|
|
111
|
+
"""
|
|
112
|
+
Configure the global ElephantQ instance.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
database_url: Database connection URL
|
|
116
|
+
concurrency: Worker concurrency (1-100)
|
|
117
|
+
max_retries: Default max retry attempts (0-10)
|
|
118
|
+
queues: Default queues to process
|
|
119
|
+
pool_min_size: Minimum connection pool size
|
|
120
|
+
pool_max_size: Maximum connection pool size
|
|
121
|
+
result_ttl: Seconds to keep completed job results
|
|
122
|
+
debug: Enable debug mode
|
|
123
|
+
environment: Environment name (development, testing, production)
|
|
124
|
+
**extra: Additional ElephantQSettings fields
|
|
125
|
+
"""
|
|
126
|
+
global _global_app
|
|
127
|
+
|
|
128
|
+
settings_kwargs = {}
|
|
129
|
+
explicit = {
|
|
130
|
+
"database_url": database_url,
|
|
131
|
+
"concurrency": concurrency,
|
|
132
|
+
"max_retries": max_retries,
|
|
133
|
+
"queues": queues,
|
|
134
|
+
"pool_min_size": pool_min_size,
|
|
135
|
+
"pool_max_size": pool_max_size,
|
|
136
|
+
"result_ttl": result_ttl,
|
|
137
|
+
"debug": debug,
|
|
138
|
+
"environment": environment,
|
|
139
|
+
}
|
|
140
|
+
for key, value in explicit.items():
|
|
141
|
+
if value is not None:
|
|
142
|
+
settings_kwargs[key] = value
|
|
143
|
+
settings_kwargs.update(extra)
|
|
144
|
+
|
|
145
|
+
if settings_kwargs:
|
|
146
|
+
settings_configure(**settings_kwargs)
|
|
147
|
+
|
|
148
|
+
# If the global app is already initialized, mark it as closed so the new
|
|
149
|
+
# app gets a fresh pool. The old pool (if any) will be closed when the
|
|
150
|
+
# old app is garbage collected or via close_pool().
|
|
151
|
+
if (
|
|
152
|
+
_global_app is not None
|
|
153
|
+
and _global_app._is_initialized
|
|
154
|
+
and not _global_app._is_closed
|
|
155
|
+
):
|
|
156
|
+
_global_app._closed = True
|
|
157
|
+
_global_app._initialized = False
|
|
158
|
+
|
|
159
|
+
_global_app = ElephantQ(**settings_kwargs) # type: ignore[arg-type]
|
|
160
|
+
|
|
161
|
+
for job_func, job_kwargs in _global_job_registry:
|
|
162
|
+
_global_app.job(**job_kwargs)(job_func)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def job(**kwargs):
|
|
166
|
+
"""
|
|
167
|
+
Global job decorator.
|
|
168
|
+
|
|
169
|
+
Equivalent to app.job() but uses the global ElephantQ instance.
|
|
170
|
+
Jobs are automatically re-registered if the global instance is recreated.
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
def decorator(func):
|
|
174
|
+
_global_job_registry.append((func, kwargs))
|
|
175
|
+
app = _get_global_app()
|
|
176
|
+
return app.job(**kwargs)(func)
|
|
177
|
+
|
|
178
|
+
return decorator
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def periodic(
|
|
182
|
+
*,
|
|
183
|
+
cron: Optional[str] = None,
|
|
184
|
+
every_seconds: Optional[int] = None,
|
|
185
|
+
every_minutes: Optional[int] = None,
|
|
186
|
+
every_hours: Optional[int] = None,
|
|
187
|
+
**job_kwargs,
|
|
188
|
+
):
|
|
189
|
+
"""
|
|
190
|
+
Decorator that registers a function as a recurring job.
|
|
191
|
+
|
|
192
|
+
Declares both the job and its schedule at definition time.
|
|
193
|
+
The scheduler picks up all @periodic functions automatically.
|
|
194
|
+
|
|
195
|
+
Examples:
|
|
196
|
+
@elephantq.periodic(cron="0 9 * * *")
|
|
197
|
+
async def daily_report():
|
|
198
|
+
...
|
|
199
|
+
|
|
200
|
+
@elephantq.periodic(every_minutes=10, queue="maintenance")
|
|
201
|
+
async def cleanup():
|
|
202
|
+
...
|
|
203
|
+
"""
|
|
204
|
+
# Determine schedule type and value
|
|
205
|
+
interval_args = [
|
|
206
|
+
("seconds", every_seconds),
|
|
207
|
+
("minutes", every_minutes),
|
|
208
|
+
("hours", every_hours),
|
|
209
|
+
]
|
|
210
|
+
interval_set = [(name, val) for name, val in interval_args if val is not None]
|
|
211
|
+
|
|
212
|
+
if cron and interval_set:
|
|
213
|
+
raise ValueError("Cannot specify both cron and every_* parameters")
|
|
214
|
+
if not cron and not interval_set:
|
|
215
|
+
raise ValueError(
|
|
216
|
+
"Must specify either cron='...' or one of every_seconds/every_minutes/every_hours"
|
|
217
|
+
)
|
|
218
|
+
if len(interval_set) > 1:
|
|
219
|
+
raise ValueError(
|
|
220
|
+
"Specify only one of every_seconds, every_minutes, every_hours"
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
if cron:
|
|
224
|
+
schedule_type = "cron"
|
|
225
|
+
schedule_value: Union[str, int] = cron
|
|
226
|
+
else:
|
|
227
|
+
name, val = interval_set[0]
|
|
228
|
+
schedule_type = "interval"
|
|
229
|
+
multipliers = {"seconds": 1, "minutes": 60, "hours": 3600}
|
|
230
|
+
schedule_value = val * multipliers[name] # type: ignore[operator]
|
|
231
|
+
|
|
232
|
+
def decorator(func):
|
|
233
|
+
# Register as a job first
|
|
234
|
+
wrapped = job(**job_kwargs)(func)
|
|
235
|
+
|
|
236
|
+
# Store schedule metadata on the function
|
|
237
|
+
wrapped._elephantq_periodic = { # type: ignore[attr-defined]
|
|
238
|
+
"type": schedule_type,
|
|
239
|
+
"value": schedule_value,
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
return wrapped
|
|
243
|
+
|
|
244
|
+
return decorator
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
async def enqueue(job_func, connection=None, **kwargs):
|
|
248
|
+
"""Enqueue a job using the global ElephantQ instance."""
|
|
249
|
+
app = _get_global_app()
|
|
250
|
+
return await app.enqueue(job_func, connection=connection, **kwargs)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
async def schedule(
|
|
254
|
+
job_func,
|
|
255
|
+
*,
|
|
256
|
+
run_at: Optional[datetime] = None,
|
|
257
|
+
run_in: Optional[Union[int, float, timedelta]] = None,
|
|
258
|
+
connection=None,
|
|
259
|
+
**kwargs,
|
|
260
|
+
):
|
|
261
|
+
"""
|
|
262
|
+
Schedule a job for future execution using the global ElephantQ instance.
|
|
263
|
+
|
|
264
|
+
Use `run_at` for absolute datetime or `run_in` for relative delay.
|
|
265
|
+
"""
|
|
266
|
+
if run_at is None and run_in is None:
|
|
267
|
+
raise ValueError("Must specify either run_at (absolute) or run_in (relative)")
|
|
268
|
+
if run_at is not None and run_in is not None:
|
|
269
|
+
raise ValueError("Cannot specify both run_at and run_in")
|
|
270
|
+
|
|
271
|
+
if run_in is not None:
|
|
272
|
+
if isinstance(run_in, (int, float)):
|
|
273
|
+
run_at = datetime.now(timezone.utc) + timedelta(seconds=run_in)
|
|
274
|
+
elif isinstance(run_in, timedelta):
|
|
275
|
+
run_at = datetime.now(timezone.utc) + run_in
|
|
276
|
+
else:
|
|
277
|
+
raise ValueError("run_in must be int, float (seconds), or timedelta")
|
|
278
|
+
|
|
279
|
+
return await enqueue(job_func, connection=connection, scheduled_at=run_at, **kwargs)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
async def run_worker(
|
|
283
|
+
concurrency: int = 4,
|
|
284
|
+
run_once: bool = False,
|
|
285
|
+
queues: Optional[list] = None,
|
|
286
|
+
):
|
|
287
|
+
"""Run a worker using the global ElephantQ instance."""
|
|
288
|
+
app = _get_global_app()
|
|
289
|
+
return await app.run_worker(
|
|
290
|
+
concurrency=concurrency, run_once=run_once, queues=queues
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
async def _setup() -> int:
|
|
295
|
+
"""Set up ElephantQ — create database (if needed) and run migrations."""
|
|
296
|
+
app = _get_global_app()
|
|
297
|
+
return await app._setup()
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
async def _reset() -> None:
|
|
301
|
+
"""Delete all jobs and workers. Used in test fixtures."""
|
|
302
|
+
app = _get_global_app()
|
|
303
|
+
return await app._reset()
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
async def get_job(job_id: str):
|
|
307
|
+
"""Get information for a specific job."""
|
|
308
|
+
app = _get_global_app()
|
|
309
|
+
return await app.get_job_status(job_id)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
# Backward-compatible alias
|
|
313
|
+
get_job_status = get_job
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
async def get_result(job_id: str):
|
|
317
|
+
"""Get the return value of a completed job, or None."""
|
|
318
|
+
app = _get_global_app()
|
|
319
|
+
return await app.get_result(job_id)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
async def cancel_job(job_id: str):
|
|
323
|
+
"""Cancel a queued job."""
|
|
324
|
+
app = _get_global_app()
|
|
325
|
+
return await app.cancel_job(job_id)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
async def retry_job(job_id: str):
|
|
329
|
+
"""Retry a failed job."""
|
|
330
|
+
app = _get_global_app()
|
|
331
|
+
return await app.retry_job(job_id)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
async def delete_job(job_id: str):
|
|
335
|
+
"""Delete a job from the queue."""
|
|
336
|
+
app = _get_global_app()
|
|
337
|
+
return await app.delete_job(job_id)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
async def list_jobs(
|
|
341
|
+
queue: Optional[str] = None,
|
|
342
|
+
status: Optional[str] = None,
|
|
343
|
+
limit: int = 100,
|
|
344
|
+
offset: int = 0,
|
|
345
|
+
):
|
|
346
|
+
"""List jobs with optional filtering."""
|
|
347
|
+
app = _get_global_app()
|
|
348
|
+
return await app.list_jobs(queue=queue, status=status, limit=limit, offset=offset)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
async def get_queue_stats():
|
|
352
|
+
"""Get statistics for all queues."""
|
|
353
|
+
app = _get_global_app()
|
|
354
|
+
return await app.get_queue_stats()
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
# Feature namespace (advanced features live under elephantq.features)
|
|
358
|
+
from . import features # noqa: E402
|
|
359
|
+
|
|
360
|
+
# Lazy imports for top-level scheduling functions to avoid circular import.
|
|
361
|
+
# elephantq.features.recurring imports from elephantq at module level,
|
|
362
|
+
# so we can't import from it at the top of this file.
|
|
363
|
+
_LAZY_IMPORTS = {
|
|
364
|
+
"every": ("elephantq.features.recurring", "every"),
|
|
365
|
+
"cron": ("elephantq.features.recurring", "cron"),
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def __getattr__(name: str):
|
|
370
|
+
if name in _LAZY_IMPORTS:
|
|
371
|
+
module_path, attr = _LAZY_IMPORTS[name]
|
|
372
|
+
import importlib
|
|
373
|
+
|
|
374
|
+
mod = importlib.import_module(module_path)
|
|
375
|
+
return getattr(mod, attr)
|
|
376
|
+
raise AttributeError(f"module 'elephantq' has no attribute {name!r}")
|