FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. flowerpower/__init__.py +2 -6
  2. flowerpower/cfg/__init__.py +7 -14
  3. flowerpower/cfg/base.py +29 -25
  4. flowerpower/cfg/pipeline/__init__.py +8 -6
  5. flowerpower/cfg/pipeline/_schedule.py +32 -0
  6. flowerpower/cfg/pipeline/adapter.py +0 -5
  7. flowerpower/cfg/pipeline/builder.py +377 -0
  8. flowerpower/cfg/pipeline/run.py +36 -0
  9. flowerpower/cfg/project/__init__.py +11 -24
  10. flowerpower/cfg/project/adapter.py +0 -12
  11. flowerpower/cli/__init__.py +2 -21
  12. flowerpower/cli/cfg.py +0 -3
  13. flowerpower/cli/mqtt.py +0 -6
  14. flowerpower/cli/pipeline.py +22 -415
  15. flowerpower/cli/utils.py +0 -1
  16. flowerpower/flowerpower.py +345 -146
  17. flowerpower/pipeline/__init__.py +2 -0
  18. flowerpower/pipeline/base.py +21 -12
  19. flowerpower/pipeline/io.py +58 -54
  20. flowerpower/pipeline/manager.py +165 -726
  21. flowerpower/pipeline/pipeline.py +643 -0
  22. flowerpower/pipeline/registry.py +285 -18
  23. flowerpower/pipeline/visualizer.py +5 -6
  24. flowerpower/plugins/io/__init__.py +8 -0
  25. flowerpower/plugins/mqtt/__init__.py +7 -11
  26. flowerpower/settings/__init__.py +0 -2
  27. flowerpower/settings/{backend.py → _backend.py} +0 -21
  28. flowerpower/settings/logging.py +1 -1
  29. flowerpower/utils/logging.py +24 -12
  30. flowerpower/utils/misc.py +17 -256
  31. flowerpower/utils/monkey.py +1 -83
  32. flowerpower-0.21.0.dist-info/METADATA +463 -0
  33. flowerpower-0.21.0.dist-info/RECORD +44 -0
  34. flowerpower/cfg/pipeline/schedule.py +0 -74
  35. flowerpower/cfg/project/job_queue.py +0 -238
  36. flowerpower/cli/job_queue.py +0 -1061
  37. flowerpower/fs/__init__.py +0 -29
  38. flowerpower/fs/base.py +0 -662
  39. flowerpower/fs/ext.py +0 -2143
  40. flowerpower/fs/storage_options.py +0 -1420
  41. flowerpower/job_queue/__init__.py +0 -294
  42. flowerpower/job_queue/apscheduler/__init__.py +0 -11
  43. flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
  44. flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
  45. flowerpower/job_queue/apscheduler/manager.py +0 -1051
  46. flowerpower/job_queue/apscheduler/setup.py +0 -554
  47. flowerpower/job_queue/apscheduler/trigger.py +0 -169
  48. flowerpower/job_queue/apscheduler/utils.py +0 -311
  49. flowerpower/job_queue/base.py +0 -413
  50. flowerpower/job_queue/rq/__init__.py +0 -10
  51. flowerpower/job_queue/rq/_trigger.py +0 -37
  52. flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -226
  53. flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -231
  54. flowerpower/job_queue/rq/manager.py +0 -1582
  55. flowerpower/job_queue/rq/setup.py +0 -154
  56. flowerpower/job_queue/rq/utils.py +0 -69
  57. flowerpower/mqtt.py +0 -12
  58. flowerpower/pipeline/job_queue.py +0 -583
  59. flowerpower/pipeline/runner.py +0 -603
  60. flowerpower/plugins/io/base.py +0 -2520
  61. flowerpower/plugins/io/helpers/datetime.py +0 -298
  62. flowerpower/plugins/io/helpers/polars.py +0 -875
  63. flowerpower/plugins/io/helpers/pyarrow.py +0 -570
  64. flowerpower/plugins/io/helpers/sql.py +0 -202
  65. flowerpower/plugins/io/loader/__init__.py +0 -28
  66. flowerpower/plugins/io/loader/csv.py +0 -37
  67. flowerpower/plugins/io/loader/deltatable.py +0 -190
  68. flowerpower/plugins/io/loader/duckdb.py +0 -19
  69. flowerpower/plugins/io/loader/json.py +0 -37
  70. flowerpower/plugins/io/loader/mqtt.py +0 -159
  71. flowerpower/plugins/io/loader/mssql.py +0 -26
  72. flowerpower/plugins/io/loader/mysql.py +0 -26
  73. flowerpower/plugins/io/loader/oracle.py +0 -26
  74. flowerpower/plugins/io/loader/parquet.py +0 -35
  75. flowerpower/plugins/io/loader/postgres.py +0 -26
  76. flowerpower/plugins/io/loader/pydala.py +0 -19
  77. flowerpower/plugins/io/loader/sqlite.py +0 -23
  78. flowerpower/plugins/io/metadata.py +0 -244
  79. flowerpower/plugins/io/saver/__init__.py +0 -28
  80. flowerpower/plugins/io/saver/csv.py +0 -36
  81. flowerpower/plugins/io/saver/deltatable.py +0 -186
  82. flowerpower/plugins/io/saver/duckdb.py +0 -19
  83. flowerpower/plugins/io/saver/json.py +0 -36
  84. flowerpower/plugins/io/saver/mqtt.py +0 -28
  85. flowerpower/plugins/io/saver/mssql.py +0 -26
  86. flowerpower/plugins/io/saver/mysql.py +0 -26
  87. flowerpower/plugins/io/saver/oracle.py +0 -26
  88. flowerpower/plugins/io/saver/parquet.py +0 -36
  89. flowerpower/plugins/io/saver/postgres.py +0 -26
  90. flowerpower/plugins/io/saver/pydala.py +0 -20
  91. flowerpower/plugins/io/saver/sqlite.py +0 -24
  92. flowerpower/plugins/mqtt/cfg.py +0 -17
  93. flowerpower/plugins/mqtt/manager.py +0 -962
  94. flowerpower/settings/job_queue.py +0 -87
  95. flowerpower/utils/scheduler.py +0 -311
  96. flowerpower-0.11.6.20.dist-info/METADATA +0 -537
  97. flowerpower-0.11.6.20.dist-info/RECORD +0 -102
  98. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/WHEEL +0 -0
  99. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/entry_points.txt +0 -0
  100. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/licenses/LICENSE +0 -0
  101. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/top_level.txt +0 -0
@@ -1,1582 +0,0 @@
1
- """
2
- RQSchedulerBackend implementation for FlowerPower using RQ and rq-scheduler.
3
-
4
- This module implements the scheduler backend using RQ (Redis Queue) and rq-scheduler.
5
- """
6
-
7
- import datetime as dt
8
- import multiprocessing
9
- import platform
10
- import sys
11
- import time
12
- import uuid
13
- from typing import Any, Callable
14
-
15
- import duration_parser
16
- from cron_descriptor import get_description
17
- from humanize import precisedelta
18
- from loguru import logger
19
- from rq import Queue, Repeat, Retry
20
- from rq.job import Callback, Job
21
- from rq.results import Result
22
- from rq.worker import Worker
23
- from rq.worker_pool import WorkerPool
24
- from rq_scheduler import Scheduler
25
-
26
- from ...fs import AbstractFileSystem
27
- from ...utils.logging import setup_logging
28
- from ..base import BaseJobQueueManager
29
- from .setup import RQBackend
30
-
31
- setup_logging()
32
-
33
- if sys.platform == "darwin" and platform.machine() == "arm64":
34
- try:
35
- # Check if the start method has already been set to avoid errors
36
- if multiprocessing.get_start_method(allow_none=True) is None:
37
- multiprocessing.set_start_method("fork")
38
- logger.debug("Set multiprocessing start method to 'fork' for macOS ARM.")
39
- elif multiprocessing.get_start_method() != "fork":
40
- logger.warning(
41
- f"Multiprocessing start method already set to '{multiprocessing.get_start_method()}'. "
42
- f"Cannot set to 'fork'. This might cause issues on macOS ARM."
43
- )
44
- except RuntimeError as e:
45
- # Handle cases where the context might already be started
46
- logger.warning(f"Could not set multiprocessing start method to 'fork': {e}")
47
-
48
-
49
- class RQManager(BaseJobQueueManager):
50
- """Implementation of BaseScheduler using Redis Queue (RQ) and rq-scheduler.
51
-
52
- This worker class uses RQ and rq-scheduler as the backend to manage jobs and schedules.
53
- It supports multiple queues, background workers, and job scheduling capabilities.
54
-
55
- Typical usage:
56
- ```python
57
- worker = RQManager(name="my_rq_worker")
58
- worker.start_worker(background=True)
59
-
60
- # Add a job
61
- def my_job(x: int) -> int:
62
- return x * 2
63
-
64
- job_id = worker.add_job(my_job, func_args=(10,))
65
- ```
66
- """
67
-
68
- def __init__(
69
- self,
70
- name: str = "rq_scheduler",
71
- base_dir: str | None = None,
72
- backend: RQBackend | None = None,
73
- storage_options: dict[str, Any] | None = None,
74
- fs: AbstractFileSystem | None = None,
75
- log_level: str | None = None,
76
- ):
77
- """Initialize the RQ scheduler backend.
78
-
79
- Args:
80
- name: Name of the scheduler instance. Used for identification in logs
81
- and queue names.
82
- base_dir: Base directory for the FlowerPower project. Used for finding
83
- configuration files.
84
- backend: RQBackend instance for Redis connection configuration.
85
- If None, configuration is loaded from project settings.
86
- storage_options: Options for configuring file system storage access.
87
- Example: {"mode": "async", "root": "/tmp"}
88
- fs: Custom filesystem implementation for storage operations.
89
- log_level: Logging level to use for this worker instance.
90
- Example: "DEBUG", "INFO", "WARNING", etc.
91
-
92
- Raises:
93
- RuntimeError: If backend setup fails due to Redis connection issues
94
- or missing configurations.
95
- ImportError: If required dependencies are not installed.
96
-
97
- Example:
98
- ```python
99
- # Basic initialization
100
- worker = RQManager(name="my_worker")
101
-
102
- # With custom backend and logging
103
- backend = RQBackend(
104
- uri="redis://localhost:6379/0",
105
- queues=["high", "default", "low"]
106
- )
107
- worker = RQManager(
108
- name="custom_worker",
109
- backend=backend,
110
- log_level="DEBUG"
111
- )
112
- ```
113
- """
114
- if log_level:
115
- setup_logging(level=log_level)
116
- self._log_level = log_level or "INFO"
117
-
118
- super().__init__(
119
- type="rq",
120
- name=name,
121
- base_dir=base_dir,
122
- backend=backend,
123
- fs=fs,
124
- storage_options=storage_options,
125
- )
126
-
127
- if self._backend is None:
128
- self._setup_backend()
129
-
130
- redis_conn = self._backend.client
131
- self._queues = {}
132
-
133
- self._queue_names = self._backend.queues # [:-1]
134
- for queue_name in self._queue_names:
135
- queue = Queue(name=queue_name, connection=redis_conn)
136
- self._queues[queue_name] = queue
137
- self._queues[queue_name].log = logger
138
- logger.debug(f"Created queue and scheduler for '{queue_name}'")
139
-
140
- self._scheduler_name = self._backend.queues[-1]
141
- self._scheduler = Scheduler(
142
- connection=redis_conn, queue_name=self._backend.queues[-1], interval=60
143
- )
144
- self._scheduler.log = logger
145
-
146
- def _setup_backend(self) -> None:
147
- """Set up the Redis backend for the scheduler.
148
-
149
- This internal method initializes the Redis connection and queues based on
150
- project configuration. It validates configuration, handles errors, and logs
151
- the setup process.
152
-
153
- Raises:
154
- RuntimeError: If Redis connection fails or configuration is invalid.
155
- """
156
- backend_cfg = getattr(self.cfg, "backend", None)
157
- if not backend_cfg:
158
- logger.error(
159
- "Backend configuration is missing in project.worker.rq_backend.backend."
160
- )
161
- raise RuntimeError("Backend configuration is missing.")
162
- try:
163
- self._backend = RQBackend(**backend_cfg.to_dict())
164
- logger.info(
165
- f"RQ backend setup successful (type: {self._backend.type}, uri: {self._backend.uri})"
166
- )
167
- except Exception as exc:
168
- logger.exception(
169
- f"Failed to set up RQ backend (type: {getattr(self._backend, 'type', None)}, uri: {getattr(self._backend, 'uri', None)}): {exc}"
170
- )
171
- raise RuntimeError(f"Failed to set up RQ backend: {exc}") from exc
172
-
173
- def start_worker(
174
- self,
175
- background: bool = False,
176
- queue_names: list[str] | None = None,
177
- with_scheduler: bool = True,
178
- **kwargs: Any,
179
- ) -> None:
180
- """Start a worker process for processing jobs from the queues.
181
-
182
- Args:
183
- background: If True, runs the worker in a non-blocking background mode.
184
- If False, runs in the current process and blocks until stopped.
185
- queue_names: List of queue names to process. If None, processes all
186
- queues defined in the backend configuration.
187
- with_scheduler: Whether to include the scheduler queue for processing
188
- scheduled jobs.
189
- **kwargs: Additional arguments passed to RQ's Worker class.
190
- Example: {"burst": True, "logging_level": "INFO", "job_monitoring_interval": 30}
191
-
192
- Raises:
193
- RuntimeError: If worker fails to start or if Redis connection fails.
194
-
195
- Example:
196
- ```python
197
- # Start worker in background processing all queues
198
- worker.start_worker(background=True)
199
-
200
- # Start worker for specific queues
201
- worker.start_worker(
202
- background=True,
203
- queue_names=["high", "default"],
204
- with_scheduler=False
205
- )
206
-
207
- # Start worker with custom settings
208
- worker.start_worker(
209
- background=True,
210
- max_jobs=100,
211
- job_monitoring_interval=30
212
- )
213
- ```
214
- """
215
- import multiprocessing
216
-
217
- logging_level = kwargs.pop("logging_level", self._log_level)
218
- burst = kwargs.pop("burst", False)
219
- max_jobs = kwargs.pop("max_jobs", None)
220
- # Determine which queues to process
221
- if queue_names is None:
222
- # Use all queues by default
223
- queue_names = self._queue_names
224
- queue_names_str = ", ".join(queue_names)
225
- else:
226
- # Filter to only include valid queue names
227
- queue_names = [name for name in queue_names if name in self._queue_names]
228
- queue_names_str = ", ".join(queue_names)
229
-
230
- if not queue_names:
231
- logger.error("No valid queues specified, cannot start worker")
232
- return
233
-
234
- if with_scheduler:
235
- # Add the scheduler queue to the list of queues
236
- queue_names.append(self._scheduler_name)
237
- queue_names_str = ", ".join(queue_names)
238
-
239
- # Create a worker instance with queue names (not queue objects)
240
- worker = Worker(queue_names, connection=self._backend.client, **kwargs)
241
-
242
- if background:
243
- # We need to use a separate process rather than a thread because
244
- # RQ's signal handler registration only works in the main thread
245
- def run_worker_process(queue_names_arg):
246
- # Import RQ inside the process to avoid connection sharing issues
247
- from redis import Redis
248
- from rq import Worker
249
-
250
- # Create a fresh Redis connection in this process
251
- redis_conn = Redis.from_url(self._backend.uri)
252
-
253
- # Create a worker instance with queue names
254
- worker_proc = Worker(queue_names_arg, connection=redis_conn)
255
-
256
- # Disable the default signal handlers in RQ worker by patching
257
- # the _install_signal_handlers method to do nothing
258
- worker_proc._install_signal_handlers = lambda: None
259
-
260
- # Work until terminated
261
- worker_proc.work(
262
- with_scheduler=True,
263
- logging_level=logging_level,
264
- burst=burst,
265
- max_jobs=max_jobs,
266
- )
267
-
268
- # Create and start the process
269
- process = multiprocessing.Process(
270
- target=run_worker_process,
271
- args=(queue_names,),
272
- name=f"rq-worker-{self.name}",
273
- )
274
- # Don't use daemon=True to avoid the "daemonic processes are not allowed to have children" error
275
- process.start()
276
- self._worker_process = process
277
- logger.info(
278
- f"Started RQ worker in background process (PID: {process.pid}) for queues: {queue_names_str}"
279
- )
280
- else:
281
- # Start worker in the current process (blocking)
282
- logger.info(
283
- f"Starting RQ worker in current process (blocking) for queues: {queue_names_str}"
284
- )
285
- worker.work(
286
- with_scheduler=True,
287
- logging_level=logging_level,
288
- burst=burst,
289
- max_jobs=max_jobs,
290
- )
291
-
292
- def stop_worker(self) -> None:
293
- """Stop the worker process.
294
-
295
- This method stops the worker process if running in background mode and
296
- performs cleanup. It should be called before program exit.
297
-
298
- Example:
299
- ```python
300
- try:
301
- worker.start_worker(background=True)
302
- # ... do work ...
303
- finally:
304
- worker.stop_worker()
305
- ```
306
- """
307
- if hasattr(self, "_worker_process") and self._worker_process is not None:
308
- if self._worker_process.is_alive():
309
- self._worker_process.terminate()
310
- self._worker_process.join(timeout=5)
311
- logger.info("RQ worker process terminated")
312
- self._worker_process = None
313
- else:
314
- logger.warning("No worker process to stop")
315
-
316
- def start_worker_pool(
317
- self,
318
- num_workers: int | None = None,
319
- background: bool = False,
320
- queue_names: list[str] | None = None,
321
- with_scheduler: bool = True,
322
- **kwargs: Any,
323
- ) -> None:
324
- """Start a pool of worker processes to handle jobs in parallel.
325
-
326
- This implementation uses RQ's WorkerPool class which provides robust worker
327
- management with proper monitoring and graceful shutdown.
328
-
329
- Args:
330
- num_workers: Number of worker processes to start. If None, uses CPU
331
- count or configuration value.
332
- background: If True, runs the worker pool in background mode.
333
- If False, runs in the current process and blocks.
334
- queue_names: List of queue names to process. If None, processes all
335
- queues defined in the backend configuration.
336
- with_scheduler: Whether to include the scheduler queue for processing
337
- scheduled jobs.
338
- **kwargs: Additional arguments passed to RQ's WorkerPool class.
339
- Example: {"max_jobs": 100, "job_monitoring_interval": 30}
340
-
341
- Raises:
342
- RuntimeError: If worker pool fails to start or Redis connection fails.
343
-
344
- Example:
345
- ```python
346
- # Start pool with default settings
347
- worker.start_worker_pool(num_workers=4, background=True)
348
-
349
- # Start pool for specific queues
350
- worker.start_worker_pool(
351
- num_workers=4,
352
- background=True,
353
- queue_names=["high", "default"],
354
- with_scheduler=False
355
- )
356
-
357
- # Start pool with custom settings
358
- worker.start_worker_pool(
359
- num_workers=4,
360
- background=True,
361
- max_jobs=100,
362
- job_monitoring_interval=30
363
- )
364
- ```
365
- """
366
- import multiprocessing
367
-
368
- logging_level = kwargs.pop("logging_level", self._log_level)
369
- burst = kwargs.pop("burst", False)
370
- max_jobs = kwargs.pop("max_jobs", None)
371
-
372
- # if num_workers is None:
373
- # backend = getattr(self.cfg, "backend", None)
374
- # if backend is not None:
375
- # num_workers = getattr(backend, "num_workers", None)
376
- if num_workers is None:
377
- num_workers = self.cfg.num_workers or multiprocessing.cpu_count()
378
- # Determine which queues to process
379
- if queue_names is None:
380
- # Use all queues by default
381
- queue_list = self._queue_names
382
- queue_names_str = ", ".join(queue_list)
383
- else:
384
- # Filter to only include valid queue names
385
- queue_list = [name for name in queue_names if name in self._queue_names]
386
- queue_names_str = ", ".join(queue_list)
387
-
388
- if not queue_list:
389
- logger.error("No valid queues specified, cannot start worker pool")
390
- return
391
- if with_scheduler:
392
- # Add the scheduler queue to the list of queues
393
- queue_list.append(self._scheduler_name)
394
- queue_names_str = ", ".join(queue_list)
395
-
396
- # Initialize RQ's WorkerPool
397
- worker_pool = WorkerPool(
398
- queues=queue_list,
399
- connection=self._backend.client,
400
- num_workers=num_workers,
401
- **kwargs,
402
- )
403
- # worker_pool.log = logger
404
-
405
- self._worker_pool = worker_pool
406
-
407
- if background:
408
- # Start the worker pool process using multiprocessing to avoid signal handler issues
409
- def run_pool_process():
410
- worker_pool.start(
411
- burst=burst, logging_level=logging_level, max_jobs=max_jobs
412
- )
413
-
414
- self._pool_process = multiprocessing.Process(
415
- target=run_pool_process,
416
- name=f"rq-worker-pool-{self.name}",
417
- )
418
- self._pool_process.start()
419
- logger.info(
420
- f"Worker pool started with {num_workers} workers across queues: {queue_names_str} in background process (PID: {self._pool_process.pid})"
421
- )
422
- else:
423
- # Start the worker pool in the current process (blocking)
424
- logger.info(
425
- f"Starting worker pool with {num_workers} workers across queues: {queue_names_str} in foreground (blocking)"
426
- )
427
- worker_pool.start(burst=burst, logging_level=logging_level)
428
-
429
- def stop_worker_pool(self) -> None:
430
- """Stop all worker processes in the pool.
431
-
432
- This method stops all worker processes in the pool and performs cleanup.
433
- It ensures a graceful shutdown of all workers.
434
-
435
- Example:
436
- ```python
437
- try:
438
- worker.start_worker_pool(num_workers=4, background=True)
439
- # ... do work ...
440
- finally:
441
- worker.stop_worker_pool()
442
- ```
443
- """
444
- if hasattr(self, "_worker_pool"):
445
- logger.info("Stopping RQ worker pool")
446
- self._worker_pool.stop_workers()
447
-
448
- if hasattr(self, "_pool_process") and self._pool_process.is_alive():
449
- # Terminate the worker pool process
450
- self._pool_process.terminate()
451
- self._pool_process.join(timeout=10)
452
- if self._pool_process.is_alive():
453
- logger.warning(
454
- "Worker pool process did not terminate within timeout"
455
- )
456
-
457
- self._worker_pool = None
458
-
459
- if hasattr(self, "_pool_process"):
460
- self._pool_process = None
461
- else:
462
- logger.warning("No worker pool to stop")
463
-
464
- def start_scheduler(self, background: bool = False, interval: int = 60) -> None:
465
- """Start the RQ scheduler process.
466
-
467
- The scheduler process manages scheduled and recurring jobs. It must be
468
- running for scheduled jobs to execute.
469
-
470
- Args:
471
- background: If True, runs the scheduler in a non-blocking background mode.
472
- If False, runs in the current process and blocks.
473
- interval: How often to check for scheduled jobs, in seconds.
474
-
475
- Raises:
476
- RuntimeError: If scheduler fails to start or Redis connection fails.
477
-
478
- Example:
479
- ```python
480
- # Start scheduler in background checking every 30 seconds
481
- worker.start_scheduler(background=True, interval=30)
482
-
483
- # Start scheduler in foreground (blocking)
484
- worker.start_scheduler(background=False)
485
- ```
486
- """
487
- # Create a scheduler instance with the queue name
488
- if not hasattr(self, "_scheduler"):
489
- self._scheduler = Scheduler(
490
- connection=self._backend.client,
491
- queue_name=self._backend.queues[-1],
492
- interval=interval,
493
- )
494
- self._scheduler.log = logger
495
-
496
- elif self._scheduler._interval != interval:
497
- self._scheduler = Scheduler(
498
- connection=self._backend.client,
499
- queue_name=self._backend.queues[-1],
500
- interval=interval,
501
- )
502
- self._scheduler.log = logger
503
-
504
- if background:
505
-
506
- def run_scheduler():
507
- self._scheduler.run()
508
-
509
- self._scheduler_process = multiprocessing.Process(
510
- target=run_scheduler, name=f"rq-scheduler-{self.name}"
511
- )
512
- self._scheduler_process.start()
513
- logger.info(
514
- f"Started RQ scheduler in background (PID: {self._scheduler_process.pid})"
515
- )
516
- else:
517
- logger.info("Starting RQ scheduler in current process (blocking)")
518
- self._scheduler.run()
519
-
520
- def stop_scheduler(self) -> None:
521
- """Stop the RQ scheduler process.
522
-
523
- This method stops the scheduler process if running in background mode
524
- and performs cleanup.
525
-
526
- Example:
527
- ```python
528
- try:
529
- worker.start_scheduler(background=True)
530
- # ... do work ...
531
- finally:
532
- worker.stop_scheduler()
533
- ```
534
- """
535
- if hasattr(self, "_scheduler_process") and self._scheduler_process is not None:
536
- if self._scheduler_process.is_alive():
537
- self._scheduler_process.terminate()
538
- self._scheduler_process.join(timeout=5)
539
- logger.info("RQ scheduler process terminated")
540
- self._scheduler_process = None
541
- else:
542
- logger.debug("No scheduler process to stop")
543
-
544
- ## Jobs ###
545
-
546
- def add_job(
547
- self,
548
- func: Callable,
549
- func_args: tuple | None = None,
550
- func_kwargs: dict[str, Any] | None = None,
551
- job_id: str | None = None,
552
- result_ttl: int | dt.timedelta | None = None,
553
- ttl: int | dt.timedelta | None = None,
554
- timeout: int | dt.timedelta | None = None,
555
- queue_name: str | None = None,
556
- run_at: dt.datetime | str | None = None,
557
- run_in: dt.timedelta | int | str | None = None,
558
- retry: int | dict | None = None,
559
- repeat: int | dict | None = None,
560
- meta: dict | None = None,
561
- failure_ttl: int | dt.timedelta | None = None,
562
- group_id: str | None = None,
563
- on_success: Callback | Callable | str | None = None,
564
- on_failure: Callback | Callable | str | None = None,
565
- on_stopped: Callback | Callable | str | None = None,
566
- **job_kwargs,
567
- ) -> Job:
568
- """Add a job for immediate or scheduled execution.
569
-
570
- Args:
571
- func: Function to execute. Must be importable from the worker process.
572
- func_args: Positional arguments to pass to the function.
573
- func_kwargs: Keyword arguments to pass to the function.
574
- job_id: Optional unique identifier for the job. If None, a UUID is generated.
575
- result_ttl: Time to live for the job result, as seconds or timedelta.
576
- After this time, the result may be removed from Redis.
577
- ttl: Maximum time the job can exist in Redis, as seconds or timedelta.
578
- After this time, the job will be removed even if not complete.
579
- timeout: Maximum time the job can run before being killed, as seconds or timedelta.
580
- queue_name: Name of the queue to place the job in. If None, uses the
581
- first queue from configuration.
582
- run_at: Schedule the job to run at a specific datetime.
583
- run_in: Schedule the job to run after a delay.
584
- retry: Number of retries or retry configuration dictionary.
585
- Example dict: {"max": 3, "interval": 60}
586
- repeat: Number of repetitions or repeat configuration dictionary.
587
- Example dict: {"max": 5, "interval": 3600}
588
- meta: Additional metadata to store with the job.
589
- failure_ttl: Time to live for the job failure result, as seconds or timedelta.
590
- group_id: Optional group ID to associate this job with a group.
591
- on_success: Callback to run on job success. Can be a function, string,
592
- or RQ Callback instance.
593
- on_failure: Callback to run on job failure. Can be a function, string,
594
- or RQ Callback instance.
595
- on_stopped: Callback to run when the job is stopped. Can be a function,
596
- string, or RQ Callback instance.
597
- **job_kwargs: Additional arguments for RQ's Job class.
598
-
599
- Returns:
600
- Job: The created job instance.
601
-
602
- Raises:
603
- ValueError: If the function is not serializable or arguments are invalid.
604
- RuntimeError: If Redis connection fails.
605
-
606
- Example:
607
- ```python
608
- def my_task(x: int, y: int = 0) -> int:
609
- return x + y
610
-
611
- # Add immediate job
612
- job = worker.add_job(
613
- my_task,
614
- func_args=(1,),
615
- func_kwargs={"y": 2},
616
- result_ttl=3600 # Keep result for 1 hour
617
- )
618
-
619
- # Add scheduled job
620
- tomorrow = dt.datetime.now() + dt.timedelta(days=1)
621
- job = worker.add_job(
622
- my_task,
623
- func_args=(1, 2),
624
- run_at=tomorrow,
625
- queue_name="scheduled"
626
- )
627
-
628
- # Add job with retries
629
- job = worker.add_job(
630
- my_task,
631
- func_args=(1, 2),
632
- retry={"max": 3, "interval": 60} # 3 retries, 1 minute apart
633
- )
634
-
635
- # Add repeating job
636
- job = worker.add_job(
637
- my_task,
638
- func_args=(1, 2),
639
- repeat={"max": 5, "interval": 3600} # 5 times, hourly
640
- )
641
- ```
642
- """
643
- job_id = job_id or str(uuid.uuid4())
644
- if isinstance(result_ttl, (int, float)):
645
- result_ttl = dt.timedelta(seconds=result_ttl)
646
- # args = args or ()
647
- # kwargs = kwargs or {}
648
- if queue_name is None:
649
- queue_name = self._queue_names[0]
650
- elif queue_name not in self._queue_names:
651
- logger.warning(
652
- f"Queue '{queue_name}' not found, using '{self._queue_names[0]}'"
653
- )
654
- queue_name = self._queue_names[0]
655
-
656
- if repeat:
657
- # If repeat is an int, convert it to a Repeat instance
658
- if isinstance(repeat, int):
659
- repeat = Repeat(max=repeat)
660
- elif isinstance(repeat, dict):
661
- # If repeat is a dict, convert it to a Repeat instance
662
- repeat = Repeat(**repeat)
663
- else:
664
- raise ValueError("Invalid repeat value. Must be int or dict.")
665
- if retry:
666
- if isinstance(retry, int):
667
- retry = Retry(max=retry)
668
- elif isinstance(retry, dict):
669
- # If retry is a dict, convert it to a Retry instance
670
- retry = Retry(**retry)
671
- else:
672
- raise ValueError("Invalid retry value. Must be int or dict.")
673
-
674
- if isinstance(ttl, dt.timedelta):
675
- ttl = ttl.total_seconds()
676
- if isinstance(timeout, dt.timedelta):
677
- timeout = timeout.total_seconds()
678
- if isinstance(result_ttl, dt.timedelta):
679
- result_ttl = result_ttl.total_seconds()
680
- if isinstance(failure_ttl, dt.timedelta):
681
- failure_ttl = failure_ttl.total_seconds()
682
-
683
- if isinstance(on_success, (str, Callable)):
684
- on_success = Callback(on_success)
685
- if isinstance(on_failure, (str, Callable)):
686
- on_failure = Callback(on_failure)
687
- if isinstance(on_stopped, (str, Callable)):
688
- on_stopped = Callback(on_stopped)
689
-
690
- queue = self._queues[queue_name]
691
- if run_at:
692
- # Schedule the job to run at a specific time
693
- run_at = (
694
- dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
695
- )
696
- job = queue.enqueue_at(
697
- run_at,
698
- func,
699
- args=func_args,
700
- kwargs=func_kwargs,
701
- job_id=job_id,
702
- result_ttl=int(result_ttl) if result_ttl else None,
703
- ttl=int(ttl) if ttl else None,
704
- failure_ttl=int(failure_ttl) if failure_ttl else None,
705
- timeout=int(timeout) if timeout else None,
706
- retry=retry,
707
- repeat=repeat,
708
- meta=meta,
709
- group_id=group_id,
710
- on_success=on_success,
711
- on_failure=on_failure,
712
- on_stopped=on_stopped,
713
- **job_kwargs,
714
- )
715
- logger.info(
716
- f"Enqueued job {job.id} ({func.__name__}) on queue '{queue_name}'. Scheduled to run at {run_at}."
717
- )
718
- elif run_in:
719
- # Schedule the job to run after a delay
720
- run_in = (
721
- duration_parser.parse(run_in) if isinstance(run_in, str) else run_in
722
- )
723
- run_in = (
724
- dt.timedelta(seconds=run_in)
725
- if isinstance(run_in, (int, float))
726
- else run_in
727
- )
728
- job = queue.enqueue_in(
729
- run_in,
730
- func,
731
- args=func_args,
732
- kwargs=func_kwargs,
733
- job_id=job_id,
734
- result_ttl=int(result_ttl) if result_ttl else None,
735
- ttl=int(ttl) if ttl else None,
736
- failure_ttl=int(failure_ttl) if failure_ttl else None,
737
- timeout=int(timeout) if timeout else None,
738
- retry=retry,
739
- repeat=repeat,
740
- meta=meta,
741
- group_id=group_id,
742
- on_success=on_success,
743
- on_failure=on_failure,
744
- on_stopped=on_stopped,
745
- **job_kwargs,
746
- )
747
- logger.info(
748
- f"Enqueued job {job.id} ({func.__name__}) on queue '{queue_name}'. Scheduled to run in {precisedelta(run_in)}."
749
- )
750
- else:
751
- # Enqueue the job for immediate execution
752
- job = queue.enqueue(
753
- func,
754
- args=func_args,
755
- kwargs=func_kwargs,
756
- job_id=job_id,
757
- result_ttl=int(result_ttl) if result_ttl else None,
758
- ttl=int(ttl) if ttl else None,
759
- failure_ttl=int(failure_ttl) if failure_ttl else None,
760
- timeout=int(timeout) if timeout else None,
761
- retry=retry,
762
- repeat=repeat,
763
- meta=meta,
764
- group_id=group_id,
765
- on_success=on_success,
766
- on_failure=on_failure,
767
- on_stopped=on_stopped,
768
- **job_kwargs,
769
- )
770
- logger.info(
771
- f"Enqueued job {job.id} ({func.__name__}) on queue '{queue_name}'"
772
- )
773
- return job
774
-
775
- def run_job(
776
- self,
777
- func: Callable,
778
- func_args: tuple | None = None,
779
- func_kwargs: dict[str, Any] | None = None,
780
- job_id: str | None = None,
781
- result_ttl: int | dt.timedelta | None = None,
782
- ttl: int | dt.timedelta | None = None,
783
- queue_name: str | None = None,
784
- retry: int | dict | None = None,
785
- repeat: int | dict | None = None,
786
- meta: dict | None = None,
787
- failure_ttl: int | dt.timedelta | None = None,
788
- group_id: str | None = None,
789
- on_success: Callback | Callable | str | None = None,
790
- on_failure: Callback | Callable | str | None = None,
791
- on_stopped: Callback | Callable | str | None = None,
792
- **job_kwargs,
793
- ) -> Any:
794
- """Run a job immediately and return its result.
795
-
796
- This method is a wrapper around add_job that waits for the job to complete
797
- and returns its result.
798
-
799
- Args:
800
- func: Function to execute. Must be importable from the worker process.
801
- func_args: Positional arguments to pass to the function.
802
- func_kwargs: Keyword arguments to pass to the function.
803
- job_id: Optional unique identifier for the job.
804
- result_ttl: Time to live for the job result.
805
- ttl: Maximum time the job can exist.
806
- queue_name: Name of the queue to use.
807
- retry: Number of retries or retry configuration.
808
- repeat: Number of repetitions or repeat configuration.
809
- meta: Additional metadata to store with the job.
810
- failure_ttl: Time to live for the job failure result.
811
- group_id: Optional group ID to associate this job with a group.
812
- on_success: Callback to run on job success.
813
- on_failure: Callback to run on job failure.
814
- on_stopped: Callback to run when the job is stopped.
815
- **job_kwargs: Additional arguments for RQ's Job class.
816
-
817
- Returns:
818
- Any: The result returned by the executed function.
819
-
820
- Raises:
821
- Exception: Any exception raised by the executed function.
822
- TimeoutError: If the job times out before completion.
823
-
824
- Example:
825
- ```python
826
- def add(x: int, y: int) -> int:
827
- return x + y
828
-
829
- # Run job and get result immediately
830
- result = worker.run_job(
831
- add,
832
- func_args=(1, 2),
833
- retry=3 # Retry up to 3 times on failure
834
- )
835
- assert result == 3
836
- ```
837
- """
838
- job = self.add_job(
839
- func=func,
840
- func_args=func_args,
841
- func_kwargs=func_kwargs,
842
- job_id=job_id,
843
- result_ttl=result_ttl,
844
- ttl=ttl,
845
- queue_name=queue_name,
846
- retry=retry,
847
- repeat=repeat,
848
- meta=meta,
849
- failure_ttl=failure_ttl,
850
- group_id=group_id,
851
- on_success=on_success,
852
- on_failure=on_failure,
853
- on_stopped=on_stopped,
854
- **job_kwargs,
855
- )
856
- while not job.is_finished:
857
- job.refresh()
858
- time.sleep(0.1)
859
- return job.result
860
-
861
- def _get_job_queue_name(self, job: str | Job) -> str | None:
862
- """Get the queue name for a job.
863
-
864
- Args:
865
- job: Job ID or Job object.
866
-
867
- Returns:
868
- str | None: Name of the queue containing the job, or None if not found.
869
- """
870
- job_id = job if isinstance(job, str) else job.id
871
- for queue_name in self.job_ids:
872
- if job_id in self.job_ids[queue_name]:
873
- return queue_name
874
- return None
875
-
876
- def get_jobs(
877
- self, queue_name: str | list[str] | None = None
878
- ) -> dict[str, list[Job]]:
879
- """Get all jobs from specified queues.
880
-
881
- Args:
882
- queue_name: Optional queue name or list of queue names to get jobs from.
883
- If None, gets jobs from all queues.
884
-
885
- Returns:
886
- dict[str, list[Job]]: Dictionary mapping queue names to lists of jobs.
887
-
888
- Example:
889
- ```python
890
- # Get jobs from all queues
891
- jobs = worker.get_jobs()
892
- for queue_name, queue_jobs in jobs.items():
893
- print(f"Queue {queue_name}: {len(queue_jobs)} jobs")
894
-
895
- # Get jobs from specific queues
896
- jobs = worker.get_jobs(["high", "default"])
897
- ```
898
- """
899
- if queue_name is None:
900
- queue_name = self._queue_names
901
- elif isinstance(queue_name, str):
902
- queue_name = [queue_name]
903
- jobs = {
904
- queue_name: self._queues[queue_name].get_jobs() for queue_name in queue_name
905
- }
906
- return jobs
907
-
908
- def get_job(self, job_id: str) -> Job | None:
909
- """Get a specific job by its ID.
910
-
911
- Args:
912
- job_id: Unique identifier of the job to retrieve.
913
-
914
- Returns:
915
- Job | None: The job object if found, None otherwise.
916
-
917
- Example:
918
- ```python
919
- job = worker.get_job("550e8400-e29b-41d4-a716-446655440000")
920
- if job:
921
- print(f"Job status: {job.get_status()}")
922
- ```
923
- """
924
- queue_name = self._get_job_queue_name(job=job_id)
925
- if queue_name is None:
926
- logger.error(f"Job {job_id} not found in any queue")
927
- return None
928
- job = self._queues[queue_name].fetch_job(job_id)
929
- if job is None:
930
- logger.error(f"Job {job_id} not found in queue '{queue_name}'")
931
- return None
932
- return job
933
-
934
- def get_job_result(self, job: str | Job, delete_result: bool = False) -> Any:
935
- """Get the result of a completed job.
936
-
937
- Args:
938
- job: Job ID or Job object.
939
- delete_result: If True, deletes the job and its result after retrieval.
940
-
941
- Returns:
942
- Any: The result of the job if available.
943
-
944
- Example:
945
- ```python
946
- # Get result and keep the job
947
- result = worker.get_job_result("job-123")
948
-
949
- # Get result and clean up
950
- result = worker.get_job_result("job-123", delete_result=True)
951
- ```
952
- """
953
- if isinstance(job, str):
954
- job = self.get_job(job_id=job)
955
-
956
- if job is None:
957
- logger.error(f"Job {job} not found in any queue")
958
- return None
959
-
960
- if delete_result:
961
- self.delete_job(job)
962
-
963
- return job.result
964
-
965
- def cancel_job(self, job: str | Job) -> bool:
966
- """
967
- Cancel a job in the queue.
968
-
969
- Args:
970
- job: Job ID or Job object
971
-
972
- Returns:
973
- bool: True if the job was canceled, False otherwise
974
- """
975
- if isinstance(job, str):
976
- job = self.get_job(job_id=job)
977
- if job is None:
978
- logger.error(f"Job {job} not found in any queue")
979
- return False
980
-
981
- job.cancel()
982
- logger.info(f"Canceled job {job.id} from queue '{job.origin}'")
983
- return True
984
-
985
- def delete_job(self, job: str | Job, ttl: int = 0, **kwargs) -> bool:
986
- """
987
- Remove a job from the queue.
988
-
989
- Args:
990
- job: Job ID or Job object
991
- ttl: Optional time to live for the job (in seconds). 0 means no TTL.
992
- Remove the job immediately.
993
- **kwargs: Additional parameters for the job removal
994
-
995
- Returns:
996
- bool: True if the job was removed, False otherwise
997
- """
998
- if isinstance(job, str):
999
- job = self.get_job(job)
1000
- if job is None:
1001
- return False
1002
- if ttl:
1003
- job.cleanup(ttl=ttl, **kwargs)
1004
- logger.info(
1005
- f"Removed job {job.id} from queue '{job.origin}' with TTL {ttl}"
1006
- )
1007
- else:
1008
- job.delete(**kwargs)
1009
- logger.info(f"Removed job {job.id} from queue '{job.origin}'")
1010
-
1011
- return True
1012
-
1013
- def cancel_all_jobs(self, queue_name: str | None = None) -> None:
1014
- """
1015
- Cancel all jobs in a queue.
1016
-
1017
- Args:
1018
- queue_name (str | None): Optional name of the queue to cancel jobs from.
1019
- If None, cancels jobs from all queues.
1020
- """
1021
- if queue_name is None:
1022
- queue_name = self._queue_names
1023
- elif isinstance(queue_name, str):
1024
- queue_name = [queue_name]
1025
-
1026
- for queue_name in queue_name:
1027
- if queue_name not in self._queue_names:
1028
- logger.warning(f"Queue '{queue_name}' not found, skipping")
1029
- continue
1030
-
1031
- for job in self.get_jobs(queue_name=queue_name):
1032
- self.cancel_job(job)
1033
-
1034
- def delete_all_jobs(self, queue_name: str | None = None, ttl: int = 0) -> None:
1035
- """
1036
- Remove all jobs from a queue.
1037
-
1038
- Args:
1039
- queue_name (str | None): Optional name of the queue to remove jobs from.
1040
- If None, removes jobs from all queues.
1041
- ttl: Optional time to live for the job (in seconds). 0 means no TTL.
1042
- Remove the job immediately.
1043
-
1044
- """
1045
- if queue_name is None:
1046
- queue_name = self._queue_names
1047
- elif isinstance(queue_name, str):
1048
- queue_name = [queue_name]
1049
-
1050
- for queue_name in queue_name:
1051
- if queue_name not in self._queue_names:
1052
- logger.warning(f"Queue '{queue_name}' not found, skipping")
1053
- continue
1054
-
1055
- for job in self.get_jobs(queue_name=queue_name):
1056
- self.delete_job(job, ttl=ttl)
1057
-
1058
- @property
1059
- def job_ids(self):
1060
- """Get all job IDs from all queues.
1061
-
1062
- Returns:
1063
- dict[str, list[str]]: Dictionary mapping queue names to lists of job IDs.
1064
-
1065
- Example:
1066
- ```python
1067
- all_ids = worker.job_ids
1068
- for queue_name, ids in all_ids.items():
1069
- print(f"Queue {queue_name}: {len(ids)} jobs")
1070
- ```
1071
- """
1072
- job_ids = {}
1073
- for queue_name in self._queue_names:
1074
- job_ids[queue_name] = self._queues[queue_name].job_ids
1075
-
1076
- return job_ids
1077
-
1078
- @property
1079
- def jobs(self):
1080
- """Get all jobs from all queues.
1081
-
1082
- Returns:
1083
- dict[str, list[Job]]: Dictionary mapping queue names to lists of jobs.
1084
-
1085
- Example:
1086
- ```python
1087
- all_jobs = worker.jobs
1088
- for queue_name, queue_jobs in all_jobs.items():
1089
- print(f"Queue {queue_name}: {len(queue_jobs)} jobs")
1090
- ```
1091
- """
1092
- jobs = {}
1093
- for queue_name in self._queue_names:
1094
- jobs[queue_name] = self._queues[queue_name].get_jobs()
1095
-
1096
- return jobs
1097
-
1098
- ### Schedules ###
1099
-
1100
- def add_schedule(
1101
- self,
1102
- func: Callable,
1103
- func_args: tuple | None = None,
1104
- func_kwargs: dict[str, Any] | None = None,
1105
- cron: str | None = None, # Cron expression for scheduling
1106
- interval: int | None = None, # Interval in seconds
1107
- date: dt.datetime | None = None, # Date to run the job
1108
- queue_name: str | None = None,
1109
- schedule_id: str | None = None,
1110
- ttl: int | dt.timedelta | None = None,
1111
- result_ttl: int | dt.timedelta | None = None,
1112
- repeat: int | None = None,
1113
- timeout: int | dt.timedelta | None = None,
1114
- meta: dict | None = None,
1115
- on_success: Callback | Callable | str | None = None,
1116
- on_failure: Callback | Callable | str | None = None,
1117
- on_stopped: Callback | Callable | str | None = None,
1118
- **schedule_kwargs,
1119
- ) -> Job:
1120
- """Schedule a job for repeated or one-time execution.
1121
-
1122
- Args:
1123
- func: Function to execute. Must be importable from the worker process.
1124
- func_args: Positional arguments to pass to the function.
1125
- func_kwargs: Keyword arguments to pass to the function.
1126
- cron: Cron expression for scheduling (e.g. "0 * * * *" for hourly).
1127
- interval: Interval in seconds for recurring execution.
1128
- date: Specific datetime for one-time execution.
1129
- queue_name: Name of the queue to use for the scheduled job.
1130
- schedule_id: Optional unique identifier for the schedule.
1131
- ttl: Time to live for the schedule, as seconds or timedelta.
1132
- result_ttl: Time to live for the job result, as seconds or timedelta.
1133
- repeat: Number of repetitions
1134
- timeout: Maximum time the job can run before being killed, as seconds or timedelta.
1135
- meta: Additional metadata to store with the schedule.
1136
- on_success: Callback to run on schedule success. Can be a function,
1137
- string, or RQ Callback instance.
1138
- on_failure: Callback to run on schedule failure. Can be a function,
1139
- string, or RQ Callback instance.
1140
- on_stopped: Callback to run when the schedule is stopped. Can be a function,
1141
- string, or RQ Callback instance.
1142
- **schedule_kwargs: Additional scheduling parameters:
1143
- - repeat: Number of repetitions (int or dict)
1144
- - result_ttl: Time to live for results (float or timedelta)
1145
- - ttl: Time to live for the schedule (float or timedelta)
1146
- - use_local_time_zone: Whether to use local time (bool)
1147
- - queue_name: Queue to use for the scheduled jobs
1148
-
1149
- Returns:
1150
- Job: The scheduled job instance.
1151
-
1152
- Raises:
1153
- ValueError: If no scheduling method specified or invalid cron expression.
1154
- RuntimeError: If Redis connection fails.
1155
-
1156
- Example:
1157
- ```python
1158
- def my_task(msg: str) -> None:
1159
- print(f"Task: {msg}")
1160
-
1161
- # Schedule with cron (every hour)
1162
- job = worker.add_schedule(
1163
- my_task,
1164
- func_kwargs={"msg": "Hourly check"},
1165
- cron="0 * * * *"
1166
- )
1167
-
1168
- # Schedule with interval (every 5 minutes)
1169
- job = worker.add_schedule(
1170
- my_task,
1171
- func_kwargs={"msg": "Regular check"},
1172
- interval=300
1173
- )
1174
-
1175
- # Schedule for specific time
1176
- tomorrow = dt.datetime.now() + dt.timedelta(days=1)
1177
- job = worker.add_schedule(
1178
- my_task,
1179
- func_kwargs={"msg": "One-time task"},
1180
- date=tomorrow
1181
- )
1182
- ```
1183
- """
1184
- schedule_id = schedule_id or str(uuid.uuid4())
1185
- func_args = func_args or ()
1186
- func_kwargs = func_kwargs or {}
1187
-
1188
- # Use the specified scheduler or default to the first one
1189
-
1190
- scheduler = self._scheduler
1191
-
1192
- use_local_time_zone = schedule_kwargs.get("use_local_time_zone", True)
1193
- # repeat = schedule_kwargs.get("repeat", None)
1194
- # result_ttl = schedule_kwargs.get("result_ttl", None)
1195
- # ttl = schedule_kwargs.get("ttl", None)
1196
- if isinstance(result_ttl, dt.timedelta):
1197
- result_ttl = result_ttl.total_seconds()
1198
- if isinstance(ttl, dt.timedelta):
1199
- ttl = ttl.total_seconds()
1200
- if isinstance(timeout, dt.timedelta):
1201
- timeout = timeout.total_seconds()
1202
- if isinstance(interval, dt.timedelta):
1203
- interval = interval.total_seconds()
1204
-
1205
- if isinstance(on_failure, (str, Callable)):
1206
- on_failure = Callback(on_failure)
1207
- if isinstance(on_success, (str, Callable)):
1208
- on_success = Callback(on_success)
1209
- if isinstance(on_stopped, (str, Callable)):
1210
- on_stopped = Callback(on_stopped)
1211
-
1212
- if cron:
1213
- if meta:
1214
- meta.update({"cron": cron})
1215
- else:
1216
- meta = {"cron": cron}
1217
- schedule = scheduler.cron(
1218
- cron_string=cron,
1219
- func=func,
1220
- args=func_args,
1221
- kwargs=func_kwargs,
1222
- id=schedule_id,
1223
- repeat=repeat, # Infinite by default
1224
- result_ttl=int(result_ttl) if result_ttl else None,
1225
- ttl=int(ttl) if ttl else None,
1226
- timeout=int(timeout) if timeout else None,
1227
- meta=meta,
1228
- use_local_time_zone=use_local_time_zone,
1229
- queue_name=queue_name or self._scheduler_name,
1230
- on_success=on_success,
1231
- on_failure=on_failure,
1232
- **schedule_kwargs,
1233
- )
1234
- logger.info(
1235
- f"Scheduled job {schedule.id} ({func.__name__}) with cron '{get_description(cron)}'"
1236
- )
1237
-
1238
- if interval:
1239
- if meta:
1240
- meta.update({"interval": int(interval)})
1241
- else:
1242
- meta = {"interval": int(interval)}
1243
- schedule = scheduler.schedule(
1244
- scheduled_time=dt.datetime.now(dt.timezone.utc),
1245
- func=func,
1246
- args=func_args,
1247
- kwargs=func_kwargs,
1248
- interval=int(interval),
1249
- id=schedule_id,
1250
- repeat=repeat, # Infinite by default
1251
- result_ttl=int(result_ttl) if result_ttl else None,
1252
- ttl=int(ttl) if ttl else None,
1253
- timeout=int(timeout) if timeout else None,
1254
- meta=meta,
1255
- queue_name=queue_name or self._scheduler_name,
1256
- on_success=on_success,
1257
- on_failure=on_failure,
1258
- **schedule_kwargs,
1259
- )
1260
- logger.info(
1261
- f"Scheduled job {schedule.id} ({func.__name__}) with interval '{precisedelta(interval)}'"
1262
- )
1263
-
1264
- if date:
1265
- if meta:
1266
- meta.update({"date": date})
1267
- else:
1268
- meta = {"date": date}
1269
- schedule = scheduler.schedule(
1270
- scheduled_time=date,
1271
- func=func,
1272
- args=func_args,
1273
- kwargs=func_kwargs,
1274
- id=schedule_id,
1275
- repeat=1, # Infinite by default
1276
- result_ttl=int(result_ttl) if result_ttl else None,
1277
- ttl=int(ttl) if ttl else None,
1278
- timeout=int(timeout) if timeout else None,
1279
- meta=meta,
1280
- queue_name=queue_name or self._scheduler_name,
1281
- on_success=on_success,
1282
- on_failure=on_failure,
1283
- on_stopped=on_stopped,
1284
- )
1285
- logger.info(
1286
- f"Scheduled job {schedule.id} ({func.__name__}) to run at '{date}'"
1287
- )
1288
-
1289
- return schedule
1290
-
1291
- def _get_schedule_queue_name(self, schedule: str | Job) -> str | None:
1292
- """Get the queue name for a schedule.
1293
-
1294
- Args:
1295
- schedule: Schedule ID or Job object.
1296
-
1297
- Returns:
1298
- str | None: Name of the scheduler queue.
1299
- """
1300
- return self._scheduler_name
1301
-
1302
- def get_schedules(
1303
- self,
1304
- until: Any | None = None,
1305
- with_times: bool = False,
1306
- offset: Any | None = None,
1307
- length: Any | None = None,
1308
- ) -> dict[str, list[Job]]:
1309
- """Get all schedules from the scheduler.
1310
-
1311
- Args:
1312
- until: Get schedules until this time.
1313
- with_times: Include next run times in the results.
1314
- offset: Number of schedules to skip.
1315
- length: Maximum number of schedules to return.
1316
-
1317
- Returns:
1318
- dict[str, list[Job]]: Dictionary mapping queue names to lists of schedules.
1319
-
1320
- Example:
1321
- ```python
1322
- # Get all schedules
1323
- schedules = worker.get_schedules()
1324
-
1325
- # Get next 10 schedules with run times
1326
- schedules = worker.get_schedules(
1327
- with_times=True,
1328
- length=10
1329
- )
1330
- ```
1331
- """
1332
- schedules = list(
1333
- self._scheduler.get_jobs(
1334
- until=until, with_times=with_times, offset=offset, length=length
1335
- )
1336
- )
1337
- if not schedules:
1338
- logger.info("No schedules found")
1339
- return []
1340
- return schedules
1341
-
1342
- def get_schedule(self, schedule_id: str) -> Job | None:
1343
- """
1344
- Get a schedule by its ID.
1345
-
1346
- Args:
1347
- schedule_id: ID of the schedule
1348
-
1349
- Returns:
1350
- Job | None: Schedule object if found, None otherwise
1351
- """
1352
- schedule = self.get_job(job_id=schedule_id)
1353
- return schedule
1354
-
1355
- def _get_schedule_results(self, schedule: str | Job) -> list[Result]:
1356
- """Get all results from a schedule's execution history.
1357
-
1358
- Args:
1359
- schedule: Schedule ID or Job object.
1360
-
1361
- Returns:
1362
- list[Result]: List of all results from the schedule's executions.
1363
-
1364
- Raises:
1365
- ValueError: If schedule not found.
1366
- """
1367
- if isinstance(schedule, str):
1368
- schedule = self.get_schedule(schedule_id=schedule)
1369
-
1370
- if schedule is None:
1371
- logger.error(f"Schedule {schedule} not found in any queue")
1372
- return None
1373
-
1374
- return [res.return_value for res in schedule.results()]
1375
-
1376
- def get_schedule_latest_result(
1377
- self, schedule: str | Job, delete_result: bool = False
1378
- ) -> Any:
1379
- """Get the most recent result of a schedule.
1380
-
1381
- Args:
1382
- schedule: Schedule ID or Job object.
1383
- delete_result: If True, deletes the schedule and results after retrieval.
1384
-
1385
- Returns:
1386
- Any: The most recent result of the schedule if available.
1387
-
1388
- Example:
1389
- ```python
1390
- # Get latest result
1391
- result = worker.get_schedule_latest_result("schedule-123")
1392
-
1393
- # Get result and clean up
1394
- result = worker.get_schedule_latest_result(
1395
- "schedule-123",
1396
- delete_result=True
1397
- )
1398
- ```
1399
- """
1400
- result = self._get_schedule_result(schedule=schedule)[-1]
1401
-
1402
- if delete_result:
1403
- self.delete_schedule(schedule)
1404
-
1405
- return result
1406
-
1407
- def get_schedule_result(
1408
- self, schedule: str | Job, index: int | list[str] | slice | str
1409
- ) -> list[Result]:
1410
- """Get specific results from a schedule's execution history.
1411
-
1412
- Args:
1413
- schedule: Schedule ID or Job object.
1414
- index: Which results to retrieve. Can be:
1415
- - int: Specific index
1416
- - list[str]: List of indices
1417
- - slice: Range of indices
1418
- - str: "all", "latest", or "earliest"
1419
-
1420
- Returns:
1421
- list[Result]: List of requested results.
1422
-
1423
- Example:
1424
- ```python
1425
- # Get all results
1426
- results = worker.get_schedule_result("schedule-123", "all")
1427
-
1428
- # Get latest result
1429
- result = worker.get_schedule_result("schedule-123", "latest")
1430
-
1431
- # Get specific results
1432
- results = worker.get_schedule_result("schedule-123", [0, 2, 4])
1433
-
1434
- # Get range of results
1435
- results = worker.get_schedule_result("schedule-123", slice(0, 5))
1436
- ```
1437
- """
1438
- results = self._get_schedule_results(schedule=schedule)
1439
- if not results:
1440
- return []
1441
-
1442
- if isinstance(index, str):
1443
- if ":" in index:
1444
- index = [int(i) for i in index.split(":")]
1445
- index = slice(index[0], index[1])
1446
- return [result for result in results[index]]
1447
-
1448
- if index == "all":
1449
- return [result for result in results]
1450
- if index == "latest":
1451
- return results[-1]
1452
- if index == "earliest":
1453
- return results[0]
1454
-
1455
- elif isinstance(index, list):
1456
- return [results[i].return_value for i in index if i < len(results)]
1457
- elif isinstance(index, slice):
1458
- return [result.return_value for result in results[index]]
1459
- elif isinstance(index, int):
1460
- if index >= len(results):
1461
- logger.error(f"Index {index} out of range for schedule {schedule.id}")
1462
- return []
1463
- return results[index].return_value
1464
-
1465
- def cancel_schedule(self, schedule: str | Job) -> bool:
1466
- """Cancel a schedule.
1467
-
1468
- This method stops any future executions of the schedule without removing
1469
- past results.
1470
-
1471
- Args:
1472
- schedule: Schedule ID or Job object to cancel.
1473
-
1474
- Returns:
1475
- bool: True if successfully canceled, False if schedule not found.
1476
-
1477
- Example:
1478
- ```python
1479
- # Cancel by ID
1480
- worker.cancel_schedule("schedule-123")
1481
-
1482
- # Cancel using job object
1483
- schedule = worker.get_schedule("schedule-123")
1484
- if schedule:
1485
- worker.cancel_schedule(schedule)
1486
- ```
1487
- """
1488
- if schedule is None:
1489
- logger.error(f"Schedule {schedule} not found")
1490
- return False
1491
-
1492
- self._scheduler.cancel(schedule)
1493
- logger.info(
1494
- f"Canceled schedule {schedule.id if isinstance(schedule, Job) else schedule}"
1495
- )
1496
- return True
1497
-
1498
- def cancel_all_schedules(self) -> None:
1499
- """Cancel all schedules in the scheduler.
1500
-
1501
- This method stops all future executions of all schedules without removing
1502
- past results.
1503
-
1504
- Example:
1505
- ```python
1506
- # Stop all scheduled jobs
1507
- worker.cancel_all_schedules()
1508
- ```
1509
- """
1510
- for job in self._scheduler.get_jobs():
1511
- self._scheduler.cancel(job.id)
1512
- logger.info(f"Canceled schedule {job.id} ")
1513
- logger.info("Canceled all schedules from all queues.")
1514
-
1515
- def delete_schedule(self, schedule: str | Job) -> bool:
1516
- """Delete a schedule and optionally its results.
1517
-
1518
- This method removes the schedule and optionally its execution history
1519
- from Redis.
1520
-
1521
- Args:
1522
- schedule: Schedule ID or Job object to delete.
1523
-
1524
- Returns:
1525
- bool: True if successfully deleted, False if schedule not found.
1526
-
1527
- Example:
1528
- ```python
1529
- # Delete schedule and its history
1530
- worker.delete_schedule("schedule-123")
1531
- ```
1532
- """
1533
- return self.delete_job(schedule)
1534
-
1535
- def delete_all_schedules(self) -> None:
1536
- """Delete all schedules and their results.
1537
-
1538
- This method removes all schedules and their execution histories from Redis.
1539
-
1540
- Example:
1541
- ```python
1542
- # Remove all schedules and their histories
1543
- worker.delete_all_schedules()
1544
- ```
1545
- """
1546
- for schedule in self.schedule_ids:
1547
- self.delete_schedule(schedule)
1548
- logger.info(f"Deleted schedule {schedule}")
1549
- logger.info("Deleted all schedules from all queues.")
1550
-
1551
- @property
1552
- def schedules(self):
1553
- """Get all schedules from all schedulers.
1554
-
1555
- Returns:
1556
- list[Job]: List of all scheduled jobs.
1557
-
1558
- Example:
1559
- ```python
1560
- all_schedules = worker.schedules
1561
- print(f"Total schedules: {len(all_schedules)}")
1562
- ```
1563
- """
1564
- schedules = self.get_schedules()
1565
-
1566
- return schedules
1567
-
1568
- @property
1569
- def schedule_ids(self):
1570
- """Get all schedule IDs.
1571
-
1572
- Returns:
1573
- list[str]: List of unique identifiers for all schedules.
1574
-
1575
- Example:
1576
- ```python
1577
- ids = worker.schedule_ids
1578
- print(f"Schedule IDs: {', '.join(ids)}")
1579
- ```
1580
- """
1581
- schedule_ids = [schedule.id for schedule in self.schedules]
1582
- return schedule_ids