FlowerPower 0.11.6.19__py3-none-any.whl → 0.20.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. flowerpower/cfg/__init__.py +3 -3
  2. flowerpower/cfg/pipeline/__init__.py +5 -3
  3. flowerpower/cfg/project/__init__.py +3 -3
  4. flowerpower/cfg/project/job_queue.py +1 -128
  5. flowerpower/cli/__init__.py +5 -5
  6. flowerpower/cli/cfg.py +0 -3
  7. flowerpower/cli/job_queue.py +401 -133
  8. flowerpower/cli/pipeline.py +14 -413
  9. flowerpower/cli/utils.py +0 -1
  10. flowerpower/flowerpower.py +537 -28
  11. flowerpower/job_queue/__init__.py +5 -94
  12. flowerpower/job_queue/base.py +201 -3
  13. flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -3
  14. flowerpower/job_queue/rq/manager.py +388 -77
  15. flowerpower/pipeline/__init__.py +2 -0
  16. flowerpower/pipeline/base.py +2 -2
  17. flowerpower/pipeline/io.py +14 -16
  18. flowerpower/pipeline/manager.py +21 -642
  19. flowerpower/pipeline/pipeline.py +571 -0
  20. flowerpower/pipeline/registry.py +242 -10
  21. flowerpower/pipeline/visualizer.py +1 -2
  22. flowerpower/plugins/_io/__init__.py +8 -0
  23. flowerpower/plugins/mqtt/manager.py +6 -6
  24. flowerpower/settings/backend.py +0 -2
  25. flowerpower/settings/job_queue.py +1 -57
  26. flowerpower/utils/misc.py +0 -256
  27. flowerpower/utils/monkey.py +1 -83
  28. {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/METADATA +308 -152
  29. flowerpower-0.20.0.dist-info/RECORD +58 -0
  30. flowerpower/fs/__init__.py +0 -29
  31. flowerpower/fs/base.py +0 -662
  32. flowerpower/fs/ext.py +0 -2143
  33. flowerpower/fs/storage_options.py +0 -1420
  34. flowerpower/job_queue/apscheduler/__init__.py +0 -11
  35. flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
  36. flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
  37. flowerpower/job_queue/apscheduler/manager.py +0 -1051
  38. flowerpower/job_queue/apscheduler/setup.py +0 -554
  39. flowerpower/job_queue/apscheduler/trigger.py +0 -169
  40. flowerpower/job_queue/apscheduler/utils.py +0 -311
  41. flowerpower/pipeline/job_queue.py +0 -583
  42. flowerpower/pipeline/runner.py +0 -603
  43. flowerpower/plugins/io/base.py +0 -2520
  44. flowerpower/plugins/io/helpers/datetime.py +0 -298
  45. flowerpower/plugins/io/helpers/polars.py +0 -875
  46. flowerpower/plugins/io/helpers/pyarrow.py +0 -570
  47. flowerpower/plugins/io/helpers/sql.py +0 -202
  48. flowerpower/plugins/io/loader/__init__.py +0 -28
  49. flowerpower/plugins/io/loader/csv.py +0 -37
  50. flowerpower/plugins/io/loader/deltatable.py +0 -190
  51. flowerpower/plugins/io/loader/duckdb.py +0 -19
  52. flowerpower/plugins/io/loader/json.py +0 -37
  53. flowerpower/plugins/io/loader/mqtt.py +0 -159
  54. flowerpower/plugins/io/loader/mssql.py +0 -26
  55. flowerpower/plugins/io/loader/mysql.py +0 -26
  56. flowerpower/plugins/io/loader/oracle.py +0 -26
  57. flowerpower/plugins/io/loader/parquet.py +0 -35
  58. flowerpower/plugins/io/loader/postgres.py +0 -26
  59. flowerpower/plugins/io/loader/pydala.py +0 -19
  60. flowerpower/plugins/io/loader/sqlite.py +0 -23
  61. flowerpower/plugins/io/metadata.py +0 -244
  62. flowerpower/plugins/io/saver/__init__.py +0 -28
  63. flowerpower/plugins/io/saver/csv.py +0 -36
  64. flowerpower/plugins/io/saver/deltatable.py +0 -186
  65. flowerpower/plugins/io/saver/duckdb.py +0 -19
  66. flowerpower/plugins/io/saver/json.py +0 -36
  67. flowerpower/plugins/io/saver/mqtt.py +0 -28
  68. flowerpower/plugins/io/saver/mssql.py +0 -26
  69. flowerpower/plugins/io/saver/mysql.py +0 -26
  70. flowerpower/plugins/io/saver/oracle.py +0 -26
  71. flowerpower/plugins/io/saver/parquet.py +0 -36
  72. flowerpower/plugins/io/saver/postgres.py +0 -26
  73. flowerpower/plugins/io/saver/pydala.py +0 -20
  74. flowerpower/plugins/io/saver/sqlite.py +0 -24
  75. flowerpower/utils/scheduler.py +0 -311
  76. flowerpower-0.11.6.19.dist-info/RECORD +0 -102
  77. {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/WHEEL +0 -0
  78. {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/entry_points.txt +0 -0
  79. {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/licenses/LICENSE +0 -0
  80. {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/top_level.txt +0 -0
@@ -1,1051 +0,0 @@
1
- """
2
- APScheduler implementation for FlowerPower scheduler.
3
-
4
- This module implements the scheduler interfaces using APScheduler as the backend.
5
- """
6
-
7
- import datetime as dt
8
- import importlib.util
9
- from typing import Any, Callable
10
- from uuid import UUID
11
-
12
- import duration_parser
13
- from apscheduler import Job, Scheduler
14
- from apscheduler.executors.async_ import AsyncJobExecutor
15
- from apscheduler.executors.subprocess import ProcessPoolJobExecutor
16
- from apscheduler.executors.thread import ThreadPoolJobExecutor
17
- from fsspec.spec import AbstractFileSystem
18
- from loguru import logger
19
-
20
- from ...utils.logging import setup_logging
21
- from ..base import BaseJobQueueManager
22
- from .setup import APSBackend, APSDataStore, APSEventBroker
23
- from .trigger import APSTrigger
24
- from .utils import display_jobs, display_schedules
25
-
26
- # Check if APScheduler is available
27
- # if not importlib.util.find_spec("apscheduler"):
28
- # raise ImportError(
29
- # "APScheduler is not installed. Please install it using `pip install "
30
- # "'apscheduler>4.0.0a1'`, 'conda install apscheduler4' or `pip install flowerpower[apscheduler]`"
31
- # )
32
-
33
-
34
- setup_logging()
35
-
36
- # Patch pickle if needed
37
- try:
38
- from ...utils.monkey import patch_pickle
39
-
40
- patch_pickle()
41
- except Exception as e:
42
- logger.warning(f"Failed to patch pickle: {e}")
43
-
44
-
45
- class APSManager(BaseJobQueueManager):
46
- """Implementation of BaseScheduler using APScheduler.
47
-
48
- This worker class uses APScheduler 4.0+ as the backend to schedule and manage jobs.
49
- It supports different job executors including async, thread pool, and process pool.
50
-
51
- Typical usage:
52
- ```python
53
- worker = APSManager(name="my_scheduler")
54
- worker.start_worker(background=True)
55
-
56
- # Add a job
57
- def my_job(x: int) -> int:
58
- return x * 2
59
-
60
- job_id = worker.add_job(my_job, func_args=(10,))
61
- ```
62
- """
63
-
64
- def __init__(
65
- self,
66
- name: str | None = "flowerpower_apscheduler",
67
- base_dir: str | None = None,
68
- backend: APSBackend | dict | None = None,
69
- storage_options: dict[str, Any] = None,
70
- fs: AbstractFileSystem | None = None,
71
- log_level: str | None = None,
72
- ):
73
- """Initialize the APScheduler backend.
74
-
75
- Args:
76
- name: Name of the scheduler instance. Used for identification in logs and data stores.
77
- base_dir: Base directory for the FlowerPower project. Used for finding configuration files.
78
- backend: APSBackend instance with data store and event broker configurations,
79
- or a dictionary with configuration parameters.
80
- storage_options: Options for configuring file system storage access.
81
- Example: {"mode": "async", "root": "/tmp"}
82
- fs: Custom filesystem implementation for storage operations.
83
- log_level: Logging level to use for this worker instance.
84
- Example: "DEBUG", "INFO", "WARNING", etc.
85
-
86
- Raises:
87
- RuntimeError: If backend setup fails due to missing or invalid configurations.
88
- ImportError: If required dependencies are not installed.
89
-
90
- Example:
91
- ```python
92
- # Basic initialization
93
- worker = APSManager(name="my_scheduler")
94
-
95
- # With custom backend and logging
96
-
97
- # Create a custom backend configuration using dictionaries for data store and event broker
98
- backend_config = {
99
- "data_store": {"type": "postgresql", "uri": "postgresql+asyncpg://user:pass@localhost/db"},
100
- "event_broker": {"type": "redis", "uri": "redis://localhost:6379/0"}
101
- }
102
-
103
- # Create a custom backend configuration using APSBackend, APSDataStore, and APSEventBroker classes
104
- from flowerpower.worker.aps import APSBackend, APSDataStore, APSEventBroker
105
- data_store = APSDataStore(
106
- type="postgresql",
107
- uri="postgresql+asyncpg://user:pass@localhost/db"
108
- )
109
- event_broker = APSEventBroker(
110
- from_ds_sqla=True
111
- )
112
- backend_config = APSBackend(
113
- data_store=data_store,
114
- event_broker=event_broker
115
- )
116
-
117
- worker = APSManager(
118
- name="custom_scheduler",
119
- backend=backend_config,
120
- log_level="DEBUG"
121
- )
122
- ```
123
- """
124
- if log_level:
125
- setup_logging(level=log_level)
126
-
127
- super().__init__(
128
- type="apscheduler",
129
- name=name,
130
- base_dir=base_dir,
131
- fs=fs,
132
- backend=backend,
133
- storage_options=storage_options,
134
- )
135
-
136
- if not isinstance(backend, APSBackend):
137
- self._setup_backend(backend)
138
- else:
139
- self._backend = backend
140
-
141
- # Set up job executors
142
- self._job_executors = {
143
- "async": AsyncJobExecutor(),
144
- "threadpool": ThreadPoolJobExecutor(),
145
- "processpool": ProcessPoolJobExecutor(),
146
- }
147
- self._worker = Scheduler(
148
- job_executors=self._job_executors,
149
- event_broker=self._backend.event_broker._client,
150
- data_store=self._backend.data_store._client,
151
- identity=self.name,
152
- logger=logger,
153
- cleanup_interval=self._backend.cleanup_interval,
154
- max_concurrent_jobs=self._backend.max_concurrent_jobs,
155
- default_job_executor=self._backend.default_job_executor,
156
- )
157
-
158
- def _setup_backend(self, backend: dict | None) -> None:
159
- """
160
- Set up the data store and SQLAlchemy engine for the scheduler.
161
-
162
- This method initializes the data store and SQLAlchemy engine using configuration
163
- values. It validates configuration, handles errors, and logs the setup process.
164
-
165
- Raises:
166
- RuntimeError: If the data store setup fails due to misconfiguration or connection errors.
167
- """
168
- if backend is None:
169
- self._backend = APSBackend(**self.cfg.backend.to_dict())
170
- elif isinstance(backend, dict):
171
- backend_cfg = self.cfg.backend.to_dict()
172
- backend_cfg.update(backend)
173
- self._backend = APSBackend(**backend_cfg)
174
-
175
- if (
176
- self._backend.data_store._client is not None
177
- and self._backend.event_broker._client is not None
178
- ):
179
- logger.info(
180
- f"Data store and event broker set up successfully: data store type"
181
- f" '{self._backend.data_store.type}', event broker type '{self._backend.event_broker.type}'"
182
- )
183
-
184
- def start_worker(
185
- self, background: bool = False, num_workers: int | None = None
186
- ) -> None:
187
- """Start the APScheduler worker process.
188
-
189
- This method initializes and starts the worker process that executes scheduled jobs.
190
- The worker can be started in foreground (blocking) or background mode.
191
-
192
- Args:
193
- background: If True, runs the worker in a non-blocking background mode.
194
- If False, runs in the current process and blocks until stopped.
195
- num_workers: Number of worker processes for the executor pools.
196
- If None, uses the value from config or defaults to CPU count.
197
-
198
- Raises:
199
- RuntimeError: If worker fails to start or if multiprocessing setup fails.
200
-
201
- Example:
202
- ```python
203
- # Start worker in background with 4 processes
204
- worker.start_worker(background=True, num_workers=4)
205
-
206
- # Start worker in foreground (blocking)
207
- worker.start_worker(background=False)
208
-
209
- # Use as a context manager
210
- with worker.start_worker(background=False):
211
- # Do some work
212
- pass
213
- ```
214
- """
215
- import multiprocessing
216
-
217
- # Allow configuration override for pool sizes
218
- if num_workers is None:
219
- num_workers = self.cfg.num_workers or multiprocessing.cpu_count()
220
-
221
- # Adjust thread and process pool executor sizes
222
- if "processpool" in self._job_executors:
223
- self._job_executors["processpool"].max_workers = num_workers
224
- if "threadpool" in self._job_executors:
225
- threadpool_size = getattr(
226
- self.cfg.backend, "threadpool_size", num_workers * 5
227
- )
228
- self._job_executors["threadpool"].max_workers = threadpool_size
229
-
230
- logger.info(f"Configured worker pool with {num_workers} workers.")
231
-
232
- if background:
233
- logger.info("Starting APScheduler worker in background mode.")
234
- self._worker.start_in_background()
235
- else:
236
- logger.info("Starting APScheduler worker in foreground mode.")
237
- self._worker.run_until_stopped()
238
-
239
- def stop_worker(self) -> None:
240
- """Stop the APScheduler worker process.
241
-
242
- This method stops the worker process and cleans up resources.
243
- It should be called before program exit to ensure proper cleanup.
244
-
245
- Raises:
246
- RuntimeError: If worker fails to stop cleanly.
247
-
248
- Example:
249
- ```python
250
- try:
251
- worker.start_worker(background=True)
252
- # ... do work ...
253
- finally:
254
- worker.stop_worker()
255
- ```
256
- """
257
- logger.info("Stopping APScheduler worker.")
258
- self._worker.stop()
259
- self._worker._exit_stack.close()
260
-
261
- def start_worker_pool(
262
- self,
263
- background: bool = False,
264
- num_workers: int | None = None,
265
- ) -> None:
266
- """
267
- Start a pool of worker processes to handle jobs in parallel.
268
-
269
- APScheduler 4.0 already handles concurrency internally through its executors,
270
- so this method simply starts a single worker with the appropriate configuration.
271
-
272
- Args:
273
- num_workers: Number of worker processes (affects executor pool sizes)
274
- background: Whether to run in background
275
- """
276
-
277
- # Start a single worker which will use the configured executors
278
- self.start_worker(background=background, num_workers=num_workers)
279
-
280
- def stop_worker_pool(self) -> None:
281
- """
282
- Stop the worker pool.
283
-
284
- Since APScheduler manages concurrency internally, this just stops the worker.
285
- """
286
-
287
- logger.info("Stopping APScheduler worker pool.")
288
- self.stop_worker()
289
-
290
- ## Jobs
291
-
292
- def add_job(
293
- self,
294
- func: Callable,
295
- func_args: tuple | None = None,
296
- func_kwargs: dict[str, Any] | None = None,
297
- result_ttl: float | dt.timedelta = 0,
298
- run_at: dt.datetime | None = None,
299
- run_in: int | float | None = None,
300
- job_executor: str | None = None,
301
- ) -> str:
302
- """Add a job for immediate or scheduled execution.
303
-
304
- This method adds a job to the scheduler. The job can be executed immediately
305
- or scheduled for later execution using run_at or run_in parameters.
306
-
307
- Args:
308
- func: Function to execute. Must be importable from the worker process.
309
- func_args: Positional arguments to pass to the function.
310
- func_kwargs: Keyword arguments to pass to the function.
311
- result_ttl: Time to live for the job result, as seconds or timedelta.
312
- After this time, the result may be removed from storage.
313
- run_at: Schedule the job to run at a specific datetime.
314
- Takes precedence over run_in if both are specified.
315
- run_in: Schedule the job to run after a delay (in seconds).
316
- Only used if run_at is not specified.
317
- job_executor: Name of the executor to run the job ("async", "threadpool",
318
- or "processpool"). If None, uses the default from config.
319
-
320
- Returns:
321
- str: Unique identifier for the job.
322
-
323
- Raises:
324
- ValueError: If the function is not serializable or arguments are invalid.
325
- RuntimeError: If the job cannot be added to the scheduler.
326
-
327
- Note:
328
- When using run_at or run_in, the job results will not be stored in the data store.
329
-
330
- Example:
331
- ```python
332
- # Add immediate job
333
- def my_task(x: int, y: int) -> int:
334
- return x + y
335
-
336
- job_id = worker.add_job(
337
- my_task,
338
- func_args=(1, 2),
339
- result_ttl=3600 # Keep result for 1 hour
340
- )
341
-
342
- # Schedule job for later
343
- tomorrow = dt.datetime.now() + dt.timedelta(days=1)
344
- job_id = worker.add_job(
345
- my_task,
346
- func_kwargs={"x": 1, "y": 2},
347
- run_at=tomorrow
348
- )
349
-
350
- # Run after delay
351
- job_id = worker.add_job(
352
- my_task,
353
- func_args=(1, 2),
354
- run_in=3600 # Run in 1 hour
355
- )
356
- ```
357
- """
358
- job_executor = job_executor or self.cfg.backend.default_job_executor
359
-
360
- # Convert result_expiration_time to datetime.timedelta if it's not already
361
- if isinstance(result_ttl, (int, float)):
362
- result_ttl = dt.timedelta(seconds=result_ttl)
363
-
364
- run_at = (
365
- dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
366
- )
367
- run_in = duration_parser.parse(run_in) if isinstance(run_in, str) else run_in
368
-
369
- if run_in:
370
- run_at = dt.datetime.now() + dt.timedelta(seconds=run_in)
371
-
372
- if run_at:
373
- job_id = self.add_schedule(
374
- func,
375
- func_args=func_args,
376
- func_kwargs=func_kwargs,
377
- date=run_at,
378
- job_executor=job_executor,
379
- )
380
- else:
381
- job_id = self._worker.add_job(
382
- func,
383
- args=func_args or (),
384
- kwargs=func_kwargs or {},
385
- job_executor=job_executor,
386
- result_expiration_time=result_ttl,
387
- )
388
-
389
- return str(job_id)
390
-
391
- def run_job(
392
- self,
393
- func: Callable,
394
- func_args: tuple | None = None,
395
- func_kwargs: dict[str, Any] | None = None,
396
- job_executor: str | None = None,
397
- ) -> Any:
398
- """Run a job immediately and wait for its result.
399
-
400
- This method executes the job synchronously and returns its result.
401
-
402
- Args:
403
- func: Function to execute. Must be importable from the worker process.
404
- func_args: Positional arguments to pass to the function.
405
- func_kwargs: Keyword arguments to pass to the function.
406
- job_executor: Name of the executor to run the job ("async", "threadpool",
407
- or "processpool"). If None, uses the default from config.
408
-
409
- Returns:
410
- Any: The result returned by the executed function.
411
-
412
- Raises:
413
- Exception: Any exception raised by the executed function.
414
-
415
- Example:
416
- ```python
417
- def add(x: int, y: int) -> int:
418
- return x + y
419
-
420
- result = worker.run_job(add, func_args=(1, 2))
421
- assert result == 3
422
- ```
423
- """
424
- job_executor = job_executor or self.cfg.backend.default_job_executor
425
-
426
- return self._worker.run_job(
427
- func,
428
- args=func_args or (),
429
- kwargs=func_kwargs or {},
430
- )
431
-
432
- def get_jobs(self) -> list[Job]:
433
- """Get all jobs from the scheduler.
434
-
435
- Returns:
436
- list[Job]: List of all jobs in the scheduler, including pending,
437
- running, and completed jobs.
438
-
439
- Example:
440
- ```python
441
- jobs = worker.get_jobs()
442
- for job in jobs:
443
- print(f"Job {job.id}: {job.status}")
444
- ```
445
- """
446
- return self._worker.get_jobs()
447
-
448
- def get_job(self, job_id: str | UUID) -> Job | None:
449
- """Get a specific job by its ID.
450
-
451
- Args:
452
- job_id: Unique identifier of the job, as string or UUID.
453
-
454
- Returns:
455
- Job | None: The job object if found, None otherwise.
456
-
457
- Example:
458
- ```python
459
- # Get job using string ID
460
- job = worker.get_job("550e8400-e29b-41d4-a716-446655440000")
461
-
462
- # Get job using UUID
463
- from uuid import UUID
464
- job = worker.get_job(UUID("550e8400-e29b-41d4-a716-446655440000"))
465
- ```
466
- """
467
- jobs = self._worker.get_jobs()
468
- if isinstance(job_id, str):
469
- job_id = UUID(job_id)
470
-
471
- for job in jobs:
472
- if job.id == job_id:
473
- return job
474
- return None
475
-
476
- def get_job_result(self, job_id: str | UUID, wait: bool = True) -> Any:
477
- """Get the result of a specific job.
478
-
479
- Args:
480
- job_id: Unique identifier of the job, as string or UUID.
481
- wait: If True, waits for the job to complete before returning.
482
- If False, returns None if the job is not finished.
483
-
484
- Returns:
485
- Any: The result of the job if available, None if the job is not
486
- finished and wait=False.
487
-
488
- Raises:
489
- ValueError: If the job ID is invalid.
490
- TimeoutError: If the job takes too long to complete (when waiting).
491
-
492
- Example:
493
- ```python
494
- # Wait for result
495
- result = worker.get_job_result("550e8400-e29b-41d4-a716-446655440000")
496
-
497
- # Check result without waiting
498
- result = worker.get_job_result(
499
- "550e8400-e29b-41d4-a716-446655440000",
500
- wait=False
501
- )
502
- if result is None:
503
- print("Job still running")
504
- ```
505
- """
506
- if isinstance(job_id, str):
507
- job_id = UUID(job_id)
508
- return self._worker.get_job_result(job_id, wait=wait)
509
-
510
- def cancel_job(self, job_id: str | UUID) -> bool:
511
- """Cancel a running or pending job.
512
-
513
- Note:
514
- Not currently implemented for APScheduler backend. Jobs must be removed
515
- manually from the data store.
516
-
517
- Args:
518
- job_id: Unique identifier of the job to cancel, as string or UUID.
519
-
520
- Returns:
521
- bool: Always returns False as this operation is not implemented.
522
-
523
- Example:
524
- ```python
525
- # This operation is not supported
526
- success = worker.cancel_job("job-123")
527
- assert not success
528
- ```
529
- """
530
- logger.info(
531
- "Not implemented for apscheduler yet. You have to remove the job manually from the data_store."
532
- )
533
- return False
534
-
535
- def delete_job(self, job_id: str | UUID) -> bool:
536
- """
537
- Delete a job and its results from storage.
538
-
539
- Note:
540
- Not currently implemented for APScheduler backend. Jobs must be removed
541
- manually from the data store.
542
-
543
- Args:
544
- job_id: Unique identifier of the job to delete, as string or UUID.
545
-
546
- Returns:
547
- bool: Always returns False as this operation is not implemented.
548
-
549
- Example:
550
- ```python
551
- # This operation is not supported
552
- success = worker.delete_job("job-123")
553
- assert not success
554
- ```
555
- """
556
- logger.info(
557
- "Not implemented for apscheduler yet. You have to remove the job manually from the data_store."
558
- )
559
- return False
560
-
561
- def cancel_all_jobs(self) -> None:
562
- """Cancel all running and pending jobs.
563
-
564
- Note:
565
- Not currently implemented for APScheduler backend. Jobs must be removed
566
- manually from the data store.
567
-
568
- Example:
569
- ```python
570
- # This operation is not supported
571
- worker.cancel_all_jobs() # No effect
572
- ```
573
- """
574
- logger.info(
575
- "Not implemented for apscheduler yet. You have to remove the jobs manually from the data_store."
576
- )
577
- return None
578
-
579
- def delete_all_jobs(self) -> None:
580
- """
581
- Delete all jobs and their results from storage.
582
-
583
- Note:
584
- Not currently implemented for APScheduler backend. Jobs must be removed
585
- manually from the data store.
586
-
587
- Example:
588
- ```python
589
- # This operation is not supported
590
- worker.delete_all_jobs() # No effect
591
- ```
592
- """
593
- logger.info(
594
- "Not implemented for apscheduler yet. You have to remove the jobs manually from the data_store."
595
- )
596
- return None
597
-
598
- @property
599
- def jobs(self) -> list[Job]:
600
- """Get all jobs from the scheduler.
601
-
602
- Returns:
603
- list[Job]: List of all job objects in the scheduler.
604
-
605
- Example:
606
- ```python
607
- all_jobs = worker.jobs
608
- print(f"Total jobs: {len(all_jobs)}")
609
- for job in all_jobs:
610
- print(f"Job {job.id}: {job.status}")
611
- ```
612
- """
613
- return self._worker.get_jobs()
614
-
615
- @property
616
- def job_ids(self) -> list[str]:
617
- """Get all job IDs from the scheduler.
618
-
619
- Returns:
620
- list[str]: List of unique identifiers for all jobs.
621
-
622
- Example:
623
- ```python
624
- ids = worker.job_ids
625
- print(f"Job IDs: {', '.join(ids)}")
626
- ```
627
- """
628
- return [str(job.id) for job in self._worker.get_jobs()]
629
-
630
- ## Schedules
631
- def add_schedule(
632
- self,
633
- func: Callable,
634
- func_args: tuple | None = None,
635
- func_kwargs: dict[str, Any] | None = None,
636
- cron: str | dict[str, str | int] | None = None,
637
- interval: int | str | dict[str, str | int] | None = None,
638
- date: dt.datetime | None = None,
639
- schedule_id: str | None = None,
640
- job_executor: str | None = None,
641
- **schedule_kwargs,
642
- ) -> str:
643
- """Schedule a job for repeated or one-time execution.
644
-
645
- This method adds a scheduled job to the scheduler. The schedule can be defined
646
- using cron expressions, intervals, or specific dates.
647
-
648
- Args:
649
- func: Function to execute. Must be importable from the worker process.
650
- func_args: Positional arguments to pass to the function.
651
- func_kwargs: Keyword arguments to pass to the function.
652
- cron: Cron expression for scheduling. Can be a string (e.g. "* * * * *")
653
- or a dict with cron parameters. Only one of cron, interval, or date
654
- should be specified.
655
- interval: Interval for recurring execution in seconds, or a dict with
656
- interval parameters. Only one of cron, interval, or date should
657
- be specified.
658
- date: Specific datetime for one-time execution. Only one of cron,
659
- interval, or date should be specified.
660
- schedule_id: Optional unique identifier for the schedule.
661
- If None, a UUID will be generated.
662
- job_executor: Name of the executor to run the job ("async", "threadpool",
663
- or "processpool"). If None, uses the default from config.
664
- **schedule_kwargs: Additional scheduling parameters:
665
- - coalesce: CoalescePolicy = CoalescePolicy.latest
666
- - misfire_grace_time: float | timedelta | None = None
667
- - max_jitter: float | timedelta | None = None
668
- - max_running_jobs: int | None = None
669
- - conflict_policy: ConflictPolicy = ConflictPolicy.do_nothing
670
- - paused: bool = False
671
-
672
- Returns:
673
- str: Unique identifier for the schedule.
674
-
675
- Raises:
676
- ValueError: If no trigger type is specified or if multiple triggers
677
- are specified.
678
- RuntimeError: If the schedule cannot be added to the scheduler.
679
-
680
- Example:
681
- ```python
682
- def my_task(msg: str) -> None:
683
- print(f"Running task: {msg}")
684
-
685
- # Using cron expression (run every minute)
686
- schedule_id = worker.add_schedule(
687
- my_task,
688
- func_kwargs={"msg": "Cron job"},
689
- cron="* * * * *"
690
- )
691
-
692
- # Using cron dict
693
- schedule_id = worker.add_schedule(
694
- my_task,
695
- func_kwargs={"msg": "Cron job"},
696
- cron={
697
- "minute": "*/15", # Every 15 minutes
698
- "hour": "9-17" # During business hours
699
- }
700
- )
701
-
702
- # Using interval (every 5 minutes)
703
- schedule_id = worker.add_schedule(
704
- my_task,
705
- func_kwargs={"msg": "Interval job"},
706
- interval=300 # 5 minutes in seconds
707
- )
708
-
709
- # Using interval dict
710
- schedule_id = worker.add_schedule(
711
- my_task,
712
- func_kwargs={"msg": "Interval job"},
713
- interval={
714
- "hours": 1,
715
- "minutes": 30
716
- }
717
- )
718
-
719
- # One-time future execution
720
- import datetime as dt
721
- future_date = dt.datetime.now() + dt.timedelta(days=1)
722
- schedule_id = worker.add_schedule(
723
- my_task,
724
- func_kwargs={"msg": "One-time job"},
725
- date=future_date
726
- )
727
-
728
- # With additional options
729
- from apscheduler import CoalescePolicy
730
- schedule_id = worker.add_schedule(
731
- my_task,
732
- func_kwargs={"msg": "Advanced job"},
733
- interval=300,
734
- coalesce=CoalescePolicy.latest,
735
- max_jitter=dt.timedelta(seconds=30)
736
- )
737
- ```
738
- """
739
- job_executor = job_executor or self.cfg.backend.default_job_executor
740
-
741
- if cron:
742
- trigger_instance = APSTrigger("cron")
743
- if isinstance(cron, str):
744
- cron = {"crontab": cron}
745
- trigger = trigger_instance.get_trigger_instance(**cron)
746
- elif interval:
747
- trigger_instance = APSTrigger("interval")
748
- if isinstance(interval, str | int):
749
- interval = {"seconds": int(interval)}
750
- trigger = trigger_instance.get_trigger_instance(**interval)
751
-
752
- if date:
753
- trigger_instance = APSTrigger("date")
754
- trigger = trigger_instance.get_trigger_instance(run_time=date)
755
-
756
- schedule_id = self._worker.add_schedule(
757
- func,
758
- trigger=trigger,
759
- id=schedule_id,
760
- args=func_args or (),
761
- kwargs=func_kwargs or {},
762
- job_executor=job_executor,
763
- **schedule_kwargs,
764
- )
765
-
766
- return schedule_id
767
-
768
- def get_schedules(self, as_dict: bool = False) -> list[Any]:
769
- """Get all schedules from the scheduler.
770
-
771
- Args:
772
- as_dict: If True, returns schedules as dictionaries instead of
773
- Schedule objects.
774
-
775
- Returns:
776
- list[Any]: List of all schedules, either as Schedule objects or
777
- dictionaries depending on as_dict parameter.
778
-
779
- Example:
780
- ```python
781
- # Get schedule objects
782
- schedules = worker.get_schedules()
783
- for schedule in schedules:
784
- print(f"Schedule {schedule.id}: Next run at {schedule.next_run_time}")
785
-
786
- # Get as dictionaries
787
- schedules = worker.get_schedules(as_dict=True)
788
- for schedule in schedules:
789
- print(f"Schedule {schedule['id']}: {schedule['trigger']}")
790
- ```
791
- """
792
- return self._worker.get_schedules()
793
-
794
- def get_schedule(self, schedule_id: str) -> Any:
795
- """Get a specific schedule by its ID.
796
-
797
- Args:
798
- schedule_id: Unique identifier of the schedule.
799
-
800
- Returns:
801
- Any: The schedule object if found, None otherwise.
802
-
803
- Example:
804
- ```python
805
- schedule = worker.get_schedule("my-daily-job")
806
- if schedule:
807
- print(f"Next run at: {schedule.next_run_time}")
808
- else:
809
- print("Schedule not found")
810
- ```
811
- """
812
- if schedule_id in self.schedule_ids:
813
- return self._worker.get_schedule(schedule_id)
814
-
815
- logger.error(f"Schedule {schedule_id} not found.")
816
- return None
817
-
818
- def cancel_schedule(self, schedule_id: str) -> bool:
819
- """Cancel a schedule.
820
-
821
- This method removes the schedule from the scheduler. This is equivalent
822
- to delete_schedule and stops any future executions of the schedule.
823
-
824
- Args:
825
- schedule_id: Unique identifier of the schedule to cancel.
826
-
827
- Returns:
828
- bool: True if the schedule was successfully canceled,
829
- False if the schedule was not found.
830
-
831
- Example:
832
- ```python
833
- if worker.cancel_schedule("my-daily-job"):
834
- print("Schedule canceled successfully")
835
- else:
836
- print("Schedule not found")
837
- ```
838
- """
839
- if schedule_id not in self.schedule_ids:
840
- logger.error(f"Schedule {schedule_id} not found.")
841
- return False
842
- self._worker.remove_schedule(schedule_id)
843
- logger.info(f"Schedule {schedule_id} canceled.")
844
-
845
- def delete_schedule(self, schedule_id: str) -> bool:
846
- """Remove a schedule.
847
-
848
- This method removes the schedule from the scheduler. This is equivalent
849
- to cancel_schedule and stops any future executions of the schedule.
850
-
851
- Args:
852
- schedule_id: Unique identifier of the schedule to remove.
853
-
854
- Returns:
855
- bool: True if the schedule was successfully removed,
856
- False if the schedule was not found.
857
-
858
- Raises:
859
- RuntimeError: If removal fails due to data store errors.
860
-
861
- Example:
862
- ```python
863
- try:
864
- if worker.delete_schedule("my-daily-job"):
865
- print("Schedule deleted successfully")
866
- else:
867
- print("Schedule not found")
868
- except RuntimeError as e:
869
- print(f"Failed to delete schedule: {e}")
870
- ```
871
- """
872
- self.cancel_schedule(schedule_id)
873
-
874
- def cancel_all_schedules(self) -> None:
875
- """Cancel all schedules in the scheduler.
876
-
877
- This method removes all schedules from the scheduler, stopping all future
878
- executions. This operation cannot be undone.
879
-
880
- Example:
881
- ```python
882
- # Cancel all schedules
883
- worker.cancel_all_schedules()
884
- assert len(worker.schedules) == 0
885
- ```
886
- """
887
- for sched in self.schedule_ids:
888
- self.cancel_schedule(sched)
889
- logger.info("All schedules canceled.")
890
- return None
891
-
892
- def delete_all_schedules(self) -> None:
893
- """
894
- Delete all schedules from the scheduler.
895
-
896
- This method removes all schedules from the scheduler, stopping all future
897
- executions. This operation cannot be undone.
898
-
899
- Example:
900
- ```python
901
- # Delete all schedules
902
- worker.delete_all_schedules()
903
- assert len(worker.schedules) == 0
904
- ```
905
- """
906
- for sched in self.schedule_ids:
907
- self.delete_schedule(sched)
908
- logger.info("All schedules deleted.")
909
- return None
910
-
911
- @property
912
- def schedules(self) -> list[Any]:
913
- """Get all schedules from the scheduler.
914
-
915
- Returns:
916
- list[Any]: List of all schedule objects in the scheduler.
917
-
918
- Example:
919
- ```python
920
- schedules = worker.schedules
921
- print(f"Total schedules: {len(schedules)}")
922
- ```
923
- """
924
- return self._worker.get_schedules()
925
-
926
- @property
927
- def schedule_ids(self) -> list[str]:
928
- """Get all schedule IDs from the scheduler.
929
-
930
- Returns:
931
- list[str]: List of unique identifiers for all schedules.
932
-
933
- Example:
934
- ```python
935
- ids = worker.schedule_ids
936
- print(f"Schedule IDs: {', '.join(ids)}")
937
- ```
938
- """
939
- return [str(sched.id) for sched in self._worker.get_schedules()]
940
-
941
- def pause_schedule(self, schedule_id: str) -> bool:
942
- """Pause a schedule temporarily.
943
-
944
- This method pauses the schedule without removing it. The schedule can be
945
- resumed later using resume_schedule.
946
-
947
- Args:
948
- schedule_id: Unique identifier of the schedule to pause.
949
-
950
- Returns:
951
- bool: True if the schedule was successfully paused,
952
- False if the schedule was not found.
953
-
954
- Example:
955
- ```python
956
- # Pause a schedule temporarily
957
- if worker.pause_schedule("daily-backup"):
958
- print("Schedule paused")
959
- ```
960
- """
961
- if schedule_id not in self.schedule_ids:
962
- logger.error(f"Schedule {schedule_id} not found.")
963
- return False
964
- self._worker.pause_schedule(schedule_id)
965
- logger.info(f"Schedule {schedule_id} paused.")
966
- return True
967
-
968
- def resume_schedule(self, schedule_id: str) -> bool:
969
- """Resume a paused schedule.
970
-
971
- Args:
972
- schedule_id: Unique identifier of the schedule to resume.
973
-
974
- Returns:
975
- bool: True if the schedule was successfully resumed,
976
- False if the schedule was not found.
977
-
978
- Example:
979
- ```python
980
- # Resume a paused schedule
981
- if worker.resume_schedule("daily-backup"):
982
- print("Schedule resumed")
983
- ```
984
- """
985
- if schedule_id not in self.schedule_ids:
986
- logger.error(f"Schedule {schedule_id} not found.")
987
- return False
988
- self._worker.unpause_schedule(schedule_id)
989
- logger.info(f"Schedule {schedule_id} resumed.")
990
- return True
991
-
992
- def pause_all_schedules(self) -> None:
993
- """Pause all schedules in the scheduler.
994
-
995
- This method pauses all schedules without removing them. They can be
996
- resumed using resume_all_schedules.
997
-
998
- Example:
999
- ```python
1000
- # Pause all schedules temporarily
1001
- worker.pause_all_schedules()
1002
- ```
1003
- """
1004
- for sched in self.schedule_ids:
1005
- self.pause_schedule(sched)
1006
- logger.info("All schedules paused.")
1007
- return None
1008
-
1009
- def resume_all_schedules(self) -> None:
1010
- """Resume all paused schedules.
1011
-
1012
- This method resumes all paused schedules in the scheduler.
1013
-
1014
- Example:
1015
- ```python
1016
- # Resume all paused schedules
1017
- worker.resume_all_schedules()
1018
- ```
1019
- """
1020
- for sched in self.schedule_ids:
1021
- self.resume_schedule(sched)
1022
- logger.info("All schedules resumed.")
1023
- return None
1024
-
1025
- def show_schedules(self) -> None:
1026
- """Display all schedules in a user-friendly format.
1027
-
1028
- This method prints a formatted view of all schedules including their
1029
- status, next run time, and other relevant information.
1030
-
1031
- Example:
1032
- ```python
1033
- # Show all schedules in a readable format
1034
- worker.show_schedules()
1035
- ```
1036
- """
1037
- display_schedules(self._worker.get_schedules())
1038
-
1039
- def show_jobs(self) -> None:
1040
- """Display all jobs in a user-friendly format.
1041
-
1042
- This method prints a formatted view of all jobs including their
1043
- status, result, and other relevant information.
1044
-
1045
- Example:
1046
- ```python
1047
- # Show all jobs in a readable format
1048
- worker.show_jobs()
1049
- ```
1050
- """
1051
- display_jobs(self._worker.get_jobs())