queuerPy 0.9.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {queuerpy-0.9.0/queuerPy.egg-info → queuerpy-0.11.0}/PKG-INFO +1 -1
- {queuerpy-0.9.0 → queuerpy-0.11.0}/__init__.py +12 -15
- {queuerpy-0.9.0 → queuerpy-0.11.0}/_version.py +1 -1
- {queuerpy-0.9.0 → queuerpy-0.11.0}/core/listener.py +6 -6
- {queuerpy-0.9.0 → queuerpy-0.11.0}/core/retryer.py +1 -6
- {queuerpy-0.9.0 → queuerpy-0.11.0}/core/runner.py +5 -5
- {queuerpy-0.9.0 → queuerpy-0.11.0}/core/scheduler.py +5 -5
- {queuerpy-0.9.0 → queuerpy-0.11.0}/core/ticker.py +5 -5
- {queuerpy-0.9.0 → queuerpy-0.11.0}/database/__init__.py +4 -5
- {queuerpy-0.9.0 → queuerpy-0.11.0}/database/db_job.py +11 -11
- {queuerpy-0.9.0 → queuerpy-0.11.0}/database/db_listener.py +2 -2
- {queuerpy-0.9.0 → queuerpy-0.11.0}/database/db_master.py +6 -6
- {queuerpy-0.9.0 → queuerpy-0.11.0}/database/db_worker.py +4 -4
- {queuerpy-0.9.0 → queuerpy-0.11.0}/helper/database.py +1 -2
- {queuerpy-0.9.0 → queuerpy-0.11.0}/helper/error.py +2 -1
- {queuerpy-0.9.0 → queuerpy-0.11.0}/helper/logging.py +15 -9
- {queuerpy-0.9.0 → queuerpy-0.11.0}/helper/sql.py +2 -2
- {queuerpy-0.9.0 → queuerpy-0.11.0}/helper/task.py +4 -2
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/job.py +1 -1
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/options.py +1 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/task.py +1 -1
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/worker.py +1 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/pyproject.toml +4 -9
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuer.py +27 -25
- {queuerpy-0.9.0 → queuerpy-0.11.0/queuerPy.egg-info}/PKG-INFO +1 -1
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuerPy.egg-info/SOURCES.txt +27 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuer_global.py +11 -10
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuer_job.py +22 -22
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuer_listener.py +2 -2
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuer_master.py +8 -8
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuer_next_interval.py +3 -6
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuer_task.py +24 -18
- {queuerpy-0.9.0 → queuerpy-0.11.0}/LICENSE +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/MANIFEST.in +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/README.md +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/core/__init__.py +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/core/broadcaster.py +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/helper/__init__.py +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/__init__.py +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/batch_job.py +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/connection.py +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/master.py +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/model/options_on_error.py +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/py.typed +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuerPy.egg-info/dependency_links.txt +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuerPy.egg-info/requires.txt +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/queuerPy.egg-info/top_level.txt +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/setup.cfg +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/sql/job.sql +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/sql/master.sql +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/sql/notify.sql +0 -0
- {queuerpy-0.9.0 → queuerpy-0.11.0}/sql/worker.sql +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: queuerPy
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
|
|
5
5
|
Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
|
|
6
6
|
Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
|
|
@@ -10,46 +10,42 @@ A job queuing and processing system with PostgreSQL backend that provides:
|
|
|
10
10
|
- Event-driven notifications
|
|
11
11
|
"""
|
|
12
12
|
|
|
13
|
-
import
|
|
14
|
-
import os
|
|
15
|
-
|
|
16
|
-
# Add the package directory to Python path to enable direct imports
|
|
17
|
-
_package_dir = os.path.dirname(__file__)
|
|
18
|
-
if _package_dir not in sys.path:
|
|
19
|
-
sys.path.insert(0, _package_dir)
|
|
20
|
-
|
|
21
|
-
from _version import __version__
|
|
13
|
+
from ._version import __version__
|
|
22
14
|
|
|
23
15
|
__author__ = "Simon Herrmann"
|
|
24
16
|
__email__ = "siherrmann@users.noreply.github.com"
|
|
25
17
|
|
|
26
18
|
# Core exports
|
|
27
|
-
from queuer import (
|
|
19
|
+
from .queuer import (
|
|
28
20
|
Queuer,
|
|
29
21
|
new_queuer,
|
|
30
22
|
new_queuer_with_db,
|
|
31
23
|
)
|
|
32
24
|
|
|
33
|
-
from helper.database import (
|
|
25
|
+
from .helper.database import (
|
|
34
26
|
DatabaseConfiguration,
|
|
35
27
|
)
|
|
36
28
|
|
|
37
|
-
from model.job import (
|
|
29
|
+
from .model.job import (
|
|
38
30
|
Job,
|
|
39
31
|
JobStatus,
|
|
40
32
|
)
|
|
41
33
|
|
|
42
|
-
from model.worker import (
|
|
34
|
+
from .model.worker import (
|
|
43
35
|
Worker,
|
|
44
36
|
WorkerStatus,
|
|
45
37
|
)
|
|
46
38
|
|
|
47
|
-
from model.
|
|
39
|
+
from .model.master import (
|
|
40
|
+
Master,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
from .model.options_on_error import (
|
|
48
44
|
OnError,
|
|
49
45
|
RetryBackoff,
|
|
50
46
|
)
|
|
51
47
|
|
|
52
|
-
from helper.error import (
|
|
48
|
+
from .helper.error import (
|
|
53
49
|
QueuerError,
|
|
54
50
|
)
|
|
55
51
|
|
|
@@ -72,6 +68,7 @@ __all__ = [
|
|
|
72
68
|
"JobStatus",
|
|
73
69
|
"Worker",
|
|
74
70
|
"WorkerStatus",
|
|
71
|
+
"Master",
|
|
75
72
|
"OnError",
|
|
76
73
|
"RetryBackoff",
|
|
77
74
|
# Exceptions
|
|
@@ -9,8 +9,8 @@ import asyncio
|
|
|
9
9
|
import threading
|
|
10
10
|
from typing import TypeVar, Generic, Callable, Optional, Any
|
|
11
11
|
|
|
12
|
-
from
|
|
13
|
-
from
|
|
12
|
+
from .broadcaster import Broadcaster
|
|
13
|
+
from .runner import go_func
|
|
14
14
|
|
|
15
15
|
T = TypeVar("T")
|
|
16
16
|
|
|
@@ -50,7 +50,7 @@ class Listener(Generic[T]):
|
|
|
50
50
|
self._stop_event = stop_event
|
|
51
51
|
|
|
52
52
|
# Define the main coroutine that handles subscription and message reception
|
|
53
|
-
async def _listen_async():
|
|
53
|
+
async def _listen_async() -> None:
|
|
54
54
|
ch = await self.broadcaster.subscribe()
|
|
55
55
|
if ready_event is not None:
|
|
56
56
|
ready_event.set()
|
|
@@ -67,7 +67,7 @@ class Listener(Generic[T]):
|
|
|
67
67
|
self._active_notifications += 1
|
|
68
68
|
self._completion_event.clear()
|
|
69
69
|
|
|
70
|
-
def process_notification():
|
|
70
|
+
def process_notification() -> None:
|
|
71
71
|
"""Executes the user's callback in a separate thread."""
|
|
72
72
|
try:
|
|
73
73
|
notify_function(msg)
|
|
@@ -96,7 +96,7 @@ class Listener(Generic[T]):
|
|
|
96
96
|
Sends notification directly by broadcasting the data in a separate thread.
|
|
97
97
|
"""
|
|
98
98
|
|
|
99
|
-
async def _notify():
|
|
99
|
+
async def _notify() -> None:
|
|
100
100
|
await self.broadcaster.broadcast(data)
|
|
101
101
|
|
|
102
102
|
go_func(_notify, use_mp=False)
|
|
@@ -105,7 +105,7 @@ class Listener(Generic[T]):
|
|
|
105
105
|
"""Wait for all notifications to be processed."""
|
|
106
106
|
return self._completion_event.wait(timeout=timeout)
|
|
107
107
|
|
|
108
|
-
def stop(self):
|
|
108
|
+
def stop(self) -> None:
|
|
109
109
|
"""Stop listening."""
|
|
110
110
|
if self._stop_event:
|
|
111
111
|
self._stop_event.set()
|
|
@@ -6,16 +6,11 @@ for reliable task execution with various backoff strategies.
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import logging
|
|
9
|
-
import sys
|
|
10
|
-
import os
|
|
11
9
|
import asyncio
|
|
12
10
|
import inspect
|
|
13
11
|
from typing import Any, Callable, Optional
|
|
14
12
|
|
|
15
|
-
|
|
16
|
-
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
17
|
-
|
|
18
|
-
from model.options_on_error import OnError, RetryBackoff
|
|
13
|
+
from ..model.options_on_error import OnError, RetryBackoff
|
|
19
14
|
|
|
20
15
|
|
|
21
16
|
# Configure logging
|
|
@@ -10,7 +10,7 @@ import traceback
|
|
|
10
10
|
import queue
|
|
11
11
|
from typing import Callable, Any, Optional, Union
|
|
12
12
|
|
|
13
|
-
from helper.logging import get_logger
|
|
13
|
+
from ..helper.logging import get_logger
|
|
14
14
|
|
|
15
15
|
logger = get_logger(__name__)
|
|
16
16
|
|
|
@@ -44,11 +44,11 @@ class Runner(multiprocessing.Process):
|
|
|
44
44
|
self.is_async = inspect.iscoroutinefunction(task)
|
|
45
45
|
self._result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue(1)
|
|
46
46
|
|
|
47
|
-
def go(self):
|
|
47
|
+
def go(self) -> None:
|
|
48
48
|
"""Starts the Runner process in the background."""
|
|
49
49
|
self.start()
|
|
50
50
|
|
|
51
|
-
def run(self):
|
|
51
|
+
def run(self) -> None:
|
|
52
52
|
"""
|
|
53
53
|
The main execution method, executed in the child process.
|
|
54
54
|
It executes the assigned task, sends the result to the queue, and exits.
|
|
@@ -155,11 +155,11 @@ class SmallRunner(threading.Thread):
|
|
|
155
155
|
self.is_async = inspect.iscoroutinefunction(task)
|
|
156
156
|
self._result_queue: queue.Queue[Any] = queue.Queue(maxsize=1)
|
|
157
157
|
|
|
158
|
-
def go(self):
|
|
158
|
+
def go(self) -> None:
|
|
159
159
|
"""Starts the SmallRunner thread in the background."""
|
|
160
160
|
self.start()
|
|
161
161
|
|
|
162
|
-
def run(self):
|
|
162
|
+
def run(self) -> None:
|
|
163
163
|
"""
|
|
164
164
|
The main execution method for the thread.
|
|
165
165
|
"""
|
|
@@ -6,9 +6,9 @@ import time
|
|
|
6
6
|
from datetime import datetime
|
|
7
7
|
from typing import Callable, Dict, Optional, Any, Tuple
|
|
8
8
|
|
|
9
|
-
from
|
|
10
|
-
from helper.logging import get_logger
|
|
11
|
-
from helper.task import check_valid_task_with_parameters
|
|
9
|
+
from .runner import go_func
|
|
10
|
+
from ..helper.logging import get_logger
|
|
11
|
+
from ..helper.task import check_valid_task_with_parameters
|
|
12
12
|
|
|
13
13
|
logger = get_logger(__name__)
|
|
14
14
|
|
|
@@ -45,7 +45,7 @@ class Scheduler:
|
|
|
45
45
|
self.kwargs = kwargs
|
|
46
46
|
self.start_time = start_time
|
|
47
47
|
|
|
48
|
-
def go(self):
|
|
48
|
+
def go(self) -> None:
|
|
49
49
|
"""
|
|
50
50
|
Start the scheduler to run the task at the specified start time.
|
|
51
51
|
Mirrors Go's Go(ctx) method exactly.
|
|
@@ -57,7 +57,7 @@ class Scheduler:
|
|
|
57
57
|
else:
|
|
58
58
|
duration = 0
|
|
59
59
|
|
|
60
|
-
def scheduled_runner_task():
|
|
60
|
+
def scheduled_runner_task() -> None:
|
|
61
61
|
"""
|
|
62
62
|
This synchronous function runs in the SmallRunner thread.
|
|
63
63
|
It executes the delay and then runs the user's task.
|
|
@@ -9,8 +9,8 @@ import time
|
|
|
9
9
|
from datetime import timedelta
|
|
10
10
|
from typing import Any, Callable, Dict, Optional, Union
|
|
11
11
|
|
|
12
|
-
from
|
|
13
|
-
from helper.logging import get_logger
|
|
12
|
+
from .runner import Runner, SmallRunner, go_func
|
|
13
|
+
from ..helper.logging import get_logger
|
|
14
14
|
|
|
15
15
|
logger = get_logger(__name__)
|
|
16
16
|
|
|
@@ -58,7 +58,7 @@ class Ticker:
|
|
|
58
58
|
f"Ticker created: task={task.__name__}, interval={self.interval_seconds}s, use_mp={use_mp}, stop_event_created={id(self._stop_event)}"
|
|
59
59
|
)
|
|
60
60
|
|
|
61
|
-
def _ticker_function(self):
|
|
61
|
+
def _ticker_function(self) -> None:
|
|
62
62
|
"""
|
|
63
63
|
The continuous loop function executed by the Runner process/thread.
|
|
64
64
|
It handles all the fixed-rate scheduling logic (sleep math).
|
|
@@ -101,7 +101,7 @@ class Ticker:
|
|
|
101
101
|
f"Interval {self.interval_seconds}s exceeded. Skipping sleep."
|
|
102
102
|
)
|
|
103
103
|
|
|
104
|
-
def go(self):
|
|
104
|
+
def go(self) -> None:
|
|
105
105
|
"""
|
|
106
106
|
Starts the background runner process/thread, executing the internal scheduling loop.
|
|
107
107
|
"""
|
|
@@ -113,7 +113,7 @@ class Ticker:
|
|
|
113
113
|
use_mp=self._use_mp,
|
|
114
114
|
)
|
|
115
115
|
|
|
116
|
-
def stop(self):
|
|
116
|
+
def stop(self) -> None:
|
|
117
117
|
"""
|
|
118
118
|
Stops the background runner process/thread.
|
|
119
119
|
"""
|
|
@@ -4,7 +4,7 @@ Provides database handlers and SQL loading using helper.database classes.
|
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
# Import from helper for core database functionality
|
|
7
|
-
from helper.database import (
|
|
7
|
+
from ..helper.database import (
|
|
8
8
|
Database,
|
|
9
9
|
DatabaseConfiguration,
|
|
10
10
|
new_database,
|
|
@@ -12,7 +12,7 @@ from helper.database import (
|
|
|
12
12
|
)
|
|
13
13
|
|
|
14
14
|
# Import deprecated SQL loader from helper.sql module
|
|
15
|
-
from helper.sql import (
|
|
15
|
+
from ..helper.sql import (
|
|
16
16
|
SQLLoader,
|
|
17
17
|
)
|
|
18
18
|
|
|
@@ -42,7 +42,6 @@ __all__ = [
|
|
|
42
42
|
# Worker handler
|
|
43
43
|
"WorkerDBHandler",
|
|
44
44
|
# Listeners
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"DBListenerWorker",
|
|
45
|
+
"QueuerListener",
|
|
46
|
+
"new_queuer_db_listener",
|
|
48
47
|
]
|
|
@@ -9,11 +9,11 @@ from typing import List, Optional, Tuple
|
|
|
9
9
|
from uuid import UUID
|
|
10
10
|
from psycopg.rows import dict_row
|
|
11
11
|
|
|
12
|
-
from helper.database import Database
|
|
13
|
-
from helper.error import QueuerError
|
|
14
|
-
from helper.sql import SQLLoader, run_ddl
|
|
15
|
-
from model.job import Job
|
|
16
|
-
from model.worker import Worker
|
|
12
|
+
from ..helper.database import Database
|
|
13
|
+
from ..helper.error import QueuerError
|
|
14
|
+
from ..helper.sql import SQLLoader, run_ddl
|
|
15
|
+
from ..model.job import Job
|
|
16
|
+
from ..model.worker import Worker
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class JobDBHandler:
|
|
@@ -67,14 +67,14 @@ class JobDBHandler:
|
|
|
67
67
|
result = cur.fetchone()
|
|
68
68
|
return result[0] if result else False
|
|
69
69
|
|
|
70
|
-
def create_table(self):
|
|
70
|
+
def create_table(self) -> None:
|
|
71
71
|
"""Create job table using SQL init function."""
|
|
72
72
|
if self.db.instance is None:
|
|
73
73
|
raise ValueError("Database connection is not established")
|
|
74
74
|
|
|
75
75
|
run_ddl(self.db.instance, "SELECT init_job();")
|
|
76
76
|
|
|
77
|
-
def drop_tables(self):
|
|
77
|
+
def drop_tables(self) -> None:
|
|
78
78
|
"""Drop job tables with DDL deadlock protection."""
|
|
79
79
|
if self.db.instance is None:
|
|
80
80
|
raise ValueError("Database connection is not established")
|
|
@@ -128,7 +128,7 @@ class JobDBHandler:
|
|
|
128
128
|
# In full implementation, this would handle explicit transaction context
|
|
129
129
|
return self.insert_job(job)
|
|
130
130
|
|
|
131
|
-
def batch_insert_jobs(self, jobs: List[Job]):
|
|
131
|
+
def batch_insert_jobs(self, jobs: List[Job]) -> None:
|
|
132
132
|
"""
|
|
133
133
|
Insert multiple jobs in a batch operation using executemany.
|
|
134
134
|
Mirrors Go's BatchInsertJobs method with direct INSERT statements.
|
|
@@ -248,7 +248,7 @@ class JobDBHandler:
|
|
|
248
248
|
except Exception as e:
|
|
249
249
|
raise QueuerError("update stale jobs", e)
|
|
250
250
|
|
|
251
|
-
def delete_job(self, rid: UUID):
|
|
251
|
+
def delete_job(self, rid: UUID) -> None:
|
|
252
252
|
"""
|
|
253
253
|
Delete a job by RID using SQL function.
|
|
254
254
|
Mirrors Go's DeleteJob method.
|
|
@@ -361,7 +361,7 @@ class JobDBHandler:
|
|
|
361
361
|
|
|
362
362
|
return jobs
|
|
363
363
|
|
|
364
|
-
def add_retention_archive(self, days: int):
|
|
364
|
+
def add_retention_archive(self, days: int) -> None:
|
|
365
365
|
"""
|
|
366
366
|
Add retention policy for archive cleanup.
|
|
367
367
|
Mirrors Go's AddRetentionArchive method.
|
|
@@ -375,7 +375,7 @@ class JobDBHandler:
|
|
|
375
375
|
cur.execute("SELECT add_retention_archive(%s);", (days,))
|
|
376
376
|
self.db.instance.commit()
|
|
377
377
|
|
|
378
|
-
def remove_retention_archive(self):
|
|
378
|
+
def remove_retention_archive(self) -> None:
|
|
379
379
|
"""
|
|
380
380
|
Remove retention policy for archive cleanup.
|
|
381
381
|
Mirrors Go's RemoveRetentionArchive method.
|
|
@@ -9,8 +9,8 @@ from typing import Optional, Callable, Awaitable
|
|
|
9
9
|
import psycopg
|
|
10
10
|
from psycopg import AsyncConnection
|
|
11
11
|
|
|
12
|
-
from helper.database import DatabaseConfiguration
|
|
13
|
-
from helper.error import QueuerError
|
|
12
|
+
from ..helper.database import DatabaseConfiguration
|
|
13
|
+
from ..helper.error import QueuerError
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class QueuerListener:
|
|
@@ -7,10 +7,10 @@ import json
|
|
|
7
7
|
from typing import Optional
|
|
8
8
|
from psycopg.rows import dict_row
|
|
9
9
|
|
|
10
|
-
from helper.database import Database
|
|
11
|
-
from helper.sql import SQLLoader, run_ddl
|
|
12
|
-
from model.master import Master, MasterSettings
|
|
13
|
-
from model.worker import Worker
|
|
10
|
+
from ..helper.database import Database
|
|
11
|
+
from ..helper.sql import SQLLoader, run_ddl
|
|
12
|
+
from ..model.master import Master, MasterSettings
|
|
13
|
+
from ..model.worker import Worker
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class MasterDBHandler:
|
|
@@ -54,7 +54,7 @@ class MasterDBHandler:
|
|
|
54
54
|
result = cur.fetchone()
|
|
55
55
|
return bool(result[0]) if result else False
|
|
56
56
|
|
|
57
|
-
def create_table(self):
|
|
57
|
+
def create_table(self) -> None:
|
|
58
58
|
"""Create master table. Mirrors Go's CreateTable method."""
|
|
59
59
|
if self.db.instance is None:
|
|
60
60
|
raise ValueError("Database connection is not established")
|
|
@@ -63,7 +63,7 @@ class MasterDBHandler:
|
|
|
63
63
|
cur.execute("SELECT init_master();")
|
|
64
64
|
self.db.instance.commit()
|
|
65
65
|
|
|
66
|
-
def drop_table(self):
|
|
66
|
+
def drop_table(self) -> None:
|
|
67
67
|
"""Drop master table. Mirrors Go's DropTable method."""
|
|
68
68
|
if self.db.instance is None:
|
|
69
69
|
raise ValueError("Database connection is not established")
|
|
@@ -9,10 +9,10 @@ from typing import List, Optional
|
|
|
9
9
|
from uuid import UUID
|
|
10
10
|
from psycopg.rows import dict_row
|
|
11
11
|
|
|
12
|
-
from helper.database import Database
|
|
13
|
-
from helper.sql import SQLLoader, run_ddl
|
|
14
|
-
from model.worker import Worker, WorkerStatus
|
|
15
|
-
from model.connection import Connection as ConnectionModel
|
|
12
|
+
from ..helper.database import Database
|
|
13
|
+
from ..helper.sql import SQLLoader, run_ddl
|
|
14
|
+
from ..model.worker import Worker, WorkerStatus
|
|
15
|
+
from ..model.connection import Connection as ConnectionModel
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
class WorkerDBHandler:
|
|
@@ -5,6 +5,7 @@ Mirrors Go's helper/error.go with Python exception handling.
|
|
|
5
5
|
|
|
6
6
|
import inspect
|
|
7
7
|
from types import FrameType
|
|
8
|
+
from typing import Optional
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class QueuerError(Exception):
|
|
@@ -18,7 +19,7 @@ class QueuerError(Exception):
|
|
|
18
19
|
traceWithFunction = trace
|
|
19
20
|
current_frame = inspect.currentframe()
|
|
20
21
|
|
|
21
|
-
frame: FrameType
|
|
22
|
+
frame: Optional[FrameType] = None
|
|
22
23
|
if current_frame is not None:
|
|
23
24
|
frame = current_frame.f_back
|
|
24
25
|
|
|
@@ -98,7 +98,7 @@ class QueuerLogger:
|
|
|
98
98
|
self.logger.addHandler(handler)
|
|
99
99
|
self.logger.propagate = False
|
|
100
100
|
|
|
101
|
-
def debug(self, message: str, **kwargs: Any):
|
|
101
|
+
def debug(self, message: str, **kwargs: Any) -> None:
|
|
102
102
|
"""
|
|
103
103
|
Log debug message with optional context.
|
|
104
104
|
|
|
@@ -107,7 +107,7 @@ class QueuerLogger:
|
|
|
107
107
|
"""
|
|
108
108
|
self.log_with_context(logging.DEBUG, message, **kwargs)
|
|
109
109
|
|
|
110
|
-
def info(self, message: str, **kwargs: Any):
|
|
110
|
+
def info(self, message: str, **kwargs: Any) -> None:
|
|
111
111
|
"""
|
|
112
112
|
Log info message with optional context.
|
|
113
113
|
|
|
@@ -116,7 +116,7 @@ class QueuerLogger:
|
|
|
116
116
|
"""
|
|
117
117
|
self.log_with_context(logging.INFO, message, **kwargs)
|
|
118
118
|
|
|
119
|
-
def warning(self, message: str, **kwargs: Any):
|
|
119
|
+
def warning(self, message: str, **kwargs: Any) -> None:
|
|
120
120
|
"""
|
|
121
121
|
Log warning message with optional context.
|
|
122
122
|
|
|
@@ -125,7 +125,9 @@ class QueuerLogger:
|
|
|
125
125
|
"""
|
|
126
126
|
self.log_with_context(logging.WARNING, message, **kwargs)
|
|
127
127
|
|
|
128
|
-
def error(
|
|
128
|
+
def error(
|
|
129
|
+
self, message: str, error: Optional[Exception] = None, **kwargs: Any
|
|
130
|
+
) -> None:
|
|
129
131
|
"""
|
|
130
132
|
Log error message with optional exception and context.
|
|
131
133
|
|
|
@@ -137,7 +139,9 @@ class QueuerLogger:
|
|
|
137
139
|
message = f"{message}: {error}"
|
|
138
140
|
self.log_with_context(logging.ERROR, message, **kwargs)
|
|
139
141
|
|
|
140
|
-
def critical(
|
|
142
|
+
def critical(
|
|
143
|
+
self, message: str, error: Optional[Exception] = None, **kwargs: Any
|
|
144
|
+
) -> None:
|
|
141
145
|
"""
|
|
142
146
|
Log critical message with optional exception and context.
|
|
143
147
|
|
|
@@ -149,7 +153,7 @@ class QueuerLogger:
|
|
|
149
153
|
message = f"{message}: {error}"
|
|
150
154
|
self.log_with_context(logging.CRITICAL, message, **kwargs)
|
|
151
155
|
|
|
152
|
-
def log_with_context(self, level: int, message: str, **kwargs: Any):
|
|
156
|
+
def log_with_context(self, level: int, message: str, **kwargs: Any) -> None:
|
|
153
157
|
"""
|
|
154
158
|
Log message with additional context information.
|
|
155
159
|
|
|
@@ -164,7 +168,7 @@ class QueuerLogger:
|
|
|
164
168
|
|
|
165
169
|
self.logger.log(level, message)
|
|
166
170
|
|
|
167
|
-
def set_level(self, level: int):
|
|
171
|
+
def set_level(self, level: int) -> None:
|
|
168
172
|
"""Set the logging level."""
|
|
169
173
|
self.logger.setLevel(level)
|
|
170
174
|
|
|
@@ -214,7 +218,7 @@ class PerformanceLogger:
|
|
|
214
218
|
self.logger = logger
|
|
215
219
|
self.start_time: Optional[datetime] = None
|
|
216
220
|
|
|
217
|
-
def start_timer(self, operation: str):
|
|
221
|
+
def start_timer(self, operation: str) -> None:
|
|
218
222
|
"""Start timing an operation."""
|
|
219
223
|
self.start_time = datetime.now()
|
|
220
224
|
self.logger.debug(f"Started {operation}")
|
|
@@ -233,7 +237,9 @@ class PerformanceLogger:
|
|
|
233
237
|
return duration
|
|
234
238
|
|
|
235
239
|
|
|
236
|
-
def time_operation(
|
|
240
|
+
def time_operation(
|
|
241
|
+
operation: str, logger: Optional[QueuerLogger] = None
|
|
242
|
+
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
|
237
243
|
"""
|
|
238
244
|
Decorator to time function execution.
|
|
239
245
|
|
|
@@ -10,7 +10,7 @@ from typing import List, Optional
|
|
|
10
10
|
from psycopg import Connection, errors
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
|
|
13
|
-
from
|
|
13
|
+
from .logging import get_logger
|
|
14
14
|
|
|
15
15
|
logger = get_logger(__name__)
|
|
16
16
|
|
|
@@ -18,7 +18,7 @@ logger = get_logger(__name__)
|
|
|
18
18
|
_DDL_LOCK = threading.RLock()
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
def run_ddl(conn: Connection, sql_statement: str, max_retries: int = 3):
|
|
21
|
+
def run_ddl(conn: Connection, sql_statement: str, max_retries: int = 3) -> None:
|
|
22
22
|
"""
|
|
23
23
|
Executes DDL under a process lock, retrying on deadlocks or serialization errors.
|
|
24
24
|
|
|
@@ -7,7 +7,7 @@ import inspect
|
|
|
7
7
|
from typing import Any, Callable, List, Type
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def check_valid_task(task: Any):
|
|
10
|
+
def check_valid_task(task: Any) -> None:
|
|
11
11
|
"""
|
|
12
12
|
Check if the provided task is a valid function.
|
|
13
13
|
Raises ValueError if the task is invalid.
|
|
@@ -23,7 +23,9 @@ def check_valid_task(task: Any):
|
|
|
23
23
|
raise ValueError(f"task must be a function, got {type(task).__name__}")
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
def check_valid_task_with_parameters(
|
|
26
|
+
def check_valid_task_with_parameters(
|
|
27
|
+
task: Callable[..., Any], *parameters: Any
|
|
28
|
+
) -> None:
|
|
27
29
|
"""
|
|
28
30
|
Check if the provided task and parameters are valid.
|
|
29
31
|
Validates that the task is a valid function and parameters match input types.
|
|
@@ -9,7 +9,7 @@ from datetime import datetime
|
|
|
9
9
|
from typing import List, Any, Optional, Callable, Dict, Union
|
|
10
10
|
from uuid import UUID, uuid4
|
|
11
11
|
|
|
12
|
-
from helper.task import get_task_name_from_interface
|
|
12
|
+
from ..helper.task import get_task_name_from_interface
|
|
13
13
|
from .options import Options
|
|
14
14
|
|
|
15
15
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "queuerPy"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.11.0"
|
|
8
8
|
description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -62,15 +62,10 @@ packages = ["queuerPy", "queuerPy.core", "queuerPy.database", "queuerPy.helper",
|
|
|
62
62
|
|
|
63
63
|
[tool.setuptools.package-dir]
|
|
64
64
|
"queuerPy" = "."
|
|
65
|
-
"queuerPy.core" = "core"
|
|
66
|
-
"queuerPy.database" = "database"
|
|
67
|
-
"queuerPy.helper" = "helper"
|
|
68
|
-
"queuerPy.model" = "model"
|
|
69
|
-
"queuerPy.sql" = "sql"
|
|
70
65
|
|
|
71
66
|
[tool.setuptools.package-data]
|
|
72
|
-
"queuerPy" = ["
|
|
73
|
-
"
|
|
67
|
+
"queuerPy" = ["py.typed"]
|
|
68
|
+
"queuerPy.sql" = ["*.sql"]
|
|
74
69
|
|
|
75
70
|
[tool.pytest.ini_options]
|
|
76
71
|
testpaths = ["."]
|
|
@@ -97,7 +92,7 @@ pythonVersion = "3.8"
|
|
|
97
92
|
pythonPlatform = "All"
|
|
98
93
|
|
|
99
94
|
[tool.mypy]
|
|
100
|
-
python_version = "0.
|
|
95
|
+
python_version = "0.11.0"
|
|
101
96
|
warn_return_any = true
|
|
102
97
|
warn_unused_configs = true
|
|
103
98
|
disallow_untyped_defs = true
|
|
@@ -14,20 +14,20 @@ from datetime import datetime, timedelta
|
|
|
14
14
|
from typing import Optional, Dict, Callable, Any
|
|
15
15
|
from uuid import UUID
|
|
16
16
|
|
|
17
|
-
from core.ticker import Ticker
|
|
18
|
-
from core.runner import Runner, SmallRunner, go_func
|
|
19
|
-
from database.db_listener import QueuerListener, new_queuer_db_listener
|
|
20
|
-
from helper.logging import get_logger
|
|
21
|
-
from helper.database import DatabaseConfiguration
|
|
22
|
-
from model.job import Job, JobStatus
|
|
23
|
-
from model.worker import Worker, new_worker, new_worker_with_options, WorkerStatus
|
|
24
|
-
from model.master import MasterSettings
|
|
25
|
-
from model.options_on_error import OnError
|
|
26
|
-
from queuer_job import QueuerJobMixin
|
|
27
|
-
from queuer_task import QueuerTaskMixin
|
|
28
|
-
from queuer_next_interval import QueuerNextIntervalMixin
|
|
29
|
-
from queuer_listener import QueuerListenerMixin
|
|
30
|
-
from queuer_master import QueuerMasterMixin
|
|
17
|
+
from .core.ticker import Ticker
|
|
18
|
+
from .core.runner import Runner, SmallRunner, go_func
|
|
19
|
+
from .database.db_listener import QueuerListener, new_queuer_db_listener
|
|
20
|
+
from .helper.logging import get_logger
|
|
21
|
+
from .helper.database import DatabaseConfiguration
|
|
22
|
+
from .model.job import Job, JobStatus
|
|
23
|
+
from .model.worker import Worker, new_worker, new_worker_with_options, WorkerStatus
|
|
24
|
+
from .model.master import MasterSettings
|
|
25
|
+
from .model.options_on_error import OnError
|
|
26
|
+
from .queuer_job import QueuerJobMixin
|
|
27
|
+
from .queuer_task import QueuerTaskMixin
|
|
28
|
+
from .queuer_next_interval import QueuerNextIntervalMixin
|
|
29
|
+
from .queuer_listener import QueuerListenerMixin
|
|
30
|
+
from .queuer_master import QueuerMasterMixin
|
|
31
31
|
|
|
32
32
|
logger = get_logger(__name__)
|
|
33
33
|
|
|
@@ -155,7 +155,7 @@ class Queuer(
|
|
|
155
155
|
f"Queuer with worker created: {new_worker_obj.name} (RID: {self.worker.rid})"
|
|
156
156
|
)
|
|
157
157
|
|
|
158
|
-
def start(self, master_settings: Optional[MasterSettings] = None):
|
|
158
|
+
def start(self, master_settings: Optional[MasterSettings] = None) -> None:
|
|
159
159
|
"""Start the queuer."""
|
|
160
160
|
if self.running:
|
|
161
161
|
raise RuntimeError("Queuer is already running")
|
|
@@ -214,7 +214,7 @@ class Queuer(
|
|
|
214
214
|
|
|
215
215
|
logger.info("Queuer started")
|
|
216
216
|
|
|
217
|
-
def stop(self):
|
|
217
|
+
def stop(self) -> None:
|
|
218
218
|
"""Stop the queuer."""
|
|
219
219
|
# Store whether we were running to decide if we need to close connections
|
|
220
220
|
was_running = getattr(self, "_running", False)
|
|
@@ -388,7 +388,7 @@ class Queuer(
|
|
|
388
388
|
logger.info(f"Queuer '{self.worker.name}' stopped")
|
|
389
389
|
|
|
390
390
|
# Job notification listeners
|
|
391
|
-
async def _handle_job_notification(self, notification: str):
|
|
391
|
+
async def _handle_job_notification(self, notification: str) -> None:
|
|
392
392
|
"""
|
|
393
393
|
Handle job database notifications.
|
|
394
394
|
|
|
@@ -423,7 +423,7 @@ class Queuer(
|
|
|
423
423
|
except Exception as e:
|
|
424
424
|
logger.error(f"Error handling job notification: {e}")
|
|
425
425
|
|
|
426
|
-
async def _handle_job_archive_notification(self, notification: str):
|
|
426
|
+
async def _handle_job_archive_notification(self, notification: str) -> None:
|
|
427
427
|
"""
|
|
428
428
|
Handle job archive database notifications.
|
|
429
429
|
|
|
@@ -452,7 +452,7 @@ class Queuer(
|
|
|
452
452
|
logger.error(f"Error handling job archive notification: {e}")
|
|
453
453
|
traceback.print_exc()
|
|
454
454
|
|
|
455
|
-
def _start_listeners(self):
|
|
455
|
+
def _start_listeners(self) -> None:
|
|
456
456
|
"""Start database listeners."""
|
|
457
457
|
# Start job listener
|
|
458
458
|
if self.job_db_listener:
|
|
@@ -482,7 +482,7 @@ class Queuer(
|
|
|
482
482
|
except Exception as e:
|
|
483
483
|
logger.error(f"Error starting job archive listener: {e}")
|
|
484
484
|
|
|
485
|
-
def _wait_for_listeners_ready(self, timeout_seconds: float = 3.0):
|
|
485
|
+
def _wait_for_listeners_ready(self, timeout_seconds: float = 3.0) -> None:
|
|
486
486
|
"""
|
|
487
487
|
Wait for both database listeners to be ready by checking their connection status.
|
|
488
488
|
|
|
@@ -520,7 +520,7 @@ class Queuer(
|
|
|
520
520
|
)
|
|
521
521
|
|
|
522
522
|
# Tickers
|
|
523
|
-
def _heartbeat_func(self):
|
|
523
|
+
def _heartbeat_func(self) -> None:
|
|
524
524
|
"""Send periodic heartbeats - only updates database, not queuer state."""
|
|
525
525
|
try:
|
|
526
526
|
# Get current worker from database
|
|
@@ -535,7 +535,7 @@ class Queuer(
|
|
|
535
535
|
except Exception as e:
|
|
536
536
|
logger.error(f"Heartbeat error: {e}")
|
|
537
537
|
|
|
538
|
-
def _start_heartbeat_ticker(self):
|
|
538
|
+
def _start_heartbeat_ticker(self) -> None:
|
|
539
539
|
"""Start heartbeat ticker using threading."""
|
|
540
540
|
self.heartbeat_ticker = Ticker(
|
|
541
541
|
timedelta(seconds=30),
|
|
@@ -544,7 +544,7 @@ class Queuer(
|
|
|
544
544
|
)
|
|
545
545
|
self.heartbeat_ticker.go()
|
|
546
546
|
|
|
547
|
-
def _poll_jobs_func(self):
|
|
547
|
+
def _poll_jobs_func(self) -> None:
|
|
548
548
|
"""Poll for jobs as backup to notification system."""
|
|
549
549
|
try:
|
|
550
550
|
logger.debug("Running backup job polling")
|
|
@@ -552,7 +552,7 @@ class Queuer(
|
|
|
552
552
|
except Exception as e:
|
|
553
553
|
logger.warning(f"Backup job polling error: {e}")
|
|
554
554
|
|
|
555
|
-
def _start_poll_job_ticker(self):
|
|
555
|
+
def _start_poll_job_ticker(self) -> None:
|
|
556
556
|
"""
|
|
557
557
|
Start job polling ticker as backup mechanism.
|
|
558
558
|
This provides a safety net in case notification-based processing fails.
|
|
@@ -565,7 +565,9 @@ class Queuer(
|
|
|
565
565
|
self.poll_job_ticker.go()
|
|
566
566
|
|
|
567
567
|
# Database listeners
|
|
568
|
-
async def _start_job_listener(
|
|
568
|
+
async def _start_job_listener(
|
|
569
|
+
self, handle_job_notification: Callable[[str], Any]
|
|
570
|
+
) -> None:
|
|
569
571
|
"""
|
|
570
572
|
Start job database listener.
|
|
571
573
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: queuerPy
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
|
|
5
5
|
Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
|
|
6
6
|
Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
|
|
@@ -22,6 +22,33 @@ queuer_task.py
|
|
|
22
22
|
./queuer_master.py
|
|
23
23
|
./queuer_next_interval.py
|
|
24
24
|
./queuer_task.py
|
|
25
|
+
./core/__init__.py
|
|
26
|
+
./core/broadcaster.py
|
|
27
|
+
./core/listener.py
|
|
28
|
+
./core/retryer.py
|
|
29
|
+
./core/runner.py
|
|
30
|
+
./core/scheduler.py
|
|
31
|
+
./core/ticker.py
|
|
32
|
+
./database/__init__.py
|
|
33
|
+
./database/db_job.py
|
|
34
|
+
./database/db_listener.py
|
|
35
|
+
./database/db_master.py
|
|
36
|
+
./database/db_worker.py
|
|
37
|
+
./helper/__init__.py
|
|
38
|
+
./helper/database.py
|
|
39
|
+
./helper/error.py
|
|
40
|
+
./helper/logging.py
|
|
41
|
+
./helper/sql.py
|
|
42
|
+
./helper/task.py
|
|
43
|
+
./model/__init__.py
|
|
44
|
+
./model/batch_job.py
|
|
45
|
+
./model/connection.py
|
|
46
|
+
./model/job.py
|
|
47
|
+
./model/master.py
|
|
48
|
+
./model/options.py
|
|
49
|
+
./model/options_on_error.py
|
|
50
|
+
./model/task.py
|
|
51
|
+
./model/worker.py
|
|
25
52
|
./sql/job.sql
|
|
26
53
|
./sql/master.sql
|
|
27
54
|
./sql/notify.sql
|
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import threading
|
|
2
2
|
from typing import Any, Callable, Dict, Optional
|
|
3
|
-
|
|
4
|
-
from core.
|
|
5
|
-
from
|
|
6
|
-
from database.
|
|
7
|
-
from database.
|
|
8
|
-
from
|
|
9
|
-
from helper.
|
|
10
|
-
from
|
|
11
|
-
from model.
|
|
12
|
-
from model.
|
|
3
|
+
|
|
4
|
+
from .core.broadcaster import new_broadcaster
|
|
5
|
+
from .core.listener import Listener
|
|
6
|
+
from .database.db_job import JobDBHandler
|
|
7
|
+
from .database.db_master import MasterDBHandler
|
|
8
|
+
from .database.db_worker import WorkerDBHandler
|
|
9
|
+
from .helper.database import Database, DatabaseConfiguration, new_database
|
|
10
|
+
from .helper.logging import get_logger
|
|
11
|
+
from .model.job import Job
|
|
12
|
+
from .model.task import Task
|
|
13
|
+
from .model.worker import Worker
|
|
13
14
|
|
|
14
15
|
logger = get_logger()
|
|
15
16
|
|
|
@@ -9,17 +9,17 @@ from datetime import datetime
|
|
|
9
9
|
from typing import Dict, List, Optional, Any, Tuple, Union, Callable
|
|
10
10
|
from uuid import UUID
|
|
11
11
|
|
|
12
|
-
from core.broadcaster import BroadcasterQueue
|
|
13
|
-
from core.runner import Runner, SmallRunner
|
|
14
|
-
from core.retryer import Retryer
|
|
15
|
-
from core.scheduler import Scheduler
|
|
16
|
-
from helper.error import QueuerError
|
|
17
|
-
from helper.task import get_task_name_from_interface
|
|
18
|
-
from model.batch_job import BatchJob
|
|
19
|
-
from model.job import Job, JobStatus, new_job as create_job
|
|
20
|
-
from model.options import Options
|
|
21
|
-
from model.worker import Worker
|
|
22
|
-
from queuer_global import QueuerGlobalMixin
|
|
12
|
+
from .core.broadcaster import BroadcasterQueue
|
|
13
|
+
from .core.runner import Runner, SmallRunner
|
|
14
|
+
from .core.retryer import Retryer
|
|
15
|
+
from .core.scheduler import Scheduler
|
|
16
|
+
from .helper.error import QueuerError
|
|
17
|
+
from .helper.task import get_task_name_from_interface
|
|
18
|
+
from .model.batch_job import BatchJob
|
|
19
|
+
from .model.job import Job, JobStatus, new_job as create_job
|
|
20
|
+
from .model.options import Options
|
|
21
|
+
from .model.worker import Worker
|
|
22
|
+
from .queuer_global import QueuerGlobalMixin
|
|
23
23
|
|
|
24
24
|
# Set up logger
|
|
25
25
|
logger = logging.getLogger(__name__)
|
|
@@ -71,7 +71,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
71
71
|
logger.error(f"Error adding job with options: {str(e)}")
|
|
72
72
|
raise Exception(f"Adding job: {str(e)}")
|
|
73
73
|
|
|
74
|
-
def add_jobs(self, batch_jobs: List[BatchJob]):
|
|
74
|
+
def add_jobs(self, batch_jobs: List[BatchJob]) -> None:
|
|
75
75
|
"""
|
|
76
76
|
Add a batch of jobs to the queue.
|
|
77
77
|
|
|
@@ -207,7 +207,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
207
207
|
self._cancel_job(job)
|
|
208
208
|
return job
|
|
209
209
|
|
|
210
|
-
def cancel_all_jobs_by_worker(self, worker_rid: UUID, entries: int = 100):
|
|
210
|
+
def cancel_all_jobs_by_worker(self, worker_rid: UUID, entries: int = 100) -> None:
|
|
211
211
|
"""
|
|
212
212
|
Cancel all jobs assigned to a specific worker by its RID.
|
|
213
213
|
|
|
@@ -336,7 +336,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
336
336
|
except Exception as e:
|
|
337
337
|
raise Exception(f"Creating or inserting job: {str(e)}")
|
|
338
338
|
|
|
339
|
-
def _run_job_initial(self):
|
|
339
|
+
def _run_job_initial(self) -> None:
|
|
340
340
|
"""
|
|
341
341
|
Called to run the next job in the queue.
|
|
342
342
|
Updates job status to running with worker.
|
|
@@ -388,7 +388,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
388
388
|
|
|
389
389
|
async def _wait_for_job(
|
|
390
390
|
self, job: Job
|
|
391
|
-
) ->
|
|
391
|
+
) -> Tuple[List[Any], bool, Optional[Exception]]:
|
|
392
392
|
"""
|
|
393
393
|
Execute the job and return the results or an error.
|
|
394
394
|
|
|
@@ -434,7 +434,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
434
434
|
logger.error(f"Error setting up runner for task {job.task_name}: {e}")
|
|
435
435
|
return [], False, e
|
|
436
436
|
|
|
437
|
-
async def _retry_job(self, job: Job, job_error: Exception):
|
|
437
|
+
async def _retry_job(self, job: Job, job_error: Exception) -> None:
|
|
438
438
|
"""
|
|
439
439
|
Retry the job with the given job error.
|
|
440
440
|
|
|
@@ -467,7 +467,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
467
467
|
results, _, _ = await self._wait_for_job(job)
|
|
468
468
|
await self._succeed_job(job, results)
|
|
469
469
|
|
|
470
|
-
async def _run_job(self, job: Job):
|
|
470
|
+
async def _run_job(self, job: Job) -> None:
|
|
471
471
|
"""
|
|
472
472
|
Run the job.
|
|
473
473
|
|
|
@@ -483,7 +483,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
483
483
|
else:
|
|
484
484
|
await self._succeed_job(job, results)
|
|
485
485
|
|
|
486
|
-
def _run_job_sync(self, job: Job):
|
|
486
|
+
def _run_job_sync(self, job: Job) -> None:
|
|
487
487
|
"""
|
|
488
488
|
Synchronous wrapper for _run_job that can be called by the Scheduler.
|
|
489
489
|
Mirrors the Go pattern where scheduler calls the function directly.
|
|
@@ -499,7 +499,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
499
499
|
except Exception as e:
|
|
500
500
|
logger.error(f"Error executing scheduled job {job.rid}: {e}")
|
|
501
501
|
|
|
502
|
-
def _cancel_job(self, job: Job):
|
|
502
|
+
def _cancel_job(self, job: Job) -> None:
|
|
503
503
|
"""
|
|
504
504
|
Cancel a job.
|
|
505
505
|
|
|
@@ -524,7 +524,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
524
524
|
except Exception as e:
|
|
525
525
|
logger.error(f"Error updating job status to cancelled: {e}")
|
|
526
526
|
|
|
527
|
-
async def _succeed_job(self, job: Job, results: List[Any]):
|
|
527
|
+
async def _succeed_job(self, job: Job, results: List[Any]) -> None:
|
|
528
528
|
"""
|
|
529
529
|
Update the job status to succeeded and run the next job if available.
|
|
530
530
|
|
|
@@ -536,7 +536,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
536
536
|
logger.debug(f"_succeed_job: Setting job {job.rid} results to {results}")
|
|
537
537
|
await self._end_job(job)
|
|
538
538
|
|
|
539
|
-
async def _fail_job(self, job: Job, job_error: Exception):
|
|
539
|
+
async def _fail_job(self, job: Job, job_error: Exception) -> None:
|
|
540
540
|
"""
|
|
541
541
|
Update the job status to failed.
|
|
542
542
|
|
|
@@ -547,7 +547,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
547
547
|
job.error = str(job_error)
|
|
548
548
|
await self._end_job(job)
|
|
549
549
|
|
|
550
|
-
async def _end_job(self, job: Job):
|
|
550
|
+
async def _end_job(self, job: Job) -> None:
|
|
551
551
|
"""
|
|
552
552
|
End a job and potentially schedule it again if it's a recurring job.
|
|
553
553
|
|
|
@@ -6,8 +6,8 @@ Mirrors Go's queuerListener.go functionality.
|
|
|
6
6
|
import logging
|
|
7
7
|
from typing import Callable
|
|
8
8
|
|
|
9
|
-
from queuer_global import QueuerGlobalMixin
|
|
10
|
-
from model.job import Job
|
|
9
|
+
from .queuer_global import QueuerGlobalMixin
|
|
10
|
+
from .model.job import Job
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
@@ -6,12 +6,12 @@ Mirrors the Go queuerMaster.go with Python async patterns.
|
|
|
6
6
|
from datetime import timedelta
|
|
7
7
|
from typing import Optional
|
|
8
8
|
|
|
9
|
-
from core.runner import go_func
|
|
10
|
-
from core.ticker import Ticker
|
|
11
|
-
from helper.logging import get_logger
|
|
12
|
-
from helper.error import QueuerError
|
|
13
|
-
from model.master import Master, MasterSettings
|
|
14
|
-
from queuer_global import QueuerGlobalMixin
|
|
9
|
+
from .core.runner import go_func
|
|
10
|
+
from .core.ticker import Ticker
|
|
11
|
+
from .helper.logging import get_logger
|
|
12
|
+
from .helper.error import QueuerError
|
|
13
|
+
from .model.master import Master, MasterSettings
|
|
14
|
+
from .queuer_global import QueuerGlobalMixin
|
|
15
15
|
|
|
16
16
|
logger = get_logger(__name__)
|
|
17
17
|
|
|
@@ -51,7 +51,7 @@ class QueuerMasterMixin(QueuerGlobalMixin):
|
|
|
51
51
|
|
|
52
52
|
try:
|
|
53
53
|
|
|
54
|
-
def ticker_task():
|
|
54
|
+
def ticker_task() -> None:
|
|
55
55
|
"""Periodic task executed by the master ticker."""
|
|
56
56
|
try:
|
|
57
57
|
try:
|
|
@@ -129,7 +129,7 @@ class QueuerMasterMixin(QueuerGlobalMixin):
|
|
|
129
129
|
"""
|
|
130
130
|
try:
|
|
131
131
|
|
|
132
|
-
def ticker_task():
|
|
132
|
+
def ticker_task() -> None:
|
|
133
133
|
"""Periodic task that polls for master role."""
|
|
134
134
|
try:
|
|
135
135
|
logger.info("Polling master...")
|
|
@@ -6,8 +6,8 @@ Mirrors Go's queuerNextInterval.go functionality.
|
|
|
6
6
|
import logging
|
|
7
7
|
from typing import Any, Callable
|
|
8
8
|
|
|
9
|
-
from queuer_global import QueuerGlobalMixin
|
|
10
|
-
from model.worker import Worker
|
|
9
|
+
from .queuer_global import QueuerGlobalMixin
|
|
10
|
+
from .model.worker import Worker
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
@@ -32,10 +32,7 @@ class QueuerNextIntervalMixin(QueuerGlobalMixin):
|
|
|
32
32
|
:raises ValueError: If function is None
|
|
33
33
|
:raises RuntimeError: If function already exists or database update fails
|
|
34
34
|
"""
|
|
35
|
-
from helper.task import get_task_name_from_interface
|
|
36
|
-
|
|
37
|
-
if nif is None:
|
|
38
|
-
raise ValueError("NextIntervalFunc cannot be None")
|
|
35
|
+
from .helper.task import get_task_name_from_interface
|
|
39
36
|
|
|
40
37
|
try:
|
|
41
38
|
nif_name = get_task_name_from_interface(nif)
|
|
@@ -6,9 +6,9 @@ Mirrors Go's queuerTask.go functionality.
|
|
|
6
6
|
import logging
|
|
7
7
|
from typing import Any, Callable, Optional, TypeVar, overload
|
|
8
8
|
|
|
9
|
-
from model.task import new_task, new_task_with_name
|
|
10
|
-
from model.task import Task
|
|
11
|
-
from queuer_global import QueuerGlobalMixin
|
|
9
|
+
from .model.task import new_task, new_task_with_name
|
|
10
|
+
from .model.task import Task
|
|
11
|
+
from .queuer_global import QueuerGlobalMixin
|
|
12
12
|
|
|
13
13
|
logger = logging.getLogger(__name__)
|
|
14
14
|
|
|
@@ -91,24 +91,33 @@ class QueuerTaskMixin(QueuerGlobalMixin):
|
|
|
91
91
|
logger.info(f"Task added: {new_task_obj.name}")
|
|
92
92
|
return new_task_obj
|
|
93
93
|
|
|
94
|
-
|
|
95
|
-
|
|
94
|
+
def task(self) -> Callable[[F], F]:
|
|
95
|
+
"""
|
|
96
|
+
Decorator to register a task function with the queuer using the function name.
|
|
97
|
+
This is equivalent to calling add_task().
|
|
96
98
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
+
Usage:
|
|
100
|
+
@queuer.task()
|
|
101
|
+
def example_task(param1, param2):
|
|
102
|
+
return param1 + param2
|
|
99
103
|
|
|
100
|
-
|
|
104
|
+
:return: The decorator function that preserves the original function's type
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
def decorator(func: F) -> F:
|
|
108
|
+
self.add_task(func)
|
|
109
|
+
return func
|
|
110
|
+
|
|
111
|
+
return decorator
|
|
112
|
+
|
|
113
|
+
def task_with_name(self, name: str) -> Callable[[F], F]:
|
|
101
114
|
"""
|
|
102
115
|
Decorator to register a task function with the queuer.
|
|
103
116
|
This is equivalent to calling add_task() or add_task_with_name().
|
|
104
117
|
|
|
105
118
|
Usage:
|
|
106
|
-
@queuer.
|
|
107
|
-
def
|
|
108
|
-
return param1 + param2
|
|
109
|
-
|
|
110
|
-
@queuer.task(name="custom_task_name")
|
|
111
|
-
def my_other_task(param):
|
|
119
|
+
@queuer.task_with_name(name="custom_task_name")
|
|
120
|
+
def example_task(param):
|
|
112
121
|
return param * 2
|
|
113
122
|
|
|
114
123
|
:param name: Optional custom name for the task. If not provided, uses function name.
|
|
@@ -116,10 +125,7 @@ class QueuerTaskMixin(QueuerGlobalMixin):
|
|
|
116
125
|
"""
|
|
117
126
|
|
|
118
127
|
def decorator(func: F) -> F:
|
|
119
|
-
|
|
120
|
-
self.add_task_with_name(func, name)
|
|
121
|
-
else:
|
|
122
|
-
self.add_task(func)
|
|
128
|
+
self.add_task_with_name(func, name)
|
|
123
129
|
return func
|
|
124
130
|
|
|
125
131
|
return decorator
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|