dsw-command-queue 4.9.6__tar.gz → 4.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/PKG-INFO +2 -3
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw/command_queue/build_info.py +4 -4
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw/command_queue/command_queue.py +8 -47
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw/command_queue/query.py +0 -9
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw_command_queue.egg-info/PKG-INFO +2 -3
- dsw_command_queue-4.10.0/dsw_command_queue.egg-info/requires.txt +1 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/pyproject.toml +2 -3
- dsw_command_queue-4.9.6/dsw_command_queue.egg-info/requires.txt +0 -2
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/LICENSE +0 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/README.md +0 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw/command_queue/__init__.py +0 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw_command_queue.egg-info/SOURCES.txt +0 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw_command_queue.egg-info/dependency_links.txt +0 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw_command_queue.egg-info/not-zip-safe +0 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw_command_queue.egg-info/top_level.txt +0 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/setup.cfg +0 -0
- {dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dsw-command-queue
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.10.0
|
|
4
4
|
Summary: Library for working with command queue and persistent commands
|
|
5
5
|
Author-email: Marek Suchánek <marek.suchanek@ds-wizard.org>
|
|
6
6
|
License: Apache License 2.0
|
|
@@ -19,8 +19,7 @@ Classifier: Topic :: Utilities
|
|
|
19
19
|
Requires-Python: <4,>=3.10
|
|
20
20
|
Description-Content-Type: text/markdown
|
|
21
21
|
License-File: LICENSE
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
Requires-Dist: dsw-database==4.9.6
|
|
22
|
+
Requires-Dist: dsw-database==4.10.0
|
|
24
23
|
|
|
25
24
|
# Data Stewardship Wizard: Command Queue
|
|
26
25
|
|
|
@@ -9,9 +9,9 @@ BuildInfo = namedtuple(
|
|
|
9
9
|
)
|
|
10
10
|
|
|
11
11
|
BUILD_INFO = BuildInfo(
|
|
12
|
-
version='v4.
|
|
13
|
-
built_at='2024-09-
|
|
14
|
-
sha='
|
|
12
|
+
version='v4.10.0~ed867ed',
|
|
13
|
+
built_at='2024-09-03 06:47:20Z',
|
|
14
|
+
sha='ed867eddd9f00958893ec14928ab160e2168e230',
|
|
15
15
|
branch='HEAD',
|
|
16
|
-
tag='v4.
|
|
16
|
+
tag='v4.10.0',
|
|
17
17
|
)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import abc
|
|
2
2
|
import datetime
|
|
3
|
-
import func_timeout
|
|
4
3
|
import logging
|
|
5
4
|
import os
|
|
6
5
|
import platform
|
|
@@ -46,26 +45,21 @@ class CommandWorker:
|
|
|
46
45
|
def work(self, payload: PersistentCommand):
|
|
47
46
|
pass
|
|
48
47
|
|
|
49
|
-
def
|
|
50
|
-
pass
|
|
51
|
-
|
|
52
|
-
def process_exception(self, e: BaseException):
|
|
48
|
+
def process_exception(self, e: Exception):
|
|
53
49
|
pass
|
|
54
50
|
|
|
55
51
|
|
|
56
52
|
class CommandQueue:
|
|
57
53
|
|
|
58
54
|
def __init__(self, worker: CommandWorker, db: Database,
|
|
59
|
-
channel: str, component: str,
|
|
60
|
-
work_timeout: int | None = None):
|
|
55
|
+
channel: str, component: str, timeout: float):
|
|
61
56
|
self.worker = worker
|
|
62
57
|
self.db = db
|
|
63
58
|
self.queries = CommandQueries(
|
|
64
59
|
channel=channel,
|
|
65
60
|
component=component
|
|
66
61
|
)
|
|
67
|
-
self.
|
|
68
|
-
self.work_timeout = work_timeout
|
|
62
|
+
self.timeout = timeout
|
|
69
63
|
|
|
70
64
|
@tenacity.retry(
|
|
71
65
|
reraise=True,
|
|
@@ -90,7 +84,7 @@ class CommandQueue:
|
|
|
90
84
|
self._fetch_and_process_queued()
|
|
91
85
|
|
|
92
86
|
LOG.debug('Waiting for notifications')
|
|
93
|
-
w = select.select(fds, [], [], self.
|
|
87
|
+
w = select.select(fds, [], [], self.timeout)
|
|
94
88
|
|
|
95
89
|
if INTERRUPTED:
|
|
96
90
|
LOG.debug('Interrupt signal received, ending...')
|
|
@@ -98,7 +92,7 @@ class CommandQueue:
|
|
|
98
92
|
|
|
99
93
|
if w == ([], [], []):
|
|
100
94
|
LOG.debug(f'Nothing received in this cycle '
|
|
101
|
-
f'(timeouted after {self.
|
|
95
|
+
f'(timeouted after {self.timeout} seconds)')
|
|
102
96
|
else:
|
|
103
97
|
notifications = 0
|
|
104
98
|
for n in psycopg.generators.notifies(queue_conn.connection.pgconn):
|
|
@@ -147,46 +141,13 @@ class CommandQueue:
|
|
|
147
141
|
LOG.debug(f'Previous state: {command.state}')
|
|
148
142
|
LOG.debug(f'Attempts: {command.attempts} / {command.max_attempts}')
|
|
149
143
|
LOG.debug(f'Last error: {command.last_error_message}')
|
|
150
|
-
attempt_number = command.attempts + 1
|
|
151
144
|
|
|
152
145
|
try:
|
|
153
|
-
self.
|
|
154
|
-
query=self.queries.query_command_start(),
|
|
155
|
-
attempts=attempt_number,
|
|
156
|
-
updated_at=datetime.datetime.now(tz=datetime.UTC),
|
|
157
|
-
uuid=command.uuid,
|
|
158
|
-
)
|
|
159
|
-
self.db.conn_query.connection.commit()
|
|
160
|
-
|
|
161
|
-
def work():
|
|
162
|
-
self.worker.work(command)
|
|
163
|
-
|
|
164
|
-
if self.work_timeout is None:
|
|
165
|
-
LOG.info('Processing (without any timeout set)')
|
|
166
|
-
work()
|
|
167
|
-
else:
|
|
168
|
-
LOG.info(f'Processing (with timeout set to {self.work_timeout} seconds)')
|
|
169
|
-
func_timeout.func_timeout(
|
|
170
|
-
timeout=self.work_timeout,
|
|
171
|
-
func=work,
|
|
172
|
-
args=(),
|
|
173
|
-
kwargs=None,
|
|
174
|
-
)
|
|
146
|
+
self.worker.work(command)
|
|
175
147
|
|
|
176
148
|
self.db.execute_query(
|
|
177
149
|
query=self.queries.query_command_done(),
|
|
178
|
-
attempts=
|
|
179
|
-
updated_at=datetime.datetime.now(tz=datetime.UTC),
|
|
180
|
-
uuid=command.uuid,
|
|
181
|
-
)
|
|
182
|
-
except func_timeout.exceptions.FunctionTimedOut as e:
|
|
183
|
-
msg = f'Processing exceeded time limit ({self.work_timeout} seconds)'
|
|
184
|
-
LOG.warning(msg)
|
|
185
|
-
self.worker.process_timeout(e)
|
|
186
|
-
self.db.execute_query(
|
|
187
|
-
query=self.queries.query_command_error(),
|
|
188
|
-
attempts=attempt_number,
|
|
189
|
-
error_message=msg,
|
|
150
|
+
attempts=command.attempts + 1,
|
|
190
151
|
updated_at=datetime.datetime.now(tz=datetime.UTC),
|
|
191
152
|
uuid=command.uuid,
|
|
192
153
|
)
|
|
@@ -196,7 +157,7 @@ class CommandQueue:
|
|
|
196
157
|
self.worker.process_exception(e)
|
|
197
158
|
self.db.execute_query(
|
|
198
159
|
query=self.queries.query_command_error(),
|
|
199
|
-
attempts=
|
|
160
|
+
attempts=command.attempts + 1,
|
|
200
161
|
error_message=msg,
|
|
201
162
|
updated_at=datetime.datetime.now(tz=datetime.UTC),
|
|
202
163
|
uuid=command.uuid,
|
|
@@ -47,12 +47,3 @@ class CommandQueries:
|
|
|
47
47
|
updated_at = %(updated_at)s
|
|
48
48
|
WHERE uuid = %(uuid)s;
|
|
49
49
|
"""
|
|
50
|
-
|
|
51
|
-
@staticmethod
|
|
52
|
-
def query_command_start() -> str:
|
|
53
|
-
return """
|
|
54
|
-
UPDATE persistent_command
|
|
55
|
-
SET attempts = %(attempts)s,
|
|
56
|
-
updated_at = %(updated_at)s
|
|
57
|
-
WHERE uuid = %(uuid)s;
|
|
58
|
-
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dsw-command-queue
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.10.0
|
|
4
4
|
Summary: Library for working with command queue and persistent commands
|
|
5
5
|
Author-email: Marek Suchánek <marek.suchanek@ds-wizard.org>
|
|
6
6
|
License: Apache License 2.0
|
|
@@ -19,8 +19,7 @@ Classifier: Topic :: Utilities
|
|
|
19
19
|
Requires-Python: <4,>=3.10
|
|
20
20
|
Description-Content-Type: text/markdown
|
|
21
21
|
License-File: LICENSE
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
Requires-Dist: dsw-database==4.9.6
|
|
22
|
+
Requires-Dist: dsw-database==4.10.0
|
|
24
23
|
|
|
25
24
|
# Data Stewardship Wizard: Command Queue
|
|
26
25
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dsw-database==4.10.0
|
|
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = 'dsw-command-queue'
|
|
7
|
-
version = "4.
|
|
7
|
+
version = "4.10.0"
|
|
8
8
|
description = 'Library for working with command queue and persistent commands'
|
|
9
9
|
readme = 'README.md'
|
|
10
10
|
keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing']
|
|
@@ -24,9 +24,8 @@ classifiers = [
|
|
|
24
24
|
]
|
|
25
25
|
requires-python = '>=3.10, <4'
|
|
26
26
|
dependencies = [
|
|
27
|
-
'func-timeout',
|
|
28
27
|
# DSW
|
|
29
|
-
"dsw-database==4.
|
|
28
|
+
"dsw-database==4.10.0",
|
|
30
29
|
]
|
|
31
30
|
|
|
32
31
|
[project.urls]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw_command_queue.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw_command_queue.egg-info/not-zip-safe
RENAMED
|
File without changes
|
{dsw_command_queue-4.9.6 → dsw_command_queue-4.10.0}/dsw_command_queue.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|