dsw-command-queue 4.13.0__tar.gz → 4.14.0__tar.gz

{dsw_command_queue-4.13.0 → dsw_command_queue-4.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsw-command-queue
-Version: 4.13.0
+Version: 4.14.0
 Summary: Library for working with command queue and persistent commands
 Author-email: Marek Suchánek <marek.suchanek@ds-wizard.org>
 License: Apache License 2.0
@@ -11,16 +11,16 @@ Keywords: dsw,subscriber,publisher,database,queue,processing
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Database
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Utilities
-Requires-Python: <4,>=3.10
+Requires-Python: <4,>=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: func-timeout
-Requires-Dist: dsw-database==4.13.0
+Requires-Dist: dsw-database==4.14.0
 
 # Data Stewardship Wizard: Command Queue
 

dsw_command_queue-4.14.0/dsw/command_queue/__init__.py

@@ -0,0 +1,3 @@
+from .command_queue import CommandJobError, CommandQueue, CommandWorker
+
+__all__ = ['CommandJobError', 'CommandQueue', 'CommandWorker']

{dsw_command_queue-4.13.0 → dsw_command_queue-4.14.0}/dsw/command_queue/build_info.py

@@ -9,9 +9,9 @@ BuildInfo = namedtuple(
 )
 
 BUILD_INFO = BuildInfo(
-    version='v4.13.0~c8e1cb3',
-    built_at='2024-12-05 08:07:11Z',
-    sha='c8e1cb3b7e0cc93bdf509eef715cd0a926d907c3',
+    version='v4.14.0~213910f',
+    built_at='2025-01-07 08:17:30Z',
+    sha='213910ffb32a7cea98942ccd0f6c52cb6cf79128',
     branch='HEAD',
-    tag='v4.13.0',
+    tag='v4.14.0',
 )

{dsw_command_queue-4.13.0 → dsw_command_queue-4.14.0}/dsw/command_queue/command_queue.py

@@ -1,13 +1,14 @@
 import abc
 import datetime
-import func_timeout
 import logging
 import os
 import platform
-import psycopg
-import psycopg.generators
 import select
 import signal
+
+import func_timeout
+import psycopg
+import psycopg.generators
 import tenacity
 
 from dsw.database import Database
@@ -22,7 +23,6 @@ RETRY_QUERY_TRIES = 3
 RETRY_QUEUE_MULTIPLIER = 0.5
 RETRY_QUEUE_TRIES = 5
 
-INTERRUPTED = False
 IS_LINUX = platform == 'Linux'
 
 if IS_LINUX:
@@ -30,20 +30,48 @@ if IS_LINUX:
     signal.set_wakeup_fd(_QUEUE_PIPE_W)
 
 
-def signal_handler(recv_signal, frame):
-    global INTERRUPTED
-    LOG.warning(f'Received interrupt signal: {recv_signal}')
-    INTERRUPTED = True
-
-
-signal.signal(signal.SIGINT, signal_handler)
-signal.signal(signal.SIGABRT, signal_handler)
+class CommandJobError(BaseException):
+
+    def __init__(self, job_id: str, message: str, try_again: bool,
+                 exc: BaseException | None = None):
+        self.job_id = job_id
+        self.message = message
+        self.try_again = try_again
+        self.exc = exc
+        super().__init__(message)
+
+    def __str__(self):
+        return self.message
+
+    def log_message(self):
+        if self.exc is None:
+            return self.message
+        return f'{self.message} (caused by: [{type(self.exc).__name__}] {str(self.exc)})'
+
+    def db_message(self):
+        if self.exc is None:
+            return self.message
+        return f'{self.message}\n\n' \
+               f'Caused by: {type(self.exc).__name__}\n' \
+               f'{str(self.exc)}'
+
+    @staticmethod
+    def create(job_id: str, message: str, try_again: bool = True,
+               exc: BaseException | None = None):
+        if isinstance(exc, CommandJobError):
+            return exc
+        return CommandJobError(
+            job_id=job_id,
+            message=message,
+            try_again=try_again,
+            exc=exc,
+        )
 
 
 class CommandWorker:
 
     @abc.abstractmethod
-    def work(self, payload: PersistentCommand):
+    def work(self, command: PersistentCommand):
         pass
 
     def process_timeout(self, e: BaseException):
@@ -55,7 +83,7 @@ class CommandWorker:
 
 class CommandQueue:
 
-    def __init__(self, worker: CommandWorker, db: Database,
+    def __init__(self, *, worker: CommandWorker, db: Database,
                  channel: str, component: str, wait_timeout: float,
                  work_timeout: int | None = None):
         self.worker = worker
@@ -66,6 +94,10 @@ class CommandQueue:
         )
         self.wait_timeout = wait_timeout
         self.work_timeout = work_timeout
+        self._interrupted = False
+
+        signal.signal(signal.SIGINT, self._signal_handler)
+        signal.signal(signal.SIGABRT, self._signal_handler)
 
     @tenacity.retry(
         reraise=True,
@@ -92,19 +124,19 @@ class CommandQueue:
             LOG.debug('Waiting for notifications')
             w = select.select(fds, [], [], self.wait_timeout)
 
-            if INTERRUPTED:
+            if self._interrupted:
                 LOG.debug('Interrupt signal received, ending...')
                 break
 
             if w == ([], [], []):
-                LOG.debug(f'Nothing received in this cycle '
-                          f'(timeouted after {self.wait_timeout} seconds)')
+                LOG.debug('Nothing received in this cycle (timeout %s seconds)',
+                          self.wait_timeout)
             else:
                 notifications = 0
                 for n in psycopg.generators.notifies(queue_conn.connection.pgconn):
                     notifications += 1
                     LOG.debug(str(n))
-                LOG.info(f'Notifications received ({notifications} in total)')
+                LOG.info('Notifications received (%s in total)', notifications)
         LOG.debug('Exiting command queue')
 
     @tenacity.retry(
@@ -123,11 +155,12 @@ class CommandQueue:
         count = 0
         while self.fetch_and_process():
             count += 1
-        LOG.info(f'There are no more commands to process ({count} processed)')
+        LOG.info('There are no more commands to process (%s processed)',
+                 count)
 
     def accept_notification(self, payload: psycopg.Notify) -> bool:
-        LOG.debug(f'Accepting notification from channel "{payload.channel}" '
-                  f'(PID = {payload.pid}) {payload.payload}')
+        LOG.debug('Accepting notification from channel "%s" (PID = %s) %s',
+                  payload.channel, payload.pid, payload.payload)
         LOG.debug('Trying to fetch a new job')
         return self.fetch_and_process()
 
@@ -139,16 +172,25 @@ class CommandQueue:
         )
         result = cursor.fetchall()
         if len(result) != 1:
-            LOG.debug(f'Fetched {len(result)} persistent commands')
+            LOG.debug('Fetched %s persistent commands', len(result))
             return False
 
         command = PersistentCommand.from_dict_row(result[0])
-        LOG.info(f'Retrieved persistent command {command.uuid} for processing')
-        LOG.debug(f'Previous state: {command.state}')
-        LOG.debug(f'Attempts: {command.attempts} / {command.max_attempts}')
-        LOG.debug(f'Last error: {command.last_error_message}')
-        attempt_number = command.attempts + 1
+        LOG.info('Retrieved persistent command %s for processing', command.uuid)
+        LOG.debug('Previous state: %s', command.state)
+        LOG.debug('Attempts: %s / %s', command.attempts, command.max_attempts)
+        LOG.debug('Last error: %s', command.last_error_message)
 
+        self._process(command)
+
+        LOG.debug('Committing transaction')
+        self.db.conn_query.connection.commit()
+        cursor.close()
+        LOG.info('Notification processing finished')
+        return True
+
+    def _process(self, command: PersistentCommand):
+        attempt_number = command.attempts + 1
         try:
             self.db.execute_query(
                 query=self.queries.query_command_start(),
@@ -165,7 +207,8 @@ class CommandQueue:
                 LOG.info('Processing (without any timeout set)')
                 work()
             else:
-                LOG.info(f'Processing (with timeout set to {self.work_timeout} seconds)')
+                LOG.info('Processing (with timeout set to %s seconds)',
+                         self.work_timeout)
                 func_timeout.func_timeout(
                     timeout=self.work_timeout,
                     func=work,
@@ -190,8 +233,27 @@ class CommandQueue:
                 updated_at=datetime.datetime.now(tz=datetime.UTC),
                 uuid=command.uuid,
             )
+        except CommandJobError as e:
+            if e.try_again and attempt_number < command.max_attempts:
+                query = self.queries.query_command_error()
+                msg = f'Failed with job error: {e.message} (will try again)'
+            else:
+                query = self.queries.query_command_error_stop()
+                msg = f'Failed with job error: {e.message}'
+            LOG.warning(msg)
+            self.worker.process_exception(e)
+            self.db.execute_query(
+                query=query,
+                attempts=attempt_number,
+                error_message=msg,
+                updated_at=datetime.datetime.now(tz=datetime.UTC),
+                uuid=command.uuid,
+            )
         except Exception as e:
-            msg = f'Failed with exception: {str(e)} ({type(e).__name__})'
+            if attempt_number < command.max_attempts:
+                msg = f'Failed with exception [{type(e).__name__}]: {str(e)} (will try again)'
+            else:
+                msg = f'Failed with exception [{type(e).__name__}]: {str(e)}'
             LOG.warning(msg)
             self.worker.process_exception(e)
             self.db.execute_query(
@@ -202,8 +264,7 @@ class CommandQueue:
                 uuid=command.uuid,
             )
 
-        LOG.debug('Committing transaction')
-        self.db.conn_query.connection.commit()
-        cursor.close()
-        LOG.info('Notification processing finished')
-        return True
+    def _signal_handler(self, recv_signal, frame):
+        LOG.warning('Received interrupt signal: %s (frame: %s)',
+                    recv_signal, frame)
+        self._interrupted = True

{dsw_command_queue-4.13.0 → dsw_command_queue-4.14.0}/dsw/command_queue/query.py

@@ -1,4 +1,7 @@
-class CommandState:
+import enum
+
+
+class CommandState(enum.Enum):
     NEW = 'NewPersistentCommandState'
     DONE = 'DonePersistentCommandState'
     ERROR = 'ErrorPersistentCommandState'
@@ -20,9 +23,11 @@ class CommandQueries:
             FROM persistent_command
             WHERE component = '{self.component}'
               AND attempts < max_attempts
-              AND state != '{CommandState.DONE}'
-              AND state != '{CommandState.IGNORE}'
-              AND (updated_at AT TIME ZONE 'UTC') < (%(now)s - ({exp} ^ attempts - 1) * INTERVAL '{interval}')
+              AND state != '{CommandState.DONE.value}'
+              AND state != '{CommandState.IGNORE.value}'
+              AND (updated_at AT TIME ZONE 'UTC')
+                    <
+                  (%(now)s - ({exp} ^ attempts - 1) * INTERVAL '{interval}')
             ORDER BY attempts ASC, updated_at DESC
             LIMIT 1 FOR UPDATE SKIP LOCKED;
         """
@@ -32,6 +37,18 @@ class CommandQueries:
         return f"""
             UPDATE persistent_command
             SET attempts = %(attempts)s,
+                last_error_message = %(error_message)s,
+                state = '{CommandState.ERROR.value}',
+                updated_at = %(updated_at)s
+            WHERE uuid = %(uuid)s;
+        """
+
+    @staticmethod
+    def query_command_error_stop() -> str:
+        return f"""
+            UPDATE persistent_command
+            SET attempts = %(attempts)s,
+                max_attempts = %(attempts)s,
                 last_error_message = %(error_message)s,
                 state = '{CommandState.ERROR}',
                 updated_at = %(updated_at)s
@@ -43,7 +60,7 @@ class CommandQueries:
         return f"""
             UPDATE persistent_command
             SET attempts = %(attempts)s,
-                state = '{CommandState.DONE}',
+                state = '{CommandState.DONE.value}',
                 updated_at = %(updated_at)s
             WHERE uuid = %(uuid)s;
         """

{dsw_command_queue-4.13.0 → dsw_command_queue-4.14.0}/dsw_command_queue.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsw-command-queue
-Version: 4.13.0
+Version: 4.14.0
 Summary: Library for working with command queue and persistent commands
 Author-email: Marek Suchánek <marek.suchanek@ds-wizard.org>
 License: Apache License 2.0
@@ -11,16 +11,16 @@ Keywords: dsw,subscriber,publisher,database,queue,processing
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Database
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Utilities
-Requires-Python: <4,>=3.10
+Requires-Python: <4,>=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: func-timeout
-Requires-Dist: dsw-database==4.13.0
+Requires-Dist: dsw-database==4.14.0
 
 # Data Stewardship Wizard: Command Queue
 

dsw_command_queue-4.14.0/dsw_command_queue.egg-info/requires.txt

@@ -0,0 +1,2 @@
+func-timeout
+dsw-database==4.14.0

{dsw_command_queue-4.13.0 → dsw_command_queue-4.14.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-command-queue'
-version = "4.13.0"
+version = "4.14.0"
 description = 'Library for working with command queue and persistent commands'
 readme = 'README.md'
 keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing']
@@ -16,17 +16,17 @@ classifiers = [
     'Development Status :: 5 - Production/Stable',
     'License :: OSI Approved :: Apache Software License',
     'Programming Language :: Python',
-    'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
     'Topic :: Database',
     'Topic :: Text Processing',
     'Topic :: Utilities',
 ]
-requires-python = '>=3.10, <4'
+requires-python = '>=3.11, <4'
 dependencies = [
     'func-timeout',
     # DSW
-    "dsw-database==4.13.0",
+    "dsw-database==4.14.0",
 ]
 
 [project.urls]

dsw_command_queue-4.13.0/dsw/command_queue/__init__.py

@@ -1,3 +0,0 @@
-from .command_queue import CommandQueue, CommandWorker
-
-__all__ = ['CommandQueue', 'CommandWorker']

dsw_command_queue-4.13.0/dsw_command_queue.egg-info/requires.txt

@@ -1,2 +0,0 @@
-func-timeout
-dsw-database==4.13.0