dsw-command-queue 4.27.0__py3-none-any.whl

dsw/command_queue/__init__.py

@@ -0,0 +1,4 @@
+from .command_queue import CommandJobError, CommandQueue, CommandWorker
+
+
+__all__ = ['CommandJobError', 'CommandQueue', 'CommandWorker']

dsw/command_queue/build_info.py

@@ -0,0 +1,17 @@
+# Generated file
+# - do not overwrite
+# - do not include in git
+from collections import namedtuple
+
+BuildInfo = namedtuple(
+    'BuildInfo',
+    ['version', 'built_at', 'sha', 'branch', 'tag'],
+)
+
+BUILD_INFO = BuildInfo(
+    version='v4.27.0~8ec71bd',
+    built_at='2026-02-03 08:43:16Z',
+    sha='8ec71bd85dfbea66adedb6590f7d76ae5143bbaa',
+    branch='HEAD',
+    tag='v4.27.0',
+)

dsw/command_queue/command_queue.py

@@ -0,0 +1,267 @@
+import abc
+import datetime
+import logging
+import os
+import platform
+import select
+import signal
+
+import func_timeout
+import psycopg.generators
+import tenacity
+
+from dsw.database import Database
+from dsw.database.model import PersistentCommand
+
+from .query import CommandQueries
+
+
+LOG = logging.getLogger(__name__)
+
+RETRY_QUERY_MULTIPLIER = 0.5
+RETRY_QUERY_TRIES = 3
+RETRY_QUEUE_MULTIPLIER = 0.5
+RETRY_QUEUE_TRIES = 5
+
+IS_LINUX = platform == 'Linux'
+
+if IS_LINUX:
+    _QUEUE_PIPE_R, _QUEUE_PIPE_W = os.pipe()
+    signal.set_wakeup_fd(_QUEUE_PIPE_W)
+
+
+class CommandJobError(BaseException):
+
+    def __init__(self, job_id: str, message: str, try_again: bool,
+                 exc: BaseException | None = None):
+        self.job_id = job_id
+        self.message = message
+        self.try_again = try_again
+        self.exc = exc
+        super().__init__(message)
+
+    def __str__(self):
+        return self.message
+
+    def log_message(self):
+        if self.exc is None:
+            return self.message
+        return f'{self.message} (caused by: [{type(self.exc).__name__}] {str(self.exc)})'
+
+    def db_message(self):
+        if self.exc is None:
+            return self.message
+        return f'{self.message}\n\n' \
+               f'Caused by: {type(self.exc).__name__}\n' \
+               f'{str(self.exc)}'
+
+    @staticmethod
+    def create(job_id: str, message: str, try_again: bool = True,
+               exc: BaseException | None = None):
+        if isinstance(exc, CommandJobError):
+            return exc
+        return CommandJobError(
+            job_id=job_id,
+            message=message,
+            try_again=try_again,
+            exc=exc,
+        )
+
+
+class CommandWorker:
+
+    @abc.abstractmethod
+    def work(self, command: PersistentCommand):
+        pass
+
+    def process_timeout(self, e: BaseException):
+        pass
+
+    def process_exception(self, e: BaseException):
+        pass
+
+
+class CommandQueue:
+
+    def __init__(self, *, worker: CommandWorker, db: Database,
+                 channel: str, component: str, wait_timeout: float,
+                 work_timeout: int | None = None):
+        self.worker = worker
+        self.db = db
+        self.queries = CommandQueries(
+            channel=channel,
+        )
+        self.component = component
+        self.wait_timeout = wait_timeout
+        self.work_timeout = work_timeout
+        self._interrupted = False
+
+        signal.signal(signal.SIGINT, self._signal_handler)
+        signal.signal(signal.SIGABRT, self._signal_handler)
+
+    @tenacity.retry(
+        reraise=True,
+        wait=tenacity.wait_exponential(multiplier=RETRY_QUEUE_MULTIPLIER),
+        stop=tenacity.stop_after_attempt(RETRY_QUEUE_TRIES),
+        before=tenacity.before_log(LOG, logging.INFO),
+        after=tenacity.after_log(LOG, logging.INFO),
+    )
+    def run(self):
+        LOG.info('Preparing to listen to command queue (issuing LISTEN)')
+        queue_conn = self.db.conn_queue
+        queue_conn.connection.execute(
+            query=self.queries.query_listen().encode(),
+        )
+        queue_conn.listening = True
+        LOG.info('Listening to notifications in command queue')
+        fds = [queue_conn.connection.pgconn.socket]
+        if IS_LINUX:
+            fds.append(_QUEUE_PIPE_R)
+
+        while True:
+            self._fetch_and_process_queued()
+
+            LOG.info('Waiting for notifications (up to %s seconds)', self.wait_timeout)
+            w = select.select(fds, [], [], self.wait_timeout)
+
+            if self._interrupted:
+                LOG.debug('Interrupt signal received, ending...')
+                break
+
+            if w == ([], [], []):
+                LOG.info('Nothing received in this cycle (timeout %s seconds)',
+                         self.wait_timeout)
+            else:
+                notifications = 0
+                for notification in psycopg.generators.notifies(queue_conn.connection.pgconn):
+                    notifications += 1
+                    LOG.info('Notification received: %s', notification)
+                LOG.info('Notifications received (%s in total)', notifications)
+        LOG.info('Exiting command queue')
+
+    @tenacity.retry(
+        reraise=True,
+        wait=tenacity.wait_exponential(multiplier=RETRY_QUEUE_MULTIPLIER),
+        stop=tenacity.stop_after_attempt(RETRY_QUEUE_TRIES),
+        before=tenacity.before_log(LOG, logging.INFO),
+        after=tenacity.after_log(LOG, logging.INFO),
+    )
+    def run_once(self):
+        LOG.info('Processing the command queue once')
+        self._fetch_and_process_queued()
+
+    def _fetch_and_process_queued(self):
+        LOG.info('Fetching the commands')
+        count = 0
+        while self.fetch_and_process():
+            count += 1
+        LOG.info('There are no more commands to process (%s processed)',
+                 count)
+
+    def fetch_and_process(self) -> bool:
+        cursor = self.db.conn_query.new_cursor(use_dict=True)
+        cursor.execute(
+            query=self.queries.query_get_command(),
+            params={
+                'component': self.component,
+                'now': datetime.datetime.now(tz=datetime.UTC),
+            },
+        )
+        result = cursor.fetchall()
+        if len(result) != 1:
+            LOG.info('Fetched %s persistent commands', len(result))
+            return False
+
+        command = PersistentCommand.from_dict_row(result[0])
+        LOG.info('Retrieved persistent command %s for processing', command.uuid)
+        LOG.info('Previous state: %s', command.state)
+        LOG.info('Attempts: %s / %s', command.attempts, command.max_attempts)
+        LOG.info('Last error: %s', command.last_error_message)
+
+        self._process(command)
+
+        LOG.debug('Committing transaction')
+        self.db.conn_query.connection.commit()
+        cursor.close()
+        LOG.info('Notification processing finished')
+        return True
+
+    def _process(self, command: PersistentCommand):
+        attempt_number = command.attempts + 1
+        try:
+            self.db.execute_query(
+                query=self.queries.query_command_start(),
+                attempts=attempt_number,
+                updated_at=datetime.datetime.now(tz=datetime.UTC),
+                uuid=command.uuid,
+            )
+            self.db.conn_query.connection.commit()
+
+            def work():
+                self.worker.work(command)
+
+            if self.work_timeout is None:
+                LOG.info('Processing (without any timeout set)')
+                work()
+            else:
+                LOG.info('Processing (with timeout set to %s seconds)',
+                         self.work_timeout)
+                func_timeout.func_timeout(
+                    timeout=self.work_timeout,
+                    func=work,
+                    args=(),
+                    kwargs=None,
+                )
+
+            self.db.execute_query(
+                query=self.queries.query_command_done(),
+                attempts=attempt_number,
+                updated_at=datetime.datetime.now(tz=datetime.UTC),
+                uuid=command.uuid,
+            )
+        except func_timeout.exceptions.FunctionTimedOut as e:
+            msg = f'Processing exceeded time limit ({self.work_timeout} seconds)'
+            LOG.warning(msg)
+            self.worker.process_timeout(e)
+            self.db.execute_query(
+                query=self.queries.query_command_error(),
+                attempts=attempt_number,
+                error_message=msg,
+                updated_at=datetime.datetime.now(tz=datetime.UTC),
+                uuid=command.uuid,
+            )
+        except CommandJobError as e:
+            if e.try_again and attempt_number < command.max_attempts:
+                query = self.queries.query_command_error()
+                msg = f'Failed with job error: {e.message} (will try again)'
+            else:
+                query = self.queries.query_command_error_stop()
+                msg = f'Failed with job error: {e.message}'
+            LOG.warning(msg)
+            self.worker.process_exception(e)
+            self.db.execute_query(
+                query=query,
+                attempts=attempt_number,
+                error_message=msg,
+                updated_at=datetime.datetime.now(tz=datetime.UTC),
+                uuid=command.uuid,
+            )
+        except Exception as e:
+            if attempt_number < command.max_attempts:
+                msg = f'Failed with exception [{type(e).__name__}]: {str(e)} (will try again)'
+            else:
+                msg = f'Failed with exception [{type(e).__name__}]: {str(e)}'
+            LOG.warning(msg)
+            self.worker.process_exception(e)
+            self.db.execute_query(
+                query=self.queries.query_command_error(),
+                attempts=attempt_number,
+                error_message=msg,
+                updated_at=datetime.datetime.now(tz=datetime.UTC),
+                uuid=command.uuid,
+            )
+
+    def _signal_handler(self, recv_signal, frame):
+        LOG.warning('Received interrupt signal: %s (frame: %s)',
+                    recv_signal, frame)
+        self._interrupted = True

dsw/command_queue/query.py

@@ -0,0 +1,74 @@
+import enum
+
+
+class CommandState(enum.Enum):
+    NEW = 'NewPersistentCommandState'
+    DONE = 'DonePersistentCommandState'
+    ERROR = 'ErrorPersistentCommandState'
+    IGNORE = 'IgnorePersistentCommandState'
+
+
+class CommandQueries:
+
+    def __init__(self, channel: str):
+        self.channel = channel
+
+    def query_listen(self) -> str:
+        return f'LISTEN persistent_command_channel__{self.channel};'
+
+    def query_get_command(self) -> str:
+        return """
+            SELECT *
+            FROM persistent_command
+            WHERE component = %(component)s
+              AND attempts < max_attempts
+              AND state != 'DonePersistentCommandState'
+              AND state != 'IgnorePersistentCommandState'
+              AND (created_at AT TIME ZONE 'UTC')
+                    <
+                  (%(now)s - (2 ^ attempts - 1) * INTERVAL '1 min')
+            ORDER BY attempts ASC, updated_at DESC
+            LIMIT 1 FOR UPDATE SKIP LOCKED;
+        """
+
+    @staticmethod
+    def query_command_error() -> str:
+        return """
+            UPDATE persistent_command
+            SET attempts = %(attempts)s,
+                last_error_message = %(error_message)s,
+                state = 'ErrorPersistentCommandState',
+                updated_at = %(updated_at)s
+            WHERE uuid = %(uuid)s;
+        """
+
+    @staticmethod
+    def query_command_error_stop() -> str:
+        return """
+            UPDATE persistent_command
+            SET attempts = %(attempts)s,
+                max_attempts = %(attempts)s,
+                last_error_message = %(error_message)s,
+                state = 'ErrorPersistentCommandState',
+                updated_at = %(updated_at)s
+            WHERE uuid = %(uuid)s;
+        """
+
+    @staticmethod
+    def query_command_done() -> str:
+        return """
+            UPDATE persistent_command
+            SET attempts = %(attempts)s,
+                state = 'DonePersistentCommandState',
+                updated_at = %(updated_at)s
+            WHERE uuid = %(uuid)s;
+        """
+
+    @staticmethod
+    def query_command_start() -> str:
+        return """
+            UPDATE persistent_command
+            SET attempts = %(attempts)s,
+                updated_at = %(updated_at)s
+            WHERE uuid = %(uuid)s;
+        """

dsw_command_queue-4.27.0.dist-info/METADATA

@@ -0,0 +1,44 @@
+Metadata-Version: 2.3
+Name: dsw-command-queue
+Version: 4.27.0
+Summary: Library for working with command queue and persistent commands
+Keywords: dsw,subscriber,publisher,database,queue,processing
+Author: Marek Suchánek
+Author-email: Marek Suchánek <marek.suchanek@ds-wizard.org>
+License: Apache License 2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Requires-Dist: func-timeout
+Requires-Dist: dsw-database==4.27.0
+Requires-Python: >=3.12, <4
+Project-URL: Homepage, https://ds-wizard.org
+Project-URL: Repository, https://github.com/ds-wizard/engine-tools
+Project-URL: Documentation, https://guide.ds-wizard.org
+Project-URL: Issues, https://github.com/ds-wizard/ds-wizard/issues
+Description-Content-Type: text/markdown
+
+# Data Stewardship Wizard: Command Queue
+
+[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/ds-wizard/engine-tools)](https://github.com/ds-wizard/engine-tools/releases)
+[![PyPI](https://img.shields.io/pypi/v/dsw-command-queue)](https://pypi.org/project/dsw-command-queue/)
+[![LICENSE](https://img.shields.io/github/license/ds-wizard/engine-tools)](LICENSE)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4975/badge)](https://bestpractices.coreinfrastructure.org/projects/4975)
+[![Python Version](https://img.shields.io/badge/Python-%E2%89%A5%203.7-blue)](https://python.org)
+
+*Library for working with DSW command queue and persistent commands*
+
+## Usage
+
+Currently, this library is intended for internal use of DSW tooling only. 
+Enhancements for use in custom scripts are planned for future development.
+
+## License
+
+This project is licensed under the Apache License v2.0 - see the
+[LICENSE](LICENSE) file for more details.

dsw_command_queue-4.27.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+dsw/command_queue/__init__.py,sha256=JmYcLGvSusyDAhoKwM9u5VgK2Zzv7h4MrMiThOuMRJ4,137
+dsw/command_queue/build_info.py,sha256=HmTWdilH1Z3vzqKd83M66jFCU1MwfhLHj2ugpm84buA,381
+dsw/command_queue/command_queue.py,sha256=l1U-8hvxf3hWdKq8kw5QNfiPUCfKNIIoi3jizYZMhx8,9090
+dsw/command_queue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dsw/command_queue/query.py,sha256=s1KlaU6ttFt8ily2xykfCGZHoUYOnLQi4h72Cz7cBzw,2281
+dsw_command_queue-4.27.0.dist-info/WHEEL,sha256=fAguSjoiATBe7TNBkJwOjyL1Tt4wwiaQGtNtjRPNMQA,80
+dsw_command_queue-4.27.0.dist-info/METADATA,sha256=LnIbJVIi37lmR0dLKWtwzDuafZJhuskIC_cZfxuGxBw,2019
+dsw_command_queue-4.27.0.dist-info/RECORD,,

dsw_command_queue-4.27.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.28
+Root-Is-Purelib: true
+Tag: py3-none-any