queuerPy 0.19.0__tar.gz → 0.21.0__tar.gz

{queuerpy-0.19.0/queuerPy.egg-info → queuerpy-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.19.0
+Version: 0.21.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -454,6 +454,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.19.0 → queuerpy-0.21.0}/README.md

@@ -411,6 +411,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.19.0 → queuerpy-0.21.0}/_version.py

@@ -1,6 +1,6 @@
 """Version information for queuerPy package."""
 
-__version__ = "0.19.0"
+__version__ = "0.21.0"
 __version_info__ = tuple(int(part) for part in __version__.split("."))
 
 # For compatibility

{queuerpy-0.19.0 → queuerpy-0.21.0}/core/runner.py

@@ -15,115 +15,170 @@ from ..helper.logging import get_logger
 logger = get_logger(__name__)
 
 
-class Runner(multiprocessing.Process):
+class Runner:
     """
     The core execution environment. Manages the execution of a single task
-    in a separate, reliably killable multiprocessing context, and retrieves its result.
+    in a multiprocessing pool for efficient process reuse.
+
+    Supports cancellation with automatic periodic checking:
+    - Async tasks are monitored every 0.1s and can be cancelled mid-execution
+    - Sync tasks can only be cancelled before they start (Python limitation)
 
     :param task: The synchronous or asynchronous function to execute.
     :param args: Arguments to pass to the task function.
-    :param name_prefix: Prefix for the process name (e.g., "Job-Runner").
+    :param pool: Multiprocessing.Pool for process reuse (required).
+    :param kwargs: Keyword arguments to pass to the task function.
     """
 
     def __init__(
         self,
         task: Callable[..., Any],
         *args: Any,
+        pool: Any,
         **kwargs: Any,
     ):
         """
-        Initializes the Runner process.
+        Initializes the Runner.
 
         :param task: The synchronous or asynchronous function to execute.
         :param args: Arguments to pass to the task function.
+        :param pool: Multiprocessing.Pool for process reuse (required).
+        :param kwargs: Keyword arguments to pass to the task function.
         """
-        super().__init__(name=task.__name__, daemon=True)
         self.task = task
         self.args = args
         self.kwargs = kwargs
         self.is_async = inspect.iscoroutinefunction(task)
-        self._result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue(1)
-
-    def go(self) -> None:
-        """Starts the Runner process in the background."""
-        self.start()
+        self.name = task.__name__
+        self._pool = pool
+        self._async_result: Optional[Any] = None
+        # Use Manager Event for cross-process sharing (pickleable)
+        manager = multiprocessing.Manager()
+        self._cancel_event = manager.Event()
+
+    @property
+    def pid(self) -> Optional[int]:
+        """Process ID (not available in pool mode)."""
+        return None
+
+    @property
+    def daemon(self) -> bool:
+        """Daemon status (always True for pool workers)."""
+        return True
+
+    @property
+    def exitcode(self) -> Optional[int]:
+        """Exit code (not available in pool mode)."""
+        return None
+
+    @staticmethod
+    def _execute_in_pool_worker(
+        task: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        is_async: bool,
+        cancel_event: Any,
+    ) -> Any:
+        """
+        Static method to execute tasks in pool workers.
+        Handles both sync and async tasks, returns result directly.
+        Checks for cancellation before and during execution.
+        """
+        # Check if task was cancelled before execution
+        if cancel_event.is_set():
+            raise asyncio.CancelledError("Task was cancelled before execution")
+
+        if is_async:
+            # For async tasks, monitor cancellation during execution
+            result = asyncio.run(
+                Runner._run_async_with_cancellation(task, args, kwargs, cancel_event)
+            )
+        else:
+            # For sync tasks, just execute (can't interrupt mid-execution easily)
+            result = task(*args, **kwargs)
+        return result
 
-    def run(self) -> None:
+    @staticmethod
+    async def _run_async_with_cancellation(
+        task: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        cancel_event: Any,
+    ) -> Any:
         """
-        The main execution method, executed in the child process.
-        It executes the assigned task, sends the result to the queue, and exits.
+        Run async task with periodic cancellation checking.
+        Monitors cancel_event and cancels the task if it's set.
         """
+        # Create the task
+        task_coro = task(*args, **kwargs)
+        task_obj = asyncio.create_task(task_coro)
+
+        # Monitor for cancellation every second
+        async def monitor_cancellation():
+            while not task_obj.done():
+                if cancel_event.is_set():
+                    task_obj.cancel()
+                    break
+                await asyncio.sleep(1)
+
+        monitor = asyncio.create_task(monitor_cancellation())
         try:
-            if self.is_async:
-                result = asyncio.run(self.task(*self.args, **self.kwargs))
-            else:
-                result = self.task(*self.args, **self.kwargs)
-
-            try:
-                self._result_queue.put(result)
-            except BrokenPipeError as e:
-                pass
-        except Exception as e:
-            logger.debug(traceback.format_exc())
+            result = await task_obj
+            monitor.cancel()
+            return result
+        except asyncio.CancelledError:
+            monitor.cancel()
+            raise asyncio.CancelledError("Task was cancelled during execution")
+        finally:
             try:
-                self._result_queue.put(e)
-            except BrokenPipeError as e:
+                await monitor
+            except asyncio.CancelledError:
                 pass
 
+    def go(self) -> None:
+        """Starts the Runner in the background using the pool."""
+        self._async_result = self._pool.apply_async(
+            Runner._execute_in_pool_worker,
+            (self.task, self.args, self.kwargs, self.is_async, self._cancel_event),
+        )
+
     def get_results(self, timeout: Optional[float] = None) -> Any:
         """
-        Waits for the runner process to complete and returns the result.
+        Waits for the runner to complete and returns the result.
 
         :param timeout: Time in seconds to wait for the result. If None, waits indefinitely.
         :returns: The result returned by the executed task function.
         :raises TimeoutError: If the result is not available within the specified timeout.
-        :raises Exception: If the task execution in the runner process failed.
+        :raises asyncio.CancelledError: If the task was cancelled.
+        :raises Exception: If the task execution failed.
         """
         logger.debug(f"Runner.get_results() called for {self.name}, timeout={timeout}")
+        assert (
+            self._async_result is not None
+        ), "go() must be called before get_results()"
         try:
-            self.join(timeout=timeout)
-
-            if self.is_alive():
-                raise TimeoutError(f"runner timed out after {timeout} seconds.")
-
-            if self._result_queue.empty():
-                if self.exitcode == 0:
-                    return None
-                else:
-                    raise Exception(f"runner failed with code {self.exitcode}")
-
-            try:
-                result = self._result_queue.get_nowait()
-            except Exception as e:
-                raise
-
-            if isinstance(result, Exception):
-                raise result
-
+            result = self._async_result.get(timeout=timeout)
             return result
-        except Exception as e:
-            raise e
+        except multiprocessing.TimeoutError:
+            raise TimeoutError(f"runner timed out after {timeout} seconds.")
+        except asyncio.CancelledError:
+            logger.debug(f"Runner {self.name} was cancelled")
+            raise
 
     def cancel(self) -> bool:
         """
-        Attempts to cancel the running process.
+        Attempts to cancel the running task by setting the cancellation event.
 
-        :returns: True if the process was successfully terminated, False otherwise.
-        :raises: Exception if an error occurs during termination.
-        """
-        try:
-            if not self.is_alive():
-                logger.debug(f"Runner {self.name}: Already finished")
-                return True
+        Cancellation behavior:
+        - Async tasks: Checked every second during execution, task will be cancelled
+        - Sync tasks: Only checked before execution starts (cannot interrupt mid-execution)
+        - Queued tasks: Cancelled before they begin executing
 
-            logger.debug(f"Cancelling runner {self.name}")
-            self.terminate()
-            self.join(timeout=5.0)  # Wait up to 5 seconds for clean termination
-            result = not self.is_alive()
-            return result
-        except Exception as e:
-            logger.warning(f"Error cancelling runner {self.name}: {e}")
-            return False
+        :returns: True if cancellation signal was sent successfully.
+        """
+        logger.debug(f"Cancelling runner {self.name}")
+        self._cancel_event.set()
+        return True
 
 
 class SmallRunner(threading.Thread):
@@ -253,7 +308,11 @@ class SmallRunner(threading.Thread):
 
 
 def go_func(
-    func: Callable[..., Any], use_mp: bool = True, *args: Any, **kwargs: Any
+    func: Callable[..., Any],
+    use_mp: bool = True,
+    *args: Any,
+    pool: Any = None,
+    **kwargs: Any,
 ) -> Union[Runner, SmallRunner]:
     """
     Go-like async function execution factory. Selects the appropriate runner.
@@ -262,11 +321,14 @@ def go_func(
     :param use_mp: If True (default), uses the multiprocessing Runner (isolated).
                    If False, uses the threading SmallRunner (shared state).
     :param args: Positional arguments for the function.
+    :param pool: Multiprocessing.Pool for process reuse (required if use_mp=True).
     :param kwargs: Keyword arguments for the function.
     :returns: A running Runner or SmallRunner instance.
     """
     if use_mp:
-        runner = Runner(func, *args, **kwargs)
+        if pool is None:
+            raise ValueError("pool is required when use_mp=True")
+        runner = Runner(func, *args, pool=pool, **kwargs)
     else:
         runner = SmallRunner(func, *args, **kwargs)
 

{queuerpy-0.19.0 → queuerpy-0.21.0}/database/db_job.py

@@ -9,7 +9,7 @@ from typing import List, Optional, Tuple
 from uuid import UUID
 from psycopg.rows import dict_row
 
-from ..helper.database import Database
+from ..helper.database import Database, DatabaseConfiguration
 from ..helper.error import QueuerError
 from ..helper.sql import SQLLoader, run_ddl
 from ..model.job import Job
@@ -25,7 +25,7 @@ class JobDBHandler:
     def __init__(
         self,
         db_connection: Database,
-        with_table_drop: bool = False,
+        db_config: DatabaseConfiguration,
         encryption_key: str = "",
     ):
         """Initialize job database handler."""
@@ -35,12 +35,13 @@ class JobDBHandler:
             raise ValueError("Database connection is not established")
 
         self.encryption_key: str = encryption_key
+        self.with_timescale: bool = db_config.with_timescale
 
         sql_loader: SQLLoader = SQLLoader()
-        sql_loader.load_notify_sql(self.db.instance, force=with_table_drop)
-        sql_loader.load_job_sql(self.db.instance, force=with_table_drop)
+        sql_loader.load_notify_sql(self.db.instance, force=db_config.with_table_drop)
+        sql_loader.load_job_sql(self.db.instance, force=db_config.with_table_drop)
 
-        if with_table_drop:
+        if db_config.with_table_drop:
             self.drop_tables()
 
         self.create_table()
@@ -72,7 +73,7 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        run_ddl(self.db.instance, "SELECT init_job();")
+        run_ddl(self.db.instance, f"SELECT init_job({self.with_timescale});")
 
     def drop_tables(self) -> None:
         """Drop job tables with DDL deadlock protection."""
@@ -379,9 +380,15 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor() as cur:
-            cur.execute("SELECT add_retention_archive(%s);", (days,))
-        self.db.instance.commit()
+        try:
+            with self.db.instance.cursor() as cur:
+                cur.execute(
+                    "SELECT add_retention_archive(%s, %s);", (days, self.with_timescale)
+                )
+            self.db.instance.commit()
+        except Exception as e:
+            self.db.instance.rollback()
+            raise e
 
     def remove_retention_archive(self) -> None:
         """
@@ -391,9 +398,15 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor() as cur:
-            cur.execute("SELECT remove_retention_archive();")
-        self.db.instance.commit()
+        try:
+            with self.db.instance.cursor() as cur:
+                cur.execute(
+                    "SELECT remove_retention_archive(%s);", (self.with_timescale,)
+                )
+            self.db.instance.commit()
+        except Exception as e:
+            self.db.instance.rollback()
+            raise e
 
     def select_job_from_archive(self, rid: UUID) -> Optional[Job]:
         """

{queuerpy-0.19.0 → queuerpy-0.21.0}/helper/database.py

@@ -30,6 +30,7 @@ class DatabaseConfiguration:
     schema: str = "public"
     sslmode: str = "require"
     with_table_drop: bool = False
+    with_timescale: bool = True
 
     @classmethod
     def from_env(cls) -> "DatabaseConfiguration":
@@ -45,6 +46,9 @@ class DatabaseConfiguration:
         with_table_drop = (
             os.getenv("QUEUER_DB_WITH_TABLE_DROP", "false").lower() == "true"
         )
+        with_timescale = (
+            os.getenv("QUEUER_DB_WITH_TIMESCALE", "true").lower() != "false"
+        )
 
         # Validate required fields
         if not all(
@@ -72,6 +76,7 @@ class DatabaseConfiguration:
             schema=schema,
             sslmode=sslmode,
             with_table_drop=with_table_drop,
+            with_timescale=with_timescale,
         )
 
     def connection_string(self) -> str:

{queuerpy-0.19.0 → queuerpy-0.21.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "queuerPy"
-version = "0.19.0"
+version = "0.21.0"
 description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
 readme = "README.md"
 license = "Apache-2.0"
@@ -92,7 +92,7 @@ pythonVersion = "3.8"
 pythonPlatform = "All"
 
 [tool.mypy]
-python_version = "0.19.0"
+python_version = "0.21.0"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{queuerpy-0.19.0 → queuerpy-0.21.0}/queuer.py

@@ -6,6 +6,7 @@ Mirrors the Go Queuer struct with Python async patterns.
 # Standard library imports
 import asyncio
 import json
+import multiprocessing
 import os
 import threading
 import time
@@ -121,6 +122,9 @@ class Queuer(
         # Active runners (jobs currently executing)
         self.active_runners: Dict[UUID, Runner] = {}
 
+        # Set pool size based on max_concurrency
+        self._pool_size = max_concurrency if max_concurrency > 0 else 4
+
         # Database listeners (will be initialized in start())
         self.job_db_listener: Optional[QueuerListener] = None
         self.job_archive_db_listener: Optional[QueuerListener] = None
@@ -170,6 +174,17 @@ class Queuer(
         self._stopped.clear()
         self._cancel_event = asyncio.Event()
 
+        # Create process pool for job execution
+        try:
+            self._process_pool = multiprocessing.Pool(
+                processes=self._pool_size,
+                maxtasksperchild=100,
+            )
+            logger.info(f"Created process pool with {self._pool_size} workers")
+        except Exception as e:
+            logger.error(f"Failed to create process pool: {e}")
+            raise
+
         # Set up database listeners
         try:
             self.job_db_listener = new_queuer_db_listener(
@@ -293,9 +308,25 @@ class Queuer(
 
         # Cancel all active runners
         for runner_id, runner in list(self.active_runners.items()):
-            runner.cancel()
+            cancelled = runner.cancel()
+            if not cancelled:
+                # Pool-based runner, need to restart pool
+                logger.warning("Pool-based runner cannot be individually cancelled")
             del self.active_runners[runner_id]
 
+        # Clean up process pool
+        if self._process_pool:
+            logger.info("Closing process pool")
+            try:
+                self._process_pool.close()
+                self._process_pool.join(timeout=5.0)
+                # Force terminate if workers didn't finish gracefully
+                self._process_pool.terminate()
+                self._process_pool.join()
+                self._process_pool = None
+            except Exception as e:
+                logger.warning(f"Error closing process pool: {e}")
+
         # Cancel the context to stop the queuer
         if self._cancel_event:
             self._cancel_event.set()

{queuerpy-0.19.0 → queuerpy-0.21.0/queuerPy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.19.0
+Version: 0.21.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -454,6 +454,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.19.0 → queuerpy-0.21.0}/queuer_global.py

@@ -1,3 +1,4 @@
+import multiprocessing
 import threading
 from typing import Any, Callable, Dict, Optional
 
@@ -45,6 +46,10 @@ class QueuerGlobalMixin:
         self.tasks: Dict[str, Task] = {}
         self.next_interval_funcs: Dict[str, Callable[..., Any]] = {}
 
+        # Process pool for runner reuse (initialized in start())
+        self._process_pool: Optional[Any] = None
+        self._pool_size: int = 4  # Default, will be set based on max_concurrency
+
     def initialise(
         self,
         db_config: DatabaseConfiguration,
@@ -56,7 +61,7 @@ class QueuerGlobalMixin:
 
         # Database handlers
         self.db_job: JobDBHandler = JobDBHandler(
-            self.database, db_config.with_table_drop, encryption_key
+            self.database, db_config, encryption_key
         )
         self.db_worker: WorkerDBHandler = WorkerDBHandler(
             self.database, db_config.with_table_drop

{queuerpy-0.19.0 → queuerpy-0.21.0}/queuer_job.py

@@ -110,6 +110,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
         runner = Runner(
             task=self._wait_for_job_added_inner,
             args=(timeout_seconds,),
+            pool=self._process_pool,
             kwargs={},
         )
         runner.go()
@@ -421,7 +422,9 @@ class QueuerJobMixin(QueuerGlobalMixin):
         try:
             parameters = getattr(job, "parameters", [])
             parameters_keyed = getattr(job, "parameters_keyed", {})
-            runner = Runner(task, *parameters, **parameters_keyed)
+            runner = Runner(
+                task, *parameters, pool=self._process_pool, **parameters_keyed
+            )
 
             logger.info(f"Created runner for job {job.rid}")
 
@@ -507,9 +510,9 @@ class QueuerJobMixin(QueuerGlobalMixin):
         """
         try:
             # Create a Runner process for the async job
-            runner = Runner(self._run_job, job)
+            runner = Runner(self._run_job, job, pool=self._process_pool)
             runner.go()
-            logger.info(f"Scheduled job {job.rid} started in process {runner.pid}")
+            logger.info(f"Scheduled job {job.rid} started")
 
         except Exception as e:
             logger.error(f"Error executing scheduled job {job.rid}: {e}")

{queuerpy-0.19.0 → queuerpy-0.21.0}/queuer_master.py

@@ -38,14 +38,27 @@ class QueuerMasterMixin(QueuerGlobalMixin):
 
         # Handle retention archive configuration
         try:
-            if old_master.settings.retention_archive == 0:
-                self.db_job.add_retention_archive(master_settings.retention_archive)
-            elif (
-                old_master.settings.retention_archive
-                != master_settings.retention_archive
-            ):
-                self.db_job.remove_retention_archive()
-                self.db_job.add_retention_archive(master_settings.retention_archive)
+            if master_settings.retention_archive > 0:
+                if old_master.settings.retention_archive == 0:
+                    self.db_job.add_retention_archive(master_settings.retention_archive)
+                elif (
+                    old_master.settings.retention_archive
+                    != master_settings.retention_archive
+                ):
+                    try:
+                        self.db_job.remove_retention_archive()
+                    except Exception as e:
+                        logger.warning(
+                            f"Could not remove old retention policy (may not exist): {e}"
+                        )
+                    self.db_job.add_retention_archive(master_settings.retention_archive)
+            elif old_master.settings.retention_archive > 0:
+                try:
+                    self.db_job.remove_retention_archive()
+                except Exception as e:
+                    logger.warning(
+                        f"Could not remove retention policy (may not exist): {e}"
+                    )
         except Exception as e:
             raise QueuerError("managing retention archive", e)
 
@@ -83,10 +96,9 @@ class QueuerMasterMixin(QueuerGlobalMixin):
                     logger.error(f"Error in master ticker task: {e}")
 
             ticker = Ticker(
-                task=ticker_task,
-                name="_master_ticker",
                 interval=timedelta(seconds=master_settings.master_poll_interval),
-                func=ticker_task,
+                task=ticker_task,
+                use_mp=False,  # Use threading instead of multiprocessing to avoid pickling issues
             )
             logger.info("Starting master ticker...")
             ticker.go()
@@ -150,10 +162,9 @@ class QueuerMasterMixin(QueuerGlobalMixin):
 
             # Create and start the polling ticker
             ticker = Ticker(
-                task=ticker_task,
-                name="_poll_master_ticker",
                 interval=timedelta(seconds=master_settings.master_poll_interval),
-                func=ticker_task,
+                task=ticker_task,
+                use_mp=False,  # Use threading instead of multiprocessing to avoid pickling issues
             )
 
             logger.info("Starting master poll ticker...")

{queuerpy-0.19.0 → queuerpy-0.21.0}/queuer_worker.py

@@ -15,6 +15,19 @@ from .queuer_global import QueuerGlobalMixin
 logger = logging.getLogger(__name__)
 
 
+def get_current_worker_rid(self) -> Optional[UUID]:
+    """
+    Get the current worker's RID safely with mutex locking.
+    Mirrors Go's GetCurrentWorkerRID method.
+
+    :return: Worker RID or None if no worker
+    """
+    with self.worker_mutex:
+        if self.worker is not None:
+            return self.worker.rid
+        return None
+
+
 class QueuerWorkerMixin(QueuerGlobalMixin):
     """
     Mixin class containing worker-related methods for the Queuer.

{queuerpy-0.19.0 → queuerpy-0.21.0}/sql/job.sql

@@ -1,7 +1,9 @@
-CREATE OR REPLACE FUNCTION init_job() RETURNS VOID AS $$
+CREATE OR REPLACE FUNCTION init_job(enable_timescaledb BOOLEAN DEFAULT TRUE) RETURNS VOID AS $$
 BEGIN
     -- Create TimescaleDB extension
-    CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+    IF enable_timescaledb THEN
+        CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+    END IF;
 
     -- Create pgcrypto extension
     CREATE EXTENSION IF NOT EXISTS pgcrypto;
@@ -37,7 +39,9 @@ BEGIN
     );
     
     -- Create hypertable (TimescaleDB feature)
-    PERFORM create_hypertable('job_archive', by_range('updated_at'), if_not_exists => TRUE);
+    IF enable_timescaledb THEN
+        PERFORM create_hypertable('job_archive', by_range('updated_at'), if_not_exists => TRUE);
+    END IF;
     
     -- Create triggers for notifications
     CREATE OR REPLACE TRIGGER job_notify_event
@@ -698,17 +702,64 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION add_retention_archive(input_retention_days INT)
+CREATE OR REPLACE FUNCTION add_retention_archive(input_retention_days INT, enable_timescaledb BOOLEAN DEFAULT TRUE)
 RETURNS VOID AS $$
 BEGIN
-    PERFORM add_retention_policy('job_archive', (input_retention_days * INTERVAL '1 day'));
+    IF enable_timescaledb THEN
+        -- Use TimescaleDB retention policy
+        PERFORM add_retention_policy('job_archive', (input_retention_days * INTERVAL '1 day'));
+    ELSE
+        -- Create sequence for tracking inserts
+        CREATE SEQUENCE IF NOT EXISTS job_archive_insert_counter;
+        
+        -- Create trigger function that deletes old records every 10th insert
+        EXECUTE format('
+            CREATE OR REPLACE FUNCTION cleanup_job_archive_trigger()
+            RETURNS TRIGGER AS $func$
+            DECLARE
+                insert_count BIGINT;
+                deleted_count INT;
+            BEGIN
+                -- Only run cleanup every 10th insert
+                insert_count := nextval(''job_archive_insert_counter'');
+                IF insert_count %% 10 = 0 THEN
+                    DELETE FROM job_archive
+                    WHERE updated_at < (CURRENT_TIMESTAMP - INTERVAL ''%s days'');
+                    
+                    GET DIAGNOSTICS deleted_count = ROW_COUNT;
+                    
+                    -- Log cleanup if any rows were deleted (optional)
+                    IF deleted_count > 0 THEN
+                        RAISE NOTICE ''Cleaned up %% old job_archive records'', deleted_count;
+                    END IF;
+                END IF;
+                RETURN NEW;
+            END;
+            $func$ LANGUAGE plpgsql;
+        ', input_retention_days);
+        
+        -- Create trigger on job_archive
+        DROP TRIGGER IF EXISTS job_archive_cleanup_trigger ON job_archive;
+        CREATE TRIGGER job_archive_cleanup_trigger
+            AFTER INSERT ON job_archive
+            FOR EACH ROW
+            EXECUTE FUNCTION cleanup_job_archive_trigger();
+    END IF;
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION remove_retention_archive()
+CREATE OR REPLACE FUNCTION remove_retention_archive(enable_timescaledb BOOLEAN DEFAULT TRUE)
 RETURNS VOID AS $$
 BEGIN
-    PERFORM remove_retention_policy('job_archive');
+    IF enable_timescaledb THEN
+        -- Remove TimescaleDB retention policy
+        PERFORM remove_retention_policy('job_archive');
+    ELSE
+        -- Drop trigger and related objects
+        DROP TRIGGER IF EXISTS job_archive_cleanup_trigger ON job_archive;
+        DROP FUNCTION IF EXISTS cleanup_job_archive_trigger();
+        DROP SEQUENCE IF EXISTS job_archive_insert_counter;
+    END IF;
 END;
 $$ LANGUAGE plpgsql;