queuerPy 0.20.0__tar.gz → 0.22.0__tar.gz

{queuerpy-0.20.0/queuerPy.egg-info → queuerpy-0.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.20.0
+Version: 0.22.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -43,7 +43,7 @@ Dynamic: license-file
 
 # queuerPy
 
-[![PyPi](https://pythonico.leapcell.app/pypi/queuerPy.svg?style=shields&data=n,v,d)](https://pypi.org/project/queuerPy/)
+[![PyPi](https://img.shields.io/pypi/v/queuerPy.svg)](https://pypi.org/project/queuerPy)
 [![Python](https://img.shields.io/badge/Python-3.10+-blue.svg)](https://www.python.org/)
 ![Coverage](https://raw.githubusercontent.com/siherrmann/queuerPy/refs/heads/main/coverage-badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/siherrmann/queuer/blob/master/LICENSE)
@@ -454,6 +454,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.20.0 → queuerpy-0.22.0}/README.md

@@ -1,6 +1,6 @@
 # queuerPy
 
-[![PyPi](https://pythonico.leapcell.app/pypi/queuerPy.svg?style=shields&data=n,v,d)](https://pypi.org/project/queuerPy/)
+[![PyPi](https://img.shields.io/pypi/v/queuerPy.svg)](https://pypi.org/project/queuerPy)
 [![Python](https://img.shields.io/badge/Python-3.10+-blue.svg)](https://www.python.org/)
 ![Coverage](https://raw.githubusercontent.com/siherrmann/queuerPy/refs/heads/main/coverage-badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/siherrmann/queuer/blob/master/LICENSE)
@@ -411,6 +411,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.20.0 → queuerpy-0.22.0}/_version.py

@@ -1,6 +1,6 @@
 """Version information for queuerPy package."""
 
-__version__ = "0.20.0"
+__version__ = "0.22.0"
 __version_info__ = tuple(int(part) for part in __version__.split("."))
 
 # For compatibility

queuerpy-0.22.0/core/context.py

@@ -0,0 +1,11 @@
+"""
+Context management for the queuer implementation.
+Provides a way to store tracking information across asynchronous and concurrent flows.
+"""
+
+from contextvars import ContextVar
+from typing import Optional
+from uuid import UUID
+
+# Context variable to track the RID of the currently executing job.
+current_job_rid: ContextVar[Optional[UUID]] = ContextVar("current_job_rid", default=None)

{queuerpy-0.20.0 → queuerpy-0.22.0}/core/runner.py

@@ -15,115 +15,177 @@ from ..helper.logging import get_logger
 logger = get_logger(__name__)
 
 
-class Runner(multiprocessing.Process):
+class Runner:
     """
     The core execution environment. Manages the execution of a single task
-    in a separate, reliably killable multiprocessing context, and retrieves its result.
+    in a multiprocessing pool for efficient process reuse.
+
+    Supports cancellation with automatic periodic checking:
+    - Async tasks are monitored every 0.1s and can be cancelled mid-execution
+    - Sync tasks can only be cancelled before they start (Python limitation)
 
     :param task: The synchronous or asynchronous function to execute.
     :param args: Arguments to pass to the task function.
-    :param name_prefix: Prefix for the process name (e.g., "Job-Runner").
+    :param pool: Multiprocessing.Pool for process reuse (required).
+    :param setup_func: Optional callable to run before the task (e.g., for setting ContextVars).
+    :param kwargs: Keyword arguments to pass to the task function.
     """
 
     def __init__(
         self,
         task: Callable[..., Any],
         *args: Any,
+        pool: Any,
+        setup_func: Optional[Callable[[], None]] = None,
         **kwargs: Any,
     ):
         """
-        Initializes the Runner process.
+        Initializes the Runner.
 
         :param task: The synchronous or asynchronous function to execute.
         :param args: Arguments to pass to the task function.
+        :param pool: Multiprocessing.Pool for process reuse (required).
+        :param kwargs: Keyword arguments to pass to the task function.
         """
-        super().__init__(name=task.__name__, daemon=True)
         self.task = task
         self.args = args
         self.kwargs = kwargs
         self.is_async = inspect.iscoroutinefunction(task)
-        self._result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue(1)
+        self.name = task.__name__
+        self._pool = pool
+        self._setup_func = setup_func
+        self._async_result: Optional[Any] = None
+        # Use Manager Event for cross-process sharing (pickleable)
+        manager = multiprocessing.Manager()
+        self._cancel_event = manager.Event()
+
+    @property
+    def pid(self) -> Optional[int]:
+        """Process ID (not available in pool mode)."""
+        return None
+
+    @property
+    def daemon(self) -> bool:
+        """Daemon status (always True for pool workers)."""
+        return True
+
+    @property
+    def exitcode(self) -> Optional[int]:
+        """Exit code (not available in pool mode)."""
+        return None
+
+    @staticmethod
+    def _execute_in_pool_worker(
+        task: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        is_async: bool,
+        cancel_event: Any,
+        setup_func: Optional[Callable[[], None]] = None,
+    ) -> Any:
+        """
+        Static method to execute tasks in pool workers.
+        Handles both sync and async tasks, returns result directly.
+        Checks for cancellation before and during execution.
+        """
+        # Check if task was cancelled before execution
+        if cancel_event.is_set():
+            raise asyncio.CancelledError("Task was cancelled before execution")
 
-    def go(self) -> None:
-        """Starts the Runner process in the background."""
-        self.start()
+        if setup_func is not None:
+            setup_func()
 
-    def run(self) -> None:
+        if is_async:
+            # For async tasks, monitor cancellation during execution
+            result = asyncio.run(
+                Runner._run_async_with_cancellation(task, args, kwargs, cancel_event)
+            )
+        else:
+            # For sync tasks, just execute (can't interrupt mid-execution easily)
+            result = task(*args, **kwargs)
+        return result
+
+    @staticmethod
+    async def _run_async_with_cancellation(
+        task: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        cancel_event: Any,
+    ) -> Any:
         """
-        The main execution method, executed in the child process.
-        It executes the assigned task, sends the result to the queue, and exits.
+        Run async task with periodic cancellation checking.
+        Monitors cancel_event and cancels the task if it's set.
         """
+        # Create the task
+        task_coro = task(*args, **kwargs)
+        task_obj = asyncio.create_task(task_coro)
+
+        # Monitor for cancellation every second
+        async def monitor_cancellation():
+            while not task_obj.done():
+                if cancel_event.is_set():
+                    task_obj.cancel()
+                    break
+                await asyncio.sleep(1)
+
+        monitor = asyncio.create_task(monitor_cancellation())
         try:
-            if self.is_async:
-                result = asyncio.run(self.task(*self.args, **self.kwargs))
-            else:
-                result = self.task(*self.args, **self.kwargs)
-
-            try:
-                self._result_queue.put(result)
-            except BrokenPipeError as e:
-                pass
-        except Exception as e:
-            logger.debug(traceback.format_exc())
+            result = await task_obj
+            monitor.cancel()
+            return result
+        except asyncio.CancelledError:
+            monitor.cancel()
+            raise asyncio.CancelledError("Task was cancelled during execution")
+        finally:
             try:
-                self._result_queue.put(e)
-            except BrokenPipeError as e:
+                await monitor
+            except asyncio.CancelledError:
                 pass
 
+    def go(self) -> None:
+        """Starts the Runner in the background using the pool."""
+        self._async_result = self._pool.apply_async(
+            Runner._execute_in_pool_worker,
+            (self.task, self.args, self.kwargs, self.is_async, self._cancel_event, self._setup_func),
+        )
+
     def get_results(self, timeout: Optional[float] = None) -> Any:
         """
-        Waits for the runner process to complete and returns the result.
+        Waits for the runner to complete and returns the result.
 
         :param timeout: Time in seconds to wait for the result. If None, waits indefinitely.
         :returns: The result returned by the executed task function.
         :raises TimeoutError: If the result is not available within the specified timeout.
-        :raises Exception: If the task execution in the runner process failed.
+        :raises asyncio.CancelledError: If the task was cancelled.
+        :raises Exception: If the task execution failed.
         """
         logger.debug(f"Runner.get_results() called for {self.name}, timeout={timeout}")
+        assert (
+            self._async_result is not None
+        ), "go() must be called before get_results()"
         try:
-            self.join(timeout=timeout)
-
-            if self.is_alive():
-                raise TimeoutError(f"runner timed out after {timeout} seconds.")
-
-            if self._result_queue.empty():
-                if self.exitcode == 0:
-                    return None
-                else:
-                    raise Exception(f"runner failed with code {self.exitcode}")
-
-            try:
-                result = self._result_queue.get_nowait()
-            except Exception as e:
-                raise
-
-            if isinstance(result, Exception):
-                raise result
-
+            result = self._async_result.get(timeout=timeout)
             return result
-        except Exception as e:
-            raise e
+        except multiprocessing.TimeoutError:
+            raise TimeoutError(f"runner timed out after {timeout} seconds.")
+        except asyncio.CancelledError:
+            logger.debug(f"Runner {self.name} was cancelled")
+            raise
 
     def cancel(self) -> bool:
         """
-        Attempts to cancel the running process.
+        Attempts to cancel the running task by setting the cancellation event.
 
-        :returns: True if the process was successfully terminated, False otherwise.
-        :raises: Exception if an error occurs during termination.
-        """
-        try:
-            if not self.is_alive():
-                logger.debug(f"Runner {self.name}: Already finished")
-                return True
+        Cancellation behavior:
+        - Async tasks: Checked every second during execution, task will be cancelled
+        - Sync tasks: Only checked before execution starts (cannot interrupt mid-execution)
+        - Queued tasks: Cancelled before they begin executing
 
-            logger.debug(f"Cancelling runner {self.name}")
-            self.terminate()
-            self.join(timeout=5.0)  # Wait up to 5 seconds for clean termination
-            result = not self.is_alive()
-            return result
-        except Exception as e:
-            logger.warning(f"Error cancelling runner {self.name}: {e}")
-            return False
+        :returns: True if cancellation signal was sent successfully.
+        """
+        logger.debug(f"Cancelling runner {self.name}")
+        self._cancel_event.set()
+        return True
 
 
 class SmallRunner(threading.Thread):
@@ -253,7 +315,11 @@ class SmallRunner(threading.Thread):
 
 
 def go_func(
-    func: Callable[..., Any], use_mp: bool = True, *args: Any, **kwargs: Any
+    func: Callable[..., Any],
+    use_mp: bool = True,
+    *args: Any,
+    pool: Any = None,
+    **kwargs: Any,
 ) -> Union[Runner, SmallRunner]:
     """
     Go-like async function execution factory. Selects the appropriate runner.
@@ -262,11 +328,14 @@ def go_func(
     :param use_mp: If True (default), uses the multiprocessing Runner (isolated).
                    If False, uses the threading SmallRunner (shared state).
     :param args: Positional arguments for the function.
+    :param pool: Multiprocessing.Pool for process reuse (required if use_mp=True).
     :param kwargs: Keyword arguments for the function.
     :returns: A running Runner or SmallRunner instance.
     """
     if use_mp:
-        runner = Runner(func, *args, **kwargs)
+        if pool is None:
+            raise ValueError("pool is required when use_mp=True")
+        runner = Runner(func, *args, pool=pool, **kwargs)
     else:
         runner = SmallRunner(func, *args, **kwargs)
 

{queuerpy-0.20.0 → queuerpy-0.22.0}/database/db_job.py

@@ -186,19 +186,22 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                """
-                SELECT * FROM update_job_initial(%s);
-            """,
-                (worker.id,),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    """
+                    SELECT * FROM update_job_initial(%s);
+                """,
+                    (worker.id,),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("update jobs initial", e)
 
     def update_job_final(self, job: Job) -> Job:
         """
@@ -303,17 +306,20 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs(%s, %s, %s);",
-                (self.encryption_key or "", last_id, entries),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs(%s, %s, %s);",
+                    (self.encryption_key or "", last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs", e)
 
     def select_all_jobs_by_worker_rid(
         self, worker_rid: UUID, last_id: int = 0, entries: int = 100
@@ -330,17 +336,20 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs_by_worker_rid(%s, %s, %s, %s);",
-                (self.encryption_key or "", worker_rid, last_id, entries),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs_by_worker_rid(%s, %s, %s, %s);",
+                    (self.encryption_key or "", str(worker_rid), last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs by worker rid", e)
 
     def select_all_jobs_by_search(
         self, search: str, last_id: int = 0, entries: int = 100
@@ -358,17 +367,48 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs_by_search(%s, %s, %s, %s);",
-                (self.encryption_key or "", search, last_id, entries),
-            )
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs_by_search(%s, %s, %s, %s);",
+                    (self.encryption_key or "", search, last_id, entries),
+                )
 
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
 
-            return jobs
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs by search", e)
+
+    def select_jobs_without_worker(
+        self, last_id: int = 0, entries: int = 100
+    ) -> List[Job]:
+        """
+        Select all jobs that are not assigned to any worker.
+
+        :param last_id: Last job ID from previous page.
+        :param entries: Number of entries to retrieve.
+        :return: List of Job instances.
+        """
+        if self.db.instance is None:
+            raise ValueError("Database connection is not established")
+
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_jobs_without_worker(%s, %s, %s);",
+                    (self.encryption_key or "", last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select jobs without worker", e)
 
     def add_retention_archive(self, days: int) -> None:
         """
@@ -441,17 +481,20 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs_from_archive(%s, %s, %s);",
-                (self.encryption_key or "", last_id, entries),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs_from_archive(%s, %s, %s);",
+                    (self.encryption_key or "", last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs from archive", e)
 
     def select_all_jobs_from_archive_by_search(
         self, search: str, last_id: int = 0, entries: int = 100
@@ -468,14 +511,17 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs_from_archive_by_search(%s, %s, %s, %s);",
-                (self.encryption_key or "", search, last_id, entries),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs_from_archive_by_search(%s, %s, %s, %s);",
+                    (self.encryption_key or "", search, last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs from archive by search", e)

{queuerpy-0.20.0 → queuerpy-0.22.0}/database/db_worker.py

@@ -195,14 +195,17 @@ class WorkerDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute("SELECT * FROM select_all_workers(%s, %s);", (last_id, entries))
-
-            workers: List[Worker] = []
-            for row in cur.fetchall():
-                workers.append(Worker.from_row(row))
-
-            return workers
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute("SELECT * FROM select_all_workers(%s, %s);", (last_id, entries))
+    
+                workers: List[Worker] = []
+                for row in cur.fetchall():
+                    workers.append(Worker.from_row(row))
+    
+                return workers
+        except Exception as e:
+            raise QueuerError("select all workers", e)
 
     def select_all_workers_by_search(
         self, search: str, last_id: int = 0, entries: int = 100
@@ -215,17 +218,20 @@ class WorkerDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_workers_by_search(%s, %s, %s);",
-                (search, last_id, entries),
-            )
-
-            workers: List[Worker] = []
-            for row in cur.fetchall():
-                workers.append(Worker.from_row(row))
-
-            return workers
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_workers_by_search(%s, %s, %s);",
+                    (search, last_id, entries),
+                )
+    
+                workers: List[Worker] = []
+                for row in cur.fetchall():
+                    workers.append(Worker.from_row(row))
+    
+                return workers
+        except Exception as e:
+            raise QueuerError("select all workers by search", e)
 
     def select_all_connections(self) -> List[ConnectionModel]:
         """
@@ -235,19 +241,22 @@ class WorkerDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute("SELECT * FROM select_all_connections();")
-
-            connections: List[ConnectionModel] = []
-            for row in cur.fetchall():
-                connection = ConnectionModel(
-                    pid=row["output_pid"],
-                    database=row["output_datname"],
-                    username=row["output_usename"],
-                    application_name=row["output_application_name"],
-                    query=row["output_query"],
-                    state=row["output_state"],
-                )
-                connections.append(connection)
-
-            return connections
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute("SELECT * FROM select_all_connections();")
+    
+                connections: List[ConnectionModel] = []
+                for row in cur.fetchall():
+                    connection = ConnectionModel(
+                        pid=row["output_pid"],
+                        database=row["output_datname"],
+                        username=row["output_usename"],
+                        application_name=row["output_application_name"],
+                        query=row["output_query"],
+                        state=row["output_state"],
+                    )
+                    connections.append(connection)
+    
+                return connections
+        except Exception as e:
+            raise QueuerError("select all connections", e)

{queuerpy-0.20.0 → queuerpy-0.22.0}/helper/database.py

@@ -88,8 +88,9 @@ class DatabaseConfiguration:
             f"user={self.username} "
             f"password={self.password} "
             f"sslmode={self.sslmode} "
+            f"connect_timeout=10 "
             f"application_name=queuer "
-            f"options='-c search_path={self.schema}'"
+            f"options='-c search_path={self.schema} -c statement_timeout=30000'"
         )
 
 

{queuerpy-0.20.0 → queuerpy-0.22.0}/model/options.py

@@ -78,6 +78,7 @@ class Options:
 
     on_error: Optional[OnError] = None
     schedule: Optional[Schedule] = None
+    parent_rid: Optional[str] = None
 
     def is_valid(self) -> bool:
         """
@@ -100,6 +101,7 @@ class Options:
         return {
             "on_error": self.on_error.to_dict() if self.on_error else None,
             "schedule": self.schedule.to_dict() if self.schedule else None,
+            "parent_rid": self.parent_rid,
         }
 
     @classmethod
@@ -115,11 +117,15 @@ class Options:
             options.on_error = OnError.from_dict(data["on_error"])
         if data.get("schedule"):
             options.schedule = Schedule.from_dict(data["schedule"])
+        if data.get("parent_rid"):
+            options.parent_rid = data["parent_rid"]
         return options
 
 
 def new_options(
-    on_error: Optional[OnError] = None, schedule: Optional[Schedule] = None
+    on_error: Optional[OnError] = None,
+    schedule: Optional[Schedule] = None,
+    parent_rid: Optional[str] = None,
 ) -> Options:
     """
     Create new options. Mirrors Go's pattern.
@@ -128,4 +134,4 @@ def new_options(
     :param schedule: Schedule options.
     :return: Options instance.
     """
-    return Options(on_error=on_error, schedule=schedule)
+    return Options(on_error=on_error, schedule=schedule, parent_rid=parent_rid)

{queuerpy-0.20.0 → queuerpy-0.22.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "queuerPy"
-version = "0.20.0"
+version = "0.22.0"
 description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
 readme = "README.md"
 license = "Apache-2.0"
@@ -92,7 +92,7 @@ pythonVersion = "3.8"
 pythonPlatform = "All"
 
 [tool.mypy]
-python_version = "0.20.0"
+python_version = "0.22.0"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{queuerpy-0.20.0 → queuerpy-0.22.0}/queuer.py

@@ -6,6 +6,7 @@ Mirrors the Go Queuer struct with Python async patterns.
 # Standard library imports
 import asyncio
 import json
+import multiprocessing
 import os
 import threading
 import time
@@ -121,6 +122,9 @@ class Queuer(
         # Active runners (jobs currently executing)
         self.active_runners: Dict[UUID, Runner] = {}
 
+        # Set pool size based on max_concurrency
+        self._pool_size = max_concurrency if max_concurrency > 0 else 4
+
         # Database listeners (will be initialized in start())
         self.job_db_listener: Optional[QueuerListener] = None
         self.job_archive_db_listener: Optional[QueuerListener] = None
@@ -170,6 +174,17 @@ class Queuer(
         self._stopped.clear()
         self._cancel_event = asyncio.Event()
 
+        # Create process pool for job execution
+        try:
+            self._process_pool = multiprocessing.Pool(
+                processes=self._pool_size,
+                maxtasksperchild=100,
+            )
+            logger.info(f"Created process pool with {self._pool_size} workers")
+        except Exception as e:
+            logger.error(f"Failed to create process pool: {e}")
+            raise
+
         # Set up database listeners
         try:
             self.job_db_listener = new_queuer_db_listener(
@@ -293,9 +308,24 @@ class Queuer(
 
         # Cancel all active runners
         for runner_id, runner in list(self.active_runners.items()):
-            runner.cancel()
+            cancelled = runner.cancel()
+            if not cancelled:
+                # Pool-based runner, need to restart pool
+                logger.warning("Pool-based runner cannot be individually cancelled")
             del self.active_runners[runner_id]
 
+        # Clean up process pool
+        if self._process_pool:
+            logger.info("Closing process pool")
+            try:
+                self._process_pool.close()
+                # Force terminate if workers didn't finish gracefully
+                self._process_pool.terminate()
+                self._process_pool.join()
+                self._process_pool = None
+            except Exception as e:
+                logger.warning(f"Error closing process pool: {e}")
+
         # Cancel the context to stop the queuer
         if self._cancel_event:
             self._cancel_event.set()
@@ -304,12 +334,11 @@ class Queuer(
         time.sleep(0.1)
 
         # Close database connection
-        if self.database:
-            logger.info("Closing database connection")
-            try:
-                self.database.close()
-            except Exception as e:
-                logger.error(f"Error closing database connection: {e}")
+        logger.info("Closing database connection")
+        try:
+            self.database.close()
+        except Exception as e:
+            logger.error(f"Error closing database connection: {e}")
 
         logger.info("Queuer stopped")
 

{queuerpy-0.20.0 → queuerpy-0.22.0/queuerPy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.20.0
+Version: 0.22.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -43,7 +43,7 @@ Dynamic: license-file
 
 # queuerPy
 
-[![PyPi](https://pythonico.leapcell.app/pypi/queuerPy.svg?style=shields&data=n,v,d)](https://pypi.org/project/queuerPy/)
+[![PyPi](https://img.shields.io/pypi/v/queuerPy.svg)](https://pypi.org/project/queuerPy)
 [![Python](https://img.shields.io/badge/Python-3.10+-blue.svg)](https://www.python.org/)
 ![Coverage](https://raw.githubusercontent.com/siherrmann/queuerPy/refs/heads/main/coverage-badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/siherrmann/queuer/blob/master/LICENSE)
@@ -454,6 +454,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.20.0 → queuerpy-0.22.0}/queuerPy.egg-info/SOURCES.txt

@@ -26,6 +26,7 @@ queuer_worker.py
 ./queuer_worker.py
 ./core/__init__.py
 ./core/broadcaster.py
+./core/context.py
 ./core/listener.py
 ./core/retryer.py
 ./core/runner.py
@@ -57,6 +58,7 @@ queuer_worker.py
 ./sql/worker.sql
 core/__init__.py
 core/broadcaster.py
+core/context.py
 core/listener.py
 core/retryer.py
 core/runner.py

{queuerpy-0.20.0 → queuerpy-0.22.0}/queuer_global.py

@@ -1,3 +1,4 @@
+import multiprocessing
 import threading
 from typing import Any, Callable, Dict, Optional
 
@@ -45,6 +46,10 @@ class QueuerGlobalMixin:
         self.tasks: Dict[str, Task] = {}
         self.next_interval_funcs: Dict[str, Callable[..., Any]] = {}
 
+        # Process pool for runner reuse (initialized in start())
+        self._process_pool: Optional[Any] = None
+        self._pool_size: int = 4  # Default, will be set based on max_concurrency
+
     def initialise(
         self,
         db_config: DatabaseConfiguration,

{queuerpy-0.20.0 → queuerpy-0.22.0}/queuer_job.py

@@ -20,11 +20,19 @@ from .model.job import Job, JobStatus, new_job as create_job
 from .model.options import Options
 from .model.worker import Worker
 from .queuer_global import QueuerGlobalMixin
+from .core.context import current_job_rid
+from functools import partial
 
 # Set up logger
 logger = logging.getLogger(__name__)
 
 
+def _setup_job_context(job_rid: UUID) -> None:
+    """Helper to set the context variable in the worker process."""
+    current_job_rid.set(job_rid)
+
+
+
 class QueuerJobMixin(QueuerGlobalMixin):
     """
     Mixin class containing job-related methods for the Queuer.
@@ -50,6 +58,12 @@ class QueuerJobMixin(QueuerGlobalMixin):
         :raises Exception: If something goes wrong
         """
         options: Optional[Options] = self._merge_options(None)
+        current_rid = current_job_rid.get()
+        if current_rid:
+            if not options:
+                options = Options()
+            options.parent_rid = str(current_rid)
+
         job: Job = self._add_job(task, options, *parameters, **parameters_keyed)
 
         logger.info(f"Job added: {job.rid}")
@@ -67,6 +81,12 @@ class QueuerJobMixin(QueuerGlobalMixin):
         # Merge default options with provided options
         options_merged: Optional[Options] = self._merge_options(options)
 
+        current_rid = current_job_rid.get()
+        if current_rid:
+            if not options_merged:
+                options_merged = Options()
+            options_merged.parent_rid = str(current_rid)
+
         try:
             # Create new job
             new_job: Job = create_job(
@@ -110,6 +130,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
         runner = Runner(
             task=self._wait_for_job_added_inner,
             args=(timeout_seconds,),
+            pool=self._process_pool,
             kwargs={},
         )
         runner.go()
@@ -421,7 +442,10 @@ class QueuerJobMixin(QueuerGlobalMixin):
         try:
             parameters = getattr(job, "parameters", [])
             parameters_keyed = getattr(job, "parameters_keyed", {})
-            runner = Runner(task, *parameters, **parameters_keyed)
+            setup_func = partial(_setup_job_context, job.rid)
+            runner = Runner(
+                task, *parameters, pool=self._process_pool, setup_func=setup_func, **parameters_keyed
+            )
 
             logger.info(f"Created runner for job {job.rid}")
 
@@ -436,8 +460,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
                 return [results], False, None
 
             except asyncio.CancelledError:
-                if runner:
-                    runner.cancel()
+                runner.cancel()
                 return [], True, None
             except Exception as e:
                 logger.error(f"Process runner task {job.task_name} failed: {e}")
@@ -507,9 +530,9 @@ class QueuerJobMixin(QueuerGlobalMixin):
         """
         try:
             # Create a Runner process for the async job
-            runner = Runner(self._run_job, job)
+            runner = Runner(self._run_job, job, pool=self._process_pool)
             runner.go()
-            logger.info(f"Scheduled job {job.rid} started in process {runner.pid}")
+            logger.info(f"Scheduled job {job.rid} started")
 
         except Exception as e:
             logger.error(f"Error executing scheduled job {job.rid}: {e}")

{queuerpy-0.20.0 → queuerpy-0.22.0}/sql/job.sql

@@ -57,6 +57,10 @@ BEGIN
         ON job
         USING HASH ((options->'schedule'->>'next_interval'));
     
+    CREATE INDEX IF NOT EXISTS idx_job_parent_rid
+        ON job
+        USING BTREE ((options->>'parent_rid'));
+    
     CREATE INDEX IF NOT EXISTS idx_job_worker_id ON job (worker_id);
     CREATE INDEX IF NOT EXISTS idx_job_worker_rid ON job (worker_rid);
     CREATE INDEX IF NOT EXISTS idx_job_status ON job (status);