queuerPy 0.20.0__tar.gz → 0.21.0__tar.gz

{queuerpy-0.20.0/queuerPy.egg-info → queuerpy-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.20.0
+Version: 0.21.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -454,6 +454,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.20.0 → queuerpy-0.21.0}/README.md

@@ -411,6 +411,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.20.0 → queuerpy-0.21.0}/_version.py

@@ -1,6 +1,6 @@
 """Version information for queuerPy package."""
 
-__version__ = "0.20.0"
+__version__ = "0.21.0"
 __version_info__ = tuple(int(part) for part in __version__.split("."))
 
 # For compatibility

{queuerpy-0.20.0 → queuerpy-0.21.0}/core/runner.py

@@ -15,115 +15,170 @@ from ..helper.logging import get_logger
 logger = get_logger(__name__)
 
 
-class Runner(multiprocessing.Process):
+class Runner:
     """
     The core execution environment. Manages the execution of a single task
-    in a separate, reliably killable multiprocessing context, and retrieves its result.
+    in a multiprocessing pool for efficient process reuse.
+
+    Supports cancellation with automatic periodic checking:
+    - Async tasks are monitored every 0.1s and can be cancelled mid-execution
+    - Sync tasks can only be cancelled before they start (Python limitation)
 
     :param task: The synchronous or asynchronous function to execute.
     :param args: Arguments to pass to the task function.
-    :param name_prefix: Prefix for the process name (e.g., "Job-Runner").
+    :param pool: Multiprocessing.Pool for process reuse (required).
+    :param kwargs: Keyword arguments to pass to the task function.
     """
 
     def __init__(
         self,
         task: Callable[..., Any],
         *args: Any,
+        pool: Any,
         **kwargs: Any,
     ):
         """
-        Initializes the Runner process.
+        Initializes the Runner.
 
         :param task: The synchronous or asynchronous function to execute.
         :param args: Arguments to pass to the task function.
+        :param pool: Multiprocessing.Pool for process reuse (required).
+        :param kwargs: Keyword arguments to pass to the task function.
         """
-        super().__init__(name=task.__name__, daemon=True)
         self.task = task
         self.args = args
         self.kwargs = kwargs
         self.is_async = inspect.iscoroutinefunction(task)
-        self._result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue(1)
-
-    def go(self) -> None:
-        """Starts the Runner process in the background."""
-        self.start()
+        self.name = task.__name__
+        self._pool = pool
+        self._async_result: Optional[Any] = None
+        # Use Manager Event for cross-process sharing (pickleable)
+        manager = multiprocessing.Manager()
+        self._cancel_event = manager.Event()
+
+    @property
+    def pid(self) -> Optional[int]:
+        """Process ID (not available in pool mode)."""
+        return None
+
+    @property
+    def daemon(self) -> bool:
+        """Daemon status (always True for pool workers)."""
+        return True
+
+    @property
+    def exitcode(self) -> Optional[int]:
+        """Exit code (not available in pool mode)."""
+        return None
+
+    @staticmethod
+    def _execute_in_pool_worker(
+        task: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        is_async: bool,
+        cancel_event: Any,
+    ) -> Any:
+        """
+        Static method to execute tasks in pool workers.
+        Handles both sync and async tasks, returns result directly.
+        Checks for cancellation before and during execution.
+        """
+        # Check if task was cancelled before execution
+        if cancel_event.is_set():
+            raise asyncio.CancelledError("Task was cancelled before execution")
+
+        if is_async:
+            # For async tasks, monitor cancellation during execution
+            result = asyncio.run(
+                Runner._run_async_with_cancellation(task, args, kwargs, cancel_event)
+            )
+        else:
+            # For sync tasks, just execute (can't interrupt mid-execution easily)
+            result = task(*args, **kwargs)
+        return result
 
-    def run(self) -> None:
+    @staticmethod
+    async def _run_async_with_cancellation(
+        task: Callable[..., Any],
+        args: tuple,
+        kwargs: dict,
+        cancel_event: Any,
+    ) -> Any:
         """
-        The main execution method, executed in the child process.
-        It executes the assigned task, sends the result to the queue, and exits.
+        Run async task with periodic cancellation checking.
+        Monitors cancel_event and cancels the task if it's set.
         """
+        # Create the task
+        task_coro = task(*args, **kwargs)
+        task_obj = asyncio.create_task(task_coro)
+
+        # Monitor for cancellation every second
+        async def monitor_cancellation():
+            while not task_obj.done():
+                if cancel_event.is_set():
+                    task_obj.cancel()
+                    break
+                await asyncio.sleep(1)
+
+        monitor = asyncio.create_task(monitor_cancellation())
         try:
-            if self.is_async:
-                result = asyncio.run(self.task(*self.args, **self.kwargs))
-            else:
-                result = self.task(*self.args, **self.kwargs)
-
-            try:
-                self._result_queue.put(result)
-            except BrokenPipeError as e:
-                pass
-        except Exception as e:
-            logger.debug(traceback.format_exc())
+            result = await task_obj
+            monitor.cancel()
+            return result
+        except asyncio.CancelledError:
+            monitor.cancel()
+            raise asyncio.CancelledError("Task was cancelled during execution")
+        finally:
             try:
-                self._result_queue.put(e)
-            except BrokenPipeError as e:
+                await monitor
+            except asyncio.CancelledError:
                 pass
 
+    def go(self) -> None:
+        """Starts the Runner in the background using the pool."""
+        self._async_result = self._pool.apply_async(
+            Runner._execute_in_pool_worker,
+            (self.task, self.args, self.kwargs, self.is_async, self._cancel_event),
+        )
+
     def get_results(self, timeout: Optional[float] = None) -> Any:
         """
-        Waits for the runner process to complete and returns the result.
+        Waits for the runner to complete and returns the result.
 
         :param timeout: Time in seconds to wait for the result. If None, waits indefinitely.
         :returns: The result returned by the executed task function.
         :raises TimeoutError: If the result is not available within the specified timeout.
-        :raises Exception: If the task execution in the runner process failed.
+        :raises asyncio.CancelledError: If the task was cancelled.
+        :raises Exception: If the task execution failed.
         """
         logger.debug(f"Runner.get_results() called for {self.name}, timeout={timeout}")
+        assert (
+            self._async_result is not None
+        ), "go() must be called before get_results()"
         try:
-            self.join(timeout=timeout)
-
-            if self.is_alive():
-                raise TimeoutError(f"runner timed out after {timeout} seconds.")
-
-            if self._result_queue.empty():
-                if self.exitcode == 0:
-                    return None
-                else:
-                    raise Exception(f"runner failed with code {self.exitcode}")
-
-            try:
-                result = self._result_queue.get_nowait()
-            except Exception as e:
-                raise
-
-            if isinstance(result, Exception):
-                raise result
-
+            result = self._async_result.get(timeout=timeout)
             return result
-        except Exception as e:
-            raise e
+        except multiprocessing.TimeoutError:
+            raise TimeoutError(f"runner timed out after {timeout} seconds.")
+        except asyncio.CancelledError:
+            logger.debug(f"Runner {self.name} was cancelled")
+            raise
 
     def cancel(self) -> bool:
         """
-        Attempts to cancel the running process.
+        Attempts to cancel the running task by setting the cancellation event.
 
-        :returns: True if the process was successfully terminated, False otherwise.
-        :raises: Exception if an error occurs during termination.
-        """
-        try:
-            if not self.is_alive():
-                logger.debug(f"Runner {self.name}: Already finished")
-                return True
+        Cancellation behavior:
+        - Async tasks: Checked every second during execution, task will be cancelled
+        - Sync tasks: Only checked before execution starts (cannot interrupt mid-execution)
+        - Queued tasks: Cancelled before they begin executing
 
-            logger.debug(f"Cancelling runner {self.name}")
-            self.terminate()
-            self.join(timeout=5.0)  # Wait up to 5 seconds for clean termination
-            result = not self.is_alive()
-            return result
-        except Exception as e:
-            logger.warning(f"Error cancelling runner {self.name}: {e}")
-            return False
+        :returns: True if cancellation signal was sent successfully.
+        """
+        logger.debug(f"Cancelling runner {self.name}")
+        self._cancel_event.set()
+        return True
 
 
 class SmallRunner(threading.Thread):
@@ -253,7 +308,11 @@ class SmallRunner(threading.Thread):
 
 
 def go_func(
-    func: Callable[..., Any], use_mp: bool = True, *args: Any, **kwargs: Any
+    func: Callable[..., Any],
+    use_mp: bool = True,
+    *args: Any,
+    pool: Any = None,
+    **kwargs: Any,
 ) -> Union[Runner, SmallRunner]:
     """
     Go-like async function execution factory. Selects the appropriate runner.
@@ -262,11 +321,14 @@ def go_func(
     :param use_mp: If True (default), uses the multiprocessing Runner (isolated).
                    If False, uses the threading SmallRunner (shared state).
     :param args: Positional arguments for the function.
+    :param pool: Multiprocessing.Pool for process reuse (required if use_mp=True).
     :param kwargs: Keyword arguments for the function.
     :returns: A running Runner or SmallRunner instance.
     """
     if use_mp:
-        runner = Runner(func, *args, **kwargs)
+        if pool is None:
+            raise ValueError("pool is required when use_mp=True")
+        runner = Runner(func, *args, pool=pool, **kwargs)
     else:
         runner = SmallRunner(func, *args, **kwargs)
 

{queuerpy-0.20.0 → queuerpy-0.21.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "queuerPy"
-version = "0.20.0"
+version = "0.21.0"
 description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
 readme = "README.md"
 license = "Apache-2.0"
@@ -92,7 +92,7 @@ pythonVersion = "3.8"
 pythonPlatform = "All"
 
 [tool.mypy]
-python_version = "0.20.0"
+python_version = "0.21.0"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{queuerpy-0.20.0 → queuerpy-0.21.0}/queuer.py

@@ -6,6 +6,7 @@ Mirrors the Go Queuer struct with Python async patterns.
 # Standard library imports
 import asyncio
 import json
+import multiprocessing
 import os
 import threading
 import time
@@ -121,6 +122,9 @@ class Queuer(
         # Active runners (jobs currently executing)
         self.active_runners: Dict[UUID, Runner] = {}
 
+        # Set pool size based on max_concurrency
+        self._pool_size = max_concurrency if max_concurrency > 0 else 4
+
         # Database listeners (will be initialized in start())
         self.job_db_listener: Optional[QueuerListener] = None
         self.job_archive_db_listener: Optional[QueuerListener] = None
@@ -170,6 +174,17 @@ class Queuer(
         self._stopped.clear()
         self._cancel_event = asyncio.Event()
 
+        # Create process pool for job execution
+        try:
+            self._process_pool = multiprocessing.Pool(
+                processes=self._pool_size,
+                maxtasksperchild=100,
+            )
+            logger.info(f"Created process pool with {self._pool_size} workers")
+        except Exception as e:
+            logger.error(f"Failed to create process pool: {e}")
+            raise
+
         # Set up database listeners
         try:
             self.job_db_listener = new_queuer_db_listener(
@@ -293,9 +308,25 @@ class Queuer(
 
         # Cancel all active runners
         for runner_id, runner in list(self.active_runners.items()):
-            runner.cancel()
+            cancelled = runner.cancel()
+            if not cancelled:
+                # Pool-based runner, need to restart pool
+                logger.warning("Pool-based runner cannot be individually cancelled")
             del self.active_runners[runner_id]
 
+        # Clean up process pool
+        if self._process_pool:
+            logger.info("Closing process pool")
+            try:
+                self._process_pool.close()
+                self._process_pool.join(timeout=5.0)
+                # Force terminate if workers didn't finish gracefully
+                self._process_pool.terminate()
+                self._process_pool.join()
+                self._process_pool = None
+            except Exception as e:
+                logger.warning(f"Error closing process pool: {e}")
+
         # Cancel the context to stop the queuer
         if self._cancel_event:
             self._cancel_event.set()

{queuerpy-0.20.0 → queuerpy-0.21.0/queuerPy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.20.0
+Version: 0.21.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -454,6 +454,7 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Keyword Arguments Support**: Python functions can use both positional arguments (`*args`) and keyword arguments (`**kwargs`), while maintaining compatibility with Go jobs that use only positional parameters.
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
+- **Multiprocessing pool**: Usage of a multiprocessing pool to reuse memory so global variables can be reused after first load.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
 - **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
 - **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.

{queuerpy-0.20.0 → queuerpy-0.21.0}/queuer_global.py

@@ -1,3 +1,4 @@
+import multiprocessing
 import threading
 from typing import Any, Callable, Dict, Optional
 
@@ -45,6 +46,10 @@ class QueuerGlobalMixin:
         self.tasks: Dict[str, Task] = {}
         self.next_interval_funcs: Dict[str, Callable[..., Any]] = {}
 
+        # Process pool for runner reuse (initialized in start())
+        self._process_pool: Optional[Any] = None
+        self._pool_size: int = 4  # Default, will be set based on max_concurrency
+
     def initialise(
         self,
         db_config: DatabaseConfiguration,

{queuerpy-0.20.0 → queuerpy-0.21.0}/queuer_job.py

@@ -110,6 +110,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
         runner = Runner(
             task=self._wait_for_job_added_inner,
             args=(timeout_seconds,),
+            pool=self._process_pool,
             kwargs={},
         )
         runner.go()
@@ -421,7 +422,9 @@ class QueuerJobMixin(QueuerGlobalMixin):
         try:
             parameters = getattr(job, "parameters", [])
             parameters_keyed = getattr(job, "parameters_keyed", {})
-            runner = Runner(task, *parameters, **parameters_keyed)
+            runner = Runner(
+                task, *parameters, pool=self._process_pool, **parameters_keyed
+            )
 
             logger.info(f"Created runner for job {job.rid}")
 
@@ -507,9 +510,9 @@ class QueuerJobMixin(QueuerGlobalMixin):
         """
         try:
             # Create a Runner process for the async job
-            runner = Runner(self._run_job, job)
+            runner = Runner(self._run_job, job, pool=self._process_pool)
             runner.go()
-            logger.info(f"Scheduled job {job.rid} started in process {runner.pid}")
+            logger.info(f"Scheduled job {job.rid} started")
 
         except Exception as e:
             logger.error(f"Error executing scheduled job {job.rid}: {e}")