queuerPy 0.21.0__tar.gz → 0.22.0__tar.gz

{queuerpy-0.21.0/queuerPy.egg-info → queuerpy-0.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.21.0
+Version: 0.22.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -43,7 +43,7 @@ Dynamic: license-file
 
 # queuerPy
 
-[![PyPi](https://pythonico.leapcell.app/pypi/queuerPy.svg?style=shields&data=n,v,d)](https://pypi.org/project/queuerPy/)
+[![PyPi](https://img.shields.io/pypi/v/queuerPy.svg)](https://pypi.org/project/queuerPy)
 [![Python](https://img.shields.io/badge/Python-3.10+-blue.svg)](https://www.python.org/)
 ![Coverage](https://raw.githubusercontent.com/siherrmann/queuerPy/refs/heads/main/coverage-badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/siherrmann/queuer/blob/master/LICENSE)

{queuerpy-0.21.0 → queuerpy-0.22.0}/README.md

@@ -1,6 +1,6 @@
 # queuerPy
 
-[![PyPi](https://pythonico.leapcell.app/pypi/queuerPy.svg?style=shields&data=n,v,d)](https://pypi.org/project/queuerPy/)
+[![PyPi](https://img.shields.io/pypi/v/queuerPy.svg)](https://pypi.org/project/queuerPy)
 [![Python](https://img.shields.io/badge/Python-3.10+-blue.svg)](https://www.python.org/)
 ![Coverage](https://raw.githubusercontent.com/siherrmann/queuerPy/refs/heads/main/coverage-badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/siherrmann/queuer/blob/master/LICENSE)

{queuerpy-0.21.0 → queuerpy-0.22.0}/_version.py

@@ -1,6 +1,6 @@
 """Version information for queuerPy package."""
 
-__version__ = "0.21.0"
+__version__ = "0.22.0"
 __version_info__ = tuple(int(part) for part in __version__.split("."))
 
 # For compatibility

queuerpy-0.22.0/core/context.py

@@ -0,0 +1,11 @@
+"""
+Context management for the queuer implementation.
+Provides a way to store tracking information across asynchronous and concurrent flows.
+"""
+
+from contextvars import ContextVar
+from typing import Optional
+from uuid import UUID
+
+# Context variable to track the RID of the currently executing job.
+current_job_rid: ContextVar[Optional[UUID]] = ContextVar("current_job_rid", default=None)

{queuerpy-0.21.0 → queuerpy-0.22.0}/core/runner.py

@@ -27,6 +27,7 @@ class Runner:
     :param task: The synchronous or asynchronous function to execute.
     :param args: Arguments to pass to the task function.
     :param pool: Multiprocessing.Pool for process reuse (required).
+    :param setup_func: Optional callable to run before the task (e.g., for setting ContextVars).
     :param kwargs: Keyword arguments to pass to the task function.
     """
 
@@ -35,6 +36,7 @@ class Runner:
         task: Callable[..., Any],
         *args: Any,
         pool: Any,
+        setup_func: Optional[Callable[[], None]] = None,
         **kwargs: Any,
     ):
         """
@@ -51,6 +53,7 @@ class Runner:
         self.is_async = inspect.iscoroutinefunction(task)
         self.name = task.__name__
         self._pool = pool
+        self._setup_func = setup_func
         self._async_result: Optional[Any] = None
         # Use Manager Event for cross-process sharing (pickleable)
         manager = multiprocessing.Manager()
@@ -78,6 +81,7 @@ class Runner:
         kwargs: dict,
         is_async: bool,
         cancel_event: Any,
+        setup_func: Optional[Callable[[], None]] = None,
     ) -> Any:
         """
         Static method to execute tasks in pool workers.
@@ -88,6 +92,9 @@ class Runner:
         if cancel_event.is_set():
             raise asyncio.CancelledError("Task was cancelled before execution")
 
+        if setup_func is not None:
+            setup_func()
+
         if is_async:
             # For async tasks, monitor cancellation during execution
             result = asyncio.run(
@@ -139,7 +146,7 @@ class Runner:
         """Starts the Runner in the background using the pool."""
         self._async_result = self._pool.apply_async(
             Runner._execute_in_pool_worker,
-            (self.task, self.args, self.kwargs, self.is_async, self._cancel_event),
+            (self.task, self.args, self.kwargs, self.is_async, self._cancel_event, self._setup_func),
         )
 
     def get_results(self, timeout: Optional[float] = None) -> Any:

{queuerpy-0.21.0 → queuerpy-0.22.0}/database/db_job.py

@@ -186,19 +186,22 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                """
-                SELECT * FROM update_job_initial(%s);
-            """,
-                (worker.id,),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    """
+                    SELECT * FROM update_job_initial(%s);
+                """,
+                    (worker.id,),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("update jobs initial", e)
 
     def update_job_final(self, job: Job) -> Job:
         """
@@ -303,17 +306,20 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs(%s, %s, %s);",
-                (self.encryption_key or "", last_id, entries),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs(%s, %s, %s);",
+                    (self.encryption_key or "", last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs", e)
 
     def select_all_jobs_by_worker_rid(
         self, worker_rid: UUID, last_id: int = 0, entries: int = 100
@@ -330,17 +336,20 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs_by_worker_rid(%s, %s, %s, %s);",
-                (self.encryption_key or "", worker_rid, last_id, entries),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs_by_worker_rid(%s, %s, %s, %s);",
+                    (self.encryption_key or "", str(worker_rid), last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs by worker rid", e)
 
     def select_all_jobs_by_search(
         self, search: str, last_id: int = 0, entries: int = 100
@@ -358,17 +367,48 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs_by_search(%s, %s, %s, %s);",
-                (self.encryption_key or "", search, last_id, entries),
-            )
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs_by_search(%s, %s, %s, %s);",
+                    (self.encryption_key or "", search, last_id, entries),
+                )
 
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
 
-            return jobs
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs by search", e)
+
+    def select_jobs_without_worker(
+        self, last_id: int = 0, entries: int = 100
+    ) -> List[Job]:
+        """
+        Select all jobs that are not assigned to any worker.
+
+        :param last_id: Last job ID from previous page.
+        :param entries: Number of entries to retrieve.
+        :return: List of Job instances.
+        """
+        if self.db.instance is None:
+            raise ValueError("Database connection is not established")
+
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_jobs_without_worker(%s, %s, %s);",
+                    (self.encryption_key or "", last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select jobs without worker", e)
 
     def add_retention_archive(self, days: int) -> None:
         """
@@ -441,17 +481,20 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs_from_archive(%s, %s, %s);",
-                (self.encryption_key or "", last_id, entries),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs_from_archive(%s, %s, %s);",
+                    (self.encryption_key or "", last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs from archive", e)
 
     def select_all_jobs_from_archive_by_search(
         self, search: str, last_id: int = 0, entries: int = 100
@@ -468,14 +511,17 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_jobs_from_archive_by_search(%s, %s, %s, %s);",
-                (self.encryption_key or "", search, last_id, entries),
-            )
-
-            jobs: List[Job] = []
-            for row in cur.fetchall():
-                jobs.append(Job.from_row(row))
-
-            return jobs
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_jobs_from_archive_by_search(%s, %s, %s, %s);",
+                    (self.encryption_key or "", search, last_id, entries),
+                )
+    
+                jobs: List[Job] = []
+                for row in cur.fetchall():
+                    jobs.append(Job.from_row(row))
+    
+                return jobs
+        except Exception as e:
+            raise QueuerError("select all jobs from archive by search", e)

{queuerpy-0.21.0 → queuerpy-0.22.0}/database/db_worker.py

@@ -195,14 +195,17 @@ class WorkerDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute("SELECT * FROM select_all_workers(%s, %s);", (last_id, entries))
-
-            workers: List[Worker] = []
-            for row in cur.fetchall():
-                workers.append(Worker.from_row(row))
-
-            return workers
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute("SELECT * FROM select_all_workers(%s, %s);", (last_id, entries))
+    
+                workers: List[Worker] = []
+                for row in cur.fetchall():
+                    workers.append(Worker.from_row(row))
+    
+                return workers
+        except Exception as e:
+            raise QueuerError("select all workers", e)
 
     def select_all_workers_by_search(
         self, search: str, last_id: int = 0, entries: int = 100
@@ -215,17 +218,20 @@ class WorkerDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute(
-                "SELECT * FROM select_all_workers_by_search(%s, %s, %s);",
-                (search, last_id, entries),
-            )
-
-            workers: List[Worker] = []
-            for row in cur.fetchall():
-                workers.append(Worker.from_row(row))
-
-            return workers
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute(
+                    "SELECT * FROM select_all_workers_by_search(%s, %s, %s);",
+                    (search, last_id, entries),
+                )
+    
+                workers: List[Worker] = []
+                for row in cur.fetchall():
+                    workers.append(Worker.from_row(row))
+    
+                return workers
+        except Exception as e:
+            raise QueuerError("select all workers by search", e)
 
     def select_all_connections(self) -> List[ConnectionModel]:
         """
@@ -235,19 +241,22 @@ class WorkerDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor(row_factory=dict_row) as cur:
-            cur.execute("SELECT * FROM select_all_connections();")
-
-            connections: List[ConnectionModel] = []
-            for row in cur.fetchall():
-                connection = ConnectionModel(
-                    pid=row["output_pid"],
-                    database=row["output_datname"],
-                    username=row["output_usename"],
-                    application_name=row["output_application_name"],
-                    query=row["output_query"],
-                    state=row["output_state"],
-                )
-                connections.append(connection)
-
-            return connections
+        try:
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.execute("SELECT * FROM select_all_connections();")
+    
+                connections: List[ConnectionModel] = []
+                for row in cur.fetchall():
+                    connection = ConnectionModel(
+                        pid=row["output_pid"],
+                        database=row["output_datname"],
+                        username=row["output_usename"],
+                        application_name=row["output_application_name"],
+                        query=row["output_query"],
+                        state=row["output_state"],
+                    )
+                    connections.append(connection)
+    
+                return connections
+        except Exception as e:
+            raise QueuerError("select all connections", e)

{queuerpy-0.21.0 → queuerpy-0.22.0}/helper/database.py

@@ -88,8 +88,9 @@ class DatabaseConfiguration:
             f"user={self.username} "
             f"password={self.password} "
             f"sslmode={self.sslmode} "
+            f"connect_timeout=10 "
             f"application_name=queuer "
-            f"options='-c search_path={self.schema}'"
+            f"options='-c search_path={self.schema} -c statement_timeout=30000'"
         )
 
 

{queuerpy-0.21.0 → queuerpy-0.22.0}/model/options.py

@@ -78,6 +78,7 @@ class Options:
 
     on_error: Optional[OnError] = None
     schedule: Optional[Schedule] = None
+    parent_rid: Optional[str] = None
 
     def is_valid(self) -> bool:
         """
@@ -100,6 +101,7 @@ class Options:
         return {
             "on_error": self.on_error.to_dict() if self.on_error else None,
             "schedule": self.schedule.to_dict() if self.schedule else None,
+            "parent_rid": self.parent_rid,
         }
 
     @classmethod
@@ -115,11 +117,15 @@ class Options:
             options.on_error = OnError.from_dict(data["on_error"])
         if data.get("schedule"):
             options.schedule = Schedule.from_dict(data["schedule"])
+        if data.get("parent_rid"):
+            options.parent_rid = data["parent_rid"]
         return options
 
 
 def new_options(
-    on_error: Optional[OnError] = None, schedule: Optional[Schedule] = None
+    on_error: Optional[OnError] = None,
+    schedule: Optional[Schedule] = None,
+    parent_rid: Optional[str] = None,
 ) -> Options:
     """
     Create new options. Mirrors Go's pattern.
@@ -128,4 +134,4 @@ def new_options(
     :param schedule: Schedule options.
     :return: Options instance.
     """
-    return Options(on_error=on_error, schedule=schedule)
+    return Options(on_error=on_error, schedule=schedule, parent_rid=parent_rid)

{queuerpy-0.21.0 → queuerpy-0.22.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "queuerPy"
-version = "0.21.0"
+version = "0.22.0"
 description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
 readme = "README.md"
 license = "Apache-2.0"
@@ -92,7 +92,7 @@ pythonVersion = "3.8"
 pythonPlatform = "All"
 
 [tool.mypy]
-python_version = "0.21.0"
+python_version = "0.22.0"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{queuerpy-0.21.0 → queuerpy-0.22.0}/queuer.py

@@ -319,7 +319,6 @@ class Queuer(
             logger.info("Closing process pool")
             try:
                 self._process_pool.close()
-                self._process_pool.join(timeout=5.0)
                 # Force terminate if workers didn't finish gracefully
                 self._process_pool.terminate()
                 self._process_pool.join()
@@ -335,12 +334,11 @@ class Queuer(
         time.sleep(0.1)
 
         # Close database connection
-        if self.database:
-            logger.info("Closing database connection")
-            try:
-                self.database.close()
-            except Exception as e:
-                logger.error(f"Error closing database connection: {e}")
+        logger.info("Closing database connection")
+        try:
+            self.database.close()
+        except Exception as e:
+            logger.error(f"Error closing database connection: {e}")
 
         logger.info("Queuer stopped")
 

{queuerpy-0.21.0 → queuerpy-0.22.0/queuerPy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.21.0
+Version: 0.22.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -43,7 +43,7 @@ Dynamic: license-file
 
 # queuerPy
 
-[![PyPi](https://pythonico.leapcell.app/pypi/queuerPy.svg?style=shields&data=n,v,d)](https://pypi.org/project/queuerPy/)
+[![PyPi](https://img.shields.io/pypi/v/queuerPy.svg)](https://pypi.org/project/queuerPy)
 [![Python](https://img.shields.io/badge/Python-3.10+-blue.svg)](https://www.python.org/)
 ![Coverage](https://raw.githubusercontent.com/siherrmann/queuerPy/refs/heads/main/coverage-badge.svg)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/siherrmann/queuer/blob/master/LICENSE)

{queuerpy-0.21.0 → queuerpy-0.22.0}/queuerPy.egg-info/SOURCES.txt

@@ -26,6 +26,7 @@ queuer_worker.py
 ./queuer_worker.py
 ./core/__init__.py
 ./core/broadcaster.py
+./core/context.py
 ./core/listener.py
 ./core/retryer.py
 ./core/runner.py
@@ -57,6 +58,7 @@ queuer_worker.py
 ./sql/worker.sql
 core/__init__.py
 core/broadcaster.py
+core/context.py
 core/listener.py
 core/retryer.py
 core/runner.py

{queuerpy-0.21.0 → queuerpy-0.22.0}/queuer_job.py

@@ -20,11 +20,19 @@ from .model.job import Job, JobStatus, new_job as create_job
 from .model.options import Options
 from .model.worker import Worker
 from .queuer_global import QueuerGlobalMixin
+from .core.context import current_job_rid
+from functools import partial
 
 # Set up logger
 logger = logging.getLogger(__name__)
 
 
+def _setup_job_context(job_rid: UUID) -> None:
+    """Helper to set the context variable in the worker process."""
+    current_job_rid.set(job_rid)
+
+
+
 class QueuerJobMixin(QueuerGlobalMixin):
     """
     Mixin class containing job-related methods for the Queuer.
@@ -50,6 +58,12 @@ class QueuerJobMixin(QueuerGlobalMixin):
         :raises Exception: If something goes wrong
         """
         options: Optional[Options] = self._merge_options(None)
+        current_rid = current_job_rid.get()
+        if current_rid:
+            if not options:
+                options = Options()
+            options.parent_rid = str(current_rid)
+
         job: Job = self._add_job(task, options, *parameters, **parameters_keyed)
 
         logger.info(f"Job added: {job.rid}")
@@ -67,6 +81,12 @@ class QueuerJobMixin(QueuerGlobalMixin):
         # Merge default options with provided options
         options_merged: Optional[Options] = self._merge_options(options)
 
+        current_rid = current_job_rid.get()
+        if current_rid:
+            if not options_merged:
+                options_merged = Options()
+            options_merged.parent_rid = str(current_rid)
+
         try:
             # Create new job
             new_job: Job = create_job(
@@ -422,8 +442,9 @@ class QueuerJobMixin(QueuerGlobalMixin):
         try:
             parameters = getattr(job, "parameters", [])
             parameters_keyed = getattr(job, "parameters_keyed", {})
+            setup_func = partial(_setup_job_context, job.rid)
             runner = Runner(
-                task, *parameters, pool=self._process_pool, **parameters_keyed
+                task, *parameters, pool=self._process_pool, setup_func=setup_func, **parameters_keyed
             )
 
             logger.info(f"Created runner for job {job.rid}")
@@ -439,8 +460,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
                 return [results], False, None
 
             except asyncio.CancelledError:
-                if runner:
-                    runner.cancel()
+                runner.cancel()
                 return [], True, None
             except Exception as e:
                 logger.error(f"Process runner task {job.task_name} failed: {e}")

{queuerpy-0.21.0 → queuerpy-0.22.0}/sql/job.sql

@@ -57,6 +57,10 @@ BEGIN
         ON job
         USING HASH ((options->'schedule'->>'next_interval'));
     
+    CREATE INDEX IF NOT EXISTS idx_job_parent_rid
+        ON job
+        USING BTREE ((options->>'parent_rid'));
+    
     CREATE INDEX IF NOT EXISTS idx_job_worker_id ON job (worker_id);
     CREATE INDEX IF NOT EXISTS idx_job_worker_rid ON job (worker_rid);
     CREATE INDEX IF NOT EXISTS idx_job_status ON job (status);