queuerPy 0.6.0__tar.gz → 0.8.0__tar.gz

{queuerpy-0.6.0/queuerPy.egg-info → queuerpy-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.6.0
+Version: 0.8.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -86,6 +86,14 @@ q.add_task(example_task)
 q.start()
 ```
 
+You can also add a task with a decorator so the tasks don't have to be added to some central function:
+
+```python
+@queuer.task(name="optionalName")
+def example_task():
+...
+```
+
 That's easy, right? Adding a job is just as easy:
 
 ```python
@@ -329,19 +337,24 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
-- **Batch Job Processing**: Insert job batches efficiently.
-- **Transaction Support**: Insert jobs within transactions for rollback capability.
-- **Error Recovery**: Comprehensive error handling and retry mechanisms.
-- **Multiple Workers**: Multiple queuer instances can run across different services while maintaining job order and isolation.
+- **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
+- **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.
+- **Error Handling**: Comprehensive error handling by checking last output parameter for errors.
+- **Multiple Workers**: Multiple queuer instances can run across different microservices while maintaining job start order and isolation.
 - **Scheduled Jobs**: Support for scheduled and periodic jobs with custom intervals.
 - **Job Lifecycle Management**: Easy functions to get jobs and workers, track job status.
-- **Event Listeners**: Listen for job updates, completion, and deletion events.
-- **Retry Mechanisms**: Configurable retry logic with different backoff strategies.
-- **Custom Scheduling**: Custom NextInterval functions for complex scheduling needs.
-- **Heartbeat System**: Worker heartbeat monitoring and automatic cleanup of stale workers.
+- **Event Listeners**: Listen for job updates, completion, and deletion events (ended jobs).
+- **Job Completion Helpers**: Helper functions to listen for specific finished jobs.
+- **Retry Mechanisms**: Retry mechanism for ended jobs which creates a new job with the same parameters, with configurable retry logic and different backoff strategies.
+- **Custom Scheduling**: Custom NextInterval functions to address custom needs for scheduling (e.g., scheduling with timezone offset).
+- **Master Worker Management**: Automatic master worker setting retention and other central settings. Automatic switch to new master if old worker stops.
+- **Heartbeat System**: Worker heartbeat monitoring and automatic stale worker detection and cancellation by the master.
+- **Encryption Support**: Encryption support for sensitive job data stored in the database.
 - **Database Integration**: Seamless PostgreSQL integration with automatic schema management.
 - **Type Safety**: Full type hints and dataclass-based models for better development experience.
 
+> **Note**: Transactional job insert is the only feature that is not yet implemented in the Python implementation of the queuer.
+
 ---
 
 # 🧪 Testing

{queuerpy-0.6.0 → queuerpy-0.8.0}/README.md

@@ -43,6 +43,14 @@ q.add_task(example_task)
 q.start()
 ```
 
+You can also add a task with a decorator so the tasks don't have to be added to some central function:
+
+```python
+@queuer.task(name="optionalName")
+def example_task():
+...
+```
+
 That's easy, right? Adding a job is just as easy:
 
 ```python
@@ -286,19 +294,24 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
-- **Batch Job Processing**: Insert job batches efficiently.
-- **Transaction Support**: Insert jobs within transactions for rollback capability.
-- **Error Recovery**: Comprehensive error handling and retry mechanisms.
-- **Multiple Workers**: Multiple queuer instances can run across different services while maintaining job order and isolation.
+- **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
+- **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.
+- **Error Handling**: Comprehensive error handling by checking last output parameter for errors.
+- **Multiple Workers**: Multiple queuer instances can run across different microservices while maintaining job start order and isolation.
 - **Scheduled Jobs**: Support for scheduled and periodic jobs with custom intervals.
 - **Job Lifecycle Management**: Easy functions to get jobs and workers, track job status.
-- **Event Listeners**: Listen for job updates, completion, and deletion events.
-- **Retry Mechanisms**: Configurable retry logic with different backoff strategies.
-- **Custom Scheduling**: Custom NextInterval functions for complex scheduling needs.
-- **Heartbeat System**: Worker heartbeat monitoring and automatic cleanup of stale workers.
+- **Event Listeners**: Listen for job updates, completion, and deletion events (ended jobs).
+- **Job Completion Helpers**: Helper functions to listen for specific finished jobs.
+- **Retry Mechanisms**: Retry mechanism for ended jobs which creates a new job with the same parameters, with configurable retry logic and different backoff strategies.
+- **Custom Scheduling**: Custom NextInterval functions to address custom needs for scheduling (e.g., scheduling with timezone offset).
+- **Master Worker Management**: Automatic master worker setting retention and other central settings. Automatic switch to new master if old worker stops.
+- **Heartbeat System**: Worker heartbeat monitoring and automatic stale worker detection and cancellation by the master.
+- **Encryption Support**: Encryption support for sensitive job data stored in the database.
 - **Database Integration**: Seamless PostgreSQL integration with automatic schema management.
 - **Type Safety**: Full type hints and dataclass-based models for better development experience.
 
+> **Note**: Transactional job insert is the only feature that is not yet implemented in the Python implementation of the queuer.
+
 ---
 
 # 🧪 Testing

{queuerpy-0.6.0 → queuerpy-0.8.0}/_version.py

@@ -1,6 +1,6 @@
 """Version information for queuerPy package."""
 
-__version__ = "0.6.0"
+__version__ = "0.8.0"
 __version_info__ = tuple(int(part) for part in __version__.split("."))
 
 # For compatibility

{queuerpy-0.6.0 → queuerpy-0.8.0}/database/db_job.py

@@ -3,8 +3,9 @@ Job database handler for Python queuer implementation.
 Mirrors Go's database/dbJob.go with psycopg3 and references same SQL functions.
 """
 
+from datetime import datetime
 import json
-from typing import List, Optional
+from typing import List, Optional, Tuple
 from uuid import UUID
 from psycopg.rows import dict_row
 
@@ -127,21 +128,44 @@ class JobDBHandler:
         # In full implementation, this would handle explicit transaction context
         return self.insert_job(job)
 
-    def batch_insert_jobs(self, jobs: List[Job]) -> List[Job]:
+    def batch_insert_jobs(self, jobs: List[Job]):
         """
-        Insert multiple jobs in a batch operation.
-        Placeholder method - mirrors Go's BatchInsertJobs method.
-        TODO: Implement efficient batch insert logic.
+        Insert multiple jobs in a batch operation using executemany.
+        Mirrors Go's BatchInsertJobs method with direct INSERT statements.
 
         :param jobs: List of Job instances to insert.
         :return: List of inserted Job instances.
+        :raises QueuerError: If batch insertion fails.
         """
-        # For now, insert jobs one by one
-        # In full implementation, this would use batch SQL operations
-        result_jobs: List[Job] = []
-        for job in jobs:
-            result_jobs.append(self.insert_job(job))
-        return result_jobs
+        if not jobs:
+            return
+
+        if self.db.instance is None:
+            raise ValueError("Database connection is not established")
+
+        try:
+            batch_data: List[Tuple[str, str, str, Optional[datetime]]] = []
+            for job in jobs:
+                options_json = (
+                    json.dumps(job.options.to_dict()) if job.options else "{}"
+                )
+                parameters_json = json.dumps(job.parameters)
+                batch_data.append(
+                    (options_json, job.task_name, parameters_json, job.scheduled_at)
+                )
+
+            with self.db.instance.cursor(row_factory=dict_row) as cur:
+                cur.executemany(
+                    """
+                    INSERT INTO job (options, task_name, parameters, scheduled_at) 
+                    VALUES (%s, %s, %s, %s)
+                    """,
+                    batch_data,
+                )
+                self.db.instance.commit()
+        except Exception as e:
+            self.db.instance.rollback()
+            raise QueuerError("batch insert jobs", e)
 
     def update_jobs_initial(self, worker: Worker) -> List[Job]:
         """

{queuerpy-0.6.0 → queuerpy-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "queuerPy"
-version = "0.6.0"
+version = "0.8.0"
 description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
 readme = "README.md"
 license = "Apache-2.0"

{queuerpy-0.6.0 → queuerpy-0.8.0/queuerPy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.6.0
+Version: 0.8.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -86,6 +86,14 @@ q.add_task(example_task)
 q.start()
 ```
 
+You can also add a task with a decorator so the tasks don't have to be added to some central function:
+
+```python
+@queuer.task(name="optionalName")
+def example_task():
+...
+```
+
 That's easy, right? Adding a job is just as easy:
 
 ```python
@@ -329,19 +337,24 @@ worker_jobs = queuer.get_jobs_by_worker_rid(worker.rid)
 
 - **Async/Await Support**: Full asyncio integration with threading fallbacks.
 - **PostgreSQL NOTIFY/LISTEN**: Real-time job notifications without polling overhead.
-- **Batch Job Processing**: Insert job batches efficiently.
-- **Transaction Support**: Insert jobs within transactions for rollback capability.
-- **Error Recovery**: Comprehensive error handling and retry mechanisms.
-- **Multiple Workers**: Multiple queuer instances can run across different services while maintaining job order and isolation.
+- **Batch Job Processing**: Insert job batches efficiently using PostgreSQL's `COPY FROM` feature.
+- **Panic Recovery**: Automatic recovery for all running jobs in case of unexpected failures.
+- **Error Handling**: Comprehensive error handling by checking last output parameter for errors.
+- **Multiple Workers**: Multiple queuer instances can run across different microservices while maintaining job start order and isolation.
 - **Scheduled Jobs**: Support for scheduled and periodic jobs with custom intervals.
 - **Job Lifecycle Management**: Easy functions to get jobs and workers, track job status.
-- **Event Listeners**: Listen for job updates, completion, and deletion events.
-- **Retry Mechanisms**: Configurable retry logic with different backoff strategies.
-- **Custom Scheduling**: Custom NextInterval functions for complex scheduling needs.
-- **Heartbeat System**: Worker heartbeat monitoring and automatic cleanup of stale workers.
+- **Event Listeners**: Listen for job updates, completion, and deletion events (ended jobs).
+- **Job Completion Helpers**: Helper functions to listen for specific finished jobs.
+- **Retry Mechanisms**: Retry mechanism for ended jobs which creates a new job with the same parameters, with configurable retry logic and different backoff strategies.
+- **Custom Scheduling**: Custom NextInterval functions to address custom needs for scheduling (e.g., scheduling with timezone offset).
+- **Master Worker Management**: Automatic master worker setting retention and other central settings. Automatic switch to new master if old worker stops.
+- **Heartbeat System**: Worker heartbeat monitoring and automatic stale worker detection and cancellation by the master.
+- **Encryption Support**: Encryption support for sensitive job data stored in the database.
 - **Database Integration**: Seamless PostgreSQL integration with automatic schema management.
 - **Type Safety**: Full type hints and dataclass-based models for better development experience.
 
+> **Note**: Transactional job insert is the only feature that is not yet implemented in the Python implementation of the queuer.
+
 ---
 
 # 🧪 Testing

{queuerpy-0.6.0 → queuerpy-0.8.0}/queuer_job.py

@@ -6,7 +6,7 @@ Mirrors Go's queuerJob.go functionality.
 import asyncio
 import logging
 from datetime import datetime
-from typing import Dict, List, Optional, Any, Tuple, Union, Callable, TYPE_CHECKING
+from typing import Dict, List, Optional, Any, Tuple, Union, Callable
 from uuid import UUID
 
 from core.broadcaster import BroadcasterQueue
@@ -15,14 +15,12 @@ from core.retryer import Retryer
 from core.scheduler import Scheduler
 from helper.error import QueuerError
 from helper.task import get_task_name_from_interface
+from model.batch_job import BatchJob
 from model.job import Job, JobStatus, new_job as create_job
 from model.options import Options
 from model.worker import Worker
 from queuer_global import QueuerGlobalMixin
 
-if TYPE_CHECKING:
-    from model.batch_job import BatchJob
-
 # Set up logger
 logger = logging.getLogger(__name__)
 
@@ -73,7 +71,7 @@ class QueuerJobMixin(QueuerGlobalMixin):
             logger.error(f"Error adding job with options: {str(e)}")
             raise Exception(f"Adding job: {str(e)}")
 
-    def add_jobs(self, batch_jobs: List["BatchJob"]):
+    def add_jobs(self, batch_jobs: List[BatchJob]):
         """
         Add a batch of jobs to the queue.
 

{queuerpy-0.6.0 → queuerpy-0.8.0}/queuer_task.py

@@ -4,7 +4,7 @@ Mirrors Go's queuerTask.go functionality.
 """
 
 import logging
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Optional, TypeVar, overload
 
 from model.task import new_task, new_task_with_name
 from model.task import Task
@@ -12,6 +12,9 @@ from queuer_global import QueuerGlobalMixin
 
 logger = logging.getLogger(__name__)
 
+# TypeVar for maintaining function type through decorator
+F = TypeVar("F", bound=Callable[..., Any])
+
 
 class QueuerTaskMixin(QueuerGlobalMixin):
     """
@@ -87,3 +90,36 @@ class QueuerTaskMixin(QueuerGlobalMixin):
 
         logger.info(f"Task added: {new_task_obj.name}")
         return new_task_obj
+
+    @overload
+    def task(self, name: None = None) -> Callable[[F], F]: ...
+
+    @overload
+    def task(self, name: str) -> Callable[[F], F]: ...
+
+    def task(self, name: Optional[str] = None) -> Callable[[F], F]:
+        """
+        Decorator to register a task function with the queuer.
+        This is equivalent to calling add_task() or add_task_with_name().
+
+        Usage:
+            @queuer.task()
+            def my_task(param1, param2):
+                return param1 + param2
+
+            @queuer.task(name="custom_task_name")
+            def my_other_task(param):
+                return param * 2
+
+        :param name: Optional custom name for the task. If not provided, uses function name.
+        :return: The decorator function that preserves the original function's type
+        """
+
+        def decorator(func: F) -> F:
+            if name is not None:
+                self.add_task_with_name(func, name)
+            else:
+                self.add_task(func)
+            return func
+
+        return decorator