queuerPy 0.1.0__tar.gz → 0.3.0__tar.gz

{queuerpy-0.1.0 → queuerpy-0.3.0}/MANIFEST.in

@@ -2,6 +2,13 @@
 include README.md
 include LICENSE
 include pyproject.toml
+include _version.py
+
+# Include all Python modules and packages
+recursive-include core *.py
+recursive-include database *.py
+recursive-include helper *.py
+recursive-include model *.py
 
 # Include SQL files that are needed for database setup
 recursive-include sql *.sql

{queuerpy-0.1.0/queuerPy.egg-info → queuerpy-0.3.0}/PKG-INFO

@@ -1,23 +1,10 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.1.0
+Version: 0.3.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
-License: Copyright 2025 Simon Herrmann
-        
-        Licensed under the Apache License, Version 2.0 (the "License");
-        you may not use this file except in compliance with the License.
-        You may obtain a copy of the License at
-        
-           http://www.apache.org/licenses/LICENSE-2.0
-        
-        Unless required by applicable law or agreed to in writing, software
-        distributed under the License is distributed on an "AS IS" BASIS,
-        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-        See the License for the specific language governing permissions and
-        limitations under the License.
-        
+License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/siherrmann/queuerPy
 Project-URL: Repository, https://github.com/siherrmann/queuerPy
 Project-URL: Issues, https://github.com/siherrmann/queuerPy/issues
@@ -25,7 +12,6 @@ Project-URL: Documentation, https://github.com/siherrmann/queuerPy#readme
 Keywords: queue,job,task,postgresql,worker,async
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8

{queuerpy-0.1.0 → queuerpy-0.3.0}/_version.py

@@ -1,6 +1,6 @@
 """Version information for queuerPy package."""
 
-__version__ = "0.1.0"
+__version__ = "0.3.0"
 __version_info__ = tuple(int(part) for part in __version__.split("."))
 
 # For compatibility

queuerpy-0.3.0/core/__init__.py

@@ -0,0 +1,19 @@
+"""
+Core components using goless channels - Python port of Go queuer core.
+"""
+
+from .broadcaster import Broadcaster
+from .listener import Listener
+from .runner import Runner
+from .scheduler import Scheduler
+from .ticker import Ticker
+from .retryer import Retryer
+
+__all__ = [
+    'Broadcaster',
+    'Listener', 
+    'Runner',
+    'Scheduler',
+    'Ticker',
+    'Retryer'
+]

queuerpy-0.3.0/core/broadcaster.py

@@ -0,0 +1,77 @@
+"""
+Broadcaster using asyncio for fully async-aware notifications.
+
+This implementation uses asyncio.Queue for consistent async behavior and follows
+the singleton broadcaster pattern for sharing broadcasters across the system.
+"""
+
+import asyncio
+from typing import Any, TypeVar, Generic, Dict
+import uuid
+
+T = TypeVar("T")
+
+# Global registry for singleton broadcasters
+broadcaster_registry: Dict[str, "Broadcaster[Any]"] = {}
+
+
+class BroadcasterQueue(asyncio.Queue[T]):
+    """An asyncio.Queue with a broadcaster ID for tracking."""
+
+    def __init__(self):
+        super().__init__()
+        self.broadcaster_id: str = ""
+
+
+class Broadcaster(Generic[T]):
+    """A broadcaster that manages async listeners and broadcasts messages."""
+
+    def __new__(cls, name: str) -> "Broadcaster[T]":
+        """Singleton pattern: return existing instance if it exists."""
+        if name not in broadcaster_registry:
+            instance = super().__new__(cls)
+            broadcaster_registry[name] = instance
+            return instance
+        return broadcaster_registry[name]
+
+    def __init__(self, name: str):
+        """Initialize a new broadcaster (only called once per unique name)."""
+        if not hasattr(self, "name"):
+            self.name = name
+            self.listeners: Dict[str, asyncio.Queue[T]] = {}
+            self._lock = asyncio.Lock()
+            self.broadcaster_id: str = ""
+
+    async def subscribe(self) -> BroadcasterQueue[T]:
+        """Subscribe to broadcasts and return a queue for receiving messages."""
+        queue = BroadcasterQueue[T]()
+        queue.broadcaster_id = str(uuid.uuid4())
+
+        async with self._lock:
+            self.listeners[queue.broadcaster_id] = queue
+
+        return queue
+
+    async def unsubscribe(self, queue: BroadcasterQueue[T]) -> None:
+        """Unsubscribe a queue from broadcasts."""
+        queue_id = queue.broadcaster_id
+        async with self._lock:
+            if queue_id in self.listeners:
+                del self.listeners[queue_id]
+
+    async def broadcast(self, message: T) -> None:
+        """Broadcast a message to all subscribers."""
+        async with self._lock:
+            current_listeners = list(self.listeners.values())
+
+        # Send to all listeners without holding the lock
+        for queue in current_listeners:
+            try:
+                queue.put_nowait(message)
+            except asyncio.QueueFull:
+                pass
+
+
+def new_broadcaster(name: str) -> Broadcaster[Any]:
+    """Get or create a singleton broadcaster with the given name."""
+    return Broadcaster(name)

queuerpy-0.3.0/core/listener.py

@@ -0,0 +1,116 @@
+"""
+Listener using asyncio for fully async-aware notifications, cleaned up using a
+SmallRunner (threading.Thread) abstraction for non-blocking execution of callbacks.
+
+This implementation uses asyncio throughout for consistent async behavior.
+"""
+
+import asyncio
+import threading
+from typing import TypeVar, Generic, Callable, Optional, Any
+
+from core.broadcaster import Broadcaster
+from core.runner import go_func
+
+T = TypeVar("T")
+
+
+class Listener(Generic[T]):
+    """A listener that receives broadcasts and calls notification functions."""
+
+    def __init__(self, broadcaster: Broadcaster[T]):
+        """Initialize listener."""
+        self.broadcaster: Broadcaster[T] = broadcaster
+        self._stop_event = None
+        self._listening = False
+        self._active_notifications = 0
+        self._notification_lock = threading.Lock()
+        self._completion_event = threading.Event()
+
+    def listen(
+        self,
+        stop_event: Optional[threading.Event] = None,
+        ready_event: Optional[threading.Event] = None,
+        notify_function: Optional[Callable[[T], Any]] = None,
+    ) -> None:
+        """
+        Starts listening for broadcasts and calls the notify function when received.
+
+        The core asyncio loop is launched in a separate thread (via SmallRunner).
+        """
+        if notify_function is None:
+            if ready_event is not None:
+                ready_event.set()
+            return
+
+        # Create internal stop event if not provided
+        if stop_event is None:
+            stop_event = threading.Event()
+
+        self._stop_event = stop_event
+
+        # Define the main coroutine that handles subscription and message reception
+        async def _listen_async():
+            ch = await self.broadcaster.subscribe()
+            if ready_event is not None:
+                ready_event.set()
+
+            if self._stop_event is None:
+                self._stop_event = threading.Event()
+
+            try:
+                while not self._stop_event.is_set():
+                    try:
+                        msg = await asyncio.wait_for(ch.get(), timeout=0.1)
+
+                        with self._notification_lock:
+                            self._active_notifications += 1
+                            self._completion_event.clear()
+
+                        def process_notification():
+                            """Executes the user's callback in a separate thread."""
+                            try:
+                                notify_function(msg)
+                            finally:
+                                with self._notification_lock:
+                                    self._active_notifications -= 1
+                                    if self._active_notifications == 0:
+                                        self._completion_event.set()
+
+                        go_func(process_notification, use_mp=False)
+                    except asyncio.TimeoutError:
+                        # Timeout is expected; loop checks stop event and continues
+                        continue
+                    except Exception as e:
+                        print(f"Error receiving message: {e}")
+                        break
+
+            finally:
+                await self.broadcaster.unsubscribe(ch)
+
+        go_func(_listen_async, use_mp=False)
+        self._listening = True
+
+    def notify(self, data: T) -> None:
+        """
+        Sends notification directly by broadcasting the data in a separate thread.
+        """
+
+        async def _notify():
+            await self.broadcaster.broadcast(data)
+
+        go_func(_notify, use_mp=False)
+
+    def wait_for_notifications_processed(self, timeout: float = 5.0) -> bool:
+        """Wait for all notifications to be processed."""
+        return self._completion_event.wait(timeout=timeout)
+
+    def stop(self):
+        """Stop listening."""
+        if self._stop_event:
+            self._stop_event.set()
+        self._listening = False
+
+    def is_listening(self) -> bool:
+        """Check if currently listening."""
+        return self._listening

queuerpy-0.3.0/core/retryer.py

@@ -0,0 +1,86 @@
+"""
+Retryer component for the queuer system.
+
+This module provides retry functionality mirroring the Go implementation
+for reliable task execution with various backoff strategies.
+"""
+
+import logging
+import sys
+import os
+import asyncio
+import inspect
+from typing import Any, Callable, Optional
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from model.options_on_error import OnError, RetryBackoff
+
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+
+class Retryer:
+    """Simple retryer mirroring Go's Retryer implementation."""
+
+    def __init__(self, function: Callable[..., Any], options: Optional[OnError]):
+        """Initialize the retryer.
+
+        :param function: The function to execute with retries (can be sync or async)
+        :param options: OnError options for retry behavior
+        :raises ValueError: If options are invalid
+        """
+        if options is None or options.max_retries <= 0 or options.retry_delay < 0:
+            raise ValueError("No valid retry options provided")
+
+        self.function = function
+        self.options = options
+        self.sleep_duration = options.retry_delay
+        self.is_async = inspect.iscoroutinefunction(function)
+
+    async def retry(self) -> Optional[Exception]:
+        """Attempt to execute the function up to MaxRetries times.
+
+        The retry behavior is determined by the RetryBackoff option.
+        If the function raises an exception, it will retry according to the specified backoff strategy.
+        If all retries fail, it returns the last exception encountered.
+
+        The backoff strategies are:
+        - RETRY_BACKOFF_NONE: No backoff, retries immediately.
+        - RETRY_BACKOFF_LINEAR: Increases the sleep duration linearly by the initial delay.
+        - RETRY_BACKOFF_EXPONENTIAL: Doubles the sleep duration after each retry.
+
+        :returns: The last exception if all retries fail, otherwise None on success.
+        """
+        last_error: Optional[Exception] = None
+
+        for i in range(self.options.max_retries):
+            try:
+                if self.is_async:
+                    await self.function()
+                else:
+                    self.function()
+                return None  # Success
+            except Exception as err:
+                last_error = err
+                logger.warning(
+                    f"Retry attempt {i + 1}/{self.options.max_retries} failed: {err}"
+                )
+
+                # Sleep between retries (except for the last attempt)
+                if i < self.options.max_retries - 1:
+                    await asyncio.sleep(self.sleep_duration)
+
+                    # Apply backoff strategy
+                    if self.options.retry_backoff == RetryBackoff.NONE:
+                        continue
+                    elif self.options.retry_backoff == RetryBackoff.LINEAR:
+                        self.sleep_duration += self.options.retry_delay
+                        continue
+                    elif self.options.retry_backoff == RetryBackoff.EXPONENTIAL:
+                        self.sleep_duration *= 2
+                        continue
+
+        return last_error

queuerpy-0.3.0/core/runner.py

@@ -0,0 +1,274 @@
+"""
+Threadsafe Runner - Go-like async task execution with result channel.
+"""
+
+import multiprocessing
+import threading
+import inspect
+import asyncio
+import traceback
+import queue
+from typing import Callable, Any, Optional, Union
+
+from helper.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class Runner(multiprocessing.Process):
+    """
+    The core execution environment. Manages the execution of a single task
+    in a separate, reliably killable multiprocessing context, and retrieves its result.
+
+    :param task: The synchronous or asynchronous function to execute.
+    :param args: Arguments to pass to the task function.
+    :param name_prefix: Prefix for the process name (e.g., "Job-Runner").
+    """
+
+    def __init__(
+        self,
+        task: Callable[..., Any],
+        *args: Any,
+        **kwargs: Any,
+    ):
+        """
+        Initializes the Runner process.
+
+        :param task: The synchronous or asynchronous function to execute.
+        :param args: Arguments to pass to the task function.
+        """
+        super().__init__(name=task.__name__, daemon=True)
+        self.task = task
+        self.args = args
+        self.kwargs = kwargs
+        self.is_async = inspect.iscoroutinefunction(task)
+        self._result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue(1)
+
+    def go(self):
+        """Starts the Runner process in the background."""
+        self.start()
+
+    def run(self):
+        """
+        The main execution method, executed in the child process.
+        It executes the assigned task, sends the result to the queue, and exits.
+        """
+        try:
+            if self.is_async:
+                result = asyncio.run(self.task(*self.args, **self.kwargs))
+            else:
+                result = self.task(*self.args, **self.kwargs)
+
+            try:
+                self._result_queue.put(result)
+            except BrokenPipeError as e:
+                pass
+        except Exception as e:
+            logger.debug(traceback.format_exc())
+            try:
+                self._result_queue.put(e)
+            except BrokenPipeError as e:
+                pass
+
+    def get_results(self, timeout: Optional[float] = None) -> Any:
+        """
+        Waits for the runner process to complete and returns the result.
+
+        :param timeout: Time in seconds to wait for the result. If None, waits indefinitely.
+        :returns: The result returned by the executed task function.
+        :raises TimeoutError: If the result is not available within the specified timeout.
+        :raises Exception: If the task execution in the runner process failed.
+        """
+        logger.debug(f"Runner.get_results() called for {self.name}, timeout={timeout}")
+        try:
+            self.join(timeout=timeout)
+
+            if self.is_alive():
+                raise TimeoutError(f"runner timed out after {timeout} seconds.")
+
+            if self._result_queue.empty():
+                if self.exitcode == 0:
+                    return None
+                else:
+                    raise Exception(f"runner failed with code {self.exitcode}")
+
+            try:
+                result = self._result_queue.get_nowait()
+            except Exception as e:
+                raise
+
+            if isinstance(result, Exception):
+                raise result
+
+            return result
+        except Exception as e:
+            raise e
+
+    def cancel(self) -> bool:
+        """
+        Attempts to cancel the running process.
+
+        :returns: True if the process was successfully terminated, False otherwise.
+        :raises: Exception if an error occurs during termination.
+        """
+        try:
+            if not self.is_alive():
+                logger.debug(f"Runner {self.name}: Already finished")
+                return True
+
+            logger.debug(f"Cancelling runner {self.name}")
+            self.terminate()
+            self.join(timeout=5.0)  # Wait up to 5 seconds for clean termination
+            result = not self.is_alive()
+            return result
+        except Exception as e:
+            logger.warning(f"Error cancelling runner {self.name}: {e}")
+            return False
+
+
+class SmallRunner(threading.Thread):
+    """
+    A lightweight, thread-based runner suitable for tasks needing
+    direct access to the parent process's shared state (like heartbeats).
+
+    :param task: The synchronous or asynchronous function to execute.
+    :param args: Arguments to pass to the task function.
+    :param name_prefix: Prefix for the thread name (e.g., "Job-SmallRunner").
+    """
+
+    def __init__(
+        self,
+        task: Callable[..., Any],
+        *args: Any,
+        **kwargs: Any,
+    ):
+        """
+        Initializes the SmallRunner thread.
+
+        :param task: The synchronous or asynchronous function to execute.
+        :param args: Arguments to pass to the task function.
+        """
+        super().__init__(name=task.__name__, daemon=True)
+        self.task = task
+        self.args = args
+        self.kwargs = kwargs
+        self.is_async = inspect.iscoroutinefunction(task)
+        self._result_queue: queue.Queue[Any] = queue.Queue(maxsize=1)
+
+    def go(self):
+        """Starts the SmallRunner thread in the background."""
+        self.start()
+
+    def run(self):
+        """
+        The main execution method for the thread.
+        """
+        loop_created = False
+        loop = None
+
+        try:
+            if self.is_async:
+                try:
+                    loop = asyncio.get_running_loop()
+                except RuntimeError:
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+                    loop_created = True
+
+                result = loop.run_until_complete(self.task(*self.args, **self.kwargs))
+            else:
+                result = self.task(*self.args, **self.kwargs)
+
+            self._result_queue.put(result)
+
+        except Exception as e:
+            logger.error(f"SmallRunner {self.name} task failed: {e}")
+            self._result_queue.put(e)
+        finally:
+            if loop_created and loop and not loop.is_closed():
+                try:
+                    pending_tasks = asyncio.all_tasks(loop)
+                    for task in pending_tasks:
+                        task.cancel()
+                    if pending_tasks:
+                        loop.run_until_complete(
+                            asyncio.gather(*pending_tasks, return_exceptions=True)
+                        )
+                    loop.close()
+                except Exception as cleanup_error:
+                    logger.debug(f"error cleaning SmallRunner loop {cleanup_error}")
+
+    def get_results(self, timeout: Optional[float] = None) -> Any:
+        """
+        Waits for the thread to complete and returns the result.
+
+        :param timeout: Time in seconds to wait for the result. If None, waits indefinitely.
+        :returns: The result returned by the executed task function.
+        :raises TimeoutError: If the result is not available within the specified timeout.
+        :raises Exception: If the task execution in the thread failed.
+        """
+        try:
+            self.join(timeout=timeout)
+
+            if self.is_alive():
+                raise TimeoutError(f"small runner timed out after {timeout} seconds")
+
+            try:
+                result = self._result_queue.get(block=False)
+            except queue.Empty:
+                # If queue is empty and thread is done, return None
+                return None
+            except Exception as e:
+                raise
+
+            if isinstance(result, Exception):
+                raise result
+
+            return result
+        except Exception as e:
+            raise e
+
+    def cancel(self) -> bool:
+        """
+        Attempts to cancel the running thread.
+        Note: Python threads cannot be forcefully terminated, so this is a no-op
+        that provides interface consistency with Runner.
+
+        :returns: True if the thread is not alive, False if still running.
+        """
+        try:
+            if not self.is_alive():
+                logger.debug(f"SmallRunner {self.name}: Already finished")
+                return True
+
+            logger.debug(
+                f"SmallRunner {self.name}: Cannot force terminate thread, waiting for natural completion"
+            )
+            # We can't actually cancel a thread in Python, just check if it's done
+            return not self.is_alive()
+
+        except Exception as e:
+            logger.warning(f"Error checking SmallRunner {self.name}: {e}")
+            raise e
+
+
+def go_func(
+    func: Callable[..., Any], use_mp: bool = True, *args: Any, **kwargs: Any
+) -> Union[Runner, SmallRunner]:
+    """
+    Go-like async function execution factory. Selects the appropriate runner.
+
+    :param func: The function to execute.
+    :param use_mp: If True (default), uses the multiprocessing Runner (isolated).
+                   If False, uses the threading SmallRunner (shared state).
+    :param args: Positional arguments for the function.
+    :param kwargs: Keyword arguments for the function.
+    :returns: A running Runner or SmallRunner instance.
+    """
+    if use_mp:
+        runner = Runner(func, *args, **kwargs)
+    else:
+        runner = SmallRunner(func, *args, **kwargs)
+
+    runner.go()
+    return runner