queuemgr 1.0.0__py3-none-any.whl

queuemgr/__init__.py

@@ -0,0 +1,13 @@
+"""
+Queue Manager - Full-featured job queue system with multiprocessing support for Linux.
+
+A production-ready job queue system with automatic process management,
+real-time monitoring, systemd integration, and multiple interfaces (CLI, web).
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+__version__ = "1.0.1"
+__author__ = "Vasiliy Zdanovskiy"
+__email__ = "vasilyvz@gmail.com"

queuemgr/constants.py

@@ -0,0 +1,70 @@
+"""
+Global constants and configuration keys for the queue manager system.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+from typing import Final
+
+# Default timeout values (in seconds)
+DEFAULT_STOP_TIMEOUT: Final[float] = 10.0
+DEFAULT_SHUTDOWN_TIMEOUT: Final[float] = 30.0
+DEFAULT_TERMINATE_TIMEOUT: Final[float] = 5.0
+
+# Progress limits
+MIN_PROGRESS: Final[int] = 0
+MAX_PROGRESS: Final[int] = 100
+
+# Registry configuration
+DEFAULT_REGISTRY_PATH: Final[str] = "runtime/registry.jsonl"
+REGISTRY_ENCODING: Final[str] = "utf-8"
+
+# Process configuration
+DEFAULT_PROCESS_NAME_PREFIX: Final[str] = "Job-"
+DEFAULT_JOB_LOOP_DELAY: Final[float] = 0.1
+
+# File system configuration
+DEFAULT_RUNTIME_DIR: Final[str] = "runtime"
+DEFAULT_EXAMPLES_DIR: Final[str] = "examples"
+
+# Error messages
+ERROR_JOB_NOT_FOUND: Final[str] = "Job with ID '{job_id}' not found"
+ERROR_JOB_ALREADY_EXISTS: Final[str] = "Job with ID '{job_id}' already exists"
+ERROR_INVALID_JOB_STATE: Final[str] = (
+    "Cannot {operation} job '{job_id}' in state '{state}'"
+)
+ERROR_JOB_EXECUTION_FAILED: Final[str] = "Job '{job_id}' execution failed"
+ERROR_REGISTRY_OPERATION_FAILED: Final[str] = "Registry operation failed: {message}"
+ERROR_PROCESS_CONTROL_FAILED: Final[str] = (
+    "Process control error for job '{job_id}' during {operation}"
+)
+ERROR_VALIDATION_FAILED: Final[str] = "Validation error for field '{field}': {reason}"
+ERROR_OPERATION_TIMEOUT: Final[str] = (
+    "Operation '{operation}' timed out after {timeout} seconds"
+)
+
+# Status descriptions
+DESCRIPTION_JOB_CREATED: Final[str] = "Job created"
+DESCRIPTION_JOB_STARTED: Final[str] = "Job started"
+DESCRIPTION_JOB_STOPPED: Final[str] = "Job stopped by user"
+DESCRIPTION_JOB_DELETED: Final[str] = "Job deleted"
+DESCRIPTION_JOB_COMPLETED: Final[str] = "Job completed successfully"
+DESCRIPTION_JOB_FAILED: Final[str] = "Job failed: {error}"
+DESCRIPTION_JOB_INTERRUPTED: Final[str] = "Job interrupted"
+
+# Configuration keys
+CONFIG_REGISTRY_PATH: Final[str] = "registry_path"
+CONFIG_MAX_CONCURRENT_JOBS: Final[str] = "max_concurrent_jobs"
+CONFIG_DEFAULT_TIMEOUT: Final[str] = "default_timeout"
+CONFIG_LOG_LEVEL: Final[str] = "log_level"
+CONFIG_LOG_FORMAT: Final[str] = "log_format"
+
+# Logging configuration
+DEFAULT_LOG_LEVEL: Final[str] = "INFO"
+DEFAULT_LOG_FORMAT: Final[str] = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Version information
+VERSION: Final[str] = "0.1.0"
+AUTHOR: Final[str] = "Vasiliy Zdanovskiy"
+EMAIL: Final[str] = "vasilyvz@gmail.com"

queuemgr/core/__init__.py

@@ -0,0 +1,6 @@
+"""
+Core components for the queue manager system.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""

queuemgr/core/exceptions.py

@@ -0,0 +1,61 @@
+"""
+Core exceptions for IPC and registry operations.
+
+This module contains specific exceptions for core functionality.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+from ..exceptions import QueueManagerError
+
+
+class IPCError(QueueManagerError):
+    """Raised when IPC operations fail."""
+
+    def __init__(self, operation: str, message: str) -> None:
+        """
+        Initialize IPCError.
+        
+        Args:
+            operation: The IPC operation that failed.
+            message: Error message.
+        """
+        self.operation = operation
+        super().__init__(f"IPC error during {operation}: {message}")
+
+
+class RegistryError(QueueManagerError):
+    """Raised when registry operations fail."""
+
+    def __init__(self, message: str, original_error: Exception = None) -> None:
+        """
+        Initialize RegistryError.
+        
+        Args:
+            message: Error message.
+            original_error: The original exception that caused the failure.
+        """
+        self.original_error = original_error
+        super().__init__(f"Registry error: {message}")
+
+
+class ProcessControlError(QueueManagerError):
+    """Raised when process control operations fail."""
+
+    def __init__(self, job_id: str, operation: str, original_error: Exception = None) -> None:
+        """
+        Initialize ProcessControlError.
+        
+        Args:
+            job_id: The job ID.
+            operation: The operation that failed.
+            original_error: The original exception that caused the failure.
+        """
+        self.job_id = job_id
+        self.operation = operation
+        self.original_error = original_error
+        error_msg = f"Process control error for job '{job_id}' during {operation}"
+        if original_error:
+            error_msg += f": {original_error}"
+        super().__init__(error_msg)

queuemgr/core/ipc.py

@@ -0,0 +1,38 @@
+"""
+Inter-process communication for Queue Manager.
+
+This module provides IPC functionality for job state management,
+commands, and progress tracking using multiprocessing.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+# Import all functionality from separate modules
+from .ipc_manager import get_manager, create_job_shared_state
+from .ipc_operations import (
+    with_job_lock,
+    update_job_state,
+    read_job_state,
+    set_command,
+    get_command,
+    clear_command,
+    set_status,
+    set_progress,
+    get_progress
+)
+
+# Re-export for backward compatibility
+__all__ = [
+    'get_manager',
+    'create_job_shared_state',
+    'with_job_lock',
+    'update_job_state',
+    'read_job_state',
+    'set_command',
+    'get_command',
+    'clear_command',
+    'set_status',
+    'set_progress',
+    'get_progress'
+]

queuemgr/core/ipc_manager.py

@@ -0,0 +1,54 @@
+"""
+IPC manager for creating and managing shared state.
+
+This module contains the manager creation and shared state
+management functionality.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+from multiprocessing import Manager
+from typing import Dict, Any
+
+
+def get_manager() -> Dict[str, Any]:
+    """
+    Return a process-shared Manager instance for the queue runtime.
+
+    Returns:
+        Manager: A multiprocessing Manager instance for creating shared
+        objects.
+    """
+    return Manager()
+
+
+def create_job_shared_state(manager: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Create and return shared variables for a job.
+
+    Creates shared state including status, command, progress, description,
+    result, and mutex. All shared variables are thread/process safe and can
+    be accessed from multiple processes.
+
+    Args:
+        manager: Multiprocessing Manager instance.
+
+    Returns:
+        Dict containing shared state variables:
+        - status: Shared integer for job status
+        - command: Shared integer for job command
+        - progress: Shared integer for job progress (0-100)
+        - description: Shared string for job description
+        - result: Shared value for job result
+        - lock: Shared mutex for thread safety
+    """
+    shared_state = {
+        "status": manager.Value("i", 0),  # JobStatus enum value
+        "command": manager.Value("i", 0),  # JobCommand enum value
+        "progress": manager.Value("i", 0),  # 0-100
+        "description": manager.Value("c", b""),  # UTF-8 encoded string
+        "result": manager.Value("O", None),  # Any Python object
+        "lock": manager.Lock(),  # Mutex for thread safety
+    }
+    return shared_state

queuemgr/core/ipc_operations.py

@@ -0,0 +1,352 @@
+"""
+IPC operations for job state management.
+
+This module contains the IPC operations for managing job state,
+commands, and progress.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+from contextlib import contextmanager
+from typing import Dict, Any, Generator, Optional, Union, List
+from .types import JobCommand, JobStatus
+from .exceptions import IPCError
+
+
+@contextmanager
+def with_job_lock(shared_state: Dict[str, Any]) -> Generator[None, None, None]:
+    """
+    Context manager for acquiring the job's mutex for consistent updates/reads.
+
+    Ensures atomic access to multiple shared fields by acquiring the job's lock
+    before performing operations and releasing it when done.
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+
+    Yields:
+        None: Context manager yields control to the caller.
+
+    Raises:
+        ValueError: If shared_state doesn't contain a 'lock' key.
+    """
+    lock = shared_state.get("lock")
+    if lock is None:
+        raise ValueError("Shared state must contain a 'lock' key")
+
+    try:
+        lock.acquire()
+        yield
+    finally:
+        try:
+            lock.release()
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+
+
+def update_job_state(
+    shared_state: Dict[str, Any],
+    status: JobStatus = None,
+    command: JobCommand = None,
+    progress: int = None,
+    description: str = None,
+    result: Optional[Union[str, int, float, bool, Dict[str, Any], List[Any]]] = None,
+) -> None:
+    """
+    Update job state variables atomically.
+
+    Updates multiple shared state variables in a single atomic operation
+    to ensure consistency across processes.
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+        status: New job status (optional).
+        command: New job command (optional).
+        progress: New job progress 0-100 (optional).
+        description: New job description (optional).
+        result: New job result (optional).
+
+    Raises:
+        ValueError: If shared_state doesn't contain required keys.
+        ValueError: If progress is not between 0 and 100.
+    """
+    lock = shared_state.get("lock")
+    if lock is None:
+        raise ValueError("Shared state must contain a 'lock' key")
+
+    if progress is not None and not (0 <= progress <= 100):
+        raise ValueError("Progress must be between 0 and 100")
+
+    try:
+        lock.acquire()
+        
+        if status is not None:
+            try:
+                shared_state["status"].value = status.value
+            except (BrokenPipeError, ConnectionResetError):
+                # Process is shutting down, ignore
+                pass
+        
+        if command is not None:
+            try:
+                shared_state["command"].value = command.value
+            except (BrokenPipeError, ConnectionResetError):
+                # Process is shutting down, ignore
+                pass
+        
+        if progress is not None:
+            try:
+                shared_state["progress"].value = progress
+            except (BrokenPipeError, ConnectionResetError):
+                # Process is shutting down, ignore
+                pass
+        
+        if description is not None:
+            try:
+                shared_state["description"].value = description.encode("utf-8")
+            except (BrokenPipeError, ConnectionResetError):
+                # Process is shutting down, ignore
+                pass
+        
+        if result is not None:
+            try:
+                shared_state["result"].value = result
+            except (BrokenPipeError, ConnectionResetError):
+                # Process is shutting down, ignore
+                pass
+                
+    finally:
+        try:
+            lock.release()
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+
+
+def read_job_state(shared_state: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Read all job state variables atomically.
+
+    Reads all shared state variables in a single atomic operation
+    to ensure consistency across processes.
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+
+    Returns:
+        Dictionary containing current job state:
+        - status: Current job status
+        - command: Current job command
+        - progress: Current job progress (0-100)
+        - description: Current job description
+        - result: Current job result
+
+    Raises:
+        ValueError: If shared_state doesn't contain required keys.
+    """
+    lock = shared_state.get("lock")
+    if lock is None:
+        raise ValueError("Shared state must contain a 'lock' key")
+
+    try:
+        lock.acquire()
+        try:
+            return {
+                "status": JobStatus(shared_state["status"].value),
+                "command": JobCommand(shared_state["command"].value),
+                "progress": shared_state["progress"].value,
+                "description": (
+                    shared_state["description"].value.decode("utf-8")
+                    if shared_state["description"].value
+                    else ""
+                ),
+                "result": shared_state["result"].value,
+            }
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, return default state
+            return {
+                "status": JobStatus.PENDING,
+                "command": JobCommand.NONE,
+                "progress": 0,
+                "description": "",
+                "result": None,
+            }
+    finally:
+        try:
+            lock.release()
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+
+
+def set_command(shared_state: Dict[str, Any], command: JobCommand) -> None:
+    """
+    Set the job command.
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+        command: Command to set.
+
+    Raises:
+        ValueError: If shared_state doesn't contain required keys.
+    """
+    lock = shared_state.get("lock")
+    if lock is None:
+        raise ValueError("Shared state must contain a 'lock' key")
+
+    try:
+        lock.acquire()
+        try:
+            shared_state["command"].value = command.value
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+    finally:
+        try:
+            lock.release()
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+
+
+def get_command(shared_state: Dict[str, Any]) -> JobCommand:
+    """
+    Get the current command for the job.
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+
+    Returns:
+        Current job command.
+
+    Raises:
+        ValueError: If shared_state doesn't contain required keys.
+    """
+    lock = shared_state.get("lock")
+    if lock is None:
+        raise ValueError("Shared state must contain a 'lock' key")
+
+    try:
+        lock.acquire()
+        try:
+            return JobCommand(shared_state["command"].value)
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, return NONE command
+            return JobCommand.NONE
+    finally:
+        try:
+            lock.release()
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+
+
+def clear_command(shared_state: Dict[str, Any]) -> None:
+    """
+    Clear the job command (set to NONE).
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+
+    Raises:
+        ValueError: If shared_state doesn't contain required keys.
+    """
+    set_command(shared_state, JobCommand.NONE)
+
+
+def set_status(shared_state: Dict[str, Any], status: JobStatus) -> None:
+    """
+    Set the job status.
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+        status: Status to set.
+
+    Raises:
+        ValueError: If shared_state doesn't contain required keys.
+    """
+    lock = shared_state.get("lock")
+    if lock is None:
+        raise ValueError("Shared state must contain a 'lock' key")
+
+    try:
+        lock.acquire()
+        try:
+            shared_state["status"].value = status.value
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+    finally:
+        try:
+            lock.release()
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+
+
+def set_progress(shared_state: Dict[str, Any], progress: int) -> None:
+    """
+    Set the job progress.
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+        progress: Progress value (0-100).
+
+    Raises:
+        ValueError: If shared_state doesn't contain required keys.
+        ValueError: If progress is not between 0 and 100.
+    """
+    if not (0 <= progress <= 100):
+        raise ValueError("Progress must be between 0 and 100")
+
+    lock = shared_state.get("lock")
+    if lock is None:
+        raise ValueError("Shared state must contain a 'lock' key")
+
+    try:
+        lock.acquire()
+        try:
+            shared_state["progress"].value = progress
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+    finally:
+        try:
+            lock.release()
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass
+
+
+def get_progress(shared_state: Dict[str, Any]) -> int:
+    """
+    Get job progress with minimal lock time.
+
+    Args:
+        shared_state: Dictionary containing shared state variables.
+
+    Returns:
+        Current job progress (0-100).
+
+    Raises:
+        ValueError: If shared_state doesn't contain required keys.
+    """
+    lock = shared_state.get("lock")
+    if lock is None:
+        raise ValueError("Shared state must contain a 'lock' key")
+
+    try:
+        lock.acquire()
+        try:
+            return int(shared_state["progress"].value)
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, return default progress
+            return 0
+    finally:
+        try:
+            lock.release()
+        except (BrokenPipeError, ConnectionResetError):
+            # Process is shutting down, ignore
+            pass