aegis-stack 0.2.0rc2__py3-none-any.whl

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/queues/system.py

@@ -0,0 +1,44 @@
+"""
+System worker queue configuration.
+
+Handles system maintenance and monitoring tasks using native arq patterns.
+"""
+
+# Import system tasks
+from app.components.worker.tasks.simple_system_tasks import (
+    cleanup_temp_files,
+    system_health_check,
+)
+from app.core.config import settings
+from arq.connections import RedisSettings
+
+
+class WorkerSettings:
+    """System maintenance worker configuration."""
+
+    # Human-readable description
+    description = "System maintenance and monitoring tasks"
+
+    # Task functions for this queue
+    functions = [
+        system_health_check,
+        cleanup_temp_files,
+    ]
+
+    # arq configuration with improved connection settings
+    base_settings = RedisSettings.from_dsn(settings.REDIS_URL)
+    redis_settings = RedisSettings(
+        host=base_settings.host,
+        port=base_settings.port,
+        database=base_settings.database,
+        password=base_settings.password,
+        conn_timeout=settings.REDIS_CONN_TIMEOUT,
+        conn_retries=settings.REDIS_CONN_RETRIES,
+        conn_retry_delay=settings.REDIS_CONN_RETRY_DELAY,
+    )
+    queue_name = "arq:queue:system"
+    max_jobs = 15  # Moderate concurrency for administrative operations
+    job_timeout = 300  # 5 minutes
+    keep_result = settings.WORKER_KEEP_RESULT_SECONDS
+    max_tries = settings.WORKER_MAX_TRIES
+    health_check_interval = settings.WORKER_HEALTH_CHECK_INTERVAL

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/registry.py

@@ -0,0 +1,139 @@
+"""
+Worker queue registry with dynamic discovery.
+
+Pure arq implementation - WorkerSettings classes are the single source of truth.
+No configuration files, no abstractions, just arq as intended.
+"""
+
+import importlib
+from pathlib import Path
+from typing import Any
+
+from app.core.log import logger
+
+
+def get_worker_settings(queue_name: str) -> Any:
+    """Import and return WorkerSettings class for a queue.
+
+    Args:
+        queue_name: Name of the queue (e.g., 'system', 'load_test')
+
+    Returns:
+        WorkerSettings class from the queue module
+
+    Raises:
+        ImportError: If queue module doesn't exist
+        AttributeError: If WorkerSettings class not found
+    """
+    try:
+        module = importlib.import_module(f"app.components.worker.queues.{queue_name}")
+        return module.WorkerSettings
+    except ImportError as e:
+        logger.error(f"Failed to import worker queue '{queue_name}': {e}")
+        raise
+    except AttributeError as e:
+        logger.error(f"WorkerSettings class not found in '{queue_name}' queue: {e}")
+        raise
+
+
+def discover_worker_queues() -> list[str]:
+    """Discover all worker queues from the queues directory.
+
+    Scans app/components/worker/queues/ for Python files and treats each
+    file as a potential queue. Excludes __init__.py and other non-queue files.
+
+    Returns:
+        Sorted list of queue names
+    """
+    queues_dir = Path(__file__).parent / "queues"
+
+    if not queues_dir.exists():
+        logger.warning(f"Worker queues directory not found: {queues_dir}")
+        return []
+
+    queue_files = queues_dir.glob("*.py")
+    queues = []
+
+    for file in queue_files:
+        # Skip __init__.py and other special files
+        if file.stem not in ["__init__", "__pycache__"]:
+            # Verify the file has a WorkerSettings class
+            try:
+                get_worker_settings(file.stem)
+                queues.append(file.stem)
+            except (ImportError, AttributeError):
+                logger.debug(f"Skipping '{file.stem}' - no valid WorkerSettings class")
+                continue
+
+    return sorted(queues)
+
+
+def get_queue_metadata(queue_name: str) -> dict[str, Any]:
+    """Get metadata for a queue from its WorkerSettings class.
+
+    Args:
+        queue_name: Name of the queue
+
+    Returns:
+        Dictionary with queue metadata:
+        - queue_name: Redis queue name
+        - max_jobs: Maximum concurrent jobs
+        - timeout: Job timeout in seconds
+        - functions: List of function names in this queue
+        - description: Human-readable description (if available)
+    """
+    try:
+        settings_class = get_worker_settings(queue_name)
+
+        metadata = {
+            "queue_name": getattr(
+                settings_class, "queue_name", f"arq:queue:{queue_name}"
+            ),
+            "max_jobs": getattr(settings_class, "max_jobs", 10),
+            "timeout": getattr(settings_class, "job_timeout", 300),
+            "functions": [f.__name__ for f in getattr(settings_class, "functions", [])],
+        }
+
+        # Add description if available
+        if hasattr(settings_class, "description"):
+            metadata["description"] = settings_class.description
+        elif hasattr(settings_class, "__doc__") and settings_class.__doc__:
+            metadata["description"] = settings_class.__doc__.strip()
+        else:
+            metadata["description"] = f"{queue_name.title()} worker queue"
+
+        return metadata
+
+    except (ImportError, AttributeError) as e:
+        logger.error(f"Failed to get metadata for queue '{queue_name}': {e}")
+        return {
+            "queue_name": f"arq:queue:{queue_name}",
+            "max_jobs": 10,
+            "timeout": 300,
+            "functions": [],
+            "description": f"Unknown queue: {queue_name}",
+        }
+
+
+def get_all_queue_metadata() -> dict[str, dict[str, Any]]:
+    """Get metadata for all discovered worker queues.
+
+    Returns:
+        Dictionary mapping queue names to their metadata
+    """
+    metadata = {}
+    for queue_name in discover_worker_queues():
+        metadata[queue_name] = get_queue_metadata(queue_name)
+    return metadata
+
+
+def validate_queue_name(queue_name: str) -> bool:
+    """Check if a queue name is valid (has a corresponding WorkerSettings).
+
+    Args:
+        queue_name: Name to validate
+
+    Returns:
+        True if queue exists and has valid WorkerSettings
+    """
+    return queue_name in discover_worker_queues()

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/tasks/__init__.py

@@ -0,0 +1,120 @@
+"""
+Worker tasks registry.
+
+This module collects all available worker tasks and exports them for the arq worker.
+Only includes production-ready, actually useful tasks.
+"""
+
+from collections.abc import Callable
+from typing import Any
+
+from app.core.config import get_default_queue
+
+from .load_tasks import (
+    cpu_intensive_task,
+    failure_testing_task,
+    io_simulation_task,
+    memory_operations_task,
+)
+from .system_tasks import (
+    load_test_orchestrator,
+)
+
+# All task functions available to arq workers
+TASK_FUNCTIONS: list[Callable[..., Any]] = [
+    # Load testing orchestrator
+    load_test_orchestrator,
+    # Load testing tasks
+    cpu_intensive_task,
+    io_simulation_task,
+    memory_operations_task,
+    failure_testing_task,
+]
+
+
+def get_task_by_name(task_name: str) -> Callable[..., Any] | None:
+    """
+    Get task function by name.
+
+    Args:
+        task_name: Name of the task function
+
+    Returns:
+        Task function or None if not found
+    """
+    for task_func in TASK_FUNCTIONS:
+        if task_func.__name__ == task_name:
+            return task_func
+    return None
+
+
+def list_available_tasks() -> list[str]:
+    """
+    Get list of all available task names.
+
+    Returns:
+        List of task function names
+    """
+    return [task_func.__name__ for task_func in TASK_FUNCTIONS]
+
+
+def get_queue_functions(queue_type: str) -> list[Callable[..., Any]]:
+    """
+    Get task functions specific to a queue type.
+
+    Args:
+        queue_type: The functional queue type ("media", "system", "load_test")
+
+    Returns:
+        List of task functions appropriate for this queue
+    """
+    # Function distribution by queue type
+    queue_function_map = {
+        "system": [
+            # System queue is for actual system tasks when needed
+            # Currently empty - add real system tasks here when required
+        ],
+        "media": [
+            # Future: Image processing, video encoding, file operations
+            # Currently empty - real media tasks will be added here
+        ],
+        "load_test": [
+            # Load testing orchestrator
+            load_test_orchestrator,
+            # Load testing tasks (synthetic workloads)
+            cpu_intensive_task,
+            io_simulation_task,
+            memory_operations_task,
+            failure_testing_task,
+        ],
+    }
+
+    from typing import cast
+
+    return cast(list[Callable[..., Any]], queue_function_map.get(queue_type, []))
+
+
+def get_queue_for_task(task_name: str) -> str:
+    """
+    Get the appropriate queue type for a given task.
+
+    Args:
+        task_name: Name of the task function
+
+    Returns:
+        Queue type that should handle this task
+    """
+    # Task to queue mapping
+    task_queue_map = {
+        # Load test tasks
+        "load_test_orchestrator": "load_test",
+        "cpu_intensive_task": "load_test",
+        "io_simulation_task": "load_test",
+        "memory_operations_task": "load_test",
+        "failure_testing_task": "load_test",
+        # Future system and media tasks would go here
+    }
+
+    return task_queue_map.get(
+        task_name, get_default_queue()
+    )  # Default to configured default queue