aegis-stack 0.1.0__py3-none-any.whl

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/scheduler/main.py

@@ -0,0 +1,138 @@
+"""
+Scheduler component for {{ cookiecutter.project_name }}.
+
+Simple, explicit job scheduling - just import functions and schedule them.
+Add your own jobs by importing service functions and calling scheduler.add_job().
+"""
+
+import asyncio
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+from app.core.log import logger
+from app.services.system.health import check_system_status, register_health_check
+from app.services.system.models import ComponentStatus, ComponentStatusType
+
+# Global scheduler instance for health checking
+_scheduler: AsyncIOScheduler | None = None
+
+
+async def _check_scheduler_health() -> ComponentStatus:
+    """Health check for the scheduler component."""
+    global _scheduler
+
+    if _scheduler is None:
+        return ComponentStatus(
+            name="scheduler",
+            status=ComponentStatusType.UNHEALTHY,
+            message="Scheduler not initialized",
+            response_time_ms=None,
+        )
+
+    if not _scheduler.running:
+        return ComponentStatus(
+            name="scheduler",
+            status=ComponentStatusType.UNHEALTHY,
+            message="Scheduler is not running",
+            response_time_ms=None,
+        )
+
+    # Get scheduler statistics
+    jobs = _scheduler.get_jobs()
+    job_count = len(jobs)
+
+    # Check if scheduler is responsive
+    try:
+        state = _scheduler.state
+        healthy = state == 1  # STATE_RUNNING = 1
+
+        status = (
+            ComponentStatusType.HEALTHY
+            if healthy
+            else ComponentStatusType.UNHEALTHY
+        )
+        return ComponentStatus(
+            name="scheduler",
+            status=status,
+            message=f"Scheduler running with {job_count} jobs",
+            response_time_ms=None,
+            metadata={
+                "job_count": job_count,
+                "state": state,
+                "jobs": [{"id": job.id, "name": job.name} for job in jobs[:5]],
+            },
+        )
+    except Exception as e:
+        return ComponentStatus(
+            name="scheduler",
+            status=ComponentStatusType.UNHEALTHY,
+            message=f"Scheduler health check failed: {str(e)}",
+            response_time_ms=None,
+        )
+
+
+def create_scheduler() -> AsyncIOScheduler:
+    """Create and configure the scheduler with all jobs."""
+    scheduler = AsyncIOScheduler()
+
+    # ============================================================================
+    # JOB SCHEDULE CONFIGURATION
+    # Add your scheduled jobs below - import service functions and schedule them!
+    # ============================================================================
+
+    # System health check every 5 minutes
+    # Adjust this interval based on your monitoring needs:
+    # - Production systems: 1-5 minutes
+    # - Development: 10-15 minutes
+    # - High-availability: 30 seconds - 1 minute
+    scheduler.add_job(
+        check_system_status,
+        trigger="interval",
+        minutes=1,
+        id="system_status_check",
+        name="System Health Check",
+        max_instances=1,  # Prevent overlapping health checks
+        coalesce=True,  # Coalesce missed executions
+    )
+
+    # Add your own scheduled jobs here by importing service functions
+    # and calling scheduler.add_job() with your custom business logic
+
+    return scheduler
+
+
+async def run_scheduler() -> None:
+    """Main scheduler runner with lifecycle management."""
+    global _scheduler
+
+    logger.info("🕒 Starting {{ cookiecutter.project_name }} Scheduler")
+
+    scheduler = create_scheduler()
+    _scheduler = scheduler  # Store for health checking
+
+    try:
+        scheduler.start()
+        logger.info("✅ Scheduler started successfully")
+        logger.info(f"📋 {len(scheduler.get_jobs())} jobs scheduled:")
+
+        for job in scheduler.get_jobs():
+            logger.info(f"   • {job.name} - {job.trigger}")
+
+        # Register scheduler health check with the system health service
+        register_health_check("scheduler", _check_scheduler_health)
+        logger.info("🩺 Scheduler health check registered")
+
+        # Keep the scheduler running
+        while True:
+            await asyncio.sleep(1)
+
+    except KeyboardInterrupt:
+        logger.info("🛑 Received shutdown signal")
+    except Exception as e:
+        logger.error(f"❌ Scheduler error: {e}")
+        raise
+    finally:
+        if scheduler.running:
+            scheduler.shutdown()
+            logger.info("✅ Scheduler stopped gracefully")
+        _scheduler = None  # Clear global reference

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/CLAUDE.md

@@ -0,0 +1,213 @@
+# Worker Component Development Guide
+
+This guide covers arq worker architecture patterns and development for the Aegis Stack worker component.
+
+## Worker Architecture (arq)
+
+Aegis Stack uses pure **arq patterns** without custom wrappers, following native arq CLI and configuration patterns.
+
+### Worker Configuration Structure
+
+Each worker queue has its own `WorkerSettings` class:
+- `app/components/worker/queues/system.py` - System maintenance worker
+- `app/components/worker/queues/load_test.py` - Load testing worker  
+- `app/components/worker/queues/media.py` - Media processing worker
+
+### Worker Services in Docker
+
+Workers run as separate Docker services with specific names:
+- **`worker-system`** - System maintenance tasks (low concurrency, high reliability)
+- **`worker-load-test`** - High-concurrency load testing (up to 50 concurrent jobs)
+- **`worker-media`** - File/media processing (commented out by default)
+
+## Adding Worker Tasks
+
+### 1. Create Task Functions
+Tasks are pure async functions in `app/components/worker/tasks/`:
+```python
+# app/components/worker/tasks/my_tasks.py
+async def my_background_task() -> dict[str, str]:
+    """My custom background task."""
+    logger.info("Running my background task")
+    
+    # Your task logic here
+    await asyncio.sleep(1)  # Simulate work
+    
+    return {
+        "status": "completed",
+        "timestamp": datetime.now(UTC).isoformat(),
+        "task": "my_background_task"
+    }
+```
+
+### 2. Register with Worker Queue
+Import and add to the appropriate `WorkerSettings`:
+```python
+# app/components/worker/queues/system.py
+from app.components.worker.tasks.my_tasks import my_background_task
+
+class WorkerSettings:
+    functions = [
+        system_health_check,
+        cleanup_temp_files,
+        my_background_task,  # Add your task here
+    ]
+    
+    # Standard arq configuration
+    redis_settings = RedisSettings.from_dsn(settings.REDIS_URL)
+    queue_name = "arq:queue:system"
+    max_jobs = 15
+    job_timeout = 300
+```
+
+## Native arq CLI Usage
+
+### Worker Health Checks
+```bash
+# Check if workers can connect to Redis and validate configuration
+uv run python -m arq app.components.worker.queues.system.WorkerSettings --check
+uv run python -m arq app.components.worker.queues.load_test.WorkerSettings --check
+uv run python -m arq app.components.worker.queues.media.WorkerSettings --check
+```
+
+### Local Worker Development
+```bash
+# Run worker locally with auto-reload for development
+uv run python -m arq app.components.worker.queues.system.WorkerSettings --watch app/
+
+# Run worker in burst mode (process all jobs and exit)
+uv run python -m arq app.components.worker.queues.system.WorkerSettings --burst
+```
+
+### Worker Configuration in Health Checks
+
+The health system reads worker configuration from `app/core/config.py` but workers themselves use their own `WorkerSettings` classes:
+
+```python
+# Health system reads this for monitoring
+WORKER_QUEUES: dict[str, dict[str, Any]] = {
+    "system": {
+        "description": "System maintenance and monitoring tasks",
+        "max_jobs": 15,
+        "timeout_seconds": 300,
+        "queue_name": "arq:queue:system",
+    },
+    "load_test": {
+        "description": "Load testing and performance testing",
+        "max_jobs": 50,
+        "timeout_seconds": 60,
+        "queue_name": "arq:queue:load_test",
+    }
+}
+
+# But workers use their own WorkerSettings classes for actual configuration
+```
+
+## Key Differences from Custom Worker Systems
+
+### ✅ What We Do (Pure arq):
+- Use native arq CLI: `python -m arq WorkerSettings`
+- Standard `WorkerSettings` classes with `functions` list
+- Direct task imports into worker configurations
+- Native arq health checking and monitoring
+
+### ❌ What We Don't Do (Avoided custom patterns):
+- Custom worker wrapper classes
+- Central worker registry systems
+- Custom CLI commands for workers
+- Configuration-driven task discovery
+
+This approach keeps workers transparent and lets developers use arq exactly as documented in the official arq documentation.
+
+## Docker Worker Debugging Commands
+
+### View Worker Logs
+```bash
+# View specific worker logs
+docker compose logs worker-system             # System worker logs
+docker compose logs worker-load-test          # Load test worker logs
+docker compose logs -f worker-system          # Follow system worker in real-time
+docker compose logs -f worker-load-test       # Follow load test worker in real-time
+
+# View all workers at once
+docker compose logs -f worker-system worker-load-test
+
+# Filter for errors in specific workers
+docker compose logs worker-load-test | grep "ERROR\|failed\|TypeError"
+
+# Monitor worker processes and resources
+docker compose exec worker-system ps aux      # Check system worker processes
+docker compose exec worker-load-test ps aux   # Check load test worker processes
+docker stats worker-system worker-load-test   # Monitor resource usage
+docker compose restart worker-system          # Restart specific worker
+```
+
+### Essential Docker Log Monitoring
+
+**Check Worker Logs for Load Test Issues:**
+```bash
+# View real-time worker logs
+docker compose logs -f worker
+
+# Check specific container logs  
+docker logs <container-id>
+
+# View logs with timestamps
+docker compose logs --timestamps worker
+
+# Search logs for specific errors
+docker compose logs worker | grep "TypeError\|failed"
+
+# Check all service logs
+docker compose logs -f
+```
+
+**Load Test Debugging Workflow:**
+1. **Run Load Test**: `uv run full-stack load-test run --type io_simulation --tasks 10`
+2. **Monitor Worker Logs**: `docker compose logs -f worker-load-test` (in separate terminal)
+3. **Look for**: 
+   - `TypeError` messages indicating parameter mismatches
+   - Task completion vs failure counts (`j_complete=X j_failed=Y`)
+   - Task execution times and errors
+   - Redis connection issues
+
+**Common Error Patterns:**
+- `TypeError: function() got an unexpected keyword argument 'param'` - Parameter mismatch between orchestrator and task function
+- `j_failed=X` increasing rapidly - Worker tasks failing due to code issues
+- `Redis connection failed` - Infrastructure connectivity problems
+- `delayed=X.XXs` - Queue saturation or worker overload
+
+**System Health Verification:**
+```bash
+# Check all containers
+docker compose ps
+
+# Check system health via API
+uv run full-stack health status --detailed
+
+# Monitor Redis connection
+docker compose logs redis
+```
+
+## Worker Development Best Practices
+
+### Task Design Patterns
+1. **Pure Functions** - Tasks should be self-contained with minimal dependencies
+2. **Error Handling** - Always include try/catch with proper logging
+3. **Return Values** - Return structured data for monitoring and debugging
+4. **Timeouts** - Set appropriate timeouts for different task types
+5. **Retry Logic** - Use arq's built-in retry mechanisms
+
+### Queue Management
+1. **Separate Concerns** - Use different queues for different types of work
+2. **Concurrency Limits** - Set appropriate max_jobs for each queue type
+3. **Priority Queues** - Use different queues for different priorities
+4. **Dead Letter Queues** - Monitor failed jobs and implement recovery
+
+### Monitoring and Observability
+1. **Structured Logging** - Use structured logs for easy parsing
+2. **Metrics Collection** - Track task execution times and success rates
+3. **Health Checks** - Implement health checks for worker availability
+4. **Alerting** - Set up alerts for queue depth and failure rates
+
+This approach ensures workers are maintainable, debuggable, and follow established patterns.

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/__init__.py

@@ -0,0 +1,6 @@
+"""
+Worker component for background task processing.
+
+This component handles asynchronous background tasks using arq (Redis-based queues).
+Tasks are organized into priority queues (high, medium, low) and by functional area.
+"""

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/constants.py.j2

@@ -0,0 +1,30 @@
+"""
+Worker component constants.
+
+This module contains constants specific to the worker component,
+keeping them separate from global application constants.
+"""
+
+from enum import Enum
+
+
+class TaskNames:
+    """Worker task function names - must match actual function names in code."""
+    
+    # Orchestrator
+    LOAD_TEST_ORCHESTRATOR = "load_test_orchestrator"
+    
+    # Load test tasks
+    CPU_INTENSIVE_TASK = "cpu_intensive_task"
+    IO_SIMULATION_TASK = "io_simulation_task"
+    MEMORY_OPERATIONS_TASK = "memory_operations_task"
+    FAILURE_TESTING_TASK = "failure_testing_task"
+
+
+class LoadTestTypes(Enum):
+    """Load test type identifiers for task selection."""
+    
+    CPU_INTENSIVE = "cpu_intensive"
+    IO_SIMULATION = "io_simulation"
+    MEMORY_OPERATIONS = "memory_operations"
+    FAILURE_TESTING = "failure_testing"

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/pools.py

@@ -0,0 +1,78 @@
+"""
+Worker pool management for client-side task enqueueing.
+
+This module provides Redis connection pooling and caching for enqueueing tasks
+to worker queues. Separated from worker management to allow clean architectural
+separation between client-side enqueueing and worker-side processing.
+"""
+
+from arq import create_pool
+from arq.connections import ArqRedis, RedisSettings
+
+from app.core.config import get_default_queue, settings
+from app.core.log import logger
+
+# Global pool cache to avoid creating new Redis connections repeatedly
+_pool_cache: dict[str, ArqRedis] = {}
+
+
+async def get_queue_pool(queue_type: str | None = None) -> tuple[ArqRedis, str]:
+    """
+    Get Redis pool for enqueuing tasks to specific functional queue.
+
+    Uses connection pooling to avoid creating new Redis connections repeatedly.
+
+    Args:
+        queue_type: Functional queue type (defaults to configured default queue)
+
+    Returns:
+        Tuple of (pool, queue_name) for enqueueing tasks
+    """
+    # Use configured default queue if not specified
+    if queue_type is None:
+        queue_type = get_default_queue()
+
+    from app.core.config import is_valid_queue, get_available_queues
+    
+    if not is_valid_queue(queue_type):
+        available = get_available_queues()
+        raise ValueError(f"Invalid queue type '{queue_type}'. Available: {available}")
+
+    from app.components.worker.registry import get_queue_metadata
+    queue_name = get_queue_metadata(queue_type)["queue_name"]
+
+    # Check cache first to avoid creating new Redis connections
+    cache_key = f"{queue_type}_{settings.REDIS_URL}"
+
+    if cache_key in _pool_cache:
+        # Reuse existing pool
+        cached_pool = _pool_cache[cache_key]
+        try:
+            # Test if pool is still valid by doing a quick ping
+            await cached_pool.ping()
+            return cached_pool, queue_name
+        except Exception:
+            # Pool is stale, remove from cache and create new one
+            logger.debug(f"Removing stale pool from cache: {cache_key}")
+            del _pool_cache[cache_key]
+
+    # Create new Redis pool and cache it
+    redis_settings = RedisSettings.from_dsn(settings.REDIS_URL)
+    pool = await create_pool(redis_settings)
+    _pool_cache[cache_key] = pool
+
+    logger.debug(f"Created and cached new Redis pool: {cache_key}")
+    return pool, queue_name
+
+
+async def clear_pool_cache() -> None:
+    """Clear all cached pools. Use during shutdown or for testing."""
+    for cache_key, pool in _pool_cache.items():
+        try:
+            await pool.aclose()
+            logger.debug(f"Closed cached pool: {cache_key}")
+        except Exception as e:
+            logger.warning(f"Error closing cached pool {cache_key}: {e}")
+
+    _pool_cache.clear()
+    logger.info("Pool cache cleared")

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/queues/__init__.py

@@ -0,0 +1 @@
+"""Worker queue configurations using native arq WorkerSettings."""

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/queues/load_test.py

@@ -0,0 +1,48 @@
+"""
+Load test worker queue configuration.
+
+Handles load testing orchestration and synthetic workload tasks using native arq
+patterns.
+"""
+
+from arq.connections import RedisSettings
+
+from app.core.config import settings
+
+# Import load test tasks
+from app.components.worker.tasks.load_tasks import (
+    cpu_intensive_task,
+    failure_testing_task,
+    io_simulation_task,
+    memory_operations_task,
+)
+from app.components.worker.tasks.system_tasks import (
+    load_test_orchestrator,
+)
+
+
+class WorkerSettings:
+    """Load testing worker configuration."""
+    
+    # Human-readable description
+    description = "Load testing and performance testing"
+    
+    # Task functions for this queue
+    functions = [
+        # Load test orchestrator
+        load_test_orchestrator,
+        # Synthetic workload tasks
+        cpu_intensive_task,
+        io_simulation_task,
+        memory_operations_task,
+        failure_testing_task,
+    ]
+    
+    # arq configuration  
+    redis_settings = RedisSettings.from_dsn(settings.REDIS_URL)
+    queue_name = "arq:queue:load_test"
+    max_jobs = 50  # High concurrency for load testing
+    job_timeout = 60  # Quick tasks
+    keep_result = settings.WORKER_KEEP_RESULT_SECONDS
+    max_tries = settings.WORKER_MAX_TRIES
+    health_check_interval = 30

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/queues/media.py

@@ -0,0 +1,41 @@
+"""
+Media worker queue configuration.
+
+Handles image processing, file operations, and media transformations using native
+arq patterns.
+"""
+
+from typing import Any
+
+from arq.connections import RedisSettings
+
+from app.core.config import settings
+
+# Import media tasks (when available)
+# from app.components.worker.tasks.media_tasks import (
+#     image_resize,
+#     video_encode, 
+#     file_convert,
+# )
+
+
+class WorkerSettings:
+    """Media processing worker configuration."""
+    
+    # Human-readable description
+    description = "Image and file processing"
+    
+    # Task functions for this queue
+    functions: list[Any] = [
+        # Media processing tasks will be added here
+        # Example: image_resize, video_encode, file_convert
+    ]
+    
+    # arq configuration
+    redis_settings = RedisSettings.from_dsn(settings.REDIS_URL) 
+    queue_name = "arq:queue:media"
+    max_jobs = 10  # I/O-bound file operations
+    job_timeout = 600  # 10 minutes - file processing can take time
+    keep_result = settings.WORKER_KEEP_RESULT_SECONDS
+    max_tries = settings.WORKER_MAX_TRIES
+    health_check_interval = 30

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/queues/system.py

@@ -0,0 +1,36 @@
+"""
+System worker queue configuration.
+
+Handles system maintenance and monitoring tasks using native arq patterns.
+"""
+
+from arq.connections import RedisSettings
+
+from app.core.config import settings
+
+# Import system tasks
+from app.components.worker.tasks.simple_system_tasks import (
+    system_health_check,
+    cleanup_temp_files,
+)
+
+class WorkerSettings:
+    """System maintenance worker configuration."""
+    
+    # Human-readable description
+    description = "System maintenance and monitoring tasks"
+    
+    # Task functions for this queue
+    functions = [
+        system_health_check,
+        cleanup_temp_files,
+    ]
+    
+    # arq configuration
+    redis_settings = RedisSettings.from_dsn(settings.REDIS_URL)
+    queue_name = "arq:queue:system"
+    max_jobs = 15  # Moderate concurrency for administrative operations
+    job_timeout = 300  # 5 minutes
+    keep_result = settings.WORKER_KEEP_RESULT_SECONDS
+    max_tries = settings.WORKER_MAX_TRIES
+    health_check_interval = 30