aegis-stack 0.2.0rc2__py3-none-any.whl

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/frontend/theme.py

@@ -0,0 +1,48 @@
+"""Simple theme management for the dashboard."""
+
+import flet as ft
+
+
+class ThemeManager:
+    """Manages light/dark theme switching for the Flet page."""
+
+    def __init__(self, page: ft.Page) -> None:
+        self.page = page
+        self.is_dark_mode = True  # Default to dark
+
+    async def initialize_themes(self) -> None:
+        """Initialize theme system with dark mode as default."""
+        self.page.theme_mode = ft.ThemeMode.DARK
+        self.is_dark_mode = True
+        self.page.update()
+
+    async def toggle_theme(self) -> None:
+        """Toggle between light and dark themes."""
+        if self.is_dark_mode:
+            self.page.theme_mode = ft.ThemeMode.LIGHT
+            self.is_dark_mode = False
+        else:
+            self.page.theme_mode = ft.ThemeMode.DARK
+            self.is_dark_mode = True
+
+        self.page.update()
+
+    def get_status_colors(self, is_healthy: bool) -> tuple[str, str, str]:
+        """Get (background, text, border) colors for status indicators."""
+        if is_healthy:
+            if self.is_dark_mode:
+                return (ft.Colors.GREEN_900, ft.Colors.GREEN_100, ft.Colors.GREEN)
+            else:
+                return (ft.Colors.GREEN_100, ft.Colors.GREEN_800, ft.Colors.GREEN)
+        else:
+            if self.is_dark_mode:
+                return (ft.Colors.RED_900, ft.Colors.RED_100, ft.Colors.ERROR)
+            else:
+                return (ft.Colors.RED_100, ft.Colors.RED_800, ft.Colors.ERROR)
+
+    def get_info_colors(self) -> tuple[str, str, str]:
+        """Get (background, text, border) colors for info cards."""
+        if self.is_dark_mode:
+            return (ft.Colors.BLUE_900, ft.Colors.BLUE_100, ft.Colors.PRIMARY)
+        else:
+            return (ft.Colors.BLUE_100, ft.Colors.BLUE_800, ft.Colors.PRIMARY)

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/scheduler/__init__.py

@@ -0,0 +1 @@
+# Scheduler component

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/scheduler/main.py.jinja

@@ -0,0 +1,156 @@
+"""
+Scheduler component for {{ project_name }}.
+
+Simple, explicit job scheduling - just import functions and schedule them.
+Add your own jobs by importing service functions and calling scheduler.add_job().
+"""
+
+import asyncio
+import os
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+{% if scheduler_backend != "memory" %}
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from app.core.db import engine, init_database, db_session
+from app.services.scheduler.models import APSchedulerJob
+from app.services.system.backup import backup_database_job
+{% elif include_scheduler and include_database %}
+from app.services.system.backup import backup_database_job
+{% endif %}
+
+from app.core.log import logger
+
+{% if scheduler_backend != "memory" %}
+
+def _job_exists_in_database(job_id: str) -> bool:
+    """Check if a job already exists in the persistence database using SQLModel."""
+    try:
+        from sqlmodel import select
+
+        with db_session() as session:
+            # Query for job by ID using modern SQLModel pattern
+            query = select(APSchedulerJob).where(APSchedulerJob.id == job_id)
+            result = session.exec(query)
+            job = result.first()
+            return job is not None
+    except Exception as e:
+        logger.warning(f"Could not check for existing job {job_id}: {e}")
+        # If we can't check, assume it doesn't exist (safer to add than skip)
+        return False
+
+{% endif %}
+
+
+
+
+def create_scheduler() -> AsyncIOScheduler:
+    """Create and configure the scheduler with all jobs."""
+{% if scheduler_backend != "memory" %}
+    # Ensure database is initialized before creating jobstore
+    init_database()
+
+    # Configure SQLAlchemy jobstore for persistence
+    jobstore = SQLAlchemyJobStore(engine=engine, tablename='apscheduler_jobs')
+    jobstores = {'default': jobstore}
+    scheduler = AsyncIOScheduler(jobstores=jobstores)
+    logger.info("📊 Scheduler using SQLite database for job persistence")
+{% else %}
+    # Use in-memory jobstore (default)
+    scheduler = AsyncIOScheduler()
+    logger.info(
+        "🕒 Scheduler running in memory mode (jobs won't persist across restarts)"
+    )
+{% endif %}
+
+    # ============================================================================
+    # JOB SCHEDULE CONFIGURATION
+    #
+    # PERSISTENCE BEHAVIOR:
+    # - Jobs are checked against the database first
+    # - Existing jobs are preserved (respects runtime modifications)
+    # - New jobs are added from code configuration
+    # - Set SCHEDULER_FORCE_UPDATE=true to override all jobs from code
+    #
+    # To update schedules during deployment:
+    #   SCHEDULER_FORCE_UPDATE=true docker-compose up -d scheduler
+    # ============================================================================
+
+{% if scheduler_backend != "memory" or (include_scheduler and include_database) %}
+    # Check environment flag for force updates (useful during deployments)
+    force_update = os.getenv("SCHEDULER_FORCE_UPDATE", "false").lower() == "true"
+    # Database backup job (runs daily at 2 AM when database is available)
+    job_id = "database_backup"
+{% if scheduler_backend != "memory" %}
+    # For persistent schedulers, check database directly since
+    # scheduler hasn't loaded jobs yet
+    job_exists = _job_exists_in_database(job_id)
+{% else %}
+    # For memory schedulers, scheduler.get_job() works fine
+    existing_job = scheduler.get_job(job_id)
+    job_exists = existing_job is not None
+{% endif %}
+
+    if not job_exists or force_update:
+        if job_exists and force_update:
+            logger.info(f"🔄 Force updating job '{job_id}' from code configuration")
+        else:
+            logger.info(f"➕ Adding new job '{job_id}'")
+
+        scheduler.add_job(
+            backup_database_job,
+            trigger="cron",
+            hour=2,
+            minute=0,
+            id=job_id,
+            name="Daily Database Backup",
+            max_instances=1,
+            coalesce=True,
+            replace_existing=True  # Safe to use since we check first
+        )
+    else:
+{% if scheduler_backend != "memory" %}
+        msg = f"✅ Job '{job_id}' exists in database, preserving current configuration"
+        logger.info(msg)
+{% else %}
+        logger.info(
+            f"✅ Job '{job_id}' exists, preserving current configuration: "
+            f"{existing_job.trigger}"
+        )
+{% endif %}
+{% endif %}
+
+    # Add your own scheduled jobs here by importing service functions
+    # and calling scheduler.add_job() with your custom business logic
+
+    return scheduler
+
+
+async def run_scheduler() -> None:
+    """Main scheduler runner with lifecycle management."""
+
+    logger.info("🕒 Starting {{ project_name }} Scheduler")
+
+    scheduler = create_scheduler()
+
+    try:
+        scheduler.start()
+        logger.info("✅ Scheduler started successfully")
+        logger.info(f"📋 {len(scheduler.get_jobs())} jobs scheduled:")
+
+        for job in scheduler.get_jobs():
+            logger.info(f"   • {job.name} - {job.trigger}")
+
+
+        # Keep the scheduler running
+        while True:
+            await asyncio.sleep(1)
+
+    except KeyboardInterrupt:
+        logger.info("🛑 Received shutdown signal")
+    except Exception as e:
+        logger.error(f"❌ Scheduler error: {e}")
+        raise
+    finally:
+        if scheduler.running:
+            scheduler.shutdown()
+            logger.info("✅ Scheduler stopped gracefully")

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/worker/CLAUDE.md.jinja

@@ -0,0 +1,213 @@
+# Worker Component Development Guide
+
+This guide covers arq worker architecture patterns and development for the Aegis Stack worker component.
+
+## Worker Architecture (arq)
+
+Aegis Stack uses pure **arq patterns** without custom wrappers, following native arq CLI and configuration patterns.
+
+### Worker Configuration Structure
+
+Each worker queue has its own `WorkerSettings` class:
+- `app/components/worker/queues/system.py` - System maintenance worker
+- `app/components/worker/queues/load_test.py` - Load testing worker  
+- `app/components/worker/queues/media.py` - Media processing worker
+
+### Worker Services in Docker
+
+Workers run as separate Docker services with specific names:
+- **`worker-system`** - System maintenance tasks (low concurrency, high reliability)
+- **`worker-load-test`** - High-concurrency load testing (up to 50 concurrent jobs)
+- **`worker-media`** - File/media processing (commented out by default)
+
+## Adding Worker Tasks
+
+### 1. Create Task Functions
+Tasks are pure async functions in `app/components/worker/tasks/`:
+```python
+# app/components/worker/tasks/my_tasks.py
+async def my_background_task() -> dict[str, str]:
+    """My custom background task."""
+    logger.info("Running my background task")
+    
+    # Your task logic here
+    await asyncio.sleep(1)  # Simulate work
+    
+    return {
+        "status": "completed",
+        "timestamp": datetime.now(UTC).isoformat(),
+        "task": "my_background_task"
+    }
+```
+
+### 2. Register with Worker Queue
+Import and add to the appropriate `WorkerSettings`:
+```python
+# app/components/worker/queues/system.py
+from app.components.worker.tasks.my_tasks import my_background_task
+
+class WorkerSettings:
+    functions = [
+        system_health_check,
+        cleanup_temp_files,
+        my_background_task,  # Add your task here
+    ]
+    
+    # Standard arq configuration
+    redis_settings = RedisSettings.from_dsn(settings.REDIS_URL)
+    queue_name = "arq:queue:system"
+    max_jobs = 15
+    job_timeout = 300
+```
+
+## Native arq CLI Usage
+
+### Worker Health Checks
+```bash
+# Check if workers can connect to Redis and validate configuration
+uv run python -m arq app.components.worker.queues.system.WorkerSettings --check
+uv run python -m arq app.components.worker.queues.load_test.WorkerSettings --check
+uv run python -m arq app.components.worker.queues.media.WorkerSettings --check
+```
+
+### Local Worker Development
+```bash
+# Run worker locally with auto-reload for development
+uv run python -m arq app.components.worker.queues.system.WorkerSettings --watch app/
+
+# Run worker in burst mode (process all jobs and exit)
+uv run python -m arq app.components.worker.queues.system.WorkerSettings --burst
+```
+
+### Worker Configuration in Health Checks
+
+The health system reads worker configuration from `app/core/config.py` but workers themselves use their own `WorkerSettings` classes:
+
+```python
+# Health system reads this for monitoring
+WORKER_QUEUES: dict[str, dict[str, Any]] = {
+    "system": {
+        "description": "System maintenance and monitoring tasks",
+        "max_jobs": 15,
+        "timeout_seconds": 300,
+        "queue_name": "arq:queue:system",
+    },
+    "load_test": {
+        "description": "Load testing and performance testing",
+        "max_jobs": 50,
+        "timeout_seconds": 60,
+        "queue_name": "arq:queue:load_test",
+    }
+}
+
+# But workers use their own WorkerSettings classes for actual configuration
+```
+
+## Key Differences from Custom Worker Systems
+
+### ✅ What We Do (Pure arq):
+- Use native arq CLI: `python -m arq WorkerSettings`
+- Standard `WorkerSettings` classes with `functions` list
+- Direct task imports into worker configurations
+- Native arq health checking and monitoring
+
+### ❌ What We Don't Do (Avoided custom patterns):
+- Custom worker wrapper classes
+- Central worker registry systems
+- Custom CLI commands for workers
+- Configuration-driven task discovery
+
+This approach keeps workers transparent and lets developers use arq exactly as documented in the official arq documentation.
+
+## Docker Worker Debugging Commands
+
+### View Worker Logs
+```bash
+# View specific worker logs
+docker compose logs worker-system             # System worker logs
+docker compose logs worker-load-test          # Load test worker logs
+docker compose logs -f worker-system          # Follow system worker in real-time
+docker compose logs -f worker-load-test       # Follow load test worker in real-time
+
+# View all workers at once
+docker compose logs -f worker-system worker-load-test
+
+# Filter for errors in specific workers
+docker compose logs worker-load-test | grep "ERROR\|failed\|TypeError"
+
+# Monitor worker processes and resources
+docker compose exec worker-system ps aux      # Check system worker processes
+docker compose exec worker-load-test ps aux   # Check load test worker processes
+docker stats worker-system worker-load-test   # Monitor resource usage
+docker compose restart worker-system          # Restart specific worker
+```
+
+### Essential Docker Log Monitoring
+
+**Check Worker Logs for Load Test Issues:**
+```bash
+# View real-time worker logs
+docker compose logs -f worker
+
+# Check specific container logs  
+docker logs <container-id>
+
+# View logs with timestamps
+docker compose logs --timestamps worker
+
+# Search logs for specific errors
+docker compose logs worker | grep "TypeError\|failed"
+
+# Check all service logs
+docker compose logs -f
+```
+
+**Load Test Debugging Workflow:**
+1. **Run Load Test**: `uv run {{ project_slug }} load-test run --type io_simulation --tasks 10`
+2. **Monitor Worker Logs**: `docker compose logs -f worker-load-test` (in separate terminal)
+3. **Look for**: 
+   - `TypeError` messages indicating parameter mismatches
+   - Task completion vs failure counts (`j_complete=X j_failed=Y`)
+   - Task execution times and errors
+   - Redis connection issues
+
+**Common Error Patterns:**
+- `TypeError: function() got an unexpected keyword argument 'param'` - Parameter mismatch between orchestrator and task function
+- `j_failed=X` increasing rapidly - Worker tasks failing due to code issues
+- `Redis connection failed` - Infrastructure connectivity problems
+- `delayed=X.XXs` - Queue saturation or worker overload
+
+**System Health Verification:**
+```bash
+# Check all containers
+docker compose ps
+
+# Check system health via API
+uv run {{ project_slug }} health status --detailed
+
+# Monitor Redis connection
+docker compose logs redis
+```
+
+## Worker Development Best Practices
+
+### Task Design Patterns
+1. **Pure Functions** - Tasks should be self-contained with minimal dependencies
+2. **Error Handling** - Always include try/catch with proper logging
+3. **Return Values** - Return structured data for monitoring and debugging
+4. **Timeouts** - Set appropriate timeouts for different task types
+5. **Retry Logic** - Use arq's built-in retry mechanisms
+
+### Queue Management
+1. **Separate Concerns** - Use different queues for different types of work
+2. **Concurrency Limits** - Set appropriate max_jobs for each queue type
+3. **Priority Queues** - Use different queues for different priorities
+4. **Dead Letter Queues** - Monitor failed jobs and implement recovery
+
+### Monitoring and Observability
+1. **Structured Logging** - Use structured logs for easy parsing
+2. **Metrics Collection** - Track task execution times and success rates
+3. **Health Checks** - Implement health checks for worker availability
+4. **Alerting** - Set up alerts for queue depth and failure rates
+
+This approach ensures workers are maintainable, debuggable, and follow established patterns.

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/worker/__init__.py

@@ -0,0 +1,6 @@
+"""
+Worker component for background task processing.
+
+This component handles asynchronous background tasks using arq (Redis-based queues).
+Tasks are organized into priority queues (high, medium, low) and by functional area.
+"""

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/worker/constants.py.jinja

@@ -0,0 +1,30 @@
+"""
+Worker component constants.
+
+This module contains constants specific to the worker component,
+keeping them separate from global application constants.
+"""
+
+from enum import Enum
+
+
+class TaskNames:
+    """Worker task function names - must match actual function names in code."""
+
+    # Orchestrator
+    LOAD_TEST_ORCHESTRATOR = "load_test_orchestrator"
+
+    # Load test tasks
+    CPU_INTENSIVE_TASK = "cpu_intensive_task"
+    IO_SIMULATION_TASK = "io_simulation_task"
+    MEMORY_OPERATIONS_TASK = "memory_operations_task"
+    FAILURE_TESTING_TASK = "failure_testing_task"
+
+
+class LoadTestTypes(Enum):
+    """Load test type identifiers for task selection."""
+
+    CPU_INTENSIVE = "cpu_intensive"
+    IO_SIMULATION = "io_simulation"
+    MEMORY_OPERATIONS = "memory_operations"
+    FAILURE_TESTING = "failure_testing"

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/worker/pools.py

@@ -0,0 +1,97 @@
+"""
+Worker pool management for client-side task enqueueing.
+
+This module provides Redis connection pooling and caching for enqueueing tasks
+to worker queues. Separated from worker management to allow clean architectural
+separation between client-side enqueueing and worker-side processing.
+"""
+
+from app.core.config import get_default_queue, settings
+from app.core.log import logger
+from arq import create_pool
+from arq.connections import ArqRedis, RedisSettings
+
+# Global pool cache to avoid creating new Redis connections repeatedly
+_pool_cache: dict[str, ArqRedis] = {}
+
+
+async def get_queue_pool(queue_type: str | None = None) -> tuple[ArqRedis, str]:
+    """
+    Get Redis pool for enqueuing tasks to specific functional queue.
+
+    Uses connection pooling to avoid creating new Redis connections repeatedly.
+
+    Args:
+        queue_type: Functional queue type (defaults to configured default queue)
+
+    Returns:
+        Tuple of (pool, queue_name) for enqueueing tasks
+    """
+    # Use configured default queue if not specified
+    if queue_type is None:
+        queue_type = get_default_queue()
+
+    from app.core.config import get_available_queues, is_valid_queue
+
+    if not is_valid_queue(queue_type):
+        available = get_available_queues()
+        raise ValueError(f"Invalid queue type '{queue_type}'. Available: {available}")
+
+    from app.components.worker.registry import get_queue_metadata
+
+    queue_name = get_queue_metadata(queue_type)["queue_name"]
+
+    # Check cache first to avoid creating new Redis connections
+    redis_url = (
+        settings.redis_url_effective
+        if hasattr(settings, "redis_url_effective")
+        else settings.REDIS_URL
+    )
+    cache_key = f"{queue_type}_{redis_url}"
+
+    if cache_key in _pool_cache:
+        # Reuse existing pool
+        cached_pool = _pool_cache[cache_key]
+        try:
+            # Test if pool is still valid by doing a quick ping
+            await cached_pool.ping()
+            return cached_pool, queue_name
+        except Exception:
+            # Pool is stale, remove from cache and create new one
+            logger.debug(f"Removing stale pool from cache: {cache_key}")
+            del _pool_cache[cache_key]
+
+    # Create new Redis pool and cache it with improved connection settings
+    redis_url = (
+        settings.redis_url_effective
+        if hasattr(settings, "redis_url_effective")
+        else settings.REDIS_URL
+    )
+    base_settings = RedisSettings.from_dsn(redis_url)
+    redis_settings = RedisSettings(
+        host=base_settings.host,
+        port=base_settings.port,
+        database=base_settings.database,
+        password=base_settings.password,
+        conn_timeout=settings.REDIS_CONN_TIMEOUT,
+        conn_retries=settings.REDIS_CONN_RETRIES,
+        conn_retry_delay=settings.REDIS_CONN_RETRY_DELAY,
+    )
+    pool = await create_pool(redis_settings)
+    _pool_cache[cache_key] = pool
+
+    logger.debug(f"Created and cached new Redis pool: {cache_key}")
+    return pool, queue_name
+
+
+async def clear_pool_cache() -> None:
+    """Clear all cached pools. Use during shutdown or for testing."""
+    for cache_key, pool in _pool_cache.items():
+        try:
+            await pool.aclose()
+            logger.debug(f"Closed cached pool: {cache_key}")
+        except Exception as e:
+            logger.warning(f"Error closing cached pool {cache_key}: {e}")
+
+    _pool_cache.clear()
+    logger.info("Pool cache cleared")

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/worker/queues/__init__.py

@@ -0,0 +1 @@
+"""Worker queue configurations using native arq WorkerSettings."""

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/worker/queues/load_test.py

@@ -0,0 +1,55 @@
+"""
+Load test worker queue configuration.
+
+Handles load testing orchestration and synthetic workload tasks using native arq
+patterns.
+"""
+
+# Import load test tasks
+from app.components.worker.tasks.load_tasks import (
+    cpu_intensive_task,
+    failure_testing_task,
+    io_simulation_task,
+    memory_operations_task,
+)
+from app.components.worker.tasks.system_tasks import (
+    load_test_orchestrator,
+)
+from app.core.config import settings
+from arq.connections import RedisSettings
+
+
+class WorkerSettings:
+    """Load testing worker configuration."""
+
+    # Human-readable description
+    description = "Load testing and performance testing"
+
+    # Task functions for this queue
+    functions = [
+        # Load test orchestrator
+        load_test_orchestrator,
+        # Synthetic workload tasks
+        cpu_intensive_task,
+        io_simulation_task,
+        memory_operations_task,
+        failure_testing_task,
+    ]
+
+    # arq configuration with improved connection settings
+    base_settings = RedisSettings.from_dsn(settings.REDIS_URL)
+    redis_settings = RedisSettings(
+        host=base_settings.host,
+        port=base_settings.port,
+        database=base_settings.database,
+        password=base_settings.password,
+        conn_timeout=settings.REDIS_CONN_TIMEOUT,
+        conn_retries=settings.REDIS_CONN_RETRIES,
+        conn_retry_delay=settings.REDIS_CONN_RETRY_DELAY,
+    )
+    queue_name = "arq:queue:load_test"
+    max_jobs = 50  # High concurrency for load testing
+    job_timeout = 60  # Quick tasks
+    keep_result = settings.WORKER_KEEP_RESULT_SECONDS
+    max_tries = settings.WORKER_MAX_TRIES
+    health_check_interval = settings.WORKER_HEALTH_CHECK_INTERVAL

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/worker/queues/media.py

@@ -0,0 +1,49 @@
+"""
+Media worker queue configuration.
+
+Handles image processing, file operations, and media transformations using native
+arq patterns.
+"""
+
+from typing import Any
+
+from app.core.config import settings
+from arq.connections import RedisSettings
+
+# Import media tasks (when available)
+# from app.components.worker.tasks.media_tasks import (
+#     image_resize,
+#     video_encode,
+#     file_convert,
+# )
+
+
+class WorkerSettings:
+    """Media processing worker configuration."""
+
+    # Human-readable description
+    description = "Image and file processing"
+
+    # Task functions for this queue
+    functions: list[Any] = [
+        # Media processing tasks will be added here
+        # Example: image_resize, video_encode, file_convert
+    ]
+
+    # arq configuration with improved connection settings
+    base_settings = RedisSettings.from_dsn(settings.REDIS_URL)
+    redis_settings = RedisSettings(
+        host=base_settings.host,
+        port=base_settings.port,
+        database=base_settings.database,
+        password=base_settings.password,
+        conn_timeout=settings.REDIS_CONN_TIMEOUT,
+        conn_retries=settings.REDIS_CONN_RETRIES,
+        conn_retry_delay=settings.REDIS_CONN_RETRY_DELAY,
+    )
+    queue_name = "arq:queue:media"
+    max_jobs = 10  # I/O-bound file operations
+    job_timeout = 600  # 10 minutes - file processing can take time
+    keep_result = settings.WORKER_KEEP_RESULT_SECONDS
+    max_tries = settings.WORKER_MAX_TRIES
+    health_check_interval = settings.WORKER_HEALTH_CHECK_INTERVAL