aegis-stack 0.2.0rc2__py3-none-any.whl

aegis/templates/copier-aegis-project/{{ project_slug }}/app/components/worker/tasks/system_tasks.py

@@ -0,0 +1,281 @@
+"""
+System and orchestration tasks.
+
+Contains the load test orchestrator which spawns many tasks to measure queue throughput.
+"""
+
+import asyncio
+from datetime import datetime
+from typing import Any
+
+from app.components.worker.constants import LoadTestTypes, TaskNames
+from app.core.config import get_load_test_queue
+from app.core.log import logger
+
+
+async def load_test_orchestrator(
+    ctx: dict[str, Any],
+    num_tasks: int = 100,
+    task_type: LoadTestTypes = LoadTestTypes.CPU_INTENSIVE,
+    batch_size: int = 10,
+    delay_ms: int = 0,
+    target_queue: str | None = None,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    """
+    Load test orchestrator that spawns many lightweight tasks to measure queue
+    throughput.
+
+    This is the new approach: instead of one task doing heavy work, we spawn
+    hundreds of lightweight tasks to actually stress test the queue infrastructure
+    and measure meaningful performance metrics like tasks/second.
+
+    Args:
+        num_tasks: Number of tasks to spawn for the load test
+        task_type: Type of worker task to spawn (cpu_intensive, io_simulation,
+            memory_operations)
+        batch_size: How many tasks to send concurrently per batch
+        delay_ms: Delay between batches in milliseconds
+        target_queue: Which queue to test (defaults to configured load_test queue)
+
+    Returns:
+        Comprehensive load test results with throughput metrics
+    """
+    start_time = datetime.now()
+    test_id = ctx.get("job_id", "unknown")
+
+    # Use configured load test queue if not specified
+    if target_queue is None:
+        target_queue = get_load_test_queue()
+
+    logger.info(
+        f"🚀 Starting load test orchestrator: {num_tasks} {task_type} tasks "
+        f"(batches of {batch_size})"
+    )
+
+    # Initialize tasks_sent before try block to prevent UnboundLocalError
+    tasks_sent = 0
+
+    try:
+        # Import here to avoid circular imports
+        from app.components.worker.pools import get_queue_pool
+
+        # Get queue pool for enqueueing
+        pool, queue_name = await get_queue_pool(target_queue)
+
+        try:
+            # Spawn tasks in batches
+            task_ids = []
+
+            for batch_start in range(0, num_tasks, batch_size):
+                batch_end = min(batch_start + batch_size, num_tasks)
+                current_batch_size = batch_end - batch_start
+
+                # Enqueue batch of tasks
+                # Map task type to actual function name
+                task_func = _get_task_function_name(task_type)
+
+                batch_jobs = []
+                for _ in range(current_batch_size):
+                    job = await pool.enqueue_job(task_func, _queue_name=queue_name)
+                    if job is not None:
+                        batch_jobs.append(job)
+                        task_ids.append(job.job_id)
+
+                tasks_sent += current_batch_size
+                logger.info(
+                    f"📤 Sent batch: {current_batch_size} tasks "
+                    f"(total: {tasks_sent}/{num_tasks})"
+                )
+
+                # Add configurable delay between batches if specified
+                if delay_ms > 0 and batch_end < num_tasks:
+                    await asyncio.sleep(delay_ms / 1000.0)
+
+            logger.info(f"✅ All {tasks_sent} tasks enqueued to {queue_name}")
+
+            # Monitor task completion with timeout based on queue configuration
+            from app.components.worker.registry import get_queue_metadata
+
+            queue_metadata = get_queue_metadata(target_queue)
+            monitor_timeout = queue_metadata.get("timeout", 300)  # Use queue's timeout
+
+            logger.info(
+                f"⏱️ Monitoring task completion (timeout: {monitor_timeout}s)..."
+            )
+
+            completion_result = await _monitor_task_completion(
+                task_ids=task_ids,
+                pool=pool,
+                expected_tasks=tasks_sent,
+                timeout_seconds=monitor_timeout,  # Use configured timeout
+            )
+
+            end_time = datetime.now()
+            total_duration = (end_time - start_time).total_seconds()
+
+            # Combine orchestrator stats with completion monitoring
+            result = {
+                "test_id": test_id,
+                "task_type": task_type.value,
+                "tasks_sent": tasks_sent,
+                "task_ids": task_ids[:10],  # Sample of IDs for debugging
+                "batch_size": batch_size,
+                "delay_ms": delay_ms,
+                "target_queue": target_queue,
+                "start_time": start_time.isoformat(),
+                "end_time": end_time.isoformat(),
+                "total_duration_seconds": round(total_duration, 2),
+                **completion_result,  # Merge in the monitoring results
+            }
+
+            # Calculate overall throughput based on completed tasks
+            if result.get("tasks_completed", 0) > 0:
+                result["overall_throughput_per_second"] = round(
+                    result["tasks_completed"] / total_duration, 2
+                )
+            else:
+                result["overall_throughput_per_second"] = 0
+
+            logger.info(
+                f"🏁 Load test complete: {result['tasks_completed']}/{tasks_sent} "
+                f"tasks in {total_duration:.1f}s"
+            )
+            logger.info(
+                f"📈 Throughput: {result['overall_throughput_per_second']} tasks/sec"
+            )
+
+            return result
+
+        finally:
+            # Always close the pool, even if errors occur
+            await pool.aclose()
+
+    except Exception as e:
+        logger.error(f"Load test orchestrator failed: {e}")
+        return {"test_id": test_id, "error": str(e), "tasks_sent": tasks_sent}
+
+
+def _get_task_function_name(task_type: LoadTestTypes) -> str:
+    """Map task type to actual function name."""
+    task_map = {
+        LoadTestTypes.CPU_INTENSIVE: TaskNames.CPU_INTENSIVE_TASK,
+        LoadTestTypes.IO_SIMULATION: TaskNames.IO_SIMULATION_TASK,
+        LoadTestTypes.MEMORY_OPERATIONS: TaskNames.MEMORY_OPERATIONS_TASK,
+        LoadTestTypes.FAILURE_TESTING: TaskNames.FAILURE_TESTING_TASK,
+    }
+    return task_map.get(task_type, TaskNames.CPU_INTENSIVE_TASK)
+
+
+async def _monitor_task_completion(
+    task_ids: list[str],
+    pool: Any,
+    expected_tasks: int,
+    timeout_seconds: int = 300,
+    poll_interval: float = 2.0,
+) -> dict[str, Any]:
+    """
+    Monitor task completion by checking job results directly.
+
+    This avoids Redis queue type errors by tracking job completion
+    instead of trying to read queue internals.
+    """
+    start_monitor = datetime.now()
+    tasks_completed = 0
+    tasks_failed = 0
+    last_progress_time = start_monitor
+    last_completed = 0
+
+    # Track which task IDs we've seen complete
+    completed_ids: set[str] = set()
+    failed_ids: set[str] = set()
+
+    try:
+        while True:
+            # Check each task ID for completion
+            for task_id in task_ids:
+                if task_id in completed_ids or task_id in failed_ids:
+                    continue  # Already processed
+
+                # Check if job result exists
+                result_key = f"arq:result:{task_id}"
+                result_data = await pool.get(result_key)
+
+                if result_data:
+                    # Job completed - check if it succeeded or failed
+                    try:
+                        # arq stores results as msgpack, but we can check existence
+                        completed_ids.add(task_id)
+                        tasks_completed += 1
+                    except Exception:
+                        # If we can't parse, assume it completed
+                        completed_ids.add(task_id)
+                        tasks_completed += 1
+
+            tasks_done = tasks_completed + tasks_failed
+
+            # Calculate throughput
+            elapsed = (datetime.now() - start_monitor).total_seconds()
+            throughput = tasks_completed / elapsed if elapsed > 0 else 0
+
+            # Check if we're making progress
+            if tasks_completed > last_completed:
+                last_progress_time = datetime.now()
+                last_completed = tasks_completed
+
+            # Progress logging (less verbose)
+            progress_pct = (
+                (tasks_done / expected_tasks * 100) if expected_tasks > 0 else 0
+            )
+            if (
+                tasks_done % 10 == 0 or tasks_done == expected_tasks
+            ):  # Log every 10 tasks or at completion
+                logger.info(
+                    f"📈 Progress: {tasks_done}/{expected_tasks} "
+                    f"({progress_pct:.0f}% - completed: {tasks_completed}, "
+                    f"failed: {tasks_failed}) throughput: {throughput:.1f} tasks/sec"
+                )
+
+            # Check completion
+            if tasks_done >= expected_tasks:
+                logger.info(
+                    f"✅ All tasks completed: {tasks_completed} success, "
+                    f"{tasks_failed} failed"
+                )
+                break
+
+            # Check timeout
+            if elapsed > timeout_seconds:
+                logger.warning(f"⏱️ Load test timed out after {timeout_seconds}s")
+                break
+
+            # Check if we're stuck (no progress for 30 seconds)
+            stuck_duration = (datetime.now() - last_progress_time).total_seconds()
+            if stuck_duration > 30 and tasks_done > 0:
+                logger.warning(
+                    f"⚠️ No progress for {stuck_duration:.0f}s, stopping monitor"
+                )
+                break
+
+            await asyncio.sleep(poll_interval)
+
+    except Exception as e:
+        logger.error(f"Task monitoring error: {e}")
+
+    # Final metrics
+    final_elapsed = (datetime.now() - start_monitor).total_seconds()
+
+    return {
+        "tasks_completed": tasks_completed,
+        "tasks_failed": tasks_failed,
+        "monitor_duration_seconds": round(final_elapsed, 2),
+        "average_throughput_per_second": round(tasks_completed / final_elapsed, 2)
+        if final_elapsed > 0
+        else 0,
+        "completion_percentage": round((tasks_completed / expected_tasks * 100), 1)
+        if expected_tasks > 0
+        else 0,
+        "failure_rate_percent": round((tasks_failed / expected_tasks * 100), 1)
+        if expected_tasks > 0
+        else 0,
+    }

aegis/templates/copier-aegis-project/{{ project_slug }}/app/core/config.py.jinja

@@ -0,0 +1,178 @@
+# app/core/config.py
+"""
+Application configuration management using Pydantic's BaseSettings.
+
+This module centralizes application settings, allowing them to be loaded
+from environment variables for easy configuration in different environments.
+"""
+
+from typing import Any
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """
+    Defines application settings.
+    `model_config` is used to specify that settings should be loaded from a .env file.
+    """
+
+    # Application environment: "dev" or "prod"
+    APP_ENV: str = "dev"
+
+    # Log level for the application
+    LOG_LEVEL: str = "INFO"
+
+    # Port for the web server
+    PORT: int = 8000
+
+    # Development settings
+    AUTO_RELOAD: bool = False
+
+    # Docker settings (used by docker-compose)
+    AEGIS_STACK_TAG: str = "aegis-stack:latest"
+    AEGIS_STACK_VERSION: str = "dev"
+
+    # Health monitoring and alerting
+    # Health checks are available via API endpoints (/health/)
+    # Use external monitoring tools (Prometheus, DataDog, etc.) to poll these endpoints
+    HEALTH_CHECK_ENABLED: bool = True
+    HEALTH_CHECK_INTERVAL_MINUTES: int = 5  # Recommended interval for monitoring
+
+    # Health check performance settings
+    HEALTH_CHECK_TIMEOUT_SECONDS: float = 2.0
+    SYSTEM_METRICS_CACHE_SECONDS: int = 5
+
+    # Basic alerting configuration
+    ALERTING_ENABLED: bool = False
+    ALERT_COOLDOWN_MINUTES: int = 60  # Minutes between repeated alerts for same issue
+
+    # Health check thresholds
+    MEMORY_THRESHOLD_PERCENT: float = 90.0
+    DISK_THRESHOLD_PERCENT: float = 85.0
+    CPU_THRESHOLD_PERCENT: float = 95.0
+
+    # Flet frontend settings
+    FLET_ASSETS_DIR: str = "assets"  # Directory for Flet static assets (images, etc.)
+
+    # Authentication settings
+    SECRET_KEY: str = "change-this-secret-key-in-production-use-env-variable"
+    JWT_ALGORITHM: str = "HS256"
+    ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
+
+{% if include_redis %}
+    # Redis settings for arq background tasks
+    REDIS_URL: str = "redis://redis:6379"  # Docker service name by default
+    REDIS_URL_LOCAL: str | None = None  # Override for local CLI usage
+    REDIS_DB: int = 0
+
+    @property
+    def redis_url_effective(self) -> str:
+        """Get effective Redis URL, preferring local override when not in Docker."""
+        # If explicitly overridden for local use
+        if self.REDIS_URL_LOCAL and not self.is_docker:
+            return self.REDIS_URL_LOCAL
+        return self.REDIS_URL
+
+    @property 
+    def is_docker(self) -> bool:
+        """Detect if running inside Docker container."""
+        import os
+        return (
+            os.path.exists("/.dockerenv") or 
+            bool(os.getenv("DOCKER_CONTAINER"))
+        )
+{% endif %}
+
+{% if include_worker %}
+    # arq worker settings (shared across all workers)
+    WORKER_KEEP_RESULT_SECONDS: int = 3600  # Keep job results for 1 hour
+    WORKER_MAX_TRIES: int = 3
+
+    # Redis connection settings for arq workers
+    REDIS_CONN_TIMEOUT: int = 5  # Connection timeout in seconds (default: 1)
+    REDIS_CONN_RETRIES: int = 5  # Connection retry attempts (default: 5)
+    REDIS_CONN_RETRY_DELAY: int = 1  # Delay between retries (default: 1)
+
+    # Worker health check settings
+    WORKER_HEALTH_CHECK_INTERVAL: int = 15  # In seconds (default: 15)
+
+    # PURE ARQ IMPLEMENTATION - NO CONFIGURATION NEEDED!
+    # Worker configuration comes from individual WorkerSettings classes
+    # in app/components/worker/queues/ - just import and use as arq intended!
+{% endif %}
+
+{% if include_database %}
+    # Database settings (SQLite)
+    DATABASE_URL: str = "sqlite:///./data/app.db"
+    DATABASE_ENGINE_ECHO: bool = False
+    DATABASE_CONNECT_ARGS: dict[str, Any] = {"check_same_thread": False}
+{% endif %}
+
+{% if include_ai %}
+    # AI Service Configuration
+    # Primary service settings
+    AI_ENABLED: bool = True
+    AI_PROVIDER: str = "public"  # Default to public provider
+    AI_MODEL: str = "auto"  # Default model (public provider uses available models)
+    AI_TEMPERATURE: float = 0.7
+    AI_MAX_TOKENS: int = 1000
+    AI_TIMEOUT_SECONDS: float = 30.0
+
+    # Provider API Keys (optional - many providers offer free tiers)
+    OPENAI_API_KEY: str | None = None
+    ANTHROPIC_API_KEY: str | None = None
+    GOOGLE_API_KEY: str | None = None
+    GROQ_API_KEY: str | None = None
+    MISTRAL_API_KEY: str | None = None
+    COHERE_API_KEY: str | None = None
+
+    # Conversation settings
+    AI_MAX_CONVERSATION_LENGTH: int = 50  # Max messages per conversation
+    AI_CONVERSATION_TIMEOUT_HOURS: int = 24  # Auto-cleanup old conversations
+{% endif %}
+
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
+
+
+settings = Settings()
+
+
+{% if include_worker %}
+# Pure arq queue helper functions - use dynamic discovery
+def get_available_queues() -> list[str]:
+    """Get all available queue names via dynamic discovery."""
+    try:
+        from app.components.worker.registry import discover_worker_queues
+        queues: list[str] = discover_worker_queues()
+        return queues
+    except ImportError:
+        # Worker components not available
+        return []
+
+
+def get_default_queue() -> str:
+    """Get the default queue name for load testing."""
+    # Prefer load_test queue if it exists, otherwise use first available
+    available = get_available_queues()
+    if "load_test" in available:
+        return "load_test"
+    return available[0] if available else "system"
+
+
+def get_load_test_queue() -> str:
+    """Get the queue name for load testing."""
+    available = get_available_queues()
+    return "load_test" if "load_test" in available else get_default_queue()
+
+
+def is_valid_queue(queue_name: str) -> bool:
+    """Check if a queue name is valid."""
+    try:
+        from app.components.worker.registry import validate_queue_name
+        result: bool = validate_queue_name(queue_name)
+        return result
+    except ImportError:
+        # Worker components not available, no queues are valid
+        return False
+{% endif %}

aegis/templates/copier-aegis-project/{{ project_slug }}/app/core/constants.py

@@ -0,0 +1,58 @@
+"""
+Application constants.
+
+This module contains truly immutable values that never change across environments.
+For environment-dependent configuration, see app.core.config.
+
+Following 12-Factor App principles:
+- Constants = code (version controlled, immutable across deployments)
+- Configuration = environment (varies between dev/staging/production)
+"""
+
+
+class APIEndpoints:
+    """API endpoint paths - immutable across all environments."""
+
+    HEALTH_BASIC = "/health/"
+    HEALTH_DETAILED = "/health/detailed"
+    HEALTH_DASHBOARD = "/health/dashboard"
+
+
+class Defaults:
+    """Default values for timeouts and limits."""
+
+    # API timeouts (seconds)
+    API_TIMEOUT = 10.0
+    HEALTH_CHECK_TIMEOUT = 5.0
+
+    # Retry configuration
+    MAX_RETRIES = 3
+    RETRY_BACKOFF = 1.0
+
+    # Health check intervals (seconds)
+    HEALTH_CHECK_INTERVAL = 30
+    COMPONENT_CHECK_TIMEOUT = 2.0
+
+
+class CLI:
+    """CLI-specific constants."""
+
+    # Display limits
+    MAX_METADATA_DISPLAY_LENGTH = 30
+
+    # Output formatting
+    HEALTH_PERCENTAGE_DECIMALS = 1
+    TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+
+class HTTP:
+    """HTTP-related constants."""
+
+    # Status codes we care about
+    OK = 200
+    SERVICE_UNAVAILABLE = 503
+    INTERNAL_SERVER_ERROR = 500
+
+    # Headers
+    CONTENT_TYPE_JSON = "application/json"
+    USER_AGENT = "AegisStack-CLI/1.0"

aegis/templates/copier-aegis-project/{{ project_slug }}/app/core/db.py.jinja

@@ -0,0 +1,176 @@
+# app/core/db.py
+"""
+Database configuration and session management.
+
+This module provides SQLite database connectivity using SQLModel and SQLAlchemy.
+Includes proper session management with transaction handling and foreign key support.
+"""
+
+from collections.abc import AsyncGenerator, Generator
+from contextlib import asynccontextmanager, contextmanager
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from sqlalchemy import create_engine, event
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+from sqlalchemy.orm import sessionmaker
+from sqlmodel import Session, SQLModel
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.core.config import settings
+from app.core.log import logger
+
+
+# Extract database file path from URL for backup operations
+def _extract_database_path(database_url: str) -> str:
+    """Extract the file path from a SQLite database URL."""
+    parsed = urlparse(database_url)
+    if parsed.scheme == "sqlite":
+        # Handle both sqlite:/// and sqlite:// formats
+        path = parsed.path
+        if path.startswith("/") and len(parsed.netloc) == 0:
+            # sqlite:///./path/file.db -> ./path/file.db
+            return path[1:]
+        elif parsed.netloc == "" and not path.startswith("/"):
+            # sqlite://./path/file.db -> ./path/file.db
+            return path
+        else:
+            # sqlite:///absolute/path/file.db -> /absolute/path/file.db
+            return path
+    else:
+        raise ValueError(f"Unsupported database URL scheme: {parsed.scheme}")
+
+DATABASE_PATH = _extract_database_path(settings.DATABASE_URL)
+
+# Create SQLite engine with proper configuration (sync)
+engine = create_engine(
+    settings.DATABASE_URL,
+    connect_args=settings.DATABASE_CONNECT_ARGS,
+    echo=settings.DATABASE_ENGINE_ECHO,
+)
+
+# Create async engine for non-blocking operations
+def _get_async_database_url(database_url: str) -> str:
+    """Convert sync database URL to async version."""
+    if database_url.startswith("sqlite:///"):
+        return database_url.replace("sqlite:///", "sqlite+aiosqlite:///")
+    elif database_url.startswith("sqlite://"):
+        return database_url.replace("sqlite://", "sqlite+aiosqlite://")
+    elif database_url.startswith("postgresql://"):
+        return database_url.replace("postgresql://", "postgresql+asyncpg://")
+    elif database_url.startswith("mysql://"):
+        return database_url.replace("mysql://", "mysql+aiomysql://")
+    else:
+        # For future database types, return as-is and let SQLAlchemy handle it
+        return database_url
+
+async_engine = create_async_engine(
+    _get_async_database_url(settings.DATABASE_URL),
+    echo=settings.DATABASE_ENGINE_ECHO,
+    # Only add connect_args for SQLite
+    connect_args=(
+        settings.DATABASE_CONNECT_ARGS
+        if "sqlite" in settings.DATABASE_URL
+        else {}
+    ),
+)
+
+
+# Enable foreign key constraints for SQLite
+@event.listens_for(engine, "connect")
+def set_sqlite_pragma(dbapi_connection: Any, connection_record: Any) -> None:
+    """Enable foreign key constraints in SQLite."""
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
+
+# Configure session factory with SQLModel Session (sync)
+SessionLocal = sessionmaker(
+    class_=Session, bind=engine, autoflush=False, autocommit=False
+)
+
+# Configure async session factory using SQLModel's AsyncSession
+AsyncSessionLocal = async_sessionmaker(
+    async_engine,
+    class_=AsyncSession,
+    expire_on_commit=False,
+)
+
+
+@contextmanager
+def db_session(autocommit: bool = True) -> Generator[Session, None, None]:
+    """
+    Database session context manager with automatic transaction handling.
+
+    Args:
+        autocommit: Whether to automatically commit the transaction on success
+
+    Yields:
+        Session: Database session instance
+
+    Example:
+        with db_session() as session:
+            # Your database operations here
+            result = session.query(MyModel).first()
+    """
+    db_session: Session = SessionLocal()
+    try:
+        yield db_session
+        if autocommit:
+            db_session.commit()
+    except Exception:
+        db_session.rollback()
+        raise
+    finally:
+        db_session.close()
+
+
+@asynccontextmanager
+async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
+    """
+    Async database session context manager with automatic transaction handling.
+
+    Yields:
+        AsyncSession: Async database session instance
+
+    Example:
+        async with get_async_session() as session:
+            # Your async database operations here
+            result = await session.exec(select(MyModel))
+    """
+    async with AsyncSessionLocal() as session:
+        try:
+            yield session
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            raise
+
+
+def init_database() -> None:
+    """
+    Initialize the database by creating tables and ensuring directory structure.
+
+    This function:
+    1. Creates the database directory if it doesn't exist
+    2. Creates all tables defined by SQLModel models
+    3. Logs the initialization status
+    """
+    try:
+        # Ensure database directory exists
+        db_path = Path(DATABASE_PATH)
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Create all tables
+        SQLModel.metadata.create_all(engine)
+
+        if db_path.exists():
+            logger.info(f"✅ Database initialized: {DATABASE_PATH}")
+        else:
+            logger.info(f"✅ Database will be created on first use: {DATABASE_PATH}")
+
+    except Exception as e:
+        logger.error(f"❌ Database initialization failed: {e}")
+        raise