aegis-stack 0.1.0__py3-none-any.whl

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/components/worker/tasks/system_tasks.py

@@ -0,0 +1,279 @@
+"""
+System and orchestration tasks.
+
+Contains the load test orchestrator which spawns many tasks to measure queue throughput.
+"""
+
+import asyncio
+from datetime import datetime
+from typing import Any
+
+from app.components.worker.constants import LoadTestTypes, TaskNames
+from app.core.config import get_load_test_queue
+from app.core.log import logger
+
+
+async def load_test_orchestrator(
+    ctx: dict[str, Any],
+    num_tasks: int = 100,
+    task_type: LoadTestTypes = LoadTestTypes.CPU_INTENSIVE,
+    batch_size: int = 10,
+    delay_ms: int = 0,
+    target_queue: str | None = None,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    """
+    Load test orchestrator that spawns many lightweight tasks to measure queue
+    throughput.
+
+    This is the new approach: instead of one task doing heavy work, we spawn
+    hundreds of lightweight tasks to actually stress test the queue infrastructure
+    and measure meaningful performance metrics like tasks/second.
+
+    Args:
+        num_tasks: Number of tasks to spawn for the load test
+        task_type: Type of worker task to spawn (cpu_intensive, io_simulation,
+            memory_operations)
+        batch_size: How many tasks to send concurrently per batch
+        delay_ms: Delay between batches in milliseconds
+        target_queue: Which queue to test (defaults to configured load_test queue)
+
+    Returns:
+        Comprehensive load test results with throughput metrics
+    """
+    start_time = datetime.now()
+    test_id = ctx.get("job_id", "unknown")
+
+    # Use configured load test queue if not specified
+    if target_queue is None:
+        target_queue = get_load_test_queue()
+
+    logger.info(
+        f"🚀 Starting load test orchestrator: {num_tasks} {task_type} tasks "
+        f"(batches of {batch_size})"
+    )
+
+    try:
+        # Import here to avoid circular imports
+        from app.components.worker.pools import get_queue_pool
+
+        # Get queue pool for enqueueing
+        pool, queue_name = await get_queue_pool(target_queue)
+
+        # Spawn tasks in batches
+        task_ids = []
+        tasks_sent = 0
+
+        for batch_start in range(0, num_tasks, batch_size):
+            batch_end = min(batch_start + batch_size, num_tasks)
+            current_batch_size = batch_end - batch_start
+
+            # Enqueue batch of tasks
+            # Map task type to actual function name
+            task_func = _get_task_function_name(task_type)
+
+            batch_jobs = []
+            for _ in range(current_batch_size):
+                job = await pool.enqueue_job(task_func, _queue_name=queue_name)
+                if job is not None:
+                    batch_jobs.append(job)
+                    task_ids.append(job.job_id)
+
+            tasks_sent += current_batch_size
+            logger.info(
+                f"📤 Sent batch: {current_batch_size} tasks "
+                f"(total: {tasks_sent}/{num_tasks})"
+            )
+
+            # Add configurable delay between batches if specified
+            if delay_ms > 0 and batch_end < num_tasks:
+                await asyncio.sleep(delay_ms / 1000.0)
+
+        # Don't close the pool yet - we need it for monitoring!
+        # await pool.aclose()
+
+        logger.info(f"✅ All {tasks_sent} tasks enqueued to {queue_name}")
+
+        # Monitor task completion with timeout based on queue configuration
+        from app.core.config import settings
+
+        from app.components.worker.registry import get_queue_metadata
+        queue_metadata = get_queue_metadata(target_queue)
+        monitor_timeout = queue_metadata.get("timeout", 300)  # Use queue's timeout
+
+        logger.info(f"⏱️ Monitoring task completion (timeout: {monitor_timeout}s)...")
+
+        completion_result = await _monitor_task_completion(
+            task_ids=task_ids,
+            pool=pool,
+            expected_tasks=tasks_sent,
+            timeout_seconds=monitor_timeout,  # Use configured timeout
+        )
+
+        # NOW we can close the pool after monitoring is done
+        await pool.aclose()
+
+        end_time = datetime.now()
+        total_duration = (end_time - start_time).total_seconds()
+
+        # Combine orchestrator stats with completion monitoring
+        result = {
+            "test_id": test_id,
+            "task_type": task_type.value,
+            "tasks_sent": tasks_sent,
+            "task_ids": task_ids[:10],  # Sample of IDs for debugging
+            "batch_size": batch_size,
+            "delay_ms": delay_ms,
+            "target_queue": target_queue,
+            "start_time": start_time.isoformat(),
+            "end_time": end_time.isoformat(),
+            "total_duration_seconds": round(total_duration, 2),
+            **completion_result,  # Merge in the monitoring results
+        }
+
+        # Calculate overall throughput based on completed tasks
+        if result.get("tasks_completed", 0) > 0:
+            result["overall_throughput_per_second"] = round(
+                result["tasks_completed"] / total_duration, 2
+            )
+        else:
+            result["overall_throughput_per_second"] = 0
+
+        logger.info(
+            f"🏁 Load test complete: {result['tasks_completed']}/{tasks_sent} "
+            f"tasks in {total_duration:.1f}s"
+        )
+        logger.info(
+            f"📈 Throughput: {result['overall_throughput_per_second']} tasks/sec"
+        )
+
+        return result
+
+    except Exception as e:
+        logger.error(f"Load test orchestrator failed: {e}")
+        return {"test_id": test_id, "error": str(e), "tasks_sent": tasks_sent}
+
+
+def _get_task_function_name(task_type: LoadTestTypes) -> str:
+    """Map task type to actual function name."""
+    task_map = {
+        LoadTestTypes.CPU_INTENSIVE: TaskNames.CPU_INTENSIVE_TASK,
+        LoadTestTypes.IO_SIMULATION: TaskNames.IO_SIMULATION_TASK,
+        LoadTestTypes.MEMORY_OPERATIONS: TaskNames.MEMORY_OPERATIONS_TASK,
+        LoadTestTypes.FAILURE_TESTING: TaskNames.FAILURE_TESTING_TASK,
+    }
+    return task_map.get(task_type, TaskNames.CPU_INTENSIVE_TASK)
+
+
+async def _monitor_task_completion(
+    task_ids: list[str],
+    pool: Any,
+    expected_tasks: int,
+    timeout_seconds: int = 300,
+    poll_interval: float = 2.0,
+) -> dict[str, Any]:
+    """
+    Monitor task completion by checking job results directly.
+
+    This avoids Redis queue type errors by tracking job completion
+    instead of trying to read queue internals.
+    """
+    start_monitor = datetime.now()
+    tasks_completed = 0
+    tasks_failed = 0
+    last_progress_time = start_monitor
+    last_completed = 0
+
+    # Track which task IDs we've seen complete
+    completed_ids: set[str] = set()
+    failed_ids: set[str] = set()
+
+    try:
+        while True:
+            # Check each task ID for completion
+            for task_id in task_ids:
+                if task_id in completed_ids or task_id in failed_ids:
+                    continue  # Already processed
+
+                # Check if job result exists
+                result_key = f"arq:result:{task_id}"
+                result_data = await pool.get(result_key)
+
+                if result_data:
+                    # Job completed - check if it succeeded or failed
+                    try:
+                        # arq stores results as msgpack, but we can check existence
+                        completed_ids.add(task_id)
+                        tasks_completed += 1
+                    except Exception:
+                        # If we can't parse, assume it completed
+                        completed_ids.add(task_id)
+                        tasks_completed += 1
+
+            tasks_done = tasks_completed + tasks_failed
+
+            # Calculate throughput
+            elapsed = (datetime.now() - start_monitor).total_seconds()
+            throughput = tasks_completed / elapsed if elapsed > 0 else 0
+
+            # Check if we're making progress
+            if tasks_completed > last_completed:
+                last_progress_time = datetime.now()
+                last_completed = tasks_completed
+
+            # Progress logging (less verbose)
+            progress_pct = (
+                (tasks_done / expected_tasks * 100) if expected_tasks > 0 else 0
+            )
+            if (
+                tasks_done % 10 == 0 or tasks_done == expected_tasks
+            ):  # Log every 10 tasks or at completion
+                logger.info(
+                    f"📈 Progress: {tasks_done}/{expected_tasks} "
+                    f"({progress_pct:.0f}% - completed: {tasks_completed}, "
+                    f"failed: {tasks_failed}) throughput: {throughput:.1f} tasks/sec"
+                )
+
+            # Check completion
+            if tasks_done >= expected_tasks:
+                logger.info(
+                    f"✅ All tasks completed: {tasks_completed} success, "
+                    f"{tasks_failed} failed"
+                )
+                break
+
+            # Check timeout
+            if elapsed > timeout_seconds:
+                logger.warning(f"⏱️ Load test timed out after {timeout_seconds}s")
+                break
+
+            # Check if we're stuck (no progress for 30 seconds)
+            stuck_duration = (datetime.now() - last_progress_time).total_seconds()
+            if stuck_duration > 30 and tasks_done > 0:
+                logger.warning(
+                    f"⚠️ No progress for {stuck_duration:.0f}s, stopping monitor"
+                )
+                break
+
+            await asyncio.sleep(poll_interval)
+
+    except Exception as e:
+        logger.error(f"Task monitoring error: {e}")
+
+    # Final metrics
+    final_elapsed = (datetime.now() - start_monitor).total_seconds()
+
+    return {
+        "tasks_completed": tasks_completed,
+        "tasks_failed": tasks_failed,
+        "monitor_duration_seconds": round(final_elapsed, 2),
+        "average_throughput_per_second": round(tasks_completed / final_elapsed, 2)
+        if final_elapsed > 0
+        else 0,
+        "completion_percentage": round((tasks_completed / expected_tasks * 100), 1)
+        if expected_tasks > 0
+        else 0,
+        "failure_rate_percent": round((tasks_failed / expected_tasks * 100), 1)
+        if expected_tasks > 0
+        else 0,
+    }

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/core/config.py.j2

@@ -0,0 +1,119 @@
+# app/core/config.py
+"""
+Application configuration management using Pydantic's BaseSettings.
+
+This module centralizes application settings, allowing them to be loaded
+from environment variables for easy configuration in different environments.
+"""
+
+from typing import Any
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """
+    Defines application settings.
+    `model_config` is used to specify that settings should be loaded from a .env file.
+    """
+
+    # Application environment: "dev" or "prod"
+    APP_ENV: str = "dev"
+
+    # Log level for the application
+    LOG_LEVEL: str = "INFO"
+
+    # Port for the web server
+    PORT: int = 8000
+
+    # Development settings
+    AUTO_RELOAD: bool = False
+
+    # Docker settings (used by docker-compose)
+    AEGIS_STACK_TAG: str = "aegis-stack:latest"
+    AEGIS_STACK_VERSION: str = "dev"
+
+    # Health monitoring and alerting
+    HEALTH_CHECK_ENABLED: bool = True
+    HEALTH_CHECK_INTERVAL_MINUTES: int = 5
+
+    # Health check performance settings
+    HEALTH_CHECK_TIMEOUT_SECONDS: float = 2.0
+    SYSTEM_METRICS_CACHE_SECONDS: int = 5
+
+    # Basic alerting configuration
+    ALERTING_ENABLED: bool = False
+    ALERT_COOLDOWN_MINUTES: int = 60  # Minutes between repeated alerts for same issue
+
+    # Health check thresholds
+    MEMORY_THRESHOLD_PERCENT: float = 90.0
+    DISK_THRESHOLD_PERCENT: float = 85.0
+    CPU_THRESHOLD_PERCENT: float = 95.0
+
+{% if cookiecutter.include_redis == "yes" %}
+    # Redis settings for arq background tasks
+    REDIS_URL: str = "redis://localhost:6379"
+    REDIS_DB: int = 0
+{% endif %}
+
+{% if cookiecutter.include_worker == "yes" %}
+    # arq worker settings (shared across all workers)
+    WORKER_KEEP_RESULT_SECONDS: int = 3600  # Keep job results for 1 hour
+    WORKER_MAX_TRIES: int = 3
+
+    # PURE ARQ IMPLEMENTATION - NO CONFIGURATION NEEDED!
+    # Worker configuration comes from individual WorkerSettings classes
+    # in app/components/worker/queues/ - just import and use as arq intended!
+{% endif %}
+
+{% if cookiecutter.include_database == "yes" %}
+    # Database settings (SQLite)
+    DATABASE_URL: str = "sqlite:///./data/app.db"
+    DATABASE_ENGINE_ECHO: bool = False
+    DATABASE_CONNECT_ARGS: dict[str, Any] = {"check_same_thread": False}
+{% endif %}
+
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
+
+
+settings = Settings()
+
+
+{% if cookiecutter.include_worker == "yes" %}
+# Pure arq queue helper functions - use dynamic discovery
+def get_available_queues() -> list[str]:
+    """Get all available queue names via dynamic discovery."""
+    try:
+        from app.components.worker.registry import discover_worker_queues
+        queues: list[str] = discover_worker_queues()
+        return queues
+    except ImportError:
+        # Worker components not available
+        return []
+
+
+def get_default_queue() -> str:
+    """Get the default queue name for load testing."""
+    # Prefer load_test queue if it exists, otherwise use first available
+    available = get_available_queues()
+    if "load_test" in available:
+        return "load_test"
+    return available[0] if available else "system"
+
+
+def get_load_test_queue() -> str:
+    """Get the queue name for load testing."""
+    available = get_available_queues()
+    return "load_test" if "load_test" in available else get_default_queue()
+
+
+def is_valid_queue(queue_name: str) -> bool:
+    """Check if a queue name is valid."""
+    try:
+        from app.components.worker.registry import validate_queue_name
+        result: bool = validate_queue_name(queue_name)
+        return result
+    except ImportError:
+        # Worker components not available, no queues are valid
+        return False
+{% endif %}

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/core/constants.py

@@ -0,0 +1,60 @@
+"""
+Application constants.
+
+This module contains truly immutable values that never change across environments.
+For environment-dependent configuration, see app.core.config.
+
+Following 12-Factor App principles:
+- Constants = code (version controlled, immutable across deployments)
+- Configuration = environment (varies between dev/staging/production)
+"""
+
+
+class APIEndpoints:
+    """API endpoint paths - immutable across all environments."""
+
+    HEALTH_BASIC = "/health/"
+    HEALTH_DETAILED = "/health/detailed"
+    HEALTH_DASHBOARD = "/health/dashboard"
+
+
+class Defaults:
+    """Default values for timeouts and limits."""
+
+    # API timeouts (seconds)
+    API_TIMEOUT = 10.0
+    HEALTH_CHECK_TIMEOUT = 5.0
+
+    # Retry configuration
+    MAX_RETRIES = 3
+    RETRY_BACKOFF = 1.0
+
+    # Health check intervals (seconds)
+    HEALTH_CHECK_INTERVAL = 30
+    COMPONENT_CHECK_TIMEOUT = 2.0
+
+
+class CLI:
+    """CLI-specific constants."""
+
+    # Display limits
+    MAX_METADATA_DISPLAY_LENGTH = 30
+
+    # Output formatting
+    HEALTH_PERCENTAGE_DECIMALS = 1
+    TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+
+class HTTP:
+    """HTTP-related constants."""
+
+    # Status codes we care about
+    OK = 200
+    SERVICE_UNAVAILABLE = 503
+    INTERNAL_SERVER_ERROR = 500
+
+    # Headers
+    CONTENT_TYPE_JSON = "application/json"
+    USER_AGENT = "AegisStack-CLI/1.0"
+
+

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/core/db.py

@@ -0,0 +1,67 @@
+# app/core/db.py
+"""
+Database configuration and session management.
+
+This module provides SQLite database connectivity using SQLModel and SQLAlchemy.
+Includes proper session management with transaction handling and foreign key support.
+"""
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from typing import Any
+
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import sessionmaker
+from sqlmodel import Session
+
+from app.core.config import settings
+
+# Create SQLite engine with proper configuration
+engine = create_engine(
+    settings.DATABASE_URL,
+    connect_args=settings.DATABASE_CONNECT_ARGS,
+    echo=settings.DATABASE_ENGINE_ECHO,
+)
+
+
+# Enable foreign key constraints for SQLite
+@event.listens_for(engine, "connect")
+def set_sqlite_pragma(dbapi_connection: Any, connection_record: Any) -> None:
+    """Enable foreign key constraints in SQLite."""
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
+
+# Configure session factory with SQLModel Session
+SessionLocal = sessionmaker(
+    class_=Session, bind=engine, autoflush=False, autocommit=False
+)
+
+
+@contextmanager
+def db_session(autocommit: bool = True) -> Generator[Session, None, None]:
+    """
+    Database session context manager with automatic transaction handling.
+
+    Args:
+        autocommit: Whether to automatically commit the transaction on success
+
+    Yields:
+        Session: Database session instance
+
+    Example:
+        with db_session() as session:
+            # Your database operations here
+            result = session.query(MyModel).first()
+    """
+    db_session: Session = SessionLocal()
+    try:
+        yield db_session
+        if autocommit:
+            db_session.commit()
+    except Exception:
+        db_session.rollback()
+        raise
+    finally:
+        db_session.close()

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/core/log.py

@@ -0,0 +1,85 @@
+# app/core/log.py
+"""
+Core logging configuration for the application.
+
+This module sets up structlog to provide structured, context-aware logging.
+It supports both human-readable console output for development and JSON
+output for production environments.
+"""
+
+import logging
+import sys
+
+import structlog
+from structlog.types import Processor
+
+from app.core.config import settings
+
+# A global logger instance for easy access throughout the application
+logger: structlog.stdlib.BoundLogger = structlog.get_logger()
+
+
+def setup_logging() -> None:
+    """
+    Configures logging for the entire application.
+
+    This function sets up structlog with processors for structured logging.
+    It routes all standard library logging through structlog to ensure
+    consistent log formats. The output format is determined by the APP_ENV
+    setting (dev-friendly console format or production-ready JSON format).
+    """
+    # Type hint for the list of processors
+    shared_processors: list[Processor] = [
+        structlog.stdlib.add_logger_name,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+    ]
+
+    # Configure structlog
+    structlog.configure(
+        processors=shared_processors
+        + [
+            # Prepare event dict for `ProcessorFormatter`.
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+    # Define the formatter based on the environment
+    if settings.APP_ENV == "dev":
+        formatter = structlog.stdlib.ProcessorFormatter(
+            # The final processor formats the log entry for console output.
+            processor=structlog.dev.ConsoleRenderer(colors=True),
+        )
+    else:
+        formatter = structlog.stdlib.ProcessorFormatter(
+            # The final processor formats the log entry as JSON.
+            processor=structlog.processors.JSONRenderer(),
+            # Remove metadata added by ProcessorFormatter
+            foreign_pre_chain=shared_processors,
+        )
+
+    # Configure the root logger
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+    root_logger = logging.getLogger()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(settings.LOG_LEVEL.upper())
+
+    # Adjust log levels for noisy third-party libraries
+    logging.getLogger("flet_core").setLevel(logging.INFO)
+    logging.getLogger("flet_runtime").setLevel(logging.INFO)
+    logging.getLogger("flet_fastapi").setLevel(logging.INFO)
+    logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
+
+    log_format = "DEV" if settings.APP_ENV == "dev" else "JSON"
+    logger.info(
+        "Logging setup complete",
+        level=settings.LOG_LEVEL,
+        log_format=log_format,
+    )

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/entrypoints/__init__.py

@@ -0,0 +1 @@
+# Entry points for different Aegis Stack execution modes

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/entrypoints/webserver.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""
+Web server entry point for Aegis Stack.
+Runs FastAPI + Flet only (clean separation of concerns).
+"""
+
+import uvicorn
+
+from app.core.config import settings
+from app.core.log import logger, setup_logging
+from app.integrations.main import create_integrated_app
+
+
+def main() -> None:
+    """Main webserver entry point"""
+    setup_logging()
+    logger.info("Starting Aegis Stack Web Server...")
+
+    # Run the web server
+    if settings.AUTO_RELOAD:
+        # When reload is enabled, uvicorn requires an import string
+        uvicorn.run(
+            "app.integrations.main:create_integrated_app",
+            factory=True,
+            host="0.0.0.0",
+            port=settings.PORT,
+            reload=True,
+        )
+    else:
+        # Use the integration layer (handles webserver hooks, service discovery, etc.)
+        app = create_integrated_app()
+        uvicorn.run(
+            app,
+            host="0.0.0.0",
+            port=settings.PORT,
+        )
+
+
+if __name__ == "__main__":
+    main()

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/entrypoints/{% if cookiecutter.include_scheduler == /"yes/" %}scheduler.py{% endif %}"

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""
+Scheduler entrypoint for {{ cookiecutter.project_name }}.
+
+This entrypoint starts the scheduler component.
+"""
+
+import asyncio
+
+from app.components.scheduler.main import run_scheduler
+from app.core.log import setup_logging
+
+
+async def main() -> None:
+    """Main scheduler entry point"""
+    setup_logging()
+    await run_scheduler()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())