aegis-stack 0.2.0rc2__py3-none-any.whl

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/services/load_test_models.py

@@ -0,0 +1,266 @@
+"""
+Pydantic models for load test data structures.
+
+Provides type safety and validation for load test configurations,
+results, and analysis data.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field, ValidationInfo, field_validator, model_validator
+
+from app.components.worker.constants import LoadTestTypes
+from app.core.config import get_load_test_queue
+
+
+class LoadTestError(Exception):
+    """Custom exception for load test operations."""
+
+    pass
+
+
+class LoadTestConfiguration(BaseModel):
+    """Load test configuration with validation and defaults."""
+
+    num_tasks: int = Field(
+        default=100, ge=10, le=10000, description="Number of tasks to spawn"
+    )
+    task_type: LoadTestTypes = Field(
+        default=LoadTestTypes.CPU_INTENSIVE, description="Type of load test to run"
+    )
+    batch_size: int = Field(default=10, ge=1, le=100, description="Tasks per batch")
+    delay_ms: int = Field(
+        default=0, ge=0, le=5000, description="Delay between batches (ms)"
+    )
+    target_queue: str | None = Field(
+        default=None, description="Target queue for testing"
+    )
+
+    @field_validator("target_queue")
+    @classmethod
+    def set_default_queue(cls, v: str | None) -> str:
+        """Set default queue if not specified."""
+        return v if v is not None else get_load_test_queue()
+
+    def model_dump(self, **kwargs: Any) -> dict[str, Any]:
+        """Convert configuration to dictionary for task enqueueing."""
+        data = super().model_dump(**kwargs)
+        # Convert enum to string value for task enqueueing
+        data["task_type"] = self.task_type.value
+        return data
+
+
+class LoadTestMetrics(BaseModel):
+    """Metrics from load test execution."""
+
+    tasks_sent: int = Field(..., ge=0, description="Total tasks enqueued")
+    tasks_completed: int = Field(..., ge=0, description="Successfully completed tasks")
+    tasks_failed: int = Field(0, ge=0, description="Failed tasks")
+    total_duration_seconds: float = Field(..., ge=0, description="Total test duration")
+    overall_throughput: float = Field(
+        0, ge=0, description="Overall throughput (tasks/sec)"
+    )
+    failure_rate_percent: float = Field(
+        0, ge=0, le=100, description="Failure rate percentage"
+    )
+    completion_percentage: float = Field(
+        0, ge=0, le=100, description="Completion percentage"
+    )
+    average_throughput_per_second: float = Field(
+        0, ge=0, description="Average throughput"
+    )
+    monitor_duration_seconds: float = Field(0, ge=0, description="Monitoring duration")
+
+    @field_validator("tasks_completed")
+    @classmethod
+    def completed_not_exceed_sent(cls, v: int, info: ValidationInfo) -> int:
+        """Ensure completed tasks don't exceed sent tasks."""
+        if info.data and "tasks_sent" in info.data and v > info.data["tasks_sent"]:
+            raise ValueError(
+                f"Completed tasks ({v}) cannot exceed sent tasks "
+                f"({info.data['tasks_sent']})"
+            )
+        return v
+
+    @field_validator("tasks_failed")
+    @classmethod
+    def failed_not_exceed_sent(cls, v: int, info: ValidationInfo) -> int:
+        """Ensure failed tasks don't exceed sent tasks."""
+        if info.data and "tasks_sent" in info.data and v > info.data["tasks_sent"]:
+            raise ValueError(
+                f"Failed tasks ({v}) cannot exceed sent tasks "
+                f"({info.data['tasks_sent']})"
+            )
+        return v
+
+    @field_validator("failure_rate_percent")
+    @classmethod
+    def validate_failure_rate_consistency(cls, v: float, info: ValidationInfo) -> float:
+        """Ensure failure rate percentage matches task counts."""
+        if info.data and "tasks_sent" in info.data and "tasks_failed" in info.data:
+            tasks_sent = info.data["tasks_sent"]
+            tasks_failed = info.data["tasks_failed"]
+            if tasks_sent > 0:
+                calculated_rate = (tasks_failed / tasks_sent) * 100
+                # Allow small floating point differences (within 0.1%)
+                if abs(v - calculated_rate) > 0.1:
+                    raise ValueError(
+                        f"Failure rate {v}% doesn't match task counts "
+                        f"({tasks_failed}/{tasks_sent} = {calculated_rate:.1f}%)"
+                    )
+        return v
+
+
+class PerformanceAnalysis(BaseModel):
+    """Performance analysis results."""
+
+    throughput_rating: str = Field(
+        ...,
+        pattern=r"^(unknown|poor|fair|good|excellent)$",
+        description="Throughput performance rating",
+    )
+    efficiency_rating: str = Field(
+        ...,
+        pattern=r"^(unknown|poor|fair|good|excellent)$",
+        description="Task completion efficiency",
+    )
+    queue_pressure: str = Field(
+        ...,
+        pattern=r"^(unknown|low|medium|high)$",
+        description="Queue saturation level",
+    )
+
+
+class ValidationStatus(BaseModel):
+    """Test execution validation status."""
+
+    test_type_verified: bool = Field(
+        default=False, description="Test type executed correctly"
+    )
+    expected_metrics_present: bool = Field(
+        default=False, description="Expected metrics are present"
+    )
+    performance_signature_match: str = Field(
+        default="unknown",
+        pattern=r"^(unknown|verified|partial|failed)$",
+        description="Performance matches expected patterns",
+    )
+    issues: list[str] = Field(default_factory=list, description="Validation issues")
+
+
+class TestTypeInfo(BaseModel):
+    """Information about a specific test type."""
+
+    name: str = Field(..., description="Human-readable test name")
+    description: str = Field(..., description="Test description")
+    expected_metrics: list[str] = Field(..., description="Expected result metrics")
+    performance_signature: str = Field(..., description="Expected performance pattern")
+    typical_duration_ms: str = Field(..., description="Typical execution time")
+    concurrency_impact: str = Field(..., description="Concurrency characteristics")
+    validation_keys: list[str] = Field(..., description="Keys for result validation")
+
+
+class LoadTestAnalysis(BaseModel):
+    """Complete load test analysis."""
+
+    test_type_info: TestTypeInfo = Field(..., description="Test type information")
+    performance_analysis: PerformanceAnalysis = Field(
+        ..., description="Performance analysis"
+    )
+    validation_status: ValidationStatus = Field(..., description="Validation results")
+    recommendations: list[str] = Field(..., description="Improvement recommendations")
+
+
+class LoadTestResult(BaseModel):
+    """Complete load test result with analysis."""
+
+    task: str = Field(default="load_test_orchestrator", description="Task name")
+    status: str = Field(
+        ...,
+        pattern=r"^(completed|failed|timed_out)$",
+        description="Test execution status",
+    )
+    test_id: str = Field(..., description="Unique test identifier")
+    configuration: LoadTestConfiguration = Field(..., description="Test configuration")
+    metrics: LoadTestMetrics = Field(..., description="Execution metrics")
+    start_time: str | None = Field(None, description="Test start time")
+    end_time: str | None = Field(None, description="Test end time")
+    task_ids: list[str] = Field(default_factory=list, description="Individual task IDs")
+    error: str | None = Field(None, description="Error message if failed")
+    analysis: LoadTestAnalysis | None = Field(None, description="Performance analysis")
+
+    @model_validator(mode="after")
+    def validate_status_consistency(self) -> "LoadTestResult":
+        """Validate status consistency with error field."""
+        if self.status == "failed" and not self.error:
+            raise ValueError("Failed status requires error message")
+        return self
+
+
+class OrchestratorRawResult(BaseModel):
+    """Raw orchestrator result format for transformation."""
+
+    test_id: str = Field(..., description="Test identifier")
+    task_type: str = Field(..., description="Task type executed")
+    tasks_sent: int = Field(..., description="Tasks enqueued")
+    tasks_completed: int = Field(0, description="Successfully completed")
+    tasks_failed: int = Field(0, description="Failed tasks")
+    total_duration_seconds: float = Field(..., description="Total duration")
+    overall_throughput_per_second: float = Field(0, description="Overall throughput")
+    failure_rate_percent: float = Field(0, description="Failure rate")
+    completion_percentage: float = Field(0, description="Completion rate")
+    average_throughput_per_second: float = Field(0, description="Average throughput")
+    monitor_duration_seconds: float = Field(0, description="Monitor duration")
+    batch_size: int = Field(1, description="Batch size used")
+    delay_ms: int = Field(0, description="Delay between batches")
+    target_queue: str = Field(..., description="Target queue")
+    start_time: str | None = Field(None, description="Start time")
+    end_time: str | None = Field(None, description="End time")
+    task_ids: list[str] = Field(default_factory=list, description="Task IDs")
+
+    def to_load_test_result(self) -> LoadTestResult:
+        """Transform to standard LoadTestResult format."""
+        configuration = LoadTestConfiguration(
+            task_type=LoadTestTypes(self.task_type),
+            num_tasks=self.tasks_sent,
+            batch_size=self.batch_size,
+            delay_ms=self.delay_ms,
+            target_queue=self.target_queue,
+        )
+
+        metrics = LoadTestMetrics(
+            tasks_sent=self.tasks_sent,
+            tasks_completed=self.tasks_completed,
+            tasks_failed=self.tasks_failed,
+            total_duration_seconds=self.total_duration_seconds,
+            overall_throughput=self.overall_throughput_per_second,
+            failure_rate_percent=self.failure_rate_percent,
+            completion_percentage=self.completion_percentage,
+            average_throughput_per_second=self.average_throughput_per_second,
+            monitor_duration_seconds=self.monitor_duration_seconds,
+        )
+
+        return LoadTestResult(
+            status="completed",
+            test_id=self.test_id,
+            configuration=configuration,
+            metrics=metrics,
+            start_time=self.start_time,
+            end_time=self.end_time,
+            task_ids=self.task_ids,
+            error=None,
+            analysis=None,
+        )
+
+
+class LoadTestErrorModel(BaseModel):
+    """Load test error result with partial information."""
+
+    task: str = Field(default="load_test_orchestrator", description="Task name")
+    status: str = Field(
+        ..., pattern=r"^(failed|timed_out)$", description="Error status"
+    )
+    test_id: str = Field(..., description="Unique test identifier")
+    error: str = Field(..., description="Error message")
+    partial_info: str | None = Field(None, description="Partial completion info")
+    tasks_sent: int | None = Field(None, ge=0, description="Tasks that were sent")

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/services/scheduler/__init__.py.j2

@@ -0,0 +1,21 @@
+{% if cookiecutter.scheduler_backend != "memory" %}
+"""Scheduler service layer for async task management."""
+
+from .models import (
+    APSchedulerJob,
+    ScheduledTask,
+    TaskStatistics,
+    SchedulerHealthMetadata,
+    UpcomingTask,
+)
+from .scheduled_task_manager import ScheduledTaskManager
+
+__all__ = [
+    "ScheduledTaskManager",
+    "ScheduledTask",
+    "TaskStatistics",
+    "APSchedulerJob",
+    "SchedulerHealthMetadata",
+    "UpcomingTask",
+]
+{% endif %}

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/services/scheduler/models.py.j2

@@ -0,0 +1,119 @@
+"""Models for scheduled task management - both database and service layer."""
+
+import pickle
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+{% if cookiecutter.scheduler_backend != "memory" %}
+from sqlalchemy import Column, Float, LargeBinary, String
+from sqlmodel import Field as SQLField, SQLModel
+{% endif %}
+
+
+{% if cookiecutter.scheduler_backend != "memory" %}
+# ============================================================================
+# DATABASE MODELS (SQLModel)
+# ============================================================================
+
+class APSchedulerJob(SQLModel, table=True):
+    """
+    Mirror of APScheduler's job persistence table.
+    This model allows us to read APScheduler data using SQLModel's async capabilities.
+
+    Note: APScheduler creates this table, we only read from it.
+    The table structure matches APScheduler 3.x exactly.
+    """
+    __tablename__ = "apscheduler_jobs"
+
+    # Use SA Column types for exact compatibility with APScheduler's table structure
+    id: str = SQLField(
+        sa_column=Column(String(191), primary_key=True)
+    )
+    next_run_time: float | None = SQLField(
+        sa_column=Column(Float(25), nullable=True, index=True)
+    )
+    job_state: bytes = SQLField(
+        sa_column=Column(LargeBinary, nullable=False)
+    )
+
+    def get_job_data(self) -> dict[str, Any]:
+        """
+        Deserialize the pickled job state.
+
+        Returns:
+            dict: Job data with standard APScheduler keys.
+
+        Note:
+            This is safe because we're reading APScheduler's own data format.
+            APScheduler writes pickled job data to this table - we're just reading it.
+        """
+        return pickle.loads(self.job_state)  # type: ignore[no-any-return]
+
+
+# ============================================================================
+# SERVICE LAYER MODELS (Pydantic)
+# ============================================================================
+
+class ScheduledTask(BaseModel):
+    """
+    Represents a scheduled task with all relevant information.
+    This is the clean data model used by CLI/API consumers.
+    """
+    job_id: str = Field(..., description="Unique task identifier")
+    name: str = Field(..., description="Human-readable task name")
+    function: str = Field(..., description="Function reference (module:function)")
+    schedule: str = Field(..., description="Human-readable schedule (e.g., 'Every 5m')")
+    trigger_type: Literal["interval", "cron", "date", "unknown"] = Field(
+        ..., description="Type of scheduling trigger"
+    )
+    next_run_time: datetime | None = Field(
+        None, description="Next scheduled execution time"
+    )
+    status: Literal["active", "paused"] = Field(
+        ..., description="Current task status"
+    )
+    max_instances: int = Field(
+        1, description="Maximum concurrent instances allowed"
+    )
+    coalesce: bool = Field(
+        True, description="Whether to coalesce missed runs"
+    )
+
+    @property
+    def is_active(self) -> bool:
+        """Check if the task is currently active."""
+        return self.status == "active"
+
+
+class TaskStatistics(BaseModel):
+    """Statistics about scheduled tasks in the system."""
+    total_tasks: int = Field(0, description="Total number of scheduled tasks")
+    active_tasks: int = Field(0, description="Number of active tasks")
+    paused_tasks: int = Field(0, description="Number of paused tasks")
+
+
+{% endif %}
+
+
+# ============================================================================
+# HEALTH MONITORING MODELS (Available for all scheduler modes)
+# ============================================================================
+
+class UpcomingTask(BaseModel):
+    """Information about an upcoming scheduled task."""
+    job_id: str = Field(..., description="Task identifier")
+    name: str = Field(..., description="Human-readable task name")
+    next_run: str = Field(..., description="ISO datetime string of next execution")
+    schedule: str = Field(..., description="Human-readable schedule description")
+
+
+class SchedulerHealthMetadata(BaseModel):
+    """Health metadata for scheduler components."""
+    total_tasks: int = Field(0, description="Total number of scheduled tasks")
+    active_tasks: int = Field(0, description="Number of active tasks")
+    paused_tasks: int = Field(0, description="Number of paused tasks")
+    upcoming_tasks: list[UpcomingTask] = Field(
+        default_factory=list, description="List of upcoming tasks (max 5)"
+    )
+    scheduler_state: str = Field("unknown", description="Scheduler state description")

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/services/scheduler/scheduled_task_manager.py.j2

@@ -0,0 +1,273 @@
+{% if cookiecutter.scheduler_backend != "memory" %}
+"""Async scheduled task manager using SQLModel for database abstraction."""
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import inspect
+from sqlmodel import select
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.core.db import async_engine, get_async_session
+from app.core.log import logger
+
+from .models import APSchedulerJob, ScheduledTask, TaskStatistics
+
+
+class ScheduledTaskManager:
+    """
+    Manages scheduled tasks via async database operations.
+
+    Uses SQLModel for database abstraction, supporting SQLite, PostgreSQL, MySQL.
+    Only available when scheduler persistence is enabled - reads from
+    apscheduler_jobs table.
+    """
+    async def has_persistence(self) -> bool:
+        """Check if apscheduler_jobs table exists."""
+        try:
+            async with async_engine.begin() as conn:
+                # Use inspector to check tables
+                tables = await conn.run_sync(
+                    lambda sync_conn: inspect(sync_conn).get_table_names()
+                )
+                return "apscheduler_jobs" in tables
+        except Exception as e:
+            logger.error(f"Error checking persistence: {e}")
+            return False
+    async def list_tasks(self) -> list[ScheduledTask]:
+        """
+        List all scheduled tasks from the database.
+
+        Returns:
+            list[ScheduledTask]: List of all scheduled tasks with their details.
+
+        Raises:
+            RuntimeError: If persistence is not available.
+        """
+        if not await self.has_persistence():
+            raise RuntimeError(
+                "Scheduled task listing requires persistence. "
+                "The apscheduler_jobs table was not found in the database."
+            )
+
+        async with get_async_session() as session:
+            # Query all jobs ordered by next run time (nulls last)
+            result = await session.exec(
+                select(APSchedulerJob).order_by(
+                    APSchedulerJob.next_run_time.desc().nulls_last()  # type: ignore[union-attr]
+                )
+            )
+            jobs = result.all()
+
+            tasks = []
+            for job in jobs:
+                try:
+                    job_data = job.get_job_data()
+                    task = ScheduledTask(
+                        job_id=job.id,
+                        name=job_data.get("name", job.id),
+                        function=job_data.get("func", "unknown"),
+                        schedule=self._format_trigger(job_data.get("trigger")),
+                        trigger_type=self._get_trigger_type(job_data.get("trigger")),
+                        next_run_time=(
+                            datetime.fromtimestamp(job.next_run_time)
+                            if job.next_run_time else None
+                        ),
+                        status="active" if job.next_run_time else "paused",
+                        max_instances=job_data.get("max_instances", 1),
+                        coalesce=job_data.get("coalesce", True),
+                    )
+                    tasks.append(task)
+                except Exception as e:
+                    logger.error(f"Error processing job {job.id}: {e}")
+                    continue
+
+            return tasks
+
+    async def get_task(self, task_id: str) -> ScheduledTask | None:
+        """
+        Get a specific task by ID.
+
+        Args:
+            task_id: The unique identifier of the task.
+
+        Returns:
+            ScheduledTask | None: The task if found, None otherwise.
+        """
+        async with get_async_session() as session:
+            result = await session.exec(
+                select(APSchedulerJob).where(APSchedulerJob.id == task_id)
+            )
+            job = result.first()
+
+            if not job:
+                return None
+
+            try:
+                job_data = job.get_job_data()
+                return ScheduledTask(
+                    job_id=job.id,
+                    name=job_data.get("name", job.id),
+                    function=job_data.get("func", "unknown"),
+                    schedule=self._format_trigger(job_data.get("trigger")),
+                    trigger_type=self._get_trigger_type(job_data.get("trigger")),
+                    next_run_time=(
+                        datetime.fromtimestamp(job.next_run_time)
+                        if job.next_run_time else None
+                    ),
+                    status="active" if job.next_run_time else "paused",
+                    max_instances=job_data.get("max_instances", 1),
+                    coalesce=job_data.get("coalesce", True),
+                )
+            except Exception as e:
+                logger.error(f"Error processing job {task_id}: {e}")
+                return None
+
+    async def get_statistics(self) -> TaskStatistics:
+        """
+        Get task statistics.
+
+        Returns:
+            TaskStatistics: Summary statistics about all scheduled tasks.
+        """
+        # TODO: Optimize with SQL aggregation queries instead of fetching all tasks
+        tasks = await self.list_tasks()
+        active = sum(1 for t in tasks if t.status == "active")
+        paused = sum(1 for t in tasks if t.status == "paused")
+
+        return TaskStatistics(
+            total_tasks=len(tasks),
+            active_tasks=active,
+            paused_tasks=paused,
+        )
+
+    def _format_trigger(self, trigger: Any) -> str:
+        """
+        Convert APScheduler trigger to human-readable string.
+
+        Args:
+            trigger: APScheduler trigger object (IntervalTrigger, CronTrigger, etc.)
+
+        Returns:
+            str: Human-readable schedule description.
+        """
+        if not trigger:
+            return "Unknown"
+
+        trigger_type = type(trigger).__name__
+
+        if trigger_type == "IntervalTrigger":
+            if hasattr(trigger, "interval"):
+                seconds = trigger.interval.total_seconds()
+                if seconds < 60:
+                    return f"Every {int(seconds)}s"
+                elif seconds < 3600:
+                    minutes = int(seconds / 60)
+                    return f"Every {minutes}m"
+                elif seconds < 86400:
+                    hours = seconds / 3600
+                    if hours == int(hours):
+                        return f"Every {int(hours)}h"
+                    else:
+                        return f"Every {hours:.1f}h"
+                else:
+                    days = int(seconds / 86400)
+                    return f"Every {days}d"
+
+        elif trigger_type == "CronTrigger":
+            if hasattr(trigger, "fields"):
+                parts = []
+                for field in trigger.fields:
+                    field_str = str(field)
+                    if field_str != "*":
+                        parts.append(f"{field.name}={field_str}")
+                if parts:
+                    return "Cron: " + ", ".join(parts)
+                return "Cron: * * * * *"
+
+        elif trigger_type == "DateTrigger":
+            if hasattr(trigger, "run_date"):
+                return f"Once at {trigger.run_date.strftime('%Y-%m-%d %H:%M:%S')}"
+
+        return trigger_type.replace("Trigger", "")
+
+    def _get_trigger_type(self, trigger: Any) -> str:
+        """
+        Extract trigger type as string.
+
+        Args:
+            trigger: APScheduler trigger object.
+
+        Returns:
+            str: Trigger type name (interval, cron, date, unknown).
+        """
+        if not trigger:
+            return "unknown"
+
+        trigger_name = type(trigger).__name__
+        if "Interval" in trigger_name:
+            return "interval"
+        elif "Cron" in trigger_name:
+            return "cron"
+        elif "Date" in trigger_name:
+            return "date"
+        return "unknown"
+
+    # Placeholder methods for CLI functionality
+    async def get_job_statistics(self, job_id: str) -> dict[str, Any]:
+        """Get statistics for a specific job."""
+        # TODO: Implement job-specific statistics
+        task = await self.get_task(job_id)
+        if not task:
+            return {}
+
+        return {
+            "job_id": job_id,
+            "total_executions": 0,  # Would need execution history table
+            "successful_executions": 0,
+            "failed_executions": 0,
+            "success_rate_percent": 0.0,
+            "avg_duration_ms": 0.0,
+            "min_duration_ms": 0.0,
+            "max_duration_ms": 0.0,
+            "last_execution": "Never",
+            "next_run": (
+                task.next_run_time.isoformat()
+                if task.next_run_time
+                else "Not scheduled"
+            ),
+        }
+
+    async def get_overall_statistics(self) -> dict[str, Any]:
+        """Get overall scheduler statistics."""
+        stats = await self.get_statistics()
+        return {
+            "total_jobs": stats.total_tasks,
+            "active_jobs": stats.active_tasks,
+            "paused_jobs": stats.paused_tasks,
+            "total_executions": 0,  # Would need execution history table
+            "successful_executions": 0,
+            "failed_executions": 0,
+            "success_rate_percent": 0.0,
+            "avg_duration_ms": 0.0,
+            "uptime": "Unknown",
+            "last_activity": "No recent activity",
+        }
+
+    async def get_job_history(
+        self, limit: int, job_id: str | None = None
+    ) -> list[dict[str, Any]]:
+        """Get recent job execution history."""
+        # TODO: Implement execution history - would need additional table
+        return []
+
+    async def trigger_job(
+        self, job_id: str, wait: bool = True, timeout: int = 30
+    ) -> dict[str, Any]:
+        """Trigger manual job execution."""
+        # TODO: Implement job triggering - would need scheduler integration
+        return {
+            "status": "error",
+            "error": "Manual job triggering not yet implemented",
+        }
+{% endif %}