django-cfg 1.5.1__py3-none-any.whl → 1.5.3__py3-none-any.whl

django_cfg/apps/rq/services/models/queue.py

@@ -0,0 +1,92 @@
+"""
+Pydantic models for RQ Queues.
+
+Internal models for queue validation and business logic.
+NO NESTED JSON - all fields are flat!
+"""
+
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class RQQueueModel(BaseModel):
+    """
+    Internal model for RQ Queue statistics.
+
+    FLAT STRUCTURE - no nested objects!
+    All fields are primitive types.
+    """
+
+    # Basic info
+    name: str = Field(..., description="Queue name")
+    is_async: bool = Field(default=True, description="Queue is in async mode")
+
+    # Job counts by status
+    count: int = Field(default=0, ge=0, description="Total jobs in queue")
+    queued_jobs: int = Field(default=0, ge=0, description="Jobs waiting to be processed")
+    started_jobs: int = Field(default=0, ge=0, description="Jobs currently being processed")
+    finished_jobs: int = Field(default=0, ge=0, description="Completed jobs")
+    failed_jobs: int = Field(default=0, ge=0, description="Failed jobs")
+    deferred_jobs: int = Field(default=0, ge=0, description="Deferred jobs")
+    scheduled_jobs: int = Field(default=0, ge=0, description="Scheduled jobs")
+
+    # Worker info
+    workers: int = Field(default=0, ge=0, description="Number of workers for this queue")
+
+    # Metadata
+    oldest_job_timestamp: Optional[str] = Field(
+        None, description="Timestamp of oldest job in queue (ISO 8601)"
+    )
+
+    # Connection info (flat!)
+    connection_host: Optional[str] = Field(None, description="Redis host")
+    connection_port: Optional[int] = Field(None, description="Redis port")
+    connection_db: Optional[int] = Field(None, description="Redis DB number")
+
+    @property
+    def total_jobs(self) -> int:
+        """Calculate total jobs across all statuses."""
+        return (
+            self.queued_jobs
+            + self.started_jobs
+            + self.finished_jobs
+            + self.failed_jobs
+            + self.deferred_jobs
+            + self.scheduled_jobs
+        )
+
+    @property
+    def completed_jobs(self) -> int:
+        """Total completed jobs (finished + failed)."""
+        return self.finished_jobs + self.failed_jobs
+
+    @property
+    def failure_rate(self) -> float:
+        """Calculate failure rate percentage (0-100%)."""
+        completed = self.completed_jobs
+        if completed == 0:
+            return 0.0
+        return (self.failed_jobs / completed) * 100
+
+    @property
+    def is_empty(self) -> bool:
+        """Check if queue is empty."""
+        return self.count == 0
+
+    @property
+    def has_workers(self) -> bool:
+        """Check if queue has any workers."""
+        return self.workers > 0
+
+    @property
+    def is_healthy(self) -> bool:
+        """
+        Check if queue is healthy.
+
+        Queue is healthy if:
+        - Has workers
+        - Failure rate < 50%
+        - Not too many queued jobs (< 1000)
+        """
+        return self.has_workers and self.failure_rate < 50 and self.queued_jobs < 1000

django_cfg/apps/rq/services/models/worker.py

@@ -0,0 +1,104 @@
+"""
+Pydantic models for RQ Workers.
+
+Internal models for worker validation and business logic.
+NO NESTED JSON - all fields are flat!
+"""
+
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class WorkerState(str, Enum):
+    """Worker state enumeration."""
+
+    IDLE = "idle"
+    BUSY = "busy"
+    SUSPENDED = "suspended"
+
+
+class RQWorkerModel(BaseModel):
+    """
+    Internal model for RQ Worker.
+
+    FLAT STRUCTURE - no nested objects!
+    Queues are comma-separated string, timestamps are ISO strings.
+    """
+
+    # Basic info
+    name: str = Field(..., description="Worker name/ID")
+    state: WorkerState = Field(..., description="Worker state (idle/busy/suspended)")
+
+    # Queues (comma-separated for flat structure)
+    queues: str = Field(..., description="Comma-separated queue names (e.g., 'default,high,low')")
+
+    # Current job
+    current_job_id: Optional[str] = Field(None, description="Current job ID if busy")
+
+    # Timestamps (as ISO strings)
+    birth: str = Field(..., description="Worker start time (ISO 8601)")
+    last_heartbeat: str = Field(..., description="Last heartbeat timestamp (ISO 8601)")
+
+    # Statistics
+    successful_job_count: int = Field(default=0, ge=0, description="Total successful jobs")
+    failed_job_count: int = Field(default=0, ge=0, description="Total failed jobs")
+    total_working_time: float = Field(default=0.0, ge=0.0, description="Total working time in seconds")
+
+    @property
+    def is_alive(self) -> bool:
+        """
+        Check if worker is alive.
+
+        Worker is considered alive if heartbeat was within last 60 seconds.
+        """
+        try:
+            last_hb = datetime.fromisoformat(self.last_heartbeat)
+            now = datetime.now(last_hb.tzinfo) if last_hb.tzinfo else datetime.now()
+            delta = (now - last_hb).total_seconds()
+            return delta < 60
+        except (ValueError, TypeError):
+            return False
+
+    @property
+    def is_busy(self) -> bool:
+        """Check if worker is busy."""
+        return self.state == WorkerState.BUSY
+
+    @property
+    def is_idle(self) -> bool:
+        """Check if worker is idle."""
+        return self.state == WorkerState.IDLE
+
+    def get_uptime_seconds(self) -> Optional[float]:
+        """Calculate worker uptime in seconds."""
+        try:
+            birth_dt = datetime.fromisoformat(self.birth)
+            now = datetime.now(birth_dt.tzinfo) if birth_dt.tzinfo else datetime.now()
+            return (now - birth_dt).total_seconds()
+        except (ValueError, TypeError):
+            return None
+
+    def get_queue_list(self) -> list[str]:
+        """Get list of queue names."""
+        return [q.strip() for q in self.queues.split(",") if q.strip()]
+
+    @property
+    def total_job_count(self) -> int:
+        """Total jobs processed (successful + failed)."""
+        return self.successful_job_count + self.failed_job_count
+
+    @property
+    def success_rate(self) -> float:
+        """Calculate success rate (0-100%)."""
+        total = self.total_job_count
+        if total == 0:
+            return 0.0
+        return (self.successful_job_count / total) * 100
+
+    class Config:
+        """Pydantic config."""
+
+        use_enum_values = True

django_cfg/apps/rq/services/rq_converters.py

@@ -0,0 +1,183 @@
+"""
+RQ Object to Pydantic Model Converters.
+
+Converts RQ objects (Job, Queue, Worker) to type-safe Pydantic models
+for internal business logic.
+"""
+
+import json
+from typing import Optional
+
+from rq import Queue, Worker
+from rq.job import Job
+
+from .models import RQJobModel, RQQueueModel, RQWorkerModel, JobStatus, WorkerState
+
+
+def job_to_model(job: Job, queue_name: Optional[str] = None) -> RQJobModel:
+    """
+    Convert RQ Job to Pydantic RQJobModel.
+
+    Args:
+        job: RQ Job instance
+        queue_name: Queue name (optional, will try to get from job.origin)
+
+    Returns:
+        Validated RQJobModel instance
+    """
+    # Get queue name
+    if not queue_name:
+        queue_name = getattr(job, 'origin', 'unknown')
+
+    # Map RQ status to JobStatus enum
+    rq_status = job.get_status()
+    status_map = {
+        'queued': JobStatus.QUEUED,
+        'started': JobStatus.STARTED,
+        'finished': JobStatus.FINISHED,
+        'failed': JobStatus.FAILED,
+        'deferred': JobStatus.DEFERRED,
+        'scheduled': JobStatus.SCHEDULED,
+        'canceled': JobStatus.CANCELED,
+    }
+    status = status_map.get(rq_status, JobStatus.QUEUED)
+
+    # Serialize args/kwargs/meta to JSON strings (flat!)
+    args_json = json.dumps(list(job.args or []))
+    kwargs_json = json.dumps(job.kwargs or {})
+    meta_json = json.dumps(job.meta or {})
+
+    # Serialize result to JSON string if available
+    result_json = None
+    if job.result is not None:
+        try:
+            result_json = json.dumps(job.result)
+        except (TypeError, ValueError):
+            # If result is not JSON serializable, convert to string
+            result_json = json.dumps(str(job.result))
+
+    # Get dependency IDs as comma-separated string
+    dependency_ids = ""
+    if hasattr(job, '_dependency_ids') and job._dependency_ids:
+        dependency_ids = ",".join(job._dependency_ids)
+
+    return RQJobModel(
+        id=job.id,
+        func_name=job.func_name or "unknown",
+        queue=queue_name,
+        status=status,
+        created_at=job.created_at.isoformat() if job.created_at else "",
+        enqueued_at=job.enqueued_at.isoformat() if job.enqueued_at else None,
+        started_at=job.started_at.isoformat() if job.started_at else None,
+        ended_at=job.ended_at.isoformat() if job.ended_at else None,
+        worker_name=job.worker_name,
+        timeout=job.timeout,
+        result_ttl=job.result_ttl,
+        failure_ttl=job.failure_ttl,
+        result_json=result_json,
+        exc_info=job.exc_info,
+        args_json=args_json,
+        kwargs_json=kwargs_json,
+        meta_json=meta_json,
+        dependency_ids=dependency_ids,
+    )
+
+
+def worker_to_model(worker: Worker) -> RQWorkerModel:
+    """
+    Convert RQ Worker to Pydantic RQWorkerModel.
+
+    Args:
+        worker: RQ Worker instance
+
+    Returns:
+        Validated RQWorkerModel instance
+    """
+    # Get worker state
+    rq_state = worker.get_state()
+    state_map = {
+        'idle': WorkerState.IDLE,
+        'busy': WorkerState.BUSY,
+        'suspended': WorkerState.SUSPENDED,
+    }
+    state = state_map.get(rq_state, WorkerState.IDLE)
+
+    # Get queues as comma-separated string (flat!)
+    queue_names = [q.name for q in worker.queues]
+    queues_str = ",".join(queue_names)
+
+    # Get current job ID
+    current_job_id = worker.get_current_job_id()
+
+    return RQWorkerModel(
+        name=worker.name,
+        state=state,
+        queues=queues_str,
+        current_job_id=current_job_id,
+        birth=worker.birth_date.isoformat() if worker.birth_date else "",
+        last_heartbeat=worker.last_heartbeat.isoformat() if worker.last_heartbeat else "",
+        successful_job_count=worker.successful_job_count,
+        failed_job_count=worker.failed_job_count,
+        total_working_time=worker.total_working_time,
+    )
+
+
+def queue_to_model(queue: Queue, queue_name: str) -> RQQueueModel:
+    """
+    Convert RQ Queue to Pydantic RQQueueModel.
+
+    Args:
+        queue: RQ Queue instance
+        queue_name: Queue name
+
+    Returns:
+        Validated RQQueueModel instance
+    """
+    # Get job counts from registries
+    queued_jobs = len(queue.get_job_ids())
+    started_jobs = len(queue.started_job_registry)
+    finished_jobs = len(queue.finished_job_registry)
+    failed_jobs = len(queue.failed_job_registry)
+    deferred_jobs = len(queue.deferred_job_registry)
+    scheduled_jobs = len(queue.scheduled_job_registry)
+
+    # Get worker count
+    workers = Worker.all(queue=queue)
+    worker_count = len(workers)
+
+    # Get oldest job timestamp
+    oldest_job_timestamp = None
+    if queue.count > 0:
+        try:
+            oldest_job = queue.get_jobs(0, 1)[0]
+            oldest_job_timestamp = oldest_job.created_at.isoformat() if oldest_job.created_at else None
+        except (IndexError, AttributeError):
+            pass
+
+    # Get connection info (flat!)
+    connection_host = None
+    connection_port = None
+    connection_db = None
+    if hasattr(queue.connection, 'connection_pool'):
+        pool = queue.connection.connection_pool
+        connection_kwargs = pool.connection_kwargs
+        connection_host = connection_kwargs.get('host', 'unknown')
+        connection_port = connection_kwargs.get('port', 6379)
+        connection_db = connection_kwargs.get('db', 0)
+
+    return RQQueueModel(
+        name=queue_name,
+        is_async=queue.is_async,
+        count=queue.count,
+        queued_jobs=queued_jobs,
+        started_jobs=started_jobs,
+        finished_jobs=finished_jobs,
+        failed_jobs=failed_jobs,
+        deferred_jobs=deferred_jobs,
+        scheduled_jobs=scheduled_jobs,
+        workers=worker_count,
+        oldest_job_timestamp=oldest_job_timestamp,
+        connection_host=connection_host,
+        connection_port=connection_port,
+        connection_db=connection_db,
+    )

django_cfg/apps/rq/tasks/__init__.py

@@ -0,0 +1,23 @@
+"""
+Demo and test tasks for RQ testing and simulation.
+
+These tasks are used for testing RQ functionality from the frontend.
+"""
+
+from .demo_tasks import (
+    demo_success_task,
+    demo_failure_task,
+    demo_slow_task,
+    demo_progress_task,
+    demo_crash_task,
+    demo_retry_task,
+)
+
+__all__ = [
+    'demo_success_task',
+    'demo_failure_task',
+    'demo_slow_task',
+    'demo_progress_task',
+    'demo_crash_task',
+    'demo_retry_task',
+]

django_cfg/apps/rq/tasks/demo_tasks.py

@@ -0,0 +1,284 @@
+"""
+Demo tasks for RQ testing and simulation.
+
+These tasks provide various scenarios for testing RQ functionality:
+- Successful completion
+- Failure scenarios
+- Long-running tasks
+- Progress tracking
+- Crash simulation
+- Retry logic
+"""
+
+import time
+import random
+from django_cfg.modules.django_logging import get_logger
+
+logger = get_logger("rq.demo_tasks")
+
+
+def demo_success_task(duration: int = 2, message: str = "Demo task completed"):
+    """
+    Simple task that always succeeds.
+
+    Args:
+        duration: How long to run (seconds)
+        message: Custom message to return
+
+    Returns:
+        dict: Success result with message and metadata
+    """
+    logger.info(f"Starting demo_success_task (duration={duration}s)")
+    time.sleep(duration)
+
+    result = {
+        "status": "success",
+        "message": message,
+        "duration": duration,
+        "timestamp": time.time(),
+    }
+
+    logger.info(f"Completed demo_success_task: {result}")
+    return result
+
+
+def demo_failure_task(error_message: str = "Simulated failure"):
+    """
+    Task that always fails with an exception.
+
+    Args:
+        error_message: Custom error message
+
+    Raises:
+        ValueError: Always raises this exception
+    """
+    logger.warning(f"Starting demo_failure_task (will fail with: {error_message})")
+    time.sleep(1)
+    raise ValueError(error_message)
+
+
+def demo_slow_task(duration: int = 30, step_interval: int = 5):
+    """
+    Long-running task for testing worker timeouts and monitoring.
+
+    Args:
+        duration: Total duration (seconds)
+        step_interval: How often to log progress (seconds)
+
+    Returns:
+        dict: Success result with timing info
+    """
+    logger.info(f"Starting demo_slow_task (duration={duration}s)")
+
+    steps = duration // step_interval
+    for i in range(steps):
+        time.sleep(step_interval)
+        progress = ((i + 1) / steps) * 100
+        logger.info(f"Progress: {progress:.1f}%")
+
+    result = {
+        "status": "success",
+        "message": "Slow task completed",
+        "duration": duration,
+        "steps": steps,
+    }
+
+    logger.info(f"Completed demo_slow_task: {result}")
+    return result
+
+
+def demo_progress_task(total_items: int = 100):
+    """
+    Task that updates progress in job meta.
+
+    Args:
+        total_items: Number of items to process
+
+    Returns:
+        dict: Success result with processed items count
+    """
+    from rq import get_current_job
+
+    logger.info(f"Starting demo_progress_task (items={total_items})")
+
+    job = get_current_job()
+
+    for i in range(total_items):
+        # Simulate work
+        time.sleep(0.1)
+
+        # Update progress in job meta
+        if job:
+            job.meta['progress'] = {
+                'current': i + 1,
+                'total': total_items,
+                'percentage': ((i + 1) / total_items) * 100,
+            }
+            job.save_meta()
+
+        if (i + 1) % 10 == 0:
+            logger.info(f"Processed {i + 1}/{total_items} items")
+
+    result = {
+        "status": "success",
+        "message": "Progress task completed",
+        "processed_items": total_items,
+    }
+
+    logger.info(f"Completed demo_progress_task: {result}")
+    return result
+
+
+def demo_crash_task():
+    """
+    Task that simulates a worker crash (raises SystemExit).
+
+    WARNING: This will actually crash the worker! Use with caution.
+
+    Raises:
+        SystemExit: Simulates worker crash
+    """
+    logger.warning("Starting demo_crash_task (will crash worker!)")
+    time.sleep(1)
+    raise SystemExit("Simulated worker crash")
+
+
+def demo_retry_task(max_attempts: int = 3, fail_until_attempt: int = 2):
+    """
+    Task that fails N times before succeeding.
+
+    Useful for testing retry logic.
+
+    Args:
+        max_attempts: Maximum retry attempts
+        fail_until_attempt: Succeed on this attempt
+
+    Returns:
+        dict: Success result if attempt succeeds
+
+    Raises:
+        ValueError: If attempt should fail
+    """
+    from rq import get_current_job
+
+    job = get_current_job()
+
+    # Track attempt count in job meta
+    if job:
+        attempts = job.meta.get('attempts', 0) + 1
+        job.meta['attempts'] = attempts
+        job.save_meta()
+    else:
+        attempts = 1
+
+    logger.info(f"demo_retry_task attempt {attempts}/{max_attempts}")
+
+    if attempts < fail_until_attempt:
+        error_msg = f"Retry attempt {attempts} - failing until attempt {fail_until_attempt}"
+        logger.warning(error_msg)
+        raise ValueError(error_msg)
+
+    result = {
+        "status": "success",
+        "message": f"Succeeded on attempt {attempts}",
+        "attempts": attempts,
+    }
+
+    logger.info(f"Completed demo_retry_task: {result}")
+    return result
+
+
+def demo_random_task(success_rate: float = 0.7):
+    """
+    Task that randomly succeeds or fails.
+
+    Args:
+        success_rate: Probability of success (0.0 to 1.0)
+
+    Returns:
+        dict: Success result if task succeeds
+
+    Raises:
+        ValueError: If task randomly fails
+    """
+    logger.info(f"Starting demo_random_task (success_rate={success_rate})")
+    time.sleep(2)
+
+    if random.random() < success_rate:
+        result = {
+            "status": "success",
+            "message": "Random task succeeded",
+            "success_rate": success_rate,
+        }
+        logger.info(f"demo_random_task succeeded: {result}")
+        return result
+    else:
+        error_msg = f"Random task failed (success_rate={success_rate})"
+        logger.warning(error_msg)
+        raise ValueError(error_msg)
+
+
+def demo_memory_intensive_task(mb_to_allocate: int = 100):
+    """
+    Task that allocates memory for testing worker memory limits.
+
+    Args:
+        mb_to_allocate: Megabytes of memory to allocate
+
+    Returns:
+        dict: Success result with memory info
+    """
+    logger.info(f"Starting demo_memory_intensive_task (allocating {mb_to_allocate}MB)")
+
+    # Allocate memory (1MB = 1024*1024 bytes, store as list of integers)
+    data = []
+    chunk_size = 1024 * 1024  # 1MB chunks
+
+    for i in range(mb_to_allocate):
+        chunk = [0] * (chunk_size // 8)  # 8 bytes per integer
+        data.append(chunk)
+
+        if (i + 1) % 10 == 0:
+            logger.info(f"Allocated {i + 1}MB")
+
+    time.sleep(2)
+
+    result = {
+        "status": "success",
+        "message": "Memory intensive task completed",
+        "mb_allocated": mb_to_allocate,
+    }
+
+    logger.info(f"Completed demo_memory_intensive_task: {result}")
+    return result
+
+
+def demo_cpu_intensive_task(iterations: int = 1000000):
+    """
+    CPU-intensive task for testing worker CPU limits.
+
+    Args:
+        iterations: Number of iterations to compute
+
+    Returns:
+        dict: Success result with computation info
+    """
+    logger.info(f"Starting demo_cpu_intensive_task (iterations={iterations})")
+
+    # Perform CPU-intensive calculation
+    result_value = 0
+    for i in range(iterations):
+        result_value += i ** 2
+
+        if (i + 1) % 100000 == 0:
+            logger.info(f"Processed {i + 1}/{iterations} iterations")
+
+    result = {
+        "status": "success",
+        "message": "CPU intensive task completed",
+        "iterations": iterations,
+        "result": result_value,
+    }
+
+    logger.info(f"Completed demo_cpu_intensive_task: {result}")
+    return result