django-cfg 1.4.106__py3-none-any.whl → 1.4.108__py3-none-any.whl

django_cfg/models/tasks/backends.py

@@ -1,253 +1,122 @@
 """
 Backend-specific configurations.
 
-Contains Dramatiq and Worker configuration models.
-Size: ~200 lines (focused on backend settings)
+Contains ReArq configuration model.
+Size: ~100 lines (focused on backend settings)
 """
 
 import logging
-import os
-from typing import List, Literal, Optional
+from typing import Any, Dict
 
 from pydantic import BaseModel, Field, field_validator
 
 logger = logging.getLogger(__name__)
 
 
-class DramatiqConfig(BaseModel):
+class RearqConfig(BaseModel):
     """
-    Dramatiq-specific configuration with production-ready defaults.
+    ReArq-specific configuration with production-ready defaults.
 
-    This model provides comprehensive configuration for Dramatiq background
+    This model provides comprehensive configuration for ReArq async background
     task processing, including Redis settings, worker configuration,
-    middleware stack, and monitoring options.
+    and job retry policies.
 
     Example:
         ```python
-        from django_cfg.models.tasks import DramatiqConfig
+        from django_cfg.models.tasks import RearqConfig
 
-        dramatiq = DramatiqConfig(
-            redis_db=1,
-            processes=4,
-            threads=8,
+        rearq = RearqConfig(
+            redis_url="redis://localhost:6379/0",
+            db_url="sqlite://./rearq.db",
+            max_jobs=10,
+            job_timeout=300,
         )
         ```
     """
 
-    # === Redis Configuration ===
-    redis_db: int = Field(
-        default=1,
-        ge=0,
-        le=15,
-        description="Redis database number for tasks (separate from cache)"
+    # === Core Settings ===
+    redis_url: str = Field(
+        default="redis://localhost:6379/0",
+        description="Redis connection URL for task queue"
     )
-    redis_key_prefix: str = Field(
-        default="dramatiq",
-        description="Redis key prefix for task data"
+    db_url: str = Field(
+        default="sqlite://./rearq.db",
+        description="Database URL for job persistence (Tortoise ORM)"
     )
 
-    # === Task Configuration ===
-    max_retries: int = Field(
-        default=3,
-        ge=0,
-        le=10,
-        description="Default maximum retry count for failed tasks"
-    )
-    default_priority: int = Field(
-        default=5,
-        ge=0,
-        le=10,
-        description="Default task priority (0=highest, 10=lowest)"
-    )
-    max_age_seconds: int = Field(
-        default=3600,
-        ge=60,
-        description="Maximum age for tasks before they expire"
+    # === Worker Settings ===
+    max_jobs: int = Field(
+        default=10,
+        ge=1,
+        le=100,
+        description="Maximum concurrent jobs per worker"
     )
-    time_limit_seconds: int = Field(
-        default=600,
+    job_timeout: int = Field(
+        default=300,
         ge=30,
-        description="Maximum execution time per task"
+        le=3600,
+        description="Default job timeout in seconds"
     )
-
-    # === Worker Configuration ===
-    processes: int = Field(
-        default=4,
-        ge=1,
-        le=32,
-        description="Number of worker processes"
+    job_retry: int = Field(
+        default=3,
+        ge=0,
+        le=10,
+        description="Default number of retries for failed jobs"
     )
-    threads: int = Field(
-        default=8,
+    job_retry_after: int = Field(
+        default=60,
         ge=1,
-        le=64,
-        description="Number of threads per worker process"
-    )
-    queues: List[str] = Field(
-        default=["default", "high", "low"],
-        description="Available task queues"
-    )
-
-    # === Middleware Stack ===
-    middleware: List[str] = Field(
-        default=[
-            "dramatiq.middleware.AgeLimit",
-            "dramatiq.middleware.TimeLimit",
-            "dramatiq.middleware.Callbacks",
-            "dramatiq.middleware.Retries",
-            # NOTE: Results middleware requires backend configuration
-            # "dramatiq.results.middleware.Results",
-            "dramatiq.middleware.Prometheus",
-            "django_dramatiq.middleware.AdminMiddleware",
-            "django_dramatiq.middleware.DbConnectionsMiddleware",
-        ],
-        description="Middleware stack for task processing"
+        description="Delay in seconds before retrying failed job"
     )
 
-    # === Monitoring & Admin ===
-    prometheus_enabled: bool = Field(
-        default=True,
-        description="Enable Prometheus metrics collection"
-    )
-    admin_enabled: bool = Field(
-        default=True,
-        description="Enable Django admin interface integration"
-    )
-
-    # === Performance Tuning ===
-    prefetch_multiplier: int = Field(
-        default=2,
+    # === Cleanup Settings ===
+    keep_job_days: int | None = Field(
+        default=7,
         ge=1,
-        le=10,
-        description="Message prefetch multiplier for workers"
-    )
-    max_memory_mb: Optional[int] = Field(
-        default=512,
-        ge=128,
-        description="Maximum memory usage per worker (MB)"
+        description="Days to keep job history (None = forever)"
     )
 
-    @field_validator("processes")
+    @field_validator("redis_url")
     @classmethod
-    def validate_processes(cls, v: int) -> int:
-        """Ensure reasonable process count based on CPU cores."""
-        cpu_count = os.cpu_count() or 4
-        max_recommended = cpu_count * 2
-
-        if v > max_recommended:
-            logger.warning(
-                f"Process count ({v}) exceeds recommended maximum ({max_recommended}). "
-                f"Consider reducing to avoid resource contention."
-            )
-
+    def validate_redis_url(cls, v: str) -> str:
+        """Validate Redis URL format."""
+        if not v.startswith(("redis://", "rediss://")):
+            raise ValueError("Redis URL must start with redis:// or rediss://")
         return v
 
-    @field_validator("queues")
+    @field_validator("db_url")
     @classmethod
-    def validate_queues(cls, v: List[str]) -> List[str]:
-        """Ensure queue names are valid and include default."""
-        if not v:
-            raise ValueError("At least one queue must be specified")
-
-        # Ensure 'default' queue exists
-        if "default" not in v:
-            v.append("default")
-
-        # Validate queue names (alphanumeric + underscore/hyphen)
-        import re
-        pattern = re.compile(r'^[a-zA-Z0-9_-]+$')
-
-        for queue in v:
-            if not pattern.match(queue):
-                raise ValueError(f"Invalid queue name: {queue}. Use only alphanumeric, underscore, and hyphen.")
-
+    def validate_db_url(cls, v: str) -> str:
+        """Validate database URL format."""
+        valid_schemes = ("sqlite://", "postgres://", "postgresql://", "mysql://")
+        if not v.startswith(valid_schemes):
+            raise ValueError(f"Database URL must start with one of: {valid_schemes}")
         return v
 
-    @field_validator("middleware")
-    @classmethod
-    def validate_middleware(cls, v: List[str]) -> List[str]:
-        """Ensure essential middleware is included."""
-        essential_middleware = [
-            "dramatiq.middleware.Retries",
-            "django_dramatiq.middleware.DbConnectionsMiddleware",
-        ]
-
-        for middleware in essential_middleware:
-            if middleware not in v:
-                logger.warning(f"Adding essential middleware: {middleware}")
-                v.append(middleware)
-
-        return v
-
-
-class WorkerConfig(BaseModel):
-    """
-    Worker process and resource configuration.
-
-    Provides fine-grained control over worker behavior, resource limits,
-    and health monitoring settings.
-
-    Example:
-        ```python
-        from django_cfg.models.tasks import WorkerConfig
-
-        worker = WorkerConfig(
-            shutdown_timeout=30,
-            max_memory_mb=512,
-            health_check_enabled=True,
-        )
-        ```
-    """
-
-    # === Process Management ===
-    shutdown_timeout: int = Field(
-        default=30,
-        ge=5,
-        le=300,
-        description="Graceful shutdown timeout in seconds"
-    )
-    heartbeat_interval: int = Field(
-        default=5,
-        ge=1,
-        le=60,
-        description="Worker heartbeat interval in seconds"
-    )
-
-    # === Resource Limits ===
-    max_memory_mb: Optional[int] = Field(
-        default=512,
-        ge=128,
-        description="Maximum memory per worker process (MB)"
-    )
-    max_cpu_percent: Optional[float] = Field(
-        default=80.0,
-        ge=10.0,
-        le=100.0,
-        description="Maximum CPU usage per worker (%)"
-    )
-
-    # === Health Monitoring ===
-    health_check_enabled: bool = Field(
-        default=True,
-        description="Enable worker health monitoring"
-    )
-    restart_on_memory_limit: bool = Field(
-        default=True,
-        description="Restart worker if memory limit exceeded"
-    )
-
-    # === Logging ===
-    log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = Field(
-        default="INFO",
-        description="Worker log level"
-    )
-    log_format: str = Field(
-        default="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
-        description="Log message format"
-    )
+    def to_django_settings(self) -> Dict[str, Any]:
+        """
+        Generate Django settings dictionary.
+
+        Returns:
+            Dictionary with ReArq configuration for Django settings
+
+        Example:
+            >>> config = RearqConfig()
+            >>> settings = config.to_django_settings()
+            >>> "REARQ_REDIS_URL" in settings
+            True
+        """
+        return {
+            "REARQ_REDIS_URL": self.redis_url,
+            "REARQ_DB_URL": self.db_url,
+            "REARQ_MAX_JOBS": self.max_jobs,
+            "REARQ_JOB_TIMEOUT": self.job_timeout,
+            "REARQ_JOB_RETRY": self.job_retry,
+            "REARQ_JOB_RETRY_AFTER": self.job_retry_after,
+            "REARQ_KEEP_JOB_DAYS": self.keep_job_days,
+        }
 
 
 __all__ = [
-    "DramatiqConfig",
-    "WorkerConfig",
+    "RearqConfig",
 ]

django_cfg/models/tasks/config.py

@@ -19,8 +19,7 @@ logger = logging.getLogger(__name__)
 
 class TaskBackend(str, Enum):
     """Supported task backends."""
-    DRAMATIQ = "dramatiq"
-    # Future: CELERY = "celery"
+    REARQ = "rearq"
 
 
 class QueuePriority(str, Enum):
@@ -56,7 +55,7 @@ class TaskConfig(BaseModel, BaseCfgAutoModule):
         description="Enable background task processing"
     )
     backend: TaskBackend = Field(
-        default=TaskBackend.DRAMATIQ,
+        default=TaskBackend.REARQ,
         description="Task processing backend"
     )
 
@@ -67,23 +66,16 @@ class TaskConfig(BaseModel, BaseCfgAutoModule):
         self._config = None
 
     # === Backend-Specific Configuration ===
-    dramatiq: 'DramatiqConfig' = Field(
+    rearq: 'RearqConfig' = Field(
         default_factory=lambda: None,
-        description="Dramatiq-specific configuration"
-    )
-    worker: 'WorkerConfig' = Field(
-        default_factory=lambda: None,
-        description="Worker configuration"
+        description="ReArq-specific configuration"
     )
 
     def model_post_init(self, __context: Any) -> None:
         """Initialize backend configs with defaults after model creation."""
-        if self.dramatiq is None:
-            from .backends import DramatiqConfig
-            self.dramatiq = DramatiqConfig()
-        if self.worker is None:
-            from .backends import WorkerConfig
-            self.worker = WorkerConfig()
+        if self.rearq is None:
+            from .backends import RearqConfig
+            self.rearq = RearqConfig()
 
     # === Environment-Specific Overrides ===
     dev_processes: Optional[int] = Field(
@@ -120,124 +112,26 @@ class TaskConfig(BaseModel, BaseCfgAutoModule):
 
         return v
 
-    def get_effective_processes(self, debug: bool = False) -> int:
-        """
-        Get effective number of processes based on environment.
-
-        Args:
-            debug: Whether in debug mode
-
-        Returns:
-            Number of worker processes to use
-
-        Example:
-            >>> config = TaskConfig()
-            >>> config.get_effective_processes(debug=True)
-            2
-        """
-        if debug and self.dev_processes is not None:
-            return self.dev_processes
-        elif not debug and self.prod_processes is not None:
-            return self.prod_processes
-        else:
-            return self.dramatiq.processes
-
-    def get_effective_queues(self) -> List[str]:
-        """
-        Get effective queue configuration.
-
-        Returns:
-            List of queue names
-
-        Example:
-            >>> config = TaskConfig()
-            >>> config.get_effective_queues()
-            ['default', 'high', 'low']
-        """
-        return self.dramatiq.queues
-
-    def get_redis_config(self, redis_url: str) -> Dict[str, Any]:
+    def to_django_settings(self) -> Dict[str, Any]:
         """
-        Generate Redis configuration for Dramatiq.
-
-        Args:
-            redis_url: Redis connection URL
+        Generate Django settings for task system.
 
         Returns:
-            Dictionary with Redis connection parameters
+            Dictionary with task configuration for Django settings
 
         Example:
             >>> config = TaskConfig()
-            >>> config.get_redis_config("redis://localhost:6379/1")
-            {'host': 'localhost', 'port': 6379, 'db': 1, 'password': None}
-        """
-        from urllib.parse import urlparse
-
-        # Parse Redis URL
-        parsed = urlparse(redis_url)
-
-        # Build Redis config
-        config = {
-            "host": parsed.hostname or "localhost",
-            "port": parsed.port or 6379,
-            "db": self.dramatiq.redis_db,
-            "password": parsed.password,
-        }
-
-        # Add SSL if specified
-        if parsed.scheme == "rediss":
-            config["ssl"] = True
-
-        return config
-
-    def get_dramatiq_settings(self, redis_url: str) -> Dict[str, Any]:
+            >>> settings = config.to_django_settings()
+            >>> "REARQ_REDIS_URL" in settings
+            True
         """
-        Generate complete Dramatiq settings for Django.
+        if not self.enabled:
+            return {}
 
-        Args:
-            redis_url: Redis connection URL
-
-        Returns:
-            Dictionary with complete Dramatiq configuration
+        if self.backend == TaskBackend.REARQ:
+            return self.rearq.to_django_settings()
 
-        Example:
-            >>> config = TaskConfig()
-            >>> settings = config.get_dramatiq_settings("redis://localhost:6379/1")
-            >>> "DRAMATIQ_BROKER" in settings
-            True
-        """
-        from urllib.parse import urlparse
-
-        redis_config = self.get_redis_config(redis_url)
-        parsed = urlparse(redis_url)
-
-        # Build Redis URL with correct database
-        redis_url_with_db = redis_url
-        if parsed.path and parsed.path != "/":
-            # Replace existing database in URL
-            redis_url_with_db = redis_url.replace(parsed.path, f"/{self.dramatiq.redis_db}")
-        else:
-            # Add database to URL
-            redis_url_with_db = f"{redis_url.rstrip('/')}/{self.dramatiq.redis_db}"
-
-        return {
-            "DRAMATIQ_BROKER": {
-                "BROKER": "dramatiq.brokers.redis.RedisBroker",
-                "OPTIONS": {
-                    "url": redis_url_with_db,
-                    **redis_config
-                },
-            },
-            "DRAMATIQ_RESULT_BACKEND": {
-                "BACKEND": "dramatiq.results.backends.redis.RedisBackend",
-                "BACKEND_OPTIONS": {
-                    "url": redis_url_with_db,
-                    **redis_config
-                },
-            },
-            "DRAMATIQ_MIDDLEWARE": self.dramatiq.middleware,
-            "DRAMATIQ_QUEUES": self.dramatiq.queues,
-        }
+        return {}
 
     def get_smart_defaults(self):
         """
@@ -303,7 +197,7 @@ class TaskConfig(BaseModel, BaseCfgAutoModule):
 
 
 # Resolve forward references for Pydantic v2
-from .backends import DramatiqConfig, WorkerConfig
+from .backends import RearqConfig
 
 TaskConfig.model_rebuild()
 
@@ -311,6 +205,5 @@ __all__ = [
     "TaskConfig",
     "TaskBackend",
     "QueuePriority",
-    "DramatiqConfig",
-    "WorkerConfig",
+    "RearqConfig",
 ]

django_cfg/models/tasks/utils.py

@@ -84,41 +84,30 @@ def get_default_task_config(debug: bool = False) -> 'TaskConfig':
 
     Example:
         >>> config = get_default_task_config(debug=True)
-        >>> config.dramatiq.processes
-        2
+        >>> config.rearq.max_jobs
+        10
     """
-    from .backends import DramatiqConfig, WorkerConfig
+    from .backends import RearqConfig
     from .config import TaskConfig
 
-    smart_queues = get_smart_queues(debug)
-
     if debug:
         # Development defaults
         return TaskConfig(
-            dramatiq=DramatiqConfig(
-                processes=2,
-                threads=4,
-                prometheus_enabled=False,
-                queues=smart_queues,
-            ),
-            worker=WorkerConfig(
-                log_level="DEBUG",
-                health_check_enabled=False,
+            rearq=RearqConfig(
+                redis_url="redis://localhost:6379/0",
+                db_url="sqlite://./rearq.db",
+                max_jobs=5,
+                job_timeout=300,
             )
         )
     else:
         # Production defaults
         return TaskConfig(
-            dramatiq=DramatiqConfig(
-                processes=8,
-                threads=16,
-                prometheus_enabled=True,
-                queues=smart_queues,
-            ),
-            worker=WorkerConfig(
-                log_level="INFO",
-                health_check_enabled=True,
-                restart_on_memory_limit=True,
+            rearq=RearqConfig(
+                redis_url="redis://localhost:6379/0",
+                db_url="postgresql://localhost/rearq",
+                max_jobs=20,
+                job_timeout=600,
             )
         )
 
@@ -138,7 +127,7 @@ def validate_task_config(config: 'TaskConfig', redis_url: Optional[str] = None)
 
     Example:
         >>> config = get_default_task_config()
-        >>> validate_task_config(config, "redis://localhost:6379/1")
+        >>> validate_task_config(config, "redis://localhost:6379/0")
         True
     """
     if not config.enabled:
@@ -156,12 +145,11 @@ def validate_task_config(config: 'TaskConfig', redis_url: Optional[str] = None)
             logger.error(f"Invalid Redis URL: {e}")
             return False
 
-    # Check if Dramatiq is available
+    # Check if ReArq is available
     try:
-        import django_dramatiq
-        import dramatiq
+        import rearq
     except ImportError as e:
-        logger.error(f"Dramatiq dependencies not available: {e}")
+        logger.error(f"ReArq dependencies not available: {e}")
         return False
 
     return True