horsies 0.1.0a4__py3-none-any.whl

horsies/core/models/app.py

@@ -0,0 +1,268 @@
+# app/core/models/app.py
+from typing import List, Optional, Any
+from pydantic import BaseModel, model_validator, Field
+from horsies.core.models.queues import QueueMode, CustomQueueConfig
+from horsies.core.models.broker import PostgresConfig
+from horsies.core.models.recovery import RecoveryConfig
+from horsies.core.models.schedule import ScheduleConfig
+from horsies.core.errors import ConfigurationError, ErrorCode, ValidationReport, raise_collected
+from urllib.parse import urlparse, urlunparse
+import logging
+import os
+
+
+class AppConfig(BaseModel):
+    queue_mode: QueueMode = QueueMode.DEFAULT
+    custom_queues: Optional[List[CustomQueueConfig]] = None
+    broker: PostgresConfig
+    # Cluster-wide cap on concurrently RUNNING tasks across all queues. None = unlimited.
+    cluster_wide_cap: Optional[int] = None
+    # Prefetch buffer: 0 = hard cap mode (count RUNNING + CLAIMED), >0 = soft cap with lease
+    prefetch_buffer: int = 0
+    # Claim lease duration in ms. Required when prefetch_buffer > 0.
+    # Prefetched claims expire after this duration and can be reclaimed by other workers.
+    claim_lease_ms: Optional[int] = None
+    recovery: RecoveryConfig = Field(default_factory=RecoveryConfig)
+    schedule: Optional[ScheduleConfig] = Field(
+        default=None, description='Scheduler configuration'
+    )
+
+    @model_validator(mode='after')
+    def validate_queue_configuration(self):
+        """Validate queue configuration based on queue mode.
+
+        Collects all independent errors and raises them together.
+        """
+        report = ValidationReport('config')
+
+        if self.queue_mode == QueueMode.DEFAULT:
+            if self.custom_queues is not None:
+                report.add(ConfigurationError(
+                    message='custom_queues must be None in DEFAULT mode',
+                    code=ErrorCode.CONFIG_INVALID_QUEUE_MODE,
+                    notes=['queue_mode=DEFAULT but custom_queues was provided'],
+                    help_text='either remove custom_queues or set queue_mode=CUSTOM',
+                ))
+        elif self.queue_mode == QueueMode.CUSTOM:
+            if self.custom_queues is None or len(self.custom_queues) == 0:
+                report.add(ConfigurationError(
+                    message='custom_queues required in CUSTOM mode',
+                    code=ErrorCode.CONFIG_INVALID_QUEUE_MODE,
+                    notes=['queue_mode=CUSTOM but custom_queues is empty or None'],
+                    help_text='provide at least one CustomQueueConfig in custom_queues',
+                ))
+            else:
+                # Validate unique queue names (only if queues exist)
+                queue_names = [q.name for q in self.custom_queues]
+                if len(queue_names) != len(set(queue_names)):
+                    report.add(ConfigurationError(
+                        message='duplicate queue names in custom_queues',
+                        code=ErrorCode.CONFIG_INVALID_QUEUE_MODE,
+                        notes=[f'queue names: {queue_names}'],
+                        help_text='each queue name must be unique',
+                    ))
+
+        # Validate cluster_wide_cap if provided
+        if self.cluster_wide_cap is not None and self.cluster_wide_cap <= 0:
+            report.add(ConfigurationError(
+                message='cluster_wide_cap must be positive',
+                code=ErrorCode.CONFIG_INVALID_CLUSTER_CAP,
+                notes=[f'got cluster_wide_cap={self.cluster_wide_cap}'],
+                help_text='use a positive integer or None for unlimited',
+            ))
+
+        # Validate prefetch_buffer
+        if self.prefetch_buffer < 0:
+            report.add(ConfigurationError(
+                message='prefetch_buffer must be non-negative',
+                code=ErrorCode.CONFIG_INVALID_PREFETCH,
+                notes=[f'got prefetch_buffer={self.prefetch_buffer}'],
+                help_text='use 0 for hard cap mode or positive integer for soft cap',
+            ))
+
+        # Validate claim_lease_ms
+        if self.prefetch_buffer > 0 and self.claim_lease_ms is None:
+            report.add(ConfigurationError(
+                message='claim_lease_ms required when prefetch_buffer > 0',
+                code=ErrorCode.CONFIG_INVALID_PREFETCH,
+                notes=[
+                    f'prefetch_buffer={self.prefetch_buffer} but claim_lease_ms is None',
+                ],
+                help_text='set claim_lease_ms (e.g., 30000 for 30 seconds)',
+            ))
+        if self.claim_lease_ms is not None and self.claim_lease_ms <= 0:
+            report.add(ConfigurationError(
+                message='claim_lease_ms must be positive',
+                code=ErrorCode.CONFIG_INVALID_PREFETCH,
+                notes=[f'got claim_lease_ms={self.claim_lease_ms}'],
+                help_text='use a positive integer in milliseconds',
+            ))
+
+        # Forbid claim_lease_ms in hard cap mode
+        if self.prefetch_buffer == 0 and self.claim_lease_ms is not None:
+            report.add(ConfigurationError(
+                message='claim_lease_ms incompatible with hard cap mode',
+                code=ErrorCode.CONFIG_INVALID_PREFETCH,
+                notes=[
+                    'prefetch_buffer=0 (hard cap mode)',
+                    f'but claim_lease_ms={self.claim_lease_ms} was set',
+                ],
+                help_text='remove claim_lease_ms or set prefetch_buffer > 0',
+            ))
+
+        # Validate prefetch_buffer vs cluster_wide_cap conflict
+        if self.cluster_wide_cap is not None and self.prefetch_buffer > 0:
+            report.add(ConfigurationError(
+                message='cluster_wide_cap incompatible with prefetch mode',
+                code=ErrorCode.CONFIG_INVALID_CLUSTER_CAP,
+                notes=[
+                    f'cluster_wide_cap={self.cluster_wide_cap}',
+                    f'prefetch_buffer={self.prefetch_buffer}',
+                    'cluster_wide_cap requires hard cap mode (prefetch_buffer=0)',
+                ],
+                help_text='set prefetch_buffer=0 when using cluster_wide_cap',
+            ))
+
+        raise_collected(report)
+        return self
+
+    def log_config(self, logger: Optional[logging.Logger] = None) -> None:
+        """
+        Log the AppConfig in a human-readable format.
+        Masks sensitive data like database passwords.
+
+        Args:
+            logger: Logger instance to use. If None, uses root logger.
+        """
+        if os.getenv('HORSIES_CHILD_PROCESS') == '1':
+            return
+        if logger is None:
+            logger = logging.getLogger()
+
+        formatted = self._format_for_logging()
+        logger.info('AppConfig:\n%s', formatted)
+
+    def _format_for_logging(self) -> str:
+        """Internal helper to format the AppConfig for human-readable logging."""
+        lines: list[str] = []
+
+        # Queue mode and queues
+        lines.append(f'  queue_mode: {self.queue_mode.value.upper()}')
+        if self.queue_mode == QueueMode.CUSTOM and self.custom_queues:
+            lines.append('  custom_queues:')
+            for queue in self.custom_queues:
+                lines.append(
+                    f'    - {queue.name} (priority={queue.priority}, max_concurrency={queue.max_concurrency})'
+                )
+
+        # Cluster-wide cap and prefetch settings
+        if self.cluster_wide_cap is not None:
+            lines.append(f'  cluster_wide_cap: {self.cluster_wide_cap}')
+        lines.append(f'  prefetch_buffer: {self.prefetch_buffer}')
+        if self.claim_lease_ms is not None:
+            lines.append(f'  claim_lease_ms: {self.claim_lease_ms}ms')
+
+        # Broker config with masked password
+        masked_url = self._mask_database_url(self.broker.database_url)
+        lines.append(f'  broker:')
+        lines.append(f'    database_url: {masked_url}')
+        lines.append(f'    pool_size: {self.broker.pool_size}')
+        lines.append(f'    max_overflow: {self.broker.max_overflow}')
+
+        # Recovery config
+        lines.append('  recovery:')
+        lines.append(
+            f'    auto_requeue_stale_claimed: {self.recovery.auto_requeue_stale_claimed}'
+        )
+        lines.append(
+            f'    claimed_stale_threshold: {self.recovery.claimed_stale_threshold_ms}ms'
+        )
+        lines.append(
+            f'    auto_fail_stale_running: {self.recovery.auto_fail_stale_running}'
+        )
+        lines.append(
+            f'    running_stale_threshold: {self.recovery.running_stale_threshold_ms}ms'
+        )
+        lines.append(f'    check_interval: {self.recovery.check_interval_ms}ms')
+        lines.append(
+            f'    heartbeat_intervals: runner={self.recovery.runner_heartbeat_interval_ms}ms, claimer={self.recovery.claimer_heartbeat_interval_ms}ms'
+        )
+
+        # Schedule config
+        if self.schedule is not None:
+            lines.append('  schedule:')
+            lines.append(f'    enabled: {self.schedule.enabled}')
+            lines.append(f'    schedules: {len(self.schedule.schedules)} schedule(s)')
+            if self.schedule.schedules:
+                for sched in self.schedule.schedules:
+                    pattern_desc = self._format_schedule_pattern(sched.pattern)
+                    lines.append(
+                        f'      - {sched.name}: {sched.task_name} {pattern_desc}'
+                    )
+            lines.append(f'    check_interval: {self.schedule.check_interval_seconds}s')
+
+        return '\n'.join(lines)
+
+    @staticmethod
+    def _mask_database_url(url: str) -> str:
+        """Mask password in database URL for secure logging."""
+        try:
+            parsed = urlparse(url)
+            if parsed.password:
+                # Replace password with asterisks
+                netloc = parsed.netloc.replace(f':{parsed.password}@', ':***@')
+                masked = urlunparse(
+                    (
+                        parsed.scheme,
+                        netloc,
+                        parsed.path,
+                        parsed.params,
+                        parsed.query,
+                        parsed.fragment,
+                    )
+                )
+                return masked
+            return url
+        except Exception:
+            # If parsing fails, just return a generic masked version
+            return (
+                url.split('@')[0].rsplit(':', 1)[0] + ':***@' + url.split('@')[1]
+                if '@' in url
+                else url
+            )
+
+    @staticmethod
+    def _format_schedule_pattern(pattern: Any) -> str:
+        """Format schedule pattern for concise logging."""
+        from horsies.core.models.schedule import (
+            IntervalSchedule,
+            HourlySchedule,
+            DailySchedule,
+            WeeklySchedule,
+            MonthlySchedule,
+        )
+
+        if isinstance(pattern, IntervalSchedule):
+            parts: list[str] = []
+            if pattern.days:
+                parts.append(f'{pattern.days}d')
+            if pattern.hours:
+                parts.append(f'{pattern.hours}h')
+            if pattern.minutes:
+                parts.append(f'{pattern.minutes}m')
+            if pattern.seconds:
+                parts.append(f'{pattern.seconds}s')
+            return f"every {' '.join(parts)}"
+        elif isinstance(pattern, HourlySchedule):
+            return f'hourly at :{pattern.minute:02d}:{pattern.second:02d}'
+        elif isinstance(pattern, DailySchedule):
+            return f"daily at {pattern.time.strftime('%H:%M:%S')}"
+        elif isinstance(pattern, WeeklySchedule):
+            days = ', '.join(d.value for d in pattern.days)
+            return f"weekly on {days} at {pattern.time.strftime('%H:%M:%S')}"
+        elif isinstance(pattern, MonthlySchedule):
+            return (
+                f"monthly on day {pattern.day} at {pattern.time.strftime('%H:%M:%S')}"
+            )
+        else:
+            return str(pattern)

horsies/core/models/broker.py

@@ -0,0 +1,79 @@
+import warnings
+from pydantic import BaseModel, Field, field_validator
+from horsies.core.errors import ConfigurationError, ErrorCode
+
+
+class PostgresConfig(BaseModel):
+    database_url: str = Field(..., description='The URL of the PostgreSQL database')
+    pool_pre_ping: bool = Field(
+        default=True, description='Whether to pre-ping the database connection pool'
+    )
+    pool_size: int = Field(
+        default=30, description='The size of the database connection pool'
+    )
+    max_overflow: int = Field(
+        default=30, description='The maximum number of connections to allow in the pool'
+    )
+    pool_timeout: int = Field(
+        default=30, description='The timeout for acquiring a connection from the pool'
+    )
+    pool_recycle: int = Field(
+        default=1800, description='The number of seconds to recycle connections'
+    )
+    echo: bool = Field(default=False, description='Whether to echo the SQL statements')
+
+    @field_validator('database_url')
+    def validate_database_url(cls, v: str) -> str:
+        if not v.startswith('postgresql+psycopg'):
+            raise ConfigurationError(
+                message='invalid database URL scheme',
+                code=ErrorCode.BROKER_INVALID_URL,
+                notes=[
+                    f"got: {v.split('://')[0] if '://' in v else v[:20]}://...",
+                    'horsies only supports psycopg3 (async PostgreSQL driver)',
+                ],
+                help_text="use 'postgresql+psycopg://user:pass@host/db'",
+            )
+        return v
+
+
+# =============================================================================
+# DEPRECATED: These exceptions are no longer raised by the broker.
+# Use TaskResult with LibraryErrorCode.TASK_NOT_FOUND and LibraryErrorCode.WAIT_TIMEOUT instead.
+# =============================================================================
+
+
+class TaskNotFoundError(Exception):
+    """
+    DEPRECATED: No longer raised by broker.get_result().
+
+    The broker now returns TaskResult(err=TaskError(error_code=LibraryErrorCode.TASK_NOT_FOUND)).
+    Check result.is_err() and result.err.error_code instead of catching this exception.
+    """
+
+    def __init__(self, *args: object) -> None:
+        warnings.warn(
+            'TaskNotFoundError is deprecated. '
+            'Use TaskResult with LibraryErrorCode.TASK_NOT_FOUND instead.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(*args)
+
+
+class TaskTimeoutError(Exception):
+    """
+    DEPRECATED: No longer raised by broker.get_result().
+
+    The broker now returns TaskResult(err=TaskError(error_code=LibraryErrorCode.WAIT_TIMEOUT)).
+    Check result.is_err() and result.err.error_code instead of catching this exception.
+    """
+
+    def __init__(self, *args: object) -> None:
+        warnings.warn(
+            'TaskTimeoutError is deprecated. '
+            'Use TaskResult with LibraryErrorCode.WAIT_TIMEOUT instead.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(*args)

horsies/core/models/queues.py

@@ -0,0 +1,23 @@
+# app/core/models/queues.py
+from enum import Enum
+from pydantic import BaseModel, Field
+
+
+class QueueMode(Enum):
+    CUSTOM = 'custom'
+    DEFAULT = 'default'
+
+
+class CustomQueueConfig(BaseModel):
+    """
+    name: name of the queue. Usage: `@task(queue="name")`
+    priority: 1 means first to be executed, 100 means last to be executed.
+    max_concurrency: max number of tasks that can be executed at the same time for this queue.
+        App level concurrency is still respected.
+        If you have set app level concurrency to 5 but queue level to 10,
+        it will still be limited to 5.
+    """
+
+    name: str
+    priority: int = Field(default=1, ge=1, le=100)
+    max_concurrency: int = 5

horsies/core/models/recovery.py

@@ -0,0 +1,101 @@
+# horsies/core/models/recovery.py
+from __future__ import annotations
+from typing import Annotated, Self
+from pydantic import BaseModel, Field, model_validator
+from horsies.core.errors import ConfigurationError, ErrorCode, ValidationReport, raise_collected
+
+
+class RecoveryConfig(BaseModel):
+    """
+    Configuration for automatic stale task handling and crash recovery.
+
+    Stale task detection and recovery:
+    - CLAIMED tasks that never start executing: safe to auto-requeue
+    - RUNNING tasks that go stale: mark as FAILED (may not be idempotent)
+
+    All time values are in milliseconds for consistency with timeout_ms.
+
+    Fields:
+    - auto_requeue_stale_claimed: If True, automatically requeue tasks stuck in CLAIMED
+    - claimed_stale_threshold_ms: Milliseconds without heartbeat before CLAIMED task is stale
+    - auto_fail_stale_running: If True, automatically mark stale RUNNING tasks as FAILED
+    - running_stale_threshold_ms: Milliseconds without heartbeat before RUNNING task is stale
+    - check_interval_ms: How often the reaper checks for stale tasks
+    - runner_heartbeat_interval_ms: How often RUNNING tasks send heartbeats from inside the task process
+    - claimer_heartbeat_interval_ms: How often CLAIMED tasks send heartbeats
+    """
+
+    auto_requeue_stale_claimed: bool = Field(
+        default=True,
+        description='Automatically requeue tasks stuck in CLAIMED (safe - user code never ran)',
+    )
+    claimed_stale_threshold_ms: Annotated[int, Field(ge=1_000, le=3_600_000)] = Field(
+        default=120_000,  # 2 minutes
+        description='Milliseconds without claimer heartbeat before CLAIMED task is considered stale (1s-1hr)',
+    )
+
+    auto_fail_stale_running: bool = Field(
+        default=True,
+        description='Automatically mark stale RUNNING tasks as FAILED (not safe to requeue)',
+    )
+    running_stale_threshold_ms: Annotated[int, Field(ge=1_000, le=7_200_000)] = Field(
+        default=300_000,  # 5 minutes
+        description='Milliseconds without runner heartbeat before RUNNING task is considered stale (1s-2hr)',
+    )
+
+    check_interval_ms: Annotated[int, Field(ge=1_000, le=600_000)] = Field(
+        default=30_000,  # 30 seconds
+        description='How often the reaper checks for stale tasks in milliseconds (1s-10min)',
+    )
+
+    runner_heartbeat_interval_ms: Annotated[int, Field(ge=1_000, le=120_000)] = Field(
+        default=30_000,  # 30 seconds
+        description=(
+            'How often RUNNING tasks send heartbeats from inside the task process in milliseconds (5s-2min); '
+            'increase stale thresholds for CPU/GIL-heavy tasks to avoid false positives'
+        ),
+    )
+
+    claimer_heartbeat_interval_ms: Annotated[int, Field(ge=1_000, le=120_000)] = Field(
+        default=30_000,  # 30 seconds
+        description='How often worker sends heartbeats for CLAIMED tasks in milliseconds (5s-2min)',
+    )
+
+    @model_validator(mode='after')
+    def validate_heartbeat_thresholds(self) -> Self:
+        """Ensure stale thresholds are at least 2x heartbeat intervals for reliability.
+
+        Collects both errors (if present) and raises them together.
+        """
+        report = ValidationReport('recovery')
+        min_running = self.runner_heartbeat_interval_ms * 2
+        min_claimed = self.claimer_heartbeat_interval_ms * 2
+
+        # Validate runner heartbeat vs running stale threshold
+        if self.running_stale_threshold_ms < min_running:
+            report.add(ConfigurationError(
+                message='running_stale_threshold_ms too low',
+                code=ErrorCode.CONFIG_INVALID_RECOVERY,
+                notes=[
+                    f'running_stale_threshold_ms={self.running_stale_threshold_ms}ms ({self.running_stale_threshold_ms/1000:.1f}s)',
+                    f'runner_heartbeat_interval_ms={self.runner_heartbeat_interval_ms}ms ({self.runner_heartbeat_interval_ms/1000:.1f}s)',
+                    'threshold must be at least 2x heartbeat interval',
+                ],
+                help_text=f'set running_stale_threshold_ms >= {min_running}ms ({min_running/1000:.1f}s)',
+            ))
+
+        # Validate claimer heartbeat vs claimed stale threshold
+        if self.claimed_stale_threshold_ms < min_claimed:
+            report.add(ConfigurationError(
+                message='claimed_stale_threshold_ms too low',
+                code=ErrorCode.CONFIG_INVALID_RECOVERY,
+                notes=[
+                    f'claimed_stale_threshold_ms={self.claimed_stale_threshold_ms}ms ({self.claimed_stale_threshold_ms/1000:.1f}s)',
+                    f'claimer_heartbeat_interval_ms={self.claimer_heartbeat_interval_ms}ms ({self.claimer_heartbeat_interval_ms/1000:.1f}s)',
+                    'threshold must be at least 2x heartbeat interval',
+                ],
+                help_text=f'set claimed_stale_threshold_ms >= {min_claimed}ms ({min_claimed/1000:.1f}s)',
+            ))
+
+        raise_collected(report)
+        return self