interloper-scheduler 0.3.0__tar.gz

interloper_scheduler-0.3.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.3
+Name: interloper-scheduler
+Version: 0.3.0
+Summary: Interloper cron scheduler and queue worker
+Author: Guillaume Onfroy
+Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
+Requires-Dist: interloper-core
+Requires-Dist: interloper-db
+Requires-Dist: croniter>=5.0.0
+Requires-Dist: interloper-docker ; extra == 'docker'
+Requires-Dist: interloper-k8s ; extra == 'k8s'
+Requires-Python: >=3.10
+Provides-Extra: docker
+Provides-Extra: k8s
+Description-Content-Type: text/markdown
+

interloper_scheduler-0.3.0/pyproject.toml

@@ -0,0 +1,51 @@
+# ###############
+# PROJECT / UV
+# ###############
+[project]
+name = "interloper-scheduler"
+version = "0.3.0"
+description = "Interloper cron scheduler and queue worker"
+readme = "README.md"
+authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
+requires-python = ">=3.10"
+dependencies = [
+    "interloper-core",
+    "interloper-db",
+    "croniter>=5.0.0",
+]
+
+[project.optional-dependencies]
+docker = ["interloper-docker"]
+k8s = ["interloper-k8s"]
+
+[build-system]
+requires = ["uv_build>=0.11.5,<0.12"]
+build-backend = "uv_build"
+
+[tool.uv.sources]
+interloper-core = { workspace = true }
+interloper-db = { workspace = true }
+interloper-docker = { workspace = true }
+interloper-k8s = { workspace = true }
+
+# ###############
+# RUFF
+# ###############
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202"]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401", "F403"]
+"tests/**" = ["ANN", "F811"]
+
+# ###############
+# PYRIGHT
+# ###############
+[tool.pyright]
+include = ["src"]
+typeCheckingMode = "basic"
+reportMissingParameterType = true
+ignore = ["tests/**"]

interloper_scheduler-0.3.0/src/interloper_scheduler/__init__.py

@@ -0,0 +1,15 @@
+from interloper_scheduler.cron import CronController
+from interloper_scheduler.executor import RunExecutor
+from interloper_scheduler.launcher import InProcessLauncher, Launcher, build_launcher
+from interloper_scheduler.queue import QueueController
+from interloper_scheduler.reaper import Reaper
+
+__all__ = [
+    "CronController",
+    "InProcessLauncher",
+    "Launcher",
+    "QueueController",
+    "Reaper",
+    "RunExecutor",
+    "build_launcher",
+]

interloper_scheduler-0.3.0/src/interloper_scheduler/cron.py

@@ -0,0 +1,201 @@
+"""Cron controller: evaluates cron jobs and creates queued runs."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import os
+from datetime import datetime, timezone
+from threading import Event
+from typing import cast
+
+from croniter import croniter
+from interloper_db import Store, get_engine
+from interloper_db.models import Backfill, Job, Run
+from sqlalchemy import or_
+from sqlmodel import Session, col, select
+
+logger = logging.getLogger(__name__)
+
+
+class CronController:
+    """Evaluates cron jobs and creates queued runs.
+
+    Runs in a loop:
+    1. ``SELECT FOR UPDATE SKIP LOCKED`` (lock jobs)
+    2. ``UPDATE next_run_at`` (calculate next)
+    3. ``INSERT run`` with ``status='queued'`` (create run)
+    4. ``COMMIT`` (release locks)
+    """
+
+    def __init__(
+        self,
+        store: Store | None = None,
+        reconcile_interval: int | None = None,
+        max_execution_delay: int | None = None,
+        batch_size: int = 50,
+    ) -> None:
+        """Initialize the cron controller.
+
+        Args:
+            store: The Store for creating backfills. Creates a default if not provided.
+            reconcile_interval: Seconds between cron evaluation cycles.
+            max_execution_delay: Max seconds a scheduled job can be late.
+            batch_size: Number of jobs to process per cycle.
+        """
+        if store is None:
+            from interloper.catalog import Catalog
+
+            store = Store(catalog=Catalog.from_settings())
+        self._store = store
+        self._batch_size = batch_size
+        self._reconcile_interval = reconcile_interval or int(os.getenv("JOB_RECONCILE_INTERVAL", "10"))
+        self._max_execution_delay = max_execution_delay or int(
+            os.getenv("MAX_JOB_EXECUTION_DELAY", str(self._reconcile_interval))
+        )
+        if self._max_execution_delay < self._reconcile_interval:
+            from interloper.errors import ConfigError
+
+            raise ConfigError("MAX_JOB_EXECUTION_DELAY must be >= JOB_RECONCILE_INTERVAL")
+        self._stop_event = Event()
+
+    def start(self) -> None:
+        """Run the cron evaluation loop until stopped."""
+        logger.info("Starting cron controller...")
+
+        try:
+            while not self._stop_event.is_set():
+                logger.info("Evaluating cron jobs...")
+                try:
+                    self._process_jobs()
+                except Exception as e:
+                    logger.error("Failed to process jobs: %s", e)
+
+                if self._stop_event.wait(self._reconcile_interval):
+                    break
+        except KeyboardInterrupt:
+            logger.info("Shutting down cron controller...")
+
+    def stop(self) -> None:
+        """Signal the loop to stop."""
+        self._stop_event.set()
+
+    def _process_jobs(self) -> None:
+        """Process a batch of due jobs in a single transaction."""
+        session = Session(get_engine())
+
+        try:
+            now = datetime.now(timezone.utc)
+
+            statement = (
+                select(Job)
+                .where(Job.enabled)
+                .where(or_(col(Job.next_run_at) <= now, col(Job.next_run_at).is_(None)))
+                .order_by(col(Job.next_run_at).asc().nulls_last())
+                .limit(self._batch_size)
+                .with_for_update(skip_locked=True)
+            )
+
+            jobs = session.exec(statement).all()
+            if not jobs:
+                return
+
+            logger.info("Found %d job(s) ready to run", len(jobs))
+
+            for job in jobs:
+                next_run_at = self._calculate_next_run(job.cron, now)
+
+                # New job: schedule for the future, don't run yet
+                if job.next_run_at is None:
+                    job.next_run_at = next_run_at
+                    session.add(job)
+                    session.flush()
+                    logger.info("Scheduling new job '%s' for %s", job.name, next_run_at)
+                    continue
+
+                # Check if too old to execute
+                scheduled_time = job.next_run_at
+                if scheduled_time.tzinfo is None:
+                    scheduled_time = scheduled_time.replace(tzinfo=timezone.utc)
+
+                delay_seconds = (now - scheduled_time).total_seconds()
+                if delay_seconds > self._max_execution_delay:
+                    logger.warning(
+                        "Skipping job '%s' - too late (%ds > %ds)",
+                        job.name,
+                        int(delay_seconds),
+                        self._max_execution_delay,
+                    )
+                    job.next_run_at = next_run_at
+                    session.add(job)
+                    session.flush()
+                    continue
+
+                # Update next_run_at
+                job.next_run_at = next_run_at
+                session.add(job)
+                session.flush()
+
+                # Create runs
+                if job.partitioned and job.backfill_days:
+                    end_date = now.date() - dt.timedelta(days=1)
+                    start_date = end_date - dt.timedelta(days=job.backfill_days - 1)
+                    backfill = Backfill(
+                        org_id=job.org_id,
+                        job_id=job.id,
+                        start_date=start_date,
+                        end_date=end_date,
+                        status="running",
+                        started_at=now,
+                    )
+                    session.add(backfill)
+                    session.flush()
+
+                    count = 0
+                    current = start_date
+                    while current <= end_date:
+                        run = Run(
+                            job_id=job.id,
+                            org_id=job.org_id,
+                            backfill_id=backfill.id,
+                            status="queued",
+                            partition_date=current,
+                        )
+                        session.add(run)
+                        count += 1
+                        current += dt.timedelta(days=1)
+                    backfill.partitions = count
+                    session.add(backfill)
+                else:
+                    run = Run(
+                        job_id=job.id,
+                        org_id=job.org_id,
+                        status="queued",
+                    )
+                    session.add(run)
+
+            session.commit()
+            logger.info("Processed %d job(s)", len(jobs))
+
+        except Exception as e:
+            logger.exception("Error processing jobs: %s", e)
+            session.rollback()
+            raise
+        finally:
+            session.close()
+
+    def _calculate_next_run(self, cron_expr: str, base_time: datetime) -> datetime:
+        """Calculate the next run time from a cron expression.
+
+        Args:
+            cron_expr: Cron expression string.
+            base_time: The reference time.
+
+        Returns:
+            The next scheduled datetime (UTC).
+        """
+        itr = croniter(cron_expr, base_time)
+        next_run = cast(datetime, itr.get_next(datetime))
+        if next_run.tzinfo is None:
+            return next_run.replace(tzinfo=timezone.utc)
+        return next_run

interloper_scheduler-0.3.0/src/interloper_scheduler/executor.py

@@ -0,0 +1,195 @@
+"""Run executor: loads a run from DB, builds the DAG, and executes it."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+from typing import Any
+from uuid import UUID
+
+import interloper as il
+from interloper.runner import ExecutionStatus
+from interloper.runner.sync_runner import SyncRunner
+from interloper_db import Store, get_engine
+from interloper_db.models import AssetDependency, Job, Run, Source
+from sqlalchemy.orm import selectinload
+from sqlmodel import Session, col, select
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: cache source and asset hydrations
+
+
+class RunExecutor:
+    """Executes a run: loads from DB, builds the DAG, runs it, tracks events.
+
+    Uses the ``Store`` for hydration so all reconstruction goes through
+    the standard framework path.
+    """
+
+    def __init__(
+        self,
+        store: Store | None = None,
+        runner_type: type[SyncRunner] = il.MultiThreadRunner,
+        runner_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        if store is None:
+            from interloper.catalog import Catalog
+
+            store = Store(catalog=Catalog.from_settings())
+        self._store = store
+        self._runner_type = runner_type
+        self._runner_kwargs = runner_kwargs or {}
+
+    def execute(self, run_id: UUID) -> bool:
+        """Execute a run with full lifecycle tracking.
+
+        Returns:
+            ``True`` if the run completed successfully, ``False`` otherwise.
+        """
+        org_id: UUID | None = None
+        backfill_id: str | None = None
+
+        try:
+            logger.info("Starting run %s", run_id)
+
+            with Session(get_engine()) as session:
+                db_run = self._load_run(session, run_id)
+                if not db_run or not db_run.job:
+                    logger.info("Run %s not found, skipping", run_id)
+                    return False
+
+                org_id = db_run.org_id
+                backfill_id = str(db_run.backfill_id) if db_run.backfill_id else None
+
+                self._mark_running(session, db_run)
+
+                assets = self._hydrate_job_assets(db_run.job)
+                if not assets:
+                    logger.info("No sources or assets for run %s, marking success", run_id)
+                    self._store.complete_run(run_id, success=True)
+                    return True
+
+                self._resolve_upstream_deps(db_run.job, assets)
+
+                dag = il.DAG(*assets)
+                partition = il.TimePartition(db_run.partition_date) if db_run.partition_date else None
+
+                result = self._run_dag(dag, partition, org_id=org_id, run_id=run_id, backfill_id=backfill_id)
+
+                success = result.status == ExecutionStatus.COMPLETED
+                logger.info("Run %s completed: %s", run_id, result.status.name)
+                self._store.complete_run(run_id, success=success)
+                return success
+
+        except Exception as e:
+            logger.exception("Run %s failed: %s", run_id, e)
+            try:
+                metadata: dict[str, Any] = {
+                    "run_id": str(run_id),
+                    "backfill_id": backfill_id,
+                    "error": str(e),
+                }
+                if org_id is not None:
+                    event = il.Event(type=il.EventType.RUN_FAILED, metadata=metadata)
+                    self._store.save_event(event, org_id=org_id, run_id=run_id)
+                self._store.complete_run(run_id, success=False)
+            except Exception:
+                logger.exception("Failed to mark run %s as failed", run_id)
+            return False
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _load_run(session: Session, run_id: UUID) -> Run | None:
+        return session.get(
+            Run,
+            run_id,
+            options=[
+                selectinload(Run.job).selectinload(Job.sources).selectinload(Source.assets),  # type: ignore[arg-type]
+                selectinload(Run.job).selectinload(Job.assets),  # type: ignore[arg-type]
+            ],
+        )
+
+    @staticmethod
+    def _mark_running(session: Session, db_run: Run) -> None:
+        db_run.status = "running"
+        db_run.started_at = dt.datetime.now(dt.timezone.utc)
+        session.add(db_run)
+        session.commit()
+
+    def _hydrate_job_assets(self, db_job: Job) -> list[il.Asset]:
+        """Hydrate job sources/assets and return only DB-registered assets."""
+        assets: list[il.Asset] = []
+
+        # Source-owned: hydrate the full source, then cherry-pick registered assets.
+        for db_source in db_job.sources:
+            assert db_source.id is not None
+            source = self._store.load_source(db_source.id)
+            registered_keys = {db_asset.key for db_asset in db_source.assets}
+            for asset in source.assets:
+                if type(asset).key in registered_keys:
+                    assets.append(asset)
+
+        # Standalone assets
+        for db_asset in db_job.assets:
+            assert db_asset.id is not None
+            assets.append(self._store.load_asset(db_asset.id))
+
+        return assets
+
+    def _resolve_upstream_deps(self, db_job: Job, assets: list[il.Asset]) -> None:
+        """Add transitive upstream deps to *assets* as non-materializable."""
+        db_asset_ids: set[UUID] = set()
+        for db_source in db_job.sources:
+            for db_asset in db_source.assets:
+                assert db_asset.id is not None
+                db_asset_ids.add(db_asset.id)
+        for db_asset in db_job.assets:
+            assert db_asset.id is not None
+            db_asset_ids.add(db_asset.id)
+
+        frontier = list(db_asset_ids)
+        visited = set(db_asset_ids)
+        with Session(get_engine()) as session:
+            while frontier:
+                dependency_rows = session.exec(
+                    select(AssetDependency).where(col(AssetDependency.asset_id).in_(frontier))
+                ).all()
+                next_frontier: list[UUID] = []
+                for dependency in dependency_rows:
+                    if dependency.upstream_asset_id not in visited:
+                        visited.add(dependency.upstream_asset_id)
+                        next_frontier.append(dependency.upstream_asset_id)
+                        upstream_asset = self._store.load_asset(dependency.upstream_asset_id)
+                        upstream_asset.materializable = False
+                        assets.append(upstream_asset)
+                frontier = next_frontier
+
+    def _run_dag(
+        self,
+        dag: il.DAG,
+        partition: il.TimePartition | None,
+        *,
+        org_id: UUID,
+        run_id: UUID,
+        backfill_id: str | None,
+    ) -> il.RunResult:
+        def handle_event(event: il.Event) -> None:
+            self._store.save_event(event, org_id=org_id, run_id=run_id)  # type: ignore[arg-type]
+
+        with self._runner_type(
+            **self._runner_kwargs,
+            on_event=handle_event,
+        ) as runner:
+            return runner.run(
+                dag,
+                partition,
+                metadata={
+                    "run_id": str(run_id),
+                    "backfill_id": backfill_id,
+                },
+            )

interloper_scheduler-0.3.0/src/interloper_scheduler/launcher.py

@@ -0,0 +1,203 @@
+"""Launcher interface and in-process implementation."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import TYPE_CHECKING, Any
+from uuid import UUID
+
+from interloper_db import Store
+
+if TYPE_CHECKING:
+    from interloper.catalog.base import Catalog
+    from interloper.settings import LauncherSettings, PostgresSettings, RunnerSettings
+
+logger = logging.getLogger(__name__)
+
+
+class RunStatus(str, Enum):
+    """Authoritative status of a launched run, reported by the launcher."""
+
+    RUNNING = "running"
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    NOT_FOUND = "not_found"
+
+
+@dataclass
+class RunState:
+    """Authoritative state of a launched run, as reported by its launcher."""
+
+    status: RunStatus
+    error: str | None = None
+
+
+def build_launcher(
+    launcher: LauncherSettings,
+    *,
+    postgres: PostgresSettings,
+    runner: RunnerSettings,
+    catalog: Catalog,
+    store: Any | None = None,
+) -> Any:
+    """Build a launcher instance from settings.
+
+    The runner configuration is always forwarded so every launcher type
+    respects ``RunnerSettings`` uniformly.
+
+    Args:
+        launcher: Launcher settings (type + type-specific config).
+        postgres: Postgres settings forwarded to launchers that spawn
+            isolated processes (e.g. Docker).
+        runner: Runner settings forwarded to every launcher.
+        catalog: Catalog forwarded to launchers that spawn isolated
+            processes so they can reproduce an identical catalog.
+        store: Optional Store instance shared with in-process launchers.
+
+    Returns:
+        A scheduler ``Launcher`` instance.
+
+    Raises:
+        ValueError: If the launcher type is unknown.
+    """
+    match launcher.type:
+        case "in_process":
+            from interloper_scheduler import InProcessLauncher
+
+            return InProcessLauncher(
+                store=store,
+                runner_type=runner.type,
+                runner_config=runner.config,
+            )
+        case "docker":
+            from interloper_docker import DockerLauncher
+
+            postgres_kwargs = {
+                "postgres_host": postgres.host,
+                "postgres_port": postgres.port,
+                "postgres_user": postgres.user,
+                "postgres_password": postgres.password,
+                "postgres_database": postgres.database,
+            }
+            kwargs = {**postgres_kwargs, **launcher.config}
+            return DockerLauncher(
+                catalog=catalog,
+                runner_type=runner.type,
+                runner_config=runner.config,
+                **kwargs,
+            )
+        case "kubernetes":
+            try:
+                from interloper_k8s import KubernetesLauncher
+            except ImportError as exc:
+                raise ValueError(
+                    "Launcher 'kubernetes' requires the 'interloper-k8s' package to be installed."
+                ) from exc
+
+            postgres_kwargs = {
+                "postgres_host": postgres.host,
+                "postgres_port": postgres.port,
+                "postgres_user": postgres.user,
+                "postgres_password": postgres.password,
+                "postgres_database": postgres.database,
+            }
+            kwargs = {**postgres_kwargs, **launcher.config}
+            return KubernetesLauncher(
+                catalog=catalog,
+                runner_type=runner.type,
+                runner_config=runner.config,
+                **kwargs,
+            )
+        case _:
+            raise ValueError(f"Unknown launcher: {launcher.type!r}. Available: in_process, docker, kubernetes")
+
+
+class Launcher(ABC):
+    """Abstract base for run launchers.
+
+    A launcher decides *where* a run executes: in-process, Docker, Kubernetes, etc.
+    Every launcher carries a runner configuration that determines *how* the
+    DAG is executed once it reaches the execution environment.
+    """
+
+    def __init__(
+        self,
+        runner_type: str = "multi_thread",
+        runner_config: dict[str, Any] | None = None,
+    ) -> None:
+        """Initialize the launcher.
+
+        Args:
+            runner_type: Runner type name (``serial``, ``multi_thread``, ``multi_process``).
+            runner_config: Runner-specific kwargs forwarded to the runner constructor.
+        """
+        self._runner_type = runner_type
+        self._runner_config = runner_config or {}
+
+    @abstractmethod
+    def launch(self, run_id: UUID) -> None:
+        """Launch a run for execution.
+
+        Args:
+            run_id: The run UUID to execute.
+        """
+
+    def describe_run(self, run_id: UUID) -> RunState | None:
+        """Return the authoritative state of a launched run.
+
+        Args:
+            run_id: The run UUID to describe.
+
+        Returns:
+            The run's authoritative state, or ``None`` if the launcher
+            cannot introspect its runs.
+        """
+        return None
+
+
+class InProcessLauncher(Launcher):
+    """Launches runs in a detached thread using ``RunExecutor``.
+
+    Accepts an optional ``store`` so all runs share the same persistence
+    layer (encryption keys, etc.) rather than creating a fresh default.
+    """
+
+    def __init__(
+        self,
+        runner_type: str = "multi_thread",
+        runner_config: dict[str, Any] | None = None,
+        store: Store | None = None,
+    ) -> None:
+        """Initialize the launcher.
+
+        Args:
+            runner_type: Runner type name (``serial``, ``multi_thread``, ``multi_process``).
+            runner_config: Runner-specific kwargs forwarded to the runner constructor.
+            store: Optional Store instance to share with executors.
+        """
+        super().__init__(runner_type=runner_type, runner_config=runner_config)
+        self._store = store
+
+    def launch(self, run_id: UUID) -> None:
+        """Launch a run in a background thread.
+
+        Args:
+            run_id: The run UUID to execute.
+        """
+        from interloper.runner import build_runner
+
+        from interloper_scheduler.executor import RunExecutor
+
+        runner_cls, runner_kwargs = build_runner(self._runner_type, self._runner_config)
+        executor = RunExecutor(
+            store=self._store,
+            runner_type=runner_cls,
+            runner_kwargs=runner_kwargs,
+        )
+        thread = threading.Thread(target=executor.execute, args=(run_id,), daemon=True)
+        thread.start()
+        logger.info("Launched run %s in background thread", run_id)

interloper_scheduler-0.3.0/src/interloper_scheduler/queue.py

@@ -0,0 +1,88 @@
+"""Queue controller: polls for queued runs and dispatches them."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import time
+from threading import Event
+
+from interloper_db import get_engine
+from interloper_db.models import Run
+from sqlmodel import Session, col, select
+
+from interloper_scheduler.launcher import InProcessLauncher, Launcher
+
+logger = logging.getLogger(__name__)
+
+
+class QueueController:
+    """Polls the runs table for queued runs and dispatches them.
+
+    Uses ``SELECT FOR UPDATE SKIP LOCKED`` for safe concurrent polling.
+    """
+
+    def __init__(
+        self,
+        launcher: Launcher | None = None,
+        poll_interval: int = 5,
+    ) -> None:
+        """Initialize the queue controller.
+
+        Args:
+            launcher: The launcher to use for dispatching runs.
+            poll_interval: Seconds between poll cycles.
+        """
+        self._launcher = launcher or InProcessLauncher()
+        self._poll_interval = poll_interval
+        self._stop_event = Event()
+
+    def start(self) -> None:
+        """Run the polling loop until stopped."""
+        logger.info("Starting queue controller...")
+
+        while not self._stop_event.is_set():
+            logger.info("Polling for queued runs...")
+
+            try:
+                with Session(get_engine()) as session:
+                    statement = (
+                        select(Run)
+                        .where(Run.status == "queued")
+                        .order_by(col(Run.created_at).asc())
+                        .limit(1)
+                        .with_for_update(skip_locked=True)
+                    )
+                    run = session.exec(statement).first()
+
+                    if not run or not run.id:
+                        if self._stop_event.wait(self._poll_interval):
+                            break
+                        continue
+
+                    run_id = run.id
+                    run.status = "dispatched"
+                    session.add(run)
+                    session.commit()
+                    logger.info("Dispatched run %s", run_id)
+
+                try:
+                    logger.info("Launching run %s", run_id)
+                    self._launcher.launch(run_id)
+                except Exception as e:
+                    logger.exception("Failed to launch run %s: %s", run_id, e)
+                    with Session(get_engine()) as session:
+                        failed_run = session.get(Run, run_id)
+                        if failed_run:
+                            failed_run.status = "failed"
+                            failed_run.completed_at = dt.datetime.now(dt.timezone.utc)
+                            session.add(failed_run)
+                            session.commit()
+
+            except Exception as e:
+                logger.exception("Queue controller error: %s", e)
+                time.sleep(5)
+
+    def stop(self) -> None:
+        """Signal the loop to stop."""
+        self._stop_event.set()

interloper_scheduler-0.3.0/src/interloper_scheduler/reaper.py

@@ -0,0 +1,182 @@
+"""Reaper: detects terminated runs via the launcher and marks them failed.
+
+The reaper is a single background thread that periodically checks every
+``dispatched`` run's authoritative state via
+:meth:`~interloper_scheduler.launcher.Launcher.describe_run`:
+
+- ``RUNNING`` → leave alone
+- ``SUCCEEDED`` → weird (container said it succeeded but didn't update
+  the DB) — mark as failed with a descriptive error
+- ``FAILED`` → mark as failed immediately with the launcher's error
+- ``NOT_FOUND`` → container is gone without a trace — mark as failed
+
+A ``timeout`` fallback catches runs the launcher can't see (e.g. when
+the launcher itself doesn't implement ``describe_run``, or the
+infrastructure API is unreachable).  Runs older than ``timeout``
+seconds in ``dispatched`` status are reaped regardless.
+
+The pattern scales flat: one SQL query per poll cycle, plus one
+launcher API call per dispatched run (which K8s/Docker can serve
+from their local daemon cheaply).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+from threading import Event
+from typing import TYPE_CHECKING
+
+import interloper as il
+from interloper_db import Store, get_engine
+from interloper_db.models import Run
+from sqlmodel import Session, select
+
+from interloper_scheduler.launcher import RunStatus
+
+if TYPE_CHECKING:
+    from interloper_scheduler.launcher import Launcher
+
+logger = logging.getLogger(__name__)
+
+
+class Reaper:
+    """Periodically reconciles dispatched runs with the launcher's truth.
+
+    Designed to run in a background thread alongside the
+    :class:`~interloper_scheduler.queue.QueueController`::
+
+        reaper = Reaper(store=store, launcher=launcher)
+        thread = threading.Thread(target=reaper.start, daemon=True)
+        thread.start()
+    """
+
+    def __init__(
+        self,
+        store: Store,
+        launcher: Launcher | None = None,
+        timeout: int = 600,
+        poll_interval: int = 10,
+    ) -> None:
+        """Initialize the reaper.
+
+        Args:
+            store: Store used to persist the failure event and update
+                the run status.
+            launcher: Optional launcher consulted each poll cycle for
+                authoritative run state.  Launchers without
+                introspection (e.g. in-process) fall back to timeout.
+            timeout: Fallback: seconds after which a ``dispatched`` run
+                is reaped regardless of what the launcher says.
+            poll_interval: Seconds between reaper scans.
+        """
+        self._store = store
+        self._launcher = launcher
+        self._timeout = timeout
+        self._poll_interval = poll_interval
+        self._stop_event = Event()
+
+    def start(self) -> None:
+        """Run the reaper loop until stopped."""
+        logger.info(
+            "Starting reaper (poll=%ds, timeout=%ds)",
+            self._poll_interval,
+            self._timeout,
+        )
+
+        while not self._stop_event.is_set():
+            try:
+                reaped = self._reap()
+                if reaped:
+                    logger.info("Reaped %d dispatched run(s)", reaped)
+            except Exception:
+                logger.exception("Reaper error")
+
+            if self._stop_event.wait(self._poll_interval):
+                break
+
+    def stop(self) -> None:
+        """Signal the loop to stop."""
+        self._stop_event.set()
+
+    def _reap(self) -> int:
+        """Scan dispatched runs and reap any that have terminated.
+
+        Returns:
+            Number of runs reaped this cycle.
+        """
+        now = dt.datetime.now(dt.timezone.utc)
+        timeout_cutoff = now - dt.timedelta(seconds=self._timeout)
+
+        with Session(get_engine()) as session:
+            dispatched_runs = list(session.exec(select(Run).where(Run.status == "dispatched")).all())
+
+        reaped = 0
+        for run in dispatched_runs:
+            if self._reap_run(run, now, timeout_cutoff):
+                reaped += 1
+        return reaped
+
+    def _reap_run(self, run: Run, now: dt.datetime, timeout_cutoff: dt.datetime) -> bool:
+        """Decide whether to reap a single run and do so if needed.
+
+        Returns:
+            ``True`` if the run was reaped.
+        """
+        assert run.id is not None
+
+        # 1. Authoritative launcher state (preferred)
+        state = None
+        if self._launcher is not None:
+            try:
+                state = self._launcher.describe_run(run.id)
+            except Exception:
+                logger.exception("Failed to describe run %s", run.id)
+
+        if state is not None:
+            if state.status == RunStatus.RUNNING:
+                return False  # Trust the launcher — still alive
+
+            if state.status == RunStatus.SUCCEEDED:
+                error = "Run container reported SUCCEEDED but never updated the DB. Possible connectivity issue."
+                self._fail_run(run, error)
+                return True
+
+            if state.status == RunStatus.FAILED:
+                error = state.error or "Run failed (no error reported by launcher)"
+                self._fail_run(run, error)
+                return True
+
+            if state.status == RunStatus.NOT_FOUND:
+                pass
+
+        # 2. Timeout fallback — for launchers without introspection,
+        # NOT_FOUND runs, or anything else.
+        if run.created_at and run.created_at < timeout_cutoff:
+            self._fail_run(run, f"Run timed out after {self._timeout}s (still 'dispatched')")
+            return True
+
+        return False
+
+    def _fail_run(self, run: Run, error: str) -> None:
+        """Mark a run as failed and emit a ``RUN_FAILED`` event."""
+        assert run.id is not None
+        logger.warning("Reaping run %s: %s", run.id, error)
+
+        try:
+            event = il.Event(
+                type=il.EventType.RUN_FAILED,
+                metadata={
+                    "run_id": str(run.id),
+                    "backfill_id": str(run.backfill_id) if run.backfill_id else None,
+                    "error": error,
+                },
+            )
+            self._store.save_event(event, org_id=run.org_id, run_id=run.id)
+        except Exception:
+            logger.exception("Failed to save RUN_FAILED event for run %s", run.id)
+
+        try:
+            self._store.complete_run(run.id, success=False)
+        except Exception:
+            logger.exception("Failed to mark run %s as failed", run.id)