gdmcode 0.1.0__py3-none-any.whl

src/agent/daemon.py

@@ -0,0 +1,681 @@
+"""Background job daemon — processes jobs from a SQLite queue.
+
+Jobs are submitted via ``daemon.submit(job_type, payload)`` which writes
+to the ``daemon_jobs`` table.  The daemon thread claims one job at a time
+(atomically via DB transaction) and writes the result back to the same row.
+
+Falls back to legacy file-based inbox/outbox when no DB is available.
+
+Job types:
+  index    — run CodeIndex.build() on the project
+  compress — compress old transcript turns via SessionCompressor
+  scan     — run ProjectScanner.scan() and update conventions
+"""
+from __future__ import annotations
+
+import concurrent.futures
+import http.server
+import json
+import logging
+import os
+import signal
+import socketserver
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from src.exceptions import DaemonUnavailableError
+
+__all__ = ["BackgroundDaemon", "DaemonHealth", "DaemonJob", "InboxProcessor", "_resume_inflight_batches"]
+
+log = logging.getLogger(__name__)
+
+_POLL_INTERVAL_SECS: float = 5.0
+_JOB_TIMEOUT_SECS: float = 120.0
+_OUTBOX_TTL_SECS: float = 3600.0
+_MAX_RESTARTS: int = 3
+_RESTART_WINDOW_SECS: float = 60.0
+
+
+# ---------------------------------------------------------------------------
+# Public types
+# ---------------------------------------------------------------------------
+
+@dataclass
+class DaemonHealth:
+    """Snapshot of daemon thread health and queue depth."""
+    alive: bool
+    last_heartbeat: datetime | None
+    heartbeat_age_secs: float | None   # None if never heartbeated
+    pending_jobs: int
+    failed_jobs: int
+
+
+class DaemonJob:
+    """A single daemon job (from DB row or legacy inbox file)."""
+
+    __slots__ = ("job_id", "type", "payload", "priority")
+
+    def __init__(
+        self,
+        job_id: str,
+        type: str,
+        payload: dict[str, Any],
+        priority: int = 0,
+    ) -> None:
+        self.job_id = job_id
+        self.type = type
+        self.payload = payload
+        self.priority = priority
+
+    @classmethod
+    def from_db_row(cls, row: Any) -> "DaemonJob":
+        payload = json.loads(row["payload_json"]) if row["payload_json"] else {}
+        return cls(job_id=row["job_id"], type=row["job_type"], payload=payload)
+
+    @classmethod
+    def from_file(cls, path: Path) -> "DaemonJob":
+        """Legacy: load from inbox JSON file."""
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return cls(
+            job_id=data["job_id"],
+            type=data["type"],
+            payload=data.get("payload", {}),
+            priority=data.get("priority", 0),
+        )
+
+
+# ---------------------------------------------------------------------------
+# BackgroundDaemon
+# ---------------------------------------------------------------------------
+
+class BackgroundDaemon:
+    """Threaded background daemon for async heavy tasks.
+
+    Uses a SQLite ``daemon_jobs`` table as the job queue when a DB is
+    provided (preferred). Falls back to file-based inbox/outbox polling
+    when ``db`` is None (legacy mode).
+
+    Usage::
+
+        daemon = BackgroundDaemon(db=db)
+        daemon.start()
+        job_id = daemon.submit("index", {"project_id": "..."})
+        result = daemon.get_result(job_id)
+        daemon.stop()
+    """
+
+    _HEARTBEAT_KEY = "daemon:heartbeat"
+    _HEARTBEAT_INTERVAL_SECS: float = 30.0
+
+    def __init__(
+        self,
+        db_or_path: Any = None,
+        db: Any = None,
+        cfg: Any = None,
+        *,
+        memory_dir: Path | None = None,
+        max_restarts: int = 5,
+        pid_file: Path | None = None,
+    ) -> None:
+        # Support both:
+        #   new: BackgroundDaemon(db=db)
+        #   old: BackgroundDaemon(memory_dir, db, cfg)
+        if isinstance(db_or_path, Path):
+            _memory_dir: Path = db_or_path
+            _db = db
+            _cfg = cfg
+        else:
+            _db = db if db_or_path is None else db_or_path
+            _cfg = cfg
+            if memory_dir is not None:
+                _memory_dir = memory_dir
+            elif _cfg is not None and hasattr(_cfg, "context_memory_dir"):
+                _memory_dir = Path(_cfg.context_memory_dir)
+            elif hasattr(_db, "_db_path"):
+                _memory_dir = Path(_db._db_path).parent
+            else:
+                _memory_dir = Path(".context-memory")
+
+        self._memory_dir = _memory_dir
+        self._db = _db
+        self._cfg = _cfg
+        self._thread: threading.Thread | None = None
+        self._stop_event = threading.Event()
+        self._restart_timestamps: list[float] = []
+        self._restart_lock = threading.Lock()
+        self._result_callbacks: dict[str, Callable[[dict], None]] = {}
+
+        # Stability additions — graceful shutdown, crash recovery, health, PID
+        self._shutdown_event = threading.Event()
+        self._tasks_completed: int = 0
+        self._start_time: float = time.monotonic()
+        self._max_restarts: int = max_restarts
+        self._restart_count: int = 0
+        self._health_server: socketserver.TCPServer | None = None
+        self._pid_path: Path = (
+            pid_file if pid_file is not None
+            else _memory_dir / ".gdm" / "daemon.pid"
+        )
+        self._exit_fn: Callable[[int], None] = os._exit
+
+        # Legacy file inbox/outbox — only created when no DB is available
+        if self._db is None:
+            self._inbox = _memory_dir / "inbox"
+            self._outbox = _memory_dir / "outbox"
+            self._inbox.mkdir(parents=True, exist_ok=True)
+            self._outbox.mkdir(parents=True, exist_ok=True)
+        else:
+            self._inbox = _memory_dir / "inbox"   # kept for attribute access compat
+            self._outbox = _memory_dir / "outbox"
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def _check_restart_cap(self) -> bool:
+        """Return True if a restart is allowed. Must be called under _restart_lock."""
+        now = time.monotonic()
+        self._restart_timestamps = [
+            t for t in self._restart_timestamps if now - t < _RESTART_WINDOW_SECS
+        ]
+        if len(self._restart_timestamps) >= _MAX_RESTARTS:
+            log.error(
+                "Daemon restart cap hit (%d in %ds) — entering degraded mode",
+                _MAX_RESTARTS, _RESTART_WINDOW_SECS,
+            )
+            return False
+        self._restart_timestamps.append(now)
+        return True
+
+    def start(self, *, write_pid: bool = True, health_port: int = 0) -> None:
+        """Start the daemon thread."""
+        if self._thread and self._thread.is_alive():
+            return
+        self._stop_event.clear()
+        self._start_time = time.monotonic()
+        if write_pid:
+            self._write_pid()
+        if health_port > 0:
+            self.start_health_server(health_port)
+        if self._db is not None:
+            try:
+                _resume_inflight_batches(self._db)
+            except Exception as exc:  # noqa: BLE001
+                log.warning("Could not resume in-flight batches on startup: %s", exc)
+        self._thread = threading.Thread(
+            target=self._run_with_restart,
+            name="gdm-daemon",
+            daemon=True,
+        )
+        self._thread.start()
+        log.debug("Background daemon started")
+
+    def stop(self, timeout: float = 3.0) -> None:
+        """Signal the daemon to stop and wait for it."""
+        self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=timeout)
+        if self._health_server is not None:
+            try:
+                self._health_server.shutdown()
+            except Exception:  # noqa: BLE001
+                pass
+            self._health_server = None
+        self._remove_pid()
+        log.debug("Background daemon stopped")
+
+    @property
+    def is_running(self) -> bool:
+        """True if the daemon thread is alive."""
+        return bool(self._thread and self._thread.is_alive())
+
+    def running(self) -> bool:
+        """Method alias for is_running (for callers that prefer method syntax)."""
+        return self.is_running
+
+    # ------------------------------------------------------------------
+    # Stability: signal handling, crash recovery, health endpoint, PID
+    # ------------------------------------------------------------------
+
+    def register_signal_handlers(self) -> None:
+        """Register SIGTERM/SIGINT handlers that set _shutdown_event and stop the loop."""
+        def _handle(signum: int, frame: Any) -> None:
+            log.info("Received signal %d — requesting graceful shutdown", signum)
+            self._shutdown_event.set()
+            self._stop_event.set()
+
+        signal.signal(signal.SIGTERM, _handle)
+        signal.signal(signal.SIGINT, _handle)
+
+    def start_health_server(self, port: int = 7682) -> None:
+        """Start a lightweight HTTP health endpoint in a daemon thread."""
+        daemon_ref = self
+
+        class _HealthHandler(http.server.BaseHTTPRequestHandler):
+            def do_GET(self) -> None:
+                if self.path == "/health":
+                    uptime = time.monotonic() - daemon_ref._start_time
+                    body = json.dumps({
+                        "status": "ok",
+                        "uptime_s": round(uptime, 2),
+                        "tasks_completed": daemon_ref._tasks_completed,
+                    }).encode()
+                    self.send_response(200)
+                    self.send_header("Content-Type", "application/json")
+                    self.send_header("Content-Length", str(len(body)))
+                    self.end_headers()
+                    self.wfile.write(body)
+                else:
+                    self.send_response(404)
+                    self.end_headers()
+
+            def log_message(self, fmt: str, *args: Any) -> None:  # silence HTTP access log
+                pass
+
+        class _Server(socketserver.TCPServer):
+            allow_reuse_address = True
+
+        try:
+            server = _Server(("127.0.0.1", port), _HealthHandler)
+            t = threading.Thread(
+                target=server.serve_forever,
+                name="gdm-health",
+                daemon=True,
+            )
+            t.start()
+            self._health_server = server
+            log.debug("Health endpoint listening on 127.0.0.1:%d", port)
+        except OSError as exc:
+            log.warning("Could not start health server on port %d: %s", port, exc)
+
+    def _run_with_restart(self) -> None:
+        """Run _run_loop with exponential-backoff auto-restart on crash."""
+        backoff = 1.0
+        while not self._stop_event.is_set():
+            try:
+                self._run_loop()
+                return  # clean exit via _stop_event
+            except KeyboardInterrupt:
+                return
+            except Exception as exc:  # noqa: BLE001
+                if self._stop_event.is_set():
+                    return
+                self._restart_count += 1
+                log.error(
+                    "Daemon loop crashed (restart %d/%d): %s",
+                    self._restart_count, self._max_restarts, exc,
+                    exc_info=True,
+                )
+                if self._restart_count > self._max_restarts:
+                    log.error(
+                        "Daemon max restarts (%d) exceeded — exiting with code 1",
+                        self._max_restarts,
+                    )
+                    self._exit_fn(1)
+                    return  # reached only when _exit_fn is mocked in tests
+                time.sleep(backoff)
+                backoff = min(backoff * 2, 30.0)
+
+    def _write_pid(self) -> None:
+        """Write current process PID to _pid_path."""
+        try:
+            self._pid_path.parent.mkdir(parents=True, exist_ok=True)
+            self._pid_path.write_text(str(os.getpid()), encoding="utf-8")
+            log.debug("PID %d written to %s", os.getpid(), self._pid_path)
+        except Exception as exc:  # noqa: BLE001
+            log.warning("Could not write PID file %s: %s", self._pid_path, exc)
+
+    def _remove_pid(self) -> None:
+        """Remove the PID file if it exists."""
+        try:
+            if self._pid_path.exists():
+                self._pid_path.unlink()
+                log.debug("PID file removed: %s", self._pid_path)
+        except Exception as exc:  # noqa: BLE001
+            log.warning("Could not remove PID file %s: %s", self._pid_path, exc)
+
+    # ------------------------------------------------------------------
+    # Health check
+    # ------------------------------------------------------------------
+
+    def health_check(self) -> DaemonHealth:
+        """Return a snapshot of thread health and queue depth."""
+        alive = self.is_running
+        last_hb = self._read_heartbeat()
+        age: float | None = None
+        if last_hb is not None:
+            age = (datetime.now(timezone.utc) - last_hb).total_seconds()
+        pending = self.pending_count()
+        failed = self._failed_count()
+        return DaemonHealth(
+            alive=alive,
+            last_heartbeat=last_hb,
+            heartbeat_age_secs=age,
+            pending_jobs=pending,
+            failed_jobs=failed,
+        )
+
+    def _write_heartbeat(self) -> None:
+        if self._db is None:
+            return
+        try:
+            self._db.execute(
+                "INSERT OR REPLACE INTO spinner_state (key, value) VALUES (?, ?)",
+                (self._HEARTBEAT_KEY, datetime.now(timezone.utc).isoformat()),
+            )
+        except Exception as exc:  # noqa: BLE001
+            log.debug("Heartbeat write failed: %s", exc)
+
+    def _read_heartbeat(self) -> datetime | None:
+        if self._db is None:
+            return None
+        try:
+            row = self._db.execute_one(
+                "SELECT value FROM spinner_state WHERE key=?", (self._HEARTBEAT_KEY,)
+            )
+            return datetime.fromisoformat(row["value"]) if row else None
+        except Exception as exc:  # noqa: BLE001
+            log.debug("Heartbeat read failed: %s", exc)
+            return None
+
+    # ------------------------------------------------------------------
+    # Job submission
+    # ------------------------------------------------------------------
+
+    def submit(self, job_type: str, payload: dict[str, Any] | None = None, *, priority: int = 0) -> str:
+        """Submit a job. Uses DB queue when available, else legacy file inbox.
+
+        Checks restart cap before enqueuing (fails fast if degraded) and starts
+        the daemon thread after enqueuing so jobs are visible on the first poll.
+        Returns job_id.
+        """
+        # Pre-check restart cap BEFORE enqueue — reject early if degraded.
+        # We resolve the start decision here but defer the actual start()
+        # until after enqueue so the daemon thread sees the job on its first poll.
+        _should_start = False
+        if not self.is_running:
+            with self._restart_lock:
+                if not self.is_running:
+                    if self._check_restart_cap():
+                        _should_start = True
+                    else:
+                        raise DaemonUnavailableError(
+                            "Daemon crashed too frequently. Check logs."
+                        )
+
+        if self._db is not None:
+            job_id = self._db.daemon_job_submit(job_type, payload or {})
+            log.debug("Daemon job submitted to DB: %s (type=%s)", job_id, job_type)
+        else:
+            import uuid
+            job_id = str(uuid.uuid4())[:8]
+            job = {
+                "job_id": job_id,
+                "type": job_type,
+                "payload": payload or {},
+                "priority": priority,
+            }
+            path = self._inbox / f"{job_id}.json"
+            path.write_text(json.dumps(job), encoding="utf-8")
+            log.debug("Daemon job submitted to file inbox: %s (type=%s)", job_id, job_type)
+
+        if _should_start:
+            log.warning("Daemon thread was not running — starting")
+            self.start()
+
+        return job_id
+
+    def get_result(self, job_id: str) -> dict[str, Any] | None:
+        """Return result for a completed job, or None if not yet done."""
+        if self._db is not None:
+            return self._db.daemon_job_result(job_id)
+        # Legacy file fallback
+        path = self._outbox / f"{job_id}.json"
+        if not path.exists():
+            return None
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except Exception:  # noqa: BLE001
+            return None
+
+    def on_result(self, job_id: str, callback: Callable[[dict], None]) -> None:
+        """Register a callback to be fired (in a daemon thread) when job_id completes."""
+        self._result_callbacks[job_id] = callback
+
+    def _fire_callback(self, job_id: str, result: dict[str, Any]) -> None:
+        cb = self._result_callbacks.pop(job_id, None)
+        if cb is not None:
+            threading.Thread(target=cb, args=(result,), daemon=True).start()
+
+    def pending_count(self) -> int:
+        """Number of jobs waiting to be processed."""
+        if self._db is not None:
+            return self._db.daemon_job_pending_count()
+        return len(list(self._inbox.glob("*.json")))
+
+    def _failed_count(self) -> int:
+        if self._db is not None:
+            return self._db.daemon_job_failed_count()
+        return 0
+
+    # ------------------------------------------------------------------
+    # Main loop
+    # ------------------------------------------------------------------
+
+    def _run_loop(self) -> None:
+        """Daemon main loop — claim jobs from DB (or file inbox) and process them."""
+        last_hb_time = 0.0
+        while not self._stop_event.is_set():
+            # Write heartbeat every _HEARTBEAT_INTERVAL_SECS
+            now = time.monotonic()
+            if now - last_hb_time >= self._HEARTBEAT_INTERVAL_SECS:
+                self._write_heartbeat()
+                last_hb_time = now
+
+            if self._db is not None:
+                row = self._db.daemon_job_claim()
+                if row is not None:
+                    self._process_db_job(row)
+                    continue  # Check immediately for more jobs
+            else:
+                # Legacy file polling with priority sort
+                inbox_files = list(self._inbox.glob("*.json"))
+                if inbox_files:
+                    def _priority_key(f: Path) -> int:
+                        try:
+                            return json.loads(f.read_text(encoding="utf-8")).get("priority", 0)
+                        except Exception:  # noqa: BLE001
+                            return 0
+                    jobs = sorted(inbox_files, key=_priority_key, reverse=True)
+                    self._process_file_job(jobs[0])
+                    # Outbox TTL cleanup
+                    _now = time.time()
+                    for f in self._outbox.glob("*.json"):
+                        try:
+                            if _now - f.stat().st_mtime > _OUTBOX_TTL_SECS:
+                                f.unlink(missing_ok=True)
+                        except Exception:  # noqa: BLE001
+                            pass
+                    continue
+
+            self._stop_event.wait(timeout=_POLL_INTERVAL_SECS)
+
+        # Poll sentry inbox non-fatally
+        try:
+            InboxProcessor(str(self._memory_dir / "inbox")).poll()
+        except Exception:  # noqa: BLE001
+            pass
+
+    def _process_db_job(self, row: Any) -> None:
+        """Execute a job claimed from the DB and write result back."""
+        job = DaemonJob.from_db_row(row)
+        log.info("Daemon: running DB job %s (type=%s)", job.job_id, job.type)
+        start = time.monotonic()
+        executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+        future = executor.submit(self._dispatch, job)
+        try:
+            result = future.result(timeout=_JOB_TIMEOUT_SECS)
+        except concurrent.futures.TimeoutError:
+            executor.shutdown(wait=False, cancel_futures=True)
+            log.error(
+                "Daemon: DB job %s (%s) timed out after %.0fs",
+                job.job_id, job.type, _JOB_TIMEOUT_SECS,
+            )
+            try:
+                self._db.daemon_job_fail(job.job_id, f"timeout after {_JOB_TIMEOUT_SECS}s")
+            except Exception as db_exc:  # noqa: BLE001
+                log.warning("daemon_job_fail write failed: %s", db_exc)
+            self._fire_callback(job.job_id, {"error": f"timeout after {_JOB_TIMEOUT_SECS}s"})
+            return
+        except Exception as exc:  # noqa: BLE001
+            executor.shutdown(wait=False)
+            log.exception("Daemon: DB job %s failed: %s", job.job_id, exc)
+            try:
+                self._db.daemon_job_fail(job.job_id, str(exc))
+            except Exception as db_exc:  # noqa: BLE001
+                log.warning("daemon_job_fail write failed: %s", db_exc)
+            self._fire_callback(job.job_id, {"error": str(exc)})
+            return
+        executor.shutdown(wait=False)
+
+        elapsed = time.monotonic() - start
+        result.setdefault("elapsed_secs", round(elapsed, 2))
+        result.setdefault("job_id", job.job_id)
+        try:
+            self._db.daemon_job_complete(job.job_id, result)
+        except Exception as db_exc:  # noqa: BLE001
+            log.warning("daemon_job_complete write failed: %s", db_exc)
+        log.info("Daemon: DB job %s done in %.1fs", job.job_id, elapsed)
+        self._tasks_completed += 1
+        self._fire_callback(job.job_id, result)
+
+    def _process_file_job(self, inbox_path: Path) -> None:
+        """Legacy: load, execute, and archive a single file job."""
+        try:
+            job = DaemonJob.from_file(inbox_path)
+        except Exception as exc:  # noqa: BLE001
+            log.warning("Daemon: malformed job %s: %s", inbox_path.name, exc)
+            inbox_path.unlink(missing_ok=True)
+            return
+
+        log.info("Daemon: running file job %s (type=%s)", job.job_id, job.type)
+        start = time.monotonic()
+        executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+        future = executor.submit(self._dispatch, job)
+        try:
+            result = future.result(timeout=_JOB_TIMEOUT_SECS)
+        except concurrent.futures.TimeoutError:
+            executor.shutdown(wait=False, cancel_futures=True)
+            log.error(
+                "Daemon: file job %s (%s) timed out after %.0fs",
+                job.job_id, job.type, _JOB_TIMEOUT_SECS,
+            )
+            result = {"ok": False, "error": f"timeout after {_JOB_TIMEOUT_SECS}s"}
+        except Exception as exc:  # noqa: BLE001
+            executor.shutdown(wait=False)
+            log.exception("Daemon: file job %s failed: %s", job.job_id, exc)
+            result = {"ok": False, "error": str(exc)}
+        else:
+            executor.shutdown(wait=False)
+
+        elapsed = time.monotonic() - start
+        result.setdefault("elapsed_secs", round(elapsed, 2))
+        result.setdefault("job_id", job.job_id)
+        out = self._outbox / f"{job.job_id}.json"
+        out.write_text(json.dumps(result), encoding="utf-8")
+        inbox_path.unlink(missing_ok=True)
+        log.info("Daemon: file job %s done in %.1fs", job.job_id, elapsed)
+        self._tasks_completed += 1
+        self._fire_callback(job.job_id, result)
+
+    def _dispatch(self, job: DaemonJob) -> dict[str, Any]:
+        """Route job to the right handler. Returns result dict."""
+        if job.type == "index":
+            return self._handle_index(job.payload)
+        if job.type == "compress":
+            return self._handle_compress(job.payload)
+        if job.type == "scan":
+            return self._handle_scan(job.payload)
+        return {"ok": False, "error": f"Unknown job type: {job.type!r}"}
+
+    # ------------------------------------------------------------------
+    # Handlers
+    # ------------------------------------------------------------------
+
+    def _handle_index(self, payload: dict[str, Any]) -> dict[str, Any]:
+        from src.memory.code_index import CodeIndex
+        project_id = payload.get("project_id", "")
+        root = Path(payload.get("root", "."))
+        index = CodeIndex(self._db, project_id, root)
+        count = index.build(force=payload.get("force", False))
+        return {"ok": True, "files_indexed": count}
+
+    def _handle_compress(self, payload: dict[str, Any]) -> dict[str, Any]:
+        try:
+            from src.memory.compressor import SessionCompressor
+            session_id = payload.get("session_id", "")
+            compressor = SessionCompressor(self._cfg)
+            digest = compressor.compress_session(session_id)
+            return {"ok": True, "digest": digest}
+        except ImportError:
+            return {"ok": False, "error": "compressor not available"}
+
+    def _handle_scan(self, payload: dict[str, Any]) -> dict[str, Any]:
+        try:
+            from src.memory.project_scanner import ProjectScanner
+            project_id = payload.get("project_id", "")
+            root = Path(payload.get("root", "."))
+            scanner = ProjectScanner(self._db)
+            info = scanner.scan(root)
+            return {"ok": True, "tech_stack": info.tech_stack if hasattr(info, "tech_stack") else {}}
+        except ImportError:
+            return {"ok": False, "error": "project_scanner not available"}
+
+
+# ---------------------------------------------------------------------------
+# Sentry inbox processor
+# ---------------------------------------------------------------------------
+
+class InboxProcessor:
+    """Polls .context-memory/inbox/ for sentry-*.json task files."""
+
+    def __init__(self, inbox_dir: str = ".context-memory/inbox"):
+        self._inbox = Path(inbox_dir)
+
+    def poll(self) -> list[dict]:
+        """Return list of pending task dicts (does not consume them)."""
+        if not self._inbox.exists():
+            return []
+        tasks = []
+        for f in sorted(self._inbox.glob("sentry-*.json")):
+            try:
+                tasks.append(json.loads(f.read_text()))
+            except Exception:  # noqa: BLE001
+                pass
+        return tasks
+
+    def consume(self, issue_id: str) -> None:
+        """Remove inbox file after dispatch."""
+        f = self._inbox / f"sentry-{issue_id}.json"
+        if f.exists():
+            f.unlink()
+
+
+# ---------------------------------------------------------------------------
+# Batch crash recovery
+# ---------------------------------------------------------------------------
+
+def _resume_inflight_batches(db_conn: Any) -> list[dict]:
+    """On startup, re-attach to any submitted/polling batch jobs."""
+    rows = db_conn.execute(
+        "SELECT id, provider, batch_id, job_type FROM batch_jobs "
+        "WHERE status IN ('submitted', 'polling')"
+    ).fetchall()
+    if rows:
+        log.info("Resuming %d in-flight batch jobs", len(rows))
+    return [dict(zip(["id", "provider", "batch_id", "job_type"], r)) for r in rows]