remote-coder 0.5.2__tar.gz → 0.5.3__tar.gz

{remote_coder-0.5.2 → remote_coder-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: remote-coder
-Version: 0.5.2
+Version: 0.5.3
 Summary: Telegram-based remote AI coding automation server
 Author: Remote AI Coder contributors
 License:                                  Apache License
@@ -292,7 +292,8 @@ Each project uses its own bot. The webhook path contains the first 16 hex charac
 |---|---|
 | `/start`, `/help` | Open the menu or command help |
 | `/model` | View or change the chat's default model |
-| `/status [job_id]` | Inspect recent or specific jobs |
+| `/status [job_id]` | Inspect recent or specific jobs; running jobs show the latest captured AI output when available |
+| `/log [job_id]` | Pick a recent job or show captured AI stdout for a specific job |
 | `/branch [name]` | Show or switch the bound project's local branch |
 | `/pull` | Fetch remotes and pull the current branch |
 | `/rebase [branch]` | Rebase and fast-forward a completed branch into `main` or `master` |
@@ -322,6 +323,7 @@ Day-to-day setup happens in the local admin UI. Files live under `REMOTE_CODER_H
 - `projects.json` stores project records, bot tokens, allowlists, root paths, and default models.
 - `advanced_settings.json` stores global behavior such as timeouts, sandbox mode, language, worktree retention, and memory limits.
 - `worktrees/<project>/` contains managed job worktrees and logs.
+- On server startup, queued Jobs stored in SQLite are rerun; Jobs that were running when the server stopped are marked failed with `server_restart` for `/status` review.
 
 Useful overrides: `REMOTE_CODER_HOME`, `PROJECTS_CONFIG_PATH`, `CONVERSATION_DB_PATH`, and `JOB_DB_PATH`.
 

{remote_coder-0.5.2 → remote_coder-0.5.3}/README.md

@@ -71,7 +71,8 @@ Each project uses its own bot. The webhook path contains the first 16 hex charac
 |---|---|
 | `/start`, `/help` | Open the menu or command help |
 | `/model` | View or change the chat's default model |
-| `/status [job_id]` | Inspect recent or specific jobs |
+| `/status [job_id]` | Inspect recent or specific jobs; running jobs show the latest captured AI output when available |
+| `/log [job_id]` | Pick a recent job or show captured AI stdout for a specific job |
 | `/branch [name]` | Show or switch the bound project's local branch |
 | `/pull` | Fetch remotes and pull the current branch |
 | `/rebase [branch]` | Rebase and fast-forward a completed branch into `main` or `master` |
@@ -101,6 +102,7 @@ Day-to-day setup happens in the local admin UI. Files live under `REMOTE_CODER_H
 - `projects.json` stores project records, bot tokens, allowlists, root paths, and default models.
 - `advanced_settings.json` stores global behavior such as timeouts, sandbox mode, language, worktree retention, and memory limits.
 - `worktrees/<project>/` contains managed job worktrees and logs.
+- On server startup, queued Jobs stored in SQLite are rerun; Jobs that were running when the server stopped are marked failed with `server_restart` for `/status` review.
 
 Useful overrides: `REMOTE_CODER_HOME`, `PROJECTS_CONFIG_PATH`, `CONVERSATION_DB_PATH`, and `JOB_DB_PATH`.
 

{remote_coder-0.5.2 → remote_coder-0.5.3}/app/__init__.py

@@ -1,3 +1,3 @@
 """Remote AI Coder application package."""
 
-__version__ = "0.5.2"
+__version__ = "0.5.3"

{remote_coder-0.5.2 → remote_coder-0.5.3}/app/ai/base.py

@@ -4,9 +4,11 @@ import re
 import subprocess
 import threading
 from abc import ABC, abstractmethod
+from collections.abc import Callable
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
 from pathlib import Path
+from typing import Literal
 
 from app.jobs.schemas import JobMode
 from app.monitoring.events import EventLogger
@@ -95,6 +97,9 @@ class RunnerInput:
     session_id: str | None = None
     resume_token: str | None = None
     native_resume_cwd_stable: bool = True
+    output_callback: Callable[[Literal["stdout", "stderr"], str], None] | None = field(
+        default=None, compare=False
+    )
 
 
 @dataclass
@@ -213,23 +218,30 @@ class BaseCliRunner(AiRunner):
                 cancelled.set()
 
             threading.Thread(target=_watch, daemon=True).start()
-        try:
-            stdout_data, stderr_data = proc.communicate(timeout=runner_input.timeout_seconds)
-        except subprocess.TimeoutExpired as exc:
-            proc.kill()
-            stdout_data, stderr_data = proc.communicate()
-            self._log.warning(
-                "timeout after %ds stdout_len=%d stderr_len=%d",
-                runner_input.timeout_seconds,
-                len(stdout_data),
-                len(stderr_data),
+        if runner_input.output_callback is None:
+            try:
+                stdout_data, stderr_data = proc.communicate(timeout=runner_input.timeout_seconds)
+            except subprocess.TimeoutExpired as exc:
+                proc.kill()
+                stdout_data, stderr_data = proc.communicate()
+                self._log.warning(
+                    "timeout after %ds stdout_len=%d stderr_len=%d",
+                    runner_input.timeout_seconds,
+                    len(stdout_data),
+                    len(stderr_data),
+                )
+                raise RunnerExecutionError(
+                    f"runner timed out after {runner_input.timeout_seconds}s",
+                    stdout=stdout_data,
+                    stderr=stderr_data,
+                    started_at=started_at,
+                ) from exc
+        else:
+            stdout_data, stderr_data = self._communicate_with_output_callback(
+                proc,
+                runner_input,
+                started_at,
             )
-            raise RunnerExecutionError(
-                f"runner timed out after {runner_input.timeout_seconds}s",
-                stdout=stdout_data,
-                stderr=stderr_data,
-                started_at=started_at,
-            ) from exc
         finished_at = datetime.now(UTC)
         if cancelled.is_set():
             raise RunnerExecutionError(
@@ -255,3 +267,61 @@ class BaseCliRunner(AiRunner):
             finished_at=finished_at,
             session_id=self._resolve_result_session_id(runner_input, session_files_before),
         )
+
+    def _communicate_with_output_callback(
+        self,
+        proc: subprocess.Popen[str],
+        runner_input: RunnerInput,
+        started_at: datetime,
+    ) -> tuple[str, str]:
+        output_callback = runner_input.output_callback
+        if output_callback is None:
+            raise ValueError("output_callback is required")
+        stdout_parts: list[str] = []
+        stderr_parts: list[str] = []
+
+        def _read_stream(
+            stream_name: Literal["stdout", "stderr"],
+            parts: list[str],
+        ) -> None:
+            stream = proc.stdout if stream_name == "stdout" else proc.stderr
+            if stream is None:
+                return
+            while True:
+                chunk = stream.readline()
+                if chunk == "":
+                    break
+                parts.append(chunk)
+                try:
+                    output_callback(stream_name, chunk)
+                except Exception:  # pylint: disable=broad-except
+                    self._log.warning("output callback failed stream=%s", stream_name)
+
+        stdout_thread = threading.Thread(target=_read_stream, args=("stdout", stdout_parts))
+        stderr_thread = threading.Thread(target=_read_stream, args=("stderr", stderr_parts))
+        stdout_thread.start()
+        stderr_thread.start()
+        try:
+            proc.wait(timeout=runner_input.timeout_seconds)
+        except subprocess.TimeoutExpired as exc:
+            proc.kill()
+            proc.wait()
+            stdout_thread.join()
+            stderr_thread.join()
+            stdout_data = "".join(stdout_parts)
+            stderr_data = "".join(stderr_parts)
+            self._log.warning(
+                "timeout after %ds stdout_len=%d stderr_len=%d",
+                runner_input.timeout_seconds,
+                len(stdout_data),
+                len(stderr_data),
+            )
+            raise RunnerExecutionError(
+                f"runner timed out after {runner_input.timeout_seconds}s",
+                stdout=stdout_data,
+                stderr=stderr_data,
+                started_at=started_at,
+            ) from exc
+        stdout_thread.join()
+        stderr_thread.join()
+        return "".join(stdout_parts), "".join(stderr_parts)

{remote_coder-0.5.2 → remote_coder-0.5.3}/app/jobs/execution_pipeline.py

@@ -83,6 +83,7 @@ def run_job(manager, job_id: str) -> Job:
             len(job.request.instruction),
             **manager._job_ctx(job),
         )
+        runner_log = manager._start_incremental_runner_log(job, worktree_base)
         heartbeat = manager._start_heartbeat(job)
         try:
             runner_result = runner.run(
@@ -97,10 +98,12 @@ def run_job(manager, job_id: str) -> Job:
                     session_id=job.request.session_id,
                     resume_token=job.request.resume_session_token,
                     native_resume_cwd_stable=not created_worktree_for_job,
+                    output_callback=runner_log.output_callback,
                 )
             )
         finally:
             heartbeat.set()
+            runner_log.flush()
         manager._save_runner_log(job, runner_result, worktree_base)
         _joblog.info(
             "runner exit=%d stdout_len=%d stderr_len=%d",

{remote_coder-0.5.2 → remote_coder-0.5.3}/app/jobs/fix_pipeline.py

@@ -79,6 +79,7 @@ def run_fix_job(manager, job_id: str) -> Job:
         runner = manager._runner_factory.create(job.request.model)
         timeout_seconds = manager._effective_job_timeout_seconds()
         fix_prompt = manager.compose_fix_source_prompt(parent_job, job.request.instruction)
+        runner_log = manager._start_incremental_runner_log(job, worktree_base)
         heartbeat = manager._start_heartbeat(job)
         try:
             runner_result = runner.run(
@@ -93,10 +94,12 @@ def run_fix_job(manager, job_id: str) -> Job:
                     session_id=job.request.session_id,
                     resume_token=job.request.resume_session_token,
                     native_resume_cwd_stable=not created_worktree_for_job,
+                    output_callback=runner_log.output_callback,
                 )
             )
         finally:
             heartbeat.set()
+            runner_log.flush()
         manager._save_runner_log(job, runner_result, worktree_base)
         if runner_result.exit_code != 0:
             raise RuntimeError(runner_result.stderr.strip() or "runner failed")

{remote_coder-0.5.2 → remote_coder-0.5.3}/app/jobs/manager.py

@@ -26,6 +26,7 @@ from app.jobs.result_writer import (
     make_output_summary,
     preserve_partial_output,
     save_runner_log,
+    start_incremental_runner_log,
     strip_links_for_stdout_summary,
 )
 from app.jobs.schemas import FixKind, Job, JobMode, JobRequest
@@ -186,6 +187,15 @@ class JobManager:
         with self._project_lock(job.request.project):
             return run_job(self, job_id)
 
+    def recover(self, job_id: str) -> Job:
+        job = self._job_store.get(job_id)
+        if job is None:
+            return run_job(self, job_id)
+        with self._project_lock(job.request.project):
+            if job.request.mode is JobMode.AGENT_FIX:
+                return self._run_fix(job_id)
+            return run_job(self, job_id)
+
     def _project_lock(self, project: str) -> threading.Lock:
         with self._project_locks_guard:
             lock = self._project_locks.get(project)
@@ -282,6 +292,9 @@ class JobManager:
     def _save_runner_log(self, job: Job, runner_result, worktree_base: Path) -> None:
         save_runner_log(job, runner_result, worktree_base)
 
+    def _start_incremental_runner_log(self, job: Job, worktree_base: Path):
+        return start_incremental_runner_log(job, worktree_base, self._job_store.update)
+
     @classmethod
     def _strip_links_for_stdout_summary(cls, text: str) -> str:
         return strip_links_for_stdout_summary(text)

remote_coder-0.5.3/app/jobs/result_writer.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import re
+import threading
+import time
+from collections.abc import Callable
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+from app.ai.base import RunnerExecutionError, RunnerResult
+from app.ai.usage import extract_runner_usage
+from app.jobs.schemas import Job
+from app.monitoring.events import EventLogger
+
+_joblog = EventLogger("app.jobs.lifecycle", "job.lifecycle")
+
+_ANSI_ESCAPE_PATTERN = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
+_MD_LINK_PATTERN = re.compile(r"\[([^\]]*)\]\([^)]+\)")
+_HTTP_URL_PATTERN = re.compile(r"https?://[^\s\]\)>,]+", flags=re.IGNORECASE)
+_WWW_URL_PATTERN = re.compile(r"\bwww\.[^\s\]\)>,]+", flags=re.IGNORECASE)
+STDOUT_SUMMARY_LIMIT = 12000
+STDERR_SUMMARY_LIMIT = 800
+
+
+class IncrementalRunnerLog:
+    def __init__(
+        self,
+        job: Job,
+        worktree_base: Path,
+        update_job: Callable[[Job], None],
+        *,
+        update_interval_seconds: float = 1.0,
+    ) -> None:
+        self._job = job
+        self._update_job = update_job
+        self._update_interval_seconds = update_interval_seconds
+        self._stdout_parts: list[str] = []
+        self._stderr_parts: list[str] = []
+        self._lock = threading.Lock()
+        self._started_at = datetime.now(UTC)
+        self._last_update = 0.0
+        log_dir = worktree_base / "_logs"
+        log_dir.mkdir(parents=True, exist_ok=True)
+        self._path = log_dir / f"{job.id}.log"
+        self._job.log_path = self._path
+        with self._lock:
+            self._write_log_locked(exit_code="running", finished_at=None)
+            self._refresh_summaries_locked()
+        self._update_job(self._job)
+
+    def output_callback(self, stream: Literal["stdout", "stderr"], chunk: str) -> None:
+        if not chunk:
+            return
+        with self._lock:
+            if stream == "stdout":
+                self._stdout_parts.append(chunk)
+            else:
+                self._stderr_parts.append(chunk)
+            self._write_log_locked(exit_code="running", finished_at=None)
+            self._refresh_summaries_locked()
+            now = time.monotonic()
+            if now - self._last_update < self._update_interval_seconds:
+                return
+            self._last_update = now
+        self._update_job(self._job)
+
+    def flush(self) -> None:
+        with self._lock:
+            self._refresh_summaries_locked()
+        self._update_job(self._job)
+
+    def _write_log_locked(self, *, exit_code: int | str, finished_at: datetime | None) -> None:
+        log_text = _runner_log_text(
+            self._job,
+            exit_code=exit_code,
+            started_at=self._started_at,
+            finished_at=finished_at,
+            stdout="".join(self._stdout_parts),
+            stderr="".join(self._stderr_parts),
+        )
+        self._path.write_text(log_text, encoding="utf-8")
+
+    def _refresh_summaries_locked(self) -> None:
+        self._job.runner_stdout_summary = make_output_summary(
+            "".join(self._stdout_parts),
+            limit=STDOUT_SUMMARY_LIMIT,
+            strip_links=True,
+        )
+        self._job.runner_stderr_summary = make_output_summary(
+            "".join(self._stderr_parts),
+            limit=STDERR_SUMMARY_LIMIT,
+        )
+
+
+def start_incremental_runner_log(
+    job: Job,
+    worktree_base: Path,
+    update_job: Callable[[Job], None],
+) -> IncrementalRunnerLog:
+    return IncrementalRunnerLog(job, worktree_base, update_job)
+
+
+def preserve_partial_output(job: Job, exc: BaseException, worktree_base: Path) -> None:
+    # A timed-out or cancelled runner still produced output; persist it so the failure
+    # notification can surface the log path and an output summary.
+    if not isinstance(exc, RunnerExecutionError):
+        return
+    save_runner_log(
+        job,
+        RunnerResult(
+            exit_code=-1,
+            stdout=exc.stdout,
+            stderr=exc.stderr,
+            started_at=exc.started_at,
+            finished_at=exc.finished_at,
+        ),
+        worktree_base,
+    )
+
+
+def save_runner_log(job: Job, runner_result, worktree_base: Path) -> None:
+    log_dir = worktree_base / "_logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_path = log_dir / f"{job.id}.log"
+    log_text = _runner_log_text(
+        job,
+        exit_code=runner_result.exit_code,
+        started_at=runner_result.started_at,
+        finished_at=runner_result.finished_at,
+        stdout=runner_result.stdout,
+        stderr=runner_result.stderr,
+    )
+    log_path.write_text(log_text, encoding="utf-8")
+    job.log_path = log_path
+    job.runner_stdout_summary = make_output_summary(
+        runner_result.stdout,
+        limit=STDOUT_SUMMARY_LIMIT,
+        strip_links=True,
+    )
+    job.runner_stderr_summary = make_output_summary(
+        runner_result.stderr, limit=STDERR_SUMMARY_LIMIT
+    )
+    usage = extract_runner_usage(f"{runner_result.stdout}\n{runner_result.stderr}")
+    job.runner_actual_model = usage.actual_model
+    job.runner_token_usage = usage.token_usage
+    job.runner_session_id = runner_result.session_id
+    _joblog.info(
+        "runner log saved file=%s stdout_summary=%s stderr_summary=%s actual_model=%s token_usage=%s",
+        log_path.name,
+        job.runner_stdout_summary is not None,
+        job.runner_stderr_summary is not None,
+        job.runner_actual_model or "-",
+        bool(job.runner_token_usage),
+        chat_id=job.request.chat_id,
+        user_id=job.request.requested_by,
+        project=job.request.project,
+        job_id=job.id,
+    )
+
+
+def _runner_log_text(
+    job: Job,
+    *,
+    exit_code: int | str,
+    started_at: datetime | None,
+    finished_at: datetime | None,
+    stdout: str,
+    stderr: str,
+) -> str:
+    return (
+        f"job_id={job.id}\n"
+        f"model={job.request.model.value}\n"
+        f"exit_code={exit_code}\n"
+        f"started_at={started_at}\n"
+        f"finished_at={finished_at}\n\n"
+        f"[stdout]\n{stdout}\n\n"
+        f"[stderr]\n{stderr}\n"
+    )
+
+
+def extract_stdout_from_log(path: Path) -> str | None:
+    """Return the full [stdout] section of a saved runner log, ANSI-stripped."""
+    try:
+        log_text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return None
+    marker = "\n[stdout]\n"
+    start = log_text.find(marker)
+    if start == -1:
+        return None
+    start += len(marker)
+    stderr_at = log_text.rfind("\n\n[stderr]\n")
+    stdout = log_text[start:stderr_at] if stderr_at >= start else log_text[start:]
+    stdout = _ANSI_ESCAPE_PATTERN.sub("", stdout)
+    return stdout or None
+
+
+def strip_links_for_stdout_summary(text: str) -> str:
+    stripped = _MD_LINK_PATTERN.sub(r"\1", text)
+    stripped = _HTTP_URL_PATTERN.sub("", stripped)
+    stripped = _WWW_URL_PATTERN.sub("", stripped)
+    stripped = re.sub(r"[ \t]{2,}", " ", stripped)
+    return stripped
+
+
+def make_output_summary(
+    text: str,
+    limit: int,
+    *,
+    strip_links: bool = False,
+) -> str | None:
+    if not text:
+        return None
+    no_ansi = _ANSI_ESCAPE_PATTERN.sub("", text)
+    if strip_links:
+        no_ansi = strip_links_for_stdout_summary(no_ansi)
+    normalized = "\n".join(line.rstrip() for line in no_ansi.splitlines()).strip()
+    if not normalized:
+        return None
+    if len(normalized) <= limit:
+        return normalized
+    return f"{normalized[:limit].rstrip()}...(truncated)"

{remote_coder-0.5.2 → remote_coder-0.5.3}/app/jobs/store.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sqlite3
+from collections.abc import Sequence
 from pathlib import Path
 from threading import Lock
 from typing import Protocol
@@ -37,6 +38,9 @@ class JobStore(Protocol):
     def list_recent_for_project_chat(self, project: str, chat_id: int, limit: int = 20) -> list[Job]:
         ...
 
+    def list_latest_by_status(self, statuses: Sequence[JobStatus]) -> list[Job]:
+        ...
+
 
 class InMemoryJobStore:
     def __init__(self) -> None:
@@ -92,6 +96,19 @@ class InMemoryJobStore:
             values.sort(key=lambda job: job.created_at, reverse=True)
             return values[:limit]
 
+    def list_latest_by_status(self, statuses: Sequence[JobStatus]) -> list[Job]:
+        wanted = {status.value for status in statuses}
+        if not wanted:
+            return []
+        with self._lock:
+            values = [
+                jobs[-1]
+                for jobs in self._jobs.values()
+                if jobs and jobs[-1].status.value in wanted
+            ]
+            values.sort(key=lambda job: job.created_at)
+            return values
+
     def get_latest_succeeded_branch_for_project_chat(
         self, project: str, chat_id: int
     ) -> str | None:
@@ -299,6 +316,32 @@ class SQLiteJobStore:
             (project, chat_id, limit),
         )
 
+    def list_latest_by_status(self, statuses: Sequence[JobStatus]) -> list[Job]:
+        if not statuses:
+            return []
+        placeholders = ",".join("?" for _ in statuses)
+        return self._fetch_jobs(
+            f"""
+            SELECT payload
+            FROM jobs AS current
+            WHERE current.status IN ({placeholders})
+              AND NOT EXISTS (
+                  SELECT 1
+                  FROM jobs AS newer
+                  WHERE newer.job_id = current.job_id
+                    AND (
+                        newer.created_at > current.created_at
+                        OR (
+                            newer.created_at = current.created_at
+                            AND newer.row_id > current.row_id
+                        )
+                    )
+              )
+            ORDER BY current.created_at ASC, current.row_id ASC
+            """,
+            tuple(status.value for status in statuses),
+        )
+
     def get_latest_succeeded_branch_for_project_chat(
         self, project: str, chat_id: int
     ) -> str | None:

{remote_coder-0.5.2 → remote_coder-0.5.3}/app/main.py

@@ -14,6 +14,7 @@ from app.git.ai_commit import AiCommitBodyGenerator
 from app.git.branch_naming import TimestampSlugStrategy
 from app.git.service import GitWorktreeService
 from app.jobs.manager import JobManager
+from app.jobs.schemas import Job
 from app.jobs.store import SQLiteJobStore
 from app.monitoring.log_buffer import InMemoryLogBuffer, attach_app_memory_log_handler
 from app.monitoring.events import EventLogger
@@ -24,7 +25,7 @@ from app.projects.registry import (
     projects_config_path,
 )
 from app.security.auth import AllowlistAuthService
-from app.system_startup import run_startup_project_pulls
+from app.system_startup import recover_startup_jobs, run_startup_project_pulls
 from app.telegram.commands import (
     CommandContext,
     CommandRegistry,
@@ -34,6 +35,8 @@ from app.telegram.commands import (
 from app.telegram.bot_instances import BotInstance, BotInstanceManager
 from app.telegram.confirmations import InMemoryConfirmationStore
 from app.telegram.conversation import SQLiteConversationStore
+from app.telegram.handlers.job_submission import JobSubmission
+from app.telegram.handlers.session_binding import SessionBinding
 from app.telegram.notifier import Notifier, TelegramNotifier
 from app.telegram.i18n import ui_message
 from app.telegram.parser import CommandParser
@@ -138,6 +141,35 @@ webhook_registrar = (
 )
 
 
+def _record_recovered_job_result(final_job: Job) -> None:
+    JobSubmission(
+        job_manager=job_manager,
+        conversation_store=conversation_store,
+        attach_session=lambda _request: None,
+        persist_session_token=SessionBinding(conversation_store).persist_session_token,
+    ).record_final_job_result(final_job)
+
+
+def _refresh_project_command_menus() -> None:
+    # Re-push setMyCommands for every enabled project on each startup. The Telegram command
+    # menu (hamburger button) and slash autocomplete are otherwise only refreshed during
+    # `remote-coder up` webhook registration or an admin settings save, so a bot registered
+    # before a new command/mode (e.g. research) was added keeps a stale menu on plain restarts.
+    registrar = webhook_registrar
+    if registrar is None:
+        registrar = TelegramWebhookRegistrar(
+            "",
+            bot_commands=command_registry.bot_commands(advanced_settings_store.get().ui_language),
+        )
+    synced = 0
+    for record in project_registry.list_projects():
+        if not record.enabled:
+            continue
+        if registrar.sync_project_commands(record):
+            synced += 1
+    _systemlog.info("startup command menus refreshed count=%d", synced)
+
+
 def _run_startup_side_effects(instances: list[BotInstance], adv: AdvancedSettings) -> None:
     startup_chat_total = sum(len(inst.auth_service.allowed_chat_ids) for inst in instances)
     _systemlog.info(
@@ -146,6 +178,13 @@ def _run_startup_side_effects(instances: list[BotInstance], adv: AdvancedSetting
         len(project_registry.list_projects()),
         ModelName.CLAUDE.value,
     )
+    _refresh_project_command_menus()
+    recover_startup_jobs(
+        job_store=job_store,
+        run_job=job_manager.recover,
+        record_final_job_result=_record_recovered_job_result,
+        system_log=_systemlog,
+    )
     run_startup_project_pulls(
         pull_projects_on_server_startup_enabled=adv.pull_projects_on_server_startup_enabled,
         project_registry=project_registry,