researchloop 0.3.0__tar.gz → 0.3.1__tar.gz

{researchloop-0.3.0 → researchloop-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: researchloop
-Version: 0.3.0
+Version: 0.3.1
 Summary: Automated research sprint platform for HPC clusters
 License: MIT
 License-File: LICENSE

{researchloop-0.3.0 → researchloop-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "researchloop"
-version = "0.3.0"
+version = "0.3.1"
 description = "Automated research sprint platform for HPC clusters"
 readme = "README.md"
 license = {text = "MIT"}

researchloop-0.3.1/researchloop/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.1"

{researchloop-0.3.0 → researchloop-0.3.1}/researchloop/clusters/monitor.py

@@ -6,12 +6,15 @@ import asyncio
 import json
 import logging
 from datetime import datetime, timezone
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from researchloop.clusters.ssh import SSHManager
 from researchloop.db import queries
 from researchloop.schedulers.base import BaseScheduler
 
+if TYPE_CHECKING:
+    from researchloop.sprints.manager import SprintManager
+
 logger = logging.getLogger(__name__)
 
 # If a job's heartbeat is older than this many seconds AND the job is not
@@ -28,11 +31,17 @@ class JobMonitor:
         db: Any,
         schedulers: dict[str, BaseScheduler],
         config: Any = None,
+        sprint_manager: SprintManager | None = None,
     ) -> None:
         self.ssh_manager = ssh_manager
         self.db = db
         self.schedulers = schedulers
         self.config = config
+        # Optional: when set, terminal-state transitions go through
+        # sprint_manager.mark_sprint_terminal so the parent auto-loop
+        # advances. None falls back to a direct DB update (used by
+        # minimal test fixtures that don't construct a SprintManager).
+        self.sprint_manager = sprint_manager
         self._polling_task: asyncio.Task[None] | None = None
         self._stop_event = asyncio.Event()
 
@@ -143,12 +152,17 @@ class JobMonitor:
             # Persist the updated status if it changed.
             if status in ("completed", "failed"):
                 try:
-                    await queries.update_sprint(
-                        self.db,
-                        sprint_id,
-                        status=status,
-                        completed_at=datetime.now(timezone.utc).isoformat(),
-                    )
+                    if self.sprint_manager is not None:
+                        await self.sprint_manager.mark_sprint_terminal(
+                            sprint_id, status
+                        )
+                    else:
+                        await queries.update_sprint(
+                            self.db,
+                            sprint_id,
+                            status=status,
+                            completed_at=datetime.now(timezone.utc).isoformat(),
+                        )
                 except Exception:
                     logger.exception(
                         "Failed to update DB status for sprint %s", sprint_id

{researchloop-0.3.0 → researchloop-0.3.1}/researchloop/core/orchestrator.py

@@ -112,6 +112,9 @@ class Orchestrator:
             sprint_manager=self.sprint_manager,
             config=self.config,
         )
+        # Late-bind the back-reference so SprintManager.mark_sprint_terminal
+        # can advance the parent loop on every terminal transition.
+        self.sprint_manager.auto_loop = self.auto_loop
 
         # 8. Job monitor
         self.job_monitor = JobMonitor(
@@ -119,6 +122,7 @@ class Orchestrator:
             db=self.db,
             schedulers=self.schedulers,
             config=self.config,
+            sprint_manager=self.sprint_manager,
         )
         await self.job_monitor.start_polling()
 
@@ -405,6 +409,10 @@ def create_app(orchestrator: Orchestrator) -> FastAPI:
                 {"ok": True, "sprint_id": sprint_id, "tweak_id": tweak_id}
             )
 
+        # handle_completion fires auto_loop.on_sprint_complete internally
+        # via mark_sprint_terminal — single chokepoint for terminal-state
+        # transitions, so the loop also advances when the JobMonitor or a
+        # dashboard refresh is the one that detects the terminal status.
         await orchestrator.sprint_manager.handle_completion(
             sprint_id=sprint_id,
             status=status,
@@ -413,10 +421,6 @@ def create_app(orchestrator: Orchestrator) -> FastAPI:
             idea=idea,
         )
 
-        # Trigger auto-loop advancement if applicable.
-        if orchestrator.auto_loop is not None:
-            await orchestrator.auto_loop.on_sprint_complete(sprint_id)
-
         logger.info(
             "Webhook: sprint %s completion processed (status=%s)",
             sprint_id,

{researchloop-0.3.0 → researchloop-0.3.1}/researchloop/dashboard/routes.py

@@ -899,17 +899,12 @@ def add_dashboard_routes(
                         }
                         cur = sprint["status"]
                         if real_status in terminal and cur not in terminal:
-                            from datetime import (
-                                datetime,
-                                timezone,
-                            )
-
-                            now = datetime.now(timezone.utc).isoformat()
-                            await queries.update_sprint(
-                                orchestrator.db,
-                                sprint_id,
-                                status=real_status,
-                                completed_at=now,
+                            # Route through SprintManager so the parent
+                            # auto-loop is advanced too — otherwise a
+                            # webhook-less failure leaves the loop stuck
+                            # in "running".
+                            await orchestrator.sprint_manager.mark_sprint_terminal(
+                                sprint_id, real_status
                             )
 
                         # Resolve sprints_base the same way

{researchloop-0.3.0 → researchloop-0.3.1}/researchloop/dashboard/templates/sprint_detail.html

@@ -106,7 +106,7 @@
 </div>
 {% endif %}
 
-{% if sprint.status == 'completed' %}
+{% if sprint.status in ('completed', 'failed', 'cancelled') %}
 <h3>Quick Tweak</h3>
 <div class="card">
   {% if tweak_active %}
@@ -116,7 +116,11 @@
         onsubmit="var b=this.querySelector('button[type=submit]');if(b.disabled)return false;b.disabled=true;b.textContent='Submitting...';">
     <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
     <div class="form-group">
+      {% if sprint.status == 'completed' %}
       <textarea name="instruction" rows="3" placeholder="e.g. Fix the axis labels on the scatter plots, add a histogram of residuals"></textarea>
+      {% else %}
+      <textarea name="instruction" rows="3" placeholder="e.g. Retry with smaller batch size, or investigate the error and continue"></textarea>
+      {% endif %}
     </div>
     <details style="margin-bottom:0.75rem">
       <summary class="dim" style="cursor:pointer;font-size:0.85rem">Resource settings</summary>

{researchloop-0.3.0 → researchloop-0.3.1}/researchloop/runner/templates/tweak.md.j2

@@ -1,4 +1,9 @@
-You are applying a quick tweak to a completed research sprint.
+You are applying a quick tweak to a research sprint.
+
+The sprint may have completed successfully, or it may have failed or
+been cancelled before finishing. Check the existing files (findings.md,
+report.md, results/, sprint logs) to see how far it got, then apply the
+instruction below.
 
 ## Tweak Instruction
 {{ instruction }}

{researchloop-0.3.0 → researchloop-0.3.1}/researchloop/sprints/manager.py

@@ -17,6 +17,7 @@ if TYPE_CHECKING:
     from researchloop.core.config import Config
     from researchloop.db.database import Database
     from researchloop.schedulers.base import BaseScheduler
+    from researchloop.sprints.auto_loop import AutoLoopController
 
 from researchloop.core.models import (
     Sprint,
@@ -89,6 +90,9 @@ class SprintManager:
         self.schedulers = schedulers
         self.study_manager = study_manager
         self.notification_router = notification_router
+        # Late-bound by Orchestrator after AutoLoopController is built;
+        # circular dep otherwise (AutoLoopController takes a SprintManager).
+        self.auto_loop: AutoLoopController | None = None
 
     # ------------------------------------------------------------------
     # Create
@@ -613,6 +617,56 @@ class SprintManager:
     # Completion handling
     # ------------------------------------------------------------------
 
+    async def mark_sprint_terminal(
+        self,
+        sprint_id: str,
+        status: str,
+        error: str | None = None,
+        **extra_fields: Any,
+    ) -> bool:
+        """Transition a sprint to a terminal state and notify its auto-loop.
+
+        This is the single chokepoint for terminal-state transitions —
+        webhook, JobMonitor SSH polling, and dashboard refresh all flow
+        through here so a loop sprint that fails outside the webhook path
+        still advances the parent loop.
+
+        Idempotent: returns ``False`` (and skips the loop callback) if the
+        sprint is already terminal or doesn't exist, so callers can branch
+        on whether their call was the one that did the transition.
+        """
+        sprint = await queries.get_sprint(self.db, sprint_id)
+        if sprint is None:
+            return False
+        terminal = {
+            SprintStatus.COMPLETED.value,
+            SprintStatus.FAILED.value,
+            SprintStatus.CANCELLED.value,
+        }
+        if sprint.get("status") in terminal:
+            return False
+
+        update_kw: dict[str, Any] = {
+            "status": status,
+            "completed_at": datetime.now(timezone.utc).isoformat(),
+        }
+        if error is not None:
+            update_kw["error"] = error
+        update_kw.update(extra_fields)
+        await queries.update_sprint(self.db, sprint_id, **update_kw)
+
+        if self.auto_loop is not None and sprint.get("loop_id"):
+            try:
+                await self.auto_loop.on_sprint_complete(sprint_id)
+            except Exception:
+                logger.exception(
+                    "Auto-loop advance failed for sprint %s; "
+                    "loop may be stuck — manual intervention needed",
+                    sprint_id,
+                )
+
+        return True
+
     async def handle_completion(
         self,
         sprint_id: str,
@@ -624,29 +678,34 @@ class SprintManager:
         """Handle a sprint completion event.
 
         Updates the database, sends notifications, and creates an event
-        record.
+        record. Idempotent — if the sprint is already terminal (e.g. the
+        JobMonitor caught it first), the rich notification work is
+        skipped to avoid double-firing.
         """
-        now = datetime.now(timezone.utc).isoformat()
-
-        update_kw: dict[str, str | None] = {
-            "status": status,
-            "completed_at": now,
-            "summary": summary,
-            "error": error,
-        }
+        extra: dict[str, Any] = {}
+        if summary is not None:
+            extra["summary"] = summary
 
         # Update the idea if it was auto-generated (sprint had idea=None).
         sprint_before = await queries.get_sprint(self.db, sprint_id)
         if sprint_before and not sprint_before.get("idea"):
             if idea:
-                update_kw["idea"] = idea[:500]
+                extra["idea"] = idea[:500]
             else:
                 # Fallback: try to read idea.txt from the cluster.
                 fetched = await self._fetch_idea(sprint_before)
                 if fetched:
-                    update_kw["idea"] = fetched[:500]
+                    extra["idea"] = fetched[:500]
 
-        await queries.update_sprint(self.db, sprint_id, **update_kw)
+        transitioned = await self.mark_sprint_terminal(
+            sprint_id, status, error=error, **extra
+        )
+        if not transitioned:
+            logger.info(
+                "Sprint %s already terminal; skipping completion processing",
+                sprint_id,
+            )
+            return
 
         sprint = await queries.get_sprint(self.db, sprint_id)
         study_name = sprint["study_name"] if sprint else "unknown"
@@ -857,7 +916,12 @@ class SprintManager:
         job_options: dict[str, str] | None = None,
         time_limit: str | None = None,
     ) -> str:
-        """Submit a quick tweak job for a completed sprint.
+        """Submit a quick tweak job for a terminal sprint.
+
+        Allowed when the sprint is in a terminal state (completed, failed,
+        or cancelled) — failed/cancelled sprints often have partial state
+        worth iterating on with an instruction like "retry with smaller
+        batch size".
 
         If *time_limit* is None, the study's ``max_sprint_duration_hours``
         is used (same default as a regular sprint). Returns the tweak ID.
@@ -865,9 +929,15 @@ class SprintManager:
         sprint = await queries.get_sprint(self.db, sprint_id)
         if sprint is None:
             raise ValueError(f"Sprint not found: {sprint_id}")
-        if sprint["status"] != SprintStatus.COMPLETED.value:
+        terminal = {
+            SprintStatus.COMPLETED.value,
+            SprintStatus.FAILED.value,
+            SprintStatus.CANCELLED.value,
+        }
+        if sprint["status"] not in terminal:
             raise ValueError(
-                f"Sprint {sprint_id} is not completed (status={sprint['status']})"
+                f"Sprint {sprint_id} is not in a terminal state "
+                f"(status={sprint['status']}); wait for it to finish first"
             )
 
         # Reject if there's already a running tweak for this sprint.

{researchloop-0.3.0 → researchloop-0.3.1}/tests/test_sprint_manager.py

@@ -319,6 +319,175 @@ class TestSprintManagerCompletion:
         mock_notifier.notify_sprint_completed.assert_called_once()
 
 
+class TestMarkSprintTerminal:
+    """The single chokepoint for terminal-state transitions.
+
+    Webhook, JobMonitor SSH polling, and dashboard refresh all flow through
+    here so the parent auto-loop advances on every terminal transition,
+    including the no-webhook case (OOM kills, walltime, node failure).
+    """
+
+    async def test_advances_loop_on_failed(self, db_with_study, sample_config):
+        """The original bug: a failed loop sprint detected outside the
+        webhook path used to leave the loop hanging in 'running'.
+        """
+        mgr = SprintManager(
+            db=db_with_study,
+            config=sample_config,
+            ssh_manager=AsyncMock(),
+            schedulers={},
+        )
+        await queries.create_auto_loop(db_with_study, "loop-mst1", "test-study", 5)
+        sprint = await mgr.create_sprint("test-study", "idea")
+        await queries.update_sprint(
+            db_with_study,
+            sprint.id,
+            status="running",
+            loop_id="loop-mst1",
+        )
+        await queries.update_auto_loop(
+            db_with_study,
+            "loop-mst1",
+            current_sprint_id=sprint.id,
+            status="running",
+        )
+
+        from researchloop.sprints.auto_loop import AutoLoopController
+
+        mgr.auto_loop = AutoLoopController(
+            db=db_with_study, sprint_manager=mgr, config=sample_config
+        )
+
+        transitioned = await mgr.mark_sprint_terminal(sprint.id, "failed")
+        assert transitioned is True
+
+        loop = await queries.get_auto_loop(db_with_study, "loop-mst1")
+        assert loop is not None
+        assert loop["status"] == "failed"
+        sp = await queries.get_sprint(db_with_study, sprint.id)
+        assert sp["status"] == "failed"
+        assert sp["completed_at"] is not None
+
+    async def test_idempotent_when_already_terminal(self, db_with_study, sample_config):
+        """A second call must not advance the loop again — otherwise a
+        completed sprint plus a late JobMonitor sweep could submit a
+        duplicate next sprint.
+        """
+        mgr = SprintManager(
+            db=db_with_study,
+            config=sample_config,
+            ssh_manager=AsyncMock(),
+            schedulers={},
+        )
+        sprint = await mgr.create_sprint("test-study", "idea")
+        await queries.update_sprint(db_with_study, sprint.id, status="failed")
+
+        mgr.auto_loop = AsyncMock()
+        result = await mgr.mark_sprint_terminal(sprint.id, "failed")
+        assert result is False
+        mgr.auto_loop.on_sprint_complete.assert_not_called()
+
+    async def test_no_callback_when_not_in_loop(self, db_with_study, sample_config):
+        """Standalone sprints (no loop_id) don't trigger the callback."""
+        mgr = SprintManager(
+            db=db_with_study,
+            config=sample_config,
+            ssh_manager=AsyncMock(),
+            schedulers={},
+        )
+        sprint = await mgr.create_sprint("test-study", "idea")
+        await queries.update_sprint(db_with_study, sprint.id, status="running")
+
+        mgr.auto_loop = AsyncMock()
+        await mgr.mark_sprint_terminal(sprint.id, "completed")
+        mgr.auto_loop.on_sprint_complete.assert_not_called()
+
+    async def test_callback_failure_does_not_block_status_update(
+        self, db_with_study, sample_config
+    ):
+        """If the loop callback raises, the DB update still stands —
+        otherwise a transient on_sprint_complete bug would leave the
+        sprint in the wrong status forever.
+        """
+        mgr = SprintManager(
+            db=db_with_study,
+            config=sample_config,
+            ssh_manager=AsyncMock(),
+            schedulers={},
+        )
+        sprint = await mgr.create_sprint("test-study", "idea")
+        await queries.update_sprint(
+            db_with_study, sprint.id, status="running", loop_id="loop-x"
+        )
+
+        mgr.auto_loop = AsyncMock()
+        mgr.auto_loop.on_sprint_complete.side_effect = RuntimeError("boom")
+        result = await mgr.mark_sprint_terminal(sprint.id, "failed")
+        assert result is True
+        sp = await queries.get_sprint(db_with_study, sprint.id)
+        assert sp["status"] == "failed"
+
+
+class TestJobMonitorAdvancesLoop:
+    """JobMonitor failure detection must advance the parent auto-loop.
+
+    Regression: the SSH-polling fallback used to bypass the auto-loop
+    callback, so loops whose sprint died without sending a webhook were
+    stranded in 'running' with their failed sprint as current_sprint_id.
+    """
+
+    async def test_failed_sprint_marks_loop_failed(self, db_with_study, sample_config):
+        from unittest.mock import patch
+
+        from researchloop.clusters.monitor import JobMonitor
+        from researchloop.sprints.auto_loop import AutoLoopController
+
+        mgr = SprintManager(
+            db=db_with_study,
+            config=sample_config,
+            ssh_manager=AsyncMock(),
+            schedulers={},
+        )
+        mgr.auto_loop = AutoLoopController(
+            db=db_with_study, sprint_manager=mgr, config=sample_config
+        )
+
+        await queries.create_auto_loop(db_with_study, "loop-jm1", "test-study", 5)
+        sprint = await mgr.create_sprint("test-study", "idea")
+        await queries.update_sprint(
+            db_with_study,
+            sprint.id,
+            status="running",
+            job_id="123",
+            loop_id="loop-jm1",
+        )
+        await queries.update_auto_loop(
+            db_with_study,
+            "loop-jm1",
+            current_sprint_id=sprint.id,
+            status="running",
+        )
+
+        monitor = JobMonitor(
+            ssh_manager=AsyncMock(),
+            db=db_with_study,
+            schedulers={},
+            sprint_manager=mgr,
+        )
+
+        with patch.object(
+            JobMonitor, "check_job", new=AsyncMock(return_value="failed")
+        ):
+            await monitor.poll_active_jobs()
+
+        loop = await queries.get_auto_loop(db_with_study, "loop-jm1")
+        assert loop is not None
+        assert loop["status"] == "failed", (
+            "Loop should advance to failed when JobMonitor catches a failure "
+            "the runner couldn't webhook about (OOM, walltime, node death)."
+        )
+
+
 def _make_config(
     tmp_path: Path,
     global_context: str = "",

{researchloop-0.3.0 → researchloop-0.3.1}/tests/test_tweaks.py

@@ -170,10 +170,10 @@ class TestSubmitTweak:
         assert ssh_mock.run.call_count >= 2  # write script + chmod
         scheduler.submit.assert_called_once()
 
-    async def test_submit_tweak_rejects_non_completed_sprint(
+    async def test_submit_tweak_rejects_non_terminal_sprint(
         self, db_with_study, sample_config
     ):
-        """Should raise ValueError for non-completed sprints."""
+        """Should raise ValueError when the sprint is still pending/running."""
         mgr = SprintManager(
             db=db_with_study,
             config=sample_config,
@@ -186,7 +186,59 @@ class TestSubmitTweak:
             await mgr.submit_tweak(sprint.id, "some tweak")
             assert False, "Expected ValueError"
         except ValueError as e:
-            assert "not completed" in str(e)
+            assert "terminal" in str(e)
+
+    async def test_submit_tweak_on_failed_sprint(self, db_with_study, tmp_path):
+        """Failed sprints accept tweaks too — useful for "retry with X"."""
+        config = _tweak_config(tmp_path)
+        ssh_mock = AsyncMock()
+        ssh_mgr = AsyncMock()
+        ssh_mgr.get_connection.return_value = ssh_mock
+        scheduler = AsyncMock()
+        scheduler.submit.return_value = "888"
+        study_mgr = StudyManager(db_with_study, config)
+
+        mgr = SprintManager(
+            db=db_with_study,
+            config=config,
+            ssh_manager=ssh_mgr,
+            schedulers={"slurm": scheduler},
+            study_manager=study_mgr,
+        )
+        sprint = await mgr.create_sprint("test-study", "original idea")
+        await queries.update_sprint(db_with_study, sprint.id, status="failed")
+
+        tweak_id = await mgr.submit_tweak(sprint.id, "retry with smaller batch size")
+
+        tweak = await queries.get_tweak(db_with_study, tweak_id)
+        assert tweak is not None
+        assert tweak["status"] == "submitted"
+
+    async def test_submit_tweak_on_cancelled_sprint(self, db_with_study, tmp_path):
+        """Cancelled sprints accept tweaks too."""
+        config = _tweak_config(tmp_path)
+        ssh_mock = AsyncMock()
+        ssh_mgr = AsyncMock()
+        ssh_mgr.get_connection.return_value = ssh_mock
+        scheduler = AsyncMock()
+        scheduler.submit.return_value = "777"
+        study_mgr = StudyManager(db_with_study, config)
+
+        mgr = SprintManager(
+            db=db_with_study,
+            config=config,
+            ssh_manager=ssh_mgr,
+            schedulers={"slurm": scheduler},
+            study_manager=study_mgr,
+        )
+        sprint = await mgr.create_sprint("test-study", "original idea")
+        await queries.update_sprint(db_with_study, sprint.id, status="cancelled")
+
+        tweak_id = await mgr.submit_tweak(sprint.id, "pick up where you left off")
+
+        tweak = await queries.get_tweak(db_with_study, tweak_id)
+        assert tweak is not None
+        assert tweak["status"] == "submitted"
 
     async def test_submit_tweak_defaults_to_study_time_limit(
         self, db_with_study, tmp_path
@@ -621,6 +673,63 @@ class TestTweakDashboard:
             assert "Quick Tweak" in resp.text
             assert 'name="instruction"' in resp.text
 
+    async def test_tweak_form_visible_on_failed_sprint(
+        self, db_with_study, sample_config
+    ):
+        """The tweak form must appear on failed/cancelled sprints too —
+        users iterate on partial state with instructions like "retry with
+        smaller batch size".
+        """
+        import tempfile
+
+        from fastapi.testclient import TestClient
+
+        from researchloop.core.config import DashboardConfig
+        from researchloop.core.orchestrator import Orchestrator, create_app
+
+        config = Config(
+            studies=sample_config.studies,
+            clusters=sample_config.clusters,
+            db_path=":memory:",
+            artifact_dir=tempfile.mkdtemp(),
+            dashboard=DashboardConfig(password_hash=None),
+        )
+        orch = Orchestrator(config)
+        app = create_app(orch)
+        client = TestClient(app)
+
+        with client:
+            assert orch.db is not None
+            from researchloop.dashboard.auth import hash_password
+
+            pw_hash = hash_password("testpass123")
+            await orch.db.execute(
+                "INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)",
+                ("dashboard_password_hash", pw_hash),
+            )
+            resp = client.post(
+                "/dashboard/login",
+                data={"password": "testpass123"},
+                follow_redirects=False,
+            )
+            cookies = dict(resp.cookies)
+
+            for sid, status in [
+                ("sp-twk-fail", "failed"),
+                ("sp-twk-canc", "cancelled"),
+            ]:
+                await queries.create_sprint(orch.db, sid, "test-study", "idea")
+                await queries.update_sprint(orch.db, sid, status=status)
+                resp = client.get(
+                    f"/dashboard/sprints/{sid}",
+                    cookies=cookies,
+                )
+                assert resp.status_code == 200, status
+                assert "Quick Tweak" in resp.text, (
+                    f"Tweak form missing for {status} sprint"
+                )
+                assert 'name="instruction"' in resp.text
+
     async def test_tweak_form_hidden_on_running_sprint(
         self, db_with_study, sample_config
     ):

{researchloop-0.3.0 → researchloop-0.3.1}/uv.lock

@@ -1152,7 +1152,7 @@ wheels = [
 
 [[package]]
 name = "researchloop"
-version = "0.3.0"
+version = "0.3.1"
 source = { editable = "." }
 dependencies = [
     { name = "aiosqlite" },

researchloop-0.3.0/researchloop/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.0"