researchloop 0.1.0__py3-none-any.whl

researchloop/runner/main.py

@@ -0,0 +1,156 @@
+"""Entry point for the ``researchloop-runner`` CLI.
+
+This runs INSIDE a SLURM/SGE job on HPC clusters.  It executes the
+sub-agent pipeline and reports results back to the orchestrator.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import sys
+from pathlib import Path
+
+import click
+
+from researchloop.runner.pipeline import Pipeline
+from researchloop.runner.upload import send_webhook, upload_artifacts
+
+logger = logging.getLogger("researchloop.runner")
+
+
+async def _run_pipeline(
+    sprint_id: str,
+    sprint_dir: str,
+    claude_md: str,
+    idea: str,
+    orchestrator_url: str,
+    shared_secret: str,
+    red_team_rounds: int,
+    claude_command: str = "claude --dangerously-skip-permissions",
+) -> None:
+    """Execute the full pipeline and report back to the orchestrator."""
+    sprint_path = Path(sprint_dir)
+    sprint_path.mkdir(parents=True, exist_ok=True)
+    (sprint_path / ".researchloop").mkdir(parents=True, exist_ok=True)
+    (sprint_path / "results").mkdir(parents=True, exist_ok=True)
+
+    pipeline = Pipeline(
+        sprint_id=sprint_id,
+        sprint_dir=sprint_dir,
+        claude_md=claude_md,
+        idea=idea,
+        orchestrator_url=orchestrator_url,
+        shared_secret=shared_secret,
+        red_team_rounds=red_team_rounds,
+        claude_command=claude_command,
+    )
+
+    summary: str | None = None
+    error_msg: str | None = None
+    final_status = "completed"
+
+    try:
+        summary = await pipeline.run()
+    except Exception:
+        logger.exception("Pipeline failed for sprint %s", sprint_id)
+        final_status = "failed"
+        error_msg = f"Pipeline error: {sys.exc_info()[1]}"
+    finally:
+        await pipeline.stop()
+
+    # Upload artifacts before sending the completion webhook so the
+    # orchestrator can access them immediately.
+    if final_status == "completed":
+        try:
+            uploaded = await upload_artifacts(
+                sprint_dir=sprint_dir,
+                orchestrator_url=orchestrator_url,
+                shared_secret=shared_secret,
+                sprint_id=sprint_id,
+            )
+            logger.info("Uploaded %d artifact(s)", len(uploaded))
+        except Exception:
+            logger.exception("Artifact upload failed for sprint %s", sprint_id)
+
+    # Notify orchestrator of completion (or failure).
+    try:
+        await send_webhook(
+            orchestrator_url=orchestrator_url,
+            shared_secret=shared_secret,
+            sprint_id=sprint_id,
+            status=final_status,
+            summary=summary,
+            error=error_msg,
+        )
+    except Exception:
+        logger.exception("Failed to send completion webhook for sprint %s", sprint_id)
+
+
+@click.group()
+def cli() -> None:
+    """ResearchLoop sprint runner - executes inside HPC jobs."""
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
+    )
+
+
+@cli.command()
+@click.option("--sprint-id", required=True, help="Unique sprint identifier")
+@click.option("--sprint-dir", required=True, help="Working directory for this sprint")
+@click.option("--claude-md", required=True, help="Path to the study's CLAUDE.md")
+@click.option("--idea", required=True, help="Research idea / prompt for this sprint")
+@click.option(
+    "--orchestrator-url", required=True, help="Base URL of the orchestrator API"
+)
+@click.option(
+    "--shared-secret", required=True, help="Shared secret for orchestrator auth"
+)
+@click.option(
+    "--red-team-rounds",
+    default=3,
+    show_default=True,
+    help="Maximum number of red-team / fix rounds",
+)
+@click.option(
+    "--claude-command",
+    default="claude --dangerously-skip-permissions",
+    show_default=True,
+    help="Command to invoke Claude CLI",
+)
+def run(
+    sprint_id: str,
+    sprint_dir: str,
+    claude_md: str,
+    idea: str,
+    orchestrator_url: str,
+    shared_secret: str,
+    red_team_rounds: int,
+    claude_command: str,
+) -> None:
+    """Run the full research sprint pipeline."""
+    logger.info(
+        "Starting sprint %s in %s (red-team rounds: %d)",
+        sprint_id,
+        sprint_dir,
+        red_team_rounds,
+    )
+    asyncio.run(
+        _run_pipeline(
+            sprint_id=sprint_id,
+            sprint_dir=sprint_dir,
+            claude_md=claude_md,
+            idea=idea,
+            orchestrator_url=orchestrator_url,
+            shared_secret=shared_secret,
+            red_team_rounds=red_team_rounds,
+            claude_command=claude_command,
+        )
+    )
+    logger.info("Sprint %s finished.", sprint_id)
+
+
+def main() -> None:
+    """Package entry point for ``researchloop-runner``."""
+    cli()

researchloop/runner/pipeline.py

@@ -0,0 +1,272 @@
+"""Sub-agent pipeline orchestration.
+
+Each step invokes the Claude CLI as a subprocess and writes progress into
+``.researchloop/status.json`` so that the orchestrator can track liveness.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from researchloop.runner.claude import render_template, run_claude
+from researchloop.runner.upload import send_heartbeat
+
+logger = logging.getLogger(__name__)
+
+# Step labels (indexed from 1).
+_STEP_LABELS: list[str] = [
+    "research",
+    "red_team",
+    "report",
+    "summarize",
+]
+_TOTAL_STEPS = len(_STEP_LABELS) + 1  # +1 because red_team counts as 2 (loop)
+
+
+class Pipeline:
+    """Runs the full research sprint pipeline inside an HPC job."""
+
+    def __init__(
+        self,
+        sprint_id: str,
+        sprint_dir: str,
+        claude_md: str,
+        idea: str,
+        orchestrator_url: str,
+        shared_secret: str,
+        red_team_rounds: int = 3,
+        claude_command: str = "claude --dangerously-skip-permissions",
+    ) -> None:
+        self.sprint_id = sprint_id
+        self.sprint_dir = sprint_dir
+        self.claude_md = claude_md
+        self.idea = idea
+        self.orchestrator_url = orchestrator_url
+        self.shared_secret = shared_secret
+        self.red_team_rounds = red_team_rounds
+        self.claude_command = claude_command
+
+        self._started_at = datetime.now(timezone.utc).isoformat()
+        self._status_path = Path(sprint_dir) / ".researchloop" / "status.json"
+        self._heartbeat_task: asyncio.Task[None] | None = None
+        self._session_id: str | None = None
+
+    # ------------------------------------------------------------------
+    # Public interface
+    # ------------------------------------------------------------------
+
+    async def run(self) -> str | None:
+        """Execute the full pipeline and return the summary text (or None)."""
+        self._start_heartbeat()
+
+        # Read study context from CLAUDE.md if available.
+        study_context = ""
+        claude_md_path = Path(self.claude_md)
+        if claude_md_path.exists():
+            study_context = claude_md_path.read_text(encoding="utf-8")
+
+        # Step 1 - Research
+        await self._update_status("research", step=1)
+        research_prompt = render_template(
+            "research_sprint.md.j2",
+            study_context=study_context,
+            idea=self.idea,
+            sprint_dir=self.sprint_dir,
+        )
+        output, self._session_id = await run_claude(
+            prompt=research_prompt,
+            working_dir=self.sprint_dir,
+            claude_md=self.claude_md,
+            claude_command=self.claude_command,
+        )
+        logger.info("Research step complete (%d chars output)", len(output))
+
+        # Step 2 - Red-team / fix loop
+        await self._update_status("red_team", step=2)
+        for round_num in range(1, self.red_team_rounds + 1):
+            substep = f"round_{round_num}"
+            await self._update_status("red_team", step=2, substep=substep)
+
+            # Run the red-team critique.
+            rt_prompt = render_template(
+                "red_team.md.j2",
+                idea=self.idea,
+                round_number=round_num,
+                max_rounds=self.red_team_rounds,
+            )
+            rt_output, self._session_id = await run_claude(
+                prompt=rt_prompt,
+                working_dir=self.sprint_dir,
+                claude_md=self.claude_md,
+                session_id=self._session_id,
+                claude_command=self.claude_command,
+            )
+            logger.info(
+                "Red-team round %d complete (%d chars)", round_num, len(rt_output)
+            )
+
+            # Check whether the red-team found critical issues.
+            rt_file = Path(self.sprint_dir) / f"red_team_round_{round_num}.md"
+            if rt_file.exists():
+                content = rt_file.read_text(encoding="utf-8")
+                if "NO CRITICAL ISSUES" in content:
+                    logger.info(
+                        "Red-team round %d: no critical issues, stopping loop.",
+                        round_num,
+                    )
+                    break
+
+            # Run fix step for this round.
+            await self._update_status("red_team", step=2, substep=f"fix_{round_num}")
+            fix_prompt = render_template(
+                "fix_issues.md.j2",
+                round_number=round_num,
+            )
+            fix_output, self._session_id = await run_claude(
+                prompt=fix_prompt,
+                working_dir=self.sprint_dir,
+                claude_md=self.claude_md,
+                session_id=self._session_id,
+                claude_command=self.claude_command,
+            )
+            logger.info("Fix round %d complete (%d chars)", round_num, len(fix_output))
+
+        # Step 3 - Report
+        await self._update_status("report", step=3)
+        report_prompt = render_template(
+            "report.md.j2",
+            idea=self.idea,
+        )
+        report_output, self._session_id = await run_claude(
+            prompt=report_prompt,
+            working_dir=self.sprint_dir,
+            claude_md=self.claude_md,
+            session_id=self._session_id,
+        )
+        logger.info("Report step complete (%d chars)", len(report_output))
+
+        # Step 4 - Summarize
+        await self._update_status("summarize", step=4)
+        summary_prompt = render_template("summarizer.md.j2")
+        summary_output, self._session_id = await run_claude(
+            prompt=summary_prompt,
+            working_dir=self.sprint_dir,
+            claude_md=self.claude_md,
+            session_id=self._session_id,
+        )
+        logger.info("Summary step complete (%d chars)", len(summary_output))
+
+        # Read summary.txt written by the summarizer agent.
+        summary_path = Path(self.sprint_dir) / "summary.txt"
+        summary: str | None = None
+        if summary_path.exists():
+            summary = summary_path.read_text(encoding="utf-8").strip()
+
+        await self._update_status("completed", step=_TOTAL_STEPS)
+        return summary
+
+    async def stop(self) -> None:
+        """Clean up background tasks."""
+        await self._stop_heartbeat()
+
+    # ------------------------------------------------------------------
+    # Status tracking
+    # ------------------------------------------------------------------
+
+    async def _update_status(
+        self,
+        status: str,
+        step: int = 0,
+        substep: str | None = None,
+        error: str | None = None,
+    ) -> None:
+        """Write the current status to ``.researchloop/status.json``."""
+        data: dict[str, Any] = {
+            "sprint_id": self.sprint_id,
+            "status": status,
+            "step": step,
+            "total_steps": _TOTAL_STEPS,
+            "substep": substep,
+            "heartbeat": datetime.now(timezone.utc).isoformat(),
+            "started_at": self._started_at,
+            "error": error,
+        }
+        self._status_path.parent.mkdir(parents=True, exist_ok=True)
+        self._status_path.write_text(
+            json.dumps(data, indent=2) + "\n", encoding="utf-8"
+        )
+        logger.info("Status updated: %s step=%d substep=%s", status, step, substep)
+
+        # Best-effort heartbeat to orchestrator.
+        try:
+            await send_heartbeat(
+                orchestrator_url=self.orchestrator_url,
+                shared_secret=self.shared_secret,
+                sprint_id=self.sprint_id,
+                status=status,
+                step=step,
+            )
+        except Exception:
+            logger.debug("Heartbeat POST failed (non-fatal)", exc_info=True)
+
+    # ------------------------------------------------------------------
+    # Heartbeat background task
+    # ------------------------------------------------------------------
+
+    def _start_heartbeat(self) -> None:
+        """Start a background task that updates status.json every 60 seconds."""
+        if self._heartbeat_task is not None and not self._heartbeat_task.done():
+            return
+        self._heartbeat_task = asyncio.create_task(
+            self._heartbeat_loop(), name="heartbeat"
+        )
+        logger.info("Heartbeat background task started.")
+
+    async def _heartbeat_loop(self) -> None:
+        """Periodically refresh the heartbeat timestamp."""
+        try:
+            while True:
+                await asyncio.sleep(60)
+                # Re-read current status to update only the heartbeat field.
+                if self._status_path.exists():
+                    try:
+                        data = json.loads(self._status_path.read_text(encoding="utf-8"))
+                    except (json.JSONDecodeError, OSError):
+                        data = {}
+                else:
+                    data = {}
+
+                data["heartbeat"] = datetime.now(timezone.utc).isoformat()
+                self._status_path.write_text(
+                    json.dumps(data, indent=2) + "\n", encoding="utf-8"
+                )
+
+                # Also ping the orchestrator.
+                try:
+                    await send_heartbeat(
+                        orchestrator_url=self.orchestrator_url,
+                        shared_secret=self.shared_secret,
+                        sprint_id=self.sprint_id,
+                        status=data.get("status", "running"),
+                        step=data.get("step", 0),
+                    )
+                except Exception:
+                    logger.debug("Heartbeat POST failed (non-fatal)", exc_info=True)
+        except asyncio.CancelledError:
+            return
+
+    async def _stop_heartbeat(self) -> None:
+        """Cancel the heartbeat background task."""
+        if self._heartbeat_task is not None:
+            self._heartbeat_task.cancel()
+            try:
+                await self._heartbeat_task
+            except asyncio.CancelledError:
+                pass
+            self._heartbeat_task = None
+            logger.info("Heartbeat background task stopped.")

researchloop/runner/templates/fix_issues.md.j2

@@ -0,0 +1,11 @@
+You are fixing issues identified by a red-team review.
+
+## Red Team Feedback
+Review the file: red_team_round_{{ round_number }}.md
+
+## Instructions
+- Address ALL critical issues identified in the red-team review
+- Fix code bugs, improve methodology, add missing controls
+- Document what you changed and why in fixes_round_{{ round_number }}.md
+- Do not break existing working functionality
+- Update progress.md with what you're fixing and the results

researchloop/runner/templates/idea_generator.md.j2

@@ -0,0 +1,16 @@
+You are generating the next research idea for an auto-loop sprint.
+
+## Study Context
+{{ study_context }}
+
+## Previous Sprint Summaries
+{% for sprint in previous_sprints %}
+### {{ sprint.id }}
+{{ sprint.summary }}
+{% endfor %}
+
+## Instructions
+- Based on the study context and previous results, propose the next research idea
+- Write a single clear, actionable research idea (1-2 sentences)
+- Focus on: unexplored angles, following up promising leads, addressing gaps
+- Output ONLY the idea text, nothing else

researchloop/runner/templates/red_team.md.j2

@@ -0,0 +1,15 @@
+You are a critical reviewer red-teaming a research sprint.
+
+## Research Idea
+{{ idea }}
+
+## Round {{ round_number }} of {{ max_rounds }}
+
+## Instructions
+- Review ALL code and analysis in the current directory
+- Look for: bugs, methodological flaws, incorrect assumptions, missing controls, statistical errors, data leakage, unfair comparisons
+- Be thorough but fair — focus on issues that would invalidate conclusions
+- Write your critique to red_team_round_{{ round_number }}.md
+- End with a section "## Critical Issues" listing only issues that MUST be fixed
+- If no critical issues remain, write "NO CRITICAL ISSUES" in that section
+- Update progress.md with what you're reviewing and what you found

researchloop/runner/templates/report.md.j2

@@ -0,0 +1,31 @@
+You are writing a final research report.
+
+## Research Idea
+{{ idea }}
+
+## Instructions
+
+### 1. Write report.md
+Read all files in the sprint directory and write report.md. Be thorough but direct and readable — no filler, no hedging, no restating the obvious. Structure:
+
+- **Objective**: What we set out to test, in 1-2 sentences
+- **Methodology**: What we did, concisely. Include enough detail to reproduce
+- **Results**: The findings, with references to figures/tables. Lead with the key numbers
+- **Red-team feedback**: What was challenged and how it was addressed
+- **Conclusions**: What we learned. Be concrete
+- **Limitations**: What this doesn't tell us
+
+Write for a technical reader who values their time. Every sentence should earn its place.
+
+### 2. Generate a PDF report
+After writing report.md, create `report.pdf` in the current directory using Python. Write a script `generate_report_pdf.py` that:
+- Uses matplotlib (via `matplotlib.backends.backend_pdf.PdfPages`)
+- Renders the report as a formatted research paper with:
+  - Title, date, and sprint ID
+  - Section headings
+  - Body text with word wrapping
+  - All relevant plots from results/ embedded inline at appropriate points
+  - Tables where the data warrants it
+- The PDF should look like a proper research report
+
+If matplotlib is not available or PDF generation fails, skip this step — report.md is the primary output.

researchloop/runner/templates/research_sprint.md.j2

@@ -0,0 +1,51 @@
+You are a research assistant conducting a focused research sprint.
+
+## Study Context
+{{ study_context }}
+
+## Research Idea
+{{ idea }}
+
+## Instructions
+- Work in the current directory: {{ sprint_dir }}
+- Create well-organized code and analysis files
+- Document your methodology and findings
+- Save key results to results/ subdirectory
+- Write a brief findings.md summarizing what you discovered
+
+## Script Output Logging
+When running Python scripts or any long-running commands, always pipe output through `tee` so it's both visible to you AND saved to `output.log`:
+
+```bash
+python train.py 2>&1 | tee -a output.log
+```
+
+This lets the team monitor script output remotely. Always use `tee -a` (append mode) so all runs accumulate in the same log file. Do this for every script execution, training run, or evaluation.
+
+## Progress Log
+Maintain a file called `progress.md` in the sprint directory. Update it regularly as you work — it's how the team monitors your progress remotely. Keep it concise and current:
+
+- Start with your plan of attack (numbered steps)
+- Update it each time you finish a meaningful chunk of work or change direction
+- Include: what you just did, what you found, what you're doing next
+- Note any problems you hit and how you resolved them
+- When running long computations, update before and after with expected vs actual results
+
+Example format:
+```
+## Plan
+1. Set up baseline model
+2. Run experiments with X
+3. Analyze results
+
+## Log
+[08:30] Starting: setting up baseline model
+[08:45] Baseline ready. Training config: lr=3e-4, batch=1024, 20M samples
+[08:46] Kicked off training run. Expecting ~30 min.
+[09:20] Baseline done. F1=0.93, MCC=0.79. Moving to experiment X.
+[09:21] Trying X with parameters A=1, B=2
+[10:05] X finished. F1=0.95 (+0.02). Improvement is real. Trying variant...
+[10:50] Hit OOM with larger config. Reducing batch size to 512.
+```
+
+Focus on depth over breadth. Be rigorous and document assumptions.

researchloop/runner/templates/summarizer.md.j2

@@ -0,0 +1,7 @@
+You are writing a brief summary of a completed research sprint.
+
+## Instructions
+- Read report.md and the sprint directory contents
+- Write a 2-3 sentence summary to summary.txt
+- Format: what was investigated, key finding, confidence level
+- This will be sent as a notification, so be concise and informative