researchloop 0.1.0__py3-none-any.whl

researchloop/db/migrations.py

@@ -0,0 +1,172 @@
+"""Schema creation and index definitions for the researchloop database."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .database import Database
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS studies (
+    name TEXT PRIMARY KEY,
+    cluster TEXT NOT NULL,
+    description TEXT,
+    claude_md_path TEXT,
+    sprints_dir TEXT NOT NULL,
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    config_json TEXT  -- full StudyConfig as JSON
+);
+
+CREATE TABLE IF NOT EXISTS sprints (
+    id TEXT PRIMARY KEY,  -- e.g. "sp-a3f7b2"
+    study_name TEXT NOT NULL REFERENCES studies(name),
+    idea TEXT,
+    status TEXT NOT NULL DEFAULT 'pending',
+    job_id TEXT,  -- SLURM/SGE job ID
+    directory TEXT,  -- full path on cluster
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    started_at TEXT,
+    completed_at TEXT,
+    error TEXT,
+    summary TEXT,
+    session_id TEXT,  -- claude session ID for --resume
+    webhook_token TEXT,  -- per-sprint token for webhook auth
+    loop_id TEXT,  -- auto-loop ID if part of a loop
+    metadata_json TEXT,
+    FOREIGN KEY (study_name) REFERENCES studies(name)
+);
+
+CREATE TABLE IF NOT EXISTS auto_loops (
+    id TEXT PRIMARY KEY,
+    study_name TEXT NOT NULL,
+    total_count INTEGER NOT NULL,
+    completed_count INTEGER NOT NULL DEFAULT 0,
+    current_sprint_id TEXT,
+    status TEXT NOT NULL DEFAULT 'running',  -- running, completed, stopped, failed
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    stopped_at TEXT,
+    metadata_json TEXT,
+    FOREIGN KEY (study_name) REFERENCES studies(name)
+);
+
+CREATE TABLE IF NOT EXISTS artifacts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    sprint_id TEXT NOT NULL,
+    filename TEXT NOT NULL,
+    path TEXT NOT NULL,  -- local storage path
+    size INTEGER,
+    content_type TEXT,
+    uploaded_at TEXT NOT NULL DEFAULT (datetime('now')),
+    FOREIGN KEY (sprint_id) REFERENCES sprints(id)
+);
+
+CREATE TABLE IF NOT EXISTS slack_sessions (
+    thread_ts TEXT PRIMARY KEY,
+    sprint_id TEXT,
+    session_id TEXT,  -- claude --resume session ID
+    study_name TEXT,
+    messages_json TEXT,  -- JSON array of {role, text}
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    FOREIGN KEY (sprint_id) REFERENCES sprints(id)
+);
+
+CREATE TABLE IF NOT EXISTS events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    sprint_id TEXT,
+    event_type TEXT NOT NULL,
+    data_json TEXT,
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    FOREIGN KEY (sprint_id) REFERENCES sprints(id)
+);
+
+CREATE TABLE IF NOT EXISTS settings (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+);
+"""
+
+INDEXES_SQL = """
+CREATE INDEX IF NOT EXISTS idx_sprints_study_name ON sprints(study_name);
+CREATE INDEX IF NOT EXISTS idx_sprints_status ON sprints(status);
+CREATE INDEX IF NOT EXISTS idx_events_sprint_id ON events(sprint_id);
+CREATE INDEX IF NOT EXISTS idx_artifacts_sprint_id ON artifacts(sprint_id);
+"""
+
+
+async def _add_column_if_missing(
+    db: Database,
+    table: str,
+    column: str,
+    col_type: str = "TEXT",
+) -> None:
+    """Add a column to a table if it doesn't exist."""
+    assert db._conn is not None
+    cursor = await db._conn.execute(f"PRAGMA table_info({table})")
+    rows = await cursor.fetchall()
+    existing = {row[1] for row in rows}
+    if column not in existing:
+        await db._conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {col_type}")
+
+
+async def run_migrations(db: Database) -> None:
+    """Create all tables and indexes if they do not already exist."""
+    assert db._conn is not None, "Database must be connected before running migrations"
+    await db._conn.executescript(SCHEMA_SQL + INDEXES_SQL)
+
+    # Incremental column migrations for existing databases.
+    await _add_column_if_missing(db, "sprints", "webhook_token", "TEXT")
+    await _add_column_if_missing(db, "sprints", "loop_id", "TEXT")
+    await _add_column_if_missing(db, "auto_loops", "metadata_json", "TEXT")
+    await _add_column_if_missing(db, "slack_sessions", "messages_json", "TEXT")
+
+    # Make idea column nullable — already applied, skip on future runs.
+    # The migration checks if the column is still NOT NULL before acting.
+    await _make_idea_nullable(db)
+
+    await db._conn.commit()
+
+
+async def _make_idea_nullable(db: Database) -> None:
+    """Remove NOT NULL from sprints.idea if needed."""
+    assert db._conn is not None
+
+    # Recover from a previously failed migration.
+    tables = await db._conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
+    table_names = {r[0] for r in await tables.fetchall()}
+    if "_sprints_old" in table_names:
+        await db._conn.execute("PRAGMA foreign_keys=OFF")
+        if "sprints" not in table_names:
+            await db._conn.execute("ALTER TABLE _sprints_old RENAME TO sprints")
+        else:
+            # Keep whichever has more rows.
+            old_cursor = await db._conn.execute("SELECT count(*) FROM _sprints_old")
+            old_n = await old_cursor.fetchone()
+            new_cursor = await db._conn.execute("SELECT count(*) FROM sprints")
+            new_n = await new_cursor.fetchone()
+            old_count = old_n[0] if old_n else 0
+            new_count = new_n[0] if new_n else 0
+            if old_count > new_count:
+                await db._conn.execute("DROP TABLE sprints")
+                await db._conn.execute("ALTER TABLE _sprints_old RENAME TO sprints")
+            else:
+                await db._conn.execute("DROP TABLE _sprints_old")
+        await db._conn.execute("PRAGMA foreign_keys=ON")
+
+    cursor = await db._conn.execute("PRAGMA table_info(sprints)")
+    rows = await cursor.fetchall()
+    for row in rows:
+        if row[1] == "idea" and row[3] == 1:  # notnull=1
+            # Disable FK checks during rebuild.
+            await db._conn.execute("PRAGMA foreign_keys=OFF")
+            # Rebuild: rename → create new → copy → drop old.
+            await db._conn.execute("ALTER TABLE sprints RENAME TO _sprints_old")
+            for stmt in SCHEMA_SQL.strip().split(";"):
+                stmt = stmt.strip()
+                if stmt.startswith("CREATE TABLE IF NOT EXISTS sprints"):
+                    await db._conn.execute(stmt)
+                    break
+            await db._conn.execute("INSERT INTO sprints SELECT * FROM _sprints_old")
+            await db._conn.execute("DROP TABLE _sprints_old")
+            await db._conn.execute("PRAGMA foreign_keys=ON")
+            return

researchloop/db/queries.py

@@ -0,0 +1,351 @@
+"""Async query functions for the researchloop database.
+
+Every public function takes a :class:`Database` instance as its first
+argument and returns plain ``dict`` rows (converted from ``aiosqlite.Row``).
+All queries use parameterized placeholders to prevent SQL injection.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .database import Database
+
+# ---------------------------------------------------------------------------
+# Column whitelists — prevent SQL injection via dynamic column names
+# ---------------------------------------------------------------------------
+
+_STUDY_COLUMNS: frozenset[str] = frozenset(
+    {
+        "name",
+        "cluster",
+        "description",
+        "claude_md_path",
+        "sprints_dir",
+        "created_at",
+        "config_json",
+    }
+)
+
+_SPRINT_COLUMNS: frozenset[str] = frozenset(
+    {
+        "id",
+        "study_name",
+        "idea",
+        "status",
+        "job_id",
+        "directory",
+        "created_at",
+        "started_at",
+        "completed_at",
+        "error",
+        "summary",
+        "session_id",
+        "webhook_token",
+        "loop_id",
+        "metadata_json",
+    }
+)
+
+_AUTO_LOOP_COLUMNS: frozenset[str] = frozenset(
+    {
+        "id",
+        "study_name",
+        "total_count",
+        "completed_count",
+        "current_sprint_id",
+        "status",
+        "created_at",
+        "stopped_at",
+        "metadata_json",
+    }
+)
+
+
+def _validate_columns(
+    kwargs: dict[str, Any], allowed: frozenset[str], entity: str
+) -> None:
+    """Raise ValueError if any key in *kwargs* is not in *allowed*."""
+    invalid = set(kwargs.keys()) - allowed
+    if invalid:
+        raise ValueError(
+            f"Invalid column(s) for {entity}: {', '.join(sorted(invalid))}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Studies
+# ---------------------------------------------------------------------------
+
+
+async def create_study(
+    db: Database,
+    name: str,
+    cluster: str,
+    description: str | None,
+    claude_md_path: str | None,
+    sprints_dir: str,
+    config_json: str | None = None,
+) -> dict[str, Any]:
+    """Insert a new study and return it."""
+    await db.execute(
+        """
+        INSERT INTO studies
+            (name, cluster, description, claude_md_path,
+             sprints_dir, config_json)
+        VALUES (?, ?, ?, ?, ?, ?)
+        """,
+        (name, cluster, description, claude_md_path, sprints_dir, config_json),
+    )
+    return await get_study(db, name)  # type: ignore[return-value]
+
+
+async def get_study(db: Database, name: str) -> dict[str, Any] | None:
+    """Return a single study by name, or ``None``."""
+    return await db.fetch_one("SELECT * FROM studies WHERE name = ?", (name,))
+
+
+async def list_studies(db: Database) -> list[dict[str, Any]]:
+    """Return all studies ordered by creation time (newest first)."""
+    return await db.fetch_all("SELECT * FROM studies ORDER BY created_at DESC")
+
+
+async def update_study(db: Database, name: str, **kwargs: Any) -> dict[str, Any] | None:
+    """Update arbitrary columns on a study.  Returns the updated row."""
+    if not kwargs:
+        return await get_study(db, name)
+    _validate_columns(kwargs, _STUDY_COLUMNS, "study")
+    columns = ", ".join(f"{k} = ?" for k in kwargs)
+    values = list(kwargs.values())
+    values.append(name)
+    await db.execute(
+        f"UPDATE studies SET {columns} WHERE name = ?",
+        values,
+    )
+    return await get_study(db, name)
+
+
+# ---------------------------------------------------------------------------
+# Sprints
+# ---------------------------------------------------------------------------
+
+
+async def create_sprint(
+    db: Database,
+    id: str,
+    study_name: str,
+    idea: str | None = None,
+    directory: str | None = None,
+) -> dict[str, Any]:
+    """Insert a new sprint and return it."""
+    import secrets
+
+    webhook_token = secrets.token_hex(16)
+    await db.execute(
+        """
+        INSERT INTO sprints (id, study_name, idea, directory, webhook_token)
+        VALUES (?, ?, ?, ?, ?)
+        """,
+        (id, study_name, idea, directory, webhook_token),
+    )
+    return await get_sprint(db, id)  # type: ignore[return-value]
+
+
+async def get_sprint(db: Database, id: str) -> dict[str, Any] | None:
+    """Return a single sprint by id, or ``None``."""
+    return await db.fetch_one("SELECT * FROM sprints WHERE id = ?", (id,))
+
+
+async def list_sprints(
+    db: Database,
+    study_name: str | None = None,
+    status: str | None = None,
+    limit: int = 50,
+) -> list[dict[str, Any]]:
+    """Return sprints with optional filters, newest first."""
+    clauses: list[str] = []
+    params: list[Any] = []
+
+    if study_name is not None:
+        clauses.append("study_name = ?")
+        params.append(study_name)
+    if status is not None:
+        clauses.append("status = ?")
+        params.append(status)
+
+    where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
+    params.append(limit)
+
+    return await db.fetch_all(
+        f"SELECT * FROM sprints {where} ORDER BY created_at DESC LIMIT ?",
+        params,
+    )
+
+
+async def update_sprint(db: Database, id: str, **kwargs: Any) -> dict[str, Any] | None:
+    """Update arbitrary columns on a sprint.  Returns the updated row."""
+    if not kwargs:
+        return await get_sprint(db, id)
+    _validate_columns(kwargs, _SPRINT_COLUMNS, "sprint")
+    columns = ", ".join(f"{k} = ?" for k in kwargs)
+    values = list(kwargs.values())
+    values.append(id)
+    await db.execute(
+        f"UPDATE sprints SET {columns} WHERE id = ?",
+        values,
+    )
+    return await get_sprint(db, id)
+
+
+async def delete_sprint(db: Database, id: str) -> None:
+    """Delete a sprint and its related artifacts, events, and sessions."""
+    await db.execute("DELETE FROM artifacts WHERE sprint_id = ?", [id])
+    await db.execute("DELETE FROM events WHERE sprint_id = ?", [id])
+    await db.execute("DELETE FROM slack_sessions WHERE sprint_id = ?", [id])
+    await db.execute("DELETE FROM sprints WHERE id = ?", [id])
+
+
+async def get_active_sprints(db: Database) -> list[dict[str, Any]]:
+    """Return all sprints whose status is 'running'."""
+    return await db.fetch_all(
+        "SELECT * FROM sprints WHERE status = 'running' ORDER BY created_at DESC",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Auto-loops
+# ---------------------------------------------------------------------------
+
+
+async def create_auto_loop(
+    db: Database,
+    id: str,
+    study_name: str,
+    total_count: int,
+) -> dict[str, Any]:
+    """Insert a new auto-loop and return it."""
+    await db.execute(
+        """
+        INSERT INTO auto_loops (id, study_name, total_count)
+        VALUES (?, ?, ?)
+        """,
+        (id, study_name, total_count),
+    )
+    return await get_auto_loop(db, id)  # type: ignore[return-value]
+
+
+async def get_auto_loop(db: Database, id: str) -> dict[str, Any] | None:
+    """Return a single auto-loop by id, or ``None``."""
+    return await db.fetch_one("SELECT * FROM auto_loops WHERE id = ?", (id,))
+
+
+async def list_auto_loops(
+    db: Database,
+    study_name: str | None = None,
+) -> list[dict[str, Any]]:
+    """Return auto-loops, optionally filtered by study name."""
+    if study_name is not None:
+        return await db.fetch_all(
+            "SELECT * FROM auto_loops WHERE study_name = ? ORDER BY created_at DESC",
+            (study_name,),
+        )
+    return await db.fetch_all(
+        "SELECT * FROM auto_loops ORDER BY created_at DESC",
+    )
+
+
+async def update_auto_loop(
+    db: Database, id: str, **kwargs: Any
+) -> dict[str, Any] | None:
+    """Update arbitrary columns on an auto-loop.  Returns the updated row."""
+    if not kwargs:
+        return await get_auto_loop(db, id)
+    _validate_columns(kwargs, _AUTO_LOOP_COLUMNS, "auto_loop")
+    columns = ", ".join(f"{k} = ?" for k in kwargs)
+    values = list(kwargs.values())
+    values.append(id)
+    await db.execute(
+        f"UPDATE auto_loops SET {columns} WHERE id = ?",
+        values,
+    )
+    return await get_auto_loop(db, id)
+
+
+# ---------------------------------------------------------------------------
+# Artifacts
+# ---------------------------------------------------------------------------
+
+
+async def create_artifact(
+    db: Database,
+    sprint_id: str,
+    filename: str,
+    path: str,
+    size: int | None = None,
+    content_type: str | None = None,
+) -> dict[str, Any]:
+    """Insert a new artifact and return it."""
+    cursor = await db.execute(
+        """
+        INSERT INTO artifacts (sprint_id, filename, path, size, content_type)
+        VALUES (?, ?, ?, ?, ?)
+        """,
+        (sprint_id, filename, path, size, content_type),
+    )
+    row_id = cursor.lastrowid
+    return await get_artifact(db, row_id)  # type: ignore[return-value]
+
+
+async def list_artifacts(db: Database, sprint_id: str) -> list[dict[str, Any]]:
+    """Return all artifacts for a given sprint."""
+    return await db.fetch_all(
+        "SELECT * FROM artifacts WHERE sprint_id = ? ORDER BY uploaded_at DESC",
+        (sprint_id,),
+    )
+
+
+async def get_artifact(db: Database, id: int) -> dict[str, Any] | None:
+    """Return a single artifact by id, or ``None``."""
+    return await db.fetch_one("SELECT * FROM artifacts WHERE id = ?", (id,))
+
+
+# ---------------------------------------------------------------------------
+# Events
+# ---------------------------------------------------------------------------
+
+
+async def create_event(
+    db: Database,
+    sprint_id: str | None,
+    event_type: str,
+    data_json: str | None = None,
+) -> dict[str, Any]:
+    """Insert a new event and return it."""
+    cursor = await db.execute(
+        """
+        INSERT INTO events (sprint_id, event_type, data_json)
+        VALUES (?, ?, ?)
+        """,
+        (sprint_id, event_type, data_json),
+    )
+    row_id = cursor.lastrowid
+    result = await db.fetch_one("SELECT * FROM events WHERE id = ?", (row_id,))
+    return result  # type: ignore[return-value]
+
+
+async def list_events(
+    db: Database,
+    sprint_id: str | None = None,
+    limit: int = 100,
+) -> list[dict[str, Any]]:
+    """Return events, optionally filtered by sprint id, newest first."""
+    if sprint_id is not None:
+        return await db.fetch_all(
+            "SELECT * FROM events WHERE sprint_id = ? ORDER BY created_at DESC LIMIT ?",
+            (sprint_id, limit),
+        )
+    return await db.fetch_all(
+        "SELECT * FROM events ORDER BY created_at DESC LIMIT ?",
+        (limit,),
+    )

researchloop/runner/__init__.py

@@ -0,0 +1 @@
+"""Sprint runner - executes inside SLURM/SGE jobs on HPC clusters."""

researchloop/runner/claude.py

@@ -0,0 +1,169 @@
+"""Claude CLI wrapper for running sub-agent prompts.
+
+Invokes ``claude -p "..." --output-format json`` as a subprocess and
+parses the structured output.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+from pathlib import Path
+
+import jinja2
+
+logger = logging.getLogger(__name__)
+
+# Template directory lives alongside this module.
+_TEMPLATES_DIR = Path(__file__).resolve().parent / "templates"
+
+_jinja_env = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(str(_TEMPLATES_DIR)),
+    keep_trailing_newline=True,
+    undefined=jinja2.StrictUndefined,
+)
+
+
+def render_template(template_name: str, **kwargs: object) -> str:
+    """Load a Jinja2 template from the ``templates/`` directory and render it."""
+    template = _jinja_env.get_template(template_name)
+    return template.render(**kwargs)
+
+
+async def run_claude(
+    prompt: str,
+    working_dir: str,
+    claude_md: str | None = None,
+    session_id: str | None = None,
+    timeout: int = 3600,
+    claude_command: str = "claude --dangerously-skip-permissions",
+) -> tuple[str, str | None]:
+    """Run the Claude CLI with the given prompt.
+
+    Parameters
+    ----------
+    prompt:
+        The full prompt text to send.
+    working_dir:
+        The working directory in which Claude should operate.
+    claude_md:
+        Optional path to a CLAUDE.md file.  When provided the
+        ``CLAUDE_MD`` environment variable is set so the CLI picks it up.
+    session_id:
+        If continuing a conversation, the session ID from a previous call.
+    timeout:
+        Maximum seconds to wait for the process (default 3600 = 1 hour).
+
+    Returns
+    -------
+    tuple[str, str | None]
+        ``(output_text, session_id)`` parsed from the JSON output.
+    """
+    # Split the command string to support things like
+    # "singularity exec img.sif claude --dangerously-skip-permissions"
+    base_cmd = claude_command.split()
+    cmd: list[str] = [
+        *base_cmd,
+        "-p",
+        prompt,
+        "--output-format",
+        "json",
+    ]
+
+    if session_id:
+        cmd.extend(["--resume", session_id])
+
+    # Build environment with optional CLAUDE_MD override.
+    env = os.environ.copy()
+    if claude_md:
+        env["CLAUDE_MD"] = claude_md
+
+    logger.info(
+        "Running Claude CLI (session=%s, timeout=%ds, cwd=%s)",
+        session_id or "new",
+        timeout,
+        working_dir,
+    )
+    logger.debug("Prompt length: %d chars", len(prompt))
+
+    try:
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            cwd=working_dir,
+            env=env,
+        )
+
+        stdout_bytes, stderr_bytes = await asyncio.wait_for(
+            process.communicate(), timeout=timeout
+        )
+    except asyncio.TimeoutError:
+        logger.error("Claude CLI timed out after %ds", timeout)
+        # Attempt to kill the hung process.
+        try:
+            process.kill()  # type: ignore[possibly-undefined]
+        except ProcessLookupError:
+            pass
+        raise TimeoutError(f"Claude CLI did not finish within {timeout} seconds")
+
+    stdout = stdout_bytes.decode("utf-8", errors="replace")
+    stderr = stderr_bytes.decode("utf-8", errors="replace")
+
+    if process.returncode != 0:
+        logger.error(
+            "Claude CLI exited with code %d\nstderr: %s",
+            process.returncode,
+            stderr[:2000],
+        )
+        raise RuntimeError(
+            f"Claude CLI exited with code {process.returncode}: {stderr[:500]}"
+        )
+
+    if stderr:
+        logger.debug("Claude CLI stderr: %s", stderr[:1000])
+
+    # Parse JSON output.
+    output_text, new_session_id = _parse_output(stdout)
+    logger.info(
+        "Claude CLI finished: %d chars output, session=%s",
+        len(output_text),
+        new_session_id or "none",
+    )
+    return output_text, new_session_id
+
+
+def _parse_output(raw: str) -> tuple[str, str | None]:
+    """Parse the JSON output from ``claude --output-format json``.
+
+    The CLI emits a JSON object with at least a ``result`` field.
+    The ``session_id`` field may or may not be present.
+    """
+    raw = raw.strip()
+    if not raw:
+        return "", None
+
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError:
+        # If the output is not valid JSON, treat the whole thing as plain text.
+        logger.warning("Could not parse Claude CLI output as JSON; using raw text")
+        return raw, None
+
+    # The CLI may nest the text in different fields depending on version.
+    output_text: str = ""
+    if isinstance(data, dict):
+        output_text = (
+            data.get("result", "")
+            or data.get("text", "")
+            or data.get("content", "")
+            or ""
+        )
+        session_id = data.get("session_id")
+    else:
+        output_text = str(data)
+        session_id = None
+
+    return output_text, session_id