outerloop-science 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev4__py3-none-any.whl

outerloop/init.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 import argparse
 import getpass
 import json
+import logging
 import os
 import shutil
 import sys
@@ -33,6 +34,8 @@ from outerloop.cli import ENV_FILE
 from outerloop.image import ensure_image
 from outerloop.paths import write_private
 
+log = logging.getLogger(__name__)
+
 CONFIG_DIR = ENV_FILE.parent
 DEFAULT_PAT_FILE = CONFIG_DIR / "bot_pat"
 API = "https://api.github.com"
@@ -69,6 +72,9 @@ def render_env(
             lines.append(f"OUTERLOOP_ACCOUNT={a.account}")
         if a.partition:  # optional: unset lets Slurm pick its default partition
             lines.append(f"OUTERLOOP_PARTITION={a.partition}")
+        # the checkout stays put until the operator upgrades; `release` follows
+        # the release tags, `main` every merge (docs/install.md)
+        lines.append("OUTERLOOP_AUTO_UPDATE=off")
     if a.image:
         lines.append(f"OUTERLOOP_IMAGE={a.image}")
     elif a.uncontained:
@@ -356,8 +362,29 @@ def _owner_type(owner: str) -> str:
     try:
         with urllib.request.urlopen(req, timeout=15) as resp:
             return str(json.loads(resp.read()).get("type", ""))
-    except Exception:
+    except Exception as exc:
+        log.warning("could not look up the account type of %s: %s", owner, exc)
+        return ""
+
+
+def _app_owner_mismatch_note(conversion: dict[str, Any], expected_owner: str, target: str) -> str:
+    """When the App landed under a different account than intended — GitHub's
+    fallback when someone who is not an org owner tries to create an org-owned
+    App — explain the path that still works, since a personal App does not
+    install on an org repo by default. Empty when the App landed as intended."""
+    actual = str((conversion.get("owner") or {}).get("login", "")).strip()
+    if not actual or actual.casefold() == expected_owner.casefold():
         return ""
+    org = target.split("/")[0]
+    return (
+        f"  note: the App was created under '{actual}', not '{expected_owner}'. Creating an\n"
+        f"  App owned by an organization needs org-owner rights, so GitHub made a personal\n"
+        f"  one — which will not install on {target} by default. To use it there anyway:\n"
+        f"  make the App public (its Settings > 'Make public'), request its installation on\n"
+        f"  {target}, and have an owner of {org} approve it; then rerun\n"
+        f"  `outerloop init --force --github-app`. For lab ownership, you can later transfer\n"
+        f"  the App to {org} from its Advanced settings — the key keeps working."
+    )
 
 
 def _app_failure(answers: InitAnswers, slug: str, problem: str) -> int:
@@ -369,7 +396,9 @@ def _app_failure(answers: InitAnswers, slug: str, problem: str) -> int:
         f"  https://github.com/apps/{slug}/installations/new (or the App's page under\n"
         "  Settings > Installations > Configure) install it on that repository and accept\n"
         "  contents, issues and pull-request write access, then run\n"
-        "  `outerloop init --force --github-app` to re-check with these credentials.",
+        "  `outerloop init --force --github-app` to re-check with these credentials.\n"
+        "  If the App is under your personal account and the repository belongs to an\n"
+        "  organization, make the App public first and have an org owner approve the install.",
         file=sys.stderr,
     )
     return 1
@@ -442,6 +471,9 @@ def _github_app_setup(
     pem_path, app_json = appmanifest.save_app_creds(conversion, CONFIG_DIR)
     repo = answers.target.split("/", 1)[-1]
     print(f"  created App '{conversion['slug']}'; credentials in {app_json} and {pem_path} (0600)")
+    mismatch = _app_owner_mismatch_note(conversion, owner, answers.target)
+    if mismatch:
+        print(mismatch, file=sys.stderr)
     print()
     print(f"Step 2 of 3: install the App on {answers.target}.")
     print(f"  Open {appmanifest.install_url(conversion)}")
@@ -684,6 +716,17 @@ def main(argv: list[str] | None = None) -> int:
             if login:
                 write_private(env_path, render_env(answers, effective_pat, bot_login=login))
                 print(f"  posting as {login} (OUTERLOOP_BOT_LOGIN)")
+            else:
+                print(
+                    "  the token's login could not be read — set OUTERLOOP_BOT_LOGIN in "
+                    f"{env_path} before `outerloop start` (the tick skips a target without it)"
+                )
+        else:
+            print(
+                "  OUTERLOOP_BOT_LOGIN not recorded (the check did not pass) — rerun "
+                "`outerloop init --force` with network access, or set it in "
+                f"{env_path} (the tick skips a target without it)"
+            )
     else:
         print("  no PAT set — add OUTERLOOP_PAT_FILE before the agents can open PRs")
     _author_key_hint(answers)

outerloop/intake.py

@@ -13,11 +13,11 @@ from __future__ import annotations
 import logging
 from dataclasses import dataclass
 
-from outerloop.brief import MAX_TASK_CHARS, _cap, _fence
+from outerloop.brief import MAX_TASK_CHARS, cap, code_fence
 from outerloop.contract import Contract
 from outerloop.followup import QUALIFYING_ASSOCIATIONS
 from outerloop.github import is_own_login
-from outerloop.markers import has_label, has_marker, label_name, marker
+from outerloop.markers import has_label, has_marker, marker
 
 log = logging.getLogger(__name__)
 
@@ -29,9 +29,6 @@ RELEASE_MARKER = marker("claim-released")
 # failure must not claim/release (and comment) forever. Same idea as the
 # steward lane's MAX_STEWARD_ATTEMPTS.
 MAX_INTAKE_ATTEMPTS = 3
-# steward work orders carry this label; they are the STEWARD lane's,
-# never the solver's (a solver climb cannot touch env paths anyway)
-STEWARD_LABEL = label_name("steward")
 
 
 @dataclass(frozen=True)
@@ -119,8 +116,8 @@ def issue_hypothesis(task: IssueTask) -> str:
     The author passed the standing gate, so the REQUEST is legitimate; the
     fence marks where quoted text ends and the harness's authority resumes.
     """
-    quoted = _cap(f"{task.title}\n\n{task.body}".strip(), MAX_TASK_CHARS - 400)
-    fence = _fence(quoted)
+    quoted = cap(f"{task.title}\n\n{task.body}".strip(), MAX_TASK_CHARS - 400)
+    fence = code_fence(quoted)
     return (
         f"A maintainer (@{task.author}) opened issue #{task.number} requesting "
         f"work on the `{task.benchmark}` benchmark. Their request:\n"

outerloop/launchlog.py

@@ -0,0 +1,239 @@
+"""The per-run launch ledger: append-only JSON lines in the run directory, one
+record when a sleep's launches are submitted and one when each job's result
+comes back at the wake. It is what `history` reads and what labels a launch in
+the queue view (docs/design/session-watcher.md, "History"). Kernel-owned: the
+run directory is never the session's to write."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from outerloop.syscall import Launch, LaunchResult, launch_jobs
+
+LEDGER = "launches.jsonl"
+# generous: a run is depth_k launches x sleep_k sleeps x the array width, far below this
+MAX_LEDGER_BYTES = 4_000_000
+
+
+def _append(run_dir: Path, rows: list[dict[str, Any]]) -> None:
+    if not rows:
+        return
+    run_dir.mkdir(parents=True, exist_ok=True)
+    path = run_dir / LEDGER
+    # a crash mid-append leaves a torn last line; start on a fresh one so the
+    # torn line is the only record lost, never the next one too
+    torn = False
+    try:
+        with path.open("rb") as fh:
+            fh.seek(-1, 2)
+            torn = fh.read(1) != b"\n"
+    except OSError:
+        pass
+    with path.open("a", encoding="utf-8") as fh:
+        if torn:
+            fh.write("\n")
+        for row in rows:
+            fh.write(json.dumps(row, sort_keys=True) + "\n")
+
+
+def append_submitted(
+    run_dir: Path, *, sleep: int, launches: tuple[Launch, ...], job_ids: list[str], at: float
+) -> None:
+    """One record per launch of a sleep, with the job ids it fanned out to. The
+    ids are positional over `launch_jobs` order, exactly as the park recorded
+    them; a launch whose ids are missing (an older park) gets none. A launch
+    already recorded under this (sleep, name) is not written again: a run
+    re-parks the same sleep through a multi-stage gate, and the first record
+    is the one with the author's words."""
+    known = {
+        (int(row.get("sleep") or 0), str(row.get("name") or ""))
+        for row in read_ledger(run_dir)
+        if row.get("event") == "submitted"
+    }
+    # one id per launch (a sweep is one Slurm job array), or one per task for
+    # a park recorded when arrays were separate jobs
+    per_launch = len(job_ids) == len(launches)
+    rows: list[dict[str, Any]] = []
+    k = 0
+    for launch in launches:
+        n = 1 if per_launch else len(launch_jobs(launch))
+        ids = job_ids[k : k + n]
+        k += n
+        if (sleep, launch.name) in known:
+            continue
+        rows.append(
+            {
+                "event": "submitted",
+                "sleep": sleep,
+                "name": launch.name,
+                "why": launch.why,
+                "minutes": launch.minutes,
+                "array": launch.array,
+                "concurrency": launch.concurrency,
+                "job_ids": list(ids),
+                "at": at,
+            }
+        )
+    _append(run_dir, rows)
+
+
+def append_ended(
+    run_dir: Path,
+    *,
+    sleep: int,
+    results: tuple[LaunchResult, ...],
+    at: float,
+    elapsed_seconds: list[int | None] | None = None,
+) -> None:
+    """One record per job that came back at the wake, keyed to its sleep, with
+    how long it ran when the compute could say (aligned with `results`) and
+    the last line it printed — captured now, because a later launch with the
+    same name overwrites the job dir. A job already recorded as ended under
+    this (sleep, name) is not written again: a submitted park's wake and the
+    author's wake may both see the same jobs."""
+    known = {
+        (int(row.get("sleep") or 0), str(row.get("name") or ""))
+        for row in read_ledger(run_dir)
+        if row.get("event") == "ended"
+    }
+    _append(
+        run_dir,
+        [
+            {
+                "event": "ended",
+                "sleep": sleep,
+                "name": r.name,
+                "exit_code": r.exit_code,
+                "state": r.slurm_state,
+                "elapsed": (
+                    elapsed_seconds[i]
+                    if elapsed_seconds is not None and i < len(elapsed_seconds)
+                    else None
+                ),
+                "last_line": last_line(r.stdout_tail),
+                "at": at,
+            }
+            for i, r in enumerate(results)
+            if (sleep, r.name) not in known
+        ],
+    )
+
+
+def last_line(text: str, cap: int = 160) -> str:
+    """The last non-empty line of a job's output — its result line, as the
+    wake shows the author — on one line and bounded. "" when there is none."""
+    for line in reversed(text.splitlines()):
+        flat = " ".join(line.split())
+        if flat:
+            return flat[:cap]
+    return ""
+
+
+def read_ledger(run_dir: Path) -> list[dict[str, Any]]:
+    """Every record, oldest first; a malformed line is skipped, a missing file
+    is an empty history."""
+    try:
+        with (run_dir / LEDGER).open("rb") as fh:
+            raw = fh.read(MAX_LEDGER_BYTES)
+    except OSError:
+        return []
+    rows: list[dict[str, Any]] = []
+    for line in raw.decode("utf-8", "replace").splitlines():
+        try:
+            row = json.loads(line)
+        except ValueError:
+            continue
+        if isinstance(row, dict):
+            rows.append(row)
+    return rows
+
+
+def history(run_dir: Path) -> list[dict[str, Any]]:
+    """Submitted launches in order, each with the ended records of its jobs.
+    The identity is (sleep, name): a name is unique within one sleep only."""
+    rows = read_ledger(run_dir)
+    entries: dict[tuple[int, str], dict[str, Any]] = {}
+    for row in rows:
+        if row.get("event") != "submitted":
+            continue
+        key = (int(row.get("sleep") or 0), str(row.get("name") or ""))
+        entries[key] = {
+            "sleep": row.get("sleep"),
+            "name": row.get("name"),
+            "why": row.get("why", ""),
+            "minutes": row.get("minutes"),
+            "array": row.get("array", 1),
+            "concurrency": row.get("concurrency", 0),
+            "job_ids": list(row.get("job_ids") or []),
+            "submitted_at": row.get("at"),
+            "jobs": [],
+        }
+    for row in rows:
+        if row.get("event") != "ended":
+            continue
+        # an array member is `<name>.<i>`; the dot is outside the name alphabet
+        launch_name = str(row.get("name") or "").split(".", 1)[0]
+        entry = entries.get((int(row.get("sleep") or 0), launch_name))
+        if entry is not None:
+            entry["jobs"].append(
+                {
+                    "name": row.get("name"),
+                    "exit_code": row.get("exit_code"),
+                    "state": row.get("state", ""),
+                    "elapsed": row.get("elapsed"),
+                    "last_line": str(row.get("last_line") or ""),
+                    "ended_at": row.get("at"),
+                }
+            )
+    return list(entries.values())
+
+
+def experiments_rows(run_dir: Path) -> list[dict[str, Any]]:
+    """The pull request's experiments table: one row per job of every launch
+    this run — its sleep, name and why, how it ended and how long it ran, and
+    the last line it printed, all from the ledger (the job dir is overwritten
+    by a later launch of the same name). A job not back yet says so."""
+    rows: list[dict[str, Any]] = []
+    for entry in history(run_dir):
+        ended = {str(j.get("name")): j for j in entry["jobs"]}
+        array = int(entry.get("array") or 1)
+        name = str(entry.get("name") or "")
+        jobs = [name] if array <= 1 else [f"{name}.{k}" for k in range(array)]
+        for job in jobs:
+            j = ended.get(job)
+            rows.append(
+                {
+                    "sleep": entry.get("sleep"),
+                    "launch": name,
+                    "why": str(entry.get("why") or ""),
+                    "array": array,
+                    "concurrency": int(entry.get("concurrency") or 0),
+                    "job": job,
+                    "back": j is not None,
+                    "exit_code": j.get("exit_code") if j else None,
+                    "state": str(j.get("state") or "") if j else "",
+                    "elapsed": j.get("elapsed") if j else None,
+                    "result": str(j.get("last_line") or "") if j else "",
+                }
+            )
+    return rows
+
+
+def why_by_job(run_dir: Path) -> dict[str, dict[str, Any]]:
+    """Slurm job id -> the launch it belongs to (name, why, sleep): how the
+    queue view labels a launch job for every agent."""
+    out: dict[str, dict[str, Any]] = {}
+    for row in read_ledger(run_dir):
+        if row.get("event") != "submitted":
+            continue
+        for job_id in row.get("job_ids") or []:
+            out[str(job_id)] = {
+                "name": row.get("name", ""),
+                "why": row.get("why", ""),
+                "sleep": row.get("sleep"),
+                "array": int(row.get("array") or 1),
+                "concurrency": int(row.get("concurrency") or 0),
+            }
+    return out