labnotes 0.2.3__tar.gz → 0.2.4__tar.gz

{labnotes-0.2.3 → labnotes-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.2.3
+Version: 0.2.4
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

labnotes-0.2.4/src/labnotes/latest_log.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+
+def latest_log(folder: Path, stream: str) -> Path:
+    # A run that continues writes one log per invocation; follow the newest.
+    return max(folder.glob(f"{stream}-*.txt"), default=folder / f"{stream}.log")

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/models.py

@@ -67,3 +67,4 @@ class ExperimentConfig:
     setup: str | None = None
     teardown: str | None = None
     env: dict[str, str] = field(default_factory=dict)
+    continues: bool = False

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/run_output.py

@@ -5,6 +5,7 @@ from pathlib import Path
 
 from dirsql import DirSQL
 
+from labnotes.latest_log import latest_log
 from labnotes.load_run import load_run
 from labnotes.tables.read_log import STREAMS
 
@@ -19,6 +20,7 @@ async def run_output(
         return
     folder = root / str(run["folder"])
     offsets = dict.fromkeys(STREAMS, 0)
+    paths: dict[str, Path] = {}
     decoders = {
         stream: codecs.getincrementaldecoder("utf-8")("replace") for stream in STREAMS
     }
@@ -28,9 +30,12 @@ async def run_output(
     def read(streams: Iterable[str]) -> Iterator[str]:
         nonlocal last, mid_line
         for stream in streams:
-            path = folder / f"{stream}.log"
+            path = latest_log(folder, stream)
             if not path.exists():
                 continue
+            if paths.setdefault(stream, path) != path:
+                paths[stream] = path
+                offsets[stream] = 0
             with path.open("rb") as log:
                 log.seek(offsets[stream])
                 data = log.read()

labnotes-0.2.4/src/labnotes/runner/claim_run.py

@@ -0,0 +1,15 @@
+import fcntl
+import os
+from pathlib import Path
+
+LOCK_FILE = "run.lock"
+
+
+def claim_run(folder: Path) -> int | None:
+    lock = os.open(folder / LOCK_FILE, os.O_RDWR | os.O_CREAT, 0o644)
+    try:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except BlockingIOError:
+        os.close(lock)
+        return None
+    return lock

labnotes-0.2.4/src/labnotes/runner/continue_run.py

@@ -0,0 +1,21 @@
+from labnotes.runner.first_invocation import first_invocation
+
+# Per-invocation keys that a later invocation may leave unset.
+UNSET = {"args", "meta", "pid", "pid_created", "retry_of"}
+
+
+def continue_run(
+    previous: dict[str, object], run: dict[str, object], invocation: dict[str, object]
+) -> dict[str, object]:
+    invocations = previous.get("invocations")
+    if not isinstance(invocations, list):
+        invocations = [] if not previous else [first_invocation(previous)]
+    kept = {key: value for key, value in previous.items() if key not in UNSET}
+    return (
+        kept
+        | run
+        | {
+            "created": previous.get("created", run["created"]),
+            "invocations": [*invocations, invocation],
+        }
+    )

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/runner/describe_event.py

@@ -18,6 +18,8 @@ def describe_event(event: dict[str, object]) -> str:
             return f"Stopping {event['run']}."
         case "killing run":
             return f"Killing {event['run']}."
+        case "retried" if event["retry"] == event["run"]:
+            return f"{event['run']} retried in its folder."
         case "retried":
             return f"{event['run']} retried as {event['retry']}."
         case "retrying":

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/runner/execute_run.py

@@ -1,13 +1,16 @@
 import asyncio
 import contextlib
+import json
 import os
 import time
 from collections.abc import Callable, Sequence
+from datetime import UTC, datetime
 from pathlib import Path
 
 import psutil
 
 from labnotes.log_path import LOG_ENV, RUN_ENV
+from labnotes.runner.continue_run import continue_run
 from labnotes.runner.exit_signal import exit_signal
 from labnotes.runner.record_memory import record_memory
 from labnotes.runner.run_git import run_git
@@ -39,8 +42,10 @@ async def execute_run(
     memory_interval: float,
     environment: dict[str, str] | None = None,
     git_info: dict[str, str | bool] | None = None,
+    continues: bool = False,
 ) -> dict[str, object]:
     resolved = shell_command(command, folder, params, args)
+    git = git_info if git_info is not None else await asyncio.to_thread(run_git, root)
     run: dict[str, object] = {
         "params": params,
         "command": resolved,
@@ -50,9 +55,7 @@ async def execute_run(
         "error": None,
         "ended_by": None,
         "signal": None,
-        "git": git_info
-        if git_info is not None
-        else await asyncio.to_thread(run_git, root),
+        "git": git,
     }
     if args:
         run["args"] = list(args)
@@ -60,14 +63,32 @@ async def execute_run(
         run["retry_of"] = retry_of
     if meta is not None:
         run["meta"] = meta
+    logs = {stream: f"{stream}.log" for stream in ("stdout", "stderr")}
+    invocation: dict[str, object] | None = None
+    if continues:
+        stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f")
+        logs = {stream: f"{stream}-{stamp}.txt" for stream in logs}
+        invocation = {
+            "started": run["created"],
+            "ended": None,
+            "exit_code": None,
+            "ended_by": None,
+            "error": None,
+            "git": None if git is None else git["commit"],
+            "args": list(args),
+            **logs,
+        }
+        record = folder / "run.json"
+        previous = json.loads(record.read_text()) if record.exists() else {}
+        run = continue_run(previous, run, invocation)
     write_run_json(folder, run)
     run_environment = os.environ if environment is None else environment
     botbox_labels = f"labnotes.run={folder.resolve()}"
     if existing_labels := run_environment.get("BOTBOX_LABELS"):
         botbox_labels = f"{existing_labels}\n{botbox_labels}"
     with (
-        (folder / "stdout.log").open("wb") as stdout,
-        (folder / "stderr.log").open("wb") as stderr,
+        (folder / logs["stdout"]).open("wb") as stdout,
+        (folder / logs["stderr"]).open("wb") as stderr,
     ):
         try:
             process = await asyncio.create_subprocess_shell(
@@ -138,5 +159,9 @@ async def execute_run(
                     await asyncio.wait({sampler})
                 running.discard(process)
     run["ended"] = iso_time(time.time())
+    if invocation is not None:
+        invocation |= {
+            key: run[key] for key in ("ended", "exit_code", "ended_by", "error")
+        }
     write_run_json(folder, run)
     return run

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/runner/finalize_lost_runs.py

@@ -3,6 +3,8 @@ import time
 
 from labnotes.list_runs import UNFINISHED
 from labnotes.models import LabnotesConfig
+from labnotes.runner.claim_run import LOCK_FILE
+from labnotes.runner.run_locked import run_locked
 from labnotes.runner.write_run_json import write_run_json
 from labnotes.tables.iso_time import iso_time
 from labnotes.tables.open_tables import open_tables
@@ -15,10 +17,17 @@ async def finalize_lost_runs(config: LabnotesConfig) -> None:
     )
     for row in rows:
         run = json.loads(str(row["run"]))
+        folder = config.root / str(row["folder"])
         pid, created = run.get("pid"), run.get("pid_created")
-        if isinstance(pid, int) and process_alive(
+        if (folder / LOCK_FILE).exists():
+            if run_locked(folder):
+                continue
+        elif isinstance(pid, int) and process_alive(
             pid, created if isinstance(created, int | float) else None
         ):
             continue
-        run |= {"ended": iso_time(time.time()), "ended_by": "lost"}
-        write_run_json(config.root / str(row["folder"]), run)
+        lost = {"ended": iso_time(time.time()), "ended_by": "lost"}
+        run |= lost
+        if invocations := run.get("invocations"):
+            invocations[-1] |= lost
+        write_run_json(folder, run)

labnotes-0.2.4/src/labnotes/runner/first_invocation.py

@@ -0,0 +1,13 @@
+def first_invocation(run: dict[str, object]) -> dict[str, object]:
+    git = run.get("git")
+    return {
+        "started": run.get("created"),
+        "ended": run.get("ended"),
+        "exit_code": run.get("exit_code"),
+        "ended_by": run.get("ended_by"),
+        "error": run.get("error"),
+        "git": git.get("commit") if isinstance(git, dict) else None,
+        "args": run.get("args", []),
+        "stdout": "stdout.log",
+        "stderr": "stderr.log",
+    }

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/runner/read_experiment_config.py

@@ -27,6 +27,7 @@ SETTINGS = {
     "setup",
     "teardown",
     "env",
+    "continues",
 }
 HOOKS = ("setup", "teardown")
 VIEWS = {"runs"}
@@ -97,6 +98,11 @@ def read_experiment_config(
             raise ValueError(f"{path}: env.{key} is reserved for labnotes")
         if not isinstance(value, str):
             raise ValueError(f"{path}: env.{key} must be a string")
+    continues = values.get("continues", False)
+    if not isinstance(continues, bool):
+        raise ValueError(f"{path}: continues must be true or false")
+    if continues and mode != "matrix":
+        raise ValueError(f'{path}: continues needs mode = "matrix"')
     views = values.get("views", {})
     if not isinstance(views, dict):
         raise ValueError(f"{path}: views must be a table, like [views]")
@@ -120,4 +126,5 @@ def read_experiment_config(
         values.get("setup"),
         values.get("teardown"),
         env,
+        continues,
     )

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/runner/run_experiment.py

@@ -17,6 +17,7 @@ from labnotes.models import (
 )
 from labnotes.retries import RETRY_EXIT_CODE, RETRY_SECONDS
 from labnotes.runner.check_hook_placeholders import check_hook_placeholders
+from labnotes.runner.claim_run import claim_run
 from labnotes.runner.execute_run import execute_run
 from labnotes.runner.fill_experiment_env import fill_experiment_env
 from labnotes.runner.fill_dirs import fill_dirs
@@ -33,6 +34,7 @@ from labnotes.runner.signal_tree import signal_tree
 from labnotes.state_path import STATE_ENV
 from labnotes.tables.experiment_run_ids import experiment_run_ids
 from labnotes.tables.list_experiment_runs import list_experiment_runs
+from labnotes.tables.list_resumable_runs import list_resumable_runs
 from labnotes.tables.open_tables import open_tables
 
 Entry = tuple[str, dict[str, object]]
@@ -82,9 +84,37 @@ async def run_experiment(
             params, experiment.params, lambda run: taken(f"{group}/{run}")
         )
 
+    locks: dict[str, int] = {}
+
+    def claim(run: str) -> bool:
+        if run not in locks:
+            lock = claim_run(run_folder(config.root, config.runs, name, run))
+            if lock is None:
+                return False
+            locks[run] = lock
+        return True
+
+    def release(run: str) -> None:
+        if run in locks:
+            os.close(locks.pop(run))
+
+    resumable: list[Entry] = []
+    if experiment.continues:
+        resumable = [
+            (str(row["id"]), json.loads(str(row["params"])))
+            for row in await list_resumable_runs(tables, name, experiment.counts)
+        ]
     plan: list[Entry] = []
     for params in cells:
-        run = new_id(params)
+        resumed = next(
+            (
+                entry
+                for entry in resumable
+                if entry[1] == params and entry[0] not in planned and claim(entry[0])
+            ),
+            None,
+        )
+        run = new_id(params) if resumed is None else resumed[0]
         planned.add(run)
         plan.append((run, params))
     run_dirs = [
@@ -129,7 +159,7 @@ async def run_experiment(
     running: dict[str, set[asyncio.subprocess.Process]] = {}
     trees: dict[str, set[psutil.Process]] = {}
     stopped: set[str] = set()
-    results: list[dict[str, object]] = []
+    results: dict[str, dict[str, object]] = {}
     stopping = asyncio.Event()
 
     def signal_run(run: str, signal_number: int) -> None:
@@ -175,9 +205,11 @@ async def run_experiment(
     waiting: dict[str, asyncio.TimerHandle] = {}
 
     def requeue(run: str, attempt: int) -> None:
-        again = new_id(ended[run])
-        planned.add(again)
-        parents[again] = run
+        again = run
+        if not experiment.continues:
+            again = new_id(ended[run])
+            planned.add(again)
+            parents[again] = run
         attempts[again] = attempt
         plan.append((again, ended[run]))
         report({"event": "retried", "run": run, "retry": again, "params": ended[run]})
@@ -207,6 +239,7 @@ async def run_experiment(
 
     def cancel_retry(run: str) -> None:
         waiting.pop(run).cancel()
+        release(run)
         report({"event": "retry cancelled", "run": run})
 
     limit = concurrency
@@ -245,6 +278,8 @@ async def run_experiment(
     async def start(run: str, params: dict[str, object]) -> None:
         folder = run_folder(config.root, config.runs, name, run)
         folder.mkdir(parents=True, exist_ok=True)
+        if experiment.continues:
+            claim(run)
         if snapshot is not None and snapshot.sha is not None:
             (folder / "git-sha.txt").write_text(snapshot.sha + "\n")
             (folder / "git-diff.patch").write_bytes(snapshot.diff or b"")
@@ -287,15 +322,18 @@ async def run_experiment(
                 if snapshot is not None and snapshot.sha is not None
                 else None
             ),
+            continues=experiment.continues,
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})
         ended[run] = params
-        results.append(result)
+        results[run] = result
         if result["exit_code"] == RETRY_EXIT_CODE and not (
             stopping.is_set() or run in stopped
         ):
             retry_later(run)
+        if run not in waiting:
+            release(run)
 
     async def teardown() -> None:
         if experiment.teardown is not None:
@@ -336,5 +374,8 @@ async def run_experiment(
             except Exception as failure:  # noqa: BLE001
                 report({"event": "teardown failed", "error": str(failure)})
             raise
+        finally:
+            for run in list(locks):
+                release(run)
         await teardown()
-    return results
+    return list(results.values())

labnotes-0.2.4/src/labnotes/runner/run_locked.py

@@ -0,0 +1,12 @@
+import os
+from pathlib import Path
+
+from labnotes.runner.claim_run import claim_run
+
+
+def run_locked(folder: Path) -> bool:
+    lock = claim_run(folder)
+    if lock is None:
+        return True
+    os.close(lock)
+    return False

labnotes-0.2.4/src/labnotes/tables/list_resumable_runs.py

@@ -0,0 +1,16 @@
+from dirsql import DirSQL
+
+from labnotes.list_runs import UNFINISHED
+from labnotes.sql_quote import quote_string
+
+
+async def list_resumable_runs(
+    tables: DirSQL, name: str, counts: str
+) -> list[dict[str, object]]:
+    # Ended badly: a nonzero or missing exit code, not merely uncounted.
+    return await tables.query(
+        f"SELECT id, params FROM runs WHERE experiment = {quote_string(name)}"
+        f" AND NOT coalesce(({counts}), false) AND NOT {UNFINISHED}"
+        " AND (exit_code IS NULL OR exit_code <> 0)"
+        " AND NOT coalesce(run ->> 'excluded', false) ORDER BY created, id"
+    )

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/tables/open_tables.py

@@ -31,8 +31,8 @@ CREATE TABLE runs (
 # Sizes only: the server reads log text from disk, the row change is the signal.
 LOGS = """
 CREATE TABLE logs (
-  experiment TEXT, run TEXT, stream TEXT, size INTEGER,
-  PRIMARY KEY (experiment, run, stream)
+  experiment TEXT, run TEXT, stream TEXT, file TEXT, size INTEGER,
+  PRIMARY KEY (experiment, run, file)
 )
 """
 BLOCKS = """
@@ -90,7 +90,7 @@ def open_tables(config: LabnotesConfig, run_files: bool = False) -> DirSQL:
             Table(
                 name="logs",
                 ddl=LOGS,
-                glob=path_glob(f"{config.runs}/std*.log"),
+                glob=path_glob(f"{config.runs}/std*"),
                 on_file=partial(
                     read_log, root, config.runs, experiment=config.experiment
                 ),

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/tables/read_log.py

@@ -1,22 +1,26 @@
+import re
 from pathlib import Path
 
 from labnotes.tables.path_pattern import path_pattern
 
 STREAMS = ("stdout", "stderr")
+LOG_NAME = re.compile(r"(?P<stream>[^/.-]+)(?:\.log|-[^/]+\.txt)")
 
 
 def read_log(
     root: Path, template: str, path: str, experiment: str | None = None
 ) -> list[dict[str, object]]:
     relative = Path(path).relative_to(root).as_posix()
-    found = path_pattern(f"{template}/{{stream}}.log").fullmatch(relative)
-    if found is None or found["stream"] not in STREAMS:
+    found = path_pattern(f"{template}/{{log}}").fullmatch(relative)
+    name = None if found is None else LOG_NAME.fullmatch(found["log"])
+    if found is None or name is None or name["stream"] not in STREAMS:
         return []
     return [
         {
             "experiment": experiment or found["experiment"],
             "run": found["run"],
-            "stream": found["stream"],
+            "stream": name["stream"],
+            "file": found["log"],
             "size": Path(path).stat().st_size,
         }
     ]

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/tables/read_run.py

@@ -13,6 +13,10 @@ def read_run(
         return []
     run = json.loads(Path(path).read_text(encoding="utf-8"))
     folder = relative.removesuffix("/run.json")
+    logs = {"stdout": "stdout.log", "stderr": "stderr.log"}
+    match run.get("invocations"):
+        case [*_, {"stdout": str() as stdout, "stderr": str() as stderr}]:
+            logs = {"stdout": stdout, "stderr": stderr}
     return [
         {
             "experiment": experiment or found["experiment"],
@@ -22,8 +26,8 @@ def read_run(
             "ended": run.get("ended"),
             "exit_code": run.get("exit_code"),
             "error": run.get("error"),
-            "stdout": f"{folder}/stdout.log",
-            "stderr": f"{folder}/stderr.log",
+            "stdout": f"{folder}/{logs['stdout']}",
+            "stderr": f"{folder}/{logs['stderr']}",
             "params": json.dumps(run.get("params", {})),
             "meta": None if (meta := run.get("meta")) is None else json.dumps(meta),
             "command": run.get("command"),

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/tui/run_app.py

@@ -164,8 +164,12 @@ class RunApp(App[None]):
                 if self.waiting.pop(run, None) is not None:
                     outcome = f"{FAILED} exited {RETRY_EXIT_CODE}, retried"
                     self.set_cell(run, "status", Text(outcome, style="red"))
-                self.retried.add(run)
                 again = str(event["retry"])
+                if again == run:
+                    self.failed.discard(run)
+                    self.ended.discard(run)
+                else:
+                    self.retried.add(run)
                 args = " ".join(param_args(event["params"]))  # type: ignore[arg-type]
                 self.cells[again] = self.waiting_cells(again, args)
                 self.pending.add(again)

{labnotes-0.2.3 → labnotes-0.2.4}/src/labnotes/tui/run_output.py

@@ -6,6 +6,8 @@ from rich.text import Text
 from textual.geometry import Size
 from textual.widgets import RichLog
 
+from labnotes.latest_log import latest_log
+
 STREAMS = ("stdout", "stderr")
 
 
@@ -16,6 +18,7 @@ class RunOutput(RichLog):
 
     def reset(self) -> None:
         self.offsets = dict.fromkeys(STREAMS, 0)
+        self.paths: dict[str, Path] = {}
         self.decoders = {
             stream: codecs.getincrementaldecoder("utf-8")(errors="replace")
             for stream in STREAMS
@@ -34,9 +37,12 @@ class RunOutput(RichLog):
 
     def follow(self, folder: Path) -> None:
         for stream in STREAMS:
-            path = folder / f"{stream}.log"
+            path = latest_log(folder, stream)
             if not path.exists():
                 continue
+            if self.paths.setdefault(stream, path) != path:
+                self.paths[stream] = path
+                self.offsets[stream] = 0
             with path.open("rb") as log:
                 log.seek(self.offsets[stream])
                 data = log.read()