labnotes 0.0.128__tar.gz → 0.0.130__tar.gz

{labnotes-0.0.128 → labnotes-0.0.130}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.128
+Version: 0.0.130
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

labnotes-0.0.130/src/labnotes/compose_experiment.py

@@ -0,0 +1,9 @@
+def compose_experiment(
+    base: dict[str, object], layer: dict[str, object]
+) -> dict[str, object]:
+    composed = {**base, **layer}
+    for key in ("params", "env"):
+        base_values, layer_values = base.get(key), layer.get(key)
+        if isinstance(base_values, dict) and isinstance(layer_values, dict):
+            composed[key] = {**base_values, **layer_values}
+    return composed

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/context.md

@@ -15,7 +15,7 @@ path = "{experiment}/runs/{run}" # one folder per run
 
 Paths are relative to the root. `{experiment}` and `{run}` stand for folder names; the experiments and files paths use `{experiment}`, the runs path both. A project laid out differently keeps its folders and changes the paths: look at where its experiments, notes and runs live, then write the templates that match.
 
-When `experiment.toml` sits next to `.labnotes.toml`, the root is the one experiment, named after its folder. Paths then drop `{experiment}`: files default to `files/`, runs to `runs/{run}`, and `[experiments] path` is rejected. An `[experiment]` table in `.labnotes.toml` holds the same keys as `experiment.toml` and is the base each `experiment.toml` layers over (top-level keys replace, `params` merge by name). With no `[experiments] path`, it also makes the root the one experiment.
+When `experiment.toml` sits next to `.labnotes.toml`, the root is the one experiment, named after its folder. Paths then drop `{experiment}`: files default to `files/`, runs to `runs/{run}`, and `[experiments] path` is rejected. An `[experiment]` table in `.labnotes.toml` holds the same keys as `experiment.toml` and is the base each `experiment.toml` layers over (top-level keys replace, `params` and `env` merge by name). With no `[experiments] path`, it also makes the root the one experiment.
 
 `labnotes experiments` reads three tables from those paths:
 
@@ -72,10 +72,9 @@ Each run gets a new run folder. In `matrix` mode it is named for the UTC time it
 its folder the same way. One that plans more gets one folder named that way, and its runs nest
 inside it (`runs/20261020-000429-brave-fox/quiet-river`); the run id is that path under `runs/`. Either way, folder listings sort by start. `command` runs in a shell from the root, with
 `{run_dir}` replaced by the run folder's absolute path. Pass it to the
-script so it writes its outputs there. `{run_id}` becomes the run id, and
-`{invocation_id}` an id every run of one `run` command shares: the run id when
-it plans one run, else the adhoc invocation folder's name, or in `matrix` mode a
-stamped id that names no folder. All three are shell-quoted.
+script so it writes its outputs there. `{invocation_dir}` is the shared adhoc
+group folder's absolute path when several runs are planned; otherwise it equals
+each run's `{run_dir}`. Both placeholders are shell-quoted.
 Each param is appended as `--name value`, with the name as written (use
 `source-language` for a `--source-language` flag), booleans as 0 or 1, and lists
 and tables as JSON. labnotes writes `run.json` (params, `command`, created,
@@ -96,9 +95,20 @@ the value from the start, even if the code changed while they waited.
 commands run once per `run` from the root: setup before the first run starts,
 teardown after the last one ends, even if runs failed, were stopped, or setup
 failed (so teardown must cope with a partial setup). A failing setup starts no
-run; a failing teardown fails the invocation. Both get `{invocation_id}`, and
-`{run_id}` only when one run is planned; otherwise `{run_id}` there fails the
-invocation before anything starts.
+run; a failing teardown fails the invocation. Both get `{invocation_dir}` only:
+the shared folder for a multi-run adhoc invocation, or the run folder when one
+run is planned. That folder exists before setup starts. In matrix mode with
+several runs there is no shared folder, so `{invocation_dir}` in a hook fails.
+`{run_dir}` is never filled in hooks; use `{invocation_dir}` instead. An
+unavailable placeholder fails before anything starts.
+`[env]` in `experiment.toml` sets string environment values for `command`,
+`meta`, `setup`, and `teardown`. `[experiment.env]` in `.labnotes.toml`
+provides shared values; experiment values replace them by name. Variables in
+the environment that starts `labnotes experiments NAME run` win over both,
+including inline assignments. Values can use raw `{run_dir}` and
+`{invocation_dir}` placeholders. In hooks and meta, only `{invocation_dir}` is
+available, and it requires one planned run in matrix mode.
+`LABNOTES_*` names are reserved for labnotes.
 A run counts when it exited 0 and its `run.json` doesn't hold
 `"excluded": true`, so excluding a run by hand means adding that key; the next
 `run` replaces it. Other keys you add to `run.json` are kept and show in `run`.

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/daemon/daemon_app.py

@@ -109,6 +109,7 @@ def daemon_app(config: LabnotesConfig) -> Starlette:
                     body.get("timeout"),  # type: ignore[arg-type]
                     only=body.get("only"),  # type: ignore[arg-type]
                     args=body.get("args"),  # type: ignore[arg-type]
+                    environment=body.get("environment"),  # type: ignore[arg-type]
                 )
             # Clients hear why the session ended; the daemon keeps serving.
             except Exception as failure:  # noqa: BLE001

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/daemon/remote_start.py

@@ -1,5 +1,6 @@
 import asyncio
 import dataclasses
+import os
 from collections.abc import Callable
 from pathlib import Path
 
@@ -21,6 +22,8 @@ def remote_start(
     setup: str | None = None,
     teardown: str | None = None,
 ) -> Start:
+    environment = dict(os.environ)
+
     async def start(report: Report, listen: Listen) -> list[dict[str, object]]:
         try:
             return await follow(report, listen)
@@ -54,6 +57,7 @@ def remote_start(
                     "args": args,
                     "setup": setup,
                     "teardown": teardown,
+                    "environment": environment,
                 },
             )
             if reply.status_code != 202:

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/models.py

@@ -66,3 +66,4 @@ class ExperimentConfig:
     memory_interval: float = DEFAULT_MEMORY_INTERVAL
     setup: str | None = None
     teardown: str | None = None
+    env: dict[str, str] = field(default_factory=dict)

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/runner/execute_run.py

@@ -34,6 +34,7 @@ async def execute_run(
     args: Sequence[str] = (),
     *,
     memory_interval: float,
+    environment: dict[str, str] | None = None,
 ) -> dict[str, object]:
     resolved = shell_command(command, folder, params, args)
     run: dict[str, object] = {
@@ -64,7 +65,7 @@ async def execute_run(
                 env={
                     "FORCE_COLOR": "1",
                     "CLICOLOR_FORCE": "1",
-                    **os.environ,
+                    **(os.environ if environment is None else environment),
                     LOG_ENV: str(folder / "log.jsonl"),
                     RUN_ENV: str(folder.resolve()),
                 },

labnotes-0.0.130/src/labnotes/runner/fill_dirs.py

@@ -0,0 +1,8 @@
+import shlex
+from collections.abc import Mapping
+
+
+def fill_dirs(command: str, dirs: Mapping[str, str], *, quote: bool = True) -> str:
+    for name, value in dirs.items():
+        command = command.replace(f"{{{name}}}", shlex.quote(value) if quote else value)
+    return command

labnotes-0.0.130/src/labnotes/runner/fill_experiment_env.py

@@ -0,0 +1,34 @@
+from collections.abc import Mapping
+
+from labnotes.runner.fill_dirs import fill_dirs
+
+
+def fill_experiment_env(
+    configured: Mapping[str, str],
+    inherited: Mapping[str, str],
+    dirs: Mapping[str, str],
+    *,
+    hook: str | None = None,
+    runs: int = 1,
+) -> dict[str, str]:
+    caller = {
+        key: value
+        for key, value in inherited.items()
+        if not key.startswith("LABNOTES_")
+    }
+    filled: dict[str, str] = {}
+    for key, value in configured.items():
+        if key in caller:
+            continue
+        if hook is not None and "{run_dir}" in value:
+            raise ValueError(
+                f"{hook} env.{key} uses {{run_dir}}; use {{invocation_dir}} instead."
+            )
+        for name in ("run_dir", "invocation_dir"):
+            if f"{{{name}}}" in value and name not in dirs:
+                raise ValueError(
+                    f"{hook} env.{key} uses {{{name}}}, but this invocation plans "
+                    f"{runs} runs. {{{name}}} needs exactly one run in this mode."
+                )
+        filled[key] = fill_dirs(value, dirs, quote=False)
+    return {**filled, **caller}

labnotes-0.0.130/src/labnotes/runner/hook_dirs.py

@@ -0,0 +1,21 @@
+def hook_dirs(
+    hooks: dict[str, str | None], invocation_dir: str | None, run_dirs: list[str]
+) -> dict[str, str]:
+    match run_dirs:
+        case [run_dir]:
+            invocation_dir = invocation_dir or run_dir
+    available = {"invocation_dir": invocation_dir} if invocation_dir is not None else {}
+    for hook, command in hooks.items():
+        if command is None:
+            continue
+        if "{run_dir}" in command:
+            raise ValueError(
+                f"{hook} command {command} uses {{run_dir}}; "
+                "use {invocation_dir} instead."
+            )
+        if "{invocation_dir}" in command and invocation_dir is None:
+            raise ValueError(
+                f"{hook} command {command} uses {{invocation_dir}}, but this "
+                f"invocation plans {len(run_dirs)} runs and has no shared folder."
+            )
+    return available

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/runner/invocation_hook.py

@@ -2,12 +2,15 @@ import asyncio
 from pathlib import Path
 
 
-async def invocation_hook(root: Path, hook: str, command: str) -> None:
+async def invocation_hook(
+    root: Path, hook: str, command: str, environment: dict[str, str] | None = None
+) -> None:
     process = await asyncio.create_subprocess_shell(
         command,
         cwd=root,
         stdout=asyncio.subprocess.DEVNULL,
         stderr=asyncio.subprocess.PIPE,
+        env=environment,
     )
     _, stderr = await process.communicate()
     if process.returncode != 0:

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/runner/invocation_meta.py

@@ -3,12 +3,15 @@ import json
 from pathlib import Path
 
 
-async def invocation_meta(root: Path, command: str) -> dict[str, object]:
+async def invocation_meta(
+    root: Path, command: str, environment: dict[str, str] | None = None
+) -> dict[str, object]:
     process = await asyncio.create_subprocess_shell(
         command,
         cwd=root,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.PIPE,
+        env=environment,
     )
     stdout, stderr = await process.communicate()
     detail = stderr.decode(errors="replace").strip()

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/runner/read_experiment_config.py

@@ -26,6 +26,7 @@ SETTINGS = {
     "views",
     "setup",
     "teardown",
+    "env",
 }
 HOOKS = ("setup", "teardown")
 VIEWS = {"runs"}
@@ -88,6 +89,14 @@ def read_experiment_config(
         run = values.get(hook)
         if run is not None and (not isinstance(run, str) or not run.strip()):
             raise ValueError(f"{path}: {hook} must be a string")
+    env = values.get("env", {})
+    if not isinstance(env, dict):
+        raise ValueError(f"{path}: env must be a table, like [env]")
+    for key, value in env.items():
+        if key.startswith("LABNOTES_"):
+            raise ValueError(f"{path}: env.{key} is reserved for labnotes")
+        if not isinstance(value, str):
+            raise ValueError(f"{path}: env.{key} must be a string")
     views = values.get("views", {})
     if not isinstance(views, dict):
         raise ValueError(f"{path}: views must be a table, like [views]")
@@ -110,4 +119,5 @@ def read_experiment_config(
         memory_interval,
         values.get("setup"),
         values.get("teardown"),
+        env,
     )

{labnotes-0.0.128 → labnotes-0.0.130}/src/labnotes/runner/run_experiment.py

@@ -1,6 +1,8 @@
 import asyncio
 import json
+import os
 import signal
+from pathlib import Path
 
 import psutil
 
@@ -13,8 +15,9 @@ from labnotes.models import (
 )
 from labnotes.retries import RETRY_EXIT_CODE, RETRY_SECONDS
 from labnotes.runner.execute_run import execute_run
-from labnotes.runner.fill_ids import fill_ids
-from labnotes.runner.hook_ids import hook_ids
+from labnotes.runner.fill_experiment_env import fill_experiment_env
+from labnotes.runner.fill_dirs import fill_dirs
+from labnotes.runner.hook_dirs import hook_dirs
 from labnotes.runner.in_subset import in_subset
 from labnotes.runner.invocation_hook import invocation_hook
 from labnotes.runner.invocation_meta import invocation_meta
@@ -42,10 +45,9 @@ async def run_experiment(
     retry_seconds: float = RETRY_SECONDS,
     only: dict[str, str] | None = None,
     args: list[str] | None = None,
+    environment: dict[str, str] | None = None,
 ) -> list[dict[str, object]]:
-    meta = None
-    if experiment.meta is not None:
-        meta = await invocation_meta(config.root, experiment.meta)
+    inherited = dict(os.environ) if environment is None else environment
     tables = open_tables(config)
     existing = await experiment_run_ids(tables, name)
     finished: list[dict[str, object]] = []
@@ -79,14 +81,35 @@ async def run_experiment(
         run = new_id(params)
         planned.add(run)
         plan.append((run, params))
-    match plan:
-        case [(invocation, _)]:
-            pass
-        case _:
-            invocation = group or new_stamped_run_id(taken)
-    planned.add(invocation)
-    hooks = {"setup": experiment.setup, "teardown": experiment.teardown}
-    ids = hook_ids(hooks, invocation, [run for run, _ in plan])
+    run_dirs = [
+        str(run_folder(config.root, config.runs, name, run).resolve())
+        for run, _ in plan
+    ]
+    group_dir = (
+        str(run_folder(config.root, config.runs, name, group).resolve())
+        if group is not None
+        else None
+    )
+    hooks = {
+        "setup": experiment.setup,
+        "teardown": experiment.teardown,
+        "meta": experiment.meta,
+    }
+    dirs = hook_dirs(hooks, group_dir, run_dirs)
+    hook_env = {
+        hook: fill_experiment_env(
+            experiment.env, inherited, dirs, hook=hook, runs=len(plan)
+        )
+        for hook, command in hooks.items()
+        if command is not None
+    }
+    meta = None
+    if experiment.meta is not None:
+        meta = await invocation_meta(
+            config.root, fill_dirs(experiment.meta, dirs), hook_env["meta"]
+        )
+    if dirs:
+        Path(dirs["invocation_dir"]).mkdir(parents=True, exist_ok=True)
     runs = [{"run": run, "params": params} for run, params in plan]
     report(
         {
@@ -215,14 +238,21 @@ async def run_experiment(
 
     async def start(run: str, params: dict[str, object]) -> None:
         folder = run_folder(config.root, config.runs, name, run)
-        folder.mkdir(parents=True)
+        folder.mkdir(parents=True, exist_ok=True)
+        run_invocation_dir = group_dir or str(folder.resolve())
         report({"event": "started", "run": run, "params": params, "folder": folder})
         running[run] = set()
         trees[run] = set()
         result = await execute_run(
             config.root,
             folder,
-            fill_ids(experiment.command, {"run_id": run, "invocation_id": invocation}),
+            fill_dirs(
+                experiment.command,
+                {
+                    "run_dir": str(folder.resolve()),
+                    "invocation_dir": run_invocation_dir,
+                },
+            ),
             params,
             running[run],
             stopping,
@@ -234,6 +264,14 @@ async def run_experiment(
             meta=meta,
             args=args or (),
             memory_interval=experiment.memory_interval,
+            environment=fill_experiment_env(
+                experiment.env,
+                inherited,
+                {
+                    "run_dir": str(folder.resolve()),
+                    "invocation_dir": run_invocation_dir,
+                },
+            ),
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})
@@ -248,7 +286,10 @@ async def run_experiment(
         if experiment.teardown is not None:
             report({"event": "tearing down"})
             await invocation_hook(
-                config.root, "teardown", fill_ids(experiment.teardown, ids)
+                config.root,
+                "teardown",
+                fill_dirs(experiment.teardown, dirs),
+                hook_env["teardown"],
             )
 
     listen(Controls(interrupt, set_concurrency, kill, finish))
@@ -256,7 +297,12 @@ async def run_experiment(
     try:
         if experiment.setup is not None:
             report({"event": "setting up"})
-            await invocation_hook(config.root, "setup", fill_ids(experiment.setup, ids))
+            await invocation_hook(
+                config.root,
+                "setup",
+                fill_dirs(experiment.setup, dirs),
+                hook_env["setup"],
+            )
         fill()
         await done.wait()
         await asyncio.gather(*started)

labnotes-0.0.128/src/labnotes/compose_experiment.py

@@ -1,8 +0,0 @@
-def compose_experiment(
-    base: dict[str, object], layer: dict[str, object]
-) -> dict[str, object]:
-    composed = {**base, **layer}
-    base_params, layer_params = base.get("params"), layer.get("params")
-    if isinstance(base_params, dict) and isinstance(layer_params, dict):
-        composed["params"] = {**base_params, **layer_params}
-    return composed

labnotes-0.0.128/src/labnotes/runner/fill_ids.py

@@ -1,8 +0,0 @@
-import shlex
-from collections.abc import Mapping
-
-
-def fill_ids(command: str, ids: Mapping[str, str]) -> str:
-    for name, value in ids.items():
-        command = command.replace(f"{{{name}}}", shlex.quote(value))
-    return command

labnotes-0.0.128/src/labnotes/runner/hook_ids.py

@@ -1,14 +0,0 @@
-def hook_ids(
-    hooks: dict[str, str | None], invocation: str, runs: list[str]
-) -> dict[str, str]:
-    match runs:
-        case [run]:
-            return {"run_id": run, "invocation_id": invocation}
-    for hook, command in hooks.items():
-        if command is not None and "{run_id}" in command:
-            raise ValueError(
-                f"{hook} command {command} uses {{run_id}}, but this invocation plans "
-                f"{len(runs)} runs. {{run_id}} needs exactly one; "
-                "use {invocation_id} instead."
-            )
-    return {"invocation_id": invocation}