labnotes 0.0.128__tar.gz → 0.0.129__tar.gz

{labnotes-0.0.128 → labnotes-0.0.129}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.128
+Version: 0.0.129
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

labnotes-0.0.129/src/labnotes/compose_experiment.py

@@ -0,0 +1,9 @@
+def compose_experiment(
+    base: dict[str, object], layer: dict[str, object]
+) -> dict[str, object]:
+    composed = {**base, **layer}
+    for key in ("params", "env"):
+        base_values, layer_values = base.get(key), layer.get(key)
+        if isinstance(base_values, dict) and isinstance(layer_values, dict):
+            composed[key] = {**base_values, **layer_values}
+    return composed

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/context.md

@@ -15,7 +15,7 @@ path = "{experiment}/runs/{run}" # one folder per run
 
 Paths are relative to the root. `{experiment}` and `{run}` stand for folder names; the experiments and files paths use `{experiment}`, the runs path both. A project laid out differently keeps its folders and changes the paths: look at where its experiments, notes and runs live, then write the templates that match.
 
-When `experiment.toml` sits next to `.labnotes.toml`, the root is the one experiment, named after its folder. Paths then drop `{experiment}`: files default to `files/`, runs to `runs/{run}`, and `[experiments] path` is rejected. An `[experiment]` table in `.labnotes.toml` holds the same keys as `experiment.toml` and is the base each `experiment.toml` layers over (top-level keys replace, `params` merge by name). With no `[experiments] path`, it also makes the root the one experiment.
+When `experiment.toml` sits next to `.labnotes.toml`, the root is the one experiment, named after its folder. Paths then drop `{experiment}`: files default to `files/`, runs to `runs/{run}`, and `[experiments] path` is rejected. An `[experiment]` table in `.labnotes.toml` holds the same keys as `experiment.toml` and is the base each `experiment.toml` layers over (top-level keys replace, `params` and `env` merge by name). With no `[experiments] path`, it also makes the root the one experiment.
 
 `labnotes experiments` reads three tables from those paths:
 
@@ -99,6 +99,13 @@ failed (so teardown must cope with a partial setup). A failing setup starts no
 run; a failing teardown fails the invocation. Both get `{invocation_id}`, and
 `{run_id}` only when one run is planned; otherwise `{run_id}` there fails the
 invocation before anything starts.
+`[env]` in `experiment.toml` sets string environment values for `command`,
+`meta`, `setup`, and `teardown`. `[experiment.env]` in `.labnotes.toml`
+provides shared values; experiment values replace them by name. Variables in
+the environment that starts `labnotes experiments NAME run` win over both,
+including inline assignments. Values can use raw `{run_dir}`, `{run_id}` and
+`{invocation_id}` placeholders. In hooks and meta, `{run_id}` and `{run_dir}`
+require exactly one planned run. `LABNOTES_*` names are reserved for labnotes.
 A run counts when it exited 0 and its `run.json` doesn't hold
 `"excluded": true`, so excluding a run by hand means adding that key; the next
 `run` replaces it. Other keys you add to `run.json` are kept and show in `run`.

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/daemon/daemon_app.py

@@ -109,6 +109,7 @@ def daemon_app(config: LabnotesConfig) -> Starlette:
                     body.get("timeout"),  # type: ignore[arg-type]
                     only=body.get("only"),  # type: ignore[arg-type]
                     args=body.get("args"),  # type: ignore[arg-type]
+                    environment=body.get("environment"),  # type: ignore[arg-type]
                 )
             # Clients hear why the session ended; the daemon keeps serving.
             except Exception as failure:  # noqa: BLE001

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/daemon/remote_start.py

@@ -1,5 +1,6 @@
 import asyncio
 import dataclasses
+import os
 from collections.abc import Callable
 from pathlib import Path
 
@@ -21,6 +22,8 @@ def remote_start(
     setup: str | None = None,
     teardown: str | None = None,
 ) -> Start:
+    environment = dict(os.environ)
+
     async def start(report: Report, listen: Listen) -> list[dict[str, object]]:
         try:
             return await follow(report, listen)
@@ -54,6 +57,7 @@ def remote_start(
                     "args": args,
                     "setup": setup,
                     "teardown": teardown,
+                    "environment": environment,
                 },
             )
             if reply.status_code != 202:

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/models.py

@@ -66,3 +66,4 @@ class ExperimentConfig:
     memory_interval: float = DEFAULT_MEMORY_INTERVAL
     setup: str | None = None
     teardown: str | None = None
+    env: dict[str, str] = field(default_factory=dict)

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/runner/execute_run.py

@@ -34,6 +34,7 @@ async def execute_run(
     args: Sequence[str] = (),
     *,
     memory_interval: float,
+    environment: dict[str, str] | None = None,
 ) -> dict[str, object]:
     resolved = shell_command(command, folder, params, args)
     run: dict[str, object] = {
@@ -64,7 +65,7 @@ async def execute_run(
                 env={
                     "FORCE_COLOR": "1",
                     "CLICOLOR_FORCE": "1",
-                    **os.environ,
+                    **(os.environ if environment is None else environment),
                     LOG_ENV: str(folder / "log.jsonl"),
                     RUN_ENV: str(folder.resolve()),
                 },

labnotes-0.0.129/src/labnotes/runner/fill_experiment_env.py

@@ -0,0 +1,31 @@
+from collections.abc import Mapping
+
+from labnotes.runner.fill_ids import fill_ids
+
+
+def fill_experiment_env(
+    configured: Mapping[str, str],
+    inherited: Mapping[str, str],
+    ids: Mapping[str, str],
+    *,
+    hook: str | None = None,
+    runs: int = 1,
+) -> dict[str, str]:
+    caller = {
+        key: value
+        for key, value in inherited.items()
+        if not key.startswith("LABNOTES_")
+    }
+    filled: dict[str, str] = {}
+    for key, value in configured.items():
+        if key in caller:
+            continue
+        for name in ("run_dir", "run_id"):
+            if f"{{{name}}}" in value and name not in ids:
+                raise ValueError(
+                    f"{hook} env.{key} uses {{{name}}}, but this invocation plans "
+                    f"{runs} runs. {{{name}}} needs exactly one; "
+                    "use {invocation_id} instead."
+                )
+        filled[key] = fill_ids(value, ids, quote=False)
+    return {**filled, **caller}

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/runner/fill_ids.py

@@ -2,7 +2,7 @@ import shlex
 from collections.abc import Mapping
 
 
-def fill_ids(command: str, ids: Mapping[str, str]) -> str:
+def fill_ids(command: str, ids: Mapping[str, str], *, quote: bool = True) -> str:
     for name, value in ids.items():
-        command = command.replace(f"{{{name}}}", shlex.quote(value))
+        command = command.replace(f"{{{name}}}", shlex.quote(value) if quote else value)
     return command

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/runner/invocation_hook.py

@@ -2,12 +2,15 @@ import asyncio
 from pathlib import Path
 
 
-async def invocation_hook(root: Path, hook: str, command: str) -> None:
+async def invocation_hook(
+    root: Path, hook: str, command: str, environment: dict[str, str] | None = None
+) -> None:
     process = await asyncio.create_subprocess_shell(
         command,
         cwd=root,
         stdout=asyncio.subprocess.DEVNULL,
         stderr=asyncio.subprocess.PIPE,
+        env=environment,
     )
     _, stderr = await process.communicate()
     if process.returncode != 0:

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/runner/invocation_meta.py

@@ -3,12 +3,15 @@ import json
 from pathlib import Path
 
 
-async def invocation_meta(root: Path, command: str) -> dict[str, object]:
+async def invocation_meta(
+    root: Path, command: str, environment: dict[str, str] | None = None
+) -> dict[str, object]:
     process = await asyncio.create_subprocess_shell(
         command,
         cwd=root,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.PIPE,
+        env=environment,
     )
     stdout, stderr = await process.communicate()
     detail = stderr.decode(errors="replace").strip()

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/runner/read_experiment_config.py

@@ -26,6 +26,7 @@ SETTINGS = {
     "views",
     "setup",
     "teardown",
+    "env",
 }
 HOOKS = ("setup", "teardown")
 VIEWS = {"runs"}
@@ -88,6 +89,14 @@ def read_experiment_config(
         run = values.get(hook)
         if run is not None and (not isinstance(run, str) or not run.strip()):
             raise ValueError(f"{path}: {hook} must be a string")
+    env = values.get("env", {})
+    if not isinstance(env, dict):
+        raise ValueError(f"{path}: env must be a table, like [env]")
+    for key, value in env.items():
+        if key.startswith("LABNOTES_"):
+            raise ValueError(f"{path}: env.{key} is reserved for labnotes")
+        if not isinstance(value, str):
+            raise ValueError(f"{path}: env.{key} must be a string")
     views = values.get("views", {})
     if not isinstance(views, dict):
         raise ValueError(f"{path}: views must be a table, like [views]")
@@ -110,4 +119,5 @@ def read_experiment_config(
         memory_interval,
         values.get("setup"),
         values.get("teardown"),
+        env,
     )

{labnotes-0.0.128 → labnotes-0.0.129}/src/labnotes/runner/run_experiment.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import os
 import signal
 
 import psutil
@@ -13,6 +14,7 @@ from labnotes.models import (
 )
 from labnotes.retries import RETRY_EXIT_CODE, RETRY_SECONDS
 from labnotes.runner.execute_run import execute_run
+from labnotes.runner.fill_experiment_env import fill_experiment_env
 from labnotes.runner.fill_ids import fill_ids
 from labnotes.runner.hook_ids import hook_ids
 from labnotes.runner.in_subset import in_subset
@@ -42,10 +44,9 @@ async def run_experiment(
     retry_seconds: float = RETRY_SECONDS,
     only: dict[str, str] | None = None,
     args: list[str] | None = None,
+    environment: dict[str, str] | None = None,
 ) -> list[dict[str, object]]:
-    meta = None
-    if experiment.meta is not None:
-        meta = await invocation_meta(config.root, experiment.meta)
+    inherited = dict(os.environ) if environment is None else environment
     tables = open_tables(config)
     existing = await experiment_run_ids(tables, name)
     finished: list[dict[str, object]] = []
@@ -79,14 +80,36 @@ async def run_experiment(
         run = new_id(params)
         planned.add(run)
         plan.append((run, params))
+    hook_run_dir: str | None
     match plan:
         case [(invocation, _)]:
-            pass
+            hook_run_dir = str(
+                run_folder(config.root, config.runs, name, invocation).resolve()
+            )
         case _:
             invocation = group or new_stamped_run_id(taken)
+            hook_run_dir = None
     planned.add(invocation)
-    hooks = {"setup": experiment.setup, "teardown": experiment.teardown}
+    hooks = {
+        "setup": experiment.setup,
+        "teardown": experiment.teardown,
+        "meta": experiment.meta,
+    }
     ids = hook_ids(hooks, invocation, [run for run, _ in plan])
+    if hook_run_dir is not None:
+        ids["run_dir"] = hook_run_dir
+    hook_env = {
+        hook: fill_experiment_env(
+            experiment.env, inherited, ids, hook=hook, runs=len(plan)
+        )
+        for hook, command in hooks.items()
+        if command is not None
+    }
+    meta = None
+    if experiment.meta is not None:
+        meta = await invocation_meta(
+            config.root, fill_ids(experiment.meta, ids), hook_env["meta"]
+        )
     runs = [{"run": run, "params": params} for run, params in plan]
     report(
         {
@@ -234,6 +257,15 @@ async def run_experiment(
             meta=meta,
             args=args or (),
             memory_interval=experiment.memory_interval,
+            environment=fill_experiment_env(
+                experiment.env,
+                inherited,
+                {
+                    "run_dir": str(folder.resolve()),
+                    "run_id": run,
+                    "invocation_id": invocation,
+                },
+            ),
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})
@@ -248,7 +280,10 @@ async def run_experiment(
         if experiment.teardown is not None:
             report({"event": "tearing down"})
             await invocation_hook(
-                config.root, "teardown", fill_ids(experiment.teardown, ids)
+                config.root,
+                "teardown",
+                fill_ids(experiment.teardown, ids),
+                hook_env["teardown"],
             )
 
     listen(Controls(interrupt, set_concurrency, kill, finish))
@@ -256,7 +291,9 @@ async def run_experiment(
     try:
         if experiment.setup is not None:
             report({"event": "setting up"})
-            await invocation_hook(config.root, "setup", fill_ids(experiment.setup, ids))
+            await invocation_hook(
+                config.root, "setup", fill_ids(experiment.setup, ids), hook_env["setup"]
+            )
         fill()
         await done.wait()
         await asyncio.gather(*started)

labnotes-0.0.128/src/labnotes/compose_experiment.py

@@ -1,8 +0,0 @@
-def compose_experiment(
-    base: dict[str, object], layer: dict[str, object]
-) -> dict[str, object]:
-    composed = {**base, **layer}
-    base_params, layer_params = base.get("params"), layer.get("params")
-    if isinstance(base_params, dict) and isinstance(layer_params, dict):
-        composed["params"] = {**base_params, **layer_params}
-    return composed