labnotes 0.0.101__tar.gz → 0.0.103__tar.gz

{labnotes-0.0.101 → labnotes-0.0.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.101
+Version: 0.0.103
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>
@@ -51,7 +51,9 @@ labnotes reads the file from the working folder, or from the path given with
 
 When `experiment.toml` sits next to `.labnotes.toml`, the root is the one
 experiment, named after its folder. Paths then drop `{experiment}`: files
-default to `files/` and runs to `runs/{run}`.
+default to `files/` and runs to `runs/{run}`. An `[experiment]` table in
+`.labnotes.toml` takes the same keys as `experiment.toml` and is the base
+that `experiment.toml` layers over.
 
 `labnotes experiments` reads three tables from those paths:
 

{labnotes-0.0.101 → labnotes-0.0.103}/README.md

@@ -27,7 +27,9 @@ labnotes reads the file from the working folder, or from the path given with
 
 When `experiment.toml` sits next to `.labnotes.toml`, the root is the one
 experiment, named after its folder. Paths then drop `{experiment}`: files
-default to `files/` and runs to `runs/{run}`.
+default to `files/` and runs to `runs/{run}`. An `[experiment]` table in
+`.labnotes.toml` takes the same keys as `experiment.toml` and is the base
+that `experiment.toml` layers over.
 
 `labnotes experiments` reads three tables from those paths:
 

labnotes-0.0.103/src/labnotes/compose_experiment.py

@@ -0,0 +1,8 @@
+def compose_experiment(
+    base: dict[str, object], layer: dict[str, object]
+) -> dict[str, object]:
+    composed = {**base, **layer}
+    base_params, layer_params = base.get("params"), layer.get("params")
+    if isinstance(base_params, dict) and isinstance(layer_params, dict):
+        composed["params"] = {**base_params, **layer_params}
+    return composed

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/context.md

@@ -15,7 +15,7 @@ path = "{experiment}/runs/{run}" # one folder per run
 
 Paths are relative to the root. `{experiment}` and `{run}` stand for folder names; the experiments and files paths use `{experiment}`, the runs path both. A project laid out differently keeps its folders and changes the paths: look at where its experiments, notes and runs live, then write the templates that match.
 
-When `experiment.toml` sits next to `.labnotes.toml`, the root is the one experiment, named after its folder. Paths then drop `{experiment}`: files default to `files/`, runs to `runs/{run}`, and `[experiments] path` is rejected.
+When `experiment.toml` sits next to `.labnotes.toml`, the root is the one experiment, named after its folder. Paths then drop `{experiment}`: files default to `files/`, runs to `runs/{run}`, and `[experiments] path` is rejected. An `[experiment]` table in `.labnotes.toml` holds the same keys as `experiment.toml` and is the base each `experiment.toml` layers over (top-level keys replace, `params` merge by name). With no `[experiments] path`, it also makes the root the one experiment.
 
 `labnotes experiments` reads three tables from those paths:
 
@@ -117,6 +117,10 @@ going. Elsewhere, it prints a line per event instead.
   in `experiment.toml` sets it, then `concurrency` under `[experiments]` in
   `.labnotes.toml`, then 1.
 - `-e PATH` reads another file instead of `experiment.toml`.
+- `--timeout SECONDS` stops any one run still going after that many seconds:
+  SIGTERM, then SIGKILL 5 seconds later. Its
+  `run.json` gets `"error": "timed out after ..."` next to its exit code.
+  Unset, runs have no time limit.
 - `run` hands its runs to the project's labnotes daemon, starting it if
   needed, and shows or waits for them. The daemon listens on a socket in
   `/tmp/labnotes-<uid>/` and exits once nothing is running and no client has

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/daemon/daemon_app.py

@@ -102,6 +102,7 @@ def daemon_app(config: LabnotesConfig) -> Starlette:
                     concurrency,
                     lambda event: record({**event, "experiment": name}),
                     listen,
+                    body.get("timeout"),  # type: ignore[arg-type]
                 )
             # Clients hear why the session ended; the daemon keeps serving.
             except Exception as failure:  # noqa: BLE001

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/daemon/remote_start.py

@@ -15,6 +15,7 @@ def remote_start(
     name: str,
     experiment_config: Path | None,
     concurrency: int | None,
+    timeout: float | None = None,
 ) -> Start:
     async def start(report: Report, listen: Listen) -> list[dict[str, object]]:
         try:
@@ -44,6 +45,7 @@ def remote_start(
                     if experiment_config is None
                     else str(experiment_config.resolve()),
                     "concurrency": concurrency,
+                    "timeout": timeout,
                 },
             )
             if reply.status_code != 202:

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/experiments.py

@@ -40,6 +40,13 @@ from labnotes.tui.run_tui import run_tui
     type=click.Path(exists=True, dir_okay=False, path_type=Path),
     help="With run: read this file instead of the experiment's experiment.toml.",
 )
+@click.option(
+    "--timeout",
+    type=click.IntRange(min=1),
+    metavar="SECONDS",
+    help="With run: stop any run still going after this many seconds."
+    " Unset, runs have no time limit.",
+)
 @click.pass_context
 def experiments(
     ctx: click.Context,
@@ -48,6 +55,7 @@ def experiments(
     output_format: str | None,
     concurrency: int | None,
     experiment_config: Path | None,
+    timeout: int | None,
 ) -> None:
     config = load_config(ctx)
     if config is None:
@@ -63,7 +71,7 @@ def experiments(
             click.echo(f"Created {path / 'experiment.toml'}")
             return
         case str(), "run":
-            start = remote_start(config, name, experiment_config, concurrency)
+            start = remote_start(config, name, experiment_config, concurrency, timeout)
             try:
                 if sys.stdout.isatty() and sys.stderr.isatty():
                     results = run_tui(start, name)

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/load_run.py

@@ -10,7 +10,7 @@ async def load_run(
 ) -> dict[str, object] | None:
     rows = await tables.query(
         "SELECT runs.experiment, experiments.display_name, runs.id, runs.folder,"
-        " runs.created, runs.ended, runs.exit_code, runs.params"
+        " runs.created, runs.ended, runs.exit_code, runs.error, runs.params"
         " FROM runs JOIN experiments ON experiments.name = runs.experiment"
         f" WHERE runs.experiment = {quote_string(experiment)}"
         f" AND runs.id = {quote_string(run)}"

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/models.py

@@ -49,6 +49,8 @@ class LabnotesConfig:
     concurrency: int = 1
     # Set when the config's folder is itself the one experiment.
     experiment: str | None = None
+    # [experiment] in .labnotes.toml, under each experiment.toml.
+    defaults: dict[str, object] = field(default_factory=dict)
 
 
 @dataclass(frozen=True)

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/read_config.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from labnotes.models import LabnotesConfig
 from labnotes.is_positive_int import is_positive_int
 from labnotes.modes import DEFAULT_MODE, MODES
+from labnotes.runner.read_experiment_config import SETTINGS
 from labnotes.read_path import DEFAULT_PATHS, read_path
 
 CONFIG_FILE = ".labnotes.toml"
@@ -15,16 +16,27 @@ def read_config(path: Path) -> LabnotesConfig:
             values = tomllib.load(file)
     except tomllib.TOMLDecodeError as error:
         raise ValueError(f"{path}: {error}") from error
-    unknown = sorted(values.keys() - DEFAULT_PATHS.keys())
+    unknown = sorted(values.keys() - {*DEFAULT_PATHS, "experiment"})
     if unknown:
         raise ValueError(f"{path}: unknown tables {', '.join(unknown)}")
+    experiments = values.get("experiments", {})
+    defaults = values.get("experiment", {})
+    if not isinstance(defaults, dict):
+        raise ValueError(f"{path}: experiment must be a table, like [experiment]")
+    unknown = sorted(defaults.keys() - SETTINGS)
+    if unknown:
+        raise ValueError(f"{path}: unknown keys in [experiment]: {', '.join(unknown)}")
     root = path.resolve().parent
-    single = (root / "experiment.toml").is_file()
+    # An [experiments] path means many experiments, each taking [experiment] as
+    # its defaults; without one, [experiment] makes the root the experiment.
+    single = (root / "experiment.toml").is_file() or (
+        "experiment" in values
+        and not (isinstance(experiments, dict) and "path" in experiments)
+    )
     paths = {
         table: read_path(path, table, values.get(table, {}), single=single)
         for table in DEFAULT_PATHS
     }
-    experiments = values.get("experiments", {})
     mode = experiments.get("mode", DEFAULT_MODE)
     if mode not in MODES:
         raise ValueError(f"{path}: [experiments] mode must be matrix or adhoc")
@@ -40,4 +52,5 @@ def read_config(path: Path) -> LabnotesConfig:
         mode=mode,
         concurrency=concurrency,
         experiment=root.name if single else None,
+        defaults=defaults,
     )

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/runner/execute_run.py

@@ -1,15 +1,18 @@
 import asyncio
 import os
 import time
+from collections.abc import Callable
 from pathlib import Path
 
 import psutil
 
 from labnotes.log_path import LOG_ENV
 from labnotes.runner.shell_command import shell_command
+from labnotes.runner.stop_then_kill import GRACE_SECONDS
 from labnotes.runner.wait_for_tree import wait_for_tree
 from labnotes.runner.write_run_json import write_run_json
 from labnotes.tables.iso_time import iso_time
+from labnotes.tui.format_elapsed import format_elapsed
 
 
 async def execute_run(
@@ -20,6 +23,8 @@ async def execute_run(
     running: set[asyncio.subprocess.Process],
     stopping: asyncio.Event,
     tree: set[psutil.Process],
+    timeout: float | None = None,
+    stop: Callable[[], None] = lambda: None,
 ) -> dict[str, object]:
     run: dict[str, object] = {
         "params": params,
@@ -50,14 +55,27 @@ async def execute_run(
             run["error"] = str(error)
         else:
             running.add(process)
+            exited = asyncio.ensure_future(process.wait())
+            kill_later = None
             try:
-                run["exit_code"] = await process.wait()
+                if (
+                    timeout is not None
+                    and not (await asyncio.wait({exited}, timeout=timeout))[0]
+                ):
+                    run["error"] = f"timed out after {format_elapsed(timeout)}"
+                    stop()
+                    kill_later = asyncio.get_running_loop().call_later(
+                        GRACE_SECONDS, stop
+                    )
+                run["exit_code"] = await exited
                 # The shell can exit on SIGTERM before what it started. Once
                 # stopping, the run stays killable until everything signalled
                 # is gone. A normal exit ends the run even if helpers linger.
-                if stopping.is_set():
+                if stopping.is_set() or kill_later is not None:
                     await wait_for_tree(tree)
             finally:
+                if kill_later is not None:
+                    kill_later.cancel()
                 running.discard(process)
     run["ended"] = iso_time(time.time())
     write_run_json(folder, run)

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/runner/prepare_experiment.py

@@ -19,9 +19,13 @@ async def prepare_experiment(
         folder = config.experiments.replace("{experiment}", name)
         experiment_config = config.root / folder / "experiment.toml"
     source = await file_text(config.root, experiment_config)
-    if source is None:
+    if source is None and not config.defaults:
         raise ValueError(f"No experiment.toml for {name} at {experiment_config}.")
     experiment = read_experiment_config(
-        experiment_config, source, config.mode, config.concurrency
+        config.path if source is None else experiment_config,
+        source or "",
+        config.mode,
+        config.concurrency,
+        config.defaults,
     )
     return experiment, experiment.concurrency if concurrency is None else concurrency

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/runner/read_experiment_config.py

@@ -1,7 +1,9 @@
 import json
 import tomllib
 from pathlib import Path
+from typing import Any
 
+from labnotes.compose_experiment import compose_experiment
 from labnotes.default_counts import DEFAULT_COUNTS
 from labnotes.models import ExperimentConfig
 from labnotes.is_positive_int import is_positive_int
@@ -11,12 +13,17 @@ SETTINGS = {"name", "command", "repeats", "params", "counts", "mode", "concurren
 
 
 def read_experiment_config(
-    path: Path, source: str, mode: str, concurrency: int
+    path: Path,
+    source: str,
+    mode: str,
+    concurrency: int,
+    defaults: dict[str, object] | None = None,
 ) -> ExperimentConfig:
     try:
-        values = tomllib.loads(source)
+        layer = tomllib.loads(source)
     except tomllib.TOMLDecodeError as error:
         raise ValueError(f"{path}: {error}") from error
+    values: dict[str, Any] = compose_experiment(defaults or {}, layer)
     unknown = sorted(values.keys() - SETTINGS)
     if unknown:
         raise ValueError(f"{path}: unknown keys {', '.join(unknown)}")

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/runner/run_experiment.py

@@ -32,6 +32,7 @@ async def run_experiment(
     concurrency: int,
     report: Report,
     listen: Listen,
+    timeout: float | None = None,
 ) -> list[dict[str, object]]:
     tables = open_tables(config)
     existing = await experiment_run_ids(tables, name)
@@ -203,6 +204,8 @@ async def run_experiment(
             running[run],
             stopping,
             trees[run],
+            timeout=timeout,
+            stop=lambda: kill(run),
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})

labnotes-0.0.103/src/labnotes/tables/experiments_ddl.py

@@ -0,0 +1,34 @@
+import json
+
+from labnotes.models import LabnotesConfig
+from labnotes.sql_quote import quote_string
+
+# experiments is a view, so files and runs can't declare a foreign key to it.
+EXPERIMENTS = """
+CREATE TABLE experiment_configs (
+  experiment TEXT PRIMARY KEY, config TEXT, source TEXT
+);
+CREATE VIEW experiments AS
+  SELECT name, coalesce(config ->> 'name', name) AS display_name, config, source
+  FROM (
+    SELECT
+      names.name,
+      coalesce(experiment_configs.config, {defaults}) AS config,
+      experiment_configs.source
+    FROM (
+      SELECT experiment AS name FROM experiment_configs
+      UNION SELECT experiment FROM files
+      UNION SELECT experiment FROM runs{one}
+    ) AS names
+    LEFT JOIN experiment_configs ON experiment_configs.experiment = names.name
+  );
+"""
+
+
+def experiments_ddl(config: LabnotesConfig) -> str:
+    defaults = json.dumps(config.defaults, default=str) if config.defaults else None
+    one = config.experiment
+    return EXPERIMENTS.format(
+        defaults="NULL" if defaults is None else quote_string(defaults),
+        one="" if one is None else f"\n      UNION SELECT {quote_string(one)}",
+    )

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/tables/open_tables.py

@@ -4,6 +4,7 @@ from dirsql import DirSQL, Table
 
 from labnotes.models import LabnotesConfig
 from labnotes.tables.experiment_toml import experiment_toml
+from labnotes.tables.experiments_ddl import experiments_ddl
 from labnotes.tables.path_glob import path_glob
 from labnotes.tables.read_blocks import read_blocks
 from labnotes.tables.read_experiment import read_experiment
@@ -11,24 +12,6 @@ from labnotes.tables.read_file import read_file
 from labnotes.tables.read_log import read_log
 from labnotes.tables.read_run import read_run
 
-# experiments is a view, so files and runs can't declare a foreign key to it.
-EXPERIMENTS = """
-CREATE TABLE experiment_configs (
-  experiment TEXT PRIMARY KEY, config TEXT, source TEXT
-);
-CREATE VIEW experiments AS
-  SELECT
-    names.name,
-    coalesce(experiment_configs.config ->> 'name', names.name) AS display_name,
-    experiment_configs.config,
-    experiment_configs.source
-  FROM (
-    SELECT experiment AS name FROM experiment_configs
-    UNION SELECT experiment FROM files
-    UNION SELECT experiment FROM runs
-  ) AS names
-  LEFT JOIN experiment_configs ON experiment_configs.experiment = names.name;
-"""
 FILES = """
 CREATE TABLE files (
   experiment TEXT, filename TEXT, path TEXT PRIMARY KEY, ctime TEXT, mtime TEXT,
@@ -97,13 +80,14 @@ def open_tables(config: LabnotesConfig) -> DirSQL:
             ),
             Table(
                 name="experiment_configs",
-                ddl=EXPERIMENTS,
+                ddl=experiments_ddl(config),
                 glob=path_glob(experiment_toml(config.experiments)),
                 on_file=partial(
                     read_experiment,
                     root,
                     config.experiments,
                     experiment=config.experiment,
+                    defaults=config.defaults,
                 ),
             ),
         ],

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/tables/read_experiment.py

@@ -2,19 +2,24 @@ import json
 import tomllib
 from pathlib import Path
 
+from labnotes.compose_experiment import compose_experiment
 from labnotes.tables.experiment_toml import experiment_toml
 from labnotes.tables.path_pattern import path_pattern
 
 
 def read_experiment(
-    root: Path, template: str, path: str, experiment: str | None = None
+    root: Path,
+    template: str,
+    path: str,
+    experiment: str | None = None,
+    defaults: dict[str, object] | None = None,
 ) -> list[dict[str, object]]:
     relative = Path(path).relative_to(root).as_posix()
     found = path_pattern(experiment_toml(template)).fullmatch(relative)
     if found is None:
         return []
     source = Path(path).read_text(encoding="utf-8")
-    config = tomllib.loads(source)
+    config = compose_experiment(defaults or {}, tomllib.loads(source))
     return [
         {
             "experiment": experiment or found["experiment"],

{labnotes-0.0.101 → labnotes-0.0.103}/src/labnotes/templates/run.html

@@ -5,7 +5,7 @@
 <section class="intro">
   <h1 class="mono">{{ run.id }}</h1>
   <p class="config-summary">
-    {% if run.ended is none %}<span class="chip run-state">running</span>{% elif run.exit_code is none %}<span class="chip run-state bad">ended</span>{% else %}<span class="chip run-state {{ 'good' if run.exit_code == 0 else 'bad' }}">exit {{ run.exit_code }}</span>{% endif %}
+    {% if run.ended is none %}<span class="chip run-state">running</span>{% elif run.error %}<span class="chip run-state bad">{{ run.error }}</span>{% elif run.exit_code is none %}<span class="chip run-state bad">ended</span>{% else %}<span class="chip run-state {{ 'good' if run.exit_code == 0 else 'bad' }}">exit {{ run.exit_code }}</span>{% endif %}
     {% if run.created %}<span class="setting">started <code>{{ run.created }}</code></span>{% endif %}
     {% for key, value in run.params.items() %}<span class="setting">{{ key }} <code>{{ value | value }}</code></span>{% endfor %}
   </p>