labnotes 0.0.19__tar.gz → 0.0.21__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. {labnotes-0.0.19 → labnotes-0.0.21}/PKG-INFO +9 -21
  2. {labnotes-0.0.19 → labnotes-0.0.21}/README.md +8 -19
  3. {labnotes-0.0.19 → labnotes-0.0.21}/pyproject.toml +0 -2
  4. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/context.md +3 -10
  5. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/create_app.py +22 -26
  6. labnotes-0.0.21/src/labnotes/experiment_notes.py +23 -0
  7. labnotes-0.0.21/src/labnotes/experiment_progress.py +25 -0
  8. labnotes-0.0.21/src/labnotes/experiment_runs.py +13 -0
  9. labnotes-0.0.21/src/labnotes/list_experiments.py +26 -0
  10. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/list_runs.py +1 -1
  11. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/load_experiment.py +22 -29
  12. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/execute_run.py +7 -0
  13. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/run_experiment.py +4 -5
  14. labnotes-0.0.21/src/labnotes/runner/wait_for_group.py +11 -0
  15. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/serve_app.py +3 -5
  16. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/server.py +11 -20
  17. labnotes-0.0.21/src/labnotes/summarize_config.py +26 -0
  18. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/templates/experiment.html +7 -12
  19. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/templates/index.html +3 -3
  20. {labnotes-0.0.19 → labnotes-0.0.21}/uv.lock +0 -59
  21. labnotes-0.0.19/src/labnotes/list_experiments.py +0 -23
  22. labnotes-0.0.19/src/labnotes/open_experiments.py +0 -26
  23. labnotes-0.0.19/src/labnotes/open_runs.py +0 -13
  24. labnotes-0.0.19/src/labnotes/read_design.py +0 -15
  25. labnotes-0.0.19/src/labnotes/read_note.py +0 -12
  26. labnotes-0.0.19/src/labnotes/run_columns.py +0 -6
  27. labnotes-0.0.19/src/labnotes/study_progress.py +0 -38
  28. labnotes-0.0.19/src/labnotes/study_runs.py +0 -5
  29. labnotes-0.0.19/src/labnotes/summarize_design.py +0 -23
  30. {labnotes-0.0.19 → labnotes-0.0.21}/.gitignore +0 -0
  31. {labnotes-0.0.19 → labnotes-0.0.21}/CHANGELOG.md +0 -0
  32. {labnotes-0.0.19 → labnotes-0.0.21}/MIGRATIONS.md +0 -0
  33. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/__init__.py +0 -0
  34. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/asset_url.py +0 -0
  35. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/cell_progress.py +0 -0
  36. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/cli.py +0 -0
  37. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/context_guide.py +0 -0
  38. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/count_runs.py +0 -0
  39. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/create_experiment.py +0 -0
  40. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/data_html.py +0 -0
  41. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/default_config_text.py +0 -0
  42. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/drain_watch.py +0 -0
  43. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/experiments.py +0 -0
  44. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/format_value.py +0 -0
  45. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/init.py +0 -0
  46. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/load_config.py +0 -0
  47. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/models.py +0 -0
  48. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/notebook_kernel.py +0 -0
  49. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/output_html.py +0 -0
  50. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/parse_run_view.py +0 -0
  51. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/pivot_cells.py +0 -0
  52. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/print_rows.py +0 -0
  53. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/python_blocks.py +0 -0
  54. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/read_config.py +0 -0
  55. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/read_path.py +0 -0
  56. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/render_markdown.py +0 -0
  57. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/__init__.py +0 -0
  58. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/describe_event.py +0 -0
  59. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/lock_path.py +0 -0
  60. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/new_run_id.py +0 -0
  61. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/param_args.py +0 -0
  62. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/param_grid.py +0 -0
  63. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/plan_runs.py +0 -0
  64. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/read_experiment_config.py +0 -0
  65. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/run_folder.py +0 -0
  66. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/run_headless.py +0 -0
  67. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/shell_command.py +0 -0
  68. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/start_experiment.py +0 -0
  69. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/stop_then_kill.py +0 -0
  70. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/write_run_json.py +0 -0
  71. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/saved_outputs.py +0 -0
  72. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/sort_href.py +0 -0
  73. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/sql_quote.py +0 -0
  74. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/static/labnotes.css +0 -0
  75. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/static/row-link.js +0 -0
  76. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/static/run-cell.js +0 -0
  77. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/static/view-toggle.js +0 -0
  78. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/stop_idle_kernels.py +0 -0
  79. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/__init__.py +0 -0
  80. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/experiment_rows.py +0 -0
  81. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/file_prefix.py +0 -0
  82. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/iso_time.py +0 -0
  83. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/list_experiment_runs.py +0 -0
  84. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/open_tables.py +0 -0
  85. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/path_glob.py +0 -0
  86. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/path_pattern.py +0 -0
  87. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/read_experiment.py +0 -0
  88. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/read_file.py +0 -0
  89. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/read_run.py +0 -0
  90. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/summarize_experiments.py +0 -0
  91. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/templates/base.html +0 -0
  92. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/templates/macros.html +0 -0
  93. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tui/__init__.py +0 -0
  94. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tui/format_elapsed.py +0 -0
  95. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tui/run_app.py +0 -0
  96. {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tui/run_tui.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: labnotes
3
- Version: 0.0.19
3
+ Version: 0.0.21
4
4
  Summary: Browse experiment designs, notes, and run data from a folder.
5
5
  Project-URL: Repository, https://github.com/thekevinscott/labnotes
6
6
  Author-email: Kevin Scott <me@thekevinscott.com>
@@ -14,7 +14,6 @@ Requires-Dist: ipykernel>=7.4.0
14
14
  Requires-Dist: jinja2>=3.1.6
15
15
  Requires-Dist: jupyter-client>=8.10.0
16
16
  Requires-Dist: markdown-it-py>=4.2.0
17
- Requires-Dist: pyyaml>=6.0.3
18
17
  Requires-Dist: rich>=14
19
18
  Requires-Dist: starlette>=1.8.0
20
19
  Requires-Dist: textual>=8.2.8
@@ -114,29 +113,18 @@ line per event instead.
114
113
  - A second `run` of the same experiment fails while the first holds the lock.
115
114
  - It exits 1 if any run failed.
116
115
 
117
- `labnotes server` doesn't read `.labnotes.toml` yet. Give it the two folders
118
- it uses with `--experiments-folder` and `--data-folder`:
119
-
120
- - `--experiments-folder` holds one subfolder per experiment, each with a
121
- `design.yaml` (with a `study` key) and any number of markdown notes.
122
- - `--data-folder` holds the run data and a `.dirsql.toml` that declares a
123
- `runs` table with a `study` column. An experiment's runs are the rows whose
124
- `study` matches its design's.
125
-
126
- That serves a local site at http://127.0.0.1:8000: a table of experiments,
127
- and a page per experiment with its parameters, its notes, progress per cell of
128
- the design grid, and every run so far. Both folders are watched, so a reload
129
- shows new runs and edited notes.
130
-
131
- `design.yaml` keys the site reads: `study`, `fixed`, `varies` (each key a list
132
- of values), `runs_per_cell`, and `completed_run`, a SQL condition over the
133
- `runs` table that marks a run as complete. Anything else shows up as-is.
116
+ `labnotes server` serves a local site at http://127.0.0.1:8000 from the same
117
+ `.labnotes.toml`: a table of experiments, and a page per experiment with its
118
+ params, its markdown files, progress per cell of the params grid, and every
119
+ run so far. A cell is full once it has `repeats` runs that exited 0 and aren't
120
+ excluded, the same count `run` uses. The project is watched, so a reload shows
121
+ new runs and edited notes. Python blocks in a note run with the root as the
122
+ working folder.
134
123
 
135
124
  `-h`/`--host` and `-p`/`--port` change where it listens; `-h 0.0.0.0` (or a
136
125
  Tailscale IP) serves beyond localhost. Every other uvicorn option passes
137
126
  through: `--reload`, `--workers`, `--proxy-headers`, `--root-path`, the
138
- `--ssl-*` set, and the rest. `labnotes server --help` lists them. `LABNOTES_EXPERIMENTS_FOLDER` and
139
- `LABNOTES_DATA_FOLDER` can stand in for the folder flags.
127
+ `--ssl-*` set, and the rest. `labnotes server --help` lists them.
140
128
 
141
129
  ## Agent skill
142
130
 
@@ -91,29 +91,18 @@ line per event instead.
91
91
  - A second `run` of the same experiment fails while the first holds the lock.
92
92
  - It exits 1 if any run failed.
93
93
 
94
- `labnotes server` doesn't read `.labnotes.toml` yet. Give it the two folders
95
- it uses with `--experiments-folder` and `--data-folder`:
96
-
97
- - `--experiments-folder` holds one subfolder per experiment, each with a
98
- `design.yaml` (with a `study` key) and any number of markdown notes.
99
- - `--data-folder` holds the run data and a `.dirsql.toml` that declares a
100
- `runs` table with a `study` column. An experiment's runs are the rows whose
101
- `study` matches its design's.
102
-
103
- That serves a local site at http://127.0.0.1:8000: a table of experiments,
104
- and a page per experiment with its parameters, its notes, progress per cell of
105
- the design grid, and every run so far. Both folders are watched, so a reload
106
- shows new runs and edited notes.
107
-
108
- `design.yaml` keys the site reads: `study`, `fixed`, `varies` (each key a list
109
- of values), `runs_per_cell`, and `completed_run`, a SQL condition over the
110
- `runs` table that marks a run as complete. Anything else shows up as-is.
94
+ `labnotes server` serves a local site at http://127.0.0.1:8000 from the same
95
+ `.labnotes.toml`: a table of experiments, and a page per experiment with its
96
+ params, its markdown files, progress per cell of the params grid, and every
97
+ run so far. A cell is full once it has `repeats` runs that exited 0 and aren't
98
+ excluded, the same count `run` uses. The project is watched, so a reload shows
99
+ new runs and edited notes. Python blocks in a note run with the root as the
100
+ working folder.
111
101
 
112
102
  `-h`/`--host` and `-p`/`--port` change where it listens; `-h 0.0.0.0` (or a
113
103
  Tailscale IP) serves beyond localhost. Every other uvicorn option passes
114
104
  through: `--reload`, `--workers`, `--proxy-headers`, `--root-path`, the
115
- `--ssl-*` set, and the rest. `labnotes server --help` lists them. `LABNOTES_EXPERIMENTS_FOLDER` and
116
- `LABNOTES_DATA_FOLDER` can stand in for the folder flags.
105
+ `--ssl-*` set, and the rest. `labnotes server --help` lists them.
117
106
 
118
107
  ## Agent skill
119
108
 
@@ -19,7 +19,6 @@ dependencies = [
19
19
  "jinja2>=3.1.6",
20
20
  "jupyter-client>=8.10.0",
21
21
  "markdown-it-py>=4.2.0",
22
- "pyyaml>=6.0.3",
23
22
  "rich>=14",
24
23
  "starlette>=1.8.0",
25
24
  "textual>=8.2.8",
@@ -38,7 +37,6 @@ dev = [
38
37
  "mypy>=2.4.0",
39
38
  "pytest>=8.0",
40
39
  "ruff==0.8.4",
41
- "types-pyyaml>=6.0.12.20260906",
42
40
  ]
43
41
 
44
42
  [tool.hatch.version]
@@ -73,7 +73,8 @@ A script run by labnotes should:
73
73
 
74
74
  - take each param as a `--name value` flag, parsing lists and tables from JSON;
75
75
  - take its output folder as an argument, passed in `command` as `{run_dir}`, and write everything it produces there;
76
- - exit non-zero when it fails, so the run counts as failed and runs again next time.
76
+ - exit non-zero when it fails, so the run counts as failed and runs again next time;
77
+ - stop any background process it starts before it exits; the run ends when `command` exits.
77
78
 
78
79
  Runs can take a long time. Start `run` in the background, then check on it with `labnotes experiments foo runs`; a run with no `ended` is still going.
79
80
 
@@ -89,12 +90,4 @@ line per event instead.
89
90
  - A second `run` of the same experiment fails while the first holds the lock.
90
91
  - It exits 1 if any run failed.
91
92
 
92
- `labnotes server` doesn't read `.labnotes.toml` yet: pass `--experiments-folder` and `--data-folder`, or set `LABNOTES_EXPERIMENTS_FOLDER` and `LABNOTES_DATA_FOLDER`. The experiments folder has one subfolder per experiment, each with a `design.yaml` and markdown notes. The data folder holds run data and a `.dirsql.toml` declaring a `runs` table with a `study` column.
93
-
94
- The run data lives in the data folder. Query it with [dirsql](https://dirsql.dev) and the data folder's `.dirsql.toml`, which declares the `runs` table:
95
-
96
- ```sh
97
- uvx dirsql -c data/.dirsql.toml "SELECT * FROM runs LIMIT 1" --format json
98
- ```
99
-
100
- `uvx labnotes server` serves the experiments and their runs as a local site.
93
+ `uvx labnotes server` serves the experiments, their markdown files and their runs as a local site, read from the same `.labnotes.toml`. A grid cell is full once it has `repeats` runs that exited 0 and aren't excluded.
@@ -14,17 +14,17 @@ from starlette.templating import Jinja2Templates
14
14
 
15
15
  from labnotes.asset_url import asset_url
16
16
  from labnotes.drain_watch import drain_watch
17
+ from labnotes.experiment_notes import experiment_notes
17
18
  from labnotes.format_value import format_value
18
19
  from labnotes.list_experiments import list_experiments
19
20
  from labnotes.load_experiment import load_experiment
21
+ from labnotes.models import LabnotesConfig
20
22
  from labnotes.notebook_kernel import NotebookKernel
21
- from labnotes.open_experiments import open_experiments
22
- from labnotes.open_runs import open_runs
23
23
  from labnotes.pivot_cells import pivot_cells
24
24
  from labnotes.python_blocks import python_blocks
25
25
  from labnotes.saved_outputs import saved_outputs
26
- from labnotes.sql_quote import quote_string
27
26
  from labnotes.stop_idle_kernels import stop_idle_kernels
27
+ from labnotes.tables.open_tables import open_tables
28
28
 
29
29
  HERE = Path(__file__).parent
30
30
  templates = Jinja2Templates(directory=HERE / "templates")
@@ -32,20 +32,15 @@ templates.env.filters["value"] = format_value
32
32
  templates.env.globals["asset"] = asset_url
33
33
 
34
34
 
35
- def create_app(
36
- experiments_folder: Path, data_folder: Path, kernel_idle_minutes: float
37
- ) -> Starlette:
35
+ def create_app(config: LabnotesConfig, kernel_idle_minutes: float) -> Starlette:
38
36
  kernels: dict[tuple[str, str], NotebookKernel] = {}
39
37
 
40
38
  @asynccontextmanager
41
39
  async def lifespan(app: Starlette) -> AsyncIterator[None]:
42
- app.state.experiments = open_experiments(experiments_folder)
43
- app.state.runs = open_runs(data_folder)
44
- await app.state.experiments.ready()
45
- await app.state.runs.ready()
40
+ app.state.tables = open_tables(config)
41
+ await app.state.tables.ready()
46
42
  watchers = [
47
- asyncio.create_task(drain_watch(app.state.experiments)),
48
- asyncio.create_task(drain_watch(app.state.runs)),
43
+ asyncio.create_task(drain_watch(app.state.tables)),
49
44
  asyncio.create_task(stop_idle_kernels(kernels, kernel_idle_minutes)),
50
45
  ]
51
46
  yield
@@ -55,20 +50,18 @@ def create_app(
55
50
  await kernel.stop()
56
51
 
57
52
  async def index(request: Request) -> Response:
58
- state = request.app.state
59
- experiments = await list_experiments(state.experiments, state.runs)
53
+ experiments = await list_experiments(request.app.state.tables)
60
54
  return templates.TemplateResponse(
61
55
  request,
62
56
  "index.html",
63
- {"experiments": experiments, "folder": experiments_folder.resolve().name},
57
+ {"experiments": experiments, "folder": config.root.name},
64
58
  )
65
59
 
66
60
  async def experiment(request: Request) -> Response:
67
- state = request.app.state
68
61
  name = request.path_params["name"]
69
62
  loaded = await load_experiment(
70
- state.experiments,
71
- state.runs,
63
+ request.app.state.tables,
64
+ config.root,
72
65
  name,
73
66
  request.query_params,
74
67
  lambda note, content: saved_outputs(kernels.get((name, note)), content),
@@ -88,14 +81,13 @@ def create_app(
88
81
  raise HTTPException(415, "Send JSON")
89
82
  name = request.path_params["name"]
90
83
  note = request.path_params["note"]
91
- found = await request.app.state.experiments.query(
92
- "SELECT content FROM notes"
93
- f" WHERE experiment = {quote_string(name)} AND name = {quote_string(note)}"
84
+ found = await experiment_notes(
85
+ request.app.state.tables, config.root, name, note
94
86
  )
95
87
  if not found:
96
88
  raise HTTPException(404, "No such note")
97
- [row] = found
98
- return str(row["content"])
89
+ [note] = found
90
+ return note["content"]
99
91
 
100
92
  async def run_block(request: Request) -> Response:
101
93
  blocks = python_blocks(await note_content(request))
@@ -104,7 +96,7 @@ def create_app(
104
96
  if not isinstance(block, int) or not 0 <= block < len(blocks):
105
97
  raise HTTPException(400, "No such block")
106
98
  if key not in kernels:
107
- kernels[key] = NotebookKernel(data_folder)
99
+ kernels[key] = NotebookKernel(config.root)
108
100
  return HTMLResponse(await kernels[key].run(blocks[block]))
109
101
 
110
102
  async def restart_kernel(request: Request) -> Response:
@@ -118,9 +110,13 @@ def create_app(
118
110
  routes=[
119
111
  Route("/", index),
120
112
  Route("/experiments/{name}", experiment),
121
- Route("/experiments/{name}/notes/{note}/run", run_block, methods=["POST"]),
122
113
  Route(
123
- "/experiments/{name}/notes/{note}/restart",
114
+ "/experiments/{name}/notes/{note:path}/run",
115
+ run_block,
116
+ methods=["POST"],
117
+ ),
118
+ Route(
119
+ "/experiments/{name}/notes/{note:path}/restart",
124
120
  restart_kernel,
125
121
  methods=["POST"],
126
122
  ),
@@ -0,0 +1,23 @@
1
+ from pathlib import Path
2
+
3
+ from dirsql import DirSQL
4
+
5
+ from labnotes.sql_quote import quote_string
6
+
7
+
8
+ async def experiment_notes(
9
+ tables: DirSQL, root: Path, experiment: str, note: str | None = None
10
+ ) -> list[dict[str, str]]:
11
+ where = f"experiment = {quote_string(experiment)} AND filename LIKE '%.md'"
12
+ if note is not None:
13
+ where += f" AND filename = {quote_string(note)}"
14
+ rows = await tables.query(
15
+ f"SELECT filename, path FROM files WHERE {where} ORDER BY filename"
16
+ )
17
+ return [
18
+ {
19
+ "name": str(row["filename"]),
20
+ "content": (root / str(row["path"])).read_text(encoding="utf-8"),
21
+ }
22
+ for row in rows
23
+ ]
@@ -0,0 +1,25 @@
1
+ from typing import cast
2
+
3
+ from dirsql import DirSQL
4
+
5
+ from labnotes.cell_progress import cell_progress
6
+ from labnotes.count_runs import count_runs
7
+
8
+
9
+ async def experiment_progress(
10
+ tables: DirSQL, source: str, summary: dict[str, object]
11
+ ) -> dict[str, object]:
12
+ varies = cast(dict[str, list[object]], summary["varies"])
13
+ completed_run = str(summary["completed_run"])
14
+ runs_per_cell = summary["runs_per_cell"]
15
+ [totals] = await count_runs(tables, source, completed_run)
16
+ progress: dict[str, object] = {**totals, "filled": None, "cells": []}
17
+ if runs_per_cell is None:
18
+ return progress
19
+ grouped = await count_runs(tables, source, completed_run, group_by=list(varies))
20
+ cells = cell_progress(varies, grouped)
21
+ progress["cells"] = cells
22
+ progress["filled"] = sum(
23
+ min(cast(int, cell["completed"]), cast(int, runs_per_cell)) for cell in cells
24
+ )
25
+ return progress
@@ -0,0 +1,13 @@
1
+ from labnotes.sql_quote import quote_identifier, quote_string
2
+
3
+
4
+ def experiment_runs(experiment: str, params: list[str]) -> str:
5
+ columns = "".join(
6
+ f", params ->> {quote_string(f'$.{quote_identifier(key)}')}"
7
+ f" AS {quote_identifier(key)}"
8
+ for key in params
9
+ )
10
+ return (
11
+ f"(SELECT *{columns} FROM runs"
12
+ f" WHERE experiment = {quote_string(experiment)})"
13
+ )
@@ -0,0 +1,26 @@
1
+ import json
2
+ from typing import cast
3
+
4
+ from dirsql import DirSQL
5
+
6
+ from labnotes.experiment_progress import experiment_progress
7
+ from labnotes.experiment_runs import experiment_runs
8
+ from labnotes.summarize_config import summarize_config
9
+
10
+
11
+ async def list_experiments(tables: DirSQL) -> list[dict[str, object]]:
12
+ rows = await tables.query(
13
+ "SELECT name, display_name, config FROM experiments ORDER BY name"
14
+ )
15
+ listed = []
16
+ for row in rows:
17
+ name = str(row["name"])
18
+ config = None if row["config"] is None else json.loads(str(row["config"]))
19
+ summary = summarize_config(config)
20
+ varies = cast(dict[str, list[object]], summary["varies"])
21
+ source = experiment_runs(name, list(varies))
22
+ progress = await experiment_progress(tables, source, summary)
23
+ listed.append(
24
+ {"name": name, "display_name": row["display_name"], **summary, **progress}
25
+ )
26
+ return listed
@@ -23,7 +23,7 @@ async def list_runs(
23
23
  elif view.sort:
24
24
  order = f"{quote_identifier(view.sort)} {direction}"
25
25
  else:
26
- order = "2 DESC"
26
+ order = "created DESC, id DESC"
27
27
  return await runs.query(
28
28
  f"SELECT ({completed_run}) AS labnotes_completed, * FROM {source}"
29
29
  f"{filtered} ORDER BY {order}"
@@ -1,19 +1,20 @@
1
1
  import json
2
2
  from collections.abc import Callable, Mapping, Sequence
3
+ from pathlib import Path
3
4
  from typing import cast
4
5
 
5
6
  from dirsql import DirSQL
6
7
 
8
+ from labnotes.experiment_notes import experiment_notes
9
+ from labnotes.experiment_progress import experiment_progress
10
+ from labnotes.experiment_runs import experiment_runs
7
11
  from labnotes.list_runs import list_runs
8
12
  from labnotes.parse_run_view import parse_run_view
9
13
  from labnotes.python_blocks import python_blocks
10
14
  from labnotes.render_markdown import render_markdown
11
- from labnotes.run_columns import run_columns
12
15
  from labnotes.sort_href import sort_href
13
16
  from labnotes.sql_quote import quote_string
14
- from labnotes.study_progress import study_progress
15
- from labnotes.study_runs import study_runs
16
- from labnotes.summarize_design import summarize_design
17
+ from labnotes.summarize_config import summarize_config
17
18
 
18
19
 
19
20
  def _no_outputs(note: str, content: str) -> Sequence[str]:
@@ -21,53 +22,45 @@ def _no_outputs(note: str, content: str) -> Sequence[str]:
21
22
 
22
23
 
23
24
  async def load_experiment(
24
- experiments: DirSQL,
25
- runs: DirSQL,
25
+ tables: DirSQL,
26
+ root: Path,
26
27
  name: str,
27
28
  params: Mapping[str, str],
28
29
  outputs_for: Callable[[str, str], Sequence[str]] = _no_outputs,
29
30
  ) -> dict[str, object] | None:
30
- found = await experiments.query(
31
- f"SELECT name, study, design FROM experiments WHERE name = {quote_string(name)}"
31
+ found = await tables.query(
32
+ "SELECT name, display_name, config FROM experiments"
33
+ f" WHERE name = {quote_string(name)}"
32
34
  )
33
35
  if not found:
34
36
  return None
35
37
  [row] = found
36
- notes = await experiments.query(
37
- "SELECT name, content FROM notes"
38
- f" WHERE experiment = {quote_string(name)} ORDER BY name"
39
- )
40
- summary = summarize_design(json.loads(str(row["design"])))
38
+ config = None if row["config"] is None else json.loads(str(row["config"]))
39
+ summary = summarize_config(config)
41
40
  varies = cast(dict[str, list[object]], summary["varies"])
42
- study = None if row["study"] is None else str(row["study"])
43
- columns = await run_columns(runs)
44
- progress = await study_progress(runs, study, summary, columns)
45
- shown = columns[:1] + [key for key in varies if key in columns[1:]]
41
+ source = experiment_runs(name, list(varies))
42
+ progress = await experiment_progress(tables, source, summary)
43
+ shown = ["id", *(key for key in varies if key != "id")]
46
44
  view = parse_run_view(params, shown)
47
- listed = (
48
- []
49
- if study is None
50
- else await list_runs(
51
- runs, study_runs(study), str(summary["completed_run"]), view
52
- )
53
- )
45
+ notes = await experiment_notes(tables, root, name)
54
46
  return {
55
47
  "name": row["name"],
56
- "study": study,
48
+ "display_name": row["display_name"],
57
49
  **summary,
58
50
  **progress,
59
51
  "notes": [
60
52
  {
61
53
  "name": note["name"],
62
54
  "html": render_markdown(
63
- str(note["content"]),
64
- outputs_for(str(note["name"]), str(note["content"])),
55
+ note["content"], outputs_for(note["name"], note["content"])
65
56
  ),
66
- "runnable": bool(python_blocks(str(note["content"]))),
57
+ "runnable": bool(python_blocks(note["content"])),
67
58
  }
68
59
  for note in notes
69
60
  ],
70
- "runs_list": listed,
61
+ "runs_list": await list_runs(
62
+ tables, source, str(summary["completed_run"]), view
63
+ ),
71
64
  "run_columns": shown,
72
65
  "run_view": view,
73
66
  "sort_hrefs": {
@@ -3,6 +3,7 @@ import time
3
3
  from pathlib import Path
4
4
 
5
5
  from labnotes.runner.shell_command import shell_command
6
+ from labnotes.runner.wait_for_group import wait_for_group
6
7
  from labnotes.runner.write_run_json import write_run_json
7
8
  from labnotes.tables.iso_time import iso_time
8
9
 
@@ -13,6 +14,7 @@ async def execute_run(
13
14
  command: str,
14
15
  params: dict[str, object],
15
16
  running: set[asyncio.subprocess.Process],
17
+ stopping: asyncio.Event,
16
18
  ) -> dict[str, object]:
17
19
  run: dict[str, object] = {
18
20
  "params": params,
@@ -42,6 +44,11 @@ async def execute_run(
42
44
  running.add(process)
43
45
  try:
44
46
  run["exit_code"] = await process.wait()
47
+ # The shell can exit on SIGTERM before what it started. Once
48
+ # stopping, the run stays killable until its whole group is
49
+ # gone. A normal exit ends the run even if helpers linger.
50
+ if stopping.is_set():
51
+ await wait_for_group(process.pid)
45
52
  finally:
46
53
  running.discard(process)
47
54
  run["ended"] = iso_time(time.time())
@@ -32,7 +32,7 @@ async def run_experiment(
32
32
  report({"event": "planned", "runs": len(plan), "concurrency": concurrency})
33
33
  running: set[asyncio.subprocess.Process] = set()
34
34
  results: list[dict[str, object]] = []
35
- stopping = False
35
+ stopping = asyncio.Event()
36
36
 
37
37
  def signal_runs(signal_number: int) -> None:
38
38
  for process in running:
@@ -40,10 +40,9 @@ async def run_experiment(
40
40
  os.killpg(process.pid, signal_number)
41
41
 
42
42
  def interrupt() -> None:
43
- nonlocal stopping
44
43
  plan.clear()
45
- if not stopping:
46
- stopping = True
44
+ if not stopping.is_set():
45
+ stopping.set()
47
46
  report({"event": "stopping"})
48
47
  signal_runs(signal.SIGTERM)
49
48
  elif running:
@@ -60,7 +59,7 @@ async def run_experiment(
60
59
  folder.mkdir(parents=True)
61
60
  report({"event": "started", "run": run, "params": params})
62
61
  result = await execute_run(
63
- config.root, folder, experiment.command, params, running
62
+ config.root, folder, experiment.command, params, running, stopping
64
63
  )
65
64
  report({"event": "ended", "run": run, **result})
66
65
  results.append(result)
@@ -0,0 +1,11 @@
1
+ import asyncio
2
+ import os
3
+
4
+
5
+ async def wait_for_group(group: int) -> None:
6
+ while True:
7
+ try:
8
+ os.killpg(group, 0)
9
+ except ProcessLookupError:
10
+ return
11
+ await asyncio.sleep(0.1)
@@ -4,15 +4,13 @@ from pathlib import Path
4
4
  from starlette.applications import Starlette
5
5
 
6
6
  from labnotes.create_app import create_app
7
+ from labnotes.read_config import read_config
7
8
 
8
- EXPERIMENTS_FOLDER = "LABNOTES_EXPERIMENTS_FOLDER"
9
- DATA_FOLDER = "LABNOTES_DATA_FOLDER"
9
+ CONFIG = "LABNOTES_CONFIG"
10
10
  KERNEL_IDLE_MINUTES = "LABNOTES_KERNEL_IDLE_MINUTES"
11
11
 
12
12
 
13
13
  def serve_app() -> Starlette:
14
14
  return create_app(
15
- Path(os.environ[EXPERIMENTS_FOLDER]),
16
- Path(os.environ[DATA_FOLDER]),
17
- int(os.environ[KERNEL_IDLE_MINUTES]),
15
+ read_config(Path(os.environ[CONFIG])), int(os.environ[KERNEL_IDLE_MINUTES])
18
16
  )
@@ -1,29 +1,18 @@
1
1
  import os
2
- from pathlib import Path
3
2
 
4
3
  import click
5
4
  from uvicorn.main import main as uvicorn_main
6
5
 
7
- from labnotes.serve_app import DATA_FOLDER, EXPERIMENTS_FOLDER, KERNEL_IDLE_MINUTES
6
+ from labnotes.load_config import load_config
7
+ from labnotes.read_config import CONFIG_FILE
8
+ from labnotes.serve_app import CONFIG, KERNEL_IDLE_MINUTES
8
9
 
9
10
  # labnotes owns these; the rest of uvicorn's options pass straight through.
10
11
  OWN_OPTIONS = {"app", "factory", "app_dir", "version", "host", "port"}
11
12
 
12
13
 
13
- @click.command(help="Serve the experiments and their run data as a local site.")
14
- @click.option(
15
- "--experiments-folder",
16
- type=click.Path(path_type=Path),
17
- envvar=EXPERIMENTS_FOLDER,
18
- required=True,
19
- help="One subfolder per experiment, each with a design.yaml and markdown notes.",
20
- )
21
- @click.option(
22
- "--data-folder",
23
- type=click.Path(path_type=Path),
24
- envvar=DATA_FOLDER,
25
- required=True,
26
- help="Run data, with a .dirsql.toml declaring a runs table that has a study column.",
14
+ @click.command(
15
+ help="Serve the experiments, their notes and their runs as a local site."
27
16
  )
28
17
  @click.option(
29
18
  "--kernel-idle-minutes",
@@ -44,14 +33,16 @@ OWN_OPTIONS = {"app", "factory", "app_dir", "version", "host", "port"}
44
33
  @click.pass_context
45
34
  def server(
46
35
  ctx: click.Context,
47
- experiments_folder: Path,
48
- data_folder: Path,
49
36
  kernel_idle_minutes: int,
50
37
  **server_options: object,
51
38
  ) -> None:
39
+ config = load_config(ctx)
40
+ if config is None:
41
+ raise click.ClickException(
42
+ f"No {CONFIG_FILE} here. Run labnotes init, or pass -c PATH."
43
+ )
52
44
  # Through the environment so --reload and --workers child processes see them.
53
- os.environ[EXPERIMENTS_FOLDER] = str(experiments_folder.resolve())
54
- os.environ[DATA_FOLDER] = str(data_folder.resolve())
45
+ os.environ[CONFIG] = str(config.path.resolve())
55
46
  os.environ[KERNEL_IDLE_MINUTES] = str(kernel_idle_minutes)
56
47
  ctx.invoke(
57
48
  uvicorn_main,
@@ -0,0 +1,26 @@
1
+ from math import prod
2
+
3
+ # What the runner counts toward repeats.
4
+ COMPLETED_RUN = "exit_code = 0 AND NOT coalesce(excluded, 0)"
5
+
6
+
7
+ def summarize_config(config: dict[str, object] | None) -> dict[str, object]:
8
+ config = config or {}
9
+ params = config.get("params")
10
+ varies = {
11
+ key: values if isinstance(values, list) else [values]
12
+ for key, values in (params if isinstance(params, dict) else {}).items()
13
+ }
14
+ repeats = config.get("repeats", 1) if config else None
15
+ repeats = repeats if isinstance(repeats, int) else None
16
+ cells = prod(len(values) for values in varies.values())
17
+ command = config.get("command")
18
+ return {
19
+ "fixed": {},
20
+ "varies": varies,
21
+ "cell_count": cells,
22
+ "runs_per_cell": repeats,
23
+ "target": cells * repeats if repeats is not None else None,
24
+ "completed_run": COMPLETED_RUN,
25
+ "other": {} if command is None else {"command": command},
26
+ }