labnotes 0.0.19__tar.gz → 0.0.21__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {labnotes-0.0.19 → labnotes-0.0.21}/PKG-INFO +9 -21
- {labnotes-0.0.19 → labnotes-0.0.21}/README.md +8 -19
- {labnotes-0.0.19 → labnotes-0.0.21}/pyproject.toml +0 -2
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/context.md +3 -10
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/create_app.py +22 -26
- labnotes-0.0.21/src/labnotes/experiment_notes.py +23 -0
- labnotes-0.0.21/src/labnotes/experiment_progress.py +25 -0
- labnotes-0.0.21/src/labnotes/experiment_runs.py +13 -0
- labnotes-0.0.21/src/labnotes/list_experiments.py +26 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/list_runs.py +1 -1
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/load_experiment.py +22 -29
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/execute_run.py +7 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/run_experiment.py +4 -5
- labnotes-0.0.21/src/labnotes/runner/wait_for_group.py +11 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/serve_app.py +3 -5
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/server.py +11 -20
- labnotes-0.0.21/src/labnotes/summarize_config.py +26 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/templates/experiment.html +7 -12
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/templates/index.html +3 -3
- {labnotes-0.0.19 → labnotes-0.0.21}/uv.lock +0 -59
- labnotes-0.0.19/src/labnotes/list_experiments.py +0 -23
- labnotes-0.0.19/src/labnotes/open_experiments.py +0 -26
- labnotes-0.0.19/src/labnotes/open_runs.py +0 -13
- labnotes-0.0.19/src/labnotes/read_design.py +0 -15
- labnotes-0.0.19/src/labnotes/read_note.py +0 -12
- labnotes-0.0.19/src/labnotes/run_columns.py +0 -6
- labnotes-0.0.19/src/labnotes/study_progress.py +0 -38
- labnotes-0.0.19/src/labnotes/study_runs.py +0 -5
- labnotes-0.0.19/src/labnotes/summarize_design.py +0 -23
- {labnotes-0.0.19 → labnotes-0.0.21}/.gitignore +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/CHANGELOG.md +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/MIGRATIONS.md +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/__init__.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/asset_url.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/cell_progress.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/cli.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/context_guide.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/count_runs.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/create_experiment.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/data_html.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/default_config_text.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/drain_watch.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/experiments.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/format_value.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/init.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/load_config.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/models.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/notebook_kernel.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/output_html.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/parse_run_view.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/pivot_cells.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/print_rows.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/python_blocks.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/read_config.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/read_path.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/render_markdown.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/__init__.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/describe_event.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/lock_path.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/new_run_id.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/param_args.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/param_grid.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/plan_runs.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/read_experiment_config.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/run_folder.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/run_headless.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/shell_command.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/start_experiment.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/stop_then_kill.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/runner/write_run_json.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/saved_outputs.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/sort_href.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/sql_quote.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/static/labnotes.css +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/static/row-link.js +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/static/run-cell.js +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/static/view-toggle.js +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/stop_idle_kernels.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/__init__.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/experiment_rows.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/file_prefix.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/iso_time.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/list_experiment_runs.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/open_tables.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/path_glob.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/path_pattern.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/read_experiment.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/read_file.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/read_run.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tables/summarize_experiments.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/templates/base.html +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/templates/macros.html +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tui/__init__.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tui/format_elapsed.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tui/run_app.py +0 -0
- {labnotes-0.0.19 → labnotes-0.0.21}/src/labnotes/tui/run_tui.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: labnotes
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.21
|
|
4
4
|
Summary: Browse experiment designs, notes, and run data from a folder.
|
|
5
5
|
Project-URL: Repository, https://github.com/thekevinscott/labnotes
|
|
6
6
|
Author-email: Kevin Scott <me@thekevinscott.com>
|
|
@@ -14,7 +14,6 @@ Requires-Dist: ipykernel>=7.4.0
|
|
|
14
14
|
Requires-Dist: jinja2>=3.1.6
|
|
15
15
|
Requires-Dist: jupyter-client>=8.10.0
|
|
16
16
|
Requires-Dist: markdown-it-py>=4.2.0
|
|
17
|
-
Requires-Dist: pyyaml>=6.0.3
|
|
18
17
|
Requires-Dist: rich>=14
|
|
19
18
|
Requires-Dist: starlette>=1.8.0
|
|
20
19
|
Requires-Dist: textual>=8.2.8
|
|
@@ -114,29 +113,18 @@ line per event instead.
|
|
|
114
113
|
- A second `run` of the same experiment fails while the first holds the lock.
|
|
115
114
|
- It exits 1 if any run failed.
|
|
116
115
|
|
|
117
|
-
`labnotes server`
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
`study` matches its design's.
|
|
125
|
-
|
|
126
|
-
That serves a local site at http://127.0.0.1:8000: a table of experiments,
|
|
127
|
-
and a page per experiment with its parameters, its notes, progress per cell of
|
|
128
|
-
the design grid, and every run so far. Both folders are watched, so a reload
|
|
129
|
-
shows new runs and edited notes.
|
|
130
|
-
|
|
131
|
-
`design.yaml` keys the site reads: `study`, `fixed`, `varies` (each key a list
|
|
132
|
-
of values), `runs_per_cell`, and `completed_run`, a SQL condition over the
|
|
133
|
-
`runs` table that marks a run as complete. Anything else shows up as-is.
|
|
116
|
+
`labnotes server` serves a local site at http://127.0.0.1:8000 from the same
|
|
117
|
+
`.labnotes.toml`: a table of experiments, and a page per experiment with its
|
|
118
|
+
params, its markdown files, progress per cell of the params grid, and every
|
|
119
|
+
run so far. A cell is full once it has `repeats` runs that exited 0 and aren't
|
|
120
|
+
excluded, the same count `run` uses. The project is watched, so a reload shows
|
|
121
|
+
new runs and edited notes. Python blocks in a note run with the root as the
|
|
122
|
+
working folder.
|
|
134
123
|
|
|
135
124
|
`-h`/`--host` and `-p`/`--port` change where it listens; `-h 0.0.0.0` (or a
|
|
136
125
|
Tailscale IP) serves beyond localhost. Every other uvicorn option passes
|
|
137
126
|
through: `--reload`, `--workers`, `--proxy-headers`, `--root-path`, the
|
|
138
|
-
`--ssl-*` set, and the rest. `labnotes server --help` lists them.
|
|
139
|
-
`LABNOTES_DATA_FOLDER` can stand in for the folder flags.
|
|
127
|
+
`--ssl-*` set, and the rest. `labnotes server --help` lists them.
|
|
140
128
|
|
|
141
129
|
## Agent skill
|
|
142
130
|
|
|
@@ -91,29 +91,18 @@ line per event instead.
|
|
|
91
91
|
- A second `run` of the same experiment fails while the first holds the lock.
|
|
92
92
|
- It exits 1 if any run failed.
|
|
93
93
|
|
|
94
|
-
`labnotes server`
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
`study` matches its design's.
|
|
102
|
-
|
|
103
|
-
That serves a local site at http://127.0.0.1:8000: a table of experiments,
|
|
104
|
-
and a page per experiment with its parameters, its notes, progress per cell of
|
|
105
|
-
the design grid, and every run so far. Both folders are watched, so a reload
|
|
106
|
-
shows new runs and edited notes.
|
|
107
|
-
|
|
108
|
-
`design.yaml` keys the site reads: `study`, `fixed`, `varies` (each key a list
|
|
109
|
-
of values), `runs_per_cell`, and `completed_run`, a SQL condition over the
|
|
110
|
-
`runs` table that marks a run as complete. Anything else shows up as-is.
|
|
94
|
+
`labnotes server` serves a local site at http://127.0.0.1:8000 from the same
|
|
95
|
+
`.labnotes.toml`: a table of experiments, and a page per experiment with its
|
|
96
|
+
params, its markdown files, progress per cell of the params grid, and every
|
|
97
|
+
run so far. A cell is full once it has `repeats` runs that exited 0 and aren't
|
|
98
|
+
excluded, the same count `run` uses. The project is watched, so a reload shows
|
|
99
|
+
new runs and edited notes. Python blocks in a note run with the root as the
|
|
100
|
+
working folder.
|
|
111
101
|
|
|
112
102
|
`-h`/`--host` and `-p`/`--port` change where it listens; `-h 0.0.0.0` (or a
|
|
113
103
|
Tailscale IP) serves beyond localhost. Every other uvicorn option passes
|
|
114
104
|
through: `--reload`, `--workers`, `--proxy-headers`, `--root-path`, the
|
|
115
|
-
`--ssl-*` set, and the rest. `labnotes server --help` lists them.
|
|
116
|
-
`LABNOTES_DATA_FOLDER` can stand in for the folder flags.
|
|
105
|
+
`--ssl-*` set, and the rest. `labnotes server --help` lists them.
|
|
117
106
|
|
|
118
107
|
## Agent skill
|
|
119
108
|
|
|
@@ -19,7 +19,6 @@ dependencies = [
|
|
|
19
19
|
"jinja2>=3.1.6",
|
|
20
20
|
"jupyter-client>=8.10.0",
|
|
21
21
|
"markdown-it-py>=4.2.0",
|
|
22
|
-
"pyyaml>=6.0.3",
|
|
23
22
|
"rich>=14",
|
|
24
23
|
"starlette>=1.8.0",
|
|
25
24
|
"textual>=8.2.8",
|
|
@@ -38,7 +37,6 @@ dev = [
|
|
|
38
37
|
"mypy>=2.4.0",
|
|
39
38
|
"pytest>=8.0",
|
|
40
39
|
"ruff==0.8.4",
|
|
41
|
-
"types-pyyaml>=6.0.12.20260906",
|
|
42
40
|
]
|
|
43
41
|
|
|
44
42
|
[tool.hatch.version]
|
|
@@ -73,7 +73,8 @@ A script run by labnotes should:
|
|
|
73
73
|
|
|
74
74
|
- take each param as a `--name value` flag, parsing lists and tables from JSON;
|
|
75
75
|
- take its output folder as an argument, passed in `command` as `{run_dir}`, and write everything it produces there;
|
|
76
|
-
- exit non-zero when it fails, so the run counts as failed and runs again next time
|
|
76
|
+
- exit non-zero when it fails, so the run counts as failed and runs again next time;
|
|
77
|
+
- stop any background process it starts before it exits; the run ends when `command` exits.
|
|
77
78
|
|
|
78
79
|
Runs can take a long time. Start `run` in the background, then check on it with `labnotes experiments foo runs`; a run with no `ended` is still going.
|
|
79
80
|
|
|
@@ -89,12 +90,4 @@ line per event instead.
|
|
|
89
90
|
- A second `run` of the same experiment fails while the first holds the lock.
|
|
90
91
|
- It exits 1 if any run failed.
|
|
91
92
|
|
|
92
|
-
`labnotes server`
|
|
93
|
-
|
|
94
|
-
The run data lives in the data folder. Query it with [dirsql](https://dirsql.dev) and the data folder's `.dirsql.toml`, which declares the `runs` table:
|
|
95
|
-
|
|
96
|
-
```sh
|
|
97
|
-
uvx dirsql -c data/.dirsql.toml "SELECT * FROM runs LIMIT 1" --format json
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
`uvx labnotes server` serves the experiments and their runs as a local site.
|
|
93
|
+
`uvx labnotes server` serves the experiments, their markdown files and their runs as a local site, read from the same `.labnotes.toml`. A grid cell is full once it has `repeats` runs that exited 0 and aren't excluded.
|
|
@@ -14,17 +14,17 @@ from starlette.templating import Jinja2Templates
|
|
|
14
14
|
|
|
15
15
|
from labnotes.asset_url import asset_url
|
|
16
16
|
from labnotes.drain_watch import drain_watch
|
|
17
|
+
from labnotes.experiment_notes import experiment_notes
|
|
17
18
|
from labnotes.format_value import format_value
|
|
18
19
|
from labnotes.list_experiments import list_experiments
|
|
19
20
|
from labnotes.load_experiment import load_experiment
|
|
21
|
+
from labnotes.models import LabnotesConfig
|
|
20
22
|
from labnotes.notebook_kernel import NotebookKernel
|
|
21
|
-
from labnotes.open_experiments import open_experiments
|
|
22
|
-
from labnotes.open_runs import open_runs
|
|
23
23
|
from labnotes.pivot_cells import pivot_cells
|
|
24
24
|
from labnotes.python_blocks import python_blocks
|
|
25
25
|
from labnotes.saved_outputs import saved_outputs
|
|
26
|
-
from labnotes.sql_quote import quote_string
|
|
27
26
|
from labnotes.stop_idle_kernels import stop_idle_kernels
|
|
27
|
+
from labnotes.tables.open_tables import open_tables
|
|
28
28
|
|
|
29
29
|
HERE = Path(__file__).parent
|
|
30
30
|
templates = Jinja2Templates(directory=HERE / "templates")
|
|
@@ -32,20 +32,15 @@ templates.env.filters["value"] = format_value
|
|
|
32
32
|
templates.env.globals["asset"] = asset_url
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def create_app(
|
|
36
|
-
experiments_folder: Path, data_folder: Path, kernel_idle_minutes: float
|
|
37
|
-
) -> Starlette:
|
|
35
|
+
def create_app(config: LabnotesConfig, kernel_idle_minutes: float) -> Starlette:
|
|
38
36
|
kernels: dict[tuple[str, str], NotebookKernel] = {}
|
|
39
37
|
|
|
40
38
|
@asynccontextmanager
|
|
41
39
|
async def lifespan(app: Starlette) -> AsyncIterator[None]:
|
|
42
|
-
app.state.
|
|
43
|
-
app.state.
|
|
44
|
-
await app.state.experiments.ready()
|
|
45
|
-
await app.state.runs.ready()
|
|
40
|
+
app.state.tables = open_tables(config)
|
|
41
|
+
await app.state.tables.ready()
|
|
46
42
|
watchers = [
|
|
47
|
-
asyncio.create_task(drain_watch(app.state.
|
|
48
|
-
asyncio.create_task(drain_watch(app.state.runs)),
|
|
43
|
+
asyncio.create_task(drain_watch(app.state.tables)),
|
|
49
44
|
asyncio.create_task(stop_idle_kernels(kernels, kernel_idle_minutes)),
|
|
50
45
|
]
|
|
51
46
|
yield
|
|
@@ -55,20 +50,18 @@ def create_app(
|
|
|
55
50
|
await kernel.stop()
|
|
56
51
|
|
|
57
52
|
async def index(request: Request) -> Response:
|
|
58
|
-
|
|
59
|
-
experiments = await list_experiments(state.experiments, state.runs)
|
|
53
|
+
experiments = await list_experiments(request.app.state.tables)
|
|
60
54
|
return templates.TemplateResponse(
|
|
61
55
|
request,
|
|
62
56
|
"index.html",
|
|
63
|
-
{"experiments": experiments, "folder":
|
|
57
|
+
{"experiments": experiments, "folder": config.root.name},
|
|
64
58
|
)
|
|
65
59
|
|
|
66
60
|
async def experiment(request: Request) -> Response:
|
|
67
|
-
state = request.app.state
|
|
68
61
|
name = request.path_params["name"]
|
|
69
62
|
loaded = await load_experiment(
|
|
70
|
-
state.
|
|
71
|
-
|
|
63
|
+
request.app.state.tables,
|
|
64
|
+
config.root,
|
|
72
65
|
name,
|
|
73
66
|
request.query_params,
|
|
74
67
|
lambda note, content: saved_outputs(kernels.get((name, note)), content),
|
|
@@ -88,14 +81,13 @@ def create_app(
|
|
|
88
81
|
raise HTTPException(415, "Send JSON")
|
|
89
82
|
name = request.path_params["name"]
|
|
90
83
|
note = request.path_params["note"]
|
|
91
|
-
found = await
|
|
92
|
-
|
|
93
|
-
f" WHERE experiment = {quote_string(name)} AND name = {quote_string(note)}"
|
|
84
|
+
found = await experiment_notes(
|
|
85
|
+
request.app.state.tables, config.root, name, note
|
|
94
86
|
)
|
|
95
87
|
if not found:
|
|
96
88
|
raise HTTPException(404, "No such note")
|
|
97
|
-
[
|
|
98
|
-
return
|
|
89
|
+
[note] = found
|
|
90
|
+
return note["content"]
|
|
99
91
|
|
|
100
92
|
async def run_block(request: Request) -> Response:
|
|
101
93
|
blocks = python_blocks(await note_content(request))
|
|
@@ -104,7 +96,7 @@ def create_app(
|
|
|
104
96
|
if not isinstance(block, int) or not 0 <= block < len(blocks):
|
|
105
97
|
raise HTTPException(400, "No such block")
|
|
106
98
|
if key not in kernels:
|
|
107
|
-
kernels[key] = NotebookKernel(
|
|
99
|
+
kernels[key] = NotebookKernel(config.root)
|
|
108
100
|
return HTMLResponse(await kernels[key].run(blocks[block]))
|
|
109
101
|
|
|
110
102
|
async def restart_kernel(request: Request) -> Response:
|
|
@@ -118,9 +110,13 @@ def create_app(
|
|
|
118
110
|
routes=[
|
|
119
111
|
Route("/", index),
|
|
120
112
|
Route("/experiments/{name}", experiment),
|
|
121
|
-
Route("/experiments/{name}/notes/{note}/run", run_block, methods=["POST"]),
|
|
122
113
|
Route(
|
|
123
|
-
"/experiments/{name}/notes/{note}/
|
|
114
|
+
"/experiments/{name}/notes/{note:path}/run",
|
|
115
|
+
run_block,
|
|
116
|
+
methods=["POST"],
|
|
117
|
+
),
|
|
118
|
+
Route(
|
|
119
|
+
"/experiments/{name}/notes/{note:path}/restart",
|
|
124
120
|
restart_kernel,
|
|
125
121
|
methods=["POST"],
|
|
126
122
|
),
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from dirsql import DirSQL
|
|
4
|
+
|
|
5
|
+
from labnotes.sql_quote import quote_string
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
async def experiment_notes(
|
|
9
|
+
tables: DirSQL, root: Path, experiment: str, note: str | None = None
|
|
10
|
+
) -> list[dict[str, str]]:
|
|
11
|
+
where = f"experiment = {quote_string(experiment)} AND filename LIKE '%.md'"
|
|
12
|
+
if note is not None:
|
|
13
|
+
where += f" AND filename = {quote_string(note)}"
|
|
14
|
+
rows = await tables.query(
|
|
15
|
+
f"SELECT filename, path FROM files WHERE {where} ORDER BY filename"
|
|
16
|
+
)
|
|
17
|
+
return [
|
|
18
|
+
{
|
|
19
|
+
"name": str(row["filename"]),
|
|
20
|
+
"content": (root / str(row["path"])).read_text(encoding="utf-8"),
|
|
21
|
+
}
|
|
22
|
+
for row in rows
|
|
23
|
+
]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from typing import cast
|
|
2
|
+
|
|
3
|
+
from dirsql import DirSQL
|
|
4
|
+
|
|
5
|
+
from labnotes.cell_progress import cell_progress
|
|
6
|
+
from labnotes.count_runs import count_runs
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
async def experiment_progress(
|
|
10
|
+
tables: DirSQL, source: str, summary: dict[str, object]
|
|
11
|
+
) -> dict[str, object]:
|
|
12
|
+
varies = cast(dict[str, list[object]], summary["varies"])
|
|
13
|
+
completed_run = str(summary["completed_run"])
|
|
14
|
+
runs_per_cell = summary["runs_per_cell"]
|
|
15
|
+
[totals] = await count_runs(tables, source, completed_run)
|
|
16
|
+
progress: dict[str, object] = {**totals, "filled": None, "cells": []}
|
|
17
|
+
if runs_per_cell is None:
|
|
18
|
+
return progress
|
|
19
|
+
grouped = await count_runs(tables, source, completed_run, group_by=list(varies))
|
|
20
|
+
cells = cell_progress(varies, grouped)
|
|
21
|
+
progress["cells"] = cells
|
|
22
|
+
progress["filled"] = sum(
|
|
23
|
+
min(cast(int, cell["completed"]), cast(int, runs_per_cell)) for cell in cells
|
|
24
|
+
)
|
|
25
|
+
return progress
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from labnotes.sql_quote import quote_identifier, quote_string
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def experiment_runs(experiment: str, params: list[str]) -> str:
|
|
5
|
+
columns = "".join(
|
|
6
|
+
f", params ->> {quote_string(f'$.{quote_identifier(key)}')}"
|
|
7
|
+
f" AS {quote_identifier(key)}"
|
|
8
|
+
for key in params
|
|
9
|
+
)
|
|
10
|
+
return (
|
|
11
|
+
f"(SELECT *{columns} FROM runs"
|
|
12
|
+
f" WHERE experiment = {quote_string(experiment)})"
|
|
13
|
+
)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import cast
|
|
3
|
+
|
|
4
|
+
from dirsql import DirSQL
|
|
5
|
+
|
|
6
|
+
from labnotes.experiment_progress import experiment_progress
|
|
7
|
+
from labnotes.experiment_runs import experiment_runs
|
|
8
|
+
from labnotes.summarize_config import summarize_config
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
async def list_experiments(tables: DirSQL) -> list[dict[str, object]]:
|
|
12
|
+
rows = await tables.query(
|
|
13
|
+
"SELECT name, display_name, config FROM experiments ORDER BY name"
|
|
14
|
+
)
|
|
15
|
+
listed = []
|
|
16
|
+
for row in rows:
|
|
17
|
+
name = str(row["name"])
|
|
18
|
+
config = None if row["config"] is None else json.loads(str(row["config"]))
|
|
19
|
+
summary = summarize_config(config)
|
|
20
|
+
varies = cast(dict[str, list[object]], summary["varies"])
|
|
21
|
+
source = experiment_runs(name, list(varies))
|
|
22
|
+
progress = await experiment_progress(tables, source, summary)
|
|
23
|
+
listed.append(
|
|
24
|
+
{"name": name, "display_name": row["display_name"], **summary, **progress}
|
|
25
|
+
)
|
|
26
|
+
return listed
|
|
@@ -23,7 +23,7 @@ async def list_runs(
|
|
|
23
23
|
elif view.sort:
|
|
24
24
|
order = f"{quote_identifier(view.sort)} {direction}"
|
|
25
25
|
else:
|
|
26
|
-
order = "
|
|
26
|
+
order = "created DESC, id DESC"
|
|
27
27
|
return await runs.query(
|
|
28
28
|
f"SELECT ({completed_run}) AS labnotes_completed, * FROM {source}"
|
|
29
29
|
f"{filtered} ORDER BY {order}"
|
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
import json
|
|
2
2
|
from collections.abc import Callable, Mapping, Sequence
|
|
3
|
+
from pathlib import Path
|
|
3
4
|
from typing import cast
|
|
4
5
|
|
|
5
6
|
from dirsql import DirSQL
|
|
6
7
|
|
|
8
|
+
from labnotes.experiment_notes import experiment_notes
|
|
9
|
+
from labnotes.experiment_progress import experiment_progress
|
|
10
|
+
from labnotes.experiment_runs import experiment_runs
|
|
7
11
|
from labnotes.list_runs import list_runs
|
|
8
12
|
from labnotes.parse_run_view import parse_run_view
|
|
9
13
|
from labnotes.python_blocks import python_blocks
|
|
10
14
|
from labnotes.render_markdown import render_markdown
|
|
11
|
-
from labnotes.run_columns import run_columns
|
|
12
15
|
from labnotes.sort_href import sort_href
|
|
13
16
|
from labnotes.sql_quote import quote_string
|
|
14
|
-
from labnotes.
|
|
15
|
-
from labnotes.study_runs import study_runs
|
|
16
|
-
from labnotes.summarize_design import summarize_design
|
|
17
|
+
from labnotes.summarize_config import summarize_config
|
|
17
18
|
|
|
18
19
|
|
|
19
20
|
def _no_outputs(note: str, content: str) -> Sequence[str]:
|
|
@@ -21,53 +22,45 @@ def _no_outputs(note: str, content: str) -> Sequence[str]:
|
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
async def load_experiment(
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
tables: DirSQL,
|
|
26
|
+
root: Path,
|
|
26
27
|
name: str,
|
|
27
28
|
params: Mapping[str, str],
|
|
28
29
|
outputs_for: Callable[[str, str], Sequence[str]] = _no_outputs,
|
|
29
30
|
) -> dict[str, object] | None:
|
|
30
|
-
found = await
|
|
31
|
-
|
|
31
|
+
found = await tables.query(
|
|
32
|
+
"SELECT name, display_name, config FROM experiments"
|
|
33
|
+
f" WHERE name = {quote_string(name)}"
|
|
32
34
|
)
|
|
33
35
|
if not found:
|
|
34
36
|
return None
|
|
35
37
|
[row] = found
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
f" WHERE experiment = {quote_string(name)} ORDER BY name"
|
|
39
|
-
)
|
|
40
|
-
summary = summarize_design(json.loads(str(row["design"])))
|
|
38
|
+
config = None if row["config"] is None else json.loads(str(row["config"]))
|
|
39
|
+
summary = summarize_config(config)
|
|
41
40
|
varies = cast(dict[str, list[object]], summary["varies"])
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
shown = columns[:1] + [key for key in varies if key in columns[1:]]
|
|
41
|
+
source = experiment_runs(name, list(varies))
|
|
42
|
+
progress = await experiment_progress(tables, source, summary)
|
|
43
|
+
shown = ["id", *(key for key in varies if key != "id")]
|
|
46
44
|
view = parse_run_view(params, shown)
|
|
47
|
-
|
|
48
|
-
[]
|
|
49
|
-
if study is None
|
|
50
|
-
else await list_runs(
|
|
51
|
-
runs, study_runs(study), str(summary["completed_run"]), view
|
|
52
|
-
)
|
|
53
|
-
)
|
|
45
|
+
notes = await experiment_notes(tables, root, name)
|
|
54
46
|
return {
|
|
55
47
|
"name": row["name"],
|
|
56
|
-
"
|
|
48
|
+
"display_name": row["display_name"],
|
|
57
49
|
**summary,
|
|
58
50
|
**progress,
|
|
59
51
|
"notes": [
|
|
60
52
|
{
|
|
61
53
|
"name": note["name"],
|
|
62
54
|
"html": render_markdown(
|
|
63
|
-
|
|
64
|
-
outputs_for(str(note["name"]), str(note["content"])),
|
|
55
|
+
note["content"], outputs_for(note["name"], note["content"])
|
|
65
56
|
),
|
|
66
|
-
"runnable": bool(python_blocks(
|
|
57
|
+
"runnable": bool(python_blocks(note["content"])),
|
|
67
58
|
}
|
|
68
59
|
for note in notes
|
|
69
60
|
],
|
|
70
|
-
"runs_list":
|
|
61
|
+
"runs_list": await list_runs(
|
|
62
|
+
tables, source, str(summary["completed_run"]), view
|
|
63
|
+
),
|
|
71
64
|
"run_columns": shown,
|
|
72
65
|
"run_view": view,
|
|
73
66
|
"sort_hrefs": {
|
|
@@ -3,6 +3,7 @@ import time
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
5
|
from labnotes.runner.shell_command import shell_command
|
|
6
|
+
from labnotes.runner.wait_for_group import wait_for_group
|
|
6
7
|
from labnotes.runner.write_run_json import write_run_json
|
|
7
8
|
from labnotes.tables.iso_time import iso_time
|
|
8
9
|
|
|
@@ -13,6 +14,7 @@ async def execute_run(
|
|
|
13
14
|
command: str,
|
|
14
15
|
params: dict[str, object],
|
|
15
16
|
running: set[asyncio.subprocess.Process],
|
|
17
|
+
stopping: asyncio.Event,
|
|
16
18
|
) -> dict[str, object]:
|
|
17
19
|
run: dict[str, object] = {
|
|
18
20
|
"params": params,
|
|
@@ -42,6 +44,11 @@ async def execute_run(
|
|
|
42
44
|
running.add(process)
|
|
43
45
|
try:
|
|
44
46
|
run["exit_code"] = await process.wait()
|
|
47
|
+
# The shell can exit on SIGTERM before what it started. Once
|
|
48
|
+
# stopping, the run stays killable until its whole group is
|
|
49
|
+
# gone. A normal exit ends the run even if helpers linger.
|
|
50
|
+
if stopping.is_set():
|
|
51
|
+
await wait_for_group(process.pid)
|
|
45
52
|
finally:
|
|
46
53
|
running.discard(process)
|
|
47
54
|
run["ended"] = iso_time(time.time())
|
|
@@ -32,7 +32,7 @@ async def run_experiment(
|
|
|
32
32
|
report({"event": "planned", "runs": len(plan), "concurrency": concurrency})
|
|
33
33
|
running: set[asyncio.subprocess.Process] = set()
|
|
34
34
|
results: list[dict[str, object]] = []
|
|
35
|
-
stopping =
|
|
35
|
+
stopping = asyncio.Event()
|
|
36
36
|
|
|
37
37
|
def signal_runs(signal_number: int) -> None:
|
|
38
38
|
for process in running:
|
|
@@ -40,10 +40,9 @@ async def run_experiment(
|
|
|
40
40
|
os.killpg(process.pid, signal_number)
|
|
41
41
|
|
|
42
42
|
def interrupt() -> None:
|
|
43
|
-
nonlocal stopping
|
|
44
43
|
plan.clear()
|
|
45
|
-
if not stopping:
|
|
46
|
-
stopping
|
|
44
|
+
if not stopping.is_set():
|
|
45
|
+
stopping.set()
|
|
47
46
|
report({"event": "stopping"})
|
|
48
47
|
signal_runs(signal.SIGTERM)
|
|
49
48
|
elif running:
|
|
@@ -60,7 +59,7 @@ async def run_experiment(
|
|
|
60
59
|
folder.mkdir(parents=True)
|
|
61
60
|
report({"event": "started", "run": run, "params": params})
|
|
62
61
|
result = await execute_run(
|
|
63
|
-
config.root, folder, experiment.command, params, running
|
|
62
|
+
config.root, folder, experiment.command, params, running, stopping
|
|
64
63
|
)
|
|
65
64
|
report({"event": "ended", "run": run, **result})
|
|
66
65
|
results.append(result)
|
|
@@ -4,15 +4,13 @@ from pathlib import Path
|
|
|
4
4
|
from starlette.applications import Starlette
|
|
5
5
|
|
|
6
6
|
from labnotes.create_app import create_app
|
|
7
|
+
from labnotes.read_config import read_config
|
|
7
8
|
|
|
8
|
-
|
|
9
|
-
DATA_FOLDER = "LABNOTES_DATA_FOLDER"
|
|
9
|
+
CONFIG = "LABNOTES_CONFIG"
|
|
10
10
|
KERNEL_IDLE_MINUTES = "LABNOTES_KERNEL_IDLE_MINUTES"
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def serve_app() -> Starlette:
|
|
14
14
|
return create_app(
|
|
15
|
-
Path(os.environ[
|
|
16
|
-
Path(os.environ[DATA_FOLDER]),
|
|
17
|
-
int(os.environ[KERNEL_IDLE_MINUTES]),
|
|
15
|
+
read_config(Path(os.environ[CONFIG])), int(os.environ[KERNEL_IDLE_MINUTES])
|
|
18
16
|
)
|
|
@@ -1,29 +1,18 @@
|
|
|
1
1
|
import os
|
|
2
|
-
from pathlib import Path
|
|
3
2
|
|
|
4
3
|
import click
|
|
5
4
|
from uvicorn.main import main as uvicorn_main
|
|
6
5
|
|
|
7
|
-
from labnotes.
|
|
6
|
+
from labnotes.load_config import load_config
|
|
7
|
+
from labnotes.read_config import CONFIG_FILE
|
|
8
|
+
from labnotes.serve_app import CONFIG, KERNEL_IDLE_MINUTES
|
|
8
9
|
|
|
9
10
|
# labnotes owns these; the rest of uvicorn's options pass straight through.
|
|
10
11
|
OWN_OPTIONS = {"app", "factory", "app_dir", "version", "host", "port"}
|
|
11
12
|
|
|
12
13
|
|
|
13
|
-
@click.command(
|
|
14
|
-
|
|
15
|
-
"--experiments-folder",
|
|
16
|
-
type=click.Path(path_type=Path),
|
|
17
|
-
envvar=EXPERIMENTS_FOLDER,
|
|
18
|
-
required=True,
|
|
19
|
-
help="One subfolder per experiment, each with a design.yaml and markdown notes.",
|
|
20
|
-
)
|
|
21
|
-
@click.option(
|
|
22
|
-
"--data-folder",
|
|
23
|
-
type=click.Path(path_type=Path),
|
|
24
|
-
envvar=DATA_FOLDER,
|
|
25
|
-
required=True,
|
|
26
|
-
help="Run data, with a .dirsql.toml declaring a runs table that has a study column.",
|
|
14
|
+
@click.command(
|
|
15
|
+
help="Serve the experiments, their notes and their runs as a local site."
|
|
27
16
|
)
|
|
28
17
|
@click.option(
|
|
29
18
|
"--kernel-idle-minutes",
|
|
@@ -44,14 +33,16 @@ OWN_OPTIONS = {"app", "factory", "app_dir", "version", "host", "port"}
|
|
|
44
33
|
@click.pass_context
|
|
45
34
|
def server(
|
|
46
35
|
ctx: click.Context,
|
|
47
|
-
experiments_folder: Path,
|
|
48
|
-
data_folder: Path,
|
|
49
36
|
kernel_idle_minutes: int,
|
|
50
37
|
**server_options: object,
|
|
51
38
|
) -> None:
|
|
39
|
+
config = load_config(ctx)
|
|
40
|
+
if config is None:
|
|
41
|
+
raise click.ClickException(
|
|
42
|
+
f"No {CONFIG_FILE} here. Run labnotes init, or pass -c PATH."
|
|
43
|
+
)
|
|
52
44
|
# Through the environment so --reload and --workers child processes see them.
|
|
53
|
-
os.environ[
|
|
54
|
-
os.environ[DATA_FOLDER] = str(data_folder.resolve())
|
|
45
|
+
os.environ[CONFIG] = str(config.path.resolve())
|
|
55
46
|
os.environ[KERNEL_IDLE_MINUTES] = str(kernel_idle_minutes)
|
|
56
47
|
ctx.invoke(
|
|
57
48
|
uvicorn_main,
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from math import prod
|
|
2
|
+
|
|
3
|
+
# What the runner counts toward repeats.
|
|
4
|
+
COMPLETED_RUN = "exit_code = 0 AND NOT coalesce(excluded, 0)"
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def summarize_config(config: dict[str, object] | None) -> dict[str, object]:
|
|
8
|
+
config = config or {}
|
|
9
|
+
params = config.get("params")
|
|
10
|
+
varies = {
|
|
11
|
+
key: values if isinstance(values, list) else [values]
|
|
12
|
+
for key, values in (params if isinstance(params, dict) else {}).items()
|
|
13
|
+
}
|
|
14
|
+
repeats = config.get("repeats", 1) if config else None
|
|
15
|
+
repeats = repeats if isinstance(repeats, int) else None
|
|
16
|
+
cells = prod(len(values) for values in varies.values())
|
|
17
|
+
command = config.get("command")
|
|
18
|
+
return {
|
|
19
|
+
"fixed": {},
|
|
20
|
+
"varies": varies,
|
|
21
|
+
"cell_count": cells,
|
|
22
|
+
"runs_per_cell": repeats,
|
|
23
|
+
"target": cells * repeats if repeats is not None else None,
|
|
24
|
+
"completed_run": COMPLETED_RUN,
|
|
25
|
+
"other": {} if command is None else {"command": command},
|
|
26
|
+
}
|