labnotes 0.0.2__tar.gz → 0.0.3__tar.gz

{labnotes-0.0.2 → labnotes-0.0.3}/PKG-INFO

@@ -1,13 +1,17 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.2
+Version: 0.0.3
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>
 License-Expression: MIT
 Requires-Python: >=3.12
 Requires-Dist: dirsql>=0.4.110
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: markdown-it-py>=4.2.0
 Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: starlette>=1.8.0
+Requires-Dist: uvicorn>=0.54.0
 Description-Content-Type: text/markdown
 
 # labnotes
@@ -23,3 +27,15 @@ uvx labnotes --experiments-folder experiments --data-folder data
 - `--data-folder` holds the run data and a `.dirsql.toml` that declares a
   `runs` table with a `study` column. An experiment's runs are the rows whose
   `study` matches its design's.
+
+That serves a local site at http://127.0.0.1:8000: a table of experiments,
+and a page per experiment with its parameters, its notes, progress per cell of
+the design grid, and every run so far. Both folders are watched, so a reload
+shows new runs and edited notes.
+
+`design.yaml` keys the site reads: `study`, `fixed`, `varies` (each key a list
+of values), `runs_per_cell`, and `completed_run`, a SQL condition over the
+`runs` table that marks a run as complete. Anything else shows up as-is.
+
+`--host` and `--port` change where it listens. `--json` prints the experiments
+as JSON and exits.

labnotes-0.0.3/README.md

@@ -0,0 +1,25 @@
+# labnotes
+
+Browse experiment designs, notes, and run data from a folder.
+
+```sh
+uvx labnotes --experiments-folder experiments --data-folder data
+```
+
+- `--experiments-folder` holds one subfolder per experiment, each with a
+  `design.yaml` (with a `study` key) and any number of markdown notes.
+- `--data-folder` holds the run data and a `.dirsql.toml` that declares a
+  `runs` table with a `study` column. An experiment's runs are the rows whose
+  `study` matches its design's.
+
+That serves a local site at http://127.0.0.1:8000: a table of experiments,
+and a page per experiment with its parameters, its notes, progress per cell of
+the design grid, and every run so far. Both folders are watched, so a reload
+shows new runs and edited notes.
+
+`design.yaml` keys the site reads: `study`, `fixed`, `varies` (each key a list
+of values), `runs_per_cell`, and `completed_run`, a SQL condition over the
+`runs` table that marks a run as complete. Anything else shows up as-is.
+
+`--host` and `--port` change where it listens. `--json` prints the experiments
+as JSON and exits.

{labnotes-0.0.2 → labnotes-0.0.3}/pyproject.toml

@@ -12,7 +12,11 @@ license = "MIT"
 authors = [{ name = "Kevin Scott", email = "me@thekevinscott.com" }]
 dependencies = [
     "dirsql>=0.4.110",
+    "jinja2>=3.1.6",
+    "markdown-it-py>=4.2.0",
     "pyyaml>=6.0.3",
+    "starlette>=1.8.0",
+    "uvicorn>=0.54.0",
 ]
 
 [project.urls]
@@ -23,6 +27,7 @@ labnotes = "labnotes.cli:main"
 
 [dependency-groups]
 dev = [
+    "httpx2>=2.13.1",
     "mypy>=2.4.0",
     "pytest>=8.0",
     "ruff==0.8.4",

labnotes-0.0.3/src/labnotes/cell_progress.py

@@ -0,0 +1,22 @@
+from itertools import product
+
+
+def _key(value: object) -> object:
+    return int(value) if isinstance(value, bool) else value
+
+
+def cell_progress(
+    varies: dict[str, list[object]], counts: list[dict[str, object]]
+) -> list[dict[str, object]]:
+    by_cell = {tuple(_key(row[key]) for key in varies): row for row in counts}
+    cells = []
+    for values in product(*varies.values()):
+        row = by_cell.get(tuple(_key(value) for value in values), {})
+        cells.append(
+            {
+                "values": dict(zip(varies, values)),
+                "runs": row.get("runs", 0),
+                "completed": row.get("completed", 0),
+            }
+        )
+    return cells

labnotes-0.0.3/src/labnotes/cli.py

@@ -0,0 +1,51 @@
+import argparse
+import asyncio
+import json
+import sys
+from importlib.metadata import version
+from pathlib import Path
+
+import uvicorn
+
+from labnotes.create_app import create_app
+from labnotes.dump_experiments import dump_experiments
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        prog="labnotes", description="Browse experiments and their run data."
+    )
+    parser.add_argument("--version", action="version", version=version("labnotes"))
+    parser.add_argument(
+        "--experiments-folder",
+        type=Path,
+        required=True,
+        help="one subfolder per experiment, each with a design.yaml and markdown notes",
+    )
+    parser.add_argument(
+        "--data-folder",
+        type=Path,
+        required=True,
+        help="run data, with a .dirsql.toml declaring a runs table that has a study column",
+    )
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="print experiments as JSON instead of serving",
+    )
+    args = parser.parse_args(argv)
+    if args.json:
+        experiments = asyncio.run(
+            dump_experiments(args.experiments_folder, args.data_folder)
+        )
+        print(json.dumps(experiments, indent=2))
+        return
+    print(f"labnotes at http://{args.host}:{args.port}", file=sys.stderr)
+    uvicorn.run(
+        create_app(args.experiments_folder, args.data_folder),
+        host=args.host,
+        port=args.port,
+        log_level="warning",
+    )

labnotes-0.0.3/src/labnotes/count_runs.py

@@ -0,0 +1,16 @@
+from dirsql import DirSQL
+
+from labnotes.sql_quote import quote_identifier, quote_string
+
+
+async def count_runs(
+    runs: DirSQL, study: str, completed_run: str, group_by: list[str] | None = None
+) -> list[dict[str, object]]:
+    keys = [quote_identifier(key) for key in group_by or []]
+    columns = "".join(f"{key}, " for key in keys)
+    group = f" GROUP BY {', '.join(keys)}" if keys else ""
+    return await runs.query(
+        f"SELECT {columns}COUNT(*) AS runs,"
+        f" COALESCE(SUM(CASE WHEN ({completed_run}) THEN 1 ELSE 0 END), 0) AS completed"
+        f" FROM runs WHERE study = {quote_string(study)}{group}"
+    )

labnotes-0.0.3/src/labnotes/create_app.py

@@ -0,0 +1,74 @@
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import cast
+
+from starlette.applications import Starlette
+from starlette.exceptions import HTTPException
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Mount, Route
+from starlette.staticfiles import StaticFiles
+from starlette.templating import Jinja2Templates
+
+from labnotes.drain_watch import drain_watch
+from labnotes.format_value import format_value
+from labnotes.list_experiments import list_experiments
+from labnotes.load_experiment import load_experiment
+from labnotes.open_experiments import open_experiments
+from labnotes.open_runs import open_runs
+from labnotes.pivot_cells import pivot_cells
+
+HERE = Path(__file__).parent
+templates = Jinja2Templates(directory=HERE / "templates")
+templates.env.filters["value"] = format_value
+
+
+def create_app(experiments_folder: Path, data_folder: Path) -> Starlette:
+    @asynccontextmanager
+    async def lifespan(app: Starlette) -> AsyncIterator[None]:
+        app.state.experiments = open_experiments(experiments_folder)
+        app.state.runs = open_runs(data_folder)
+        await app.state.experiments.ready()
+        await app.state.runs.ready()
+        watchers = [
+            asyncio.create_task(drain_watch(app.state.experiments)),
+            asyncio.create_task(drain_watch(app.state.runs)),
+        ]
+        yield
+        for watcher in watchers:
+            watcher.cancel()
+
+    async def index(request: Request) -> Response:
+        state = request.app.state
+        experiments = await list_experiments(state.experiments, state.runs)
+        return templates.TemplateResponse(
+            request,
+            "index.html",
+            {"experiments": experiments, "folder": experiments_folder.resolve().name},
+        )
+
+    async def experiment(request: Request) -> Response:
+        state = request.app.state
+        loaded = await load_experiment(
+            state.experiments, state.runs, request.path_params["name"]
+        )
+        if loaded is None:
+            raise HTTPException(404, "No such experiment")
+        varies = cast(dict[str, list[object]], loaded["varies"])
+        cells = cast(list[dict[str, object]], loaded["cells"])
+        return templates.TemplateResponse(
+            request,
+            "experiment.html",
+            {"experiment": loaded, "pivot": pivot_cells(varies, cells)},
+        )
+
+    return Starlette(
+        routes=[
+            Route("/", index),
+            Route("/experiments/{name}", experiment),
+            Mount("/static", StaticFiles(directory=HERE / "static"), name="static"),
+        ],
+        lifespan=lifespan,
+    )

labnotes-0.0.3/src/labnotes/drain_watch.py

@@ -0,0 +1,6 @@
+from dirsql import DirSQL
+
+
+async def drain_watch(db: DirSQL) -> None:
+    async for _ in db.watch():
+        pass

labnotes-0.0.3/src/labnotes/dump_experiments.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from labnotes.list_experiments import list_experiments
+from labnotes.open_experiments import open_experiments
+from labnotes.open_runs import open_runs
+
+
+async def dump_experiments(
+    experiments_folder: Path, data_folder: Path
+) -> list[dict[str, object]]:
+    experiments = open_experiments(experiments_folder)
+    runs = open_runs(data_folder)
+    await experiments.ready()
+    await runs.ready()
+    return await list_experiments(experiments, runs)

labnotes-0.0.3/src/labnotes/format_value.py

@@ -0,0 +1,11 @@
+import json
+
+
+def format_value(value: object) -> str:
+    if value is None:
+        return "—"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, dict | list):
+        return json.dumps(value)
+    return str(value)

labnotes-0.0.3/src/labnotes/list_experiments.py

@@ -0,0 +1,23 @@
+import json
+
+from dirsql import DirSQL
+
+from labnotes.run_columns import run_columns
+from labnotes.study_progress import study_progress
+from labnotes.summarize_design import summarize_design
+
+
+async def list_experiments(
+    experiments: DirSQL, runs: DirSQL
+) -> list[dict[str, object]]:
+    rows = await experiments.query(
+        "SELECT name, study, design FROM experiments ORDER BY name"
+    )
+    columns = await run_columns(runs)
+    listed = []
+    for row in rows:
+        summary = summarize_design(json.loads(str(row["design"])))
+        study = None if row["study"] is None else str(row["study"])
+        progress = await study_progress(runs, study, summary, columns)
+        listed.append({"name": row["name"], "study": study, **summary, **progress})
+    return listed

labnotes-0.0.3/src/labnotes/list_runs.py

@@ -0,0 +1,12 @@
+from dirsql import DirSQL
+
+from labnotes.sql_quote import quote_string
+
+
+async def list_runs(
+    runs: DirSQL, study: str, completed_run: str
+) -> list[dict[str, object]]:
+    return await runs.query(
+        f"SELECT ({completed_run}) AS labnotes_completed, * FROM runs"
+        f" WHERE study = {quote_string(study)} ORDER BY 2 DESC"
+    )

labnotes-0.0.3/src/labnotes/load_experiment.py

@@ -0,0 +1,48 @@
+import json
+from typing import cast
+
+from dirsql import DirSQL
+
+from labnotes.list_runs import list_runs
+from labnotes.render_markdown import render_markdown
+from labnotes.run_columns import run_columns
+from labnotes.sql_quote import quote_string
+from labnotes.study_progress import study_progress
+from labnotes.summarize_design import summarize_design
+
+
+async def load_experiment(
+    experiments: DirSQL, runs: DirSQL, name: str
+) -> dict[str, object] | None:
+    found = await experiments.query(
+        f"SELECT name, study, design FROM experiments WHERE name = {quote_string(name)}"
+    )
+    if not found:
+        return None
+    [row] = found
+    notes = await experiments.query(
+        "SELECT name, content FROM notes"
+        f" WHERE experiment = {quote_string(name)} ORDER BY name"
+    )
+    summary = summarize_design(json.loads(str(row["design"])))
+    varies = cast(dict[str, list[object]], summary["varies"])
+    study = None if row["study"] is None else str(row["study"])
+    columns = await run_columns(runs)
+    progress = await study_progress(runs, study, summary, columns)
+    listed = (
+        []
+        if study is None
+        else await list_runs(runs, study, str(summary["completed_run"]))
+    )
+    return {
+        "name": row["name"],
+        "study": study,
+        **summary,
+        **progress,
+        "notes": [
+            {"name": note["name"], "html": render_markdown(str(note["content"]))}
+            for note in notes
+        ],
+        "runs_list": listed,
+        "run_columns": columns[:1] + [key for key in varies if key in columns[1:]],
+    }

labnotes-0.0.3/src/labnotes/pivot_cells.py

@@ -0,0 +1,21 @@
+from itertools import product
+
+
+def pivot_cells(
+    varies: dict[str, list[object]], cells: list[dict[str, object]]
+) -> dict[str, object] | None:
+    if not varies or not cells:
+        return None
+    *row_keys, column_key = varies
+    width = len(varies[column_key])
+    labels = product(*(varies[key] for key in row_keys))
+    rows = [
+        {"labels": list(label), "cells": cells[start : start + width]}
+        for label, start in zip(labels, range(0, len(cells), width))
+    ]
+    return {
+        "row_keys": row_keys,
+        "column_key": column_key,
+        "columns": varies[column_key],
+        "rows": rows,
+    }

labnotes-0.0.3/src/labnotes/render_markdown.py

@@ -0,0 +1,9 @@
+from markdown_it import MarkdownIt
+
+_markdown = MarkdownIt("commonmark", {"html": False, "typographer": True}).enable(
+    ["table", "strikethrough", "replacements", "smartquotes"]
+)
+
+
+def render_markdown(text: str) -> str:
+    return _markdown.render(text)

labnotes-0.0.3/src/labnotes/run_columns.py

@@ -0,0 +1,6 @@
+from dirsql import DirSQL
+
+
+async def run_columns(runs: DirSQL) -> list[str]:
+    rows = await runs.query("SELECT name FROM pragma_table_info('runs')")
+    return [str(row["name"]) for row in rows]

labnotes-0.0.3/src/labnotes/sql_quote.py

@@ -0,0 +1,6 @@
+def quote_string(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def quote_identifier(name: str) -> str:
+    return '"' + name.replace('"', '""') + '"'