labnotes 0.0.8__tar.gz → 0.0.10__tar.gz

{labnotes-0.0.8 → labnotes-0.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.8
+Version: 0.0.10
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>
@@ -8,7 +8,9 @@ License-Expression: MIT
 Requires-Python: >=3.12
 Requires-Dist: click>=8.5.0
 Requires-Dist: dirsql>=0.4.110
+Requires-Dist: ipykernel>=7.4.0
 Requires-Dist: jinja2>=3.1.6
+Requires-Dist: jupyter-client>=8.10.0
 Requires-Dist: markdown-it-py>=4.2.0
 Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: starlette>=1.8.0

{labnotes-0.0.8 → labnotes-0.0.10}/pyproject.toml

@@ -13,7 +13,9 @@ authors = [{ name = "Kevin Scott", email = "me@thekevinscott.com" }]
 dependencies = [
     "click>=8.5.0",
     "dirsql>=0.4.110",
+    "ipykernel>=7.4.0",
     "jinja2>=3.1.6",
+    "jupyter-client>=8.10.0",
     "markdown-it-py>=4.2.0",
     "pyyaml>=6.0.3",
     "starlette>=1.8.0",

{labnotes-0.0.8 → labnotes-0.0.10}/src/labnotes/cli.py

@@ -7,7 +7,7 @@ import click
 from uvicorn.main import main as uvicorn_main
 
 from labnotes.dump_experiments import dump_experiments
-from labnotes.serve_app import DATA_FOLDER, EXPERIMENTS_FOLDER
+from labnotes.serve_app import DATA_FOLDER, EXPERIMENTS_FOLDER, KERNEL_IDLE_MINUTES
 
 # labnotes owns these; the rest of uvicorn's options pass straight through.
 OWN_OPTIONS = {"app", "factory", "app_dir", "version", "host", "port"}
@@ -29,6 +29,14 @@ OWN_OPTIONS = {"app", "factory", "app_dir", "version", "host", "port"}
     envvar=DATA_FOLDER,
     help="Run data, with a .dirsql.toml declaring a runs table that has a study column.",
 )
+@click.option(
+    "--kernel-idle-minutes",
+    type=click.IntRange(min=1),
+    default=30,
+    show_default=True,
+    envvar=KERNEL_IDLE_MINUTES,
+    help="Stop a note's Python kernel after this long without a run.",
+)
 @click.option(
     "--json",
     "as_json",
@@ -48,6 +56,7 @@ def main(
     ctx: click.Context,
     experiments_folder: Path,
     data_folder: Path,
+    kernel_idle_minutes: int,
     as_json: bool,
     **server_options: object,
 ) -> None:
@@ -58,6 +67,7 @@ def main(
     # Through the environment so --reload and --workers child processes see them.
     os.environ[EXPERIMENTS_FOLDER] = str(experiments_folder.resolve())
     os.environ[DATA_FOLDER] = str(data_folder.resolve())
+    os.environ[KERNEL_IDLE_MINUTES] = str(kernel_idle_minutes)
     ctx.invoke(
         uvicorn_main,
         app="labnotes.serve_app:serve_app",

{labnotes-0.0.8 → labnotes-0.0.10}/src/labnotes/create_app.py

@@ -7,7 +7,7 @@ from typing import cast
 from starlette.applications import Starlette
 from starlette.exceptions import HTTPException
 from starlette.requests import Request
-from starlette.responses import Response
+from starlette.responses import HTMLResponse, Response
 from starlette.routing import Mount, Route
 from starlette.staticfiles import StaticFiles
 from starlette.templating import Jinja2Templates
@@ -17,9 +17,14 @@ from labnotes.drain_watch import drain_watch
 from labnotes.format_value import format_value
 from labnotes.list_experiments import list_experiments
 from labnotes.load_experiment import load_experiment
+from labnotes.notebook_kernel import NotebookKernel
 from labnotes.open_experiments import open_experiments
 from labnotes.open_runs import open_runs
 from labnotes.pivot_cells import pivot_cells
+from labnotes.python_blocks import python_blocks
+from labnotes.saved_outputs import saved_outputs
+from labnotes.sql_quote import quote_string
+from labnotes.stop_idle_kernels import stop_idle_kernels
 
 HERE = Path(__file__).parent
 templates = Jinja2Templates(directory=HERE / "templates")
@@ -27,7 +32,11 @@ templates.env.filters["value"] = format_value
 templates.env.globals["asset"] = asset_url
 
 
-def create_app(experiments_folder: Path, data_folder: Path) -> Starlette:
+def create_app(
+    experiments_folder: Path, data_folder: Path, kernel_idle_minutes: float
+) -> Starlette:
+    kernels: dict[tuple[str, str], NotebookKernel] = {}
+
     @asynccontextmanager
     async def lifespan(app: Starlette) -> AsyncIterator[None]:
         app.state.experiments = open_experiments(experiments_folder)
@@ -37,10 +46,13 @@ def create_app(experiments_folder: Path, data_folder: Path) -> Starlette:
         watchers = [
             asyncio.create_task(drain_watch(app.state.experiments)),
             asyncio.create_task(drain_watch(app.state.runs)),
+            asyncio.create_task(stop_idle_kernels(kernels, kernel_idle_minutes)),
         ]
         yield
         for watcher in watchers:
             watcher.cancel()
+        for kernel in kernels.values():
+            await kernel.stop()
 
     async def index(request: Request) -> Response:
         state = request.app.state
@@ -53,11 +65,13 @@ def create_app(experiments_folder: Path, data_folder: Path) -> Starlette:
 
     async def experiment(request: Request) -> Response:
         state = request.app.state
+        name = request.path_params["name"]
         loaded = await load_experiment(
             state.experiments,
             state.runs,
-            request.path_params["name"],
+            name,
             request.query_params,
+            lambda note, content: saved_outputs(kernels.get((name, note)), content),
         )
         if loaded is None:
             raise HTTPException(404, "No such experiment")
@@ -69,10 +83,47 @@ def create_app(experiments_folder: Path, data_folder: Path) -> Starlette:
             {"experiment": loaded, "pivot": pivot_cells(varies, cells)},
         )
 
+    async def note_content(request: Request) -> str:
+        if request.headers.get("content-type") != "application/json":
+            raise HTTPException(415, "Send JSON")
+        name = request.path_params["name"]
+        note = request.path_params["note"]
+        found = await request.app.state.experiments.query(
+            "SELECT content FROM notes"
+            f" WHERE experiment = {quote_string(name)} AND name = {quote_string(note)}"
+        )
+        if not found:
+            raise HTTPException(404, "No such note")
+        [row] = found
+        return str(row["content"])
+
+    async def run_block(request: Request) -> Response:
+        blocks = python_blocks(await note_content(request))
+        key = (request.path_params["name"], request.path_params["note"])
+        block = (await request.json()).get("block")
+        if not isinstance(block, int) or not 0 <= block < len(blocks):
+            raise HTTPException(400, "No such block")
+        if key not in kernels:
+            kernels[key] = NotebookKernel(data_folder)
+        return HTMLResponse(await kernels[key].run(blocks[block]))
+
+    async def restart_kernel(request: Request) -> Response:
+        await note_content(request)
+        kernel = kernels.get((request.path_params["name"], request.path_params["note"]))
+        if kernel is not None:
+            await kernel.restart()
+        return Response(status_code=204)
+
     return Starlette(
         routes=[
             Route("/", index),
             Route("/experiments/{name}", experiment),
+            Route("/experiments/{name}/notes/{note}/run", run_block, methods=["POST"]),
+            Route(
+                "/experiments/{name}/notes/{note}/restart",
+                restart_kernel,
+                methods=["POST"],
+            ),
             Mount("/static", StaticFiles(directory=HERE / "static"), name="static"),
         ],
         lifespan=lifespan,

labnotes-0.0.10/src/labnotes/data_html.py

@@ -0,0 +1,12 @@
+from collections.abc import Mapping
+from html import escape
+
+
+def data_html(data: Mapping[str, str]) -> str:
+    # Rich output is trusted: it comes from code in the user's own notes.
+    for mime in ("text/html", "image/svg+xml"):
+        if mime in data:
+            return data[mime]
+    if "image/png" in data:
+        return f'<img alt="" src="data:image/png;base64,{data["image/png"]}">'
+    return f"<pre>{escape(data.get('text/plain', ''))}</pre>"

{labnotes-0.0.8 → labnotes-0.0.10}/src/labnotes/load_experiment.py

@@ -1,11 +1,12 @@
 import json
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping, Sequence
 from typing import cast
 
 from dirsql import DirSQL
 
 from labnotes.list_runs import list_runs
 from labnotes.parse_run_view import parse_run_view
+from labnotes.python_blocks import python_blocks
 from labnotes.render_markdown import render_markdown
 from labnotes.run_columns import run_columns
 from labnotes.sort_href import sort_href
@@ -14,8 +15,16 @@ from labnotes.study_progress import study_progress
 from labnotes.summarize_design import summarize_design
 
 
+def _no_outputs(note: str, content: str) -> Sequence[str]:
+    return ()
+
+
 async def load_experiment(
-    experiments: DirSQL, runs: DirSQL, name: str, params: Mapping[str, str]
+    experiments: DirSQL,
+    runs: DirSQL,
+    name: str,
+    params: Mapping[str, str],
+    outputs_for: Callable[[str, str], Sequence[str]] = _no_outputs,
 ) -> dict[str, object] | None:
     found = await experiments.query(
         f"SELECT name, study, design FROM experiments WHERE name = {quote_string(name)}"
@@ -45,7 +54,14 @@ async def load_experiment(
         **summary,
         **progress,
         "notes": [
-            {"name": note["name"], "html": render_markdown(str(note["content"]))}
+            {
+                "name": note["name"],
+                "html": render_markdown(
+                    str(note["content"]),
+                    outputs_for(str(note["name"]), str(note["content"])),
+                ),
+                "runnable": bool(python_blocks(str(note["content"]))),
+            }
             for note in notes
         ],
         "runs_list": listed,

labnotes-0.0.10/src/labnotes/notebook_kernel.py

@@ -0,0 +1,74 @@
+import asyncio
+import hashlib
+from pathlib import Path
+from time import monotonic
+
+from jupyter_client.asynchronous.client import AsyncKernelClient
+from jupyter_client.manager import AsyncKernelManager
+
+from labnotes.output_html import output_html
+
+QUIET = {"status", "execute_input"}
+
+
+def block_key(code: str) -> str:
+    return hashlib.sha256(code.encode()).hexdigest()
+
+
+class NotebookKernel:
+    """One live Python kernel per notes file; blocks run only when asked."""
+
+    def __init__(self, cwd: Path) -> None:
+        self.cwd = cwd
+        self.outputs: dict[str, str] = {}
+        self._lock = asyncio.Lock()
+        self._manager: AsyncKernelManager | None = None
+        self._client: AsyncKernelClient | None = None
+        self.last_used = monotonic()
+
+    async def _start(self) -> None:
+        self._manager = AsyncKernelManager(kernel_name="python3")
+        await self._manager.start_kernel(cwd=str(self.cwd))
+        self._client = self._manager.client()
+        self._client.start_channels()
+        await self._client.wait_for_ready(timeout=60)
+
+    async def run(self, code: str) -> str:
+        async with self._lock:
+            try:
+                if self._client is None:
+                    await self._start()
+                assert self._client is not None
+                messages: list[dict[str, object]] = []
+                await self._client.execute_interactive(
+                    code,
+                    output_hook=lambda message: messages.append(message),
+                    allow_stdin=False,
+                )
+                html = output_html(
+                    [
+                        {"msg_type": message["msg_type"], "content": message["content"]}
+                        for message in messages
+                        if message["msg_type"] not in QUIET
+                    ]
+                )
+                self.outputs[block_key(code)] = html
+                return html
+            finally:
+                self.last_used = monotonic()
+
+    async def stop(self) -> None:
+        client, manager = self._client, self._manager
+        self._client = self._manager = None
+        if client is not None:
+            client.stop_channels()
+        if manager is not None:
+            await manager.shutdown_kernel(now=True)
+
+    async def restart(self) -> None:
+        async with self._lock:
+            await self.stop()
+
+    async def stop_if_idle(self, idle_seconds: float) -> None:
+        if not self._lock.locked() and monotonic() - self.last_used > idle_seconds:
+            await self.stop()

labnotes-0.0.10/src/labnotes/output_html.py

@@ -0,0 +1,25 @@
+import re
+from collections.abc import Mapping, Sequence
+from html import escape
+
+from labnotes.data_html import data_html
+
+ANSI = re.compile(r"\x1b\[[0-9;]*m")
+
+
+def output_html(outputs: Sequence[Mapping[str, object]]) -> str:
+    parts = []
+    for output in outputs:
+        kind = output["msg_type"]
+        content = output["content"]
+        assert isinstance(content, Mapping)
+        if kind == "stream":
+            parts.append(
+                f'<pre class="{escape(content["name"])}">{escape(content["text"])}</pre>'
+            )
+        elif kind in ("execute_result", "display_data"):
+            parts.append(data_html(content["data"]))
+        elif kind == "error":
+            trace = ANSI.sub("", "\n".join(content["traceback"]))
+            parts.append(f'<pre class="error">{escape(trace)}</pre>')
+    return "".join(parts)

labnotes-0.0.10/src/labnotes/python_blocks.py

@@ -0,0 +1,12 @@
+from markdown_it import MarkdownIt
+from markdown_it.token import Token
+
+_parser = MarkdownIt("commonmark")
+
+
+def is_python(token: Token) -> bool:
+    return token.type == "fence" and token.info.split()[:1] == ["python"]
+
+
+def python_blocks(text: str) -> list[str]:
+    return [token.content for token in _parser.parse(text) if is_python(token)]

labnotes-0.0.10/src/labnotes/render_markdown.py

@@ -0,0 +1,40 @@
+from collections.abc import MutableMapping, Sequence
+from typing import Any
+
+from markdown_it import MarkdownIt
+from markdown_it.renderer import RendererHTML
+from markdown_it.token import Token
+
+from labnotes.python_blocks import is_python
+
+_markdown = MarkdownIt("commonmark", {"html": False, "typographer": True}).enable(
+    ["table", "strikethrough", "replacements", "smartquotes"]
+)
+
+
+def _fence(
+    self: RendererHTML,
+    tokens: Sequence[Token],
+    idx: int,
+    options: Any,
+    env: MutableMapping[str, Any],
+) -> str:
+    code = RendererHTML.fence(self, tokens, idx, options, env)
+    if not is_python(tokens[idx]):
+        return code
+    block = env["blocks"]
+    env["blocks"] = block + 1
+    outputs = env["outputs"]
+    shown = outputs[block] if block < len(outputs) else ""
+    return (
+        f'<div class="cell" data-block="{block}">{code}'
+        '<button type="button" class="run">Run</button>'
+        f'<output class="cell-output">{shown}</output></div>\n'
+    )
+
+
+_markdown.add_render_rule("fence", _fence)
+
+
+def render_markdown(text: str, outputs: Sequence[str] = ()) -> str:
+    return _markdown.render(text, {"blocks": 0, "outputs": outputs})

labnotes-0.0.10/src/labnotes/saved_outputs.py

@@ -0,0 +1,8 @@
+from labnotes.notebook_kernel import NotebookKernel, block_key
+from labnotes.python_blocks import python_blocks
+
+
+def saved_outputs(kernel: NotebookKernel | None, content: str) -> list[str]:
+    if kernel is None:
+        return []
+    return [kernel.outputs.get(block_key(code), "") for code in python_blocks(content)]

{labnotes-0.0.8 → labnotes-0.0.10}/src/labnotes/serve_app.py

@@ -7,9 +7,12 @@ from labnotes.create_app import create_app
 
 EXPERIMENTS_FOLDER = "LABNOTES_EXPERIMENTS_FOLDER"
 DATA_FOLDER = "LABNOTES_DATA_FOLDER"
+KERNEL_IDLE_MINUTES = "LABNOTES_KERNEL_IDLE_MINUTES"
 
 
 def serve_app() -> Starlette:
     return create_app(
-        Path(os.environ[EXPERIMENTS_FOLDER]), Path(os.environ[DATA_FOLDER])
+        Path(os.environ[EXPERIMENTS_FOLDER]),
+        Path(os.environ[DATA_FOLDER]),
+        int(os.environ[KERNEL_IDLE_MINUTES]),
     )

{labnotes-0.0.8 → labnotes-0.0.10}/src/labnotes/static/labnotes.css

@@ -406,3 +406,31 @@ h2 .clear { font-family: var(--sans); font-size: 0.82rem; margin-left: 0.8rem; c
   .ledger .design { display: none; }
   .progress-col { width: auto; }
 }
+
+/* Runnable python blocks in notes */
+.cell { position: relative; margin: 1em 0; }
+.cell pre { margin: 0; }
+.cell .run {
+  position: absolute;
+  top: 0.5rem;
+  right: 0.5rem;
+}
+.cell .run, .note-name .restart {
+  font: inherit;
+  font-family: var(--sans);
+  font-size: 0.75rem;
+  padding: 0.15rem 0.6rem;
+  border: 1px solid var(--rule-strong);
+  border-radius: 5px;
+  background: var(--surface);
+  color: var(--ink-2);
+  cursor: pointer;
+}
+.cell .run:hover, .note-name .restart:hover { color: var(--accent); border-color: var(--accent); }
+.cell .run:disabled, .note-name .restart:disabled { cursor: progress; opacity: 0.6; }
+.note-name:has(.restart) { display: flex; align-items: center; justify-content: space-between; padding-top: 0.45rem; padding-bottom: 0.45rem; }
+.cell-output { display: block; }
+.cell-output:not(:empty) { margin-top: 0.4rem; padding-left: 0.9rem; border-left: 2px solid var(--accent-soft); overflow-x: auto; }
+.cell-output pre { background: none; border: 0; padding: 0.3rem 0; }
+.cell-output .stderr, .cell-output .error { color: var(--bad); }
+.cell-output img, .cell-output svg { max-width: 100%; height: auto; }

labnotes-0.0.10/src/labnotes/static/run-cell.js

@@ -0,0 +1,44 @@
+for (const note of document.querySelectorAll(".note[data-run-url]")) {
+  for (const cell of note.querySelectorAll(".cell")) {
+    const button = cell.querySelector(".run");
+    const output = cell.querySelector(".cell-output");
+    button.addEventListener("click", async () => {
+      button.disabled = true;
+      button.textContent = "Running";
+      try {
+        const response = await fetch(note.dataset.runUrl, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ block: Number(cell.dataset.block) }),
+        });
+        output.innerHTML = response.ok
+          ? await response.text()
+          : `<pre class="error">${response.status} ${response.statusText}</pre>`;
+      } finally {
+        button.disabled = false;
+        button.textContent = "Run";
+      }
+    });
+  }
+}
+
+for (const note of document.querySelectorAll(".note[data-restart-url]")) {
+  const button = note.querySelector(".restart");
+  if (!button) continue;
+  button.addEventListener("click", async () => {
+    button.disabled = true;
+    button.textContent = "Restarting";
+    try {
+      const response = await fetch(note.dataset.restartUrl, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: "{}",
+      });
+      button.textContent = response.ok ? "Restarted" : `${response.status} ${response.statusText}`;
+      await new Promise((resolve) => setTimeout(resolve, 1500));
+    } finally {
+      button.disabled = false;
+      button.textContent = "Restart";
+    }
+  });
+}

labnotes-0.0.10/src/labnotes/stop_idle_kernels.py

@@ -0,0 +1,13 @@
+import asyncio
+from collections.abc import Mapping
+
+from labnotes.notebook_kernel import NotebookKernel
+
+
+async def stop_idle_kernels(
+    kernels: Mapping[tuple[str, str], NotebookKernel], idle_minutes: float
+) -> None:
+    while True:
+        await asyncio.sleep(60)
+        for kernel in list(kernels.values()):
+            await kernel.stop_if_idle(idle_minutes * 60)

{labnotes-0.0.8 → labnotes-0.0.10}/src/labnotes/templates/experiment.html

@@ -111,12 +111,13 @@
 <section id="notes" class="block">
   <h2>Notes</h2>
   {% for note in e.notes %}
-  <article class="note">
-    <header class="note-name">{{ note.name }}</header>
+  <article class="note" data-run-url="{{ url_for('run_block', name=e.name, note=note.name) }}" data-restart-url="{{ url_for('restart_kernel', name=e.name, note=note.name) }}">
+    <header class="note-name">{{ note.name }}{% if note.runnable %}<button type="button" class="restart" title="Stop this note's Python kernel. The next Run starts a fresh one.">Restart</button>{% endif %}</header>
     <div class="prose">{{ note.html | safe }}</div>
   </article>
   {% endfor %}
 </section>
+<script src="{{ asset('run-cell.js') }}" defer></script>
 {% endif %}
 
 {% if e.study %}