labnotes 0.0.112__tar.gz → 0.0.114__tar.gz

{labnotes-0.0.112 → labnotes-0.0.114}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.112
+Version: 0.0.114
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

labnotes-0.0.114/src/labnotes/__init__.py

@@ -0,0 +1,4 @@
+from labnotes import view
+from labnotes.log import log
+
+__all__ = ["log", "view"]

{labnotes-0.0.112 → labnotes-0.0.114}/src/labnotes/load_experiment.py

@@ -19,6 +19,7 @@ from labnotes.nests_runs import nests_runs
 from labnotes.parse_run_view import parse_run_view
 from labnotes.python_blocks import python_blocks
 from labnotes.render_markdown import render_markdown
+from labnotes.runs_view_command import runs_view_command
 from labnotes.sort_href import sort_href
 from labnotes.sql_quote import quote_string
 from labnotes.summarize_config import summarize_config
@@ -88,6 +89,7 @@ async def load_experiment(
         "run_columns": shown,
         "run_view": view,
         "run_choices": choices,
+        "runs_view": runs_view_command(config),
         "sort_hrefs": {
             column: sort_href(view, column) for column in ["status", *shown]
         },

{labnotes-0.0.112 → labnotes-0.0.114}/src/labnotes/runner/read_experiment_config.py

@@ -23,7 +23,9 @@ SETTINGS = {
     "retries",
     "meta",
     "memory_interval",
+    "views",
 }
+VIEWS = {"runs"}
 
 
 def read_experiment_config(
@@ -79,6 +81,15 @@ def read_experiment_config(
     memory_interval = values.get("memory_interval", DEFAULT_MEMORY_INTERVAL)
     if not is_non_negative_number(memory_interval):
         raise ValueError(f"{path}: memory_interval must be 0 or more seconds")
+    views = values.get("views", {})
+    if not isinstance(views, dict):
+        raise ValueError(f"{path}: views must be a table, like [views]")
+    unknown = sorted(views.keys() - VIEWS)
+    if unknown:
+        raise ValueError(f"{path}: unknown keys in views: {', '.join(unknown)}")
+    runs = views.get("runs")
+    if runs is not None and (not isinstance(runs, str) or not runs.strip()):
+        raise ValueError(f"{path}: views.runs must be a command")
     return ExperimentConfig(
         command,
         repeats,

labnotes-0.0.114/src/labnotes/runs_view_command.py

@@ -0,0 +1,4 @@
+def runs_view_command(config: dict[str, object] | None) -> str | None:
+    views = (config or {}).get("views")
+    command = views.get("runs") if isinstance(views, dict) else None
+    return command if isinstance(command, str) and command.strip() else None

{labnotes-0.0.112 → labnotes-0.0.114}/src/labnotes/tables/open_tables.py

@@ -12,6 +12,7 @@ from labnotes.tables.read_file import read_file
 from labnotes.tables.read_log import read_log
 from labnotes.tables.read_memory import read_memory
 from labnotes.tables.read_run import read_run
+from labnotes.tables.read_run_file import read_run_file
 
 FILES = """
 CREATE TABLE files (
@@ -47,9 +48,26 @@ CREATE TABLE memory (
 )
 """
 
+# Every file in a run folder, so a change anywhere in it is a watch event.
+RUN_FILES = """
+CREATE TABLE run_files (
+  experiment TEXT, path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER
+)
+"""
 
-def open_tables(config: LabnotesConfig) -> DirSQL:
+
+def open_tables(config: LabnotesConfig, run_files: bool = False) -> DirSQL:
     root = config.root
+    extra = [
+        Table(
+            name="run_files",
+            ddl=RUN_FILES,
+            glob=path_glob(f"{config.runs}/**"),
+            on_file=partial(
+                read_run_file, root, config.runs, experiment=config.experiment
+            ),
+        )
+    ]
     return DirSQL(
         str(root),
         tables=[
@@ -105,6 +123,7 @@ def open_tables(config: LabnotesConfig) -> DirSQL:
                     defaults=config.defaults,
                 ),
             ),
+            *(extra if run_files else []),
         ],
         # Run folders are often gitignored (DVC, large outputs); scan them anyway.
         no_ignore=True,

labnotes-0.0.114/src/labnotes/tables/read_run_file.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+
+from labnotes.tables.path_pattern import path_pattern
+
+
+def read_run_file(
+    root: Path, template: str, path: str, experiment: str | None = None
+) -> list[dict[str, object]]:
+    relative = Path(path).relative_to(root).as_posix()
+    found = path_pattern(f"{template}/**").fullmatch(relative)
+    if found is None:
+        return []
+    stat = Path(path).stat()
+    return [
+        {
+            "experiment": experiment or found["experiment"],
+            "path": relative,
+            "size": stat.st_size,
+            "mtime_ns": stat.st_mtime_ns,
+        }
+    ]

labnotes-0.0.114/src/labnotes/tui/browse_app.py

@@ -0,0 +1,39 @@
+from textual.app import App
+from textual.binding import Binding
+
+from labnotes.models import LabnotesConfig
+from labnotes.tables.open_tables import open_tables
+from labnotes.tui.experiment_screen import ExperimentScreen
+from labnotes.tui.experiments_screen import ExperimentsScreen
+from labnotes.tui.runs_view import Request
+from labnotes.tui.view_process import ViewProcess
+
+
+class BrowseApp(App[BaseException | None]):
+    BINDINGS = [Binding("ctrl+c", "quit", "Quit")]
+
+    def __init__(self, config: LabnotesConfig, name: str | None) -> None:
+        super().__init__()
+        self.config = config
+        self.experiment_name = name
+        self.title = config.root.name
+        self.views: dict[str, ViewProcess] = {}
+
+    def on_mount(self) -> None:
+        tables = open_tables(self.config, run_files=True)
+        self.push_screen(ExperimentsScreen(tables, self.config, self.view_request))
+        if self.experiment_name is not None:
+            self.push_screen(
+                ExperimentScreen(
+                    tables, self.config, self.experiment_name, self.view_request
+                )
+            )
+
+    def view_request(self, command: str) -> Request:
+        if command not in self.views:
+            self.views[command] = ViewProcess(command, self.config.root)
+        return self.views[command].request
+
+    async def on_unmount(self) -> None:
+        for process in self.views.values():
+            await process.close()

labnotes-0.0.114/src/labnotes/tui/check_view_response.py

@@ -0,0 +1,43 @@
+import json
+from collections.abc import Mapping
+from typing import Any
+
+from labnotes.tui.view_column import view_column
+
+SHAPE = "the response must be an object with columns and rows"
+
+
+def check_view_response(
+    reply: str, request: Mapping[str, Any]
+) -> tuple[list[dict[str, str]], dict[str, dict[str, Any]]]:
+    try:
+        response = json.loads(reply)
+    except json.JSONDecodeError as error:
+        raise ValueError(f"invalid JSON: {error}") from error
+    if not isinstance(response, dict) or response.keys() != {"columns", "rows"}:
+        raise ValueError(SHAPE)
+    if not isinstance(response["columns"], list):
+        raise ValueError("columns must be a list")
+    columns = [view_column(entry) for entry in response["columns"]]
+    keys = set()
+    for each in columns:
+        if each["key"] in keys:
+            raise ValueError(f"column {each['key']} is listed twice")
+        keys.add(each["key"])
+    rows = response["rows"]
+    if not isinstance(rows, dict):
+        raise ValueError("rows must be an object")
+    asked = request["rows"]
+    for folder in asked:
+        if folder not in rows:
+            raise ValueError(f"no row for {folder}")
+    for folder, row in rows.items():
+        if folder not in asked:
+            raise ValueError(f"a row for {folder}, which wasn't asked for")
+        if not isinstance(row, dict):
+            raise ValueError(f"{folder}: the row must be an object")
+        # Native values may ride along unshown, so {**row, ...} needs no filtering.
+        stray = sorted(row.keys() - keys - asked[folder].keys())
+        if stray:
+            raise ValueError(f"{folder}: unknown columns {', '.join(stray)}")
+    return columns, rows

{labnotes-0.0.112 → labnotes-0.0.114}/src/labnotes/tui/experiment_screen.py

@@ -1,3 +1,4 @@
+from collections.abc import Callable
 from typing import Any
 
 from dirsql import DirSQL
@@ -5,7 +6,7 @@ from starlette.datastructures import QueryParams
 from textual.app import ComposeResult
 from textual.binding import Binding
 from textual.screen import Screen
-from textual.widgets import DataTable, Footer, Header
+from textual.widgets import DataTable, Footer, Header, Static
 from textual.worker import Worker, WorkerState
 
 from labnotes.load_experiment import load_experiment
@@ -13,9 +14,11 @@ from labnotes.models import LabnotesConfig
 from labnotes.tui.fill_table import fill_table
 from labnotes.tui.invocation_columns import invocation_columns
 from labnotes.tui.invocation_rows import invocation_rows
+from labnotes.tui.native_rows import native_rows
 from labnotes.tui.reload_on_change import reload_on_change
 from labnotes.tui.run_screen import RunScreen
 from labnotes.tui.runs_table import RunsTable
+from labnotes.tui.runs_view import Request, RunsView
 from labnotes.tui.stopped_title import stopped_title
 from labnotes.watch_rows import watch_rows
 
@@ -23,22 +26,35 @@ WATCHED = {"runs", "experiment_configs"}
 
 
 class ExperimentScreen(Screen[None]):
+    DEFAULT_CSS = """
+    ExperimentScreen RunsTable { height: 1fr; }
+    ExperimentScreen #view-status { color: $text-warning; }
+    """
     BINDINGS = [
         Binding("enter", "open_row", "Open", priority=True),
         Binding("escape", "app.pop_screen", "Back"),
     ]
 
-    def __init__(self, tables: DirSQL, config: LabnotesConfig, name: str) -> None:
+    def __init__(
+        self,
+        tables: DirSQL,
+        config: LabnotesConfig,
+        name: str,
+        views: Callable[[str], Request],
+    ) -> None:
         super().__init__()
         self.tables = tables
         self.config = config
         self.experiment_name = name
+        self.views = views
+        self.view: RunsView | None = None
         self.experiment: dict[str, Any] = {}
         self.opened: set[str] = set()
 
     def compose(self) -> ComposeResult:
         yield Header()
         yield RunsTable(id="runs", cursor_type="row")
+        yield Static(id="view-status")
         yield Footer()
 
     def on_mount(self) -> None:
@@ -75,14 +91,50 @@ class ExperimentScreen(Screen[None]):
             return
         self.experiment = loaded
         self.sub_title = str(loaded["display_name"])
+        command = loaded["runs_view"]
+        if command is not None and self.view is None:
+            self.view = RunsView(self.views(str(command)))
+            self.run_worker(self.watch_files(self.view), exit_on_error=False)
+        if self.view is not None:
+            self.view.update(invocation_columns(loaded), native_rows(loaded))
+            self.ask(self.view)
         self.redraw()
 
+    async def watch_files(self, view: RunsView) -> None:
+        async for row in watch_rows(self.tables, {"run_files"}, self.experiment_name):
+            view.touch(str(row["path"]))
+            self.ask(view)
+
+    def ask(self, view: RunsView) -> None:
+        request = view.next_request()
+        if request is not None:
+            self.run_worker(self.answer(view, request), exit_on_error=False)
+
+    async def answer(self, view: RunsView, request: dict[str, Any]) -> None:
+        if await view.exchange(request):
+            self.redraw()
+        else:
+            self.show_status()
+        self.ask(view)
+
     def redraw(self) -> None:
+        if self.view is None:
+            columns = invocation_columns(self.experiment)
+            rows = native_rows(self.experiment)
+        else:
+            columns, rows = self.view.table()
         fill_table(
             self.query_one(DataTable),
-            invocation_columns(self.experiment),
-            invocation_rows(self.experiment, self.opened),
+            [column["label"] for column in columns],
+            invocation_rows(self.experiment, self.opened, columns, rows),
         )
+        self.show_status()
+
+    def show_status(self) -> None:
+        text = "" if self.view is None else self.view.status()
+        status = self.query_one("#view-status", Static)
+        status.update(text)
+        status.display = bool(text)
 
     def action_open_row(self) -> None:
         table = self.query_one(DataTable)

{labnotes-0.0.112 → labnotes-0.0.114}/src/labnotes/tui/experiments_screen.py

@@ -1,3 +1,5 @@
+from collections.abc import Callable
+
 from dirsql import DirSQL
 from textual.app import ComposeResult
 from textual.screen import Screen
@@ -10,6 +12,7 @@ from labnotes.tui.experiment_screen import ExperimentScreen
 from labnotes.tui.fill_table import fill_table
 from labnotes.tui.reload_on_change import reload_on_change
 from labnotes.tui.runs_table import RunsTable
+from labnotes.tui.runs_view import Request
 from labnotes.tui.stopped_title import stopped_title
 from labnotes.watch_rows import watch_rows
 
@@ -17,10 +20,13 @@ WATCHED = {"files", "runs", "experiment_configs"}
 
 
 class ExperimentsScreen(Screen[None]):
-    def __init__(self, tables: DirSQL, config: LabnotesConfig) -> None:
+    def __init__(
+        self, tables: DirSQL, config: LabnotesConfig, views: Callable[[str], Request]
+    ) -> None:
         super().__init__()
         self.tables = tables
         self.config = config
+        self.views = views
 
     def compose(self) -> ComposeResult:
         yield Header()
@@ -57,5 +63,7 @@ class ExperimentsScreen(Screen[None]):
 
     def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
         self.app.push_screen(
-            ExperimentScreen(self.tables, self.config, str(event.row_key.value))
+            ExperimentScreen(
+                self.tables, self.config, str(event.row_key.value), self.views
+            )
         )

labnotes-0.0.114/src/labnotes/tui/format_cell.py

@@ -0,0 +1,27 @@
+from datetime import UTC, datetime
+
+from rich.text import Text
+
+from labnotes.format_value import format_value
+from labnotes.tui.dim import dim
+from labnotes.tui.format_elapsed import format_elapsed
+
+
+def format_cell(key: str, value: object, live: bool) -> object:
+    if key == "status" and value == "passed":
+        return Text("✓", style="green")
+    if key == "status" and value == "failed":
+        return Text("✕", style="red")
+    if key == "status" and value == "unfinished":
+        return Text("◐", style="yellow") if live else dim("–")
+    if key == "duration" and isinstance(value, int) and not isinstance(value, bool):
+        return format_elapsed(value)
+    if key == "started" and isinstance(value, str):
+        try:
+            started = datetime.fromisoformat(value)
+        except ValueError:
+            return value
+        if started.tzinfo is not None:
+            started = started.astimezone(UTC)
+        return started.strftime("%Y-%m-%d %H:%M:%S")
+    return format_value(value)

labnotes-0.0.114/src/labnotes/tui/group_cells.py

@@ -0,0 +1,36 @@
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from rich.text import Text
+
+from labnotes.format_value import format_value
+
+
+def group_cells(
+    group: Mapping[str, Any],
+    columns: Sequence[Mapping[str, str]],
+    live: bool,
+    opened: bool,
+) -> list[object]:
+    cells: list[object] = []
+    for column in columns:
+        key = column["key"]
+        if key == "status":
+            status = Text.assemble(
+                (f"{group['passed']}✓", "green"), " ", (f"{group['failed']}✕", "red")
+            )
+            if group["unfinished"]:
+                mark, style = ("◐", "yellow") if live else ("–", "dim")
+                status.append(f" {group['unfinished']}{mark}", style=style)
+            cells.append(status)
+        elif key == "id":
+            cells.append(f"{'▾' if opened else '▸'} {group['name']}")
+        elif key == "runs":
+            cells.append(str(len(group["runs"])))
+        elif key == "started":
+            cells.append(group["started"] or "")
+        elif key in group["shared"]:
+            cells.append(format_value(group["shared"][key]))
+        else:
+            cells.append("")
+    return cells

labnotes-0.0.114/src/labnotes/tui/invocation_columns.py

@@ -0,0 +1,9 @@
+from collections.abc import Mapping
+from typing import Any
+
+
+def invocation_columns(experiment: Mapping[str, Any]) -> list[dict[str, str]]:
+    keys = list(experiment["run_columns"])
+    if experiment["run_groups"] is not None:
+        keys.insert(1, "runs")
+    return [{"key": "status", "label": ""}, *({"key": k, "label": k} for k in keys)]

labnotes-0.0.114/src/labnotes/tui/invocation_rows.py

@@ -0,0 +1,32 @@
+from collections.abc import Collection, Mapping, Sequence
+from typing import Any
+
+from labnotes.tui.group_cells import group_cells
+from labnotes.tui.run_cells import run_cells
+
+
+def invocation_rows(
+    experiment: Mapping[str, Any],
+    opened: Collection[str],
+    columns: Sequence[Mapping[str, str]],
+    rows: Mapping[str, Mapping[str, object]],
+) -> list[tuple[str, list[object]]]:
+    live = experiment["runs_live"]
+    groups = experiment["run_groups"]
+
+    def cells(run: Mapping[str, object], nested: bool) -> list[object]:
+        return run_cells(rows[str(run["folder"])], columns, nested, live)
+
+    if groups is None:
+        return [(str(run["id"]), cells(run, False)) for run in experiment["runs_list"]]
+    shown = []
+    for group in groups:
+        if group["flat"]:
+            [run, *_] = group["runs"]
+            shown.append((str(run["id"]), cells(run, True)))
+            continue
+        is_open = group["name"] in opened
+        shown.append((group["name"], group_cells(group, columns, live, is_open)))
+        if is_open:
+            shown += [(str(run["id"]), cells(run, True)) for run in group["runs"]]
+    return shown

labnotes-0.0.114/src/labnotes/tui/native_rows.py

@@ -0,0 +1,24 @@
+import json
+from collections.abc import Mapping
+from typing import Any
+
+JSON_FIELDS = ("params", "meta", "run")
+
+
+def native_row(run: Mapping[str, Any]) -> dict[str, Any]:
+    row = {key: value for key, value in run.items() if not key.startswith("labnotes_")}
+    for key in JSON_FIELDS:
+        if isinstance(row.get(key), str):
+            row[key] = json.loads(row[key])
+    if run["labnotes_unfinished"]:
+        row["status"] = "unfinished"
+    elif run["labnotes_completed"]:
+        row["status"] = "passed"
+    else:
+        row["status"] = "failed"
+    row["started"] = run["created"]
+    return row
+
+
+def native_rows(experiment: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
+    return {str(run["folder"]): native_row(run) for run in experiment["runs_list"]}

labnotes-0.0.114/src/labnotes/tui/run_cells.py

@@ -0,0 +1,22 @@
+from collections.abc import Mapping, Sequence
+
+from labnotes.tui.format_cell import format_cell
+
+
+def run_cells(
+    row: Mapping[str, object],
+    columns: Sequence[Mapping[str, str]],
+    nested: bool,
+    live: bool,
+) -> list[object]:
+    cells: list[object] = []
+    for column in columns:
+        key = column["key"]
+        if key not in row:
+            cells.append("")
+        elif key == "id" and nested:
+            child = str(row[key]).partition("/")[2]
+            cells.append(f"  {child}" if child else str(row[key]))
+        else:
+            cells.append(format_cell(key, row[key], live))
+    return cells

labnotes-0.0.114/src/labnotes/tui/runs_view.py

@@ -0,0 +1,81 @@
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from labnotes.tui.check_view_response import check_view_response
+
+type Columns = list[dict[str, str]]
+type Rows = dict[str, dict[str, Any]]
+type Request = Callable[[dict[str, Any]], Awaitable[tuple[str, float]]]
+
+SLOW_SECONDS = 0.1
+
+
+class RunsView:
+    """Track which runs the view worker must see again, and its last good answer."""
+
+    def __init__(self, request: Request) -> None:
+        self.request = request
+        self.columns: Columns = []
+        self.native: Rows = {}
+        self.sent_columns: Columns | None = None
+        self.sent: Rows = {}
+        self.pending: set[str] = set()
+        self.busy = False
+        self.shown_columns: Columns | None = None
+        self.shown: Rows = {}
+        self.error: str | None = None
+        self.elapsed: float | None = None
+
+    def update(self, columns: Columns, native: Rows) -> None:
+        if columns != self.sent_columns:
+            self.pending |= native.keys()
+        self.pending |= {f for f, row in native.items() if self.sent.get(f) != row}
+        self.pending &= native.keys()
+        self.sent = {f: row for f, row in self.sent.items() if f in native}
+        self.shown = {f: row for f, row in self.shown.items() if f in native}
+        self.columns, self.native = columns, native
+
+    def touch(self, path: str) -> None:
+        owners = [f for f in self.native if path.startswith(f"{f}/")]
+        if owners:
+            self.pending.add(max(owners, key=len))
+
+    def next_request(self) -> dict[str, Any] | None:
+        if self.busy or not self.pending:
+            return None
+        rows = {folder: self.native[folder] for folder in sorted(self.pending)}
+        self.pending = set()
+        self.busy = True
+        self.sent_columns = self.columns
+        self.sent.update(rows)
+        return {"columns": self.columns, "rows": rows}
+
+    async def exchange(self, request: dict[str, Any]) -> bool:
+        """Send a request and keep a valid answer; report whether rows changed."""
+        try:
+            reply, self.elapsed = await self.request(request)
+            columns, rows = check_view_response(reply, request)
+        except (ValueError, OSError) as error:
+            self.error = str(error)
+            return False
+        finally:
+            self.busy = False
+        changed = columns != self.shown_columns or any(
+            self.shown.get(folder) != row for folder, row in rows.items()
+        )
+        self.error = None
+        self.shown_columns = columns
+        self.shown.update(rows)
+        return changed
+
+    def table(self) -> tuple[Columns, Rows]:
+        rows = {f: self.shown.get(f, row) for f, row in self.native.items()}
+        return self.columns if self.shown_columns is None else self.shown_columns, rows
+
+    def status(self) -> str:
+        notes = []
+        if self.error is not None:
+            notes.append(f"runs view: {self.error}")
+        if self.elapsed is not None and self.elapsed > SLOW_SECONDS:
+            notes.append(f"runs view took {round(self.elapsed * 1000)} ms")
+        return " · ".join(notes)

labnotes-0.0.114/src/labnotes/tui/view_column.py

@@ -0,0 +1,15 @@
+def view_column(entry: object) -> dict[str, str]:
+    if isinstance(entry, str):
+        return {"key": entry, "label": entry}
+    if not isinstance(entry, dict):
+        raise ValueError("a column must be a string or an object")
+    key = entry.get("key")
+    if not isinstance(key, str):
+        raise ValueError("a column needs a string key")
+    label = entry.get("label", key)
+    if not isinstance(label, str):
+        raise ValueError(f"column {key}: label must be a string")
+    unknown = sorted(entry.keys() - {"key", "label"})
+    if unknown:
+        raise ValueError(f"column {key}: unknown fields {', '.join(unknown)}")
+    return {"key": key, "label": label}