labnotes 0.0.111__tar.gz → 0.0.113__tar.gz

{labnotes-0.0.111 → labnotes-0.0.113}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.111
+Version: 0.0.113
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

labnotes-0.0.113/src/labnotes/__init__.py

@@ -0,0 +1,4 @@
+from labnotes import view
+from labnotes.log import log
+
+__all__ = ["log", "view"]

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/context.md

@@ -123,6 +123,13 @@ shows a waiting retry as `retry in 30s` and the end as `gave up (exit 75)`;
 
 Runs can take a long time. Start `run` in the background, then check on it with `labnotes experiments foo runs --format json`; a run with no `ended` is still going. Each run lists `size`, the bytes in its run folder, and a running run lists `memory`, the resident bytes of its command and everything it started.
 
+While a run goes, labnotes appends a memory sample to `memory.jsonl` in its run
+folder every 5 seconds: one JSON line, `{"time": <unix seconds>, "rss": <bytes>,
+"processes": <count>}`, for the command and everything it started. It keeps
+every sample, so a run killed for memory still shows the climb.
+`memory_interval` in `experiment.toml` (or `[experiment]` in `.labnotes.toml`)
+sets the seconds between samples; 0 turns it off.
+
 In a terminal, `run` shows a live table of runs (status, time, memory and
 folder size) and stays
 open when they finish. Failed runs show their status in red; `f` hides or shows them, and `r` on one

labnotes-0.0.113/src/labnotes/is_non_negative_number.py

@@ -0,0 +1,10 @@
+import math
+
+
+def is_non_negative_number(value: object) -> bool:
+    return (
+        isinstance(value, int | float)
+        and not isinstance(value, bool)
+        and math.isfinite(value)
+        and value >= 0
+    )

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/load_experiment.py

@@ -19,6 +19,7 @@ from labnotes.nests_runs import nests_runs
 from labnotes.parse_run_view import parse_run_view
 from labnotes.python_blocks import python_blocks
 from labnotes.render_markdown import render_markdown
+from labnotes.runs_view_command import runs_view_command
 from labnotes.sort_href import sort_href
 from labnotes.sql_quote import quote_string
 from labnotes.summarize_config import summarize_config
@@ -88,6 +89,7 @@ async def load_experiment(
         "run_columns": shown,
         "run_view": view,
         "run_choices": choices,
+        "runs_view": runs_view_command(config),
         "sort_hrefs": {
             column: sort_href(view, column) for column in ["status", *shown]
         },

labnotes-0.0.113/src/labnotes/memory_interval.py

@@ -0,0 +1,2 @@
+# Seconds between memory samples, unless experiment.toml sets `memory_interval`.
+DEFAULT_MEMORY_INTERVAL = 5.0

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/models.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from typing import Any, Protocol
 
 from labnotes.default_counts import DEFAULT_COUNTS
+from labnotes.memory_interval import DEFAULT_MEMORY_INTERVAL
 from labnotes.modes import DEFAULT_MODE
 from labnotes.retries import DEFAULT_RETRIES
 
@@ -66,3 +67,4 @@ class ExperimentConfig:
     concurrency: int = 1
     retries: int = DEFAULT_RETRIES
     meta: str | None = None
+    memory_interval: float = DEFAULT_MEMORY_INTERVAL

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/runner/execute_run.py

@@ -9,6 +9,7 @@ import psutil
 
 from labnotes.log_path import LOG_ENV
 from labnotes.runner.exit_signal import exit_signal
+from labnotes.runner.record_memory import record_memory
 from labnotes.runner.shell_command import shell_command
 from labnotes.runner.stop_then_kill import GRACE_SECONDS
 from labnotes.runner.wait_for_tree import wait_for_tree
@@ -30,6 +31,8 @@ async def execute_run(
     stop: Callable[[], None] = lambda: None,
     stopped: Callable[[], bool] = lambda: False,
     meta: dict[str, object] | None = None,
+    *,
+    memory_interval: float,
 ) -> dict[str, object]:
     resolved = shell_command(command, folder, params)
     run: dict[str, object] = {
@@ -69,9 +72,17 @@ async def execute_run(
         else:
             running.add(process)
             run["pid"] = process.pid
+            created = None
             with contextlib.suppress(psutil.Error):
-                run["pid_created"] = psutil.Process(process.pid).create_time()
+                run["pid_created"] = created = psutil.Process(process.pid).create_time()
             write_run_json(folder, run)
+            sampler = None
+            if memory_interval > 0 and created is not None:
+                sampler = asyncio.create_task(
+                    record_memory(
+                        folder / "memory.jsonl", process.pid, created, memory_interval
+                    )
+                )
             exited = asyncio.ensure_future(process.wait())
             kill_later = None
             try:
@@ -100,6 +111,9 @@ async def execute_run(
             finally:
                 if kill_later is not None:
                     kill_later.cancel()
+                if sampler is not None:
+                    sampler.cancel()
+                    await asyncio.wait({sampler})
                 running.discard(process)
     run["ended"] = iso_time(time.time())
     write_run_json(folder, run)

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/runner/read_experiment_config.py

@@ -7,6 +7,8 @@ from labnotes.compose_experiment import compose_experiment
 from labnotes.default_counts import DEFAULT_COUNTS
 from labnotes.models import ExperimentConfig
 from labnotes.is_non_negative_int import is_non_negative_int
+from labnotes.is_non_negative_number import is_non_negative_number
+from labnotes.memory_interval import DEFAULT_MEMORY_INTERVAL
 from labnotes.is_positive_int import is_positive_int
 from labnotes.modes import MODES
 
@@ -20,7 +22,10 @@ SETTINGS = {
     "concurrency",
     "retries",
     "meta",
+    "memory_interval",
+    "views",
 }
+VIEWS = {"runs"}
 
 
 def read_experiment_config(
@@ -73,6 +78,27 @@ def read_experiment_config(
     meta = values.get("meta")
     if meta is not None and (not isinstance(meta, str) or not meta.strip()):
         raise ValueError(f"{path}: meta must be a string")
+    memory_interval = values.get("memory_interval", DEFAULT_MEMORY_INTERVAL)
+    if not is_non_negative_number(memory_interval):
+        raise ValueError(f"{path}: memory_interval must be 0 or more seconds")
+    views = values.get("views", {})
+    if not isinstance(views, dict):
+        raise ValueError(f"{path}: views must be a table, like [views]")
+    unknown = sorted(views.keys() - VIEWS)
+    if unknown:
+        raise ValueError(f"{path}: unknown keys in views: {', '.join(unknown)}")
+    runs = views.get("runs")
+    if runs is not None and (not isinstance(runs, str) or not runs.strip()):
+        raise ValueError(f"{path}: views.runs must be a command")
     return ExperimentConfig(
-        command, repeats, params, name, counts, mode, concurrency, retries, meta
+        command,
+        repeats,
+        params,
+        name,
+        counts,
+        mode,
+        concurrency,
+        retries,
+        meta,
+        memory_interval,
     )

labnotes-0.0.113/src/labnotes/runner/record_memory.py

@@ -0,0 +1,18 @@
+import asyncio
+import json
+import time
+from pathlib import Path
+
+from labnotes.utils.tree_usage import tree_usage
+
+
+async def record_memory(path: Path, pid: int, created: float, interval: float) -> None:
+    with path.open("a", encoding="utf-8") as samples:
+        while True:
+            usage = tree_usage(pid, created)
+            if usage is not None:
+                rss, processes = usage
+                sample = {"time": time.time(), "rss": rss, "processes": processes}
+                samples.write(json.dumps(sample) + "\n")
+                samples.flush()
+            await asyncio.sleep(interval)

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/runner/run_experiment.py

@@ -261,6 +261,7 @@ async def run_experiment(
             stop=lambda: kill(run),
             stopped=lambda: run in stopped,
             meta=meta,
+            memory_interval=experiment.memory_interval,
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})

labnotes-0.0.113/src/labnotes/runs_view_command.py

@@ -0,0 +1,4 @@
+def runs_view_command(config: dict[str, object] | None) -> str | None:
+    views = (config or {}).get("views")
+    command = views.get("runs") if isinstance(views, dict) else None
+    return command if isinstance(command, str) and command.strip() else None

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/tables/open_tables.py

@@ -10,7 +10,9 @@ from labnotes.tables.read_blocks import read_blocks
 from labnotes.tables.read_experiment import read_experiment
 from labnotes.tables.read_file import read_file
 from labnotes.tables.read_log import read_log
+from labnotes.tables.read_memory import read_memory
 from labnotes.tables.read_run import read_run
+from labnotes.tables.read_run_file import read_run_file
 
 FILES = """
 CREATE TABLE files (
@@ -39,10 +41,33 @@ CREATE TABLE blocks (
   PRIMARY KEY (experiment, run)
 )
 """
+MEMORY = """
+CREATE TABLE memory (
+  experiment TEXT, run TEXT, line INTEGER, time REAL, rss INTEGER, processes INTEGER,
+  PRIMARY KEY (experiment, run, line)
+)
+"""
+
+# Every file in a run folder, so a change anywhere in it is a watch event.
+RUN_FILES = """
+CREATE TABLE run_files (
+  experiment TEXT, path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER
+)
+"""
 
 
-def open_tables(config: LabnotesConfig) -> DirSQL:
+def open_tables(config: LabnotesConfig, run_files: bool = False) -> DirSQL:
     root = config.root
+    extra = [
+        Table(
+            name="run_files",
+            ddl=RUN_FILES,
+            glob=path_glob(f"{config.runs}/**"),
+            on_file=partial(
+                read_run_file, root, config.runs, experiment=config.experiment
+            ),
+        )
+    ]
     return DirSQL(
         str(root),
         tables=[
@@ -78,6 +103,14 @@ def open_tables(config: LabnotesConfig) -> DirSQL:
                     read_blocks, root, config.runs, experiment=config.experiment
                 ),
             ),
+            Table(
+                name="memory",
+                ddl=MEMORY,
+                glob=path_glob(f"{config.runs}/memory.jsonl"),
+                on_file=partial(
+                    read_memory, root, config.runs, experiment=config.experiment
+                ),
+            ),
             Table(
                 name="experiment_configs",
                 ddl=experiments_ddl(config),
@@ -90,6 +123,7 @@ def open_tables(config: LabnotesConfig) -> DirSQL:
                     defaults=config.defaults,
                 ),
             ),
+            *(extra if run_files else []),
         ],
         # Run folders are often gitignored (DVC, large outputs); scan them anyway.
         no_ignore=True,

labnotes-0.0.113/src/labnotes/tables/read_memory.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+
+from labnotes.tables.path_pattern import path_pattern
+
+
+def read_memory(
+    root: Path, template: str, path: str, experiment: str | None = None
+) -> list[dict[str, object]]:
+    relative = Path(path).relative_to(root).as_posix()
+    found = path_pattern(f"{template}/memory.jsonl").fullmatch(relative)
+    if found is None:
+        return []
+    rows: list[dict[str, object]] = []
+    lines = Path(path).read_text(encoding="utf-8").splitlines()
+    for line, text in enumerate(lines):
+        # A run killed mid-write leaves a cut-off last line.
+        try:
+            sample = json.loads(text)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(sample, dict):
+            continue
+        rows.append(
+            {
+                "experiment": experiment or found["experiment"],
+                "run": found["run"],
+                "line": line,
+                "time": sample.get("time"),
+                "rss": sample.get("rss"),
+                "processes": sample.get("processes"),
+            }
+        )
+    return rows

labnotes-0.0.113/src/labnotes/tables/read_run_file.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+
+from labnotes.tables.path_pattern import path_pattern
+
+
+def read_run_file(
+    root: Path, template: str, path: str, experiment: str | None = None
+) -> list[dict[str, object]]:
+    relative = Path(path).relative_to(root).as_posix()
+    found = path_pattern(f"{template}/**").fullmatch(relative)
+    if found is None:
+        return []
+    stat = Path(path).stat()
+    return [
+        {
+            "experiment": experiment or found["experiment"],
+            "path": relative,
+            "size": stat.st_size,
+            "mtime_ns": stat.st_mtime_ns,
+        }
+    ]

labnotes-0.0.113/src/labnotes/tui/browse_app.py

@@ -0,0 +1,39 @@
+from textual.app import App
+from textual.binding import Binding
+
+from labnotes.models import LabnotesConfig
+from labnotes.tables.open_tables import open_tables
+from labnotes.tui.experiment_screen import ExperimentScreen
+from labnotes.tui.experiments_screen import ExperimentsScreen
+from labnotes.tui.runs_view import Request
+from labnotes.tui.view_process import ViewProcess
+
+
+class BrowseApp(App[BaseException | None]):
+    BINDINGS = [Binding("ctrl+c", "quit", "Quit")]
+
+    def __init__(self, config: LabnotesConfig, name: str | None) -> None:
+        super().__init__()
+        self.config = config
+        self.experiment_name = name
+        self.title = config.root.name
+        self.views: dict[str, ViewProcess] = {}
+
+    def on_mount(self) -> None:
+        tables = open_tables(self.config, run_files=True)
+        self.push_screen(ExperimentsScreen(tables, self.config, self.view_request))
+        if self.experiment_name is not None:
+            self.push_screen(
+                ExperimentScreen(
+                    tables, self.config, self.experiment_name, self.view_request
+                )
+            )
+
+    def view_request(self, command: str) -> Request:
+        if command not in self.views:
+            self.views[command] = ViewProcess(command, self.config.root)
+        return self.views[command].request
+
+    async def on_unmount(self) -> None:
+        for process in self.views.values():
+            await process.close()

labnotes-0.0.113/src/labnotes/tui/check_view_response.py

@@ -0,0 +1,43 @@
+import json
+from collections.abc import Mapping
+from typing import Any
+
+from labnotes.tui.view_column import view_column
+
+SHAPE = "the response must be an object with columns and rows"
+
+
+def check_view_response(
+    reply: str, request: Mapping[str, Any]
+) -> tuple[list[dict[str, str]], dict[str, dict[str, Any]]]:
+    try:
+        response = json.loads(reply)
+    except json.JSONDecodeError as error:
+        raise ValueError(f"invalid JSON: {error}") from error
+    if not isinstance(response, dict) or response.keys() != {"columns", "rows"}:
+        raise ValueError(SHAPE)
+    if not isinstance(response["columns"], list):
+        raise ValueError("columns must be a list")
+    columns = [view_column(entry) for entry in response["columns"]]
+    keys = set()
+    for each in columns:
+        if each["key"] in keys:
+            raise ValueError(f"column {each['key']} is listed twice")
+        keys.add(each["key"])
+    rows = response["rows"]
+    if not isinstance(rows, dict):
+        raise ValueError("rows must be an object")
+    asked = request["rows"]
+    for folder in asked:
+        if folder not in rows:
+            raise ValueError(f"no row for {folder}")
+    for folder, row in rows.items():
+        if folder not in asked:
+            raise ValueError(f"a row for {folder}, which wasn't asked for")
+        if not isinstance(row, dict):
+            raise ValueError(f"{folder}: the row must be an object")
+        # Native values may ride along unshown, so {**row, ...} needs no filtering.
+        stray = sorted(row.keys() - keys - asked[folder].keys())
+        if stray:
+            raise ValueError(f"{folder}: unknown columns {', '.join(stray)}")
+    return columns, rows

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/tui/experiment_screen.py

@@ -1,3 +1,4 @@
+from collections.abc import Callable
 from typing import Any
 
 from dirsql import DirSQL
@@ -5,7 +6,7 @@ from starlette.datastructures import QueryParams
 from textual.app import ComposeResult
 from textual.binding import Binding
 from textual.screen import Screen
-from textual.widgets import DataTable, Footer, Header
+from textual.widgets import DataTable, Footer, Header, Static
 from textual.worker import Worker, WorkerState
 
 from labnotes.load_experiment import load_experiment
@@ -13,9 +14,11 @@ from labnotes.models import LabnotesConfig
 from labnotes.tui.fill_table import fill_table
 from labnotes.tui.invocation_columns import invocation_columns
 from labnotes.tui.invocation_rows import invocation_rows
+from labnotes.tui.native_rows import native_rows
 from labnotes.tui.reload_on_change import reload_on_change
 from labnotes.tui.run_screen import RunScreen
 from labnotes.tui.runs_table import RunsTable
+from labnotes.tui.runs_view import Request, RunsView
 from labnotes.tui.stopped_title import stopped_title
 from labnotes.watch_rows import watch_rows
 
@@ -23,22 +26,35 @@ WATCHED = {"runs", "experiment_configs"}
 
 
 class ExperimentScreen(Screen[None]):
+    DEFAULT_CSS = """
+    ExperimentScreen RunsTable { height: 1fr; }
+    ExperimentScreen #view-status { color: $text-warning; }
+    """
     BINDINGS = [
         Binding("enter", "open_row", "Open", priority=True),
         Binding("escape", "app.pop_screen", "Back"),
     ]
 
-    def __init__(self, tables: DirSQL, config: LabnotesConfig, name: str) -> None:
+    def __init__(
+        self,
+        tables: DirSQL,
+        config: LabnotesConfig,
+        name: str,
+        views: Callable[[str], Request],
+    ) -> None:
         super().__init__()
         self.tables = tables
         self.config = config
         self.experiment_name = name
+        self.views = views
+        self.view: RunsView | None = None
         self.experiment: dict[str, Any] = {}
         self.opened: set[str] = set()
 
     def compose(self) -> ComposeResult:
         yield Header()
         yield RunsTable(id="runs", cursor_type="row")
+        yield Static(id="view-status")
         yield Footer()
 
     def on_mount(self) -> None:
@@ -75,14 +91,50 @@ class ExperimentScreen(Screen[None]):
             return
         self.experiment = loaded
         self.sub_title = str(loaded["display_name"])
+        command = loaded["runs_view"]
+        if command is not None and self.view is None:
+            self.view = RunsView(self.views(str(command)))
+            self.run_worker(self.watch_files(self.view), exit_on_error=False)
+        if self.view is not None:
+            self.view.update(invocation_columns(loaded), native_rows(loaded))
+            self.ask(self.view)
         self.redraw()
 
+    async def watch_files(self, view: RunsView) -> None:
+        async for row in watch_rows(self.tables, {"run_files"}, self.experiment_name):
+            view.touch(str(row["path"]))
+            self.ask(view)
+
+    def ask(self, view: RunsView) -> None:
+        request = view.next_request()
+        if request is not None:
+            self.run_worker(self.answer(view, request), exit_on_error=False)
+
+    async def answer(self, view: RunsView, request: dict[str, Any]) -> None:
+        if await view.exchange(request):
+            self.redraw()
+        else:
+            self.show_status()
+        self.ask(view)
+
     def redraw(self) -> None:
+        if self.view is None:
+            columns = invocation_columns(self.experiment)
+            rows = native_rows(self.experiment)
+        else:
+            columns, rows = self.view.table()
         fill_table(
             self.query_one(DataTable),
-            invocation_columns(self.experiment),
-            invocation_rows(self.experiment, self.opened),
+            [column["label"] for column in columns],
+            invocation_rows(self.experiment, self.opened, columns, rows),
         )
+        self.show_status()
+
+    def show_status(self) -> None:
+        text = "" if self.view is None else self.view.status()
+        status = self.query_one("#view-status", Static)
+        status.update(text)
+        status.display = bool(text)
 
     def action_open_row(self) -> None:
         table = self.query_one(DataTable)

{labnotes-0.0.111 → labnotes-0.0.113}/src/labnotes/tui/experiments_screen.py

@@ -1,3 +1,5 @@
+from collections.abc import Callable
+
 from dirsql import DirSQL
 from textual.app import ComposeResult
 from textual.screen import Screen
@@ -10,6 +12,7 @@ from labnotes.tui.experiment_screen import ExperimentScreen
 from labnotes.tui.fill_table import fill_table
 from labnotes.tui.reload_on_change import reload_on_change
 from labnotes.tui.runs_table import RunsTable
+from labnotes.tui.runs_view import Request
 from labnotes.tui.stopped_title import stopped_title
 from labnotes.watch_rows import watch_rows
 
@@ -17,10 +20,13 @@ WATCHED = {"files", "runs", "experiment_configs"}
 
 
 class ExperimentsScreen(Screen[None]):
-    def __init__(self, tables: DirSQL, config: LabnotesConfig) -> None:
+    def __init__(
+        self, tables: DirSQL, config: LabnotesConfig, views: Callable[[str], Request]
+    ) -> None:
         super().__init__()
         self.tables = tables
         self.config = config
+        self.views = views
 
     def compose(self) -> ComposeResult:
         yield Header()
@@ -57,5 +63,7 @@ class ExperimentsScreen(Screen[None]):
 
     def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
         self.app.push_screen(
-            ExperimentScreen(self.tables, self.config, str(event.row_key.value))
+            ExperimentScreen(
+                self.tables, self.config, str(event.row_key.value), self.views
+            )
         )

labnotes-0.0.113/src/labnotes/tui/format_cell.py

@@ -0,0 +1,27 @@
+from datetime import UTC, datetime
+
+from rich.text import Text
+
+from labnotes.format_value import format_value
+from labnotes.tui.dim import dim
+from labnotes.tui.format_elapsed import format_elapsed
+
+
+def format_cell(key: str, value: object, live: bool) -> object:
+    if key == "status" and value == "passed":
+        return Text("✓", style="green")
+    if key == "status" and value == "failed":
+        return Text("✕", style="red")
+    if key == "status" and value == "unfinished":
+        return Text("◐", style="yellow") if live else dim("–")
+    if key == "duration" and isinstance(value, int) and not isinstance(value, bool):
+        return format_elapsed(value)
+    if key == "started" and isinstance(value, str):
+        try:
+            started = datetime.fromisoformat(value)
+        except ValueError:
+            return value
+        if started.tzinfo is not None:
+            started = started.astimezone(UTC)
+        return started.strftime("%Y-%m-%d %H:%M:%S")
+    return format_value(value)

labnotes-0.0.113/src/labnotes/tui/group_cells.py

@@ -0,0 +1,36 @@
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from rich.text import Text
+
+from labnotes.format_value import format_value
+
+
+def group_cells(
+    group: Mapping[str, Any],
+    columns: Sequence[Mapping[str, str]],
+    live: bool,
+    opened: bool,
+) -> list[object]:
+    cells: list[object] = []
+    for column in columns:
+        key = column["key"]
+        if key == "status":
+            status = Text.assemble(
+                (f"{group['passed']}✓", "green"), " ", (f"{group['failed']}✕", "red")
+            )
+            if group["unfinished"]:
+                mark, style = ("◐", "yellow") if live else ("–", "dim")
+                status.append(f" {group['unfinished']}{mark}", style=style)
+            cells.append(status)
+        elif key == "id":
+            cells.append(f"{'▾' if opened else '▸'} {group['name']}")
+        elif key == "runs":
+            cells.append(str(len(group["runs"])))
+        elif key == "started":
+            cells.append(group["started"] or "")
+        elif key in group["shared"]:
+            cells.append(format_value(group["shared"][key]))
+        else:
+            cells.append("")
+    return cells

labnotes-0.0.113/src/labnotes/tui/invocation_columns.py

@@ -0,0 +1,9 @@
+from collections.abc import Mapping
+from typing import Any
+
+
+def invocation_columns(experiment: Mapping[str, Any]) -> list[dict[str, str]]:
+    keys = list(experiment["run_columns"])
+    if experiment["run_groups"] is not None:
+        keys.insert(1, "runs")
+    return [{"key": "status", "label": ""}, *({"key": k, "label": k} for k in keys)]