kalfa 3.0.0.dev0__py3-none-any.whl

kalfa/summary.py

@@ -0,0 +1,296 @@
+import math
+import numbers
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import pandas
+
+from .core.errors import KalfaError
+from .records import reader
+from .records import sweep as sweeping
+
+
+FIXED = ("name", "status", "turn", "total", "seconds")
+RESERVED = (*FIXED, "best", "best_turn")
+HIDDEN = ("train/", "lr/")
+SETS = {"train": 0, "val": 1, "test": 2}
+normal_mode = reader.normal_mode
+
+
+@dataclass
+class Summary:
+    records: list
+    table: object
+    by: str | None
+    mode: str
+    best: dict | None = None
+    folds: list | None = None
+    sweep: dict | None = None
+    points: list = field(default_factory=list)
+
+    def to_markdown(self):
+        return markdown_of(self)
+
+
+def under(root):
+    root = Path(root)
+    found = []
+    for entry in reader.records_under(root):
+        if entry.get("kind") == "sweep":
+            found.extend(Path(point) for point in sweeping.point_dirs(root / entry["path"]))
+        else:
+            found.append(root / entry["path"])
+    return found
+
+
+def expand(paths):
+    if isinstance(paths, (str, os.PathLike)):
+        paths = [paths]
+    directories, seen, root = [], set(), None
+    for given in paths:
+        path = Path(given)
+        if reader.is_record(path):
+            found = [path]
+        elif sweeping.is_sweep(path):
+            found = [Path(point) for point in sweeping.point_dirs(path)]
+            if not found:
+                raise KalfaError(f"no records under {given}")
+            if root is None:
+                root = path
+        elif path.is_dir():
+            found = under(path)
+            if not found:
+                raise KalfaError(f"no records under {given}")
+        else:
+            raise KalfaError(f"not a record, a sweep root or a directory of records: {given}")
+        for directory in found:
+            key = directory.resolve()
+            if key not in seen:
+                seen.add(key)
+                directories.append(str(directory))
+    sweep = sweeping.read_sweep(root) if root is not None else None
+    points = sweeping.sweep_points(root) if root is not None else []
+    return directories, sweep, points
+
+
+def default_series(summaries):
+    if not summaries:
+        return None, "min"
+    best = summaries[0].get("best")
+    if isinstance(best, dict) and best.get("series"):
+        return str(best["series"]), normal_mode(best.get("mode"))
+    last = summaries[0].get("last") or {}
+    by = next((key for key in last if str(key).startswith("val/")), None)
+    return by, "min"
+
+
+def value_of(summary, by, mode):
+    if by is None or summary.get("status") == "failed":
+        return None, None
+    best = summary.get("best")
+    if isinstance(best, dict) and best.get("series") == by and normal_mode(best.get("mode")) == normal_mode(mode):
+        return best.get("value"), best.get("turn")
+    return reader.best_in(reader.read_history(summary["path"]), by, normal_mode(mode))
+
+
+def sort_key(row, maximize):
+    value = row["best"]
+    if value is None or (isinstance(value, float) and math.isnan(value)):
+        return (1, 0.0)
+    return (0, -float(value) if maximize else float(value))
+
+
+def series_key(series):
+    name = str(series)
+    return (SETS.get(name.split("/", 1)[0], len(SETS)), name)
+
+
+def column_of(param):
+    key = str(param)
+    if key in RESERVED or key.startswith("last_"):
+        return f"param_{key}"
+    return key
+
+
+def table_of(summaries, by, mode):
+    params, series = [], []
+    for summary in summaries:
+        for key in summary.get("params") or {}:
+            if key not in params:
+                params.append(key)
+        for key in summary.get("last") or {}:
+            if key != by and key not in series:
+                series.append(key)
+    columns = [*FIXED, *(column_of(key) for key in params), "best", "best_turn"]
+    if by is not None:
+        columns.append(f"last_{by}")
+    series.sort(key=series_key)
+    columns.extend(f"last_{key}" for key in series)
+    rows = []
+    for summary in summaries:
+        given = summary.get("params") or {}
+        last = summary.get("last") or {}
+        row = {key: summary.get(key) for key in FIXED}
+        row.update({column_of(key): given.get(key) for key in params})
+        row["best"], row["best_turn"] = value_of(summary, by, mode)
+        if by is not None:
+            row[f"last_{by}"] = last.get(by)
+        row.update({f"last_{key}": last.get(key) for key in series})
+        rows.append(row)
+    maximize = normal_mode(mode) == "max"
+    rows.sort(key=lambda row: sort_key(row, maximize))
+    return pandas.DataFrame(rows, columns=columns)
+
+
+def clean(value):
+    if isinstance(value, numbers.Real) and not isinstance(value, bool) and math.isnan(value):
+        return None
+    if isinstance(value, numbers.Integral) and not isinstance(value, bool):
+        return int(value)
+    if isinstance(value, numbers.Real) and not isinstance(value, bool):
+        return float(value)
+    return value
+
+
+def summarize(paths, by=None, mode=None, top=None):
+    directories, sweep, points = expand(paths)
+    summaries = [reader.summary_of(directory) for directory in directories]
+    default_by, default_mode = default_series(summaries)
+    by = by if by is not None else default_by
+    mode = normal_mode(mode if mode is not None else default_mode)
+    table = table_of(summaries, by, mode)
+    if top is not None:
+        table = table.head(int(top)).reset_index(drop=True)
+    best = {key: clean(value) for key, value in table.iloc[0].to_dict().items()} if len(table) else None
+    if best is not None and best["best"] is None:
+        best = None
+    folds = None
+    if any("fold" in (summary.get("params") or {}) for summary in summaries):
+        folds = sweeping.group_folds(summaries)
+    return Summary(records=summaries, table=table, by=by, mode=mode, best=best, folds=folds, sweep=sweep,
+                   points=list(points or []))
+
+
+def cell(value, column=None):
+    value = clean(value)
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return str(value)
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        return format(value, ".6g" if column == "seconds" else ".4g")
+    return str(value)
+
+
+def hidden(column, by):
+    name = str(column)
+    if not name.startswith("last_"):
+        return False
+    series = name[len("last_"):]
+    return series != by and series.startswith(HIDDEN)
+
+
+def columns_of(summary, columns):
+    present = [str(column) for column in summary.table.columns]
+    if columns is None:
+        return [column for column in present if not hidden(column, summary.by)]
+    if isinstance(columns, str):
+        if columns == "all":
+            return present
+        columns = [part.strip() for part in columns.split(",") if part.strip()]
+    chosen = [str(column) for column in columns]
+    unknown = [column for column in chosen if column not in present]
+    if unknown:
+        raise KalfaError(f"unknown columns: {', '.join(unknown)}")
+    return chosen
+
+
+def rows_of(summary, top, columns=None):
+    table = summary.table
+    if top is not None:
+        table = table.head(int(top))
+    chosen = columns_of(summary, columns)
+    table = table[chosen]
+    return chosen, [[cell(value, column) for value, column in zip(row, chosen)]
+                    for row in table.itertuples(index=False, name=None)]
+
+
+def text_of(summary, top=None, columns=None):
+    columns, rows = rows_of(summary, top, columns)
+    widths = [max([len(column), *(len(row[position]) for row in rows)]) for position, column in enumerate(columns)]
+    lines = ["  ".join(column.ljust(width) for column, width in zip(columns, widths)).rstrip()]
+    for row in rows:
+        lines.append("  ".join(value.ljust(width) for value, width in zip(row, widths)).rstrip())
+    if not rows:
+        lines.append("no records")
+    return "\n".join(lines)
+
+
+def fold_label(group):
+    members = group.get("members") or []
+    folds = ", ".join(cell((member.get("params") or {}).get("fold")) for member in members)
+    where = ", ".join(f"{key}={cell(value)}" for key, value in (group.get("group") or {}).items())
+    return f"folds {folds} of {where}" if where else f"folds {folds}"
+
+
+def fold_series(summary, group, every=False):
+    names = sorted(group.get("mean") or {}, key=series_key)
+    if every:
+        return names
+    return [name for name in names if name == summary.by or not name.startswith(HIDDEN)]
+
+
+def folds_table(summary):
+    params = []
+    for group in summary.folds or []:
+        for key in group.get("group") or {}:
+            if key not in params:
+                params.append(key)
+    rows = []
+    for group in summary.folds or []:
+        given = group.get("group") or {}
+        for name in fold_series(summary, group, every=True):
+            rows.append([*(given.get(key) for key in params), len(group.get("members") or []), name,
+                         group["mean"][name], group["std"].get(name)])
+    return pandas.DataFrame(rows, columns=[*(column_of(key) for key in params), "records", "series", "mean", "std"])
+
+
+def folds_text(summary):
+    lines = []
+    for group in summary.folds or []:
+        names = fold_series(summary, group)
+        width = max([len("series"), *(len(name) for name in names)])
+        lines.extend([f"{fold_label(group)}: mean and std over {len(group.get('members') or [])} records",
+                      f"  {'series'.ljust(width)}  {'mean'.ljust(10)}  std"])
+        lines.extend(f"  {name.ljust(width)}  {cell(group['mean'][name]).ljust(10)}  {cell(group['std'].get(name))}"
+                     for name in names)
+    return "\n".join(lines)
+
+
+def escape(text):
+    return text.replace("|", "\\|")
+
+
+def markdown_of(summary, top=None, columns="all"):
+    columns, rows = rows_of(summary, top, columns)
+    lines = ["| " + " | ".join(escape(column) for column in columns) + " |", "|" + "---|" * len(columns)]
+    for row in rows:
+        lines.append("| " + " | ".join(escape(value) for value in row) + " |")
+    return "\n".join(lines)
+
+
+def write(summary, out):
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    csv = out / "summary.csv"
+    markdown = out / "summary.md"
+    summary.table.to_csv(csv, index=False, float_format="%.12g")
+    markdown.write_text(markdown_of(summary) + "\n", encoding="utf-8")
+    if not summary.folds:
+        return str(csv), str(markdown)
+    folds = out / "folds.csv"
+    folds_table(summary).to_csv(folds, index=False, float_format="%.12g")
+    return str(csv), str(markdown), str(folds)

kalfa/tui/__init__.py

@@ -0,0 +1,4 @@
+def open_tui(root=".", config=None):
+    from .app import KalfaApp
+
+    KalfaApp(root=root, config=config).run()

kalfa/tui/app.py

@@ -0,0 +1,107 @@
+import threading
+
+from kalfa.core.errors import KalfaError
+
+
+try:
+    from textual.app import App
+    from textual.widgets import Footer, Header, TabbedContent, TabPane
+
+    from kalfa.tui.config import ConfigView
+    from kalfa.tui.guide import GuideView
+    from kalfa.tui.jobs import JobOutput, OutputLog, follow, spawn
+    from kalfa.tui.legos import LegosView
+    from kalfa.tui.records import RecordsView
+except ImportError as missing:
+    App = None
+    TEXTUAL_ERROR = missing
+else:
+    TEXTUAL_ERROR = None
+
+
+PRIMARY = {"records": "#records-table", "legos": "#legos-tree", "config": "#config-output", "guide": "#guide-sections"}
+STYLE = """
+#tabs, #tabs > ContentSwitcher, TabPane, #records-view, #legos-view, #config-view, #legos-tree { height: 1fr; }
+#records-main { width: 3fr; height: 1fr; }
+#records-table { height: 1fr; }
+#records-order { height: 1; padding: 0 1; color: $text-muted; }
+#records-side, #legos-side { width: 2fr; }
+#legos-scroll { width: 3fr; }
+#legos-tree { overflow-x: hidden; }
+#records-info { height: 2fr; }
+#record-detail, #record-sparks, #legos-page, .field { height: auto; padding: 0 1; }
+.spark { height: 2; margin-bottom: 1; }
+#records-log, #config-output { height: 1fr; border-top: solid $primary; }
+.label { width: 10; padding: 1 1 0 1; }
+"""
+
+
+def require_textual():
+    if App is None:
+        raise KalfaError(f"the terminal interface needs the textual package (uv sync installs the dependencies of "
+                         f"kalfa): {TEXTUAL_ERROR}")
+
+
+class KalfaApp(App if App is not None else object):
+    TITLE = "kalfa"
+    CSS = STYLE
+    AUTO_FOCUS = "#records-table"
+    BINDINGS = [("1", "tab('records')", "records"), ("2", "tab('legos')", "legos"), ("3", "tab('config')", "config"),
+                ("4", "tab('guide')", "guide"), ("escape", "tab", "leave field"), ("q", "quit", "quit")]
+
+    def __init__(self, root=".", config=None):
+        require_textual()
+        super().__init__()
+        self.root = str(root)
+        self.config = config
+        self.board = None
+        self.closing = threading.Event()
+        self.sub_title = self.root
+
+    def compose(self):
+        yield Header()
+        with TabbedContent(initial="records", id="tabs"):
+            with TabPane("1 records", id="records"):
+                yield RecordsView(id="records-view")
+            with TabPane("2 legos", id="legos"):
+                yield LegosView(id="legos-view")
+            with TabPane("3 config", id="config"):
+                yield ConfigView(id="config-view")
+            with TabPane("4 guide", id="guide"):
+                yield GuideView(id="guide-view")
+        yield Footer()
+
+    def action_tab(self, name=None):
+        tabs = self.query_one("#tabs", TabbedContent)
+        tabs.active = name = name or tabs.active or "records"
+        self.query_one(PRIMARY[name]).focus()
+
+    def on_tabbed_content_tab_activated(self, event):
+        focused, pane = self.focused, event.pane
+        if pane is not None and pane.id in PRIMARY and (focused is None or pane not in focused.ancestors_with_self):
+            self.query_one(PRIMARY[pane.id]).focus()
+
+    def launch(self, argv, label, target, detach=True):
+        output = self.query_one(target, OutputLog)
+        try:
+            process, path = spawn(argv, label, detach=detach)
+        except OSError as exc:
+            output.say(f"{label}: could not start: {exc}")
+            return None
+        output.say(f"{label}: pid {process.pid}, output in {path}")
+
+        def emit(text):
+            try:
+                return output.post_message(JobOutput(text))
+            except RuntimeError:
+                return False
+
+        threading.Thread(target=follow, args=(path, process, emit, self.closing), kwargs={"label": label},
+                         daemon=True).start()
+        return process
+
+    def on_unmount(self):
+        self.closing.set()
+        if self.board is not None and self.board.poll() is None:
+            self.board.terminate()
+            self.board.wait(timeout=5)

kalfa/tui/config.py

@@ -0,0 +1,61 @@
+import json
+
+from textual.containers import Horizontal, Vertical
+from textual.widgets import Input, Static
+
+from kalfa.cli import record_configs
+from kalfa.core.check import check
+from kalfa.core.errors import KalfaError
+from kalfa.tui.jobs import OutputLog, cli_argv, pairs_of, words_of
+
+
+FIELDS = (("config", "config-path", "YAML files, later ones override, or a record"),
+          ("--set", "config-sets", "PATH=VALUE pairs separated by spaces"),
+          ("-p", "config-params", "NAME=VALUE pairs separated by spaces"))
+
+
+class ConfigView(Vertical):
+    BINDINGS = [("c", "checked(False)", "check"), ("p", "checked(True)", "plan"), ("x", "start('run')", "run"),
+                ("w", "start('sweep')", "sweep")]
+
+    def compose(self):
+        config = str(self.app.config or "")
+        for label, name, hint in FIELDS:
+            with Horizontal(classes="field"):
+                yield Static(label, classes="label", markup=False)
+                yield Input(value=config if name == "config-path" else "", placeholder=hint, id=name)
+        yield OutputLog(id="config-output")
+
+    def say(self, text):
+        self.query_one("#config-output", OutputLog).say(text)
+
+    def fields(self):
+        paths = words_of(self.query_one("#config-path", Input).value)
+        if not paths:
+            raise KalfaError("write a config path first")
+        sets = pairs_of(self.query_one("#config-sets", Input).value, "--set")
+        return paths, sets, pairs_of(self.query_one("#config-params", Input).value, "-p")
+
+    def action_checked(self, plan=False):
+        try:
+            paths, sets, params = self.fields()
+            self.say(f"check {' '.join(paths)}")
+            found = check(paths=record_configs(paths), sets=sets, params=params)
+        except Exception as exc:
+            self.say(f"error: {exc}" if isinstance(exc, KalfaError) else f"error: {type(exc).__name__}: {exc}")
+            return
+        self.say(found.text())
+        if plan and found.ok:
+            self.say(json.dumps(found.plan, indent=1, default=str))
+
+    def action_start(self, command):
+        try:
+            paths, sets, params = self.fields()
+        except KalfaError as exc:
+            self.say(f"error: {exc}")
+            return
+        self.app.launch(cli_argv(command, *paths, sets=sets, params=params), command, "#config-output")
+
+    def on_input_submitted(self, event):
+        self.query_one("#config-output", OutputLog).focus()
+        self.action_checked()

kalfa/tui/guide.py

@@ -0,0 +1,100 @@
+from textual.containers import Horizontal, VerticalScroll
+from textual.widgets import Input, OptionList, Static
+from textual.widgets.option_list import Option
+
+from kalfa.commands import guide
+from kalfa.core.errors import KalfaError
+from kalfa.tui.jobs import pairs_of, words_of
+
+
+EMPTY = ("the guide explains the config of the config tab, and its path is empty: write a config path there (key 3), "
+         "then come back to this tab or press r here")
+
+
+class GuideView(Horizontal):
+    BINDINGS = [("r", "reload", "reload")]
+    DEFAULT_CSS = """
+    GuideView { height: 1fr; }
+    #guide-sections { width: 24; height: 1fr; }
+    #guide-scroll { width: 1fr; height: 1fr; }
+    #guide-scroll > Static { height: auto; padding: 0 1; }
+    #guide-head, #guide-message { margin-bottom: 1; }
+    """
+
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.sections = {}
+        self.picked = None
+        self.skip = None
+        self.shown = None
+
+    def compose(self):
+        yield OptionList(id="guide-sections", markup=False)
+        yield VerticalScroll(id="guide-scroll")
+
+    async def on_show(self):
+        await self.action_reload()
+
+    def fields(self):
+        value = self.app.query_one
+        paths = words_of(value("#config-path", Input).value)
+        sets = pairs_of(value("#config-sets", Input).value, "--set")
+        return paths, sets, pairs_of(value("#config-params", Input).value, "-p")
+
+    async def action_reload(self):
+        try:
+            paths, sets, params = self.fields()
+        except KalfaError as exc:
+            await self.show_message(f"error: {exc}")
+            return
+        if not paths:
+            await self.show_message(EMPTY)
+            return
+        doc = guide.document(paths, sets, params)
+        await self.show(doc)
+
+    async def fill(self, widgets, names, shown=None):
+        scroll = self.query_one("#guide-scroll", VerticalScroll)
+        await scroll.remove_children()
+        await scroll.mount_all(widgets)
+        listing = self.query_one("#guide-sections", OptionList)
+        listing.clear_options()
+        listing.add_options([Option(name, id=name) for name in names])
+        if self.picked in names:
+            listing.highlighted = names.index(self.picked)
+        elif names:
+            self.skip = names[0]
+            listing.highlighted = 0
+        if self.picked not in names and shown != self.shown:
+            scroll.scroll_home(animate=False)
+        self.shown = shown
+
+    async def show_message(self, text):
+        self.sections = {}
+        await self.fill([Static(text, id="guide-message", markup=False)], [])
+
+    async def show(self, doc):
+        self.sections = {section["name"]: Static(guide.section_renderable(section, doc), classes="guide-section")
+                         for section in doc["sections"]}
+        head = Static(guide.head_renderable(doc), id="guide-head")
+        await self.fill([head, *self.sections.values()], list(self.sections), doc["title"])
+
+    def on_option_list_option_highlighted(self, event):
+        if event.option_list.id != "guide-sections" or event.option_id not in self.sections:
+            return
+        if event.option_id == self.skip:
+            self.skip = None
+            return
+        self.skip = None
+        self.picked = event.option_id
+        self.call_after_refresh(self.scroll_to_section, event.option_id)
+
+    def on_option_list_option_selected(self, event):
+        if event.option_list.id == "guide-sections" and event.option_id in self.sections:
+            self.picked = event.option_id
+            self.call_after_refresh(self.scroll_to_section, event.option_id)
+
+    def scroll_to_section(self, name):
+        widget = self.sections.get(name)
+        if widget is not None and widget.is_attached:
+            self.query_one("#guide-scroll", VerticalScroll).scroll_to_widget(widget, animate=False, top=True)

kalfa/tui/jobs.py

@@ -0,0 +1,97 @@
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+
+from rich.text import Text
+from textual.message import Message
+from textual.widgets import RichLog
+
+from kalfa.core.errors import KalfaError
+
+
+LOG_LINES = 5000
+
+
+def words_of(text):
+    try:
+        return shlex.split(text or "")
+    except ValueError as exc:
+        raise KalfaError(f"cannot split {text!r}: {exc}") from None
+
+
+def pairs_of(text, flag="--set"):
+    words = words_of(text)
+    bad = [word for word in words if "=" not in word or not word.partition("=")[0].strip()]
+    if bad:
+        raise KalfaError(f"{flag} takes NAME=VALUE pairs separated by spaces, got {' '.join(bad)}")
+    return words
+
+
+def cli_argv(command, *arguments, sets=(), params=()):
+    argv = [sys.executable, "-m", "kalfa.cli", command, *(str(argument) for argument in arguments)]
+    argv += [part for item in sets for part in ("--set", item)]
+    return argv + [part for item in params for part in ("--param", item)]
+
+
+def spawn(argv, label, detach=True, directory=None):
+    handle, path = tempfile.mkstemp(prefix=f"kalfa-{label}-", suffix=".log", dir=directory)
+    environment = dict(os.environ, PYTHONUNBUFFERED="1", PYTHONIOENCODING="utf-8")
+    with os.fdopen(handle, "wb") as output:
+        process = subprocess.Popen([str(part) for part in argv], stdout=output, stderr=subprocess.STDOUT,
+                                   stdin=subprocess.DEVNULL, env=environment, start_new_session=detach)
+    return process, path
+
+
+def shown_line(raw):
+    return raw.decode("utf-8", errors="replace").rstrip("\r").rsplit("\r", 1)[-1]
+
+
+def follow(path, process, emit, stop, every=0.2, label="job"):
+    offset, pending = 0, b""
+    while not stop.is_set():
+        finished = process.poll() is not None
+        try:
+            with open(path, "rb") as handle:
+                handle.seek(offset)
+                data = handle.read()
+        except OSError:
+            data = b""
+        offset += len(data)
+        *lines, pending = (pending + data).split(b"\n")
+        for raw in lines:
+            if emit(shown_line(raw)) is False:
+                return None
+        if finished and not data:
+            if pending:
+                emit(shown_line(pending))
+            emit(f"{label} exited with {process.returncode}")
+            return process.returncode
+        if not data:
+            stop.wait(every)
+    return None
+
+
+class JobOutput(Message):
+    def __init__(self, text):
+        super().__init__()
+        self.text = text
+
+
+class OutputLog(RichLog):
+    def __init__(self, **options):
+        super().__init__(max_lines=LOG_LINES, wrap=True, markup=False, highlight=False, **options)
+        self.plain = []
+
+    def say(self, text):
+        text = str(text)
+        self.plain.extend(text.splitlines() or [""])
+        del self.plain[:-LOG_LINES]
+        self.write(text)
+
+    def on_job_output(self, message):
+        self.say(message.text)
+
+    def render(self):
+        return Text("\n".join(self.plain))