loopcaptain 0.3.1__py3-none-any.whl

loopcaptain/board/discover.py

@@ -0,0 +1,249 @@
+"""Which loops the board shows: the repositories named in the config, read fresh on every request."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+import yaml
+
+from loopcaptain.protocol import store
+from loopcaptain.protocol.models import FOLDER, Manifest, now
+from loopcaptain.protocol.signing import untrusted_ids
+from loopcaptain.protocol.state import LoopState, fold
+from loopcaptain.protocol.store import Problem, StoreError
+
+#: Where the config is looked for, first hit wins. The file names this laptop's folders, so it is
+#: never committed (`*.local.yaml` is ignored).
+CONFIG_ENV = "LOOPCAPTAIN_CONFIG"
+CONFIG_NAMES = ("loopcaptain.local.yaml",)
+USER_CONFIG = Path("~/.config/loopcaptain/config.yaml")
+GIT_TIMEOUT_S = 3
+DEMO_FOLDER = "demo"
+
+
+@dataclass(frozen=True)
+class Config:
+    """`roots`: folders whose direct sub-folders are repositories. `repos`: single repositories."""
+
+    roots: tuple[Path, ...] = ()
+    repos: tuple[Path, ...] = ()
+    source: Path | None = None
+
+
+@dataclass(frozen=True)
+class Repo:
+    key: str  # unique on the board, the folder name unless two repositories share it
+    path: Path  # of worktrees of one repository, the one whose `.loops` was written last
+    others: tuple[Path, ...] = ()  # the other worktrees of the same repository that have `.loops`
+
+
+def git_common_dir(path: Path) -> Path:
+    """The `.git` folder a checkout shares with its worktrees, read from disk without running git.
+
+    A worktree's `.git` is a file, `gitdir: <common>/worktrees/<name>`; a main checkout's is the folder
+    itself. Anything else (no git, an unreadable file) stands for itself.
+    """
+    dot_git = path / ".git"
+    if dot_git.is_dir():
+        return dot_git.resolve()
+    try:
+        first = dot_git.read_text(encoding="utf-8").splitlines()[0]
+    except (OSError, IndexError, UnicodeDecodeError):
+        return path.resolve()
+    if not first.startswith("gitdir:"):
+        return path.resolve()
+    gitdir = (path / first.removeprefix("gitdir:").strip()).resolve()
+    return gitdir.parent.parent if gitdir.parent.name == "worktrees" else gitdir
+
+
+def last_written(path: Path) -> int:
+    """When anything in a repository's `.loops` was last written, in nanoseconds; 0 if never."""
+    folder = path / FOLDER
+    stamps = [0]
+    for loop_id in store.loop_ids(path):
+        for name in ("loop.json", "events.jsonl", "inbox.jsonl"):
+            try:
+                stamps.append((folder / loop_id / name).lstat().st_mtime_ns)
+            except OSError:
+                continue
+    return max(stamps)
+
+
+def one_per_repository(paths: list[Path]) -> list[tuple[Path, tuple[Path, ...]]]:
+    """Worktrees of one repository become one entry: the last written, and the others beside it."""
+    groups: dict[Path, list[Path]] = {}
+    for path in paths:
+        groups.setdefault(git_common_dir(path), []).append(path)
+    grouped = []
+    for members in groups.values():
+        ordered = sorted(members, key=last_written, reverse=True)
+        grouped.append((ordered[0], tuple(ordered[1:])))
+    return grouped
+
+
+@dataclass(frozen=True)
+class LoopView:
+    """One loop as the board shows it: where it lives, where it stands, and what could not be read."""
+
+    repo: Repo
+    loop_id: str
+    state: LoopState | None
+    problems: tuple[Problem, ...] = ()
+    error: str = ""
+
+    @property
+    def title(self) -> str:
+        return (self.state.manifest.title if self.state else "") or self.loop_id
+
+
+def load_config(cwd: Path | None = None) -> Config:
+    base = cwd or Path.cwd()
+    candidates = [Path(os.environ[CONFIG_ENV])] if os.environ.get(CONFIG_ENV) else []
+    candidates += [base / name for name in CONFIG_NAMES]
+    candidates.append(USER_CONFIG.expanduser())
+    for path in candidates:
+        if path.is_file():
+            return parse_config(path)
+    # No config yet: show the example loop if `make demo` has written it, so a fresh clone shows something.
+    demo = base / DEMO_FOLDER
+    return Config(repos=(demo.resolve(),) if demo.is_dir() else ())
+
+
+def parse_config(path: Path) -> Config:
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except (OSError, yaml.YAMLError) as error:
+        raise StoreError(f"{path.name}: {error}") from error
+    if not isinstance(data, dict):
+        raise StoreError(f"{path.name}: expected `roots:` and `repos:` lists")
+
+    def paths(key: str) -> tuple[Path, ...]:
+        value = data.get(key) or []
+        if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+            raise StoreError(f"{path.name}: `{key}` must be a list of paths")
+        return tuple((path.parent / Path(item).expanduser()).resolve() for item in value)
+
+    return Config(roots=paths("roots"), repos=paths("repos"), source=path)
+
+
+def find_repos(config: Config) -> list[Repo]:
+    """Every configured repository that has a `.loops` folder, each once, in a stable order."""
+    found: list[Path] = []
+    for root in config.roots:
+        if not root.is_dir():
+            continue
+        found.extend(child for child in sorted(root.iterdir()) if (child / FOLDER).is_dir())
+    found.extend(repo for repo in config.repos if (repo / FOLDER).is_dir())
+    grouped = one_per_repository(list(dict.fromkeys(found)))
+    names = [path.name for path, _ in grouped]
+    repos = []
+    for path, others in grouped:
+        key = path.name if names.count(path.name) == 1 else f"{path.parent.name}.{path.name}"
+        repos.append(Repo(key=safe_key(key), path=path, others=others))
+    return repos
+
+
+def safe_key(name: str) -> str:
+    """A repository key is a URL part: anything else in a folder name becomes a dash."""
+    return "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in name) or "repo"
+
+
+def load_loop(repo: Repo, loop_id: str, at: datetime | None = None, process_alive: bool = False) -> LoopView:
+    try:
+        manifest = store.read_manifest(repo.path, loop_id)
+        events, event_problems = store.read_events(repo.path, loop_id)
+        inbox, inbox_problems = store.read_inbox(repo.path, loop_id)
+    except StoreError as error:
+        return LoopView(repo, loop_id, None, error=str(error))
+    state = fold(manifest, events, inbox, at or now(), process_alive, untrusted_ids(inbox, loop_id))
+    return LoopView(repo, loop_id, state, (*event_problems, *inbox_problems))
+
+
+def load_all(config: Config) -> list[LoopView]:
+    return [load_loop(repo, loop_id) for repo in find_repos(config) for loop_id in store.loop_ids(repo.path)]
+
+
+def find_repo(config: Config, repo_key: str, loop_id: str) -> Repo | None:
+    """The repository behind a URL if it lists this loop, readable or not."""
+    return next(
+        (
+            repo
+            for repo in find_repos(config)
+            if repo.key == repo_key and loop_id in store.loop_ids(repo.path)
+        ),
+        None,
+    )
+
+
+def find_loop(config: Config, repo_key: str, loop_id: str) -> tuple[Repo, Manifest] | None:
+    """The repository and manifest behind a URL, or None. Only configured repositories are reachable."""
+    for repo in find_repos(config):
+        if repo.key == repo_key and loop_id in store.loop_ids(repo.path):
+            try:
+                return repo, store.read_manifest(repo.path, loop_id)
+            except StoreError:
+                return None
+    return None
+
+
+def git_branch(repo: Path) -> str:
+    """The checked-out branch, or "" where git cannot say (no repository, detached, too slow)."""
+    try:
+        done = subprocess.run(
+            ["git", "-C", str(repo), "rev-parse", "--abbrev-ref", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=GIT_TIMEOUT_S,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return ""
+    branch = done.stdout.strip()
+    return branch if done.returncode == 0 and branch != "HEAD" else ""
+
+
+def config_target() -> Path:
+    """The config `watch` and `unwatch` edit: the one `LOOPCAPTAIN_CONFIG` names, else the user's."""
+    named = os.environ.get(CONFIG_ENV)
+    return Path(named).expanduser() if named else USER_CONFIG.expanduser()
+
+
+def as_written(path: Path) -> str:
+    """A path as it goes into the config: absolute, the home folder as `~`."""
+    resolved = path.expanduser().resolve()
+    try:
+        return f"~/{resolved.relative_to(Path.home())}"
+    except ValueError:
+        return str(resolved)
+
+
+def edit_config(path: Path, *, add: tuple[str, str] | None = None, remove: str | None = None) -> list[str]:
+    """Add `(key, path)` to the config or remove a path from both lists; returns what changed, in words."""
+    target = config_target()
+    data: dict[str, list[str]] = {"roots": [], "repos": []}
+    if target.is_file():
+        loaded = yaml.safe_load(target.read_text(encoding="utf-8")) or {}
+        if not isinstance(loaded, dict):
+            raise StoreError(f"{target}: expected `roots:` and `repos:` lists")
+        data.update({key: list(loaded.get(key) or []) for key in ("roots", "repos")})
+    changes: list[str] = []
+    if add is not None:
+        key, entry = add
+        if entry in data[key]:
+            return [f"{entry} is already under {key}"]
+        data[key].append(entry)
+        changes.append(f"added {entry} to {key}")
+    if remove is not None:
+        for key in ("roots", "repos"):
+            if remove in data[key]:
+                data[key].remove(remove)
+                changes.append(f"removed {remove} from {key}")
+        if not changes:
+            return [f"{remove} is not in {target}"]
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
+    return [f"{target}: {change}" for change in changes]

loopcaptain/board/files.py

@@ -0,0 +1,94 @@
+"""Which files of a watched repository the board opens, and which it never shows."""
+
+from __future__ import annotations
+
+import fnmatch
+from pathlib import Path
+
+from fastapi import HTTPException
+
+#: The largest file the board renders. A round's plan or review is a few kilobytes.
+MAX_FILE_BYTES = 400_000
+
+
+#: Never shown, though inside the repository: git's own folder and files that hold secrets.
+PRIVATE_PARTS = frozenset(
+    {".git", ".ssh", ".aws", ".gnupg", ".netrc", ".npmrc", ".pypirc", ".git-credentials", ".docker", ".kube"}
+)
+
+
+#: Name patterns of files that hold keys or credentials, matched against every part of the path.
+PRIVATE_PATTERNS = (
+    ".env*",
+    "*.pem",
+    "*.key",
+    "*.p12",
+    "*.pfx",
+    "*.keystore",
+    "*.jks",
+    "*.tfstate",
+    "*.tfstate.*",
+    ".htpasswd",
+    "id_rsa*",
+    "id_ed25519*",
+    "id_ecdsa*",
+    "credentials*.json",
+    "service-account*.json",
+    "*-sa-key*.json",
+    "secrets.*",
+)
+
+
+#: The board shows a file's content only for these; anything else is named, not opened.
+TEXT_SUFFIXES = frozenset(
+    {
+        ".md",
+        ".markdown",
+        ".txt",
+        ".rst",
+        ".json",
+        ".jsonl",
+        ".yaml",
+        ".yml",
+        ".toml",
+        ".csv",
+        ".tsv",
+        ".log",
+        ".py",
+        ".sh",
+        ".js",
+        ".ts",
+        ".html",
+        ".css",
+        ".sql",
+        ".r",
+        ".tex",
+        ".bib",
+        ".diff",
+        ".patch",
+    }
+)
+
+
+def is_private(part: str) -> bool:
+    """Whether one part of a path names git's folder or a file that holds keys or credentials."""
+    lowered = part.lower()
+    return lowered in PRIVATE_PARTS or any(
+        fnmatch.fnmatchcase(lowered, pattern) for pattern in PRIVATE_PATTERNS
+    )
+
+
+def inside(repo: Path, relative: str) -> Path:
+    """A path inside the repository, or 404: no link on the board may lead out of it."""
+    root = repo.resolve()
+    try:
+        target = (root / relative).resolve()
+        found = bool(relative) and target.is_relative_to(root) and target.is_file()
+    except (ValueError, OSError):  # a null byte, a name too long
+        found = False
+    if not found:
+        raise HTTPException(404, "no such file in this repository")
+    parts = target.relative_to(root).parts
+    if any(is_private(part) for part in parts):
+        raise HTTPException(404, "no such file in this repository")
+    return target

loopcaptain/board/filters.py

@@ -0,0 +1,48 @@
+"""How numbers, times and places read on a page: the template filters."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+from loopcaptain.protocol.models import now
+
+
+def ago(value: datetime | None, at: datetime | None = None) -> str:
+    if value is None:
+        return ""
+    seconds = int(((at or now()) - value).total_seconds())
+    if seconds < 60:
+        return "just now"
+    for size, unit in ((86400, "d"), (3600, "h"), (60, "min")):
+        if seconds >= size:
+            return f"{seconds // size} {unit} ago"
+    return ""
+
+
+def usd(value: float) -> str:
+    return f"${value:,.2f}" if value else ""
+
+
+def minutes(value: int | None) -> str:
+    if not value:
+        return ""
+    return f"{value} min" if value < 120 else f"{value / 60:.1f} h"
+
+
+def tokens(value: int) -> str:
+    if not value:
+        return ""
+    return f"{value / 1_000_000:.1f}M" if value >= 1_000_000 else f"{value / 1000:.0f}k"
+
+
+def stamp(value: datetime | None) -> str:
+    return value.astimezone().strftime("%Y-%m-%d %H:%M") if value else ""
+
+
+def place(path: Path) -> str:
+    """Where a repository is, as the person would type it: the home folder as `~`."""
+    try:
+        return f"~/{path.resolve().relative_to(Path.home())}"
+    except ValueError:
+        return str(path)

loopcaptain/board/graph.py

@@ -0,0 +1,139 @@
+"""The picture of one round: the manifest's steps laid out left to right, as numbers a template draws.
+
+No drawing library: each step gets a column (one further right than the last step it waits for)
+and a row within it, and every edge is one curve.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from loopcaptain.protocol.models import Manifest
+from loopcaptain.protocol.state import RoundState
+
+NODE_W = 148
+NODE_H = 46
+GAP_X = 44
+GAP_Y = 14
+PAD = 14
+#: Room under the nodes for the arrow that leads from the last step back to the first.
+RETURN_H = 34
+
+
+@dataclass(frozen=True)
+class Node:
+    id: str
+    name: str
+    kind: str
+    x: int
+    y: int
+    #: pending, running, ok, failed, skipped: the step in the round the picture shows.
+    state: str = "pending"
+    runs: int = 0
+    findings: int = 0
+
+    @property
+    def cx(self) -> int:
+        return self.x + NODE_W // 2
+
+    @property
+    def cy(self) -> int:
+        return self.y + NODE_H // 2
+
+
+@dataclass(frozen=True)
+class Graph:
+    nodes: tuple[Node, ...]
+    edges: tuple[str, ...]  # SVG path data
+    back: str  # the path from the last column back to the first, "" for a single column
+    width: int
+    height: int
+    node_w: int = NODE_W
+    node_h: int = NODE_H
+
+
+def columns(manifest: Manifest) -> dict[str, int]:
+    """Each step's column: 0 for a step that waits for nothing, else one past its latest predecessor.
+
+    In waves, without recursion: a loop.json with thousands of chained steps must not exhaust the
+    stack. A step left over after as many waves as there are steps sits in a cycle and goes to 0.
+    """
+    before: dict[str, list[str]] = {step.id: [] for step in manifest.steps}
+    for source, target in manifest.edges():
+        before[target].append(source)
+    column: dict[str, int] = {}
+    for _ in range(len(before)):
+        ready = [
+            step_id
+            for step_id, sources in before.items()
+            if step_id not in column and all(name in column for name in sources)
+        ]
+        if not ready:
+            break
+        for step_id in ready:
+            column[step_id] = 1 + max((column[name] for name in before[step_id]), default=-1)
+    return {step_id: column.get(step_id, 0) for step_id in before}
+
+
+def step_state(round_: RoundState | None, step_id: str) -> tuple[str, int, int]:
+    """How a step stands in a round: its state, how many runs it had, how many open material findings."""
+    runs = round_.runs_of(step_id) if round_ else ()
+    if not runs:
+        return "pending", 0, 0
+    findings = sum(run.material_open for run in runs)
+    statuses = {run.status for run in runs}
+    for state in ("running", "failed", "ok"):
+        if state in statuses:
+            return state, len(runs), findings
+    return "skipped", len(runs), findings
+
+
+def layout(manifest: Manifest, round_: RoundState | None = None) -> Graph:
+    column = columns(manifest)
+    by_column: dict[int, list[str]] = {}
+    for step in manifest.steps:
+        by_column.setdefault(column[step.id], []).append(step.id)
+    tallest = max((len(ids) for ids in by_column.values()), default=1)
+    inner_h = tallest * NODE_H + (tallest - 1) * GAP_Y
+
+    nodes: dict[str, Node] = {}
+    for col, ids in by_column.items():
+        block = len(ids) * NODE_H + (len(ids) - 1) * GAP_Y
+        top = PAD + (inner_h - block) // 2
+        for row, step_id in enumerate(ids):
+            declared = manifest.step(step_id)
+            state, runs, findings = step_state(round_, step_id)
+            nodes[step_id] = Node(
+                id=step_id,
+                name=(declared.name if declared else "") or step_id,
+                kind=declared.kind if declared else "other",
+                x=PAD + col * (NODE_W + GAP_X),
+                y=top + row * (NODE_H + GAP_Y),
+                state=state,
+                runs=runs,
+                findings=findings,
+            )
+
+    edges = tuple(curve(nodes[source], nodes[target]) for source, target in manifest.edges())
+    last_col = max(by_column, default=0)
+    width = 2 * PAD + (last_col + 1) * NODE_W + last_col * GAP_X
+    height = 2 * PAD + inner_h
+    back = ""
+    if last_col > 0:
+        floor = height + RETURN_H - 12
+        start_x = PAD + last_col * (NODE_W + GAP_X) + NODE_W // 2
+        end_x = PAD + NODE_W // 2
+        low = PAD + inner_h
+        back = (
+            f"M {start_x} {low + 4} V {floor - 10} Q {start_x} {floor} {start_x - 10} {floor} "
+            f"H {end_x + 10} Q {end_x} {floor} {end_x} {floor - 10} V {low + 8}"
+        )
+        height += RETURN_H
+    return Graph(tuple(nodes.values()), edges, back, width, height)
+
+
+def curve(source: Node, target: Node) -> str:
+    x1, y1 = source.x + NODE_W, source.cy
+    x2, y2 = target.x, target.cy
+    mid = (x1 + x2) // 2
+    return f"M {x1} {y1} C {mid} {y1}, {mid} {y2}, {x2 - 5} {y2}"

loopcaptain/board/overview.py

@@ -0,0 +1,59 @@
+"""The first page: loops grouped by repository, and the filters over them."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from loopcaptain.board.discover import LoopView, Repo, git_branch
+from loopcaptain.board.filters import place
+
+
+@dataclass(frozen=True)
+class Project:
+    """One repository on the first page, with the loops that live in it."""
+
+    repo: Repo
+    place: str
+    branch: str
+    views: tuple[LoopView, ...]
+
+    @property
+    def waiting(self) -> int:
+        return sum(1 for view in self.views if view.state and view.state.question)
+
+
+def projects_of(views: list[LoopView]) -> list[Project]:
+    """The loops grouped by repository, a repository where its first loop stands in `views`."""
+    grouped: dict[str, list[LoopView]] = {}
+    for view in views:
+        grouped.setdefault(view.repo.key, []).append(view)
+    return [
+        Project(found[0].repo, place(found[0].repo.path), git_branch(found[0].repo.path), tuple(found))
+        for found in grouped.values()
+    ]
+
+
+#: Statuses that are over; the overview folds them away unless asked.
+FINISHED = frozenset({"done", "stopped"})
+
+
+def filter_views(
+    views: list[LoopView], text: str, status: str, *, show_finished: bool
+) -> tuple[list[LoopView], int]:
+    """The loops the table shows, and how many finished ones were folded away."""
+    needle = text.strip().lower()
+
+    def status_of(view: LoopView) -> str:
+        return view.state.status if view.state else "failed"
+
+    def matches(view: LoopView) -> bool:
+        haystack = f"{view.title} {view.loop_id} {view.repo.key}".lower()
+        if view.state:
+            haystack += " " + view.state.manifest.brief.goal.lower()
+        return (not needle or needle in haystack) and (not status or status_of(view) == status)
+
+    picked = [view for view in views if matches(view)]
+    if show_finished or status in FINISHED:
+        return picked, 0
+    kept = [view for view in picked if status_of(view) not in FINISHED]
+    return kept, len(picked) - len(kept)

loopcaptain/board/static/board.js

@@ -0,0 +1,41 @@
+// Two small things, no library: reload the page when a loop's files change, and copy the protocol.
+(function () {
+  "use strict";
+  var body = document.body;
+  var url = body.getAttribute("data-pulse-url");
+  var seen = body.getAttribute("data-pulse");
+  var POLL_MS = 4000;
+
+  function busy() {
+    // Never reload under someone who is typing an answer or reading a check's history.
+    var typed = Array.prototype.some.call(document.querySelectorAll("textarea[name]"), function (field) {
+      return field.value.trim() !== "" || field === document.activeElement;
+    });
+    return typed || document.querySelector("table.items details[open]") !== null;
+  }
+
+  function poll() {
+    if (document.hidden) return;
+    fetch(url, { cache: "no-store" })
+      .then(function (response) { return response.ok ? response.json() : null; })
+      .then(function (data) {
+        if (data && data.pulse !== seen && !busy()) window.location.reload();
+      })
+      .catch(function () { /* the board is restarting; the next poll tries again */ });
+  }
+
+  if (url) window.setInterval(poll, POLL_MS);
+
+  // Every copy button copies its own source and says so beside itself.
+  if (navigator.clipboard) {
+    document.querySelectorAll("[data-copy-from]").forEach(function (button) {
+      button.addEventListener("click", function () {
+        var source = document.getElementById(button.getAttribute("data-copy-from"));
+        navigator.clipboard.writeText(source.value).then(function () {
+          var done = button.parentNode.querySelector("[data-copied]");
+          if (done) done.hidden = false;
+        });
+      });
+    });
+  }
+})();