pi-background 0.2.2 → 0.3.1

package/README.md

@@ -1,6 +1,6 @@
 # pi-background
 
-Lets [pi](https://pi.dev) run long commands in the background instead of waiting on them. pi starts the job, carries on or hands back to you, and gets a message when the job finishes or prints something it's waiting for.
+Lets [Pi](https://pi.dev) run long commands in the background instead of waiting on them. Pi starts the job, carries on or hands back to you, and gets a message when the job finishes or prints something it's waiting for.
 
 <picture>
   <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Pringled/pi-extensions/main/assets/images/pi-background-dark.gif">
@@ -13,12 +13,14 @@ pi install npm:pi-background
 
 ## Features
 
-- Long commands like downloads, builds, test suites and training runs go to the background, and pi hears back with the exit code and the end of the output when they finish.
+- Long commands like downloads, builds, test suites and training runs go to the background, and Pi hears back with the exit code and the end of the output when they finish.
 - While a job runs, a line above the input shows it with its elapsed time and latest output, such as `⏵ bg1 python3 train.py · 4m12s · epoch 3/10 loss 0.41`.
-- pi can wait for a specific line, such as a dev server's `listening on` or a training run's `epoch 3`, without checking again and again.
-- Regular commands time out after 2 minutes, unless pi asks for longer (up to 10), so pi never hangs on a stuck command. Commands that start their own background process with `&` are left alone.
+- `/bg` shows a job's last 20 lines of output, updating while it runs (`/bg bg2` for a specific job). Press esc to close it.
+- Pi can wait for a specific line, such as a dev server's `listening on` or a training run's `epoch 3`, without checking again and again.
+- Jobs started together are reported together: when Pi starts several jobs at once, such as benchmarks to compare, it's woken once when the last one finishes instead of after each. You can keep chatting meanwhile, and a job that runs much longer doesn't hold the others back for more than 2 minutes.
+- Regular commands time out after 2 minutes, unless Pi asks for longer (up to 10), so Pi never hangs on a stuck command. Commands that start their own background process with `&` are left alone.
 
-You don't need to do anything differently: pi decides when a job belongs in the background. You can also ask for it, for example "run the tests in the background and tell me when they're done".
+You don't need to do anything differently: Pi decides when a job belongs in the background. You can also ask for it, for example "run the tests in the background and tell me when they're done".
 
 ## Settings
 
@@ -32,7 +34,7 @@ The running jobs line has three levels. Set one in `~/.pi/agent/settings.json`,
 
 ## Tools
 
-pi gets three tools for this: `bg_run` starts a job, `bg_status` shows how it's doing and its latest output, and `bg_kill` stops it. Jobs run with the same environment and sandbox as pi's own shell commands, and stop when you close pi.
+Pi gets three tools for this: `bg_run` starts a job, `bg_status` shows how it's doing and its latest output, and `bg_kill` stops it. Jobs run with the same environment and sandbox as Pi's own shell commands, and stop when you close Pi.
 
 ## License
 

package/extensions/background.ts

@@ -1,8 +1,8 @@
 import { createWriteStream, mkdirSync, readFileSync } from "node:fs";
-import { tmpdir } from "node:os";
+import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { createLocalBashOperations, type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent";
-import { Text, truncateToWidth } from "@earendil-works/pi-tui";
+import { Key, matchesKey, stripTerminalSequences, Text, truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
 import { Type } from "typebox";
 
 const TAIL_LINES = 20;
@@ -12,6 +12,8 @@ const BASH_MAX_TIMEOUT = 600;
 const WAKE_DEBOUNCE_MS = 1000;
 const MAX_WAKES = 10;
 const MAX_SHOWN_JOBS = 3;
+// How long finished results wait for jobs started alongside them before Pi is woken anyway
+const GROUP_HOLD_MS = 2 * 60_000;
 
 interface Job {
 	id: string;
@@ -24,6 +26,10 @@ interface Job {
 	ended?: number;
 	// The latest output line, shown while the job runs
 	lastLine?: string;
+	// The turn that started the job; jobs from the same turn are reported together
+	group: number;
+	// Started with wake_on, so it may run indefinitely and doesn't hold back the others
+	watched: boolean;
 }
 
 function tail(path: string, lines: number): string {
@@ -40,17 +46,27 @@ function elapsed(job: Job): string {
 	return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m${s % 60}s`;
 }
 
+function state(job: Job): string {
+	return job.status === "exited" ? `exit ${job.exitCode}` : job.status;
+}
+
 function summary(job: Job): string {
-	const state = job.status === "exited" ? `exit ${job.exitCode}` : job.status;
-	return `${job.id} [${state}, ${elapsed(job)}] ${job.command}\n  log: ${job.log}`;
+	return `${job.id} [${state(job)}, ${elapsed(job)}] ${job.command}\n  log: ${job.log}`;
 }
 
-/** The part of a command worth showing: its first line, without a leading cd and with a script path shortened to its name. */
+/** An output line as a terminal would last have shown it: without escape codes or control characters, and a progress bar redrawn with \r in its latest state only. */
+function cleanLine(line: string): string {
+	const shown = stripTerminalSequences(line).split("\r").filter((part) => part.trim()).at(-1) ?? "";
+	return shown.replace(/\t/g, "  ").replace(/[\x00-\x08\x0b-\x1f\x7f]/g, "").trimEnd();
+}
+
+/** The part of a command worth showing: its first line, without a leading cd, with a script path shortened to its name and at most 50 characters. */
 function shownCommand(command: string): string {
-	return command
+	const shown = command
 		.split("\n")[0]
 		.replace(/^\s*(cd\s+("[^"]*"|'[^']*'|\S+)\s*(&&|;)\s*)+/, "")
 		.replace(/^\S*\/(?=[^\s/]+(\s|$))/, "");
+	return shown.length > 50 ? `${shown.slice(0, 49)}…` : shown;
 }
 
 /** True when a command starts a job with a bare & (not &&, 2>&1, &> or |&). */
@@ -79,8 +95,40 @@ export default function (pi: ExtensionAPI) {
 	const ops = createLocalBashOperations();
 	let next = 1;
 
-	const wake = (content: string, details: { headline: string; last: string; error: boolean }) =>
-		pi.sendMessage({ customType: "background-job", content, display: true, details }, { triggerTurn: true, deliverAs: "followUp" });
+	const wake = (content: string, details: { headline: string; last: string; error: boolean }, triggerTurn = true) =>
+		pi.sendMessage({ customType: "background-job", content, display: true, details }, { triggerTurn, deliverAs: "followUp" });
+
+	// Jobs started in the same turn usually belong together, like several benchmarks to compare. While some of them
+	// still run, a successful one's result is added to the conversation without waking Pi, so it answers once at the
+	// end instead of after each job. A job that runs on too long doesn't hold the others back for more than GROUP_HOLD_MS.
+	let turn = 0;
+	pi.on("turn_start", async () => {
+		turn++;
+	});
+	const holds = new Map<number, NodeJS.Timeout>();
+	const runningInGroup = (job: Job) =>
+		[...jobs.values()].filter((other) => other !== job && other.group === job.group && other.status === "running" && !other.watched);
+	const report = (job: Job, content: string, details: { headline: string; last: string; error: boolean }) => {
+		const others = job.watched ? [] : runningInGroup(job);
+		// A failure wakes Pi straight away, so it can react before the rest of the group finishes
+		if (others.length === 0 || details.error) {
+			clearTimeout(holds.get(job.group));
+			holds.delete(job.group);
+			return wake(content, details);
+		}
+		wake(`${content}\n(Still running from the same turn: ${others.map((other) => other.id).join(", ")}; you'll be woken when they finish.)`, details, false);
+		if (holds.has(job.group)) return;
+		holds.set(job.group, setTimeout(() => {
+			holds.delete(job.group);
+			const still = runningInGroup(job);
+			if (still.length === 0) return;
+			wake(`Background jobs from the same turn are still running after ${GROUP_HOLD_MS / 60_000} minutes: ${still.map((other) => other.id).join(", ")}. The results of the ones that finished are above; you'll get a message when each of these finishes.`, {
+				headline: `${still.map((other) => other.id).join(", ")} still running`,
+				last: "",
+				error: false,
+			});
+		}, GROUP_HOLD_MS));
+	};
 
 	// Wakes the agent with output lines matching pattern, batched per second and capped per job
 	const watcher = (job: Job, pattern: RegExp) => {
@@ -115,6 +163,8 @@ export default function (pi: ExtensionAPI) {
 	// One line per running job above the input, with its elapsed time and latest output
 	let ui: ExtensionContext["ui"] | undefined;
 	const running = () => [...jobs.values()].filter((job) => job.status === "running");
+	// The job open in /bg, left out of the jobs line meanwhile so it isn't shown twice
+	let viewing: Job | undefined;
 	// "full" (the default), "compact" without the output, or "off", set with "background": { "jobs": ... } in pi's settings.json
 	const jobsLine = () => (pi.getSettings() as { background?: { jobs?: string } }).background?.jobs ?? "full";
 	const showJobs = () => {
@@ -126,12 +176,11 @@ export default function (pi: ExtensionAPI) {
 				dispose: () => clearInterval(timer),
 				invalidate() {},
 				render(width: number) {
-					const live = running();
+					const live = running().filter((job) => job !== viewing);
 					const lines = live.slice(0, MAX_SHOWN_JOBS).map((job) => {
-						const command = shownCommand(job.command);
 						const text =
 							theme.fg("accent", " ⏵ ") +
-							theme.fg("toolTitle", `${job.id} ${command.length > 50 ? `${command.slice(0, 49)}…` : command}`) +
+							theme.fg("toolTitle", `${job.id} ${shownCommand(job.command)}`) +
 							theme.fg("dim", ` · ${elapsed(job)}${jobsLine() === "full" && job.lastLine ? ` · ${job.lastLine}` : ""}`);
 						return truncateToWidth(text, width);
 					});
@@ -209,15 +258,14 @@ export default function (pi: ExtensionAPI) {
 			mkdirSync(dir, { recursive: true });
 			const log = join(dir, `${process.pid}-${id}.log`);
 			const out = createWriteStream(log);
-			const job: Job = { id, command: params.command, log, started: Date.now(), controller: new AbortController(), status: "running" };
+			const job: Job = { id, command: params.command, log, started: Date.now(), controller: new AbortController(), status: "running", group: turn, watched: Boolean(pattern) };
 			jobs.set(id, job);
 
 			const watch = pattern ? watcher(job, pattern) : undefined;
 			const onData = (d: Buffer) => {
 				out.write(d);
 				watch?.(d);
-				// Progress bars redraw with \r, so the last segment of either kind of line break is the current state
-				const last = d.toString("utf8").replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "").split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean).at(-1);
+				const last = d.toString("utf8").split("\n").map(cleanLine).filter((line) => line.trim()).at(-1)?.trim();
 				if (last) job.lastLine = last;
 			};
 
@@ -237,7 +285,7 @@ export default function (pi: ExtensionAPI) {
 					out.end(() => {
 						// A kill from session shutdown has no session left to notify
 						if (job.status === "killed" && !jobs.has(id)) return;
-						wake(`Background job finished: ${summary(job)}\nLast lines:\n${tail(log, TAIL_LINES)}`, {
+						report(job, `Background job finished: ${summary(job)}\nLast lines:\n${tail(log, TAIL_LINES)}`, {
 							headline: `${id} ${job.status === "exited" ? `exited ${job.exitCode}` : job.status} · ${elapsed(job)}`,
 							last: tail(log, 1),
 							error: job.status !== "exited" || job.exitCode !== 0,
@@ -288,7 +336,75 @@ export default function (pi: ExtensionAPI) {
 		},
 	});
 
+	// Shows a job's latest output in place of the input until esc, refreshed every second while it runs
+	pi.registerCommand("bg", {
+		description: "Show a background job's latest output",
+		getArgumentCompletions: (prefix) => {
+			const matching = [...jobs.values()].filter((job) => job.id.startsWith(prefix.trim().toLowerCase()));
+			return matching.length ? matching.map((job) => ({ value: job.id, label: job.id, description: `${shownCommand(job.command)} · ${state(job)}` })) : null;
+		},
+		handler: async (args, ctx) => {
+			if (!ctx.hasUI) return;
+			const all = [...jobs.values()].reverse();
+			if (all.length === 0) return ctx.ui.notify("No background jobs yet", "info");
+			// "/bg 2" works as well as "/bg bg2"
+			const wanted = args.trim().toLowerCase().replace(/^(\d+)$/, "bg$1");
+			let job = wanted ? jobs.get(wanted) : all.length === 1 ? all[0] : undefined;
+			if (wanted && !job) return ctx.ui.notify(`No background job ${args.trim()}; there's ${[...jobs.keys()].join(", ")}`, "warning");
+			if (!job) {
+				// Matched by position, since the elapsed time in a label changes while the list is open
+				const options = all.map((other) => `${other.id} ${shownCommand(other.command)} · ${state(other)} · ${elapsed(other)}`);
+				job = all[options.indexOf((await ctx.ui.select("Background jobs", options)) ?? "")];
+				if (!job) return;
+			}
+			const shown = job;
+			viewing = shown;
+			const log = shown.log.replace(homedir(), "~");
+			try {
+				await ctx.ui.custom<void>((tui, theme, _kb, done) => {
+					const timer = setInterval(() => tui.requestRender(), 1000);
+					return {
+						dispose: () => clearInterval(timer),
+						invalidate() {},
+						handleInput: (data: string) => {
+							if (matchesKey(data, Key.escape) || matchesKey(data, Key.ctrl("c")) || data === "q") done();
+						},
+						render(width: number) {
+							const color = shown.status === "running" ? "accent" : shown.status === "exited" && shown.exitCode === 0 ? "success" : "error";
+							const title = theme.fg(color, " ● ") + theme.fg("toolTitle", `${shown.id} ${shownCommand(shown.command)}`);
+							const status = theme.fg("muted", `${state(shown)} · ${elapsed(shown)} `);
+							const gap = Math.max(1, width - visibleWidth(title) - visibleWidth(status));
+							// At most half the terminal, so the conversation above stays in view
+							const text = tail(shown.log, Math.max(3, Math.min(TAIL_LINES, Math.floor(tui.terminal.rows / 2))));
+							const output = text === "(no output)" ? [theme.fg("dim", theme.italic("   no output yet"))] : text.split("\n").map((line) => `   ${cleanLine(line)}`);
+							return [
+								theme.fg("borderMuted", "─".repeat(width)),
+								title + " ".repeat(gap) + status,
+								...output,
+								theme.fg("dim", ` esc to close · ${log}`),
+								theme.fg("borderMuted", "─".repeat(width)),
+							].map((line) => truncateToWidth(line, width));
+						},
+					};
+				});
+			} finally {
+				viewing = undefined;
+			}
+		},
+	});
+
+	// After /reload or a resume, carry on after the ids already in the conversation so "bg3" keeps meaning one job
+	pi.on("session_start", async (_event, ctx) => {
+		for (const entry of ctx.sessionManager.getBranch()) {
+			const msg = entry.type === "message" ? (entry.message as any) : undefined;
+			const n = Number(/^bg(\d+)$/.exec(msg?.role === "toolResult" && msg.toolName === "bg_run" ? String(msg.details?.id) : "")?.[1]);
+			if (n >= next) next = n + 1;
+		}
+	});
+
 	pi.on("session_shutdown", async () => {
+		for (const timer of holds.values()) clearTimeout(timer);
+		holds.clear();
 		for (const job of jobs.values()) {
 			if (job.status === "running") job.controller.abort();
 		}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-background",
-  "version": "0.2.2",
+  "version": "0.3.1",
   "license": "MIT",
   "author": "Thomas van Dongen",
   "type": "module",
@@ -13,7 +13,7 @@
     "README.md",
     "LICENSE"
   ],
-  "description": "Lets pi run long commands in the background and wakes it when they finish, so it never blocks or hangs",
+  "description": "Lets Pi run long commands in the background and wakes it when they finish, so it never blocks or hangs",
   "keywords": [
     "pi-package",
     "pi",