pi-repl-py 0.2.6 → 0.2.8

package/README.md

@@ -96,6 +96,7 @@ The Python interpreter is auto-resolved (the venv, else `$PYTHON`/`python3`).
 - Why this design: [docs/design.md](docs/design.md)
 - How it works, the venv, and the kernel: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)
 - How to write and load helpers: [docs/helpers.md](docs/helpers.md)
+- Working examples you can copy: [example/](example/) — helpers under `example/helper/` and skills under `example/skills/`
 - Termux / Android installation: [docs/termux.md](docs/termux.md)
 
 ## It is not

package/docs/ARCHITECTURE.md

@@ -87,10 +87,12 @@ finished draining on iopub. A cell settles only when **both** the `execute_reply
 matching iopub `status idle` (published after every byte of output) have arrived. Settling
 on the reply alone would drop output that was still in flight.
 
-**Output is capped per channel.** Each channel accumulates output against a character budget
-(`maxOutputChars`). Overflow is checked within each message. A single 10 MB print trips the
-cap immediately instead of waiting for a later message to exhaust the budget. The host appends
-an explicit truncation marker so the model knows output was cut.
+**Output is capped per channel and per line.** Each channel accumulates output against a character
+budget (`maxOutputChars`), checked within each message, so overflow trips the moment a message
+exceeds the budget rather than when it churns on. Each individual line is also capped at a generous
+length (4096 chars), so a single genuinely oversized line cannot own the whole budget — while
+legitimately long REPL output (JSON, reprs, errors) still passes through whole. Both truncations
+are announced with explicit markers so the model knows output was cut.
 
 **Cancellation is real.** An abort sends an `interrupt_request` on the control channel,
 which raises a genuine `KeyboardInterrupt` in the running cell; the namespace survives. As a

package/docs/helpers.md

@@ -119,6 +119,26 @@ the workspace.
 
 Because helpers execute at kernel startup, top-level code has consequences. Definitions are fine; imports should be reasonable; network calls, prints, subprocesses, and expensive work should usually happen inside an explicit function or method call.
 
+## Third-party packages in the install venv
+
+The evaluator runs from a real Python virtualenv (`~/.pi/agent/pi-repl/venv`); `sys.path` includes
+its `site-packages`. Packages you install there are importable from helpers and from any cell:
+
+```bash
+~/.pi/agent/pi-repl/venv/bin/pip3 install -U numpy pandas
+```
+
+```python
+import numpy as np
+```
+
+That is how a helper reaches a package the repl does not ship by default (the venv is created
+minimal — it has no `requests`, `numpy`, `pandas`, etc.). ipykernel only pulls its own dependencies.
+
+Because it is a project-local venv, the packages you add are personal to you: nothing about them
+ships with pi-repl. A helper that depends on a package runs only on machines that have installed
+it, so say so in the helper description or docstring when your helper imports one.
+
 ## Choosing what belongs in a helper
 
 Write a helper when it owns a part of the work that is easy to get wrong or tedious to repeat:

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pi-repl-py",
-	"version": "0.2.6",
+	"version": "0.2.8",
 	"type": "module",
 	"description": "A pi extension with a single tool: execute, running a TypeScript host with a persistent Python (ipykernel) evaluator and a user-configurable toolbox of functions.",
 	"keywords": [

package/src/engine/index.ts

@@ -26,6 +26,10 @@ function resolvePythonPath(cwd: string | undefined): string {
 }
 
 const DEFAULT_MAX_OUTPUT_CHARS = 65536;
+/** Per-line cap: one genuinely oversized line must not own the channel budget, while legitimately long
+ * REPL output (JSON, reprs, errors) still fits under the cap in one piece. Generous enough that only
+ * pathological giant lines are trimmed, unlike pi's grep where the line cap keeps matches terse. */
+export const MAX_OUTPUT_LINE_CHARS = 4096;
 const ABORT_GRACE_MS = 500;
 const DEFAULT_SNAPSHOT_DEBOUNCE_MS = 1500;
 
@@ -98,6 +102,18 @@ function truncateWithMarker(text: string, maxChars: number, wasTruncated: boolea
 	return `${text.slice(0, maxChars)}\n[... output truncated at ${maxChars} chars ...]`;
 }
 
+/** Cap each individual line, so one giant line cannot own the whole channel budget (like grep's line cap). */
+export function capLinesForContext(text: string): { text: string; trimmed: boolean } {
+	const lines = text.split("\n");
+	let trimmed = false;
+	const mapped = lines.map((line) => {
+		if (line.length <= MAX_OUTPUT_LINE_CHARS) return line;
+		trimmed = true;
+		return line.slice(0, MAX_OUTPUT_LINE_CHARS);
+	});
+	return { text: mapped.join("\n"), trimmed };
+}
+
 export class EngineManager {
 	private readonly options: EngineOptions;
 	private kernel?: KernelClient;
@@ -233,11 +249,18 @@ export class EngineManager {
 				});
 				if (r.status === "ok") this.scheduleSnapshot();
 				const status: ExecuteResult["status"] = opts.signal?.aborted ? "aborted" : r.status;
-				const truncate = (text: string, truncated: boolean) => truncateWithMarker(text, maxChars, truncated);
+				// Channel cap (truncateWithMarker), then per-line cap; both append a marker so truncation is explicit.
+				const finalize = (text: string, channelTruncated: boolean): string => {
+					let out = truncateWithMarker(text, maxChars, channelTruncated);
+					const line = capLinesForContext(out);
+					if (line.trimmed) out = `${line.text}\n[... some lines exceeded ${MAX_OUTPUT_LINE_CHARS} chars ...]`;
+					else out = line.text;
+					return out;
+				};
 				return {
-					stdout: truncate(r.stdout, r.truncated?.stdout ?? false),
-					stderr: truncate(r.stderr, r.truncated?.stderr ?? false),
-					result: r.result !== undefined ? truncate(String(r.result), String(r.result).length > maxChars) : undefined,
+					stdout: finalize(r.stdout, r.truncated?.stdout ?? false),
+					stderr: finalize(r.stderr, r.truncated?.stderr ?? false),
+					result: r.result !== undefined ? finalize(String(r.result), String(r.result).length > maxChars) : undefined,
 					error: r.error,
 					status,
 					durationMs: Date.now() - started,

package/src/extension/prompt.ts

@@ -6,75 +6,58 @@
 // more signal; the machine reads every line every turn.
 
 export const executeToolDescription =
-	"You have one tool: a persistent Python workspace backed by a real `ipython` kernel. " +
-	"Variables, imports, and definitions survive across cells and turns — it is your working memory and action " +
-	"language. " +
-	"Helpers in `~/.pi/agent/pi-repl/helpers/` load at boot. A cell returns its final expression; printed output is " +
-	"captured separately.";
+	"You have one tool: a real `ipython` kernel that stays alive across cells and turns. " +
+	"This persistent Python workspace is your only surface — it does the work of bash, read, write, edit, " +
+	"search, and file handling, and everything you define (variables, imports, helpers loaded from " +
+	"`~/.pi/agent/pi-repl/helpers/`) survives for reuse in later cells. A cell returns its final expression; " +
+	"printed output is captured separately.";
 
 export const executePromptSnippet =
-	"Work in the workspace: keep artifacts in variables, compose related actions in Python, print only what " +
-	"the next step needs, and revise from what you observe.";
+	"The persistent Python workspace is your only tool: keep artifacts in variables across cells for reuse, use the loaded helpers, prefer surgical reads/edits over full-file dumps and rewrites, and parse before you print so context stays lean.";
 
 // --- the workspace doctrine riding the execute tool ---
 export function buildPromptGuidelines(preloaded: string[]): string[] {
 	return [
 		"## Your only workspace",
-		"`execute` is the only callable tool. Python replaces a read, shell, search, and edit tool rack. State " +
-			"persists across cells and turns.",
+		"`execute` is the only callable tool. Python replaces a read, shell, search, and edit tool rack. State persists across cells and turns.",
 		"",
 		"## Work in the workspace, not the transcript",
-		"Load files, command results, search hits, and computed artifacts into variables once; filter, compare, " +
-			"branch, edit, and verify them in later cells. Do not re-read or paste raw material back. Print only the " +
-			"small observation needed for the next decision; keep the full artifact in a variable.",
+		"Load files, command results, searches, and computed artifacts into variables once; filter, compare, branch, edit, and verify them in later cells. Do not re-read or paste raw material back. Print only the small observation you'll decide on next; keep the full artifact in a variable. A bare final expression is auto-displayed by IPython — assign instead and print only what the next step needs.",
 		"",
 		"## A cell is a small program",
-		"Compose filesystem access, shell commands, searches, transforms, checks, and edits in ordinary Python " +
-			"when they belong to the same step.",
+		"Compose filesystem access, shell commands, searches, transforms, checks, and edits in ordinary Python in the same step.",
 		"",
 		"## Revise on observations",
 		"Revise prior actions or emit new actions upon new observations.", // CodeAct core
 		"",
 		"## Probe, then build",
-		"Inspect what is present — count, print a few lines, list what is loaded — before committing. Build one " +
-			"step, run it, and use its output to choose the next.",
+		"Inspect what is present — count, print a few lines, list what is loaded — before committing; build one step, run it, and use its output to choose the next.",
 		"",
-		"## Precise file and search work",
-		"Search narrowly and inspect only the lines needed. Do not dump whole files or repeat unchanged context. " +
-			"For existing files, prefer a surgical old-text/new-text replacement over rewriting the file. Read the " +
-			"target region first, make the smallest unique replacement, then verify the changed region and file validity. " +
-			"Use complete writes only for new files or intentional full rewrites. Never leave a bare final expression: " +
-			"IPython displays it automatically; assign results and explicitly print only what you need.",
+		"## File and search work",
+		"Prefer a surgical old-text/new-text replacement over rewriting a file: read the region first, make the smallest unique replacement, verify the change and file validity. Use complete writes only for new files or intentional full rewrites. When walking directories, prune generated dirs — node_modules, .git, .venv, dist, __pycache__ — and never print a raw tree.",
 		"",
 		"## Repository discipline",
-		"Inspect before changing. Preserve project conventions and unrelated content. Make the smallest valid change, " +
-			"verify it afterward, and never invent files, APIs, conventions, or test results.",
+		"Make the smallest valid change, preserve conventions, verify afterward, and never invent files, APIs, conventions, or test results.",
 		"",
-		"## Batch and print sparingly",
-		"Batch as much independent work as reasonably possible into one call. Keep large values in variables; " +
-			"print slices, counts, and summaries.",
+		"## Context is expensive",
+		"Every printed value enters the conversation. Explore and filter in variables; print only the small, bounded slice for the next decision. Never dump a whole file, a raw result list, or an unbounded output, and never rely on truncation to control it.",
 		"",
 		...(preloaded.length
 			? [
 					"## Helpers",
-					"User helpers load from `~/.pi/agent/pi-repl/helpers/` as workspace definitions. Their descriptions " +
-						"appear below.",
+					"These helpers are given to you by the user to use directly (loaded from `~/.pi/agent/pi-repl/helpers/`). Descriptions appear below.",
 					"",
 					...preloaded,
 					"",
 				]
 			: []),
 		"## Shell and search",
-		"`subprocess.run(..., timeout=...)` when you need a result — always set a `timeout`, the evaluator does not " +
-			"kill a silent cell. Use `rg`/`grep`/`find` via `subprocess.run` for deep searches, not Python loops.",
+		"Always pass a `timeout` to `subprocess.run(...)` — a silent cell must die, not hang. Use `rg`/`grep`/`find` via the subprocess for deep searches, not Python loops.",
 		"",
-		"## Environment boundary",
-		"The evaluator runs in a project-local venv, not the system Python. Do not install a target project's " +
-			"dependencies into the evaluator. Run external projects through their own interface and normal commands.",
+		"## Environment & rescue",
+		"The evaluator runs in a project-local venv, not the system Python. Do not install a project's dependencies into the evaluator; run external projects through their own interface. If output begins with `<repl_engine_reset>`, the kernel was rebuilt — re-verify any revived variable before reusing it.",
 		"",
-		"## Engine reset guard",
-		"If output begins with `<repl_engine_reset>`, the kernel was rebuilt from the last snapshot. Re-verify a " +
-			"revived variable before reusing it — especially in a shell command. Functions, classes, and live handles " +
-			"are not snapshotted and must be redefined.",
+		"## Follow these as the operating manual",
+		"These guidelines are how this workspace works — internalize their intent and adapt to this environment by applying it to decisions they do not spell out. Follow them diligently.",
 	];
 }