pi-repl-py 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -22
- package/docs/ARCHITECTURE.md +128 -88
- package/docs/how-to-functions.md +82 -80
- package/docs/philosophy.md +67 -59
- package/index.ts +4 -22
- package/package.json +5 -6
- package/scripts/setup-venv.mjs +39 -17
- package/src/engine/index.ts +90 -424
- package/src/engine/kernel.ts +598 -0
- package/src/engine/session.ts +149 -0
- package/src/engine/zmtp.ts +251 -0
- package/src/extension/helpers.ts +46 -0
- package/src/extension/preview/candidates.ts +7 -53
- package/src/extension/preview/descriptor.ts +2 -2
- package/src/extension/preview/index.ts +3 -11
- package/src/extension/preview/shell.ts +4 -4
- package/src/extension/preview/types.ts +1 -1
- package/src/extension/prompt.ts +67 -45
- package/src/extension/render-core.ts +5 -9
- package/src/extension/render.ts +2 -11
- package/src/extension/session-engine.ts +9 -31
- package/src/extension/tool-meta.ts +6 -6
- package/src/engine/guest.py +0 -320
- package/src/engine/protocol.ts +0 -66
- package/src/engine/toolbox/bash.py +0 -72
- package/src/engine/toolbox/edit.py +0 -37
- package/src/engine/toolbox/read.py +0 -26
- package/src/engine/toolbox/write.py +0 -23
- package/src/extension/config.ts +0 -64
- package/src/extension/preview-core.ts +0 -2
- package/src/extension/toolbox.ts +0 -98
package/src/extension/prompt.ts
CHANGED
|
@@ -1,54 +1,76 @@
|
|
|
1
|
-
// --- prompt: the execute tool's model-facing contract (pure, no pi/
|
|
2
|
-
// --- mirrors pi-robust-edit's schema/domain split: content lives here; the thin adapter in tool-meta wires it in ---
|
|
1
|
+
// --- prompt: the execute tool's model-facing contract (pure, no pi/helper dep) ---
|
|
3
2
|
|
|
4
3
|
export const executeToolDescription =
|
|
5
|
-
"
|
|
6
|
-
"functions, and data survive across
|
|
7
|
-
"
|
|
8
|
-
"
|
|
4
|
+
"You have one tool: a persistent Python workspace backed by a real `ipython` kernel. " +
|
|
5
|
+
"Variables, imports, functions, and data survive across cells and turns. Use this tool to read files, " +
|
|
6
|
+
"run shell commands, search code, transform data, and build up solutions — all inside Python. " +
|
|
7
|
+
"Helpers in `~/.pi/agent/pi-repl/helpers/` are loaded at boot as functions; see what's loaded with " +
|
|
8
|
+
"`[k for k in globals() if not k.startswith('_')]`. A cell returns its final expression; printed output " +
|
|
9
|
+
"is captured separately.";
|
|
9
10
|
|
|
10
11
|
export const executePromptSnippet =
|
|
11
|
-
"
|
|
12
|
-
"
|
|
13
|
-
"ls() lists them, help(name) shows usage";
|
|
12
|
+
"Use the Python workspace: keep state in variables, batch independent reads/searches in one cell, " +
|
|
13
|
+
"edit files safely, and iterate in small cells.";
|
|
14
14
|
|
|
15
|
-
// --- the
|
|
15
|
+
// --- the workspace doctrine riding the execute tool ---
|
|
16
16
|
export function buildPromptGuidelines(preloaded: string[]): string[] {
|
|
17
17
|
return [
|
|
18
|
-
"##
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
"",
|
|
22
|
-
"##
|
|
23
|
-
"
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
"
|
|
27
|
-
"
|
|
28
|
-
"
|
|
29
|
-
|
|
30
|
-
""
|
|
31
|
-
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
"
|
|
39
|
-
"
|
|
40
|
-
""
|
|
41
|
-
|
|
42
|
-
"
|
|
43
|
-
|
|
44
|
-
"
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
18
|
+
"## This workspace is your only tool",
|
|
19
|
+
"In `--repl` mode, `execute` is the only callable tool. Read files, run shell, search, and edit " +
|
|
20
|
+
"all happen inside Python.",
|
|
21
|
+
"",
|
|
22
|
+
"## The loop is generate → execute → observe → iterate",
|
|
23
|
+
"Write a cell, run it, observe the result, then write the next cell. Build solutions incrementally.",
|
|
24
|
+
"",
|
|
25
|
+
"## State persists",
|
|
26
|
+
"Variables, imports, and functions survive across cells and turns. Assign read/search results to " +
|
|
27
|
+
"named variables and reuse them.",
|
|
28
|
+
"",
|
|
29
|
+
"## Chain big tasks into verifiable steps",
|
|
30
|
+
"Break ambitious requests into independently checkable steps. Confirm assumptions before writing " +
|
|
31
|
+
"code that depends on them.",
|
|
32
|
+
"",
|
|
33
|
+
"## Shell & files are plain Python",
|
|
34
|
+
"`!cmd` / `%%bash` for fire-and-forget shell; `subprocess.run(..., timeout=...)` when you need the result back " +
|
|
35
|
+
"as a value — always set a `timeout` on anything that could hang (the evaluator does not kill a " +
|
|
36
|
+
"silent cell automatically). `open()` / `pathlib` read and write files. For safe edits: read the full " +
|
|
37
|
+
"file, modify in memory, write once, then re-read to verify.",
|
|
38
|
+
"",
|
|
39
|
+
"## Batch independent work, keep exploratory cells small",
|
|
40
|
+
"Batch independent reads, searches, and setup steps in one cell to reduce round-trips. Keep " +
|
|
41
|
+
"exploratory/iterative cells small so you can observe and adjust.",
|
|
42
|
+
"",
|
|
43
|
+
"## Search efficiently",
|
|
44
|
+
"Use `rg`, `fd`, `grep`, `find` via `subprocess.run` for deep searches, not Python loops.",
|
|
45
|
+
"",
|
|
46
|
+
...(preloaded.length
|
|
47
|
+
? [
|
|
48
|
+
"## Helpers",
|
|
49
|
+
"User helpers load from `~/.pi/agent/pi-repl/helpers/`. Their descriptions appear below. " +
|
|
50
|
+
"List what's loaded with `[k for k in globals() if not k.startswith('_')]`.",
|
|
51
|
+
"",
|
|
52
|
+
...preloaded,
|
|
53
|
+
"",
|
|
54
|
+
]
|
|
55
|
+
: []),
|
|
56
|
+
"## Compose and reuse",
|
|
57
|
+
"If the same pattern appears more than once, wrap it in a `def` and reuse it.",
|
|
58
|
+
"",
|
|
59
|
+
"## Output discipline",
|
|
60
|
+
"Printing is a context cost: everything a cell prints stays in the transcript. Print slices, " +
|
|
61
|
+
"counts, and summaries. Keep large values in variables. End a cell with `;` to suppress the " +
|
|
62
|
+
"last-expression echo.",
|
|
63
|
+
"",
|
|
64
|
+
"## Environment boundary",
|
|
65
|
+
"The evaluator runs in a project-local venv, not the system Python. Do not install a target project's " +
|
|
66
|
+
"dependencies into the evaluator just to make that project run there. Run external projects " +
|
|
67
|
+
"through their own interface and normal commands.",
|
|
68
|
+
"",
|
|
69
|
+
"## Engine reset guard",
|
|
70
|
+
"If the output begins with `<repl_engine_reset>`, the kernel was rebuilt from the last snapshot. " +
|
|
71
|
+
"Some variables may be revived, some lost, and anything defined after the snapshot is gone. " +
|
|
72
|
+
"Re-verify variables before reusing them — never interpolate a restored variable into a shell " +
|
|
73
|
+
"command until you have confirmed it still holds what you expect. Functions, classes, and live " +
|
|
74
|
+
"handles cannot be snapshotted and must be redefined.",
|
|
53
75
|
];
|
|
54
76
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
/** Pure layout, free of pi imports so unit tests drive it directly. */
|
|
2
2
|
|
|
3
3
|
export interface ExecuteDetails {
|
|
4
4
|
status?: "ok" | "error" | "aborted" | string;
|
|
@@ -21,7 +21,7 @@ export interface ExecuteRenderState {
|
|
|
21
21
|
hasResult: boolean;
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
-
import { previewCell } from "./preview
|
|
24
|
+
import { previewCell } from "./preview/index.js";
|
|
25
25
|
|
|
26
26
|
export type StatusKind = "error" | "aborted" | "running" | "queued" | "done";
|
|
27
27
|
export type BgKind = "toolPendingBg" | "toolSuccessBg" | "toolErrorBg";
|
|
@@ -49,11 +49,7 @@ export function formatDuration(durationMs: number | undefined): string | undefin
|
|
|
49
49
|
|
|
50
50
|
const SGR_PATTERN = /\x1b\[([0-9;]*)m/g;
|
|
51
51
|
|
|
52
|
-
/**
|
|
53
|
-
* Append a reset when `line` ends with a foreground or background color still
|
|
54
|
-
* open, so a span that wrapping split across lines cannot bleed into the
|
|
55
|
-
* trailing padding or the next row.
|
|
56
|
-
*/
|
|
52
|
+
/** Close an open color so a line wrapped across words can't bleed into padding or the next row. */
|
|
57
53
|
export function closeOpenSgr(line: string): string {
|
|
58
54
|
let fgOpen = false;
|
|
59
55
|
let bgOpen = false;
|
|
@@ -134,7 +130,7 @@ function outputText(state: ExecuteRenderState): string {
|
|
|
134
130
|
function topLine(state: ExecuteRenderState, width: number, deps: RenderDeps): string {
|
|
135
131
|
const code = state.code.trimEnd();
|
|
136
132
|
const preview = previewCell(code);
|
|
137
|
-
const language = preview.kind === "shell" ? "repl · shell" :
|
|
133
|
+
const language = preview.kind === "shell" ? "repl · shell" : "repl";
|
|
138
134
|
const prefix = `${marker(state, deps)} ${deps.fg("muted", language)}`;
|
|
139
135
|
|
|
140
136
|
// --- suffix priority: expand hint > error > duration > counts, so truncation never hides the expand key ---
|
|
@@ -168,7 +164,7 @@ function topLine(state: ExecuteRenderState, width: number, deps: RenderDeps): st
|
|
|
168
164
|
// --- budget: width minus leading space, prefix, suffix, and separators ---
|
|
169
165
|
const fixed = 1 + deps.visibleWidth(prefix) + separatorWidth + deps.visibleWidth(suffix);
|
|
170
166
|
const previewBudget = Math.max(8, width - fixed - separatorWidth);
|
|
171
|
-
// --- a semantic preview is a one-line summary; highlight Python code, accent shell
|
|
167
|
+
// --- a semantic preview is a one-line summary; highlight Python code, accent shell intent ---
|
|
172
168
|
let middle = "";
|
|
173
169
|
if (preview.text) {
|
|
174
170
|
const previewText =
|
package/src/extension/render.ts
CHANGED
|
@@ -1,9 +1,4 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* TUI adapter for the `execute` cell renderer.
|
|
3
|
-
*
|
|
4
|
-
* Binds pi's theme, syntax highlighting, key hints, and width primitives to the
|
|
5
|
-
* pure layout in render-core.ts, which is unit-tested outside pi's runtime.
|
|
6
|
-
*/
|
|
1
|
+
/** TUI adapter binding pi's theme/width to the unit-tested pure layout in render-core.ts. */
|
|
7
2
|
|
|
8
3
|
import { highlightCode, keyHint, keyText, rawKeyHint, type Theme } from "@mariozechner/pi-coding-agent";
|
|
9
4
|
import { truncateToWidth, visibleWidth, wrapTextWithAnsi } from "@mariozechner/pi-tui";
|
|
@@ -35,11 +30,7 @@ function makeDeps(theme: Theme): RenderDeps {
|
|
|
35
30
|
};
|
|
36
31
|
}
|
|
37
32
|
|
|
38
|
-
/**
|
|
39
|
-
* The layout of a cell only changes when its state or the spinner frame does,
|
|
40
|
-
* but the TUI repaints on every frame. Rendering from a key of both stops the
|
|
41
|
-
* recompute-per-frame (and with it, flicker on wide panes).
|
|
42
|
-
*/
|
|
33
|
+
/** The layout only changes on state/spinner change, but the TUI repaints every frame; key by both to avoid recompute flicker. */
|
|
43
34
|
function renderVersion(state: ExecuteRenderState): string {
|
|
44
35
|
const details = state.details ? JSON.stringify(state.details) : "";
|
|
45
36
|
return [
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
//
|
|
1
|
+
// revival is part of create() so a session that gets teardown without a session_start reload still revives
|
|
2
2
|
|
|
3
3
|
import type { RestoreResult } from "../engine/index.js";
|
|
4
4
|
|
|
5
|
-
/**
|
|
6
|
-
* A revived session can carry hundreds of variables; listing them all turns
|
|
7
|
-
* the banner and the reset notice into a wall. Show enough to orient, then
|
|
8
|
-
* count the rest.
|
|
9
|
-
*/
|
|
5
|
+
/** Show enough names to orient, then count the rest (a revive can carry hundreds). */
|
|
10
6
|
export function summarizeNames(names: readonly string[], limit: number): string {
|
|
11
7
|
if (names.length <= limit) return names.join(", ");
|
|
12
8
|
return `${names.slice(0, limit).join(", ")} … and ${names.length - limit} more`;
|
|
@@ -22,23 +18,15 @@ export interface EngineLifecycleDeps<E extends RevivableEngine> {
|
|
|
22
18
|
create(): E;
|
|
23
19
|
/** Tears the current engine down, flushing its final snapshot. */
|
|
24
20
|
dispose(engine: E): Promise<void>;
|
|
25
|
-
/**
|
|
26
|
-
* Tears down an engine that cannot cooperate — a wedged guest cannot serve
|
|
27
|
-
* the snapshot flush dispose would ask of it. Falls back to dispose.
|
|
28
|
-
*/
|
|
21
|
+
/** Kill-then-rebuild when a wedged engine cannot serve the snapshot flush. */
|
|
29
22
|
discard?(engine: E): Promise<void>;
|
|
30
23
|
}
|
|
31
24
|
|
|
32
|
-
/**
|
|
33
|
-
* Why an engine came into existence. `startup` is the expected path and is
|
|
34
|
-
* already announced in the transcript; `cell` means an engine had to be built
|
|
35
|
-
* to serve a tool call, which only happens when the previous one went away
|
|
36
|
-
* mid-session — the case the model needs told about in-band.
|
|
37
|
-
*/
|
|
25
|
+
/** `startup` is announced in the transcript; `cell` means an engine was rebuilt mid-session and needs an in-band notice. */
|
|
38
26
|
export type AcquireOrigin = "startup" | "cell";
|
|
39
27
|
|
|
40
28
|
function formatEngineResetNotice(restore: RestoreResult | null): string {
|
|
41
|
-
const lines = ["<
|
|
29
|
+
const lines = ["<repl_engine_reset>"];
|
|
42
30
|
if (!restore) {
|
|
43
31
|
// --- no snapshot at all: namespace is genuinely empty ---
|
|
44
32
|
lines.push(
|
|
@@ -73,7 +61,7 @@ function formatEngineResetNotice(restore: RestoreResult | null): string {
|
|
|
73
61
|
}
|
|
74
62
|
lines.push("Anything defined after the last snapshot is also gone.");
|
|
75
63
|
}
|
|
76
|
-
lines.push("Re-verify a variable before reusing it, especially inside shell interpolation.", "</
|
|
64
|
+
lines.push("Re-verify a variable before reusing it, especially inside shell interpolation.", "</repl_engine_reset>");
|
|
77
65
|
return lines.join("\n");
|
|
78
66
|
}
|
|
79
67
|
|
|
@@ -81,25 +69,19 @@ export class EngineLifecycle<E extends RevivableEngine> {
|
|
|
81
69
|
private engine?: E;
|
|
82
70
|
private revival?: Promise<RestoreResult | null>;
|
|
83
71
|
private pendingNotice?: string;
|
|
84
|
-
/** Teardown in progress; a rebuild must not overlap the final snapshot flush. */
|
|
85
72
|
private teardown?: Promise<void>;
|
|
86
|
-
/** First-build in progress
|
|
73
|
+
/** First-build in progress. */
|
|
87
74
|
private acquiring?: Promise<{ engine: E; restore: RestoreResult | null; created: boolean }>;
|
|
88
75
|
|
|
89
76
|
constructor(private readonly deps: EngineLifecycleDeps<E>) {}
|
|
90
77
|
|
|
91
|
-
/**
|
|
92
|
-
* The live engine, built and revived if it does not exist yet.
|
|
93
|
-
* Revival is awaited here so a caller never sees an un-revived namespace.
|
|
94
|
-
*/
|
|
78
|
+
/** Built and revived on demand; awaited so callers never see an un-revived namespace. */
|
|
95
79
|
async acquire(origin: AcquireOrigin): Promise<{ engine: E; restore: RestoreResult | null; created: boolean }> {
|
|
96
80
|
if (this.engine) {
|
|
97
81
|
return { engine: this.engine, restore: await this.revival!, created: false };
|
|
98
82
|
}
|
|
99
|
-
// --- two concurrent acquires on an empty engine must share one build ---
|
|
100
83
|
if (this.acquiring) return this.acquiring;
|
|
101
84
|
const build = (async () => {
|
|
102
|
-
// --- a teardown flushing its final snapshot must finish before the rebuild reads it ---
|
|
103
85
|
while (this.teardown) await this.teardown;
|
|
104
86
|
if (this.engine) {
|
|
105
87
|
const held: E = this.engine;
|
|
@@ -131,11 +113,7 @@ export class EngineLifecycle<E extends RevivableEngine> {
|
|
|
131
113
|
await this.teardownWith((engine) => this.deps.dispose(engine));
|
|
132
114
|
}
|
|
133
115
|
|
|
134
|
-
/**
|
|
135
|
-
* Teardown for an engine that cannot cooperate (e.g. wedged in synchronous
|
|
136
|
-
* code). Skips the snapshot flush a graceful dispose would attempt; the next
|
|
137
|
-
* acquire builds a fresh engine revived from the last completed snapshot.
|
|
138
|
-
*/
|
|
116
|
+
/** Kill-then-rebuild for a wedged engine; skips the final snapshot flush, uses the last good one. */
|
|
139
117
|
async discard(): Promise<void> {
|
|
140
118
|
await this.teardownWith((engine) => (this.deps.discard ?? this.deps.dispose)(engine));
|
|
141
119
|
}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
// --- tool-meta: thin surface assembling the execute tool's prompt from pure modules ---
|
|
2
|
-
// --- the model contract lives in prompt.ts; only the
|
|
2
|
+
// --- the model contract lives in prompt.ts; only the helpers wiring stays here ---
|
|
3
3
|
|
|
4
|
+
import { buildHelpersMap } from "./helpers.js";
|
|
4
5
|
import { buildPromptGuidelines, executePromptSnippet, executeToolDescription } from "./prompt.js";
|
|
5
|
-
import { buildToolboxMap } from "./toolbox.js";
|
|
6
6
|
|
|
7
7
|
export const EXECUTE_DESCRIPTION = executeToolDescription;
|
|
8
8
|
export const EXECUTE_PROMPT_SNIPPET = executePromptSnippet;
|
|
9
9
|
|
|
10
|
-
// --- build the guidelines from the
|
|
11
|
-
export function buildExecutePromptGuidelines(
|
|
12
|
-
const map =
|
|
13
|
-
const preloaded = map.length > 0 ? map : [
|
|
10
|
+
// --- build the guidelines from the one helpers dir (default ~/.pi/agent/pi-repl/helpers) ---
|
|
11
|
+
export function buildExecutePromptGuidelines(): string[] {
|
|
12
|
+
const map = buildHelpersMap();
|
|
13
|
+
const preloaded = map.length > 0 ? map : [];
|
|
14
14
|
return buildPromptGuidelines(preloaded);
|
|
15
15
|
}
|
package/src/engine/guest.py
DELETED
|
@@ -1,320 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
guest.py — the real IPython kernel guest evaluator for pi-repl.
|
|
3
|
-
|
|
4
|
-
The host spawns this once. It starts a local ipykernel subprocess via
|
|
5
|
-
jupyter_client, keeps it for the session, and bridges the wire protocol to it
|
|
6
|
-
(stdin = commands, fd 3 = results). State survives because the kernel process
|
|
7
|
-
does. Frames carry a nonce the host mints and the guest erases, so agent code
|
|
8
|
-
cannot forge protocol traffic.
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
from __future__ import annotations
|
|
12
|
-
|
|
13
|
-
import json
|
|
14
|
-
import os
|
|
15
|
-
import sys
|
|
16
|
-
import time
|
|
17
|
-
|
|
18
|
-
# --- protocol envelope ---
|
|
19
|
-
ENVELOPE_KEY = "__rlm"
|
|
20
|
-
NONCE_ENV = "PI_RLM_NONCE"
|
|
21
|
-
PROTOCOL_FD = 3
|
|
22
|
-
|
|
23
|
-
NONCE = os.environ.get(NONCE_ENV, "")
|
|
24
|
-
os.environ.pop(NONCE_ENV, None)
|
|
25
|
-
|
|
26
|
-
# --- per-cell timeout: 0 = no cap; else a silence watchdog (no output for N seconds) ---
|
|
27
|
-
CELL_TIMEOUT_S = float(os.environ.get("PI_REPL_TIMEOUT_MS", "0") or "0") / 1000.0
|
|
28
|
-
|
|
29
|
-
# --- snapshot/restore use a fixed window, not the cell silence timer ---
|
|
30
|
-
SNAPSHOT_TIMEOUT_S = 90.0
|
|
31
|
-
|
|
32
|
-
# --- cap buffered output so a runaway print can't grow guest memory or send one giant frame ---
|
|
33
|
-
MAX_CELL_OUTPUT_CHARS = 1_000_000
|
|
34
|
-
|
|
35
|
-
# --- fd 3 protocol writer; dup'd so we don't close the caller's fd 3 on exit ---
|
|
36
|
-
_proto = os.fdopen(os.dup(PROTOCOL_FD), "w", buffering=1)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def _send(msg):
|
|
40
|
-
envelope = {ENVELOPE_KEY: 1, **msg}
|
|
41
|
-
if NONCE:
|
|
42
|
-
envelope["n"] = NONCE
|
|
43
|
-
_proto.write(json.dumps(envelope) + "\n")
|
|
44
|
-
_proto.flush()
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def _decode(line):
|
|
48
|
-
if ENVELOPE_KEY not in line:
|
|
49
|
-
return None
|
|
50
|
-
try:
|
|
51
|
-
obj = json.loads(line)
|
|
52
|
-
except Exception:
|
|
53
|
-
return None
|
|
54
|
-
if obj.get(ENVELOPE_KEY) != 1 or not isinstance(obj.get("type"), str):
|
|
55
|
-
return None
|
|
56
|
-
if NONCE and obj.get("n") != NONCE:
|
|
57
|
-
return None
|
|
58
|
-
return obj
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
# --- toolbox: one function per *.py, exec'd into every kernel (PI_TOOLBOX_DIR) ---
|
|
62
|
-
|
|
63
|
-
def _toolbox_files(directory):
|
|
64
|
-
"""Return {function_name: source} for each *.py in `directory`."""
|
|
65
|
-
if not directory:
|
|
66
|
-
return {}
|
|
67
|
-
d = os.path.expanduser(directory)
|
|
68
|
-
if not os.path.isdir(d):
|
|
69
|
-
return {}
|
|
70
|
-
names = {}
|
|
71
|
-
for entry in sorted(os.listdir(d)):
|
|
72
|
-
if not entry.endswith(".py"):
|
|
73
|
-
continue
|
|
74
|
-
name = entry[:-3]
|
|
75
|
-
if not name.isidentifier() or name.startswith("_"):
|
|
76
|
-
continue
|
|
77
|
-
try:
|
|
78
|
-
with open(os.path.join(d, entry), encoding="utf-8") as f:
|
|
79
|
-
names[name] = f.read()
|
|
80
|
-
except OSError:
|
|
81
|
-
continue
|
|
82
|
-
return names
|
|
83
|
-
|
|
84
|
-
DEFAULT_TOOLBOX_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "toolbox")
|
|
85
|
-
TOOLBOX_DIR = os.environ.get("PI_TOOLBOX_DIR", "").strip()
|
|
86
|
-
# --- Merge: built-ins are supreme; a config toolboxDir adds others and overrides on name ---
|
|
87
|
-
_TOOLBOX_SRC = _toolbox_files(DEFAULT_TOOLBOX_DIR)
|
|
88
|
-
if TOOLBOX_DIR and os.path.expanduser(TOOLBOX_DIR) != DEFAULT_TOOLBOX_DIR:
|
|
89
|
-
_TOOLBOX_SRC.update(_toolbox_files(TOOLBOX_DIR))
|
|
90
|
-
|
|
91
|
-
# --- help/ls are part of the evaluator, not the toolbox ---
|
|
92
|
-
INTRINSIC = """
|
|
93
|
-
# --- ls() filters IPython-injected names out of the tool list ---
|
|
94
|
-
_RPL_LS_NOISE = {'exit', 'quit', 'get_ipython', 'open', 'display'}
|
|
95
|
-
|
|
96
|
-
def ls():
|
|
97
|
-
return sorted(n for n in globals() if n not in _RPL_LS_NOISE and not n.startswith('_') and callable(globals()[n]))
|
|
98
|
-
|
|
99
|
-
def help(name=None):
|
|
100
|
-
if name is None:
|
|
101
|
-
return ls()
|
|
102
|
-
fn = globals().get(name)
|
|
103
|
-
if fn is None or not callable(fn):
|
|
104
|
-
return f"no such function: {name!r}"
|
|
105
|
-
return fn.__doc__ or f"{name} (no docstring)"
|
|
106
|
-
"""
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
from jupyter_client import KernelManager
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
class Kernel:
|
|
113
|
-
"""A persistent subprocess ipykernel + blocking client."""
|
|
114
|
-
|
|
115
|
-
def __init__(self):
|
|
116
|
-
self.km = KernelManager(kernel_name="python3")
|
|
117
|
-
self.km.start_kernel()
|
|
118
|
-
self.kc = self.km.client()
|
|
119
|
-
self.kc.start_channels()
|
|
120
|
-
self.kc.wait_for_ready(timeout=30)
|
|
121
|
-
self._preload()
|
|
122
|
-
|
|
123
|
-
def _preload(self):
|
|
124
|
-
"""Exec every toolbox function + the intrinsic help/ls into the kernel ns."""
|
|
125
|
-
code = INTRINSIC + "\n"
|
|
126
|
-
for src in _TOOLBOX_SRC.values():
|
|
127
|
-
code += src + "\n"
|
|
128
|
-
if code.strip():
|
|
129
|
-
self.kc.execute(code)
|
|
130
|
-
self._drain()
|
|
131
|
-
|
|
132
|
-
def _drain(self):
|
|
133
|
-
try:
|
|
134
|
-
while True:
|
|
135
|
-
m = self.kc.get_iopub_msg(timeout=1)
|
|
136
|
-
if m.get("msg_type") == "status" and m.get("content", {}).get("execution_state") == "idle":
|
|
137
|
-
break
|
|
138
|
-
except Exception:
|
|
139
|
-
pass
|
|
140
|
-
|
|
141
|
-
def _drain_execution(self, code, timeout):
|
|
142
|
-
"""Run `code`; return (stdout, stderr, error_text, result, timed_out).
|
|
143
|
-
|
|
144
|
-
`timeout <= 0` means "no cap": a cell runs until it reports idle.
|
|
145
|
-
`timeout > 0` is a SILENCE watchdog — it trips only once the cell has
|
|
146
|
-
produced no message for `timeout` seconds. A silent-but-running command
|
|
147
|
-
(e.g. `find ... | sort`) is allowed to complete; a stalled one (dead
|
|
148
|
-
kernel, or nothing for the silence window) reports `timed_out=True` so
|
|
149
|
-
the caller can surface a real hang instead of faking success.
|
|
150
|
-
"""
|
|
151
|
-
msg_id = self.kc.execute(code)
|
|
152
|
-
out, err, error, result = [], [], None, None
|
|
153
|
-
out_len, err_len = 0, 0
|
|
154
|
-
timed_out = False
|
|
155
|
-
# --- silence clock only starts once the cell begins (grace for a fresh kernel) ---
|
|
156
|
-
last_activity: float | None = None
|
|
157
|
-
while True:
|
|
158
|
-
if not self.km.is_alive():
|
|
159
|
-
timed_out = True
|
|
160
|
-
break
|
|
161
|
-
if last_activity is not None and timeout and (time.monotonic() - last_activity) >= timeout:
|
|
162
|
-
timed_out = True
|
|
163
|
-
break
|
|
164
|
-
wait = (timeout - (time.monotonic() - last_activity)) if (last_activity is not None and timeout) else 0.25
|
|
165
|
-
try:
|
|
166
|
-
m = self.kc.get_iopub_msg(timeout=max(0.01, min(0.25, wait)))
|
|
167
|
-
except Exception:
|
|
168
|
-
continue
|
|
169
|
-
if m.get("parent_header", {}).get("msg_id") != msg_id:
|
|
170
|
-
continue
|
|
171
|
-
if last_activity is None:
|
|
172
|
-
last_activity = time.monotonic()
|
|
173
|
-
else:
|
|
174
|
-
last_activity = time.monotonic()
|
|
175
|
-
mt = m.get("msg_type")
|
|
176
|
-
c = m.get("content", {})
|
|
177
|
-
if mt == "stream":
|
|
178
|
-
is_out = c.get("name") == "stdout"
|
|
179
|
-
text = c.get("text", "") or ""
|
|
180
|
-
if is_out:
|
|
181
|
-
if out_len < MAX_CELL_OUTPUT_CHARS:
|
|
182
|
-
take = text[: MAX_CELL_OUTPUT_CHARS - out_len]
|
|
183
|
-
out.append(take)
|
|
184
|
-
out_len += len(take)
|
|
185
|
-
else:
|
|
186
|
-
if err_len < MAX_CELL_OUTPUT_CHARS:
|
|
187
|
-
take = text[: MAX_CELL_OUTPUT_CHARS - err_len]
|
|
188
|
-
err.append(take)
|
|
189
|
-
err_len += len(take)
|
|
190
|
-
elif mt == "execute_result":
|
|
191
|
-
result = c.get("data", {}).get("text/plain")
|
|
192
|
-
elif mt == "error":
|
|
193
|
-
error = "\n".join(c.get("traceback", ["(no traceback)"]))
|
|
194
|
-
elif mt == "status" and c.get("execution_state") == "idle":
|
|
195
|
-
break
|
|
196
|
-
if timed_out:
|
|
197
|
-
# --- best-effort cancel so the NEXT cell doesn't queue behind this one ---
|
|
198
|
-
try:
|
|
199
|
-
self.kc.interrupt_kernel()
|
|
200
|
-
except Exception:
|
|
201
|
-
pass
|
|
202
|
-
return "".join(out), "".join(err), error, result, timed_out
|
|
203
|
-
|
|
204
|
-
def execute(self, code):
|
|
205
|
-
"""Idle-sync path used by snapshot/restore; not a user cell."""
|
|
206
|
-
return self._drain_execution(code, SNAPSHOT_TIMEOUT_S)[:4]
|
|
207
|
-
|
|
208
|
-
def run_cell(self, code):
|
|
209
|
-
"""Run a user cell under the per-cell timeout, so the model learns
|
|
210
|
-
when work did not finish."""
|
|
211
|
-
return self._drain_execution(code, CELL_TIMEOUT_S)
|
|
212
|
-
|
|
213
|
-
def snapshot_globals(self):
|
|
214
|
-
# --- snapshot only user state (skip toolbox/intrinsic functions and _ names) ---
|
|
215
|
-
tool_names = sorted(set(_TOOLBOX_SRC) | {"ls", "help", "function_description"})
|
|
216
|
-
skip_names = json.dumps(tool_names)
|
|
217
|
-
out, _, _, _ = self.execute(
|
|
218
|
-
"import pickle as _pk, base64 as _b64, json as _js\n"
|
|
219
|
-
"__rlm_skip = set(" + skip_names + ") | {'In','Out','get_ipython','exit','quit','open'}\n"
|
|
220
|
-
"__rlm_v = {}\n__rlm_f = []\n"
|
|
221
|
-
"for _k, _v in list(globals().items()):\n"
|
|
222
|
-
" # skip IPython bookkeeping and names with a leading underscore\n"
|
|
223
|
-
" if _k.startswith('__') or _k.startswith('_') or _k in __rlm_skip:\n"
|
|
224
|
-
" continue\n"
|
|
225
|
-
" try:\n"
|
|
226
|
-
" __rlm_v[_k] = _b64.b64encode(_pk.dumps(_v)).decode()\n"
|
|
227
|
-
" except Exception as _e:\n"
|
|
228
|
-
" __rlm_f.append({'name': _k, 'reason': str(_e)})\n"
|
|
229
|
-
"print('__RLC_SNAPSHOT__' + _js.dumps({'vars': __rlm_v, 'failed': __rlm_f}))\n"
|
|
230
|
-
)
|
|
231
|
-
marker = "__RLC_SNAPSHOT__"
|
|
232
|
-
if marker not in out:
|
|
233
|
-
# --- marker never printed: serialization stalled; report incomplete so the host keeps the last good file ---
|
|
234
|
-
return {}, [], False
|
|
235
|
-
try:
|
|
236
|
-
o = json.loads(out.split(marker)[-1])
|
|
237
|
-
return o.get("vars", {}), o.get("failed", []), True
|
|
238
|
-
except Exception:
|
|
239
|
-
return {}, [], False
|
|
240
|
-
|
|
241
|
-
def restore_globals(self, vars_):
|
|
242
|
-
if not vars_:
|
|
243
|
-
return [], []
|
|
244
|
-
# --- restore each variable in one atomic kernel call so one failure reports itself ---
|
|
245
|
-
code2 = (
|
|
246
|
-
"import pickle as _pk, base64 as _b64, json as _js\n"
|
|
247
|
-
"__rl_r = {'restored': [], 'failed': []}\n"
|
|
248
|
-
+ "\n".join(
|
|
249
|
-
"try:\n"
|
|
250
|
-
f" globals()[{name!r}] = _pk.loads(_b64.b64decode({b64!r}))\n"
|
|
251
|
-
f" __rl_r['restored'].append({name!r})\n"
|
|
252
|
-
"except Exception as _e:\n"
|
|
253
|
-
f" __rl_r['failed'].append({{'name': {name!r}, 'reason': str(_e)}})\n"
|
|
254
|
-
for name, b64 in vars_.items()
|
|
255
|
-
)
|
|
256
|
-
+ "\nprint('__RLC_RESTORE__' + _js.dumps(__rl_r))"
|
|
257
|
-
)
|
|
258
|
-
out, _, _, _ = self.execute(code2)
|
|
259
|
-
marker = "__RLC_RESTORE__"
|
|
260
|
-
if marker not in out:
|
|
261
|
-
return [], []
|
|
262
|
-
try:
|
|
263
|
-
obj = json.loads(out.split(marker)[-1])
|
|
264
|
-
return obj.get("restored", []), obj.get("failed", [])
|
|
265
|
-
except Exception:
|
|
266
|
-
return [], []
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
def _line_error(text):
|
|
270
|
-
lines = text.split("\n")
|
|
271
|
-
return {"name": "", "message": text, "stack": lines[:12]}
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
def main():
|
|
275
|
-
kernel = Kernel()
|
|
276
|
-
_send({"type": "ready"})
|
|
277
|
-
|
|
278
|
-
for line in sys.stdin:
|
|
279
|
-
msg = _decode(line)
|
|
280
|
-
if not msg:
|
|
281
|
-
continue
|
|
282
|
-
t = msg["type"]
|
|
283
|
-
if t == "ping":
|
|
284
|
-
_send({"type": "pong", "id": msg["id"]})
|
|
285
|
-
elif t == "snapshot":
|
|
286
|
-
vars_, failed, complete = kernel.snapshot_globals()
|
|
287
|
-
_send({"type": "snapshot_result", "id": msg["id"], "vars": vars_, "failed": failed, "complete": complete})
|
|
288
|
-
elif t == "restore":
|
|
289
|
-
restored, failed = kernel.restore_globals(msg.get("vars", {}))
|
|
290
|
-
_send({"type": "restore_result", "id": msg["id"], "restored": restored, "failed": failed})
|
|
291
|
-
elif t == "list_names":
|
|
292
|
-
names = list(kernel.snapshot_globals()[0].keys())
|
|
293
|
-
_send({"type": "names_result", "id": msg["id"], "names": names})
|
|
294
|
-
elif t == "run":
|
|
295
|
-
cell_id = msg.get("cellId")
|
|
296
|
-
stdout, stderr, error, result, timed_out = kernel.run_cell(msg.get("code", ""))
|
|
297
|
-
if stdout:
|
|
298
|
-
_send({"type": "stream", "cellId": cell_id, "name": "stdout", "chunk": stdout})
|
|
299
|
-
if stderr:
|
|
300
|
-
_send({"type": "stream", "cellId": cell_id, "name": "stderr", "chunk": stderr})
|
|
301
|
-
if timed_out:
|
|
302
|
-
tmsg = {
|
|
303
|
-
"name": "Timeout",
|
|
304
|
-
"message": f"cell did not finish within {CELL_TIMEOUT_S:g}s and may still be running",
|
|
305
|
-
"stack": ["[cell timed out]"],
|
|
306
|
-
}
|
|
307
|
-
_send({"type": "done", "cellId": cell_id, "status": "error", "error": tmsg})
|
|
308
|
-
elif error:
|
|
309
|
-
_send({"type": "done", "cellId": cell_id, "status": "error", "error": _line_error(error)})
|
|
310
|
-
else:
|
|
311
|
-
_send({"type": "done", "cellId": cell_id, "status": "ok", "result": result})
|
|
312
|
-
# --- a single-threaded guest can't read 'abort' mid-cell; the host discards+rebuilds ---
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
if __name__ == "__main__":
|
|
316
|
-
try:
|
|
317
|
-
main()
|
|
318
|
-
except Exception as e:
|
|
319
|
-
_send({"type": "done", "cellId": "", "status": "error", "error": _line_error(str(e))})
|
|
320
|
-
sys.exit(1)
|