pi-repl-py 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -20
- package/docs/ARCHITECTURE.md +182 -0
- package/docs/how-to-functions.md +82 -79
- package/docs/philosophy.md +67 -59
- package/index.ts +8 -31
- package/package.json +12 -6
- package/scripts/setup-venv.mjs +41 -19
- package/src/engine/index.ts +92 -433
- package/src/engine/kernel.ts +598 -0
- package/src/engine/session.ts +149 -0
- package/src/engine/zmtp.ts +251 -0
- package/src/extension/helpers.ts +46 -0
- package/src/extension/preview/candidates.ts +159 -0
- package/src/extension/preview/descriptor.ts +28 -0
- package/src/extension/preview/index.ts +31 -0
- package/src/extension/preview/scan.ts +59 -0
- package/src/extension/preview/shell.ts +156 -0
- package/src/extension/preview/types.ts +23 -0
- package/src/extension/prompt.ts +76 -0
- package/src/extension/render-core.ts +11 -31
- package/src/extension/render.ts +2 -11
- package/src/extension/session-engine.ts +9 -31
- package/src/extension/tool-meta.ts +11 -54
- package/ARCHITECTURE.md +0 -141
- package/src/engine/guest.py +0 -317
- package/src/engine/protocol.ts +0 -66
- package/src/engine/toolbox/bash.py +0 -72
- package/src/engine/toolbox/edit.py +0 -37
- package/src/engine/toolbox/read.py +0 -26
- package/src/engine/toolbox/write.py +0 -23
- package/src/extension/config.ts +0 -65
- package/src/extension/preview-core.ts +0 -518
- package/src/extension/toolbox.ts +0 -74
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// --- Jupyter messaging over ZMTP: [identities] <IDS|MSG> [sig, h, p, m, c] ---
|
|
2
|
+
// ids are empty for a client's own channels (kernel ROUTER strips them);
|
|
3
|
+
// sig = hex(HMAC-SHA256(key, h||p||m||c)) over the exact bytes; key from the
|
|
4
|
+
// connection file. Checked against jupyter_client's session.py.
|
|
5
|
+
|
|
6
|
+
import { createHmac, randomUUID } from "node:crypto";
|
|
7
|
+
import { readFileSync } from "node:fs";
|
|
8
|
+
|
|
9
|
+
/** The kernel's connection file: ip/ports/key, written by ipykernel at boot. */
|
|
10
|
+
export interface ConnectionFile {
|
|
11
|
+
ip: string;
|
|
12
|
+
transport: "tcp" | "ipc";
|
|
13
|
+
shell_port: number;
|
|
14
|
+
iopub_port: number;
|
|
15
|
+
stdin_port: number;
|
|
16
|
+
control_port: number;
|
|
17
|
+
hb_port: number;
|
|
18
|
+
key: string;
|
|
19
|
+
signature_scheme: string;
|
|
20
|
+
kernel_name?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function readConnectionFile(path: string): ConnectionFile {
|
|
24
|
+
return JSON.parse(readFileSync(path, "utf8")) as ConnectionFile;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const DELIM = Buffer.from("<IDS|MSG>");
|
|
28
|
+
/** The protocol version ipykernel 7 advertises; we send the same on our own headers. */
|
|
29
|
+
const PROTOCOL_VERSION = "5.3";
|
|
30
|
+
|
|
31
|
+
export interface JupyterHeader {
|
|
32
|
+
msg_id: string;
|
|
33
|
+
msg_type: string;
|
|
34
|
+
username: string;
|
|
35
|
+
session: string;
|
|
36
|
+
date: string;
|
|
37
|
+
version: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** A parsed inbound message: JSON parts + whether the signature verified. */
|
|
41
|
+
export interface ParsedMessage {
|
|
42
|
+
msg_id: string;
|
|
43
|
+
msg_type: string;
|
|
44
|
+
header: JupyterHeader;
|
|
45
|
+
parent: Record<string, unknown>;
|
|
46
|
+
metadata: Record<string, unknown>;
|
|
47
|
+
content: Record<string, unknown>;
|
|
48
|
+
signatureOk: boolean;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function pack(obj: unknown): Buffer {
|
|
52
|
+
return Buffer.from(JSON.stringify(obj));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** One client-side session: mints ids, signs and frames outbound messages. */
|
|
56
|
+
export class JupyterSession {
|
|
57
|
+
readonly sessionId: string;
|
|
58
|
+
readonly username: string;
|
|
59
|
+
private counter = 0;
|
|
60
|
+
private readonly key: Buffer;
|
|
61
|
+
|
|
62
|
+
constructor(opts: { key: string; sessionId?: string; username?: string }) {
|
|
63
|
+
this.key = Buffer.from(opts.key, "utf8");
|
|
64
|
+
this.sessionId = opts.sessionId ?? randomUUID();
|
|
65
|
+
this.username = opts.username ?? "pi-repl";
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
nextMsgId(): string {
|
|
69
|
+
// --- ids only need uniqueness; a monotone counter over a session id keeps them short ---
|
|
70
|
+
return `${this.sessionId}_${process.pid}_${this.counter++}`;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private sign(parts: Buffer[]): Buffer {
|
|
74
|
+
if (this.key.length === 0) return Buffer.alloc(0);
|
|
75
|
+
const hmac = createHmac("sha256", this.key);
|
|
76
|
+
for (const part of parts) hmac.update(part);
|
|
77
|
+
return Buffer.from(hmac.digest("hex"), "ascii");
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Build the wire frames for an outbound message: [DELIM, sig, h, p, m, c]. */
|
|
81
|
+
buildFrames(
|
|
82
|
+
msgType: string,
|
|
83
|
+
content: Record<string, unknown>,
|
|
84
|
+
parent?: JupyterHeader | null,
|
|
85
|
+
msgId?: string,
|
|
86
|
+
): Buffer[] {
|
|
87
|
+
const header: JupyterHeader = {
|
|
88
|
+
msg_id: msgId ?? this.nextMsgId(),
|
|
89
|
+
msg_type: msgType,
|
|
90
|
+
username: this.username,
|
|
91
|
+
session: this.sessionId,
|
|
92
|
+
date: new Date().toISOString(),
|
|
93
|
+
version: PROTOCOL_VERSION,
|
|
94
|
+
};
|
|
95
|
+
const h = pack(header);
|
|
96
|
+
const p = pack(parent ?? {});
|
|
97
|
+
const m = pack({});
|
|
98
|
+
const c = pack(content);
|
|
99
|
+
const signature = this.sign([h, p, m, c]);
|
|
100
|
+
return [DELIM, signature, h, p, m, c];
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Parse an inbound multipart message (identities stripped by ZMTP); null if malformed. */
|
|
104
|
+
parseMessage(frames: Buffer[]): ParsedMessage | null {
|
|
105
|
+
// --- indexOf uses ===; frames are distinct Buffers, so match by value ---
|
|
106
|
+
const delimIdx = frames.findIndex((f) => f.equals(DELIM));
|
|
107
|
+
if (delimIdx < 0) return null;
|
|
108
|
+
const rest = frames.slice(delimIdx + 1);
|
|
109
|
+
if (rest.length < 5) return null;
|
|
110
|
+
const [signature, h, p, m, c] = rest;
|
|
111
|
+
const expected = this.sign([h, p, m, c]);
|
|
112
|
+
const signatureOk = this.key.length === 0 || signature.equals(expected);
|
|
113
|
+
try {
|
|
114
|
+
const header = JSON.parse(h.toString("utf8")) as JupyterHeader;
|
|
115
|
+
const content = JSON.parse(c.toString("utf8")) as Record<string, unknown>;
|
|
116
|
+
const metadata = JSON.parse(m.toString("utf8")) as Record<string, unknown>;
|
|
117
|
+
const parent = JSON.parse(p.toString("utf8")) as Record<string, unknown>;
|
|
118
|
+
return { msg_id: header.msg_id, msg_type: header.msg_type, header, parent, metadata, content, signatureOk };
|
|
119
|
+
} catch {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function executeRequest(code: string, silent: boolean): Record<string, unknown> {
|
|
126
|
+
return {
|
|
127
|
+
code,
|
|
128
|
+
silent,
|
|
129
|
+
store_history: !silent,
|
|
130
|
+
user_expressions: {},
|
|
131
|
+
allow_stdin: false,
|
|
132
|
+
stop_on_error: true,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** A payload the kernel publishes back to us with a private MIME key. */
|
|
137
|
+
export const SNAPSHOT_MIME = "application/vnd.pi-repl.snapshot+json";
|
|
138
|
+
export const RESTORE_MIME = "application/vnd.pi-repl.restore+json";
|
|
139
|
+
export const NAMES_MIME = "application/vnd.pi-repl.names+json";
|
|
140
|
+
|
|
141
|
+
/** Read a private-MIME payload out of an execute_result/display_data content. */
|
|
142
|
+
export function readPayload(content: Record<string, unknown>, mime: string): string | null {
|
|
143
|
+
const data = content.data;
|
|
144
|
+
if (data && typeof data === "object") {
|
|
145
|
+
const value = (data as Record<string, unknown>)[mime];
|
|
146
|
+
if (typeof value === "string") return value;
|
|
147
|
+
}
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
// --- ZMTP 3.0 wire protocol, by hand (bun can't load libzmq's bindings). ---
|
|
2
|
+
// DEALER for shell/control, SUB for iopub; greeting 0xff..0x7f + READY each side.
|
|
3
|
+
|
|
4
|
+
import { connect, type Socket } from "node:net";
|
|
5
|
+
|
|
6
|
+
const GREETING_SIGNATURE = Buffer.from([0xff, 0, 0, 0, 0, 0, 0, 0, 0x01, 0x7f]);
|
|
7
|
+
const NULL_MECHANISM = Buffer.concat([Buffer.from("NULL"), Buffer.alloc(16)]);
|
|
8
|
+
|
|
9
|
+
/** The client (non-server) half of the 64-byte ZMTP 3.0 greeting. */
|
|
10
|
+
function buildGreeting(): Buffer {
|
|
11
|
+
return Buffer.concat([
|
|
12
|
+
GREETING_SIGNATURE,
|
|
13
|
+
Buffer.from([3, 0]), // version 3.0
|
|
14
|
+
NULL_MECHANISM,
|
|
15
|
+
Buffer.from([0]), // as-server: we are the connecting socket
|
|
16
|
+
Buffer.alloc(31), // filler
|
|
17
|
+
]);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const FRAME_MORE = 0x01;
|
|
21
|
+
const FRAME_LONG = 0x02;
|
|
22
|
+
const GREETING_LENGTH = 64;
|
|
23
|
+
|
|
24
|
+
/** Serialize one ZMTP frame: flags byte, short/8-byte length, body. */
|
|
25
|
+
export function encodeFrame(body: Uint8Array, more: boolean): Buffer {
|
|
26
|
+
const flags = more ? FRAME_MORE : 0;
|
|
27
|
+
if (body.length <= 255) {
|
|
28
|
+
const out = Buffer.allocUnsafe(2 + body.length);
|
|
29
|
+
out[0] = flags;
|
|
30
|
+
out[1] = body.length;
|
|
31
|
+
Buffer.from(body).copy(out, 2);
|
|
32
|
+
return out;
|
|
33
|
+
}
|
|
34
|
+
const out = Buffer.allocUnsafe(9 + body.length);
|
|
35
|
+
out[0] = flags | FRAME_LONG;
|
|
36
|
+
out.writeUInt32BE(0, 1); // length is 64-bit; we never exceed 2^32
|
|
37
|
+
out.writeUInt32BE(body.length, 5);
|
|
38
|
+
Buffer.from(body).copy(out, 9);
|
|
39
|
+
return out;
|
|
40
|
+
}
|
|
41
|
+
/** Incremental parser: `current` persists across feed() and accumulation is once-per-frame (avoid O(n²)). */
|
|
42
|
+
export class ZmtpFrameParser {
|
|
43
|
+
private chunks: Buffer[] = [];
|
|
44
|
+
private total = 0;
|
|
45
|
+
private current: Buffer[] = []; // frames of the in-progress message
|
|
46
|
+
|
|
47
|
+
/** @returns one or more complete messages consumed from `chunk`. */
|
|
48
|
+
feed(chunk: Uint8Array): Buffer[][] {
|
|
49
|
+
if (chunk.length > 0) {
|
|
50
|
+
this.chunks.push(Buffer.from(chunk));
|
|
51
|
+
this.total += chunk.length;
|
|
52
|
+
}
|
|
53
|
+
const messages: Buffer[][] = [];
|
|
54
|
+
for (;;) {
|
|
55
|
+
if (this.total < 1) break;
|
|
56
|
+
const flags = this.peekBytes(1)[0];
|
|
57
|
+
const long = (flags & FRAME_LONG) !== 0;
|
|
58
|
+
const headerLen = long ? 9 : 2;
|
|
59
|
+
if (this.total < headerLen) break;
|
|
60
|
+
const header = this.peekBytes(headerLen);
|
|
61
|
+
const length = long ? header.readUInt32BE(5) : header[1];
|
|
62
|
+
if (this.total < headerLen + length) break;
|
|
63
|
+
const frame = this.take(headerLen + length);
|
|
64
|
+
// `take` returns a subarray (or a fresh concat for multi-chunk frames); never mutated, so no copy
|
|
65
|
+
this.current.push(frame.subarray(headerLen));
|
|
66
|
+
if ((flags & FRAME_MORE) === 0) {
|
|
67
|
+
messages.push(this.current);
|
|
68
|
+
this.current = [];
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return messages;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The first `n` bytes across the chunk list, without consuming them. */
|
|
75
|
+
private peekBytes(n: number): Buffer {
|
|
76
|
+
if (this.chunks[0].length >= n) return this.chunks[0].subarray(0, n);
|
|
77
|
+
const parts: Buffer[] = [];
|
|
78
|
+
let need = n;
|
|
79
|
+
for (const c of this.chunks) {
|
|
80
|
+
const t = Math.min(c.length, need);
|
|
81
|
+
parts.push(c.subarray(0, t));
|
|
82
|
+
need -= t;
|
|
83
|
+
if (need === 0) break;
|
|
84
|
+
}
|
|
85
|
+
return Buffer.concat(parts);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Consume `n` bytes from the front of the chunk list. */
|
|
89
|
+
private take(n: number): Buffer {
|
|
90
|
+
const first = this.chunks[0];
|
|
91
|
+
if (first.length >= n) {
|
|
92
|
+
const out = first.subarray(0, n);
|
|
93
|
+
if (first.length === n) this.chunks.shift();
|
|
94
|
+
else this.chunks[0] = first.subarray(n);
|
|
95
|
+
this.total -= n;
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
const parts: Buffer[] = [];
|
|
99
|
+
let need = n;
|
|
100
|
+
for (const c of this.chunks) {
|
|
101
|
+
const t = Math.min(c.length, need);
|
|
102
|
+
parts.push(c.subarray(0, t));
|
|
103
|
+
need -= t;
|
|
104
|
+
if (need === 0) break;
|
|
105
|
+
}
|
|
106
|
+
let left = n;
|
|
107
|
+
while (left > 0) {
|
|
108
|
+
const c = this.chunks[0];
|
|
109
|
+
if (c.length <= left) {
|
|
110
|
+
this.chunks.shift();
|
|
111
|
+
left -= c.length;
|
|
112
|
+
} else {
|
|
113
|
+
this.chunks[0] = c.subarray(left);
|
|
114
|
+
left = 0;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
this.total -= n;
|
|
118
|
+
return Buffer.concat(parts);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Socket-type string carried in the READY metadata (ZMTP "Socket-Type"). */
|
|
123
|
+
export type ZmtpSocketType = "DEALER" | "SUB";
|
|
124
|
+
|
|
125
|
+
interface ReadReady {
|
|
126
|
+
resolve(): void;
|
|
127
|
+
reject(error: Error): void;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** One ZMTP client connection: TCP socket + greeting/READY handshake → complete multipart messages. */
|
|
131
|
+
export class ZmtpSocket {
|
|
132
|
+
private socket?: Socket;
|
|
133
|
+
private parser = new ZmtpFrameParser();
|
|
134
|
+
private readyResolve?: ReadReady;
|
|
135
|
+
private closed = false;
|
|
136
|
+
onMessage?: (frames: Buffer[]) => void;
|
|
137
|
+
onClose?: () => void;
|
|
138
|
+
|
|
139
|
+
private constructor(socket: Socket) {
|
|
140
|
+
this.socket = socket;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Connect to `host:port` and complete the ZMTP handshake for `socketType`. */
|
|
144
|
+
static connect(opts: { host: string; port: number; socketType: ZmtpSocketType }): Promise<ZmtpSocket> {
|
|
145
|
+
const socket = connect({ host: opts.host, port: opts.port });
|
|
146
|
+
const z = new ZmtpSocket(socket);
|
|
147
|
+
// --- the peer's 64-byte greeting is not frame-formatted; collect it before the parser sees bytes ---
|
|
148
|
+
let greeting = Buffer.alloc(0);
|
|
149
|
+
|
|
150
|
+
socket.on("data", (chunk) => {
|
|
151
|
+
if (greeting.length < GREETING_LENGTH) {
|
|
152
|
+
const take = Math.min(chunk.length, GREETING_LENGTH - greeting.length);
|
|
153
|
+
greeting = Buffer.concat([greeting, chunk.subarray(0, take)]);
|
|
154
|
+
chunk = chunk.subarray(take);
|
|
155
|
+
if (greeting.length === GREETING_LENGTH) {
|
|
156
|
+
const sig = greeting.subarray(0, GREETING_SIGNATURE.length);
|
|
157
|
+
if (!sig.equals(GREETING_SIGNATURE)) {
|
|
158
|
+
z.failHandshake(
|
|
159
|
+
new Error(`ZMTP peer at ${opts.host}:${opts.port} sent an unexpected signature (${sig.toString("hex")})`),
|
|
160
|
+
);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
// --- greeting done: announce our socket type, then await the peer's ---
|
|
164
|
+
z.send([buildReadyMetadata(opts.socketType)]);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (greeting.length === GREETING_LENGTH) {
|
|
168
|
+
for (const message of z.parser.feed(chunk)) {
|
|
169
|
+
const ready = z.readyResolve;
|
|
170
|
+
if (ready) {
|
|
171
|
+
// --- the first frame after the greeting is the peer's READY ---
|
|
172
|
+
z.readyResolve = undefined;
|
|
173
|
+
ready.resolve();
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
z.deliver(message);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
socket.on("error", (error) => {
|
|
181
|
+
if (z.readyResolve) {
|
|
182
|
+
z.failHandshake(new Error(`ZMTP connection to ${opts.host}:${opts.port} failed: ${error.message}`));
|
|
183
|
+
}
|
|
184
|
+
// --- node always follows an error with 'close', which handles teardown ---
|
|
185
|
+
});
|
|
186
|
+
socket.on("close", () => {
|
|
187
|
+
if (z.closed) return;
|
|
188
|
+
z.closed = true;
|
|
189
|
+
if (z.readyResolve) {
|
|
190
|
+
z.failHandshake(new Error(`ZMTP connection to ${opts.host}:${opts.port} closed during handshake`));
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
z.onClose?.();
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
return new Promise<ZmtpSocket>((resolve, reject) => {
|
|
197
|
+
z.readyResolve = { resolve: () => resolve(z), reject };
|
|
198
|
+
socket.on("connect", () => {
|
|
199
|
+
// --- full greeting in one write; the peer may split its reply ---
|
|
200
|
+
socket.write(buildGreeting());
|
|
201
|
+
});
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
private failHandshake(error: Error): void {
|
|
206
|
+
const ready = this.readyResolve;
|
|
207
|
+
if (!ready) return;
|
|
208
|
+
this.readyResolve = undefined;
|
|
209
|
+
ready.reject(error);
|
|
210
|
+
this.close();
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
private deliver(message: Buffer[]): void {
|
|
214
|
+
this.onMessage?.(message);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Send a multipart message (DEALER) or a single subscription frame (SUB). */
|
|
218
|
+
send(frames: Uint8Array[]): void {
|
|
219
|
+
for (let i = 0; i < frames.length; i++) {
|
|
220
|
+
this.socket?.write(encodeFrame(frames[i], i < frames.length - 1));
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** SUB only: subscribe to a topic prefix (empty = all traffic). */
|
|
225
|
+
subscribe(topic: Uint8Array): void {
|
|
226
|
+
this.send([Buffer.concat([Buffer.from([0x01]), topic])]);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
close(): void {
|
|
230
|
+
this.closed = true;
|
|
231
|
+
this.socket?.destroy();
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
get isClosed(): boolean {
|
|
235
|
+
return this.closed;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** The READY metadata frame: `\x05READY` + Socket-Type + Identity properties. */
|
|
240
|
+
function buildReadyMetadata(socketType: ZmtpSocketType): Buffer {
|
|
241
|
+
const type = Buffer.from(socketType);
|
|
242
|
+
const body = Buffer.concat([
|
|
243
|
+
Buffer.from("\x05READY"),
|
|
244
|
+
Buffer.from("\x0bSocket-Type"),
|
|
245
|
+
Buffer.from([0, 0, 0, type.length]),
|
|
246
|
+
type,
|
|
247
|
+
Buffer.from("\x08Identity"),
|
|
248
|
+
Buffer.from([0, 0, 0, 0]), // empty identity: the routing id lives in ZMTP, not Jupyter
|
|
249
|
+
]);
|
|
250
|
+
return body;
|
|
251
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/** Loads helpers from the ONE fixed dir; `helper_description` surfaces verbatim (no signature parsing). */
|
|
2
|
+
|
|
3
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
const DEFAULT_HELPERS_DIR = join(homedir(), ".pi", "agent", "pi-repl", "helpers");
|
|
8
|
+
|
|
9
|
+
interface HelperEntry {
|
|
10
|
+
name: string;
|
|
11
|
+
description: string; // full helper_description body, "" if absent
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Extract `helper_description = """..."""` (or `'''`) verbatim; no signature parsing. */
|
|
15
|
+
function parseDescription(source: string): string {
|
|
16
|
+
const m = source.match(/helper_description\s*=\s*("""|''')([\s\S]*?)\1/);
|
|
17
|
+
return m ? m[2].trim() : "";
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Load {name → entry} for each non-underscore *.py in the helpers dir. */
|
|
21
|
+
function loadHelperEntries(dir?: string): HelperEntry[] {
|
|
22
|
+
const d = dir ?? DEFAULT_HELPERS_DIR;
|
|
23
|
+
if (!existsSync(d)) return [];
|
|
24
|
+
const entries: HelperEntry[] = [];
|
|
25
|
+
for (const file of readdirSync(d).sort()) {
|
|
26
|
+
if (!file.endsWith(".py")) continue;
|
|
27
|
+
const name = file.slice(0, -3);
|
|
28
|
+
if (!/^[A-Za-z_]\w*$/.test(name)) continue;
|
|
29
|
+
// --- underscore-prefixed files are neither loaded nor advertised ---
|
|
30
|
+
if (name.startsWith("_")) continue;
|
|
31
|
+
try {
|
|
32
|
+
const source = readFileSync(join(d, file), "utf8");
|
|
33
|
+
entries.push({ name, description: parseDescription(source) });
|
|
34
|
+
} catch {}
|
|
35
|
+
}
|
|
36
|
+
return entries;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** The prompt-facing list, one bullet per loaded file (verbatim description, or an introspection pointer). */
|
|
40
|
+
export function buildHelpersMap(dir?: string): string[] {
|
|
41
|
+
return loadHelperEntries(dir).map((t) =>
|
|
42
|
+
t.description
|
|
43
|
+
? `- ${t.description.replace(/\n/g, "\n ")}`
|
|
44
|
+
: `- ${t.name} (no description — inspect it with print(${t.name}.__doc__))`,
|
|
45
|
+
);
|
|
46
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// --- candidates: the five detectors that name a cell's intent, plus generic scoring ---
|
|
2
|
+
|
|
3
|
+
import { descriptor } from "./descriptor.js";
|
|
4
|
+
import { maskSpan, scanTemplate, substituteVars } from "./scan.js";
|
|
5
|
+
import { previewShellCommand, previewShellCommandScored, SHELL_SETUP_WORDS, shellWords } from "./shell.js";
|
|
6
|
+
import { BACKTICK, type Candidate } from "./types.js";
|
|
7
|
+
|
|
8
|
+
const SHELL_OPEN_PATTERN = new RegExp(`Bun\\s*\\.\\s*\\$\\s*(?:\\([^)]*\\)\\s*)?${BACKTICK}`, "g");
|
|
9
|
+
|
|
10
|
+
export function shellCandidates(
|
|
11
|
+
source: string,
|
|
12
|
+
vars: ReadonlyMap<string, string>,
|
|
13
|
+
): { candidates: Candidate[]; masked: string } {
|
|
14
|
+
const candidates: Candidate[] = [];
|
|
15
|
+
let masked = source;
|
|
16
|
+
SHELL_OPEN_PATTERN.lastIndex = 0;
|
|
17
|
+
let match = SHELL_OPEN_PATTERN.exec(masked);
|
|
18
|
+
while (match) {
|
|
19
|
+
const span = scanTemplate(masked, match.index + match[0].length - 1);
|
|
20
|
+
const command = previewShellCommandScored(substituteVars(span.body, vars));
|
|
21
|
+
// --- the command's own strength breaks ties; setup-only drops lower ---
|
|
22
|
+
if (command.text) {
|
|
23
|
+
const setupOnly = SHELL_SETUP_WORDS.has(shellWords(command.text)[0] ?? "");
|
|
24
|
+
const score = setupOnly ? 72 : 90 + Math.min(command.strength, 200) / 25;
|
|
25
|
+
candidates.push({ kind: "shell", text: command.text, score });
|
|
26
|
+
}
|
|
27
|
+
masked = maskSpan(masked, span);
|
|
28
|
+
SHELL_OPEN_PATTERN.lastIndex = span.end;
|
|
29
|
+
match = SHELL_OPEN_PATTERN.exec(masked);
|
|
30
|
+
}
|
|
31
|
+
return { candidates, masked };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const FILE_EFFECT_PATTERN =
|
|
35
|
+
/(?:Bun\.write|\b(?:fs|fsp|promises)\.(?:writeFileSync|writeFile|appendFileSync|appendFile|mkdirSync|mkdir|rmSync|rmdirSync|unlinkSync|unlink|renameSync|rename|copyFileSync|copyFile|cpSync|cp)|\b(?:writeFileSync|writeFile|appendFileSync|mkdirSync|rmSync|unlinkSync|renameSync|copyFileSync))\s*\(\s*([^,)\n]+)/g;
|
|
36
|
+
|
|
37
|
+
const FILE_EFFECT_VERBS: ReadonlyArray<[string, string]> = [
|
|
38
|
+
["Bun.write", "write"],
|
|
39
|
+
["writeFileSync", "write"],
|
|
40
|
+
["writeFile", "write"],
|
|
41
|
+
["appendFileSync", "append"],
|
|
42
|
+
["appendFile", "append"],
|
|
43
|
+
["mkdirSync", "mkdir"],
|
|
44
|
+
["mkdir", "mkdir"],
|
|
45
|
+
["rmdirSync", "delete"],
|
|
46
|
+
["rmSync", "delete"],
|
|
47
|
+
["rm", "delete"],
|
|
48
|
+
["unlinkSync", "delete"],
|
|
49
|
+
["unlink", "delete"],
|
|
50
|
+
["renameSync", "rename"],
|
|
51
|
+
["rename", "rename"],
|
|
52
|
+
["copyFileSync", "copy"],
|
|
53
|
+
["copyFile", "copy"],
|
|
54
|
+
["cpSync", "copy"],
|
|
55
|
+
["cp", "copy"],
|
|
56
|
+
];
|
|
57
|
+
|
|
58
|
+
// --- resolve a quoted literal, a known const, or an interpolated template into a plain string ---
|
|
59
|
+
function resolveArgText(arg: string, vars: ReadonlyMap<string, string>): string | undefined {
|
|
60
|
+
const trimmed = arg.trim();
|
|
61
|
+
const literalPattern = new RegExp(`^["'${BACKTICK}]([^"'${BACKTICK}]*)["'${BACKTICK}]$`);
|
|
62
|
+
const literal = trimmed.match(literalPattern);
|
|
63
|
+
if (literal?.[1]) return literal[1];
|
|
64
|
+
if (/^[A-Za-z_$][\w$]*$/.test(trimmed)) return vars.get(trimmed);
|
|
65
|
+
if (trimmed.startsWith(BACKTICK)) return substituteVars(trimmed.slice(1, -1), vars);
|
|
66
|
+
return undefined;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const FILE_READ_PATTERN = /Bun\.file\s*\(\s*([^,)\n]+?)\s*\)\s*\.\s*(?:text|json|arrayBuffer|bytes|stream)\s*\(/g;
|
|
70
|
+
|
|
71
|
+
export function fileCandidates(source: string, vars: ReadonlyMap<string, string>): Candidate[] {
|
|
72
|
+
const candidates: Candidate[] = [];
|
|
73
|
+
for (const match of source.matchAll(FILE_EFFECT_PATTERN)) {
|
|
74
|
+
const call = match[0];
|
|
75
|
+
const verb = FILE_EFFECT_VERBS.find(([name]) => call.includes(name))?.[1];
|
|
76
|
+
if (!verb) continue;
|
|
77
|
+
const path = resolveArgText(match[1] ?? "", vars);
|
|
78
|
+
if (path) candidates.push({ kind: "ts", text: descriptor(`${verb} ${path}`), score: 95 });
|
|
79
|
+
}
|
|
80
|
+
for (const match of source.matchAll(FILE_READ_PATTERN)) {
|
|
81
|
+
const path = resolveArgText(match[1] ?? "", vars);
|
|
82
|
+
if (path) candidates.push({ kind: "ts", text: descriptor(`read ${path}`), score: 70 });
|
|
83
|
+
}
|
|
84
|
+
for (const match of source.matchAll(/\bfetch\s*\(\s*([^,)\n]+)/g)) {
|
|
85
|
+
const url = resolveArgText(match[1] ?? "", vars);
|
|
86
|
+
if (url) candidates.push({ kind: "ts", text: descriptor(`fetch ${url}`), score: 75 });
|
|
87
|
+
}
|
|
88
|
+
return candidates;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// --- per-tool: which arg names the target, the verb shown, and its scoring band ---
|
|
92
|
+
const BRIDGED_TOOLS: Record<string, { arg: string; verb: string; score: number }> = {
|
|
93
|
+
read: { arg: "path", verb: "read", score: 70 },
|
|
94
|
+
bash: { arg: "command", verb: "", score: 88 },
|
|
95
|
+
edit: { arg: "path", verb: "edit", score: 95 },
|
|
96
|
+
write: { arg: "path", verb: "write", score: 95 },
|
|
97
|
+
grep: { arg: "pattern", verb: "grep", score: 68 },
|
|
98
|
+
find: { arg: "pattern", verb: "find", score: 68 },
|
|
99
|
+
ls: { arg: "path", verb: "ls", score: 68 },
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
export function bridgedToolCandidates(source: string, vars: ReadonlyMap<string, string>): Candidate[] {
|
|
103
|
+
const candidates: Candidate[] = [];
|
|
104
|
+
for (const match of source.matchAll(/\btools\.(\w+)\s*\(\s*\{([^}]*)\}/g)) {
|
|
105
|
+
const spec = BRIDGED_TOOLS[match[1] ?? ""];
|
|
106
|
+
if (!spec) continue;
|
|
107
|
+
const props = match[2] ?? "";
|
|
108
|
+
const argMatch = props.match(new RegExp(`${spec.arg}\\s*:\\s*([^,}]+)`));
|
|
109
|
+
const target = argMatch ? resolveArgText(argMatch[1] ?? "", vars) : undefined;
|
|
110
|
+
if (!target) continue;
|
|
111
|
+
// --- a bridged bash call is a command like any other ---
|
|
112
|
+
const text = spec.verb ? `${spec.verb} ${target}` : previewShellCommand(target) || target;
|
|
113
|
+
candidates.push({ kind: "ts", text: descriptor(text), score: spec.score });
|
|
114
|
+
}
|
|
115
|
+
return candidates;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const SKIP_LINE_PATTERN = /^(?:$|\/\/|\/\*|\*|import\s|export\s+(?:type\s|\{)|[})\];,]+$)/;
|
|
119
|
+
const DEFINITION_PATTERN = /^(?:export\s+)?(?:async\s+)?(?:function\s|class\s|interface\s|type\s+\w+\s*=)/;
|
|
120
|
+
const ARROW_DEFINITION_PATTERN = /^(?:const|let)\s+[A-Za-z_$][\w$]*\s*=\s*(?:async\s*)?\(?[^)=]*\)?\s*=>/;
|
|
121
|
+
const CONTROL_PATTERN = /^(?:if|for|while|switch|try|do)\b/;
|
|
122
|
+
const CALL_STATEMENT_PATTERN = /^(?:await\s+)?[A-Za-z_$][\w$.]*\s*\(/;
|
|
123
|
+
const ASSIGNMENT_CALL_PATTERN = /^(?:const|let|var)\s+[^=]{1,60}=\s*(?:await\s+)?(?:new\s+)?[A-Za-z_$][\w$.]*\s*\(/;
|
|
124
|
+
const LOW_SIGNAL_CALL_PATTERN =
|
|
125
|
+
/^(?:await\s+)?(?:console\.\w+|String|Number|Boolean|JSON\.stringify|JSON\.parse|structuredClone)\s*\(/;
|
|
126
|
+
const LOW_SIGNAL_ASSIGNMENT_PATTERN =
|
|
127
|
+
/=\s*(?:await\s+)?(?:JSON\.parse|JSON\.stringify|String|Number|Boolean|Object\.keys|Object\.entries)\s*\(/;
|
|
128
|
+
|
|
129
|
+
function consoleInnerCall(line: string): string | undefined {
|
|
130
|
+
const inner = line.match(/^console\.\w+\(\s*(.+)\)\s*;?\s*$/)?.[1]?.trim();
|
|
131
|
+
return inner && CALL_STATEMENT_PATTERN.test(inner) && !LOW_SIGNAL_CALL_PATTERN.test(inner) ? inner : undefined;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function genericLineScore(line: string): number {
|
|
135
|
+
if (SKIP_LINE_PATTERN.test(line)) return -1;
|
|
136
|
+
if (LOW_SIGNAL_ASSIGNMENT_PATTERN.test(line)) return 25;
|
|
137
|
+
if (consoleInnerCall(line)) return 55;
|
|
138
|
+
if (LOW_SIGNAL_CALL_PATTERN.test(line)) return 15;
|
|
139
|
+
if (DEFINITION_PATTERN.test(line) || ARROW_DEFINITION_PATTERN.test(line)) return 50;
|
|
140
|
+
if (CONTROL_PATTERN.test(line)) return 20;
|
|
141
|
+
if (/^(?:return|throw)\b/.test(line)) return 45;
|
|
142
|
+
if (ASSIGNMENT_CALL_PATTERN.test(line)) return 60;
|
|
143
|
+
if (CALL_STATEMENT_PATTERN.test(line)) return 65;
|
|
144
|
+
if (/^(?:const|let|var)\s/.test(line)) return 22;
|
|
145
|
+
return 30;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function genericCandidates(masked: string): Candidate[] {
|
|
149
|
+
const candidates: Candidate[] = [];
|
|
150
|
+
for (const [index, rawLine] of masked.split("\n").entries()) {
|
|
151
|
+
const line = rawLine.trim();
|
|
152
|
+
const score = genericLineScore(line);
|
|
153
|
+
if (score < 0) continue;
|
|
154
|
+
const text = consoleInnerCall(line) ?? line;
|
|
155
|
+
// --- later lines win ties: cells read as setup-then-act, and the act is the story ---
|
|
156
|
+
candidates.push({ kind: "ts", text: descriptor(text), score: score + Math.min(index, 90) / 100 });
|
|
157
|
+
}
|
|
158
|
+
return candidates;
|
|
159
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// --- descriptor: collapse the raw line into one readable, safe, width-capped string ---
|
|
2
|
+
const DESCRIPTOR_MAX_WIDTH = 64;
|
|
3
|
+
|
|
4
|
+
function collapseWhitespace(text: string): string {
|
|
5
|
+
return text.replace(/\s+/g, " ").trim();
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
function truncateDescriptor(text: string): string {
|
|
9
|
+
if (text.length <= DESCRIPTOR_MAX_WIDTH) return text;
|
|
10
|
+
return `${text.slice(0, DESCRIPTOR_MAX_WIDTH - 1).trimEnd()}…`;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// --- strip blobs, secrets, and sk- keys before a line reaches the header ---
|
|
14
|
+
function redactNoise(text: string): string {
|
|
15
|
+
return text
|
|
16
|
+
.replace(/[A-Za-z0-9+/]{80,}={0,2}/g, "<blob>")
|
|
17
|
+
.replace(/\b((?=\w*(?:token|key|secret|password))[A-Za-z_]\w*)\s*[=:]\s*(["'])[^"']*\2/gi, "$1=<redacted>")
|
|
18
|
+
.replace(
|
|
19
|
+
/\b((?=\w*(?:token|key|secret|password))[A-Za-z_]\w*)\s*[=:]\s*(?!<redacted>)(?!["'])\S+/gi,
|
|
20
|
+
"$1=<redacted>",
|
|
21
|
+
)
|
|
22
|
+
.replace(/(["'])sk-[^"']+\1/g, "$1<redacted>$1")
|
|
23
|
+
.replace(/(["']).{160,}\1/g, "$1…$1");
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function descriptor(text: string): string {
|
|
27
|
+
return truncateDescriptor(collapseWhitespace(redactNoise(text)));
|
|
28
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// --- preview entry: score the whole cell for its one truthful line ---
|
|
2
|
+
|
|
3
|
+
import { bridgedToolCandidates, fileCandidates, genericCandidates, shellCandidates } from "./candidates.js";
|
|
4
|
+
import { descriptor } from "./descriptor.js";
|
|
5
|
+
import { stringConsts } from "./scan.js";
|
|
6
|
+
import { previewShellCommand } from "./shell.js";
|
|
7
|
+
import type { CellPreview } from "./types.js";
|
|
8
|
+
|
|
9
|
+
export type { CellPreview };
|
|
10
|
+
export { descriptor, previewShellCommand };
|
|
11
|
+
|
|
12
|
+
export function previewCell(code: string): CellPreview {
|
|
13
|
+
const source = code.trimEnd();
|
|
14
|
+
if (!source) return { kind: "ts", text: "" };
|
|
15
|
+
const vars = stringConsts(source);
|
|
16
|
+
|
|
17
|
+
// --- scan order: shell masks shell-looking syntax, then file/tool/generic ---
|
|
18
|
+
const shell = shellCandidates(source, vars);
|
|
19
|
+
const candidates = [
|
|
20
|
+
...shell.candidates,
|
|
21
|
+
...fileCandidates(shell.masked, vars),
|
|
22
|
+
...bridgedToolCandidates(shell.masked, vars),
|
|
23
|
+
...genericCandidates(shell.masked),
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
let best: { kind: CellPreview["kind"]; text: string; score: number } | undefined;
|
|
27
|
+
for (const candidate of candidates) {
|
|
28
|
+
if (candidate.text && (!best || candidate.score > best.score)) best = candidate;
|
|
29
|
+
}
|
|
30
|
+
return best ?? { kind: "ts", text: "" };
|
|
31
|
+
}
|