pi-repl-py 0.1.1 → 0.2.1

package/src/engine/guest.py

@@ -1,320 +0,0 @@
-"""
-guest.py — the real IPython kernel guest evaluator for pi-repl.
-
-The host spawns this once. It starts a local ipykernel subprocess via
-jupyter_client, keeps it for the session, and bridges the wire protocol to it
-(stdin = commands, fd 3 = results). State survives because the kernel process
-does. Frames carry a nonce the host mints and the guest erases, so agent code
-cannot forge protocol traffic.
-"""
-
-from __future__ import annotations
-
-import json
-import os
-import sys
-import time
-
-# --- protocol envelope ---
-ENVELOPE_KEY = "__rlm"
-NONCE_ENV = "PI_RLM_NONCE"
-PROTOCOL_FD = 3
-
-NONCE = os.environ.get(NONCE_ENV, "")
-os.environ.pop(NONCE_ENV, None)
-
-# --- per-cell timeout: 0 = no cap; else a silence watchdog (no output for N seconds) ---
-CELL_TIMEOUT_S = float(os.environ.get("PI_REPL_TIMEOUT_MS", "0") or "0") / 1000.0
-
-# --- snapshot/restore use a fixed window, not the cell silence timer ---
-SNAPSHOT_TIMEOUT_S = 90.0
-
-# --- cap buffered output so a runaway print can't grow guest memory or send one giant frame ---
-MAX_CELL_OUTPUT_CHARS = 1_000_000
-
-# --- fd 3 protocol writer; dup'd so we don't close the caller's fd 3 on exit ---
-_proto = os.fdopen(os.dup(PROTOCOL_FD), "w", buffering=1)
-
-
-def _send(msg):
-    envelope = {ENVELOPE_KEY: 1, **msg}
-    if NONCE:
-        envelope["n"] = NONCE
-    _proto.write(json.dumps(envelope) + "\n")
-    _proto.flush()
-
-
-def _decode(line):
-    if ENVELOPE_KEY not in line:
-        return None
-    try:
-        obj = json.loads(line)
-    except Exception:
-        return None
-    if obj.get(ENVELOPE_KEY) != 1 or not isinstance(obj.get("type"), str):
-        return None
-    if NONCE and obj.get("n") != NONCE:
-        return None
-    return obj
-
-
-# --- toolbox: one function per *.py, exec'd into every kernel (PI_TOOLBOX_DIR) ---
-
-def _toolbox_files(directory):
-    """Return {function_name: source} for each *.py in `directory`."""
-    if not directory:
-        return {}
-    d = os.path.expanduser(directory)
-    if not os.path.isdir(d):
-        return {}
-    names = {}
-    for entry in sorted(os.listdir(d)):
-        if not entry.endswith(".py"):
-            continue
-        name = entry[:-3]
-        if not name.isidentifier() or name.startswith("_"):
-            continue
-        try:
-            with open(os.path.join(d, entry), encoding="utf-8") as f:
-                names[name] = f.read()
-        except OSError:
-            continue
-    return names
-
-DEFAULT_TOOLBOX_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "toolbox")
-TOOLBOX_DIR = os.environ.get("PI_TOOLBOX_DIR", "").strip()
-# --- Merge: built-ins are supreme; a config toolboxDir adds others and overrides on name ---
-_TOOLBOX_SRC = _toolbox_files(DEFAULT_TOOLBOX_DIR)
-if TOOLBOX_DIR and os.path.expanduser(TOOLBOX_DIR) != DEFAULT_TOOLBOX_DIR:
-    _TOOLBOX_SRC.update(_toolbox_files(TOOLBOX_DIR))
-
-# --- help/ls are part of the evaluator, not the toolbox ---
-INTRINSIC = """
-# --- ls() filters IPython-injected names out of the tool list ---
-_RPL_LS_NOISE = {'exit', 'quit', 'get_ipython', 'open', 'display'}
-
-def ls():
-    return sorted(n for n in globals() if n not in _RPL_LS_NOISE and not n.startswith('_') and callable(globals()[n]))
-
-def help(name=None):
-    if name is None:
-        return ls()
-    fn = globals().get(name)
-    if fn is None or not callable(fn):
-        return f"no such function: {name!r}"
-    return fn.__doc__ or f"{name} (no docstring)"
-"""
-
-
-from jupyter_client import KernelManager
-
-
-class Kernel:
-    """A persistent subprocess ipykernel + blocking client."""
-
-    def __init__(self):
-        self.km = KernelManager(kernel_name="python3")
-        self.km.start_kernel()
-        self.kc = self.km.client()
-        self.kc.start_channels()
-        self.kc.wait_for_ready(timeout=30)
-        self._preload()
-
-    def _preload(self):
-        """Exec every toolbox function + the intrinsic help/ls into the kernel ns."""
-        code = INTRINSIC + "\n"
-        for src in _TOOLBOX_SRC.values():
-            code += src + "\n"
-        if code.strip():
-            self.kc.execute(code)
-            self._drain()
-
-    def _drain(self):
-        try:
-            while True:
-                m = self.kc.get_iopub_msg(timeout=1)
-                if m.get("msg_type") == "status" and m.get("content", {}).get("execution_state") == "idle":
-                    break
-        except Exception:
-            pass
-
-    def _drain_execution(self, code, timeout):
-        """Run `code`; return (stdout, stderr, error_text, result, timed_out).
-
-        `timeout <= 0` means "no cap": a cell runs until it reports idle.
-        `timeout > 0` is a SILENCE watchdog — it trips only once the cell has
-        produced no message for `timeout` seconds. A silent-but-running command
-        (e.g. `find ... | sort`) is allowed to complete; a stalled one (dead
-        kernel, or nothing for the silence window) reports `timed_out=True` so
-        the caller can surface a real hang instead of faking success.
-        """
-        msg_id = self.kc.execute(code)
-        out, err, error, result = [], [], None, None
-        out_len, err_len = 0, 0
-        timed_out = False
-        # --- silence clock only starts once the cell begins (grace for a fresh kernel) ---
-        last_activity: float | None = None
-        while True:
-            if not self.km.is_alive():
-                timed_out = True
-                break
-            if last_activity is not None and timeout and (time.monotonic() - last_activity) >= timeout:
-                timed_out = True
-                break
-            wait = (timeout - (time.monotonic() - last_activity)) if (last_activity is not None and timeout) else 0.25
-            try:
-                m = self.kc.get_iopub_msg(timeout=max(0.01, min(0.25, wait)))
-            except Exception:
-                continue
-            if m.get("parent_header", {}).get("msg_id") != msg_id:
-                continue
-            if last_activity is None:
-                last_activity = time.monotonic()
-            else:
-                last_activity = time.monotonic()
-            mt = m.get("msg_type")
-            c = m.get("content", {})
-            if mt == "stream":
-                is_out = c.get("name") == "stdout"
-                text = c.get("text", "") or ""
-                if is_out:
-                    if out_len < MAX_CELL_OUTPUT_CHARS:
-                        take = text[: MAX_CELL_OUTPUT_CHARS - out_len]
-                        out.append(take)
-                        out_len += len(take)
-                else:
-                    if err_len < MAX_CELL_OUTPUT_CHARS:
-                        take = text[: MAX_CELL_OUTPUT_CHARS - err_len]
-                        err.append(take)
-                        err_len += len(take)
-            elif mt == "execute_result":
-                result = c.get("data", {}).get("text/plain")
-            elif mt == "error":
-                error = "\n".join(c.get("traceback", ["(no traceback)"]))
-            elif mt == "status" and c.get("execution_state") == "idle":
-                break
-        if timed_out:
-            # --- best-effort cancel so the NEXT cell doesn't queue behind this one ---
-            try:
-                self.kc.interrupt_kernel()
-            except Exception:
-                pass
-        return "".join(out), "".join(err), error, result, timed_out
-
-    def execute(self, code):
-        """Idle-sync path used by snapshot/restore; not a user cell."""
-        return self._drain_execution(code, SNAPSHOT_TIMEOUT_S)[:4]
-
-    def run_cell(self, code):
-        """Run a user cell under the per-cell timeout, so the model learns
-        when work did not finish."""
-        return self._drain_execution(code, CELL_TIMEOUT_S)
-
-    def snapshot_globals(self):
-        # --- snapshot only user state (skip toolbox/intrinsic functions and _ names) ---
-        tool_names = sorted(set(_TOOLBOX_SRC) | {"ls", "help", "function_description"})
-        skip_names = json.dumps(tool_names)
-        out, _, _, _ = self.execute(
-            "import pickle as _pk, base64 as _b64, json as _js\n"
-            "__rlm_skip = set(" + skip_names + ") | {'In','Out','get_ipython','exit','quit','open'}\n"
-            "__rlm_v = {}\n__rlm_f = []\n"
-            "for _k, _v in list(globals().items()):\n"
-            "    # skip IPython bookkeeping and names with a leading underscore\n"
-            "    if _k.startswith('__') or _k.startswith('_') or _k in __rlm_skip:\n"
-            "        continue\n"
-            "    try:\n"
-            "        __rlm_v[_k] = _b64.b64encode(_pk.dumps(_v)).decode()\n"
-            "    except Exception as _e:\n"
-            "        __rlm_f.append({'name': _k, 'reason': str(_e)})\n"
-            "print('__RLC_SNAPSHOT__' + _js.dumps({'vars': __rlm_v, 'failed': __rlm_f}))\n"
-        )
-        marker = "__RLC_SNAPSHOT__"
-        if marker not in out:
-            # --- marker never printed: serialization stalled; report incomplete so the host keeps the last good file ---
-            return {}, [], False
-        try:
-            o = json.loads(out.split(marker)[-1])
-            return o.get("vars", {}), o.get("failed", []), True
-        except Exception:
-            return {}, [], False
-
-    def restore_globals(self, vars_):
-        if not vars_:
-            return [], []
-        # --- restore each variable in one atomic kernel call so one failure reports itself ---
-        code2 = (
-            "import pickle as _pk, base64 as _b64, json as _js\n"
-            "__rl_r = {'restored': [], 'failed': []}\n"
-            + "\n".join(
-                "try:\n"
-                f"    globals()[{name!r}] = _pk.loads(_b64.b64decode({b64!r}))\n"
-                f"    __rl_r['restored'].append({name!r})\n"
-                "except Exception as _e:\n"
-                f"    __rl_r['failed'].append({{'name': {name!r}, 'reason': str(_e)}})\n"
-                for name, b64 in vars_.items()
-            )
-            + "\nprint('__RLC_RESTORE__' + _js.dumps(__rl_r))"
-        )
-        out, _, _, _ = self.execute(code2)
-        marker = "__RLC_RESTORE__"
-        if marker not in out:
-            return [], []
-        try:
-            obj = json.loads(out.split(marker)[-1])
-            return obj.get("restored", []), obj.get("failed", [])
-        except Exception:
-            return [], []
-
-
-def _line_error(text):
-    lines = text.split("\n")
-    return {"name": "", "message": text, "stack": lines[:12]}
-
-
-def main():
-    kernel = Kernel()
-    _send({"type": "ready"})
-
-    for line in sys.stdin:
-        msg = _decode(line)
-        if not msg:
-            continue
-        t = msg["type"]
-        if t == "ping":
-            _send({"type": "pong", "id": msg["id"]})
-        elif t == "snapshot":
-            vars_, failed, complete = kernel.snapshot_globals()
-            _send({"type": "snapshot_result", "id": msg["id"], "vars": vars_, "failed": failed, "complete": complete})
-        elif t == "restore":
-            restored, failed = kernel.restore_globals(msg.get("vars", {}))
-            _send({"type": "restore_result", "id": msg["id"], "restored": restored, "failed": failed})
-        elif t == "list_names":
-            names = list(kernel.snapshot_globals()[0].keys())
-            _send({"type": "names_result", "id": msg["id"], "names": names})
-        elif t == "run":
-            cell_id = msg.get("cellId")
-            stdout, stderr, error, result, timed_out = kernel.run_cell(msg.get("code", ""))
-            if stdout:
-                _send({"type": "stream", "cellId": cell_id, "name": "stdout", "chunk": stdout})
-            if stderr:
-                _send({"type": "stream", "cellId": cell_id, "name": "stderr", "chunk": stderr})
-            if timed_out:
-                tmsg = {
-                    "name": "Timeout",
-                    "message": f"cell did not finish within {CELL_TIMEOUT_S:g}s and may still be running",
-                    "stack": ["[cell timed out]"],
-                }
-                _send({"type": "done", "cellId": cell_id, "status": "error", "error": tmsg})
-            elif error:
-                _send({"type": "done", "cellId": cell_id, "status": "error", "error": _line_error(error)})
-            else:
-                _send({"type": "done", "cellId": cell_id, "status": "ok", "result": result})
-        # --- a single-threaded guest can't read 'abort' mid-cell; the host discards+rebuilds ---
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except Exception as e:
-        _send({"type": "done", "cellId": "", "status": "error", "error": _line_error(str(e))})
-        sys.exit(1)

package/src/engine/protocol.ts

@@ -1,66 +0,0 @@
-// --- trust: fd3 is protocol-only (user output stays on stdout), and a cell can't forge frames (minted nonce) ---
-
-interface HostToGuest {
-	run: { type: "run"; cellId: string; code: string };
-	abort: { type: "abort"; cellId: string };
-	ping: { type: "ping"; id: string };
-	snapshot: { type: "snapshot"; id: string };
-	restore: { type: "restore"; id: string; vars: Record<string, string> };
-	list_names: { type: "list_names"; id: string };
-}
-
-export type HostToGuestMessage = HostToGuest[keyof HostToGuest];
-
-interface GuestToHost {
-	ready: { type: "ready" };
-	stream: { type: "stream"; cellId: string; name: "stdout" | "stderr"; chunk: string };
-	done: {
-		type: "done";
-		cellId: string;
-		status: "ok" | "error" | "aborted";
-		result?: string;
-		error?: { name: string; message: string; stack: string[] };
-	};
-	pong: { type: "pong"; id: string };
-	snapshot_result: {
-		type: "snapshot_result";
-		id: string;
-		vars: Record<string, string>;
-		failed: { name: string; reason: string }[];
-		/** False means the kernel didn't finish serializing; keep the last good file. */
-		complete?: boolean;
-	};
-	restore_result: {
-		type: "restore_result";
-		id: string;
-		restored: string[];
-		failed: { name: string; reason: string }[];
-	};
-	names_result: { type: "names_result"; id: string; names: string[] };
-}
-
-export type GuestToHostMessage = GuestToHost[keyof GuestToHost];
-
-const ENVELOPE_KEY = "__rlm";
-/** Env var carrying the per-process nonce to the guest. */
-export const NONCE_ENV = "PI_RLM_NONCE";
-/** Protocol pipe: guest → host. */
-export const PROTOCOL_FD = 3;
-
-export function encodeMessage(message: HostToGuestMessage | GuestToHostMessage, nonce?: string): string {
-	const envelope: Record<string, unknown> = { [ENVELOPE_KEY]: 1, ...message };
-	if (nonce) envelope.n = nonce;
-	return `${JSON.stringify(envelope)}\n`;
-}
-
-export function decodeMessage<T>(line: string, nonce?: string): T | null {
-	if (!line.trim()) return null;
-	try {
-		const parsed = JSON.parse(line);
-		if (parsed?.[ENVELOPE_KEY] !== 1 || typeof parsed.type !== "string") return null;
-		if (nonce && parsed.n !== nonce) return null;
-		return parsed as T;
-	} catch {
-		return null;
-	}
-}

package/src/engine/toolbox/bash.py

@@ -1,72 +0,0 @@
-function_description = """Run a shell command in a fresh subshell and return its result."""
-
-import os as _os
-import signal as _sig
-import subprocess as _sp
-
-
-def _kill_group(proc):
-    """Kill the whole process group of a child (its shell AND any grandchildren).
-
-    The shell's children inherit the fresh session's id, so they are the only
-    ones a timeout must reap; without this a `find | sort` that outlives the
-    call would keep chewing CPU long after bash() returned.
-    """
-    try:
-        _os.killpg(_os.getpgid(proc.pid), _sig.SIGKILL)
-    except Exception:
-        pass
-
-
-def bash(command, cwd=None, env=None, input=None, timeout=None):
-    """Run a shell command and return a CompletedProcess.
-
-    Argument notes:
-      command - the shell command string to run.
-      cwd     - optional directory to run it in; uses the evaluator's cwd if omitted.
-      env     - optional dict of environment variables merged into the current env.
-      input   - optional string to feed as stdin.
-      timeout - optional timeout in seconds; raises TimeoutExpired (after killing
-                the command's whole process group) if exceeded.
-
-    Result:
-      Returns subprocess.CompletedProcess. Read .stdout, .stderr, .returncode.
-      Example:  out = bash("git log --oneline"); print(out.returncode, out.stdout)
-
-    Behaviour:
-      - Runs via the shell, so pipes/&&/etc. work. Each call runs a FRESH
-        subshell: cd, export, and shell variables do NOT carry across calls.
-        Hold state in Python variables instead.
-      - The shell runs in its own process group, so a timeout kills the group —
-        no orphaned children keep running afterwards.
-      - `env` is merged into the current environment, not a replacement.
-
-    Environment:
-      This evaluator runs in a project-local Python venv, not the system
-      interpreter. A command that starts python/pip should target the same venv.
-    """
-    merged_env = dict(_os.environ)
-    if env:
-        merged_env.update(env)
-    with _sp.Popen(
-        command,
-        shell=True,
-        stdin=_sp.PIPE,
-        stdout=_sp.PIPE,
-        stderr=_sp.PIPE,
-        text=True,
-        cwd=cwd,
-        env=merged_env,
-        # --- new session => shell+children form one process group a timeout kills ---
-        start_new_session=_os.name == "posix",
-    ) as proc:
-        try:
-            stdout, stderr = proc.communicate(input=input, timeout=timeout)
-        except _sp.TimeoutExpired:
-            _kill_group(proc)
-            try:
-                proc.communicate()  # --- drain so no zombie is left ---
-            except Exception:
-                pass
-            raise
-    return _sp.CompletedProcess(args=command, returncode=proc.returncode, stdout=stdout, stderr=stderr)

package/src/engine/toolbox/edit.py

@@ -1,37 +0,0 @@
-function_description = """Replace old_text with new_text in a file; fails if old_text is not found exactly once."""
-
-
-def edit(path, old_text, new_text):
-    """Perform a targeted single replacement in a file.
-
-    Argument notes:
-      old_text - exact, unique text already in the file to replace.
-      new_text - text to substitute for it.
-
-    Behaviour:
-      - Requires old_text to appear EXACTLY once in the file. Zero or multiple
-        matches raise an error instead of guessing, so it can never silently
-        mangle a file it wasn't sure about.
-      - If it fails, the file is left untouched.
-
-    Environment:
-      This evaluator runs in a project-local Python venv, not the system
-      interpreter. For a package install that is ~/.pi/agent/pi-repl/venv; for a
-      repo checkout it is .venv/. The file edited is a real file on disk.
-    """
-    with open(path, encoding="utf-8") as f:
-        content = f.read()
-    count = content.count(old_text)
-    if count == 0:
-        raise ValueError(
-            f"edit: could not find the given old_text in {path} — it may have already been "
-            "applied or the file changed. Re-read the file and retry."
-        )
-    if count > 1:
-        raise ValueError(
-            f"edit: old_text occurs {count} times in {path} — make it more specific so it matches exactly once."
-        )
-    content = content.replace(old_text, new_text, 1)
-    with open(path, "w", encoding="utf-8") as f:
-        f.write(content)
-    return f"edited {path}"

package/src/engine/toolbox/read.py

@@ -1,26 +0,0 @@
-function_description = """Return the text of a file, optionally a slice of its lines."""
-
-
-def read(path, offset=1, limit=None):
-    """Read a file's UTF-8 text and return it, optionally a slice of its lines.
-
-    Argument notes:
-      offset - 1-based first line to return (default 1).
-      limit  - maximum number of lines to return (default: all of them).
-
-    Behaviour:
-      - Decoding errors are replaced instead of raising, so a binary-adjacent
-        file still returns most of its text.
-      - Keeps only what you ask for so a huge file can't flood context.
-
-    Environment:
-      This evaluator runs in a project-local Python venv, not the system
-      interpreter. For a package install that is ~/.pi/agent/pi-repl/venv; for a
-      repo checkout it is .venv/. python / pip on PATH may point elsewhere, so
-      do not assume the system python is what's running.
-    """
-    with open(path, encoding="utf-8", errors="replace") as f:
-        lines = f.readlines()
-    start = max(0, (offset or 1) - 1)
-    end = None if limit is None else start + limit
-    return "".join(lines[start:end])

package/src/engine/toolbox/write.py

@@ -1,23 +0,0 @@
-function_description = """Write content to a file, creating it or overwriting its entire contents."""
-
-
-def write(path, content):
-    """Write a file wholesale, creating it if missing or replacing its contents.
-
-    Argument notes:
-      content - string (or anything stringifiable) to write in full.
-
-    Behaviour:
-      - Unconditional: overwrites whatever is there. There is no size cap.
-      - Use edit() for a targeted change inside an existing file; write() is for
-        a new file or a full rewrite.
-
-    Environment:
-      This evaluator runs in a project-local Python venv, not the system
-      interpreter. For a package install that is ~/.pi/agent/pi-repl/venv; for a
-      repo checkout it is .venv/. Files you write are real files on disk in the
-      working directory, visible to the host and other processes.
-    """
-    with open(path, "w", encoding="utf-8") as f:
-        f.write(content if isinstance(content, str) else str(content))
-    return f"wrote {path} ({len(str(content))} chars)"

package/src/extension/config.ts

@@ -1,64 +0,0 @@
-/**
- * config.ts — reads the user's pi-repl config.
- *
- * Where the venv python path, the toolbox directory, and timeouts are
- * configured. First-found-wins, never throws on a missing/malformed file.
- *
- *   $PI_REPL_CONFIG            explicit env override
- *   ~/.pi/agent/pi-repl/config.json   user-global
- *
- * The loadable function set is the toolbox directory (see engine/toolbox/);
- * there is no separate helpers list. The extension ships with the four default
- * functions (read/write/edit/bash) and a user points toolboxDir at their own
- * folder to replace them.
- */
-
-import { existsSync, readFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-
-export interface ReplConfig {
-	/** Python interpreter used to spawn the guest. Defaults to the venv / PATH. */
-	pythonPath?: string;
-	/** Directory of toolbox function files, exec'd into every kernel. */
-	toolboxDir?: string;
-	/** Stall watchdog, ms: 0 = no cap, nonzero = no-output-for-N-ms trips it. */
-	timeoutMs: number;
-	/** Debounce for the auto-snapshot after an ok cell, ms. Default 1500. */
-	snapshotDebounceMs: number;
-}
-
-const DEFAULT_CONFIG: ReplConfig = {
-	// --- timeoutMs: 0 = no cap; nonzero = silence watchdog (no output for N ms) ---
-	timeoutMs: 0,
-	snapshotDebounceMs: 1500,
-};
-
-function num(v: unknown, dflt: number): number {
-	return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : dflt;
-}
-
-function configCandidates(): string[] {
-	const env = process.env.PI_REPL_CONFIG;
-	const user = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), ".pi", "agent", "pi-repl"), "config.json");
-	return [env, user].filter((p): p is string => !!p && p.length > 0);
-}
-
-/** Load + validate; return defaults on any failure. */
-export function loadConfig(): ReplConfig {
-	for (const file of configCandidates()) {
-		try {
-			if (!existsSync(file)) continue;
-			const raw: unknown = JSON.parse(readFileSync(file, "utf8"));
-			if (typeof raw !== "object" || raw === null) return { ...DEFAULT_CONFIG };
-			const r = raw as Record<string, unknown>;
-			return {
-				pythonPath: typeof r.pythonPath === "string" && r.pythonPath.length > 0 ? r.pythonPath : undefined,
-				toolboxDir: typeof r.toolboxDir === "string" && r.toolboxDir.length > 0 ? r.toolboxDir : undefined,
-				timeoutMs: num(r.timeoutMs, DEFAULT_CONFIG.timeoutMs),
-				snapshotDebounceMs: num(r.snapshotDebounceMs, DEFAULT_CONFIG.snapshotDebounceMs),
-			};
-		} catch {}
-	}
-	return { ...DEFAULT_CONFIG };
-}

package/src/extension/preview-core.ts

@@ -1,2 +0,0 @@
-// --- thin aggregator: the preview surface kept at this path so callers and tests stay put ---
-export { type CellPreview, descriptor, previewCell, previewShellCommand } from "./preview/index.js";