stcode 0.0.1a0__py3-none-any.whl

stcode/core/repl/_worker.py

@@ -0,0 +1,179 @@
+"""
+The REPL subprocess. Run as `python -u _worker.py`.
+
+One line in, one or more lines out.
+
+    in   {"type":"exec","id":"c1","code":"x = 1\nprint(x)"}
+    out  {"type":"chunk","text":"1"}            <- while the cell runs
+    out  {"type":"result","id":"c1","ok":true,"stdout":"1\n","error":null}
+
+    in   {"type":"inject","id":"i1","values":{"read_ab12": "..."}}
+    out  {"type":"ok","id":"i1"}
+
+`ns` persists between cells. `tool_out` lives in `ns` and `inject` fills it, so the
+elided half of a tool result really is reachable. Top-level `await` works, via
+`PyCF_ALLOW_TOP_LEVEL_AWAIT`.
+
+Dependency-free and imported by nothing: it is spawned, not imported.
+"""
+
+from __future__ import annotations
+
+import ast
+import asyncio
+import contextlib
+import io
+import json
+import sys
+import traceback
+from typing import Any
+
+# The real stdout, grabbed before any redirect. Cell output is redirected away from
+# sys.stdout; this handle is the protocol and must never be.
+_PROTO = sys.stdout
+_REAL_STDERR = sys.stderr
+
+MAX_FRAME_CHARS = 32768
+"""Cap on any single string this worker puts on the wire.
+
+The parent reads one JSON object per line through an `asyncio` stream, which refuses a
+line past its buffer limit — and once that happens the pipe is wedged, not merely
+noisy. A cell printing a megabyte is a cell whose author wanted a variable, so the cap
+belongs here, at the producer, rather than in a bigger buffer over there."""
+
+HEAD_SHARE = 0.75
+"""A traceback's last line and a test summary both live at the end."""
+
+ns: dict[str, Any] = {"tool_out": {}}
+
+
+def cap(text: str, limit: int = MAX_FRAME_CHARS) -> str:
+    """Keep the head and tail of `text`, and say what went missing.
+
+    Deliberately a copy of `core/common/truncate.elide` rather than an import: this
+    file is spawned, never imported, and must stay dependency-free.
+    """
+    if len(text) <= limit:
+        return text
+    marker = f"\n… [{len(text) - limit:,} chars dropped by the REPL — print less, or slice the variable] …\n"
+    body = max(limit - len(marker), 0)
+    head = int(body * HEAD_SHARE)
+    return text[:head] + marker + (text[-(body - head):] if body - head else "")
+
+_loop = asyncio.new_event_loop()
+asyncio.set_event_loop(_loop)
+
+
+def emit(message: dict[str, Any]) -> None:
+    """One JSON object, one line, flushed."""
+    _PROTO.write(json.dumps(message, ensure_ascii=False, default=str) + "\n")
+    _PROTO.flush()
+
+
+class _Tee(io.TextIOBase):
+    """Collects the cell's output, and streams it out a line at a time.
+
+    Two readers want the same bytes for different reasons: the caller wants the whole
+    thing at the end, the UI wants it now so a 90-second cell does not look frozen.
+    Streaming per line rather than per `write()` because `print` makes several writes
+    per line and one frame each would be noise.
+    """
+
+    def __init__(self) -> None:
+        self.buffer = io.StringIO()
+        self._pending = ""
+
+    def write(self, text: str) -> int:
+        self.buffer.write(text)
+        self._pending += text
+        while "\n" in self._pending:
+            line, self._pending = self._pending.split("\n", 1)
+            emit({"type": "chunk", "text": cap(line)})
+        return len(text)
+
+    def flush(self) -> None:
+        pass
+
+    def fileno(self) -> int:
+        """The real stderr's fd.
+
+        Needed because `subprocess.Popen` asks for one, and a cell that spawns a
+        process is not exotic — launching a stdio MCP server is the main thing this
+        REPL exists for. Without this, `redirect_stderr` turns every such call into
+        `io.UnsupportedOperation: fileno`.
+
+        A child's output therefore goes to the worker's own stderr rather than into
+        this buffer. The parent drains that continuously, so a chatty server cannot
+        fill the pipe and deadlock.
+        """
+        return _REAL_STDERR.fileno()
+
+    def drain(self) -> None:
+        """Emit the last line when it had no trailing newline."""
+        if self._pending:
+            emit({"type": "chunk", "text": cap(self._pending)})
+            self._pending = ""
+
+
+def run_cell(code: str) -> tuple[bool, str, str | None]:
+    """Execute one cell. Returns (ok, stdout, traceback)."""
+    tee = _Tee()
+    ok, error = True, None
+    try:
+        with contextlib.redirect_stdout(tee), contextlib.redirect_stderr(tee):
+            compiled = compile(code, "<cell>", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+            # `eval` of exec-mode code returns None, unless the cell had a top-level
+            # await — then it is a coroutine and we drive it here.
+            pending = eval(compiled, ns)  # noqa: S307 — running the model's code is the job
+            if pending is not None:
+                _loop.run_until_complete(pending)
+    except BaseException:  # noqa: BLE001 — KeyboardInterrupt is an outcome, not a crash
+        ok, error = False, traceback.format_exc()
+    tee.drain()
+    return ok, tee.buffer.getvalue(), error
+
+
+def main() -> None:
+    while True:
+        try:
+            line = sys.stdin.readline()
+        except KeyboardInterrupt:
+            # A SIGINT that landed after the cell had already returned. Nothing to
+            # interrupt; going back to waiting is the correct response.
+            continue
+        if not line:
+            return  # stdin closed — the parent is gone
+        line = line.strip()
+        if not line:
+            continue
+
+        try:
+            request = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(request, dict):
+            continue
+
+        kind = request.get("type", "exec")
+        request_id = str(request.get("id", ""))
+
+        if kind == "inject":
+            ns["tool_out"].update(request.get("values") or {})
+            emit({"type": "ok", "id": request_id})
+            continue
+        if kind == "ping":
+            emit({"type": "ok", "id": request_id})
+            continue
+
+        ok, stdout, error = run_cell(str(request.get("code", "")))
+        emit({
+            "type": "result",
+            "id": request_id,
+            "ok": ok,
+            "stdout": cap(stdout),
+            "error": cap(error) if error else error,
+        })
+
+
+if __name__ == "__main__":
+    main()

stcode/core/repl/client.py

@@ -0,0 +1,374 @@
+"""
+PyREPL — the parent half of the REPL. Owns one `python -u` subprocess.
+
+    repl = PyREPL(cwd=project_root)
+    result = await repl.execute("x = load()\nprint(len(x))")
+    await repl.inject({"read_ab12": "...the elided remainder..."})
+    await repl.aclose()
+
+A subprocess rather than `exec()` here, because a model that writes `while True:` would
+otherwise hang the whole daemon, and in a container nobody can press Ctrl-C. Here it
+costs one SIGINT.
+
+* **One cell at a time** — the namespace is shared state, and two concurrent cells in it
+  is a race with no upside.
+* **Start on first use.** Most sessions never call `repl`.
+* **A dead worker is replaced, not mourned** — killed, and the next call gets a fresh
+  one with an empty namespace, which the caller is told about.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import os
+import signal
+import sys
+import uuid
+from collections import deque
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Awaitable, Callable
+
+from stcode.core.common.truncate import DEFAULT_VIEW_LIMIT, NARROW_REQUEST_HINT, elide
+
+WORKER = Path(__file__).with_name("_worker.py")
+
+DEFAULT_TIMEOUT = 120.0
+STDERR_TAIL_LINES = 50
+"""Worker stderr kept for a crash report. Bounded — the alternative is holding every
+warning a long session produced."""
+
+STREAM_LIMIT = 4 * 1024 * 1024
+"""Bytes one protocol line may take. Belt to the worker's braces: it caps what it emits
+at 32k chars, but a char is up to four bytes, and `asyncio`'s 64 KiB default does not
+merely truncate an over-long line — the read raises and the paused transport then never
+delivers EOF, so even `aclose()` hangs."""
+
+INTERRUPT_GRACE = 5.0
+"""How long the worker gets to report back after SIGINT before it is killed. A cell
+stuck in a C call cannot be interrupted at all, and waiting forever for one is how a
+daemon becomes something you `kill -9` instead of trusting."""
+
+StreamFn = Callable[[str, str], Awaitable[None]]
+"""`(stream_name, text) -> None`. Called per line while a cell runs."""
+
+
+@dataclass
+class ExecResult:
+    """What one cell produced."""
+
+    ok: bool = True
+    outcome: str = "ok"
+    """`ok` | `error` | `timeout` | `crashed`. The model needs these apart: timeout means
+    make it smaller, error means fix it, crashed means the namespace is gone."""
+
+    stdout: str = ""
+    error: str | None = None
+    duration: float = 0.0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def text(self) -> str:
+        """Output and traceback, in the order they happened."""
+        parts = [part for part in (self.stdout.rstrip("\n"), self.error) if part]
+        return "\n".join(parts)
+
+    def view(self, limit: int = DEFAULT_VIEW_LIMIT) -> str:
+        """What the model reads. Elided at `limit`, never summarised."""
+        return elide(self.text(), limit, hint=NARROW_REQUEST_HINT)
+
+
+class ReplError(RuntimeError):
+    """The worker could not be started or could not be reached."""
+
+
+class PyREPL:
+    """A persistent Python namespace in a subprocess."""
+
+    def __init__(
+        self,
+        *,
+        cwd: str | Path | None = None,
+        env: dict[str, str] | None = None,
+        python: str = sys.executable,
+    ) -> None:
+        self.cwd = Path(cwd) if cwd else Path.cwd()
+        self.env = env or {}
+        self.python = python
+        self._process: asyncio.subprocess.Process | None = None
+        self._lock = asyncio.Lock()
+        self._stderr_task: asyncio.Task[None] | None = None
+        self._stderr_tail: deque[str] = deque(maxlen=STDERR_TAIL_LINES)
+        # Injections that arrived before the worker started. Held rather than spawning
+        # a Python for them — the session may never use one.
+        self._pending_injects: dict[str, Any] = {}
+
+    @property
+    def running(self) -> bool:
+        return self._process is not None and self._process.returncode is None
+
+    # ---- lifecycle ----
+
+    async def start(self) -> None:
+        """Spawn the worker. Idempotent."""
+        if self.running:
+            return
+        environment = {**os.environ, **self.env}
+        # The generated MCP stubs live under the workspace, so it must be importable
+        # from inside a cell — this is what makes `from mcp_servers.github import ...`
+        # work.
+        roots = [str(self.cwd), str(self.cwd / ".stcode")]
+        existing = environment.get("PYTHONPATH", "")
+        environment["PYTHONPATH"] = os.pathsep.join([*roots, existing]) if existing else os.pathsep.join(roots)
+
+        try:
+            self._process = await asyncio.create_subprocess_exec(
+                self.python, "-u", str(WORKER),
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                cwd=str(self.cwd),
+                env=environment,
+                limit=STREAM_LIMIT,
+            )
+        except OSError as exc:
+            raise ReplError(f"Could not start the REPL worker: {exc}") from exc
+
+        self._stderr_task = asyncio.create_task(self._drain_stderr(), name="stcode-repl-stderr")
+
+        if self._pending_injects:
+            values, self._pending_injects = self._pending_injects, {}
+            await self._request({"type": "inject", "values": values})
+
+    async def _drain_stderr(self) -> None:
+        """Keep the worker's stderr pipe empty, and keep the last of it.
+
+        The second half is the one that bites: a subprocess started from a cell inherits
+        this pipe, and a chatty child blocks on a full buffer and hangs its own cell.
+        """
+        process = self._process
+        if process is None or process.stderr is None:
+            return
+        with contextlib.suppress(Exception):
+            while True:
+                line = await process.stderr.readline()
+                if not line:
+                    return
+                self._stderr_tail.append(line.decode("utf-8", errors="replace").rstrip("\n"))
+
+    async def aclose(self) -> None:
+        """Close stdin, then kill if it does not leave."""
+        process = self._process
+        self._process = None
+        if self._stderr_task is not None:
+            self._stderr_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError, Exception):
+                await self._stderr_task
+            self._stderr_task = None
+        if process is None or process.returncode is not None:
+            return
+        with contextlib.suppress(Exception):
+            if process.stdin is not None:
+                process.stdin.close()
+        try:
+            await asyncio.wait_for(process.wait(), 2.0)
+        except Exception:
+            with contextlib.suppress(ProcessLookupError, OSError):
+                process.kill()
+            # Bounded, because `wait()` is not: it completes only once every pipe has
+            # reported EOF, and a stdout transport paused by an over-long line never
+            # will. An unbounded wait here is how closing a session hangs forever.
+            with contextlib.suppress(Exception):
+                await asyncio.wait_for(process.wait(), 2.0)
+
+    async def __aenter__(self) -> "PyREPL":
+        await self.start()
+        return self
+
+    async def __aexit__(self, *_exc_info: object) -> None:
+        await self.aclose()
+
+    # ---- the two verbs ----
+
+    async def execute(
+        self,
+        code: str,
+        *,
+        timeout: float = DEFAULT_TIMEOUT,
+        on_stream: StreamFn | None = None,
+    ) -> ExecResult:
+        """Run one cell and return what it printed."""
+        async with self._lock:
+            await self.start()
+            started = asyncio.get_running_loop().time()
+            request_id = uuid.uuid4().hex[:8]
+            try:
+                reply = await self._exchange(
+                    {"type": "exec", "id": request_id, "code": code},
+                    timeout=timeout,
+                    on_stream=on_stream,
+                )
+            except asyncio.TimeoutError:
+                return await self._after_timeout(request_id, timeout, started, on_stream)
+            except ReplError as exc:
+                await self.aclose()
+                return ExecResult(
+                    ok=False,
+                    outcome="crashed",
+                    error=f"{exc}\nThe namespace is gone; the next cell starts a fresh one.",
+                    duration=asyncio.get_running_loop().time() - started,
+                )
+
+        return _result_from(reply, asyncio.get_running_loop().time() - started)
+
+    async def inject(self, values: dict[str, Any]) -> None:
+        """Push values into the cell namespace's `tool_out`.
+
+        What `elide` cut is parked here, so the hint naming `tool_out["..."]` is true.
+        """
+        if not values:
+            return
+        if not self.running:
+            self._pending_injects.update(values)
+            return
+        async with self._lock:
+            with contextlib.suppress(ReplError, asyncio.TimeoutError):
+                await self._exchange({"type": "inject", "values": values}, timeout=10.0)
+
+    async def interrupt(self) -> None:
+        """SIGINT the cell in flight. The namespace survives."""
+        if self.running and self._process is not None:
+            with contextlib.suppress(ProcessLookupError, OSError):
+                self._process.send_signal(signal.SIGINT)
+
+    # ---- wire ----
+
+    async def _request(self, message: dict[str, Any]) -> dict[str, Any]:
+        return await self._exchange(message, timeout=10.0)
+
+    async def _exchange(
+        self,
+        message: dict[str, Any],
+        *,
+        timeout: float | None,
+        on_stream: StreamFn | None = None,
+    ) -> dict[str, Any]:
+        """Send one message, read until its answer. Forwards `chunk` lines on the way."""
+        process = self._process
+        if process is None or process.stdin is None or process.stdout is None:
+            raise ReplError("the REPL worker is not running")
+
+        message.setdefault("id", uuid.uuid4().hex[:8])
+        try:
+            process.stdin.write((json.dumps(message, default=str) + "\n").encode("utf-8"))
+            await process.stdin.drain()
+        except (ConnectionError, OSError, RuntimeError) as exc:
+            raise ReplError(f"the REPL worker stopped listening: {exc}") from exc
+
+        wanted = str(message["id"])
+        if timeout is None:
+            return await self._read_frame(on_stream=on_stream, wanted=wanted)
+        return await asyncio.wait_for(
+            self._read_frame(on_stream=on_stream, wanted=wanted), timeout
+        )
+
+    async def _read_frame(
+        self, *, on_stream: StreamFn | None = None, wanted: str | None = None
+    ) -> dict[str, Any]:
+        """Read until a real answer arrives. Chunks are forwarded, not returned.
+
+        `wanted` matches the request id. Without it a late frame — a cell answering its
+        SIGINT after we stopped waiting — would be handed to the *next* call, and every
+        reply after that would be off by one.
+        """
+        process = self._process
+        if process is None or process.stdout is None:
+            raise ReplError("the REPL worker is not running")
+        while True:
+            try:
+                line = await process.stdout.readline()
+            except (ValueError, asyncio.LimitOverrunError) as exc:
+                # A line past `STREAM_LIMIT`. The buffer is dropped and the pipe is no
+                # longer in sync with the protocol, so the worker is gone either way —
+                # say so as a `ReplError`, which `execute` already knows how to turn
+                # into a crashed result, rather than as a bare ValueError from a caller
+                # that only ever asked to run a cell.
+                raise ReplError(
+                    f"the REPL worker sent more than {STREAM_LIMIT:,} bytes on one line "
+                    f"({exc}). Assign the value to a variable and print a slice of it."
+                ) from exc
+            if not line:
+                raise ReplError(await self._death_note(process))
+            try:
+                frame = json.loads(line.decode("utf-8", errors="replace"))
+            except json.JSONDecodeError:
+                continue
+            if frame.get("type") == "chunk":
+                if on_stream is not None:
+                    await on_stream("stdout", str(frame.get("text", "")))
+                continue
+            if wanted and str(frame.get("id", "")) != wanted:
+                continue  # a stale answer to a call that already gave up
+            return frame  # type: ignore[no-any-return]
+
+    async def _after_timeout(
+        self, request_id: str, timeout: float, started: float, on_stream: StreamFn | None
+    ) -> ExecResult:
+        """Interrupt, then give the worker a moment to report what it had.
+
+        A cell that answers its SIGINT keeps its namespace, which is why this comes
+        before the kill. We wait for that cell's *own* result frame — anything else
+        would leave it in the pipe for the next call to trip over.
+        """
+        await self.interrupt()
+        elapsed = asyncio.get_running_loop().time() - started
+        try:
+            reply = await asyncio.wait_for(
+                self._read_frame(on_stream=on_stream, wanted=request_id), INTERRUPT_GRACE
+            )
+        except (asyncio.TimeoutError, ReplError):
+            await self.aclose()
+            return ExecResult(
+                ok=False,
+                outcome="timeout",
+                error=(
+                    f"The cell exceeded {timeout:.0f}s and would not stop, so the "
+                    "interpreter was killed. The namespace is gone. Split the work into "
+                    "smaller cells."
+                ),
+                duration=elapsed,
+            )
+        return ExecResult(
+            ok=False,
+            outcome="timeout",
+            stdout=str(reply.get("stdout", "")),
+            error=(
+                f"The cell was interrupted after {timeout:.0f}s. The namespace survived, "
+                "so whatever it managed to build is still there to inspect."
+            ),
+            duration=elapsed,
+        )
+
+    async def _death_note(self, process: asyncio.subprocess.Process) -> str:
+        """Whatever the worker said on its way out — usually the real reason."""
+        detail = "\n".join(self._stderr_tail).strip()
+        code = process.returncode
+        return f"the REPL worker exited ({code}){': ' + detail[-800:] if detail else ''}"
+
+    def __repr__(self) -> str:
+        return f"PyREPL(cwd={self.cwd}, running={self.running})"
+
+
+def _result_from(reply: dict[str, Any], duration: float) -> ExecResult:
+    ok = bool(reply.get("ok", False))
+    return ExecResult(
+        ok=ok,
+        outcome="ok" if ok else "error",
+        stdout=str(reply.get("stdout", "")),
+        error=reply.get("error"),
+        duration=duration,
+    )
+
+
+__all__ = ["DEFAULT_TIMEOUT", "ExecResult", "PyREPL", "ReplError", "StreamFn"]

stcode/core/session/__init__.py

@@ -0,0 +1,39 @@
+"""
+Session — the append-only JSONL transcript, and the only place a turn is recorded.
+
+    s = Session.create(cwd=workspace, role="backend-dev")
+    s = Session.resume("01K7...")
+    Session.list(limit=20)
+
+    s = Session.create(cwd=workspace, defer=True)   # no file until the first message
+
+    s.append(type="user", content="Add rate limiting to the API")
+    s.set_meta(model="claude-opus-5")   # a setting, appended — never a rewrite
+    s.messages()        # -> list[Message] for the gateway
+    s.tail(30)          # -> raw records for the supervisor
+    s.meta(), s.overrides()             # what it started as / what changed since
+
+One writer, one file, three readers. There is no second logger to keep in sync.
+"""
+
+from stcode.core.session.session import (
+    DEFAULT_SESSION_DIR,
+    META_BOOKKEEPING,
+    MODEL_VISIBLE,
+    Session,
+    new_id,
+    prune,
+    read_records,
+    sessions_dir,
+)
+
+__all__ = [
+    "DEFAULT_SESSION_DIR",
+    "META_BOOKKEEPING",
+    "MODEL_VISIBLE",
+    "Session",
+    "new_id",
+    "prune",
+    "read_records",
+    "sessions_dir",
+]