darkpyonix 0.3.0__py3-none-win_amd64.whl

darkpyonix/__init__.py

@@ -0,0 +1,43 @@
+"""DarkPyonix runtime API for notebook files (SPEC FR-F2..F6, FORMAT §4).
+
+Standard library only (INTENT §2). Importing this package must stay cheap and must never
+import any kernel-process module (``_server``, ``_executor``, ...): notebook code and the kernel
+import it from interpreters where nothing but the standard library is available. The public names are loaded lazily
+from private modules on first access (PEP 562).
+
+Inside a kernel the calls talk to ``darkpyonix._hostctx``; anywhere else they behave
+as under plain ``python file.py`` (SPEC FR-F5).
+"""
+
+__version__ = "0.3.0"
+
+_EXPORTS = {
+    "markdown": "_markdown",
+    "params": "_params",
+    "binding": "_binding",
+    "run_command": "_command",
+    "uv": "_command",
+    "pip": "_command",
+    "display": "_misc",
+    "run_parallel": "_misc",
+    "run_cinterop": "_misc",
+    "run_cppinterop": "_misc",
+    "run_rustinterop": "_misc",
+}
+
+__all__ = sorted(_EXPORTS)
+
+
+def __getattr__(name):
+    module = _EXPORTS.get(name)
+    if module is None:
+        raise AttributeError("module 'darkpyonix' has no attribute %r" % (name,))
+    import importlib
+
+    value = getattr(importlib.import_module("darkpyonix." + module), name)
+    globals()[name] = value
+    return value
+
+
+def __dir__():
+    return sorted(set(globals()) | set(_EXPORTS))

darkpyonix/__main__.py

@@ -0,0 +1,263 @@
+"""The DarkPyonix kernel process (INTENT D1–D3, D12; PROTOCOL §2–§3).
+
+Started as ``python -m darkpyonix`` (or by ``_launcher.bootstrap_command``). Owns the per-file
+lock, the shared document (SPEC §10a, ``_collab.DocumentState``), the run store, the executor
+(user code on the main thread), the DKP/1 control server and discovery. Standard library only;
+Python 3.8+.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import platform
+import signal
+import socket
+import sys
+import threading
+import time
+from typing import Any, Callable, Dict, List, Optional
+
+from darkpyonix import _home
+from darkpyonix import _lock as lock
+from darkpyonix._collab import DocumentState
+from darkpyonix._discovery import DiscoveryResponder
+from darkpyonix._events import EventLog
+from darkpyonix._executor import Executor
+from darkpyonix._launcher import bootstrap_command
+from darkpyonix._protocol import (
+    EXIT_ALREADY_RUNNING, KERNEL_VERSION, DKPError, canonical_path, kernel_id_for, now_iso,
+)
+from darkpyonix._runs import RunStore, runs_dir_for
+from darkpyonix._tokens import TokenStore
+
+FINISHED = ("ok", "error", "interrupted", "cancelled", "crashed")
+
+
+def _log(stream, message: str) -> None:
+    try:
+        stream.write("[darkpyonix-kernel %s] %s\n" % (now_iso(), message))
+        stream.flush()
+    except Exception:
+        pass
+
+
+class Kernel(object):
+    def __init__(self, path: str) -> None:
+        self.path = canonical_path(path)
+        self.kernel_id = kernel_id_for(self.path)
+        self.user_tag = _home.user_tag()
+        self.lock = lock.FileLock(lock.lock_path(self.kernel_id))
+        self.events = EventLog()
+        self.collab = None  # type: Optional[DocumentState]
+        self.store = None  # type: Optional[RunStore]
+        self.tokens = None  # type: Optional[TokenStore]
+        self.executor = None  # type: Optional[Executor]
+        self.server = None
+        self.discovery = None  # type: Optional[DiscoveryResponder]
+        self.extra_handlers = {}  # type: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]]
+        self.started_at = now_iso()
+        self.python = {
+            "version": platform.python_version(),
+            "implementation": platform.python_implementation(),
+            "executable": sys.executable,
+        }
+        self.host = socket.gethostname()
+        self.diag = sys.stderr
+        self._shutdown_started = threading.Event()
+
+    # -------------------------------------------------------------- discovery info
+
+    def info(self) -> Dict[str, Any]:
+        st = self.executor.status() if self.executor is not None else {"status": "starting"}
+        return {
+            "kernel_id": self.kernel_id,
+            "path": self.path,
+            "pid": os.getpid(),
+            "control": self.server.address if self.server is not None else "",
+            "status": st.get("status", "starting"),
+            "run_id": st.get("run_id"),
+            "python": self.python,
+            "kernel_version": KERNEL_VERSION,
+            "runs_dir": runs_dir_for(self.path),
+            "started_at": self.started_at,
+            "host": self.host,
+            "home": _home.home(),
+        }
+
+    def emit(self, type: str, data: Dict[str, Any]) -> None:
+        self.events.append(type, data)
+        if type == "kernel.status" and self.discovery is not None:
+            self.discovery.announce_now()
+
+    # -------------------------------------------------------------- handlers
+
+    def _status(self, params: Dict[str, Any]) -> Dict[str, Any]:
+        out = self.info()
+        out.update(self.executor.status())
+        return out
+
+    def _runs_list(self, params: Dict[str, Any]) -> Dict[str, Any]:
+        limit = int(params.get("limit", 20))
+        return {"runs": self.store.list(limit)}
+
+    def _runs_get(self, params: Dict[str, Any]) -> Dict[str, Any]:
+        ref = params.get("run_id")
+        if not isinstance(ref, str):
+            raise DKPError("bad_request", "run_id is required")
+        return self.store.get(ref)
+
+    def _runs_wait(self, params: Dict[str, Any]) -> Dict[str, Any]:
+        """SPEC FR-S7: return when the run finishes, or after ``timeout`` seconds."""
+        ref = params.get("run_id")
+        timeout = max(1.0, min(float(params.get("timeout", 60)), 300.0))
+        if ref == "current":
+            ref = self.executor.status().get("run_id")
+            if ref is None:
+                raise DKPError("not_found", "no run is executing")
+        deadline = time.monotonic() + timeout
+        while True:
+            summary = self._summary(ref)
+            if summary is not None and summary.get("status") in FINISHED:
+                return {"status": summary["status"], "run_id": summary["run_id"], "run": summary}
+            if time.monotonic() >= deadline:
+                if summary is None:
+                    raise DKPError("not_found", "unknown run %r" % ref)
+                return {"status": summary.get("status", "running"), "run_id": summary["run_id"],
+                        "run": summary}
+            time.sleep(0.1)
+
+    def _summary(self, ref: str) -> Optional[Dict[str, Any]]:
+        for s in self.store.list(500):
+            if ref == "latest" and s.get("status") in FINISHED:
+                return s
+            if s.get("run_id") == ref:
+                return s
+        st = self.executor.status()
+        if ref in st.get("queue", []):
+            return {"run_id": ref, "status": "queued"}
+        return None
+
+    def _run(self, params: Dict[str, Any]) -> Dict[str, Any]:
+        """PROTOCOL §3.3 ``run``; ``cell_ids`` (FR-S6) name cells of the shared document."""
+        params = dict(params or {}) if isinstance(params, dict) else params
+        if isinstance(params, dict) and params.get("cell_ids") is not None:
+            ids = params.pop("cell_ids")
+            if not isinstance(ids, list) or not ids or not all(isinstance(c, str) for c in ids):
+                raise DKPError("bad_request", "cell_ids must be a non-empty list of strings")
+            if params.get("cells") is not None:
+                raise DKPError("bad_request", "give cells or cell_ids, not both")
+            params["cells"] = self.collab.indices_for(ids)
+            params["mode"] = "cells"
+        self.collab.flush()  # debounced edits reach the file before the run reads it
+        return self.executor.submit(params)
+
+    def _before_load(self) -> List[str]:
+        """Executor hook, right before a (possibly queued) run reads the file."""
+        self.collab.flush()
+        return self.collab.cell_ids()
+
+    def _shutdown(self, params: Dict[str, Any]) -> Dict[str, Any]:
+        self.request_shutdown()
+        return {"shutting_down": True}
+
+    def handlers(self) -> Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]]:
+        ex = self.executor
+        h = {
+            "status": self._status,
+            "run": self._run,
+            "cancel": lambda p: ex.cancel(p.get("run_id")),
+            "interrupt": lambda p: ex.interrupt(p.get("client")),
+            "restart": lambda p: ex.restart(bool(p.get("hard", False))),
+            "shutdown": self._shutdown,
+            "namespace": lambda p: ex.namespace(int(p.get("limit", 200))),
+            "runs.list": self._runs_list,
+            "runs.get": self._runs_get,
+            "runs.wait": self._runs_wait,
+        }  # type: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]]
+        h.update(self.extra_handlers)
+        return h
+
+    def request_shutdown(self) -> None:
+        if self._shutdown_started.is_set():
+            return
+        self._shutdown_started.set()
+        threading.Thread(target=self.executor.shutdown, name="dpx-shutdown", daemon=True).start()
+
+    # -------------------------------------------------------------- lifecycle
+
+    def run(self) -> int:
+        if not self.lock.acquire():
+            from darkpyonix import _discovery as discovery
+            found = discovery.discover(self.kernel_id, timeout=0.5)
+            existing = found[0] if found else {"kernel_id": self.kernel_id}
+            sys.stderr.write(json.dumps(existing, ensure_ascii=False) + "\n")
+            sys.stderr.flush()
+            return EXIT_ALREADY_RUNNING
+        try:
+            _home.expose_package()
+            self.collab = DocumentState(self.path, self.events.append,
+                                        seq_provider=lambda: self.events.seq)
+            self.extra_handlers.update(self.collab.handlers())
+            self.tokens = TokenStore(self.path)  # FR-K10
+            self.extra_handlers.update(self.tokens.handlers())
+            self.store = RunStore(self.path, self.kernel_id)
+            crashed = self.store.recover_crashed()
+            if crashed:
+                _log(self.diag, "marked crashed runs: %s" % ", ".join(crashed))
+            self.executor = Executor(self.path, self.kernel_id, self.emit, self.store,
+                                     self.python, self.host)
+            self.executor.before_load = self._before_load
+            self.diag = getattr(self.executor, "original_stderr", sys.stderr)
+
+            from darkpyonix._server import ControlServer
+            self.server = ControlServer(self.kernel_id, self.events, self.handlers())
+            self.tokens.close_streams = self.server.streams.close_share  # FR-K9, PR-6
+            self.collab.start()
+            self.tokens.start()
+            self.server.start()
+            self.discovery = DiscoveryResponder(self.info, self.user_tag)
+            self.discovery.start()
+
+            signal.signal(signal.SIGTERM, lambda signum, frame: self.request_shutdown())
+            if hasattr(signal, "SIGHUP"):
+                signal.signal(signal.SIGHUP, signal.SIG_IGN)
+            _log(self.diag, "kernel %s for %s on %s" % (self.kernel_id, self.path, self.server.address))
+
+            self.executor.run_forever()      # main thread until shutdown / hard restart
+        finally:
+            hard = bool(self.executor is not None and self.executor.hard_restart_requested)
+            self._teardown()
+        if hard:
+            _log(sys.stderr, "hard restart")
+            cmd = bootstrap_command(sys.executable, self.path)
+            os.execv(cmd[0], cmd)
+        return 0
+
+    def _teardown(self) -> None:
+        for step in (
+            lambda: self.store.flush() if self.store is not None else None,
+            lambda: self.discovery.stop(bye=True) if self.discovery is not None else None,
+            lambda: self.server.stop() if self.server is not None else None,
+            lambda: self.collab.stop() if self.collab is not None else None,
+            lambda: self.tokens.stop() if self.tokens is not None else None,
+            lambda: self.lock.release(),
+        ):
+            try:
+                step()
+            except Exception as e:  # never let teardown keep the lock
+                _log(sys.stderr, "teardown: %r" % (e,))
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(prog="darkpyonix-kernel")
+    parser.add_argument("--file", required=True)
+    args, _unknown = parser.parse_known_args(argv)
+    if not os.path.isfile(args.file):
+        sys.stderr.write("darkpyonix-kernel: no such file: %s\n" % args.file)
+        return 2
+    return Kernel(args.file).run()
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

darkpyonix/_binding.py

@@ -0,0 +1,134 @@
+"""``@darkpyonix.binding`` (SPEC FR-F4, FORMAT §3.4, issue #6).
+
+The decorator re-evaluates the decorated ``def`` / ``class`` from its source in a snapshot of
+the defining module's globals taken at definition time. Imports and earlier bindings are in
+the snapshot; ``[code]`` variables assigned later are not, so referencing them raises
+``NameError``. Only ``binding`` (and decorators written above it, which the original statement
+applies afterwards anyway) is stripped; decorators below it are re-applied.
+
+Works the same in a kernel and under plain ``python file.py`` (SPEC FR-F5). Python 3.8
+compatible: no ``ast.unparse``; decorator lines are blanked in the original source instead, so
+line numbers in tracebacks still point at the file.
+"""
+from __future__ import annotations
+
+import __future__
+import ast
+import builtins
+import linecache
+import sys
+import textwrap
+import warnings
+
+
+_FUTURE_FLAGS = 0
+for _feature in __future__.all_feature_names:
+    _FUTURE_FLAGS |= getattr(__future__, _feature).compiler_flag
+
+
+class _Unresolved(Exception):
+    pass
+
+
+def _resolve(expr, frame):
+    """Resolve a dotted-name decorator expression without evaluating anything else."""
+    if isinstance(expr, ast.Name):
+        for scope in (frame.f_locals, frame.f_globals):
+            if expr.id in scope:
+                return scope[expr.id]
+        if hasattr(builtins, expr.id):
+            return getattr(builtins, expr.id)
+        raise _Unresolved(expr.id)
+    if isinstance(expr, ast.Attribute):
+        base = _resolve(expr.value, frame)
+        try:
+            return getattr(base, expr.attr)
+        except Exception:
+            raise _Unresolved(expr.attr) from None
+    raise _Unresolved(type(expr).__name__)
+
+
+def _first_line(node):
+    return min([node.lineno] + [d.lineno for d in node.decorator_list])
+
+
+def _find_node(tree, name, lineno):
+    """The def/class named ``name`` whose span (decorators included) contains ``lineno``."""
+    kinds = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
+    best = None
+    for node in ast.walk(tree):
+        if isinstance(node, kinds) and node.name == name and node.decorator_list:
+            end = getattr(node, "end_lineno", None) or node.lineno
+            if _first_line(node) <= lineno <= end:
+                # Innermost match wins (a nested definition with the same name).
+                if best is None or _first_line(node) >= _first_line(best):
+                    best = node
+    return best
+
+
+def _source_for(obj, frame):
+    """(filename, all lines, node) for the decorated definition, or None."""
+    filename = frame.f_code.co_filename
+    lines = linecache.getlines(filename, frame.f_globals)
+    if not lines:
+        return None
+    try:
+        tree = ast.parse("".join(lines), filename)
+    except (SyntaxError, ValueError):
+        return None
+    node = _find_node(tree, obj.__name__, frame.f_lineno)
+    if node is None:
+        code = getattr(obj, "__code__", None)
+        if code is not None and code.co_filename == filename:
+            node = _find_node(tree, obj.__name__, code.co_firstlineno)
+    if node is None:
+        return None
+    return filename, lines, node
+
+
+def binding(obj):
+    """Re-evaluate ``obj`` (a def or class) so it cannot see later ``[code]`` variables."""
+    name = getattr(obj, "__name__", None)
+    frame = sys._getframe(1)
+    try:
+        found = _source_for(obj, frame) if name else None
+        if found is None:
+            raise _Unresolved("source of %r is not available" % (name or obj,))
+        filename, lines, node = found
+
+        cut = -1
+        for i, dec in enumerate(node.decorator_list):
+            try:
+                if _resolve(dec, frame) is binding:
+                    cut = i
+            except _Unresolved:
+                continue
+        if cut < 0:
+            raise _Unresolved("could not find the binding decorator on %r" % name)
+
+        first = _first_line(node)
+        end = node.end_lineno
+        segment = list(lines[first - 1:end])
+        # Blank the binding decorator and those above it; they are applied by the original
+        # statement. Decorators below it are kept and re-applied in the snapshot.
+        for dec in node.decorator_list[:cut + 1]:
+            # The '@' sits on the decorator's first line (or the line before a parenthesised one).
+            start = dec.lineno
+            while start > first and not segment[start - first].lstrip().startswith("@"):
+                start -= 1
+            for ln in range(start, dec.end_lineno + 1):
+                segment[ln - first] = "\n"
+        src = textwrap.dedent("".join(segment))
+        src = "\n" * (first - 1) + src
+        # Same __future__ features as the defining code (e.g. ``annotations``).
+        flags = frame.f_code.co_flags & _FUTURE_FLAGS
+        code = compile(src, filename, "exec", flags, dont_inherit=True)
+    except (_Unresolved, SyntaxError, OSError, TypeError, ValueError) as exc:
+        warnings.warn("darkpyonix.binding: returning %r unchanged (%s)" % (name or obj, exc), stacklevel=2)
+        return obj
+    finally:
+        del frame
+
+    snapshot = dict(sys._getframe(1).f_globals)
+    exec(code, snapshot)
+    return snapshot[name]