starforge-kernel 0.1.0__py3-none-any.whl

starforge/core/runner.py

@@ -0,0 +1,293 @@
+"""Pipeline execution. Runs INSIDE the per-run worker process — the kernel
+never imports user code (DESIGN.md §3, process-per-run).
+
+Planning rules:
+- stale nodes (no checkpoint for their history hash) execute;
+- a fresh node is pulled back into execution if an executing descendant needs
+  one of its outputs and that output was ephemeral (never persisted) — this
+  cascades upward until every needed value is either loadable or recomputed;
+- nodes with hashing problems, and every node downstream of a problem or a
+  failure, are blocked (independent branches keep running).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import datetime as dt
+import importlib
+import sys
+import time
+import traceback
+from typing import Any, Callable
+
+import starforge
+from starforge.core import figures as figmod
+from starforge.core.checkpoints import CheckpointStore
+from starforge.core.provenance import BUILTIN_PREFIX, toposort
+from starforge.core.serializers import EphemeralValueError
+from starforge.core.spec import PipelineDoc, Node
+
+#: Minimum seconds between forwarded progress events per node — blocks may
+#: call progress() in tight loops; the canvas needs ~12fps at most.
+PROGRESS_THROTTLE_S = 0.08
+
+
+def _progress_hook(node_id: str, emit: Emit) -> Callable[[Any, Any, Any], None]:
+    last = {"at": 0.0}
+
+    def hook(current: Any, total: Any, label: Any) -> None:
+        now = time.monotonic()
+        final = total is not None and current == total
+        if not final and now - last["at"] < PROGRESS_THROTTLE_S:
+            return
+        last["at"] = now
+        event: dict[str, Any] = {"event": "node_progress", "node": node_id}
+        if current is not None:
+            event["current"] = current
+        if total is not None:
+            event["total"] = total
+        if label is not None:
+            event["label"] = str(label)
+        if isinstance(current, (int, float)) and isinstance(total, (int, float)) and total:
+            event["percent"] = max(0.0, min(1.0, current / total))
+        emit(event)
+
+    return hook
+
+Emit = Callable[[dict[str, Any]], None]
+
+
+@dataclass
+class _NodePlanInfo:
+    history_hash: str | None
+    problems: list[str]
+
+
+def _ancestor_cone(doc: PipelineDoc, target: str) -> set[str]:
+    """The target plus everything upstream of it."""
+    cone = {target}
+    frontier = [target]
+    while frontier:
+        nid = frontier.pop()
+        for edge in doc.in_edges(nid):
+            if edge.source not in cone:
+                cone.add(edge.source)
+                frontier.append(edge.source)
+    return cone
+
+
+def plan_execution(
+    doc: PipelineDoc,
+    states: dict[str, dict[str, Any]],
+    store: CheckpointStore,
+    target: str | None = None,
+) -> tuple[list[str], set[str], set[str], set[str]]:
+    """Returns (topo_order, execute, reuse, blocked).
+
+    With ``target``, planning is scoped to the target's ancestor cone —
+    "run to here": only what's needed to produce that node executes."""
+    order, cyclic = toposort(doc)
+    if target is not None and any(n.id == target for n in doc.nodes):
+        cone = _ancestor_cone(doc, target)
+        order = [nid for nid in order if nid in cone]
+    blocked: set[str] = set(cyclic) & set(order)
+    execute: set[str] = set()
+
+    for nid in order:
+        state = states.get(nid, {})
+        if state.get("problems") or state.get("history_hash") is None:
+            blocked.add(nid)
+        elif state.get("stale", True):
+            execute.add(nid)
+
+    # Block everything downstream of a blocked node.
+    for nid in order:
+        if nid in blocked:
+            continue
+        if any(edge.source in blocked for edge in doc.in_edges(nid)):
+            blocked.add(nid)
+            execute.discard(nid)
+
+    # Ephemeral cascade: pull fresh parents back in when an executing child
+    # needs an output that was never persisted.
+    changed = True
+    while changed:
+        changed = False
+        for edge in doc.edges:
+            if edge.target not in execute or edge.source in execute or edge.source in blocked:
+                continue
+            parent_hash = states.get(edge.source, {}).get("history_hash")
+            if parent_hash is None:
+                continue
+            if store.is_ephemeral(parent_hash, edge.source_output):
+                execute.add(edge.source)
+                changed = True
+
+    reuse = {nid for nid in order if nid not in execute and nid not in blocked}
+    return order, execute, reuse, blocked
+
+
+def _run_builtin(node: Node) -> dict[str, Any]:
+    if node.block == "builtin:constant":
+        return {"output": node.params.get("value")}
+    raise ValueError(f"unknown builtin '{node.block}'")
+
+
+def _import_function(module: str, qualname: str) -> Callable[..., Any]:
+    mod = importlib.import_module(module)
+    fn = getattr(mod, qualname, None)
+    if fn is None or not callable(fn):
+        raise AttributeError(f"function '{qualname}' not found in module '{module}'")
+    return fn
+
+
+def _map_outputs(result: Any, output_names: list[str], block_label: str) -> dict[str, Any]:
+    if len(output_names) == 1:
+        return {output_names[0]: result}
+    if not isinstance(result, (tuple, list)) or len(result) != len(output_names):
+        raise TypeError(
+            f"'{block_label}' declares outputs {output_names} but returned "
+            f"{type(result).__name__} (expected a {len(output_names)}-tuple)"
+        )
+    return dict(zip(output_names, result))
+
+
+def run_pipeline(
+    doc: PipelineDoc,
+    blocks: dict[str, dict[str, Any]],
+    states: dict[str, dict[str, Any]],
+    store: CheckpointStore,
+    emit: Emit,
+    pickle_enabled: bool = False,
+    target: str | None = None,
+) -> str:
+    """Execute the plan; returns terminal status: 'completed' or 'failed'."""
+    store.ensure_layout()
+    order, execute, reuse, blocked = plan_execution(doc, states, store, target=target)
+    emit(
+        {
+            "event": "run_plan",
+            "execute": sorted(execute),
+            "reuse": sorted(reuse),
+            "blocked": sorted(blocked),
+        }
+    )
+
+    # In-memory values for this run: (node_id, output_name) -> value.
+    memory: dict[tuple[str, str], Any] = {}
+    failed: set[str] = set()
+    any_failed = False
+
+    for nid in order:
+        state = states.get(nid, {})
+        node_hash = state.get("history_hash")
+
+        if nid in blocked or any(e.source in failed or e.source in blocked for e in doc.in_edges(nid)):
+            blocked.add(nid)
+            emit({"event": "node_blocked", "node": nid, "problems": state.get("problems", [])})
+            continue
+
+        if nid in reuse:
+            store.touch(node_hash)  # LRU signal for checkpoint GC
+            emit({"event": "node_skipped", "node": nid, "history_hash": node_hash})
+            continue
+
+        node = doc.node(nid)
+        emit({"event": "node_started", "node": nid, "block": node.block})
+        started = time.time()
+        try:
+            connected: dict[str, dict[str, str]] = {}
+            side_figures: list[Any] = []
+            if node.block.startswith(BUILTIN_PREFIX):
+                outputs = _run_builtin(node)
+                label = node.block.removeprefix(BUILTIN_PREFIX).title()
+                source_hash = node.block
+            else:
+                info = blocks[node.block]
+                kwargs: dict[str, Any] = {}
+                for edge in doc.in_edges(nid):
+                    key = (edge.source, edge.source_output)
+                    if key not in memory:
+                        parent_hash = states[edge.source]["history_hash"]
+                        try:
+                            memory[key] = store.load_output(parent_hash, edge.source_output)
+                        except EphemeralValueError:
+                            # plan_execution should have prevented this; surface loudly.
+                            raise RuntimeError(
+                                f"input '{edge.target_param}' of node '{nid}' is ephemeral and its "
+                                f"producer was not scheduled — planner bug, please report"
+                            )
+                    kwargs[edge.target_param] = memory[key]
+                    connected[edge.target_param] = {
+                        "node": edge.source,
+                        "output": edge.source_output,
+                        "history_hash": states[edge.source]["history_hash"],
+                    }
+                for name, value in node.params.items():
+                    if name not in kwargs:
+                        kwargs[name] = value
+                # `T | None` params with no signature default are optional by
+                # *Forge convention: inject None when nothing else fills them.
+                for name in info.get("optional_params", ()):
+                    kwargs.setdefault(name, None)
+
+                # capture() wraps the import too: module-level figure code
+                # and figures created/shown inside the call are both swept.
+                starforge._progress_hook = _progress_hook(nid, emit)
+                try:
+                    with figmod.capture() as captured:
+                        fn = _import_function(info["module"], info["qualname"])
+                        result = fn(**kwargs)
+                finally:
+                    starforge._progress_hook = None
+                outputs = _map_outputs(result, list(info["outputs"]), info.get("label", node.block))
+                side_figures = [
+                    fig for fig in captured.all_objects()
+                    if all(fig is not value for value in outputs.values())
+                ]
+                label = info.get("label", node.block)
+                source_hash = info.get("source_hash")
+
+            duration = time.time() - started
+            manifest = store.write(
+                node_hash,
+                {
+                    "block_id": node.block,
+                    "label": label,
+                    "source_hash": source_hash,
+                    "params": {k: v for k, v in node.params.items() if k not in connected},
+                    "inputs": connected,
+                    "duration_seconds": round(duration, 6),
+                    "finished_at": dt.datetime.now(dt.timezone.utc).isoformat(),
+                },
+                outputs,
+                pickle_enabled=pickle_enabled,
+                side_figures=side_figures,
+            )
+            figmod.close_figures(side_figures)
+            for name, value in outputs.items():
+                memory[(nid, name)] = value
+            emit(
+                {
+                    "event": "node_completed",
+                    "node": nid,
+                    "history_hash": node_hash,
+                    "duration_seconds": round(duration, 6),
+                    "outputs": manifest,
+                }
+            )
+        except Exception:
+            any_failed = True
+            failed.add(nid)
+            emit(
+                {
+                    "event": "node_failed",
+                    "node": nid,
+                    "duration_seconds": round(time.time() - started, 6),
+                    "traceback": traceback.format_exc(),
+                }
+            )
+
+    status = "failed" if any_failed else "completed"
+    emit({"event": "run_finished", "status": status})
+    return status

starforge/core/serializers.py

@@ -0,0 +1,141 @@
+"""Typed serializer registry for checkpoint outputs (DESIGN.md §8).
+
+Probed in order per value: parquet (DataFrame/Series) → npy (ndarray) → json
+(plain data) → pickle (opt-in, default OFF) → ephemeral.
+
+Ephemeral values flow normally to downstream nodes within the same run but
+are not persisted, so future runs recompute them on demand. That keeps the
+cost of unserializable types localized instead of poisoning the store.
+
+Import discipline: this module never imports pandas/numpy itself. If a value
+IS a DataFrame, pandas is by definition already imported in this process —
+we detect by the type's module name first, then import for free.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import pickle
+from typing import Any
+
+EPHEMERAL = "ephemeral"
+
+
+class EphemeralValueError(RuntimeError):
+    """Raised when loading an output that was never persisted."""
+
+
+def _root_type_module(value: Any) -> str:
+    return type(value).__module__.split(".")[0]
+
+
+def _meta(value: Any) -> dict[str, Any]:
+    meta: dict[str, Any] = {"type": type(value).__name__}
+    shape = getattr(value, "shape", None)
+    if isinstance(shape, tuple) and all(isinstance(d, int) for d in shape):
+        meta["shape"] = list(shape)
+    elif isinstance(value, (list, dict, str)):
+        meta["len"] = len(value)
+    return meta
+
+
+def save_value(
+    value: Any,
+    outputs_dir: Path,
+    name: str,
+    pickle_enabled: bool = False,
+) -> dict[str, Any]:
+    """Persist one output value; returns its manifest entry."""
+    entry: dict[str, Any] = {"name": name, "meta": _meta(value)}
+    outputs_dir.mkdir(parents=True, exist_ok=True)
+
+    if _root_type_module(value) == "pandas":
+        import pandas as pd
+
+        frame = value
+        if isinstance(value, pd.Series):
+            frame = value.to_frame(name=value.name if value.name is not None else "value")
+            entry["meta"]["series"] = True
+        if isinstance(frame, pd.DataFrame):
+            try:
+                filename = f"{name}.parquet"
+                frame.to_parquet(outputs_dir / filename)
+                entry.update(serializer="parquet", file=filename)
+                return entry
+            except (ImportError, ValueError, OSError):
+                pass  # no pyarrow, or unserializable dtypes — fall through
+
+    if _root_type_module(value) == "numpy":
+        import numpy as np
+
+        if isinstance(value, np.ndarray):
+            try:
+                filename = f"{name}.npy"
+                np.save(outputs_dir / filename, value, allow_pickle=False)
+                entry.update(serializer="npy", file=filename)
+                return entry
+            except (ValueError, OSError):
+                pass
+
+    try:
+        text = json.dumps(value)
+        filename = f"{name}.json"
+        (outputs_dir / filename).write_text(text, encoding="utf-8")
+        entry.update(serializer="json", file=filename)
+        return entry
+    except (TypeError, ValueError, OSError):
+        pass
+
+    # Figures can't round-trip from their rendered form, so the VALUE is
+    # ephemeral (flows within a run; downstream recomputes later via the
+    # cascade) — but it leaves a rendered artifact behind for display.
+    from starforge.core import figures
+
+    try:
+        artifact = figures.render_figure(value, outputs_dir, name)
+    except Exception:
+        artifact = None
+    if artifact is not None:
+        entry.update(serializer=EPHEMERAL, file=None, artifact=artifact)
+        return entry
+
+    if pickle_enabled:
+        try:
+            filename = f"{name}.pkl"
+            with (outputs_dir / filename).open("wb") as handle:
+                pickle.dump(value, handle, protocol=pickle.HIGHEST_PROTOCOL)
+            entry.update(serializer="pickle", file=filename)
+            return entry
+        except Exception:
+            pass
+
+    entry.update(serializer=EPHEMERAL, file=None)
+    return entry
+
+
+def load_value(outputs_dir: Path, entry: dict[str, Any]) -> Any:
+    serializer = entry.get("serializer")
+    name = entry.get("name")
+    if serializer == EPHEMERAL:
+        raise EphemeralValueError(
+            f"output '{name}' was not persisted (ephemeral); the producing node must re-run"
+        )
+    path = outputs_dir / entry["file"]
+    if serializer == "parquet":
+        import pandas as pd
+
+        frame = pd.read_parquet(path)
+        if entry.get("meta", {}).get("series"):
+            return frame.iloc[:, 0]
+        return frame
+    if serializer == "npy":
+        import numpy as np
+
+        return np.load(path, allow_pickle=False)
+    if serializer == "json":
+        return json.loads(path.read_text(encoding="utf-8"))
+    if serializer == "pickle":
+        with path.open("rb") as handle:
+            return pickle.load(handle)
+    raise ValueError(f"unknown serializer {serializer!r} for output '{name}'")

starforge/core/spec.py

@@ -0,0 +1,126 @@
+"""The ``.forge`` pipeline document model.
+
+Documents are plain JSON text — the VS Code custom editor is text-based so
+undo/redo/diff/merge come for free. Layout and notes are durable metadata and
+never participate in provenance hashing.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import json
+from pathlib import Path
+from typing import Any
+
+SCHEMA = "starforge/1"
+
+
+@dataclass
+class Node:
+    id: str
+    block: str  # "dotted.module:qualname"
+    params: dict[str, Any] = field(default_factory=dict)  # literals for unconnected params
+    position: dict[str, float] = field(default_factory=lambda: {"x": 0.0, "y": 0.0})
+    notes: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "block": self.block,
+            "params": self.params,
+            "position": self.position,
+            "notes": self.notes,
+        }
+
+
+@dataclass
+class Edge:
+    id: str
+    source: str
+    target: str
+    target_param: str  # parameter NAME on the target function — never a slot index
+    source_output: str = "output"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "source": self.source,
+            "source_output": self.source_output,
+            "target": self.target,
+            "target_param": self.target_param,
+        }
+
+
+@dataclass
+class PipelineDoc:
+    name: str = "Untitled"
+    nodes: list[Node] = field(default_factory=list)
+    edges: list[Edge] = field(default_factory=list)
+    #: Canvas annotation boxes: [{id, title, description, position, width,
+    #: height, color}]. Pure layout metadata — round-trips untouched, never
+    #: participates in hashing or execution.
+    comments: list[dict[str, Any]] = field(default_factory=list)
+
+    def node(self, node_id: str) -> Node:
+        for node in self.nodes:
+            if node.id == node_id:
+                return node
+        raise KeyError(node_id)
+
+    def in_edges(self, node_id: str) -> list[Edge]:
+        return [e for e in self.edges if e.target == node_id]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema": SCHEMA,
+            "name": self.name,
+            "nodes": [n.to_dict() for n in self.nodes],
+            "edges": [e.to_dict() for e in self.edges],
+            "comments": self.comments,
+        }
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), indent=2)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "PipelineDoc":
+        schema = d.get("schema", SCHEMA)
+        if schema != SCHEMA:
+            raise ValueError(f"unsupported .forge schema {schema!r} (expected {SCHEMA!r})")
+        nodes = [
+            Node(
+                id=n["id"],
+                block=n["block"],
+                params=dict(n.get("params", {})),
+                position=dict(n.get("position", {"x": 0.0, "y": 0.0})),
+                notes=n.get("notes", ""),
+            )
+            for n in d.get("nodes", [])
+        ]
+        edges = [
+            Edge(
+                id=e["id"],
+                source=e["source"],
+                source_output=e.get("source_output", "output"),
+                target=e["target"],
+                target_param=e["target_param"],
+            )
+            for e in d.get("edges", [])
+        ]
+        return cls(
+            name=d.get("name", "Untitled"),
+            nodes=nodes,
+            edges=edges,
+            comments=list(d.get("comments", [])),
+        )
+
+    @classmethod
+    def from_json(cls, text: str) -> "PipelineDoc":
+        return cls.from_dict(json.loads(text))
+
+    @classmethod
+    def load(cls, path: str | Path) -> "PipelineDoc":
+        return cls.from_json(Path(path).read_text(encoding="utf-8"))
+
+    def save(self, path: str | Path) -> None:
+        Path(path).write_text(self.to_json() + "\n", encoding="utf-8")

starforge/index/__init__.py

@@ -0,0 +1,9 @@
+from starforge.index.scanner import (
+    BlockInfo,
+    ModuleInfo,
+    ParamInfo,
+    WorkspaceIndex,
+    scan_workspace,
+)
+
+__all__ = ["BlockInfo", "ModuleInfo", "ParamInfo", "WorkspaceIndex", "scan_workspace"]