openprocess 0.7.0__py3-none-any.whl

openprocess/flow/runner.py

@@ -0,0 +1,357 @@
+"""Running a workflow: in order, only what changed, each box on its own.
+
+The :class:`Runner` walks the workflow in topological order.  Every box's
+result is kept under a **key** made of the box's code, its settings and the
+keys of its inputs, so
+
+* changing one setting re-runs only the boxes after it,
+* re-opening a workflow whose inputs are unchanged runs nothing at all,
+* the same box with the same inputs anywhere in any workflow is computed once.
+
+An exception stays in its box: the box fails with the message and the
+traceback, the boxes after it wait, and the rest keeps its results.
+
+Sweeps (:mod:`.sweep`): the boxes after a swept setting run once per value;
+a box that takes any number of inputs collects the results of every value.
+
+The runner is plain Python; the app calls :meth:`Runner.run` on a worker
+thread and listens to ``on_status``.  ``stop`` (a ``threading.Event``) is
+checked between boxes.  A box marked ``heavy=True`` can be run in a
+separate process with :func:`run_in_process`, which can be killed.
+"""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import threading
+import time
+import traceback
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from .box import BoxSpec, Port
+from .convert import can_convert, convert
+from .explain import Explanation, collect_result, explaining
+from .sweep import assignments, is_sweep
+from .workflow import Node, Workflow
+
+WAITING, RUNNING, DONE, FAILED, IDLE, BLOCKED, SKIPPED = (
+    "waiting", "running", "done", "failed", "idle", "blocked", "skipped")
+
+
+@dataclass
+class Result:
+    """What one box produced in one run (one per sweep variant)."""
+
+    node: str
+    status: str = WAITING
+    value: Any = None
+    values: dict[str, Any] = field(default_factory=dict)   # by output name
+    error: str = ""
+    traceback: str = ""
+    message: str = ""                                       # why it is idle
+    duration: float = 0.0
+    explanation: Explanation = field(default_factory=Explanation)
+    key: str = ""
+    cached: bool = False
+    variant: int = 0
+
+    @property
+    def ok(self) -> bool:
+        return self.status == DONE
+
+
+@dataclass
+class Run:
+    """Everything a run of a workflow produced."""
+
+    workflow: Workflow
+    #: ``(node id, variant)`` -> Result.  Variant 0 is the only one without sweeps.
+    results: dict[tuple[str, int], Result] = field(default_factory=dict)
+    #: The swept settings of each variant: ``{(node, setting): value}``.
+    variants: list[dict[tuple[str, str], Any]] = field(default_factory=list)
+    started: float = 0.0
+    finished: float = 0.0
+    stopped: bool = False
+
+    def result(self, node: Node | str, variant: int = 0) -> Result | None:
+        node_id = node if isinstance(node, str) else node.id
+        return self.results.get((node_id, variant))
+
+    def value(self, node: Node | str, output: str = "out", variant: int = 0) -> Any:
+        result = self.result(node, variant)
+        if result is None or not result.ok:
+            return None
+        return result.values.get(output, result.value)
+
+    def status(self, node: Node | str, variant: int = 0) -> str:
+        result = self.result(node, variant)
+        return result.status if result else WAITING
+
+    def failed(self) -> list[Result]:
+        return [r for r in self.results.values() if r.status == FAILED]
+
+    @property
+    def ok(self) -> bool:
+        return all(r.status == DONE for r in self.results.values())
+
+    def summary(self) -> str:
+        done = sum(1 for r in self.results.values() if r.status == DONE)
+        return (f"{done} of {len(self.results)} boxes done"
+                + (f", {len(self.failed())} failed" if self.failed() else "")
+                + (f", {len(self.variants)} sweep variants" if len(self.variants) > 1 else ""))
+
+
+class Cache:
+    """Results by key.  In memory; the app keeps one per folder."""
+
+    def __init__(self, limit: int = 500) -> None:
+        self.entries: dict[str, tuple[dict[str, Any], Explanation]] = {}
+        self.limit = limit
+
+    def get(self, key: str):
+        return self.entries.get(key)
+
+    def put(self, key: str, values: dict[str, Any], explanation: Explanation) -> None:
+        if len(self.entries) >= self.limit:
+            del self.entries[next(iter(self.entries))]
+        self.entries[key] = (values, explanation)
+
+    def clear(self) -> None:
+        self.entries.clear()
+
+
+def settings_key(settings: dict[str, Any]) -> str:
+    return json.dumps({k: (str(v) if not isinstance(v, (int, float, bool, str, type(None), list, dict)) else v)
+                       for k, v in sorted(settings.items())}, sort_keys=True, default=str)
+
+
+class Runner:
+    def __init__(self, library=None, cache: Cache | None = None, folder=None) -> None:
+        """``folder`` is where a workflow's relative file settings point (its
+        own folder in the app); without it they are taken as they are."""
+        from .library import standard_library
+        self.library = library or standard_library()
+        self.cache = cache or Cache()
+        self.folder = folder
+        self._per_variant: set[str] = set()
+
+    # -- the public call ----------------------------------------------------------------
+    def run(self, workflow: Workflow, changed: list[str] | None = None, previous: Run | None = None,
+            stop: threading.Event | None = None,
+            on_status: Callable[[Result], None] | None = None) -> Run:
+        """Run the workflow (all of it, or from ``changed`` on, reusing
+        ``previous`` for the rest).  Returns a :class:`Run`."""
+        run = Run(workflow, started=time.time())
+        order = workflow.order()
+        swept = workflow.swept()
+        run.variants = assignments(swept)
+        swept_nodes = {node for node, _, _ in swept}
+        under_sweep = workflow.descendants(swept_nodes) if swept_nodes else set()
+        collectors = {n.id for n in order if n.id in under_sweep
+                      and any(p.many for p in workflow.spec(n).inputs)
+                      and self._fed_from(workflow, n, under_sweep)}
+        after_collectors = workflow.descendants(collectors) if collectors else set()
+        per_variant = under_sweep - after_collectors
+        self._per_variant = per_variant
+        dirty = workflow.descendants(changed) if changed is not None else set(workflow.nodes)
+        if previous is None or previous.variants != run.variants:
+            dirty = set(workflow.nodes)
+        notify = on_status or (lambda r: None)
+
+        def reuse(node_id: str, variant: int) -> bool:
+            if node_id in dirty or previous is None:
+                return False
+            old = previous.result(node_id, variant)
+            if old is None:
+                return False
+            kept = copy.copy(old)
+            kept.cached = True
+            run.results[(node_id, variant)] = kept
+            return True
+
+        for variant_index, assignment in enumerate(run.variants):
+            if variant_index > 0 and not per_variant:
+                break
+            for node in order:
+                if node.id in after_collectors or node.id in collectors:
+                    continue
+                if node.id not in per_variant and variant_index > 0:
+                    continue
+                if stop is not None and stop.is_set():
+                    run.stopped = True
+                    run.finished = time.time()
+                    return run
+                if reuse(node.id, variant_index):
+                    continue
+                result = self._run_node(workflow, node, run, variant_index, assignment, notify)
+                run.results[(node.id, variant_index)] = result
+                notify(result)
+        for node in order:
+            if node.id not in collectors and node.id not in after_collectors:
+                continue
+            if stop is not None and stop.is_set():
+                run.stopped = True
+                break
+            if reuse(node.id, 0):
+                continue
+            result = self._run_node(workflow, node, run, 0, {}, notify, collect=node.id in collectors)
+            run.results[(node.id, 0)] = result
+            notify(result)
+        run.finished = time.time()
+        return run
+
+    @staticmethod
+    def _fed_from(workflow: Workflow, node: Node, nodes: set[str]) -> bool:
+        return any(e.source in nodes for e in workflow.edges if e.target == node.id)
+
+    # -- one box ---------------------------------------------------------------------------
+    def _run_node(self, workflow: Workflow, node: Node, run: Run, variant: int,
+                  assignment: dict, notify, collect: bool = False) -> Result:
+        spec = workflow.spec(node)
+        result = Result(node.id, variant=variant)
+        settings = dict(node.settings)
+        for (node_id, name), value in assignment.items():
+            if node_id == node.id:
+                settings[name] = value
+        if any(is_sweep(v) for v in settings.values()):          # a swept node outside its variants
+            for name, value in list(settings.items()):
+                if is_sweep(value):
+                    settings[name] = value.values[0]
+        if not spec.available:
+            result.status, result.message = BLOCKED, spec.unavailable_reason
+            return result
+        if self.folder is not None:
+            from pathlib import Path
+            for setting in spec.settings:
+                value = settings.get(setting.name)
+                if setting.kind == "path" and value and not Path(value).is_absolute():
+                    settings[setting.name] = Path(self.folder) / value
+        # Gather inputs
+        inputs: dict[str, Any] = {}
+        keys: list[str] = []
+        fed = workflow.inputs_of(node)
+        for port in spec.inputs:
+            sources = fed[port.name]
+            if not sources:
+                if port.optional:
+                    inputs[port.name] = [] if port.many else None
+                    continue
+                result.status = IDLE
+                result.message = f"Connect a {port.type_name.lower()} to “{port.label or port.name}”"
+                return result
+            values = []
+            for source_id, output in sources:
+                variants = (range(len(run.variants)) if collect and source_id in self._per_variant
+                            else [variant if source_id in self._per_variant else 0])
+                for v in variants:
+                    upstream = run.result(source_id, v) or run.result(source_id, 0)
+                    if upstream is None or upstream.status != DONE:
+                        if port.many and upstream is not None and upstream.status in (IDLE, FAILED, BLOCKED) and collect:
+                            continue
+                        result.status = IDLE
+                        why = {FAILED: "failed", BLOCKED: "is not allowed to run", IDLE: "is waiting"}.get(
+                            upstream.status if upstream else WAITING, "has not run")
+                        result.message = f"Waiting for “{workflow.title(source_id)}” ({why})"
+                        return result
+                    value = upstream.values.get(output, upstream.value)
+                    values.append(self._fit(value, port))
+                    keys.append(upstream.key + ":" + output)
+            inputs[port.name] = values if port.many else values[0]
+        result.key = hashlib.sha256(
+            (spec.fingerprint + "|" + settings_key(settings) + "|" + "|".join(keys)).encode()).hexdigest()
+        cached = self.cache.get(result.key)
+        if cached is not None:
+            result.values, result.explanation = cached
+            result.value = result.values.get("out", next(iter(result.values.values()), None))
+            result.status, result.cached = DONE, True
+            return result
+        # Run it
+        result.status = RUNNING
+        notify(result)
+        started = time.perf_counter()
+        with explaining() as explanation:
+            try:
+                value = spec.function(**inputs, **settings)
+            except Exception as error:   # noqa: BLE001 - whatever the box raised stays in the box
+                result.status = FAILED
+                result.error = "".join(traceback.format_exception_only(type(error), error)).strip()
+                result.traceback = traceback.format_exc()
+                result.duration = time.perf_counter() - started
+                result.explanation = explanation
+                return result
+        result.duration = time.perf_counter() - started
+        values = self._outputs(spec, value)
+        for out in values.values():
+            if node.id in self._per_variant:
+                self._tag(out, assignment, spec)
+            collect_result(out, explanation)
+        result.values, result.value, result.explanation = values, values.get("out", value), explanation
+        result.status = DONE
+        self.cache.put(result.key, values, explanation)
+        return result
+
+    @staticmethod
+    def _outputs(spec: BoxSpec, value) -> dict[str, Any]:
+        if spec.result_type is not None:
+            return {p.name: getattr(value, p.name) for p in spec.outputs}
+        if not spec.outputs:
+            return {}
+        return {"out": value}
+
+    @staticmethod
+    def _fit(value, port: Port):
+        """Convert a value to the input's type when a converter exists."""
+        if port.type is None or value is None or isinstance(value, port.type):
+            return value
+        if can_convert(value, port.type):
+            return convert(value, port.type)
+        return value
+
+    @staticmethod
+    def _tag(value, assignment: dict, spec: BoxSpec) -> None:
+        """Scores and tables made under a sweep remember the swept values."""
+        context = getattr(value, "context", None)
+        if isinstance(context, dict) and assignment:
+            for (_node, name), v in assignment.items():
+                context.setdefault(name, v)
+
+
+# ---------------------------------------------------------------------------
+# Heavy boxes in a process that can be killed
+# ---------------------------------------------------------------------------
+def _in_process(box_id: str, inputs: dict, settings: dict, queue) -> None:   # pragma: no cover - child
+    try:
+        from .library import standard_library
+        spec = standard_library().get(box_id)
+        with explaining() as explanation:
+            value = spec.function(**inputs, **settings)
+        queue.put(("ok", value, explanation))
+    except Exception as error:   # noqa: BLE001
+        queue.put(("error", "".join(traceback.format_exception_only(type(error), error)).strip(),
+                   traceback.format_exc()))
+
+
+def run_in_process(spec: BoxSpec, inputs: dict, settings: dict, timeout: float | None = None):
+    """Run one box in a separate process (standard-library boxes only, since
+    the child imports the box by id).  Returns ``(value, explanation)`` or
+    raises ``RuntimeError`` with the child's error."""
+    import multiprocessing
+    queue = multiprocessing.Queue()
+    process = multiprocessing.Process(target=_in_process, args=(spec.id, inputs, settings, queue))
+    process.start()
+    try:
+        outcome = queue.get(timeout=timeout)
+    except Exception:   # noqa: BLE001 - queue.Empty or a dead child
+        process.kill()
+        raise RuntimeError(f"{spec.name} did not finish in time and was stopped")
+    process.join()
+    if outcome[0] == "error":
+        raise RuntimeError(outcome[1])
+    return outcome[1], outcome[2]
+
+
+__all__ = ["BLOCKED", "Cache", "DONE", "FAILED", "IDLE", "RUNNING", "Result", "Run", "Runner",
+           "WAITING", "run_in_process", "settings_key"]

openprocess/flow/sweep.py

@@ -0,0 +1,92 @@
+"""Sweeps: run the boxes after a setting once per value.
+
+A sweep is a setting marked as a range or a list rather than a loop drawn on
+the canvas::
+
+    node.settings["noise"] = Sweep.parse("0..0.5 step 0.1")
+    node.settings["seed"] = Sweep([0, 1, 2, 3, 4])
+
+The runner runs every box after a swept box once per value (the cartesian
+product when several settings are swept), and a box that takes *any
+number* of inputs (Compare, Plot, Sweep table) collects the results of all
+the values.  Every :class:`~.types.Scores` and :class:`~.types.Table` made
+under a sweep carries the values in its ``context``, so a stacked table
+tells them apart.
+"""
+
+from __future__ import annotations
+
+import itertools
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+_RANGE = re.compile(r"^\s*(-?[\d.]+)\s*\.\.\s*(-?[\d.]+)\s*(?:step\s+(-?[\d.]+))?\s*$")
+
+
+@dataclass(frozen=True)
+class Sweep:
+    """The values a setting takes, one run each."""
+
+    values: tuple = field(default_factory=tuple)
+
+    @classmethod
+    def parse(cls, text: str) -> "Sweep":
+        """``"0..0.5 step 0.1"``, ``"1..5"`` (step 1) or ``"a, b, c"``."""
+        match = _RANGE.match(text)
+        if match:
+            start, stop, step = (float(match.group(1)), float(match.group(2)),
+                                 float(match.group(3) or 1))
+            if step == 0 or (stop - start) * step < 0:
+                raise ValueError(f"The range {text!r} never reaches its end")
+            values, value, count = [], start, 0
+            while (value <= stop + 1e-9 if step > 0 else value >= stop - 1e-9) and count < 10_000:
+                values.append(round(value, 10))
+                value += step
+                count += 1
+            if all(float(v).is_integer() for v in values) and all(
+                    "." not in part for part in (match.group(1), match.group(2), match.group(3) or "")):
+                values = [int(v) for v in values]
+            return cls(tuple(values))
+        parts = [p.strip() for p in text.split(",") if p.strip()]
+        if not parts:
+            raise ValueError("A sweep needs at least one value")
+        return cls(tuple(_number(p) for p in parts))
+
+    def describe(self) -> str:
+        values = self.values
+        if len(values) > 6:
+            return f"{values[0]} … {values[-1]} ({len(values)} values)"
+        return ", ".join(str(v) for v in values)
+
+    def __len__(self) -> int:
+        return len(self.values)
+
+    def to_json(self) -> dict:
+        return {"sweep": list(self.values)}
+
+
+def _number(text: str):
+    try:
+        return int(text)
+    except ValueError:
+        try:
+            return float(text)
+        except ValueError:
+            return text
+
+
+def is_sweep(value: Any) -> bool:
+    return isinstance(value, Sweep)
+
+
+def assignments(swept: list[tuple[str, str, Sweep]]) -> list[dict[tuple[str, str], Any]]:
+    """Every combination of values: ``[{(node, setting): value, ...}, ...]``."""
+    if not swept:
+        return [{}]
+    keys = [(node, setting) for node, setting, _ in swept]
+    return [dict(zip(keys, combination))
+            for combination in itertools.product(*(s.values for _, _, s in swept))]
+
+
+__all__ = ["Sweep", "assignments", "is_sweep"]