hyperstudy-perception 0.1.0__py3-none-any.whl

perception/keys.py

@@ -0,0 +1,259 @@
+"""The user's keys (API keys, tokens) for nodes and cloud places, found in this order:
+
+1. the environment (a name already known below, or any *_KEY / *_TOKEN);
+2. the system keychain (macOS Keychain, Linux Secret Service, Windows Credential Manager), where `perception keys set`
+   stores them, under the service "hyperstudy-perception";
+3. NAME=value lines in ~/.config/hyperstudy/*.env, only from files no one else can read or write (0600, the user's own).
+   A file others can read is refused, not read, and said so: a key in it may already have been read by someone else.
+
+Values are never printed, logged or written to run files. The keychain cannot list what it holds, so the names stored
+there are kept in ~/.config/hyperstudy/keys.json (names only)."""
+
+from __future__ import annotations
+
+import contextlib
+import errno
+import json
+import os
+import re
+import stat
+import sys
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+
+SERVICE = "hyperstudy-perception"
+NAME = re.compile(r"^[A-Z][A-Z0-9_]*$")
+
+
+def conf_dir() -> Path:
+    return Path.home() / ".config" / "hyperstudy"
+
+
+@dataclass
+class Found:
+    values: dict[str, str] = field(default_factory=dict)
+    sources: dict[str, str] = field(default_factory=dict)   # name -> "environment", "keychain" or the file's path
+    problems: list[str] = field(default_factory=list)
+
+
+POSIX = os.name != "nt"   # Windows has no POSIX modes (every file reads 0o666): a user's profile folder is their own there
+
+
+def folder_problem(conf: Path) -> str | None:
+    """Why nothing in the config folder is used, or None: it must be a real folder (not a link), the user's own, and one
+    no one else can write to (else a file could be swapped in after it was checked)."""
+    try:
+        st = conf.lstat()
+    except FileNotFoundError:
+        return None
+    if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
+        return f"{conf} is not a folder (a link?): its key files are not used"
+    if POSIX and st.st_uid != os.getuid():
+        return f"{conf} is not yours: its key files are not used"
+    if POSIX and st.st_mode & 0o022:
+        return f"{conf} is a folder others can write to (mode {oct(stat.S_IMODE(st.st_mode))}): its key files are not used; chmod 700 it"
+    return None
+
+
+def file_problem(path: Path, st: os.stat_result) -> str | None:
+    """Why a key file is not used, or None: a regular file, the user's own, readable and writable by no one else."""
+    if not stat.S_ISREG(st.st_mode):
+        return f"{path} is not a regular file: its keys are not used"
+    if POSIX and st.st_uid != os.getuid():
+        return f"{path} is not yours: its keys are not used"
+    if POSIX and st.st_mode & 0o077:
+        return f"{path} can be read by others (mode {oct(stat.S_IMODE(st.st_mode))}): its keys are not used; chmod 600 it"
+    return None
+
+
+def parse_env(text: str) -> dict[str, str]:
+    out = {}
+    for line in text.splitlines():
+        if "=" in line and not line.lstrip().startswith("#"):
+            k, v = line.split("=", 1)
+            out[k.strip()] = v.strip()
+    return out
+
+
+def read_private(path: Path) -> tuple[dict[str, str], str | None]:
+    """A key file's keys, or why it is refused: opened without following a link, and checked on that open file, so what
+    is checked is what is read."""
+    try:
+        fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+    except OSError as e:
+        if e.errno == errno.ELOOP:
+            return {}, f"{path} is not a regular file (a link): its keys are not used"
+        return {}, f"{path} could not be read ({e.strerror})"
+    with os.fdopen(fd) as f:
+        problem = file_problem(path, os.fstat(f.fileno()))
+        return ({}, problem) if problem else (parse_env(f.read()), None)
+
+
+def keyring_module():
+    """The keyring library, or None where no keychain can be used (a headless server, or PERCEPTION_NO_KEYCHAIN)."""
+    if os.environ.get("PERCEPTION_NO_KEYCHAIN"):
+        return None
+    import keyring
+    from keyring.backends import fail
+    return None if isinstance(keyring.get_keyring(), fail.Keyring) else keyring
+
+
+class Keys:
+    def __init__(self, conf: Path | None = None):
+        self.conf = conf or conf_dir()
+        self.index = self.conf / "keys.json"
+
+    def stored_names(self) -> list[str]:
+        """Names `keys set` stored in the keychain (keys.json: names only)."""
+        try:
+            return [n for n in json.loads(self.index.read_text()) if isinstance(n, str) and NAME.match(n)]
+        except (OSError, ValueError, TypeError):
+            return []
+
+    def _private_folder(self):
+        """The config folder, made if missing and made private if it is not (0700): refused if it is a link or not the
+        user's (folder_problem)."""
+        self.conf.mkdir(mode=0o700, parents=True, exist_ok=True)
+        st = self.conf.lstat()
+        if POSIX and not stat.S_ISLNK(st.st_mode) and st.st_uid == os.getuid() and st.st_mode & 0o077:
+            os.chmod(self.conf, 0o700)
+        problem = folder_problem(self.conf)
+        if problem:
+            raise ValueError(problem)
+
+    def _write(self, path: Path, text: str):
+        """Write a file in one step through a new temporary file of its own (0600 from its first byte, never one left
+        over), in the private config folder."""
+        self._private_folder()
+        fd, tmp = tempfile.mkstemp(dir=self.conf, prefix=f".{path.name}.")
+        try:
+            with os.fdopen(fd, "w") as f:
+                f.write(text)
+            os.replace(tmp, path)
+        except BaseException:
+            with contextlib.suppress(OSError):
+                os.unlink(tmp)
+            raise
+
+    @contextlib.contextmanager
+    def _locked(self):
+        """One change to keys.json and keys.env at a time (two `keys set` at once would drop a name otherwise)."""
+        self._private_folder()
+        try:
+            import fcntl
+        except ImportError:   # Windows: no flock; a single user's own commands
+            yield
+            return
+        fd = os.open(self.conf / ".keys.lock", os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            yield
+        finally:
+            os.close(fd)
+
+    def _write_index(self, names: list[str]):
+        self._write(self.index, json.dumps(sorted(set(names))))
+
+    def _file_keys(self) -> dict[str, str]:
+        """What keys.env holds (the file `set` writes where there is no keychain), refusing one others can read."""
+        path = self.conf / "keys.env"
+        if not path.exists():
+            return {}
+        values, problem = read_private(path)
+        if problem:
+            raise ValueError(problem)
+        return values
+
+    def _write_file_keys(self, keys: dict[str, str]):
+        self._write(self.conf / "keys.env", "".join(f"{k}={v}\n" for k, v in sorted(keys.items())))
+
+    def load(self) -> Found:
+        found = Found()
+        # 3. private files, lowest: what a keychain or the environment holds wins
+        problem = folder_problem(self.conf)
+        if problem:
+            found.problems.append(problem)
+        for path in sorted(self.conf.glob("*.env")) if self.conf.is_dir() and not problem else []:
+            values, refused = read_private(path)
+            if refused:
+                found.problems.append(refused)
+            for k, v in values.items():
+                found.values[k], found.sources[k] = v, str(path)
+        # 2. the keychain
+        kr = keyring_module()
+        if kr is not None:
+            for name in self.stored_names():
+                try:
+                    value = kr.get_password(SERVICE, name)
+                except Exception as e:   # a locked or refused keychain: said, not fatal (other sources may have it)
+                    found.problems.append(f"the keychain did not give {name} ({type(e).__name__})")
+                    continue
+                if value:
+                    found.values[name], found.sources[name] = value, "keychain"
+        # 1. the environment
+        for k, v in os.environ.items():
+            if v and (k in found.values or k.endswith(("_KEY", "_TOKEN"))):
+                found.values[k], found.sources[k] = v, "environment"
+        return found
+
+    def set(self, name: str, value: str) -> str:
+        """Store a key: in the keychain, or (no keychain here) in ~/.config/hyperstudy/keys.env, 0600. Where it went."""
+        if not NAME.match(name):
+            raise ValueError(f"a key's name is upper-case letters, digits and _ (like RUNPOD_API_KEY): {name!r}")
+        value = value.strip()
+        if not value or value.splitlines() != [value]:   # any line break (\r, \x85, \u2028...) would add a line to keys.env
+            raise ValueError("a key is one non-empty line")
+        kr = keyring_module()
+        with self._locked():
+            if kr is not None:
+                from keyring.errors import KeyringError
+                try:
+                    kr.set_password(SERVICE, name, value)
+                except KeyringError as e:   # locked, or access refused: nothing stored, said plainly
+                    raise ValueError(f"the keychain did not store {name} ({e or type(e).__name__}): unlock it and try again") from None
+                self._write_index([*self.stored_names(), name])
+                return "keychain"
+            keys = self._file_keys()
+            keys[name] = value
+            self._write_file_keys(keys)
+            return str(self.conf / "keys.env")
+
+    def remove(self, name: str) -> list[str]:
+        """Remove a key from the keychain and from keys.env (what `set` wrote); where it was. The user's own .env files
+        are theirs to edit."""
+        with self._locked():
+            return self._remove(name)
+
+    def _remove(self, name: str) -> list[str]:
+        removed = []
+        kr = keyring_module()
+        if kr is not None and name in self.stored_names():
+            from keyring.errors import KeyringError, PasswordDeleteError
+            try:
+                kr.delete_password(SERVICE, name)
+                removed.append("keychain")
+            except PasswordDeleteError:
+                pass   # already gone from the keychain (removed there by hand)
+            except KeyringError as e:   # locked, or access refused: still stored, and still in the index
+                raise ValueError(f"the keychain did not remove {name} ({e or type(e).__name__}): unlock it and try again") from None
+            self._write_index([n for n in self.stored_names() if n != name])
+        keys = self._file_keys()
+        if keys.pop(name, None) is not None:
+            self._write_file_keys(keys)
+            removed.append(str(self.conf / "keys.env"))
+        return removed
+
+
+_warned = False
+
+
+def load_secrets() -> dict[str, str]:
+    """Every key found (Keys.load), saying once on stderr which files were refused and why."""
+    global _warned
+    found = Keys().load()
+    if found.problems and not _warned:
+        _warned = True
+        for p in found.problems:
+            print(f"perception: {p}", file=sys.stderr)
+    return found.values

perception/manifests.py

@@ -0,0 +1,169 @@
+"""Node manifests: what a node needs, produces and runs on. One `manifest.yaml` per node directory under `nodes/`.
+
+A manifest is the only thing the coordinator knows about a node. The graph refers to it as `uses: <id>@<version>`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+DATA_KINDS = ("video", "audio", "frames", "faces", "transcript", "embeddings")
+
+
+class Strict(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class Runner(Strict):
+    """One way to run the node. `native`: a script in a uv environment (on this machine, a cloud machine or a lab
+    server). `self`: the coordinator's own interpreter (tests and trivial nodes only). No containers (2026-10-24):
+    every node installs with uv, and Docker on macOS cannot reach the Apple GPU."""
+    kind: Literal["native", "self"]
+    entry: str | None = None                 # native/self: script path, relative to the node directory
+    env: str | None = None                   # native: uv project, relative to the repository root
+    platforms: list[str] = Field(default_factory=lambda: ["darwin/arm64", "linux/amd64", "linux/arm64", "windows/amd64"])
+    accelerator: Literal["cpu", "mps", "cuda"] = "cpu"
+    minimum_vram_gb: float = 0
+
+
+class Resources(Strict):
+    cpu_cores: float = 1
+    memory_gb: float = 1
+    # exclusive: one such node at a time on a device (large models; measured: two at once swapped a 128 GB Mac)
+    gpu: Literal["none", "shared", "exclusive"] = "none"
+
+
+class Execution(Strict):
+    partitionable: bool = False
+    deterministic: bool = True
+
+
+class Privacy(Strict):
+    external_upload: list[Literal["video", "audio", "frames", "faces", "transcript", "embeddings"]] = Field(default_factory=list)
+
+
+class Cost(Strict):
+    """A billable node's estimate, reserved against the run's budget before it starts; what it actually spent comes
+    back in its response (usage.usd). `per` counts units in one of the node's inputs. When the price depends on a
+    config value (the model), `usd` is a table keyed by that value and `by` names the config key."""
+    usd: float | dict[str, float]                # per unit, an upper estimate
+    by: str | None = None                        # the config key `usd` is keyed by
+    per: Literal["shot", "minute", "run"]
+    input: str | None = None                     # the input holding the units (shots layer; media for minutes)
+
+    def unit_usd(self, config: dict) -> float:
+        return self.usd[config[self.by]] if isinstance(self.usd, dict) else self.usd
+
+
+class ModelRef(Strict):
+    """A model the node uses: where its weights come from and under what terms."""
+    id: str
+    source: str | None = None                # hf://repo@revision[/file], or a URL
+    revision: str | None = None
+    licence: str | None = None
+    licence_mode: Literal["academic-only", "commercial-ok", "needs-review", "api-terms"]
+    size_gb: float | None = None
+    gated: bool = False
+
+
+class Manifest(Strict):
+    id: str
+    version: str
+    title: str = ""                          # what it gives, in a few words: the app's feature list ("Face tracks")
+    service: str | None = None               # the hosted API whose machines do the work ("Anthropic API"): it runs there, not here
+    description: str = ""
+    stage: str | None = None                 # the bundle producer's stage name (default: the node id)
+    inputs: dict[str, str] = Field(default_factory=dict)      # name -> artifact type
+    outputs: dict[str, str]                                    # name -> artifact type
+    config: dict[str, dict] = Field(default_factory=dict)     # name -> JSON Schema (with default)
+    runners: list[Runner]
+    resources: Resources = Field(default_factory=Resources)
+    execution: Execution = Field(default_factory=Execution)
+    privacy: Privacy = Field(default_factory=Privacy)
+    models: list[ModelRef] = Field(default_factory=list)
+    # a script in the node's directory, run in its environment with its settings (JSON) before it is first needed, that
+    # fetches what the node would otherwise download on first use (its weights): `perception setup`, a remote run
+    prepare: str | None = None
+    secrets: list[str] = Field(default_factory=list)
+    licence: dict = Field(default_factory=dict)
+    cost: Cost | None = None
+
+    directory: Path = Field(default=Path("."), exclude=True)
+
+    @model_validator(mode="after")
+    def _cost_counts_an_input(self):
+        if self.cost and self.cost.per != "run":
+            if self.cost.input not in self.inputs:
+                raise ValueError(f"cost: per {self.cost.per} counts units in an input, but cost.input "
+                                 f"{self.cost.input!r} is not one of this node's inputs ({', '.join(self.inputs) or 'none'})")
+            # what manifest_cost reads: the shots of a shots layer, or the duration of media.v1
+            kind = self.inputs[self.cost.input]
+            if (self.cost.per == "shot") != kind.startswith("bundle.shots@") or (self.cost.per == "minute") != (kind == "media.v1"):
+                raise ValueError(f"cost: per {self.cost.per} cannot be counted in {self.cost.input} ({kind}); "
+                                 "shots come from a bundle.shots layer, minutes from media.v1")
+        return self
+
+    @model_validator(mode="after")
+    def _cost_covers_its_config(self):
+        """A price table must price every value the config allows, or a new model would be reserved at nothing."""
+        if self.cost is None or not (isinstance(self.cost.usd, dict) or self.cost.by):
+            return self
+        if not (isinstance(self.cost.usd, dict) and self.cost.by):
+            raise ValueError("cost: a price table (usd: {value: price}) and `by` go together")
+        if self.cost.by not in self.config or "enum" not in self.config[self.cost.by]:
+            raise ValueError(f"cost.by: {self.cost.by!r} is not a config key with an enum")
+        missing = [v for v in self.config[self.cost.by]["enum"] if v not in self.cost.usd]
+        if missing:
+            raise ValueError(f"cost: no price for {self.cost.by} {', '.join(map(repr, missing))}")
+        return self
+
+    @property
+    def ref(self) -> str:
+        return f"{self.id}@{self.version}"
+
+    def defaults(self) -> dict:
+        return {k: s["default"] for k, s in self.config.items() if "default" in s}
+
+    def code_hash(self) -> str:
+        """Hash of every file in the node's directory: changing node code invalidates its cache."""
+        h = hashlib.sha256()
+        for p in sorted(self.directory.rglob("*")):
+            if p.is_file() and "__pycache__" not in p.parts:
+                h.update(str(p.relative_to(self.directory)).encode())
+                h.update(p.read_bytes())
+        return h.hexdigest()
+
+
+class Registry:
+    def __init__(self, manifests: list[Manifest]):
+        self.by_ref: dict[str, Manifest] = {}
+        for m in manifests:
+            if m.ref in self.by_ref:
+                raise ValueError(f"two manifests define {m.ref}: {self.by_ref[m.ref].directory} and {m.directory}")
+            self.by_ref[m.ref] = m
+
+    @classmethod
+    def load(cls, *roots: Path) -> "Registry":
+        manifests = []
+        for root in roots:
+            for path in sorted(root.rglob("manifest.yaml")):
+                doc = yaml.safe_load(path.read_text())
+                m = Manifest.model_validate(doc)
+                m.directory = path.parent.resolve()
+                manifests.append(m)
+        return cls(manifests)
+
+    def get(self, ref: str) -> Manifest:
+        if ref not in self.by_ref:
+            same_id = [r for r in self.by_ref if r.split("@")[0] == ref.split("@")[0]]
+            hint = f" (available: {', '.join(same_id)})" if same_id else ""
+            raise KeyError(f"no node {ref}{hint}")
+        return self.by_ref[ref]
+
+    def __iter__(self):
+        return iter(self.by_ref.values())

perception/plan.py

@@ -0,0 +1,170 @@
+"""Dry run: what `perception run` would do, without doing it (spec acceptance criterion 6).
+
+For each node: where it would run (or why it cannot), whether its result is already cached, which model weights are
+not on this machine yet, which secrets are missing, and the estimated spend. Privacy conflicts are graph errors
+(ir.py), so a graph that reaches planning has none.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .estimates import estimate, media_seconds, probe_seconds
+from .ir import Graph
+from .manifests import ModelRef
+from .scheduler import Coordinator, manifest_cost
+from .store import Artifact
+
+
+@dataclass
+class NodePlan:
+    id: str
+    uses: str
+    where: str | None
+    why_not: str = ""
+    cached: bool = False
+    downloads: list[ModelRef] = field(default_factory=list)
+    missing_secrets: list[str] = field(default_factory=list)
+    cost_usd: float = 0.0
+    cost_note: str = ""
+    uploads: list[str] = field(default_factory=list)   # data kinds that leave the machine where it would run: to its API, or its inputs to another machine
+    upload_to: str | None = None                       # where they go: the API's service, or the other machine
+    place: str | None = None                           # where it would run (backend:accelerator)
+    options: list[str] = field(default_factory=list)   # every place it can run here, best first
+    estimate_s: float | None = None                    # seconds, from its measured runs here (estimates.py)
+    estimate_basis: str = ""
+    needs: list[str] = field(default_factory=list)     # the nodes whose outputs it takes (a stage picker's edges)
+    estimates: dict[str, dict] = field(default_factory=dict)   # every option -> {estimate_s, estimate_basis, cost_usd, uploads, upload_to}: to compare places
+    title: str = ""
+    service: str | None = None                                  # the hosted API that does its work, if it runs elsewhere
+    description: str = ""
+    outputs: dict[str, str] = field(default_factory=dict)      # name -> artifact type
+    models: list[ModelRef] = field(default_factory=list)       # every model it uses (downloads: the ones not here yet)
+    settings: list[dict] = field(default_factory=list)         # its fixed-choice settings: {key, description, value, choices, prices, per}
+
+
+@dataclass
+class Plan:
+    nodes: list[NodePlan]
+    places: dict[str, dict] = field(default_factory=dict)   # every place a node could go -> {title, remote, accelerator}
+    max_cost_usd: float = 0.0                               # the run's cap, as the graph sets it (--max-cost-usd replaces it)
+    allow_downloads: bool = False
+    allow_uploads: bool = False   # the run's consent; the graph's policy.allow_upload is the most it may ever send
+
+    @property
+    def ok(self) -> bool:
+        return all(n.where and not n.missing_secrets and (self.allow_downloads or not n.downloads)
+                   and (self.allow_uploads or n.cached or not n.upload_to) for n in self.nodes)
+
+    def render(self, explain: bool = False) -> str:
+        lines = []
+        for n in self.nodes:
+            status = "cached" if n.cached else (n.where or "CANNOT RUN")
+            lines.append(f"{n.id:16s} {n.uses:28s} {status}")
+            if n.why_not:
+                lines.append(f"{'':16s}   {n.why_not}")
+            for m in n.downloads:
+                size = f"{m.size_gb:.1f} GB" if m.size_gb else "size unknown"
+                lines.append(f"{'':16s}   needs download: {m.id} ({size}{', gated' if m.gated else ''}, {m.licence_mode}) from {m.source}")
+            if n.upload_to and not n.cached:
+                lines.append(f"{'':16s}   sends {', '.join(n.uploads) or 'its inputs'} to {n.upload_to}")
+            for s in n.missing_secrets:
+                lines.append(f"{'':16s}   missing secret: {s}")
+            if n.cost_usd or n.cost_note:
+                lines.append(f"{'':16s}   estimated ${n.cost_usd:.2f}" + (f" ({n.cost_note})" if n.cost_note else ""))
+        total = sum(n.cost_usd for n in self.nodes if not n.cached)
+        lines.append(f"estimated spend: ${total:.2f}")
+        downloads = [m for n in self.nodes for m in n.downloads]
+        if downloads and not self.allow_downloads:
+            lines.append("weights are missing: fetch them first (perception models pull) or run with --allow-downloads")
+        uploading = [n for n in self.nodes if n.upload_to and not n.cached]
+        if uploading and not self.allow_uploads:
+            kinds = sorted({k for n in uploading for k in n.uploads})
+            lines.append(f"{', '.join(n.id for n in uploading)} send {', '.join(kinds) or 'their inputs'} off this machine: run with --allow-uploads, or leave them out (--only)")
+        return "\n".join(lines)
+
+
+def hf_cached(source: str) -> bool:
+    """Is hf://org/name[@revision] in the local Hugging Face cache?"""
+    repo = source.removeprefix("hf://").split("@")[0]
+    parts = repo.split("/")
+    if len(parts) < 2:
+        return False
+    hub = Path(os.environ.get("HF_HUB_CACHE") or Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface")) / "hub")
+    return (hub / f"models--{parts[0]}--{parts[1]}").is_dir()
+
+
+def settings(m, config: dict) -> list[dict]:
+    """A node's settings that are a fixed list of choices (a model, an effort), with the price of each when the node's
+    cost is keyed by that setting: what the app offers as menus. Free-form settings stay at their tuned defaults."""
+    out = []
+    for key, schema in m.config.items():
+        if "enum" not in schema:
+            continue
+        priced = m.cost and m.cost.by == key and isinstance(m.cost.usd, dict)
+        out.append({"key": key, "description": schema.get("description", ""), "value": config.get(key), "choices": list(schema["enum"]),
+                    "prices": dict(m.cost.usd) if priced else None, "per": m.cost.per if priced else None})
+    return out
+
+
+def plan(coord: Coordinator, graph: Graph, allow_downloads: bool = False, allow_uploads: bool = False) -> Plan:
+    known: dict[str, dict[str, Artifact]] = {}   # node -> outputs, when its result is cached
+    graph_inputs = {name: coord._graph_input(graph, name) for name in graph.spec.inputs}
+    out, places = [], {}
+    video_s = None   # the video's length: from a cached decode below, else ffprobe on the graph input
+    for name, spec in graph.spec.inputs.items():
+        if spec.type == "video.v1" and not video_s:
+            video_s = probe_seconds(graph.input_path(name))
+    for node in graph.nodes.values():
+        m = node.manifest
+        policy = graph.spec.policy
+        placement, why = coord.place(m, node.spec.backend, node.spec.place, policy)
+        np = NodePlan(node.id, m.ref, f"{placement.backend.kind} ({placement.runner.accelerator})" if placement else None, why)
+        options = coord.placements(m, node.spec.backend, policy)[0]
+        np.options = [p.label for p in options]
+        for p in options:
+            places[p.label] = {"title": p.backend.title, "remote": p.backend.remote, "accelerator": p.runner.accelerator}
+        np.needs = sorted(node.upstream)
+        np.place = placement.label if placement else None
+        # weights not in this machine's cache; on another machine its cache is not known, so every weight is a download
+        # there (a run never downloads silently: --allow-downloads)
+        np.downloads = [x for x in m.models if x.source and x.source.startswith("hf://")
+                        and ((placement and placement.backend.remote) or not hf_cached(x.source))]
+        np.missing_secrets = [s for s in m.secrets if s not in coord.secrets]
+        np.uploads, np.upload_to = coord.uploads(m, placement) if placement else (list(m.privacy.external_upload), m.service)
+        np.title, np.service, np.description, np.outputs, np.models = m.title, m.service, m.description, dict(m.outputs), list(m.models)
+        np.settings = settings(m, node.config)
+        if placement:
+            inputs = {}
+            for name, ref in node.inputs.items():
+                if ref.node is None:
+                    inputs[name] = graph_inputs[ref.name]
+                elif ref.node in known:
+                    inputs[name] = known[ref.node][ref.name]
+            if len(inputs) == len(node.inputs):
+                key = coord.cache_key(node, placement, inputs)
+                hit = coord.store.db.execute("SELECT outputs FROM cache WHERE key=?", (key,)).fetchone()
+                if hit:
+                    outs = {k: Artifact.from_json(v) for k, v in json.loads(hit["outputs"]).items()}
+                    if all(coord.store.has(a) for a in outs.values()):
+                        np.cached = True
+                        np.downloads = []
+                        known[node.id] = outs
+                        video_s = media_seconds(json.loads(hit["outputs"])) or video_s
+            np.cost_usd = placement.backend.estimate(m, placement.runner, {}, video_s).cost_usd
+            if m.cost:   # units are known once the input exists (from the cache, or a graph input)
+                np.cost_usd += manifest_cost(m, node.config, inputs) if m.cost.per == "run" or m.cost.input in inputs else 0.0
+                np.cost_note = "" if m.cost.per == "run" or m.cost.input in inputs else f"${m.cost.unit_usd(node.config)} per {m.cost.per}, units known after {node.inputs[m.cost.input].node}"
+        for p in options:   # cached or not: moving a node elsewhere runs it again
+            seconds, basis = estimate(coord.store, m.ref, p.label, video_s)
+            sent, to = coord.uploads(m, p)
+            np.estimates[p.label] = {"estimate_s": seconds, "estimate_basis": basis,
+                                     "cost_usd": p.backend.estimate(m, p.runner, {}, video_s).cost_usd, "uploads": sent, "upload_to": to}
+        if placement and not np.cached:
+            chosen = np.estimates[placement.label]
+            np.estimate_s, np.estimate_basis = chosen["estimate_s"], chosen["estimate_basis"]
+        out.append(np)
+    return Plan(out, places, graph.spec.policy.max_cost_usd, allow_downloads, allow_uploads)

perception/release.py

@@ -0,0 +1,61 @@
+"""Where the pipeline's files are: a checkout of this repository, or the package installed from PyPI, which carries
+them in perception/_pipeline (scripts/build-dist.sh: nodes, environments, the node SDK, graphs, schemas and cloud
+scripts). The package has no .git, so what git says in a checkout (which commit this is, which files are tracked) is in
+its RELEASE.json instead. Read only where the file is: a checkout never has one (.gitignore), and the package never
+asks git (on a Mac without the developer tools, running git opens a dialog offering to install them)."""
+
+import hashlib
+import json
+import os
+from functools import cache
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+
+PACKAGED = Path(__file__).resolve().parent / "_pipeline"
+REPO = PACKAGED if PACKAGED.is_dir() else Path(__file__).resolve().parents[3]
+RELEASE = REPO / "RELEASE.json"
+
+
+def home() -> Path:
+    """The store: artifacts, run state and (for the package) node environments."""
+    return Path(os.environ.get("PERCEPTION_HOME", Path.home() / ".hyperstudy-perception"))
+
+
+@cache
+def release() -> dict | None:
+    """{"commit": the full hash, "files": every file it carries, relative to REPO}; None in a checkout."""
+    if not RELEASE.exists():
+        return None
+    data = json.loads(RELEASE.read_text())
+    if not isinstance(data.get("commit"), str) or not isinstance(data.get("files"), list):
+        raise RuntimeError(f"{RELEASE} is not a release file (commit and files)")
+    return data
+
+
+def files_in(dirs: list[str]) -> list[str]:
+    """The package's files under these directories, as `git ls-files -- dirs` lists them in a checkout."""
+    return sorted(f for f in release()["files"] if any(f.startswith(d.rstrip("/") + "/") for d in dirs))
+
+
+def env_dir(env: str) -> Path | None:
+    """Where a node environment (envs/NAME) is made: in a checkout, uv's own place (envs/NAME/.venv, None here); for
+    the package, under the store, by the lock, the commit and the package version (the package's folder may not be
+    writable, and an upgrade replaces it)."""
+    if release() is None:
+        return None
+    lock = REPO / env / "uv.lock"
+    h = hashlib.sha256(lock.read_bytes() if lock.exists() else b"no lock")
+    # the commit too: the SDK is installed editable from this copy, so two builds of one version (PyPI's and a local
+    # one) must not share an environment and point its SDK at each other's files
+    h.update(release()["commit"].encode())
+    try:
+        h.update(version("hyperstudy-perception").encode())
+    except PackageNotFoundError:   # the files without the package's metadata (a copy run in place)
+        pass
+    return home() / "envs" / f"{Path(env).name}-{h.hexdigest()[:12]}"
+
+
+def env_vars(env: str) -> dict[str, str]:
+    """What uv is told so a node environment is made where env_dir says (nothing in a checkout)."""
+    d = env_dir(env)
+    return {} if d is None else {"UV_PROJECT_ENVIRONMENT": str(d)}