hyperstudy-perception 0.1.0__py3-none-any.whl

perception/_pipeline/sdk/pyproject.toml

@@ -0,0 +1,13 @@
+[project]
+name = "perception-node"
+version = "0.1.0"
+description = "Node side of the perception graph's file protocol (standard library only)"
+requires-python = ">=3.10"
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/perception_node"]

perception/_pipeline/sdk/src/perception_node/__init__.py

@@ -0,0 +1,157 @@
+"""The node side of the perception graph's file protocol. Standard library only: it is installed into every model
+environment, which share nothing else.
+
+The coordinator makes a fresh attempt directory, writes request.json into it and runs the node with
+`--request PATH`. The node reads its inputs (paths), writes outputs under the request's out_dir, and finishes by
+writing response.json next to the request. The coordinator then validates, hashes and publishes the outputs; a node
+never writes into the artifact store, so a node that crashes or is orphaned cannot corrupt it.
+
+    from perception_node import Node, NodeError
+
+    with Node() as node:
+        video = node.input("video")              # a Path
+        shots = detect(video, node.config["threshold"])
+        node.write_json("shots", "shots.json", shots)
+        node.model("PySceneDetect AdaptiveDetector", licence_mode="commercial-ok", licence="BSD-3-Clause")
+
+Errors: raise NodeError(code, message). Codes: transient (worth retrying: a network error, a busy device),
+input (the inputs cannot be processed), schema, privacy, licence (terminal). Any other exception is reported as
+`internal` with its traceback, and is terminal: retrying a bug does not fix it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import platform
+import sys
+import time
+import traceback
+from pathlib import Path
+
+PROTOCOL = 1
+# resources: the node ran out of memory (GPU or RAM). Terminal like the others: the same input on the same device would
+# run out again, so the fix is a smaller memory setting in the graph's config, or another device.
+ERROR_CODES = ("transient", "input", "schema", "privacy", "licence", "resources", "internal")
+MEMORY_SETTINGS = ("batch_size",)   # config keys that trade memory for speed, named in the message
+
+
+def out_of_memory(exc: BaseException) -> bool:
+    """PyTorch's OutOfMemoryError (CUDA), MPS's RuntimeError, and Python's MemoryError."""
+    return (isinstance(exc, MemoryError) or type(exc).__name__ == "OutOfMemoryError"
+            or (isinstance(exc, RuntimeError) and "out of memory" in str(exc).lower()))
+
+
+class NodeError(Exception):
+    def __init__(self, code: str, message: str):
+        if code not in ERROR_CODES:
+            raise ValueError(f"unknown error code {code!r}")
+        super().__init__(message)
+        self.code = code
+
+
+class Node:
+    def __init__(self, argv: list[str] | None = None):
+        parser = argparse.ArgumentParser()
+        parser.add_argument("--request", type=Path, required=True)
+        args = parser.parse_args(argv)
+        self.request_path: Path = args.request
+        self.request: dict = json.loads(self.request_path.read_text())
+        if self.request.get("protocol") != PROTOCOL:
+            raise SystemExit(f"request protocol {self.request.get('protocol')} is not {PROTOCOL}")
+        self.out_dir = Path(self.request["out_dir"])
+        self.out_dir.mkdir(parents=True, exist_ok=True)
+        self.config: dict = self.request.get("config", {})
+        self.accelerator: str = self.request.get("accelerator", "cpu")   # cpu | mps | cuda: where placement put this run
+        self._outputs: dict[str, dict] = {}
+        self._models: list[dict] = []
+        self._usage: dict = {}
+        self._t0 = time.time()
+
+    # --- inputs ---------------------------------------------------------------------------------------------
+
+    def input(self, name: str) -> Path:
+        try:
+            return Path(self.request["inputs"][name]["path"])
+        except KeyError:
+            raise NodeError("input", f"no input {name!r} in the request") from None
+
+    def input_json(self, name: str):
+        return json.loads(self.input(name).read_text())
+
+    def secret(self, name: str) -> str:
+        """A secret the request names, from the environment the coordinator gave this process."""
+        if name not in self.request.get("secrets", []):
+            raise NodeError("input", f"secret {name} was not requested by this node's manifest")
+        value = os.environ.get(name)
+        if not value:
+            raise NodeError("input", f"secret {name} is not set")
+        return value
+
+    # --- outputs ---------------------------------------------------------------------------------------------
+
+    def path(self, name: str, relative: str) -> Path:
+        """Where to write output `name` (a file, or a directory for tree outputs). Declares the output."""
+        p = self.out_dir / relative
+        p.parent.mkdir(parents=True, exist_ok=True)
+        self._outputs[name] = {"path": relative}
+        return p
+
+    def write_json(self, name: str, relative: str, doc) -> Path:
+        p = self.path(name, relative)
+        p.write_text(json.dumps(doc, separators=(",", ":")))
+        return p
+
+    def producer(self, timings: dict | None = None, device: str | None = None) -> dict:
+        """The bundle layer `producer` block: stage and models from the manifest, and the device.
+
+        Deliberately without the run time or timings: a stored layer must be the same bytes whenever the same inputs
+        produce the same data, or every downstream node would miss the cache after an upstream rerun. Timings go to
+        the response's usage; `perception export` writes run_at and timings into the bundle's copy of the layer."""
+        if timings:
+            self._usage.setdefault("timings", {}).update(timings)
+        base = self.request.get("producer", {})
+        return {**base, "models": self._models or base.get("models", []), "config": self.config,
+                "device": device, "run_at": None, "timings": {}}
+
+    def model(self, id: str, licence_mode: str, source: str | None = None, revision: str | None = None, licence: str | None = None):
+        """Record a model this run used (the coordinator checks it against the manifest's licence mode)."""
+        self._models.append({"id": id, "source": source, "revision": revision, "licence": licence, "licence_mode": licence_mode})
+
+    def usage(self, **values):
+        """Billable or measured usage: seconds, tokens_in, tokens_out, usd, device..."""
+        self._usage.update(values)
+
+    # --- lifecycle -------------------------------------------------------------------------------------------
+
+    def __enter__(self) -> "Node":
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        error = None
+        if exc is not None:
+            if isinstance(exc, NodeError):
+                error = {"code": exc.code, "message": str(exc)}
+            elif isinstance(exc, KeyboardInterrupt):
+                error = {"code": "transient", "message": "interrupted"}
+            elif out_of_memory(exc):
+                knobs = [f"{k} (now {self.config[k]})" for k in MEMORY_SETTINGS if k in self.config]
+                hint = f"lower {' or '.join(knobs)} in the graph's config for this node" if knobs else "run it on a device with more memory"
+                error = {"code": "resources", "message": f"out of memory on {self.accelerator}: {exc}. To fix: {hint}.",
+                         "traceback": traceback.format_exc()}
+            else:
+                error = {"code": "internal", "message": f"{exc_type.__name__}: {exc}", "traceback": traceback.format_exc()}
+        response = {
+            "protocol": PROTOCOL, "ok": error is None, "outputs": self._outputs if error is None else {},
+            "models": self._models, "usage": {"seconds": round(time.time() - self._t0, 3), **self._usage},
+            "host": {"platform": sys.platform, "machine": platform.machine(), "python": platform.python_version()},
+            "error": error,
+        }
+        tmp = self.request_path.with_name("response.json.tmp")
+        tmp.write_text(json.dumps(response, indent=1))
+        os.replace(tmp, self.request_path.with_name("response.json"))   # the coordinator never sees half a response
+        if error:
+            print(f"node failed ({error['code']}): {error['message']}", file=sys.stderr)
+            sys.exit(1)
+        return False

perception/artifacts.py

@@ -0,0 +1,180 @@
+"""Artifact types: what flows between nodes, and how a node's output is checked before it is published.
+
+The bundle's layer schemas are artifact types (`bundle.<layer>@0.1`): a layer a node writes is exactly the layer the
+annotator and HyperStudy read. A few internal types carry what bundles do not (media metadata, audio, keyframes).
+A type is a file or a tree (a directory); its validator returns a list of problems (empty: valid).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+from typing import Callable, Literal
+
+from jsonschema import Draft202012Validator
+from referencing import Registry, Resource
+
+from .release import REPO
+
+SCHEMA_DIR = REPO / "schema"
+BUNDLE_SCHEMA = SCHEMA_DIR / "bundle-0.1.schema.json"
+BUNDLE_LAYERS = ("shots", "transcript", "speakers", "faces", "bodies", "hands", "gestures", "facing", "gaze", "objects",
+                 "sounds", "text", "prosody", "waveform")
+
+
+@dataclass(frozen=True)
+class ArtifactType:
+    id: str
+    shape: Literal["file", "tree", "source"]   # source: an input that stays where it is (the video), never copied
+    validate: Callable[[Path], list[str]]
+    layer: str | None = None                    # the bundle layer this type is, if any
+    # the kinds of data it holds, in policy.allow_upload's terms: what leaves the machine when a node that reads it runs
+    # elsewhere (a lab server, a cloud machine). None: anything (a JSON or text file could hold any of them)
+    kinds: tuple[str, ...] | None = None
+
+
+@cache
+def _bundle_schema():
+    schema = json.loads(BUNDLE_SCHEMA.read_text())
+    return schema, Registry().with_resource(schema["$id"], Resource.from_contents(schema))
+
+
+def _json_schema(schema: dict) -> Callable[[Path], list[str]]:
+    validator = Draft202012Validator(schema)
+
+    def check(path: Path) -> list[str]:
+        try:
+            doc = json.loads(path.read_text())
+        except (OSError, ValueError) as e:
+            return [f"not JSON: {e}"]
+        return [f"{e.json_path}: {e.message[:200]}" for e in list(validator.iter_errors(doc))[:20]]
+    return check
+
+
+def _layer(name: str) -> Callable[[Path], list[str]]:
+    def check(path: Path) -> list[str]:
+        schema, registry = _bundle_schema()
+        try:
+            doc = json.loads(path.read_text())
+        except (OSError, ValueError) as e:
+            return [f"not JSON: {e}"]
+        validator = Draft202012Validator({"$ref": f"{schema['$id']}#/$defs/layer/{name}"}, registry=registry)
+        problems = [f"{e.json_path}: {e.message[:200]}" for e in list(validator.iter_errors(doc))[:20]]
+        if doc.get("layer") != name:
+            problems.append(f"layer is {doc.get('layer')!r}, not {name!r}")
+        return problems
+    return check
+
+
+def _wav(path: Path) -> list[str]:
+    with open(path, "rb") as f:
+        head = f.read(12)
+    return [] if head[:4] == b"RIFF" and head[8:12] == b"WAVE" else ["not a RIFF/WAVE file"]
+
+
+def _tree_of(suffixes: tuple[str, ...]) -> Callable[[Path], list[str]]:
+    def check(path: Path) -> list[str]:
+        if not path.is_dir():
+            return ["not a directory"]
+        files = [p for p in path.rglob("*") if p.is_file()]
+        bad = [str(p.relative_to(path)) for p in files if p.suffix.lower() not in suffixes and p.name != "index.json"]
+        return [f"unexpected files: {bad[:5]}"] if bad else []
+    return check
+
+
+MEDIA_SCHEMA = {
+    "type": "object",
+    "required": ["sha256", "duration", "width", "height", "fps", "time_base", "variable_frame_rate"],
+    "properties": {
+        "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"},
+        "duration": {"type": "number", "exclusiveMinimum": 0},
+        "width": {"type": "integer"}, "height": {"type": "integer"},
+        "fps": {"type": "number", "exclusiveMinimum": 0, "description": "average frame rate"},
+        "r_frame_rate": {"type": "string"}, "avg_frame_rate": {"type": "string"},
+        "time_base": {"type": "string", "description": "the video stream's PTS time base, e.g. 1/30000"},
+        "start_pts": {"type": ["integer", "null"]},
+        "variable_frame_rate": {"type": "boolean"},
+        "audio": {"type": ["object", "null"]},
+        "filename": {"type": "string"},
+    },
+}
+
+# What each layer can hold. Shots carry captions, which describe and name the people on screen; prosody measures the
+# audio and carries each utterance's words; the waveform is a measure of the audio. Every layer is listed: a new one must say what it holds.
+LAYER_KINDS: dict[str, tuple[str, ...]] = {
+    "shots": ("transcript", "faces"), "transcript": ("transcript",), "speakers": ("transcript",), "text": ("transcript",),
+    "faces": ("faces", "embeddings"), "bodies": (), "hands": (), "gestures": (), "facing": (), "gaze": (), "objects": (),
+    "sounds": (), "prosody": ("audio", "transcript"), "waveform": ("audio",),
+}
+assert set(LAYER_KINDS) == set(BUNDLE_LAYERS)
+
+TYPES: dict[str, ArtifactType] = {
+    "video.v1": ArtifactType("video.v1", "source", lambda p: [] if p.is_file() else ["missing"], kinds=("video",)),
+    "media.v1": ArtifactType("media.v1", "file", _json_schema(MEDIA_SCHEMA), kinds=()),   # the video's format and timing
+    "audio.wav.v1": ArtifactType("audio.wav.v1", "file", _wav, kinds=("audio",)),
+    # small images (keyframes, crops) a layer refers to as "assets/<path>"; export merges every assets.v1 tree into the
+    # bundle's assets/ directory. Either kind of image: both kinds are counted
+    "assets.v1": ArtifactType("assets.v1", "tree", _tree_of((".jpg", ".jpeg", ".png", ".webp")), kinds=("frames", "faces")),
+    "json.v1": ArtifactType("json.v1", "file", _json_schema({})),
+    "text.v1": ArtifactType("text.v1", "file", lambda p: []),
+    **{f"bundle.{name}@0.1": ArtifactType(f"bundle.{name}@0.1", "file", _layer(name), layer=name, kinds=LAYER_KINDS[name])
+       for name in BUNDLE_LAYERS},
+}
+
+
+def get(type_id: str) -> ArtifactType:
+    if type_id not in TYPES:
+        raise KeyError(f"unknown artifact type {type_id}")
+    return TYPES[type_id]
+
+
+# --- content hashes ---------------------------------------------------------------------------------------------
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def sha256_tree(path: Path) -> str:
+    """A directory's hash: over its sorted relative paths and their file hashes (so renames and edits both count)."""
+    h = hashlib.sha256()
+    for p in sorted(x for x in path.rglob("*") if x.is_file()):
+        h.update(str(p.relative_to(path).as_posix()).encode() + b"\0" + sha256_file(p).encode() + b"\n")
+    return h.hexdigest()
+
+
+def special_files(path: Path) -> list[str]:
+    """Entries that are not plain files or directories (symlinks, sockets, devices), found without following links.
+    Outputs must have none: publishing a symlink would copy whatever it points at on the host."""
+    import os
+    import stat
+    bad = []
+    st = os.lstat(path)
+    if stat.S_ISLNK(st.st_mode):
+        return [f"{path.name} (symlink)"]
+    if stat.S_ISDIR(st.st_mode):
+        for root, dirs, files in os.walk(path, followlinks=False):
+            for name in dirs + files:
+                mode = os.lstat(os.path.join(root, name)).st_mode
+                if stat.S_ISLNK(mode):
+                    bad.append(f"{os.path.relpath(os.path.join(root, name), path)} (symlink)")
+                elif not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
+                    bad.append(f"{os.path.relpath(os.path.join(root, name), path)} (not a regular file)")
+    elif not stat.S_ISREG(st.st_mode):
+        bad.append(f"{path.name} (not a regular file)")
+    return bad
+
+
+def content_hash(path: Path) -> str:
+    return sha256_tree(path) if path.is_dir() else sha256_file(path)
+
+
+def canonical(value) -> str:
+    """JSON with sorted keys and no whitespace: equal values hash equal."""
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)

perception/backends/base.py

@@ -0,0 +1,179 @@
+"""What every execution backend provides. The graph never names a provider; placement picks a backend whose runner
+fits the node, and the backend turns the file protocol (request.json in, out/ + response.json back) into a process,
+a remote job or an API call.
+
+    probe_capabilities()   what this backend can run here (platform, accelerators, memory, tools)
+    runner_for(manifest)   the manifest runner this backend would use, or why none fits
+    identity(manifest, r)  what makes this runner's results this runner's (env lock, code, SDK): cache key
+    estimate(...)          cost and time, before anything is reserved
+    stage_inputs(...)      make the inputs readable where the node runs; returns the request's input entries
+    submit(...)            start it; returns a JSON-able handle (kept in the run state, so a restarted coordinator
+                           can find the job again)
+    poll(handle)           running | exited
+    cancel(handle)
+    collect(attempt_dir)   the node's response.json, or None if it never wrote one (a crash)
+    cleanup(attempt_dir)
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import re
+import stat
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ..manifests import Manifest, Runner
+from ..store import Artifact
+
+
+@dataclass
+class Capabilities:
+    platform: str                      # darwin/arm64, linux/amd64, windows/amd64...
+    cpu_cores: int
+    memory_gb: float
+    accelerators: set[str]             # cpu, mps, cuda
+    vram_gb: float = 0                 # CUDA: total; Apple GPU: unified memory (memory_gb)
+    tools: dict[str, str | None] = field(default_factory=dict)   # uv, ffmpeg, ffprobe -> path or None
+
+
+@dataclass
+class Estimate:
+    cost_usd: float                    # 0 for local runs; a range's upper end for billable ones
+    seconds: float | None = None       # None: not known before a first run
+
+
+def this_platform() -> str:
+    machine = platform.machine().lower()
+    arch = {"x86_64": "amd64", "amd64": "amd64", "arm64": "arm64", "aarch64": "arm64"}.get(machine, machine)
+    system = {"Darwin": "darwin", "Linux": "linux", "Windows": "windows"}.get(platform.system(), platform.system().lower())
+    return f"{system}/{arch}"
+
+
+class Backend(ABC):
+    kind: str            # names the backend in a place (`kind:accelerator`): unique among a coordinator's backends
+    runner_kind: str = ""   # the manifest runners it runs (default: its kind); a remote machine runs `native` ones
+    remote = False       # runs elsewhere: a node's inputs leave this machine to reach it
+    title = "This machine"   # where it is, for people: "gpu1 (SSH)", "Cloud GPU (AWS L4)"
+
+    @property
+    def machine(self) -> str | None:
+        """Which machine's CPU, memory and GPU a node placed here shares with other nodes; None when every node gets a
+        machine of its own (a cloud machine started for it)."""
+        return "local"
+
+    def sends(self, manifest: Manifest) -> list[str]:
+        """The kinds of data that leave this machine when the node runs here: its inputs', for a remote machine (every
+        kind, for an input that could hold anything)."""
+        if not self.remote:
+            return []
+        from ..artifacts import get
+        from ..ir import UPLOAD_KINDS
+        kinds = [get(t).kinds for t in manifest.inputs.values()]
+        return sorted(UPLOAD_KINDS) if None in kinds else sorted({k for ks in kinds for k in ks})
+
+    def remote_refusal(self, manifest: Manifest) -> str:
+        """Why a node may not run on this remote machine at all, or "" (local backends: never)."""
+        return ""
+
+    @abstractmethod
+    def probe_capabilities(self) -> Capabilities: ...
+
+    def runner_for(self, manifest: Manifest) -> tuple[Runner | None, str]:
+        """The first of the manifest's runners this backend can use here, preferring an accelerated one."""
+        runners, why = self.runners_for(manifest)
+        return (runners[0], "") if runners else (None, why)
+
+    def runners_for(self, manifest: Manifest) -> tuple[list[Runner], str]:
+        """Every runner of the manifest this backend can use here, accelerated ones first; why none, when none."""
+        caps = self.probe_capabilities()
+        reasons, usable = [], []
+        kind = self.runner_kind or self.kind
+        mine = [r for r in manifest.runners if r.kind == kind]
+        if not mine:
+            return [], f"no {kind} runner"
+        if refusal := self.remote_refusal(manifest):
+            return [], refusal
+        for r in sorted(mine, key=lambda r: r.accelerator == "cpu"):
+            if caps.platform not in r.platforms:
+                reasons.append(f"{r.accelerator} runner is for {', '.join(r.platforms)}, not {caps.platform}")
+            elif r.accelerator not in caps.accelerators:
+                reasons.append(f"{r.accelerator} runner needs {r.accelerator}, not available here")
+            elif r.minimum_vram_gb > caps.vram_gb:
+                reasons.append(f"{r.accelerator} runner needs {r.minimum_vram_gb} GB VRAM, {caps.vram_gb:.0f} here")
+            else:
+                missing = self.missing_tools(r)
+                if missing:
+                    reasons.append(f"{', '.join(missing)} not installed")
+                else:
+                    usable.append(r)
+        return usable, "" if usable else "; ".join(reasons)
+
+    def missing_tools(self, runner: Runner) -> list[str]:
+        return []
+
+    @abstractmethod
+    def identity(self, manifest: Manifest, runner: Runner) -> str: ...
+
+    def estimate(self, manifest: Manifest, runner: Runner, inputs: dict[str, Artifact], video_s: float | None = None) -> Estimate:
+        """`video_s`: the video's length when the caller knows it (a plan, before decode has run)."""
+        return Estimate(0.0)
+
+    @abstractmethod
+    def stage_inputs(self, attempt_dir: Path, inputs: dict[str, Artifact]) -> dict[str, dict]: ...
+
+    @abstractmethod
+    def submit(self, manifest: Manifest, runner: Runner, attempt_dir: Path, env: dict[str, str]) -> dict: ...
+
+    @abstractmethod
+    def poll(self, handle: dict) -> str: ...
+
+    @abstractmethod
+    def cancel(self, handle: dict) -> None: ...
+
+    def work_dir(self, attempt_dir: Path) -> Path:
+        """Where the node itself reads and writes (its request, out/, response.json)."""
+        return attempt_dir
+
+    def output_dir(self, attempt_dir: Path) -> Path:
+        return self.work_dir(attempt_dir) / "out"
+
+    def collect(self, attempt_dir: Path) -> dict | None:
+        """The node's response, read only if it is a regular file: the node controls this directory, and a symlink
+        here could point anywhere."""
+        path = self.work_dir(attempt_dir) / "response.json"
+        try:
+            st = os.lstat(path)
+        except FileNotFoundError:
+            return None
+        if not stat.S_ISREG(st.st_mode):
+            return None
+        return json.loads(path.read_text())
+
+    def cleanup(self, attempt_dir: Path) -> None:
+        return None
+
+    def recorded_handle(self, attempt_dir: Path) -> dict | None:
+        """The handle `submit` recorded in the attempt's folder as soon as the node started, if it did: a coordinator that
+        died before recording the task running leaves that node to be adopted, not started a second time."""
+        return None
+
+
+# --- the child's environment: no credentials it did not ask for -------------------------------------------------
+
+SECRET_NAME = re.compile(r"(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL)", re.I)
+# the coordinator's own interpreter settings: a node runs in its own environment, never the coordinator's
+INTERPRETER_VARS = {"VIRTUAL_ENV", "PYTHONPATH", "PYTHONHOME", "UV_PROJECT_ENVIRONMENT", "CONDA_PREFIX"}
+
+
+def child_env(requested: list[str], secrets: dict[str, str]) -> dict[str, str]:
+    """The parent's environment without anything that looks like a credential, plus the secrets the node's manifest
+    names. A node never sees a key it did not declare."""
+    env = {k: v for k, v in os.environ.items() if not SECRET_NAME.search(k) and k not in INTERPRETER_VARS}
+    for name in requested:
+        if name in secrets:
+            env[name] = secrets[name]
+    return env