hyperstudy-perception 0.1.0__tar.gz → 0.1.1__tar.gz

{hyperstudy_perception-0.1.0 → hyperstudy_perception-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hyperstudy-perception
-Version: 0.1.0
+Version: 0.1.1
 Summary: Video perception pipeline: shots, speech, faces and captions from a video, as a versioned track bundle, run locally, on cloud GPUs or lab servers
 Project-URL: Homepage, https://github.com/hyperstudyio/hyperstudy-perception
 Project-URL: Changelog, https://github.com/hyperstudyio/hyperstudy-perception/releases

{hyperstudy_perception-0.1.0 → hyperstudy_perception-0.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hyperstudy-perception"
-version = "0.1.0"
+version = "0.1.1"
 description = "Video perception pipeline: shots, speech, faces and captions from a video, as a versioned track bundle, run locally, on cloud GPUs or lab servers"
 readme = "README.md"
 license = "MIT"

{hyperstudy_perception-0.1.0 → hyperstudy_perception-0.1.1}/src/perception/_pipeline/RELEASE.json

@@ -1,5 +1,5 @@
 {
-"commit": "9beebed7f711b6c85e2a735810db16841a741f39",
+"commit": "4df6083954bc875054f55459a28001f26287a2d1",
 "files": [
 "cloud/runpod/guard.sh",
 "cloud/runpod/image/Dockerfile",

{hyperstudy_perception-0.1.0 → hyperstudy_perception-0.1.1}/src/perception/backends/remote.py

@@ -22,7 +22,7 @@ import subprocess
 import sys
 from pathlib import Path
 
-from ..hosts import Host, ssh_command
+from ..hosts import Host
 from ..manifests import Manifest, Runner
 from ..release import files_in, release
 from .base import Capabilities
@@ -127,7 +127,7 @@ class SshBackend(NativeBackend):
         files = code_files(manifest, runner)
         caps = self.host.caps
         spec = {
-            "dest": self.host.dest, "root": self.host.root, "host": self.host.name, "ssh": ssh_command(),
+            "dest": self.host.dest, "root": self.host.root, "host": self.host.name, "ssh": self.host.ssh(),
             "code": code_hash(files), "files": files,
             "node_dir": manifest.directory.resolve().relative_to(REPO).as_posix(), "entry": runner.entry, "env": runner.env,
             "runner_kind": self.runner_kind, "python": self.python, "uv": caps.tools.get("uv"), "prepare": manifest.prepare,

{hyperstudy_perception-0.1.0 → hyperstudy_perception-0.1.1}/src/perception/cli.py

@@ -41,7 +41,7 @@ from .backends.native import NativeBackend, SelfBackend
 from .backends.probe import probe
 from .backends.cloud import Cloud, CloudBackend, RunPodBackend
 from .backends.remote import SshBackend
-from .hosts import HostError, Hosts, ssh_command
+from .hosts import HostError, Hosts, probe_dest
 from .keys import load_secrets
 from .ir import GraphError, compile_graph
 from .manifests import Registry
@@ -258,8 +258,13 @@ def cmd_nodes(args):
 def cmd_models(args):
     reg = registry(args)
     if args.json:
+        # each node too: what it is called, the API it calls (service) and the keys it needs, and its fixed choices
+        # (a model, an effort) at their defaults, with prices: what the app's settings show
+        from .plan import settings
         emit("models", models=[{"node": m.ref, **x.model_dump(include={"id", "source", "size_gb", "gated", "licence_mode"})}
-                               for m in reg for x in m.models])
+                               for m in reg for x in m.models],
+             nodes=[{"node": m.ref, "title": m.title, "service": m.service, "secrets": list(m.secrets),
+                     "settings": settings(m, {k: s.get("default") for k, s in m.config.items()})} for m in reg])
         return 0
     for m in reg:
         for x in m.models:
@@ -481,7 +486,18 @@ def cmd_hosts(args):
         if args.action == "add":
             if not args.dest:
                 raise HostError("hosts add NAME USER@HOST")
-            hosts.add(args.name, args.dest, args.root)
+            hosts.add(args.name, args.dest, args.root, args.user, args.port, args.identity)
+        elif args.action == "probe":   # a connection test: nothing is added, nothing changes on the host
+            dest = args.dest or args.name
+            if not dest:
+                raise HostError("hosts probe USER@HOST")
+            caps, root = probe_dest(dest, args.root, args.user, args.port, args.identity)
+            found = {"dest": dest, "root": root, "caps": {**caps.__dict__, "accelerators": sorted(caps.accelerators)}}
+            if args.json:
+                emit("host_probe", **found)
+            else:
+                print(f"{dest}: {caps.platform}, {caps.cpu_cores} cores, {caps.memory_gb} GB, {'/'.join(sorted(caps.accelerators))}")
+            return 0
         elif args.action in ("check", "remove", "clean") and not args.name:
             raise HostError(f"hosts {args.action} NAME")
         elif args.action == "check":
@@ -510,7 +526,7 @@ def cmd_hosts(args):
                       f'if ps -ww -p "$p" -o command= 2>/dev/null | grep -qF -- "${{f%/pid}}/request.json"; then '
                       f'echo "a node is running there" >&2; exit 4; fi; done\n'
                       f"chmod -R u+w {root}/objects 2>/dev/null; rm -rf {root}/objects {root}/code {root}/work")
-            out = subprocess.run([*ssh_command(), host.dest, f"sh -c {q(script)}"], capture_output=True, text=True, timeout=600)
+            out = subprocess.run([*host.ssh(), host.dest, f"sh -c {q(script)}"], capture_output=True, text=True, timeout=600)
             if out.returncode:
                 raise HostError(f"could not clean {host.name}: {out.stderr.strip()[-300:]}")
     except HostError as e:
@@ -536,6 +552,13 @@ def cmd_cloud(args):
     (SkyPilot, your AWS credentials) or RunPod (an API key). `on` checks that the provider can be used first."""
     cloud = Cloud(home())
     problem = None
+    if args.action == "check":   # whether it could be used, changing nothing (a test)
+        problem = runpod_problem() if args.provider == "runpod" else aws_problem()
+        if args.json:
+            emit("cloud_check", provider=args.provider, ok=problem is None, detail=problem or f"{args.provider} can be used")
+        else:
+            print(problem or f"{args.provider} can be used")
+        return 0
     if args.action == "on":
         problem = runpod_problem() if args.provider == "runpod" else aws_problem()
         if problem is None:
@@ -609,6 +632,16 @@ def cmd_keys(args):
             if not args.name:
                 raise ValueError("keys remove NAME: which key?")
             result = {"name": args.name, "removed": keys.remove(args.name)}
+        elif args.action == "check":   # whether its provider accepts it: a refused key is an answer, not a failure
+            if not args.name:
+                raise ValueError("keys check NAME: which key?")
+            from .keycheck import check
+            ok, detail = check(args.name, keys.load().values.get(args.name))
+            if args.json:
+                emit("key_check", name=args.name, ok=ok, detail=detail)
+            else:
+                print(detail)
+            return 0
         else:
             found = keys.load()
             result = {"keys": [{"name": k, "source": found.sources[k]} for k in sorted(found.values)], "problems": found.problems}
@@ -743,17 +776,20 @@ def main(argv: list[str] | None = None) -> int:
     rr.add_argument("--allow-uploads", action="store_true", help="let the run's nodes send data off this machine from now on")
     rr.set_defaults(fn=cmd_rerun)
     h = sub.add_parser("hosts", help="lab servers nodes can run on, over SSH")
-    h.add_argument("action", choices=["add", "list", "check", "remove", "clean"])
-    h.add_argument("name", nargs="?")
+    h.add_argument("action", choices=["add", "list", "check", "remove", "clean", "probe"])
+    h.add_argument("name", nargs="?", help="probe: the ssh destination")
     h.add_argument("dest", nargs="?", help="add: an ssh destination (user@host, or a ~/.ssh/config alias)")
     h.add_argument("--root", default=".cache/hyperstudy-perception", help="add: where on the host its files go (under its home if relative)")
+    h.add_argument("--user", help="add, probe: the user to log in as, over what ~/.ssh/config says")
+    h.add_argument("--port", type=int, help="add, probe: the ssh port, over what ~/.ssh/config says")
+    h.add_argument("--identity", help="add, probe: a private key file on this machine, over what ~/.ssh/config says")
     h.set_defaults(fn=cmd_hosts)
     c = sub.add_parser("cloud", help="offer a cloud GPU machine to nodes that can use one")
-    c.add_argument("action", choices=["on", "off", "status"])
+    c.add_argument("action", choices=["on", "off", "status", "check"])
     c.add_argument("provider", nargs="?", choices=Cloud.providers, default="aws")
     c.set_defaults(fn=cmd_cloud)
     k = sub.add_parser("keys", help="your API keys and tokens: store one in the keychain, list where each comes from")
-    k.add_argument("action", choices=["set", "list", "remove"])
+    k.add_argument("action", choices=["set", "list", "remove", "check"])
     k.add_argument("name", nargs="?")
     k.set_defaults(fn=cmd_keys)
     st = sub.add_parser("setup", help="install what nodes need here before a run: environments and weights")

{hyperstudy_perception-0.1.0 → hyperstudy_perception-0.1.1}/src/perception/export.py

@@ -20,6 +20,16 @@ from .scheduler import Coordinator, LICENCE_ORDER
 SCHEMA_VERSION = "0.1.0"
 
 
+def _version() -> str:
+    """The installed package's version: what made the bundle ("unknown" in a checkout not installed; its git_commit
+    says which)."""
+    from importlib.metadata import PackageNotFoundError, version
+    try:
+        return version("hyperstudy-perception")
+    except PackageNotFoundError:
+        return "unknown"
+
+
 def _git_commit() -> str | None:
     if r := release():
         return r["commit"][:7]
@@ -92,7 +102,7 @@ def bundle(coord: Coordinator, run_id: str, out: Path) -> Path:
         "format": "hyperstudy-perception-bundle", "schema_version": SCHEMA_VERSION,
         "created_at": time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime()),
         "video": {k: media[k] for k in ("sha256", "duration", "fps", "width", "height")} | ({"filename": media["filename"]} if media.get("filename") else {}),
-        "pipeline": {"name": "hyperstudy-perception", "version": "0.1.0", "git_commit": _git_commit(), "source": f"perception run {run_id}"},
+        "pipeline": {"name": "hyperstudy-perception", "version": _version(), "git_commit": _git_commit(), "source": f"perception run {run_id}"},
         "layers": entries,
         "assets": assets,
     }

{hyperstudy_perception-0.1.0 → hyperstudy_perception-0.1.1}/src/perception/hosts.py

@@ -1,5 +1,9 @@
 """Other machines nodes can run on: lab servers reached over SSH (`perception hosts add NAME DEST`).
 
+DEST is what you would give ssh (user@host, or a Host from ~/.ssh/config). --user, --port and --identity override what
+~/.ssh/config says for it; every command sent to the host uses them (Host.ssh). `perception hosts probe DEST` reaches a
+destination and says what it has without adding it or changing anything there (a connection test).
+
 A host is probed when it is added (and by `hosts check`): its platform, cores, memory, GPU and tools are recorded in
 $PERCEPTION_HOME/hosts.json, so planning never waits on the network. SSH runs in batch mode with the user's own keys
 and ~/.ssh/config: nothing here asks for or stores a password. Connections are shared (ControlMaster), so the many short
@@ -36,6 +40,7 @@ DEST = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.@%+:\[\]-]*$")
 ROOT = re.compile(r"^[A-Za-z0-9_./~-]+$")
 DEFAULT_ROOT = ".cache/hyperstudy-perception"
 RESERVED = {"native", "self", "cloud", "runpod", "local"}
+USER = re.compile(r"^[A-Za-z0-9._][A-Za-z0-9._-]{0,63}$")
 
 # What the host has, as key=value lines. POSIX sh: Linux servers and Macs, no Python needed. Tools are looked for on the
 # PATH a login shell would add too (~/.local/bin is where uv installs itself), since ssh commands get a minimal PATH.
@@ -65,6 +70,10 @@ class Host:
     root: str                  # absolute, on the host
     caps: Capabilities
     probed_at: float
+    # overrides of what ~/.ssh/config says for dest (None: ssh's own)
+    user: str | None = None
+    port: int | None = None
+    identity: str | None = None   # a private key on this machine, absolute
 
     def to_json(self) -> dict:
         d = asdict(self)
@@ -74,20 +83,59 @@ class Host:
     @classmethod
     def from_json(cls, d: dict) -> "Host":
         caps = Capabilities(**{**d["caps"], "accelerators": set(d["caps"]["accelerators"])})
-        return cls(d["name"], d["dest"], d["root"], caps, d["probed_at"])
+        return cls(d["name"], d["dest"], d["root"], caps, d["probed_at"], d.get("user"), d.get("port"), d.get("identity"))
+
+    def ssh(self, fresh: bool = False) -> list[str]:
+        """ssh to this host: every command sent to it (probe, nodes, clean) goes through this, with its overrides.
+        `fresh`: its own connection, not a shared one (a check: see ssh_command)."""
+        return [*ssh_command(shared=not fresh, identity=self.identity), *ssh_options(self.user, self.port, self.identity)]
+
+
+def ssh_options(user: str | None, port: int | None, identity: str | None) -> list[str]:
+    return [*(["-l", user] if user else []), *(["-p", str(port)] if port else []),
+            *(["-i", identity, "-o", "IdentitiesOnly=yes"] if identity else [])]
 
 
-def ssh_command() -> list[str]:
+def check_overrides(dest: str, user: str | None, port: int | None, identity: str | None) -> str | None:
+    """Refuse overrides ssh would misread; the identity file as an absolute path on this machine."""
+    if user is not None:
+        if not USER.match(user):
+            raise HostError(f"not a user name: {user!r}")
+        if "@" in dest:
+            raise HostError(f"{dest} already names its user: give the user once (in the destination, or --user)")
+    if port is not None and not 1 <= port <= 65535:
+        raise HostError(f"not a port: {port}")
+    if identity is None:
+        return None
+    path = Path(identity).expanduser()
+    if not path.is_file():
+        raise HostError(f"no key file at {identity}")
+    path = path.resolve()
+    if any(ord(c) < 32 for c in str(path)):
+        raise HostError(f"not a plain path: {identity!r}")
+    if "%" in str(path) or "${" in str(path):   # ssh expands %u, ${HOME}… in -i: it would read another file than this
+        raise HostError(f"ssh would expand % or ${{ in a key file's path: rename {identity}")
+    return str(path)
+
+
+def ssh_command(shared: bool = True, identity: str | None = None) -> list[str]:
     """ssh with the options every remote command uses. PERCEPTION_SSH replaces the ssh program (the tests use a
-    stand-in that runs commands on this machine)."""
+    stand-in that runs commands on this machine). A connection is shared by host, port and user (%C), and by key file
+    too (`identity`): ssh's own sharing ignores the key, so a connection opened with one key would carry commands meant
+    for another. Not `shared`: a connection of its own, for a check (add, probe, check), whose yes must be this
+    connection's, never one another key opened."""
     control = Path(f"/tmp/perception-ssh-{os.getuid()}")   # short: a socket path must fit in 104 bytes on macOS
     control.mkdir(mode=0o700, exist_ok=True)
     st = control.stat()
     if st.st_uid != os.getuid() or st.st_mode & 0o077:
         raise HostError(f"{control} must be a directory only you can use")
-    return [os.environ.get("PERCEPTION_SSH", "ssh"), "-o", "BatchMode=yes", "-o", "ConnectTimeout=15",
-            "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=4",
-            "-o", "ControlMaster=auto", "-o", f"ControlPath={control}/%C", "-o", "ControlPersist=120"]
+    base = [os.environ.get("PERCEPTION_SSH", "ssh"), "-o", "BatchMode=yes", "-o", "ConnectTimeout=15",
+            "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=4"]
+    if not shared:
+        return [*base, "-o", "ControlMaster=no", "-o", "ControlPath=none"]
+    import hashlib
+    keyed = f"-{hashlib.sha256(identity.encode()).hexdigest()[:12]}" if identity else ""   # short: the socket path's limit
+    return [*base, "-o", "ControlMaster=auto", "-o", f"ControlPath={control}/%C{keyed}", "-o", "ControlPersist=120"]
 
 
 def check_names(name: str, dest: str, root: str):
@@ -126,11 +174,12 @@ def private_root_script(root: str) -> str:
     return f"root={path}\n" + PRIVATE_ROOT
 
 
-def probe(dest: str, root: str = DEFAULT_ROOT, ssh: list[str] | None = None) -> tuple[Capabilities, str]:
+def probe(dest: str, root: str = DEFAULT_ROOT, ssh: list[str] | None = None, make_root: bool = True) -> tuple[Capabilities, str]:
     """What the host has, and its root as an absolute path (made private there, or refused: PRIVATE_ROOT). `ssh`: the
-    command to reach it, when it needs options of its own (a RunPod pod's key and port)."""
-    out = subprocess.run([*(ssh or ssh_command()), dest, "sh -s"], input=private_root_script(root) + PROBE, capture_output=True,
-                         text=True, timeout=120)
+    command to reach it, when it needs options of its own (overrides, a RunPod pod's key and port). Without `make_root`
+    nothing on the host is changed: the root is only where it would be (a connection test)."""
+    script = (private_root_script(root) if make_root else "") + PROBE
+    out = subprocess.run([*(ssh or ssh_command()), dest, "sh -s"], input=script, capture_output=True, text=True, timeout=120)
     if out.returncode:
         raise HostError(f"could not reach {dest} (ssh exit {out.returncode}): {out.stderr.strip()[-500:]}")
     facts = dict(line.split("=", 1) for line in out.stdout.splitlines() if "=" in line)
@@ -175,10 +224,12 @@ class Hosts:
         tmp.write_text(json.dumps({n: h.to_json() for n, h in sorted(hosts.items())}, indent=1))
         os.replace(tmp, self.path)
 
-    def add(self, name: str, dest: str, root: str = DEFAULT_ROOT) -> Host:
+    def add(self, name: str, dest: str, root: str = DEFAULT_ROOT, user: str | None = None, port: int | None = None,
+            identity: str | None = None) -> Host:
         check_names(name, dest, root)
-        caps, absolute = probe(dest, root)
-        host = Host(name, dest, absolute, caps, time.time())
+        identity = check_overrides(dest, user, port, identity)
+        caps, absolute = probe(dest, root, [*ssh_command(shared=False), *ssh_options(user, port, identity)])
+        host = Host(name, dest, absolute, caps, time.time(), user, port, identity)
         hosts = self.load()
         hosts[name] = host
         self.save(hosts)
@@ -189,8 +240,8 @@ class Hosts:
         if name not in hosts:
             raise HostError(f"no host {name!r} (perception hosts list)")
         old = hosts[name]
-        caps, _ = probe(old.dest, old.root)
-        hosts[name] = Host(name, old.dest, old.root, caps, time.time())
+        caps, _ = probe(old.dest, old.root, old.ssh(fresh=True))
+        hosts[name] = Host(name, old.dest, old.root, caps, time.time(), old.user, old.port, old.identity)
         self.save(hosts)
         return hosts[name]
 
@@ -199,3 +250,11 @@ class Hosts:
         if hosts.pop(name, None) is None:
             raise HostError(f"no host {name!r}")
         self.save(hosts)
+
+
+def probe_dest(dest: str, root: str = DEFAULT_ROOT, user: str | None = None, port: int | None = None,
+               identity: str | None = None) -> tuple[Capabilities, str]:
+    """Reach a destination as `hosts add` would and say what it has, changing nothing there and adding nothing here."""
+    check_names("probe", dest, root)
+    identity = check_overrides(dest, user, port, identity)
+    return probe(dest, root, [*ssh_command(shared=False), *ssh_options(user, port, identity)], make_root=False)

hyperstudy_perception-0.1.1/src/perception/keycheck.py

@@ -0,0 +1,64 @@
+"""Whether a provider accepts a key (`perception keys check NAME`): one request that reads something small and costs
+nothing (Anthropic's model list, your Hugging Face account, your RunPod pods). The key is sent only to its own
+provider, never through a redirect (runpod.OPENER)."""
+
+from __future__ import annotations
+
+import http.client
+import json
+import urllib.error
+import urllib.request
+
+from .runpod import OPENER, RunPod, RunPodError
+
+ANTHROPIC = "https://api.anthropic.com/v1/models?limit=1"
+HUGGING_FACE = "https://huggingface.co/api/whoami-v2"
+CHECKED = ("ANTHROPIC_API_KEY", "HF_TOKEN", "RUNPOD_API_KEY")
+
+
+def get(url: str, headers: dict[str, str]) -> tuple[int, bytes]:
+    """The status and body of a GET (an HTTP error's too); a provider that cannot be reached raises OSError."""
+    try:
+        with OPENER.open(urllib.request.Request(url, headers=headers), timeout=20) as r:
+            return r.status, r.read()
+    except urllib.error.HTTPError as e:
+        return e.code, e.read()
+
+
+def account(body: bytes) -> str | None:
+    """The account name in Hugging Face's answer, if it says one (the key was accepted either way)."""
+    try:
+        d = json.loads(body or b"{}")
+    except ValueError:
+        return None
+    return d.get("name") if isinstance(d, dict) else None
+
+
+def check(name: str, value: str | None) -> tuple[bool, str]:
+    """(accepted, what to tell the user)."""
+    if name not in CHECKED:
+        raise ValueError(f"no check for {name}: perception can check {', '.join(CHECKED)}")
+    if not value:
+        return False, f"{name} is not set"
+    try:
+        if name == "RUNPOD_API_KEY":
+            try:
+                RunPod(value).pods()
+            except RunPodError as e:
+                return False, ("RunPod refused the key" if e.status in (401, 403) else f"RunPod could not be checked: {e}")
+            return True, "RunPod accepted the key"
+        if name == "ANTHROPIC_API_KEY":
+            status, _ = get(ANTHROPIC, {"x-api-key": value, "anthropic-version": "2023-06-01"})
+            provider = "Anthropic"
+        else:
+            status, body = get(HUGGING_FACE, {"Authorization": f"Bearer {value}"})
+            provider = "Hugging Face"
+            if status == 200:
+                return True, f"Hugging Face accepted the key{f' (as {who})' if (who := account(body)) else ''}"
+    except (OSError, http.client.HTTPException) as e:   # no whole answer (a reply cut short is not an OSError)
+        return False, f"could not reach the provider to check: {e}"
+    if status == 200:
+        return True, f"{provider} accepted the key"
+    if status in (401, 403):
+        return False, f"{provider} refused the key"
+    return False, f"{provider} could not be checked (HTTP {status})"