flashnode 0.3.5__tar.gz → 0.4.0__tar.gz

{flashnode-0.3.5 → flashnode-0.4.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: flashnode
-Version: 0.3.5
+Version: 0.4.0
 Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: flashruntime<0.6,>=0.4.3
+Requires-Dist: flashruntime<0.7,>=0.6
 Requires-Dist: psutil>=5.9
 Requires-Dist: websockets>=12
 Requires-Dist: cryptography>=42

{flashnode-0.3.5 → flashnode-0.4.0}/flashnode/agent/cli.py

@@ -29,6 +29,8 @@ commands:
   doctor    check this machine can run tasks (docker engine, images, mounts)
   env       see and reclaim disk held by cached dependency environments
             (list | purge — purge never touches node-id or credentials.json)
+  data      see and reclaim disk held by cached dataset shards
+            (list | purge — purge never touches node-id or credentials.json)
   join      connect this machine to a FlashML control plane (not yet implemented)
   status    show node identity, capabilities, and active leases (not yet implemented)
   leave     drain and disconnect (not yet implemented)
@@ -624,6 +626,139 @@ def _env(args: list[str]) -> int:
     return 0
 
 
+def _data(args: list[str]) -> int:
+    """`flashnode data list|purge` — the disk cached dataset shards hold.
+
+    The sibling of `_env`, with the same contract: `purge` deletes only what
+    `DatasetCache.entries()` reports — the contents of
+    `$FLASHNODE_STATE_DIR/data/` — and never the state directory itself, so
+    `node-id` and `credentials.json` survive. That is the entire difference
+    between this command and un-enrolling the machine.
+
+    Its own command rather than a flag on `env` because the two caches hold
+    different orders of magnitude for different reasons, and a host owner
+    who wants to keep 8 GB of virtualenvs while giving back 200 GB of a
+    stranger's shards must be able to say exactly that.
+    """
+    import argparse
+    import shutil
+    from datetime import datetime
+
+    from flashnode.executor.datasets import DatasetCache
+
+    parser = argparse.ArgumentParser(prog="flashnode data")
+    sub = parser.add_subparsers(dest="subcommand", required=True)
+    sub.add_parser("list", help="show cached shards, their size, and last use")
+    sub.add_parser(
+        "purge",
+        help="delete all cached dataset shards (identity and credentials "
+             "survive; stop `flashnode work` first — this can break a task "
+             "in flight)",
+    )
+    opts = parser.parse_args(args)
+
+    cache = DatasetCache()
+    entries = cache.entries()  # most recently used first
+
+    # Scratch directories from fetches that were KILLED — SIGKILL, the OOM
+    # killer, a power cut. `entries()` skips dotted names, so without this
+    # the two commands whose whole job is to account for disk would be the
+    # two that could not see a half-downloaded multi-GB shard.
+    stale = cache.stale_bytes()
+
+    if opts.subcommand == "list":
+        if not entries and not stale:
+            print(
+                "flashnode data: no dataset shards cached — nothing is using "
+                "disk yet."
+            )
+            return 0
+        for entry in entries:
+            when = datetime.fromtimestamp(entry.last_used).strftime("%Y-%m-%d %H:%M")
+            # Truncated: the key is a sha256, and 64 hex characters per line
+            # buries the two columns the owner is actually reading.
+            print(
+                f"  {entry.key[:16]}…  {_human_bytes(entry.size_bytes):>8}  "
+                f"last used {when}"
+            )
+        total = sum(e.size_bytes for e in entries) + stale
+        if stale:
+            print(
+                f"  leftover fetch directories  {_human_bytes(stale):>8}  "
+                "from interrupted fetches"
+            )
+        print(
+            f"total: {_human_bytes(total)} across {len(entries)} shard(s) "
+            f"at {cache.root}"
+        )
+        if stale:
+            print(
+                "  the leftover fetch directories are swept by the next "
+                "fetch, or now by `flashnode data purge`"
+            )
+        return 0
+
+    # purge
+    #
+    # This runs in a SEPARATE process from the agent, so it sidesteps the
+    # `_evict(protect=…)` that makes in-process eviction safe and can delete
+    # the shard a running task is reading through its hard link. Refusing
+    # would beat warning, but nothing in the state directory records a
+    # running agent — no pidfile, no lock — and a guessed signal would refuse
+    # on a machine with no agent at all. So: say it, plainly, every time, and
+    # before the deletion rather than after it.
+    print(
+        "flashnode data purge: warning — if `flashnode work` is running on "
+        "this machine, this can delete a shard a task is reading and fail "
+        "that task. This command cannot tell whether an agent is running. "
+        "Stop the agent first if you are not sure."
+    )
+    # Counted AFTER each removal, never before — the `_env` bug, which must
+    # not be reintroduced here. Totalling up front and deleting with
+    # `ignore_errors=True` prints the full figure as reclaimed when a
+    # permission error left the bytes on disk, on the one command whose
+    # entire value is that number and which a volunteer has no other way to
+    # check.
+    freed = 0
+    removed = 0
+    survivors = []
+    for entry in entries:
+        shutil.rmtree(entry.path, ignore_errors=True)
+        if entry.path.exists():
+            survivors.append(entry.path)
+        else:
+            freed += entry.size_bytes
+            removed += 1
+    stale_freed = cache.sweep_stale()  # counts only what actually went
+    freed += stale_freed
+    survivors.extend(cache.stale_dirs())
+    leftovers = (
+        f" (including {_human_bytes(stale_freed)} of leftover fetch "
+        "directories from interrupted fetches)"
+        if stale_freed
+        else ""
+    )
+    print(
+        f"flashnode data purge: freed {_human_bytes(freed)} across "
+        f"{removed} shard(s){leftovers}. node-id and credentials.json were "
+        "not touched."
+    )
+    if survivors:
+        noun = "directory" if len(survivors) == 1 else "directories"
+        print(
+            f"flashnode data purge: could not remove {len(survivors)} {noun} "
+            "— the disk it holds was NOT reclaimed:"
+        )
+        for path in survivors:
+            print(f"  {path}")
+        print(
+            "  check the permissions on those paths, or whether another "
+            "process is holding them open, and run purge again."
+        )
+        return 1
+    return 0
+
+
 def main(argv: list[str] | None = None) -> int:
     args = sys.argv[1:] if argv is None else argv
     if args and args[0] == "agent":
@@ -642,6 +777,8 @@ def main(argv: list[str] | None = None) -> int:
         return _logout(args[1:])
     if args and args[0] == "env":
         return _env(args[1:])
+    if args and args[0] == "data":
+        return _data(args[1:])
     print(USAGE.format(version=__version__), end="")
     if args and args[0] in {"join", "status", "leave"}:
         print(f"\nerror: '{args[0]}' is not implemented yet in this scaffold.", file=sys.stderr)

flashnode-0.4.0/flashnode/executor/datasets.py

@@ -0,0 +1,457 @@
+"""Cached dataset shards, one entry per shard checksum.
+
+A task runs with `--network none` and cannot fetch its own data. The agent
+fetches instead, before the sandbox closes, and this module is where the
+bytes live between tasks.
+
+Modelled on `environments.py`, and for the same reason: **disk is somebody
+else's.** Bounded (FLASHNODE_DATA_BUDGET_GB, default 16), evicted
+least-recently-USED, and a fetch is refused before it starts unless the
+filesystem still has 1 GB free.
+
+Two deliberate differences from EnvironmentCache:
+
+* **The key is the shard's own checksum**, not a hash of a request. Two
+  jobs training on the same public dataset — or two revisions of one
+  dataset sharing an unchanged shard — hit the same entry for free.
+* **Size is a written marker.** EnvironmentCache measures with a tree walk
+  on every `entries()` call, which `_evict` also calls. That is fine for a
+  handful of venvs and hopeless for a dataset of a million files.
+
+A separate budget from the venv cache on purpose: a host owner should be
+able to say yes to 8 GB of virtualenvs and no to 200 GB of a stranger's
+images. Different orders of magnitude, different consent.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import http.client
+import os
+import re
+import shutil
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+from flashnode.executor.runner import TaskExecutionError
+from flashnode.identity.store import state_dir
+
+DEFAULT_BUDGET_GB = 16.0
+# A floor on free space that must EXIST before a fetch starts. Read it
+# literally: it is not a promise the gigabyte survives the fetch.
+MIN_FREE_BYTES = 1024**3
+_USED_MARKER = "used"
+_SIZE_MARKER = "size"
+_BUILD_PREFIX = ".fetching-"
+
+#: Same shape flashnode already requires of an input name (loop.py) and a
+#: local-data label. A dataset name becomes a directory on a stranger's disk.
+_SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
+
+#: Transport-level retry, the same shape and for the same measured reason as
+#: `client._TRANSPORT_ATTEMPTS`: a single dropped connection on a volunteer's
+#: home network must not cost the submitter their task.
+#: What counts as "the origin hiccuped" rather than "the task is broken".
+#:
+#: `http.client.IncompleteRead` is here and is NOT in `client.py`'s
+#: byte-identical-looking tuple, deliberately. It descends from
+#: HTTPException, not OSError, so the obvious three-way tuple misses it —
+#: and a CDN closing a connection early is the single most likely way a
+#: large shard fetch fails. Uncaught it leaves `materialise`, misses
+#: `_execute_inner`'s `except TaskExecutionError`, and lands in
+#: `execute_one`'s last-line-of-defence handler, which says of itself:
+#: "This DOES count against the host". So a flaky ORIGIN would be charged
+#: to the volunteer and could quarantine their machine — inverting the one
+#: rule this module exists to keep, that a broken origin costs the
+#: submitter their task and never the volunteer their node.
+#:
+#: `client.py` has the same gap on the coordinator transport. Left alone
+#: here on purpose: changing shipped retry semantics is a separate change
+#: with its own evidence, not a side effect of adding datasets.
+_TRANSIENT = (
+    urllib.error.URLError,
+    TimeoutError,
+    OSError,
+    http.client.IncompleteRead,
+)
+
+_FETCH_ATTEMPTS = 4
+_FETCH_BACKOFF_S = 0.5
+_CHUNK = 1 << 20  # 1 MiB
+
+
+class DatasetFetchError(TaskExecutionError):
+    """A shard could not be fetched, verified, or stored.
+
+    Subclasses TaskExecutionError deliberately: the executor loop already
+    reports the cause through fail() before re-raising, so a broken origin
+    costs the submitter their task and never the volunteer their node.
+    """
+
+
+@dataclass(frozen=True)
+class ShardEntry:
+    key: str
+    path: Path
+    size_bytes: int
+    last_used: float
+
+
+def _free_bytes(path: Path) -> int:
+    # A module function, not an inline shutil call, so tests can patch it by
+    # dotted string — the seam test_environments.py already relies on.
+    return shutil.disk_usage(path).free
+
+
+def budget_bytes_from_env() -> int:
+    """FLASHNODE_DATA_BUDGET_GB in bytes, or 0 if it is not a usable number.
+
+    THE one place this variable is parsed. It has two readers that must
+    never disagree: this cache, which decides what the host will actually
+    hold, and `inventory.capabilities.discover`, which decides what the
+    host ADVERTISES. Advertising more than the cache holds means accepting
+    work the cache then evicts mid-job, so a second copy of this arithmetic
+    is a bug waiting for a typo — the same duplicated-derivation trap
+    `elastic.py`'s docstring warns about.
+
+    Malformed reads as 0, never as the default. Fail closed: 0 is
+    indistinguishable on the wire from the owner's deliberate opt-out and
+    costs nobody a task, whereas silently substituting 16 GB would advertise
+    capacity the owner never agreed to lend.
+
+    `int()` is inside the guard, not just `float()`: `float("nan")` parses
+    happily and raises in `int()`, and `"inf"` raises OverflowError there.
+    """
+    try:
+        return max(0, int(float(
+            os.environ.get("FLASHNODE_DATA_BUDGET_GB", DEFAULT_BUDGET_GB)
+        ) * 1024**3))
+    except (TypeError, ValueError, OverflowError):
+        return 0
+
+
+class DatasetCache:
+    """Shard checksum -> cached file, bounded and LRU-evicted."""
+
+    def __init__(self, root: Path | None = None, budget_gb: float | None = None):
+        self.root = Path(root) if root is not None else state_dir() / "data"
+        self.budget_bytes = (
+            budget_bytes_from_env()
+            if budget_gb is None
+            else max(0, int(budget_gb * 1024**3))
+        )
+
+    # -- public -----------------------------------------------------------
+    def get(self, integrity_value: str) -> Path | None:
+        """The cached shard for this checksum, or None. Records a USE."""
+        entry = self.root / integrity_value
+        blob = entry / "blob"
+        if not blob.exists():
+            return None
+        self._touch(entry)
+        return blob
+
+    def put(self, integrity_value: str, source: Path) -> Path:
+        """Publish `source` under `integrity_value`; return the stored path.
+
+        The move is a rename into place, the write-then-rename idiom
+        `environments.py` and `identity/store.py` both use, so a partial
+        entry is never published or returned.
+        """
+        self.root.mkdir(parents=True, exist_ok=True)
+        entry = self.root / integrity_value
+        size = Path(source).stat().st_size
+        scratch = self.root / f"{_BUILD_PREFIX}{integrity_value}"
+        if scratch.exists():
+            shutil.rmtree(scratch, ignore_errors=True)
+        scratch.mkdir(parents=True)
+        try:
+            os.replace(source, scratch / "blob")
+            # Read-only, and load-bearing rather than tidy. `materialise`
+            # hard-links out of this cache, so the file a task sees at
+            # /work/data/ IS this blob — one inode, many jobs. A task writing
+            # to its own input therefore rewrites the CACHE, and every later
+            # job on this host gets the corrupted bytes as a HIT, with no
+            # re-verification and no signal. Measured before this line: job 1
+            # writes, job 2 silently trains on job 1's garbage.
+            #
+            # 0o444 turns that into an immediate EACCES in the job that tried
+            # it, which is the correct blame. Eviction still works: removing a
+            # file needs write permission on the DIRECTORY, not on the file.
+            (scratch / "blob").chmod(0o444)
+            (scratch / _SIZE_MARKER).write_text(str(size))
+            self._touch(scratch)
+            if entry.exists():  # wreckage from an interrupted run
+                shutil.rmtree(entry, ignore_errors=True)
+            scratch.replace(entry)
+        except BaseException:
+            shutil.rmtree(scratch, ignore_errors=True)
+            raise
+        self._evict(protect=entry)
+        return entry / "blob"
+
+    def reserve(self, nbytes: int) -> None:
+        """Prune, then refuse unless the disk can take `nbytes`.
+
+        Pruning first is load-bearing, not an optimisation: `_evict`
+        otherwise runs only after a successful put, so a node that dips
+        under the floor can never fetch, therefore never evicts, therefore
+        never recovers.
+
+        Unlike EnvironmentCache's floor — which cannot know an install's
+        size until the resolver has run — a manifest entry states its own
+        `size`, so the bytes about to arrive are counted here as well as
+        the floor beneath them.
+        """
+        self.root.mkdir(parents=True, exist_ok=True)
+        if _free_bytes(self.root) - nbytes >= MIN_FREE_BYTES:
+            return
+        self._evict()
+        free = _free_bytes(self.root)
+        if free - nbytes < MIN_FREE_BYTES:
+            raise DatasetFetchError(
+                f"refusing to fetch {nbytes // 1024**2} MB of dataset: after "
+                f"pruning the shard cache to its "
+                f"{self.budget_bytes // 1024**2} MB budget, only "
+                f"{free // 1024**2} MB is free on disk at {self.root}, and "
+                f"this node will not start a fetch that would leave less "
+                f"than {MIN_FREE_BYTES // 1024**2} MB free. Free disk space, "
+                f"or lower FLASHNODE_DATA_BUDGET_GB so the next attempt "
+                f"prunes further."
+            )
+
+    def entries(self) -> list[ShardEntry]:
+        """Cached shards, most recently used first."""
+        if not self.root.is_dir():
+            return []
+        found = []
+        for path in self.root.iterdir():
+            if not path.is_dir() or path.name.startswith("."):
+                continue
+            # The written marker, not a tree walk. `_evict` calls this on
+            # every put, and a shard directory can hold a million files.
+            # The fallback covers an entry whose marker was lost with the
+            # process that should have written it: one stat is still cheap,
+            # and reporting 0 would make `_evict` unable to shrink.
+            try:
+                size = int((path / _SIZE_MARKER).read_text())
+            except (OSError, ValueError):
+                blob = path / "blob"
+                size = blob.stat().st_size if blob.exists() else 0
+            # Seconds, deliberately: `agent/cli.py` renders this with
+            # `datetime.fromtimestamp`. The PRECISION comes from the written
+            # nanosecond stamp — see `_touch`; the unit does not change.
+            try:
+                last_used = int((path / _USED_MARKER).read_text()) / 1e9
+            except (OSError, ValueError):
+                last_used = path.stat().st_mtime
+            found.append(ShardEntry(path.name, path, size, last_used))
+        return sorted(found, key=lambda e: e.last_used, reverse=True)
+
+    def stale_dirs(self) -> list[Path]:
+        """Scratch directories left by fetches that were killed outright.
+
+        Deliberately NOT folded into `entries()`, for the reason
+        `EnvironmentCache.stale_build_dirs` gives: those are cache entries
+        `_evict` deletes by age to stay inside the budget, and a half-written
+        shard has no `used` marker and is not a cacheable thing. Reported
+        separately so the owner sees the bytes rather than the cache quietly
+        under-reporting what it holds.
+        """
+        if not self.root.is_dir():
+            return []
+        return sorted(
+            p for p in self.root.iterdir()
+            if p.is_dir() and p.name.startswith(_BUILD_PREFIX)
+        )
+
+    def stale_bytes(self) -> int:
+        return sum(
+            sum(f.stat().st_size for f in p.rglob("*") if f.is_file())
+            for p in self.stale_dirs()
+        )
+
+    def sweep_stale(self) -> int:
+        """Delete wreckage; return bytes actually reclaimed (counted after)."""
+        freed = 0
+        for path in self.stale_dirs():
+            size = sum(f.stat().st_size for f in path.rglob("*") if f.is_file())
+            shutil.rmtree(path, ignore_errors=True)
+            if not path.exists():
+                freed += size
+        return freed
+
+    # -- internals --------------------------------------------------------
+    def _touch(self, entry: Path) -> None:
+        """Record a USE with a written nanosecond stamp.
+
+        NOT mtime. Linux stamps inodes from a coarse clock, so two events
+        microseconds apart get byte-identical st_mtime_ns; `sorted` is
+        stable, so the tie fell through to iterdir() order, which ext4
+        hashes. The result was an LRU that evicted its most recently used
+        entry. See EnvironmentCache._touch — do not simplify this.
+        """
+        (entry / _USED_MARKER).write_text(str(time.time_ns()))
+
+    def _evict(self, protect: Path | None = None) -> None:
+        """Delete least-recently-used entries until within budget.
+
+        `protect` is the entry a caller is about to be handed. Pruning to
+        make room takes no such argument: nothing is in use at that moment,
+        so the cache may empty itself if that is what the budget now says.
+        """
+        entries = self.entries()  # most recently used first
+        total = sum(e.size_bytes for e in entries)
+        for entry in reversed(entries):  # oldest first
+            if total <= self.budget_bytes:
+                return
+            if entry.path == protect:
+                continue
+            shutil.rmtree(entry.path, ignore_errors=True)
+            total -= entry.size_bytes
+
+
+def _open_url(url: str, offset: int):
+    """Open `url`, optionally resuming at `offset`. The single network seam.
+
+    Deliberately NOT `client._request`: that prefixes the coordinator base
+    URL and attaches the agent's bearer token, and a dataset origin is a
+    third party that must never see it.
+
+    A module function, and the ONLY place this module touches the network,
+    so every test above patches it by dotted string and the real thing runs
+    only under `-m integration`.
+    """
+    request = urllib.request.Request(url, headers={"User-Agent": "flashnode"})
+    if offset:
+        request.add_header("Range", f"bytes={offset}-")
+    return urllib.request.urlopen(request, timeout=60)
+
+
+def fetch_shard(entry: dict, cache: DatasetCache, scratch: Path) -> Path:
+    """Return a local path to this manifest entry's bytes, fetching if needed."""
+    integrity = entry.get("integrity") or {}
+    kind = integrity.get("kind")
+    value = integrity.get("value")
+    if not isinstance(value, str) or not value:
+        raise DatasetFetchError(
+            f"manifest entry {entry.get('path')!r} has no integrity value"
+        )
+
+    cached = cache.get(value)
+    if cached is not None:
+        return cached
+
+    size = entry.get("size")
+    cache.reserve(int(size) if isinstance(size, int) and size > 0 else 0)
+
+    scratch.mkdir(parents=True, exist_ok=True)
+    target = scratch / "blob"
+    written = 0
+    last: Exception | None = None
+    for attempt in range(_FETCH_ATTEMPTS):
+        try:
+            with _open_url(entry["url"], written) as resp:
+                mode = "ab" if written else "wb"
+                with open(target, mode) as fh:
+                    # copyfileobj, never resp.read(): a multi-GB shard must
+                    # not be assembled in RAM on a volunteer's laptop.
+                    shutil.copyfileobj(resp, fh, _CHUNK)
+            written = target.stat().st_size
+            break
+        except _TRANSIENT as exc:
+            last = exc
+            # Resume from whatever landed. An origin that ignores the Range
+            # header and re-serves the whole body will produce a wrong file,
+            # not a silently wrong one — the checksum below catches it.
+            written = target.stat().st_size if target.exists() else 0
+            if attempt < _FETCH_ATTEMPTS - 1:
+                time.sleep(_FETCH_BACKOFF_S * (2 ** attempt))
+    else:
+        raise DatasetFetchError(
+            f"could not fetch {entry.get('path')!r} after {_FETCH_ATTEMPTS} "
+            f"attempts: {last}"
+        )
+
+    # sha256 is the only kind we can prove. An ETag is md5-of-md5s for a
+    # multipart upload, so asserting on it would fail every valid multipart
+    # object; it is a change-detection token the control plane already used
+    # to pin the revision, not something to verify here.
+    # `declared-sha256` is verified exactly like `sha256`. The two differ in
+    # PROVENANCE, not in arithmetic: one hash came from the origin's own API,
+    # the other from a manifest the submitter wrote. Neither is a reason to
+    # skip the check — a declared hash is the only integrity signal an
+    # arbitrary HTTPS origin offers at all, so not checking it would leave
+    # that whole scheme with no verification of any kind.
+    #
+    # An ETag stays unverified: for a multipart upload it is md5-of-md5s, so
+    # asserting on it would fail every valid multipart object.
+    if kind in ("sha256", "declared-sha256"):
+        computed = hashlib.sha256()
+        with open(target, "rb") as fh:
+            # Streamed, like the download: hashing via read_bytes() would
+            # undo the whole point of copyfileobj above.
+            for block in iter(lambda: fh.read(_CHUNK), b""):
+                computed.update(block)
+        if computed.hexdigest() != value:
+            target.unlink(missing_ok=True)
+            raise DatasetFetchError(
+                f"checksum mismatch for {entry.get('path')!r}: the origin "
+                f"served bytes that are not what the manifest pinned "
+                f"(wanted {value[:16]}…, got {computed.hexdigest()[:16]}…). "
+                f"The dataset may have changed since the job was submitted."
+            )
+    return cache.put(value, target)
+
+
+def materialise(datasets: list[dict], cache: DatasetCache, workdir: Path) -> None:
+    """Lay every declared dataset out under `workdir/data/<name>/`.
+
+    Hard-links from the cache where the filesystem allows it and copies
+    otherwise, so N tasks sharing a shard cost one copy on disk.
+    """
+    for dataset in datasets or []:
+        name = dataset.get("name")
+        if (
+            not isinstance(name, str)
+            or not _SAFE_NAME.match(name)
+            or name in (".", "..")
+        ):
+            raise DatasetFetchError(
+                f"refusing a dataset with unsafe name {name!r}: a name is not "
+                f"a path"
+            )
+        root = (workdir / "data" / name).resolve()
+        for entry in dataset.get("entries") or []:
+            rel = str(entry.get("path") or "")
+            dest = (root / rel).resolve()
+            # Resolve THEN require containment. A naive join misses the
+            # absolute form entirely: Path("/work/data") / "/etc/x" is
+            # "/etc/x" — pathlib discards the left operand. Same reasoning
+            # as archives.py.
+            #
+            # Strictly INSIDE, never equal: an entry whose path is empty or
+            # "." names the dataset directory itself, and linking a blob
+            # onto a directory raises a bare OSError out of here. A broken
+            # manifest must fail the task as a typed refusal.
+            if not str(dest).startswith(str(root) + os.sep):
+                raise DatasetFetchError(
+                    f"refusing dataset entry {rel!r}: it resolves outside "
+                    f"{root}"
+                )
+            blob = fetch_shard(entry, cache, workdir / ".fetching" / name)
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                os.link(blob, dest)
+            except OSError:
+                # A separate filesystem, a hard-link-hostile one, or the
+                # destination already there from a previous task on this
+                # workdir. A copy is slower, never wrong.
+                shutil.copyfile(blob, dest)
+                # A copy is a NEW inode and would default to writable, so the
+                # link and copy paths would disagree about whether a task can
+                # scribble on its own input. Freeze it to match.
+                dest.chmod(0o444)

{flashnode-0.3.5 → flashnode-0.4.0}/flashnode/executor/loop.py

@@ -431,6 +431,37 @@ class ExecutorLoop:
                         f"unpack_inputs names inputs that do not exist: {sorted(unknown)}"
                     )
 
+                declared = payload.get("datasets")
+                if declared:
+                    # Shape-checked first, exactly as `unpack_inputs` is
+                    # above and for the same reason: the payload is
+                    # attacker-influenced all the way from the submission,
+                    # and `materialise` reaches for `.get` on every element.
+                    # A `datasets` that is not a list of dicts would raise a
+                    # bare AttributeError, miss the `except TaskExecutionError`
+                    # below, never reach fail(), and land in `execute_one`'s
+                    # last-line-of-defence — which counts against the HOST.
+                    # A broken payload must cost the submitter their task.
+                    if not isinstance(declared, list) or not all(
+                        isinstance(d, dict) for d in declared
+                    ):
+                        raise TaskExecutionError(
+                            "payload 'datasets' must be a list of dataset slices"
+                        )
+                    # Before the runner, always. The task runs with
+                    # `--network none`, so a fetch after the sandbox closes
+                    # is a fetch that never happens.
+                    #
+                    # Imported here, not at module scope, following the rule
+                    # trusted_runner.py already follows for `environments`.
+                    from flashnode.executor import datasets as datasets_mod
+
+                    datasets_mod.materialise(
+                        declared, datasets_mod.DatasetCache(), workdir,
+                    )
+                    log.info(_jlog("datasets ready", task=lease.task_id,
+                                   count=len(declared)))
+
                 prefix = payload.get("output_prefix", f"jobs/{lease.job_id}/{lease.task_id}/")
                 if payload.get("checkpoint") is not None:
                     # resume from the task's latest valid checkpoint, wherever