flashnode 0.2.0__py3-none-any.whl

flashnode/executor/archives.py

@@ -0,0 +1,290 @@
+"""Unpacking an untrusted archive onto a volunteer's machine.
+
+This is the one place in flashnode that writes attacker-chosen *paths* to a
+host owner's disk. The bytes arrive as a task input: any signed-in user
+points the cloud API at any repo they like, the API stages the original
+archive as an ``artifact://`` input, and this agent downloads it. Member
+names, symlink targets, member types and declared sizes are all chosen by
+whoever submitted the job — never by us.
+
+The cloud API has an equivalent guard on its own copy of the same bytes
+(``flashml_cloud_api.repo.extract_safely``). That one is *not* reusable
+here: flashnode may import ``flashruntime.protocol`` and nothing else (see
+AGENTS.md hard rule 2), and a private control-plane module must never be a
+dependency of the public host agent. So the guard is deliberately
+duplicated, and the drift that duplication invites is caught by a parity
+test in the workspace ``e2e/`` suite — the only place allowed to import
+both repos — which feeds one attack corpus to both extractors.
+
+What is refused, and why each one matters on someone else's laptop:
+
+- **zip-slip**, relative (``../../.ssh/authorized_keys``) or absolute
+  (``/etc/cron.d/evil``). Both are caught by the same check rather than
+  special-cased, because a naive join-then-compare misses the absolute
+  form entirely: ``Path("/work/inputs/code") / "/etc/evil"`` is
+  ``/etc/evil`` — pathlib silently discards the left operand when the
+  right side is absolute. Resolving first and then requiring containment
+  via ``relative_to`` has no such blind spot.
+- **escaping symlinks**. A member linking to ``/`` or ``../..`` turns a
+  later, individually-innocent write into an escape. Contained symlinks
+  are validated and then *not created at all*: nothing downstream needs a
+  link, and not materialising one removes the whole TOCTOU class (member
+  ``a`` is a symlink to a safe dir; member ``a/b`` then resolves through
+  it) for free.
+- **decompression bombs**, checked *during* extraction and in bounded
+  chunks. A 40 KB archive can hold petabytes; a total measured after the
+  fact has already filled the disk it was meant to protect.
+- **member count**, because millions of empty files exhaust inodes without
+  ever tripping a byte cap.
+- **device, fifo and hard-link members**, which have no legitimate place in
+  a code archive and every place in an attack.
+
+Nothing is executed, nothing is made executable (members are rewritten as
+plain files with the process umask, so an archive cannot smuggle in a
+setuid bit), and on any refusal everything already written is removed —
+a rejected archive must not leave half a payload behind for the next stage
+to trip over.
+"""
+
+from __future__ import annotations
+
+import shutil
+import stat
+import tarfile
+import zipfile
+from pathlib import Path, PurePosixPath
+
+#: Read/write in bounded chunks so an oversized member is caught partway
+#: through rather than only once it is fully on disk.
+_CHUNK_SIZE = 1 << 16  # 64 KiB
+
+#: Defaults for a task input. Generous enough for a real repo (the cloud
+#: API caps an extracted repo at 128 MiB), small enough that a hostile job
+#: cannot fill a volunteer's disk before the cap fires.
+DEFAULT_MAX_BYTES = 128 * 1024 * 1024
+DEFAULT_MAX_MEMBERS = 20_000
+
+
+class ArchiveError(Exception):
+    """Raised for any archive this agent refuses to unpack: a malformed
+    container, a member whose path or symlink target escapes the
+    destination, an unsupported member type, or a cap (bytes, members)
+    exceeded. The executor turns it into a task failure, never into a
+    crashed agent — a hostile input must cost the submitter their task, not
+    the volunteer their node."""
+
+
+def _is_within(path: Path, root: Path) -> bool:
+    try:
+        path.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+def _safe_target(name: str, dest_root: Path) -> Path:
+    """Resolve a member name against the extraction root, refusing anything
+    that lands outside it. Handles the relative and the absolute escape with
+    one check — see the module docstring on why joining first is not enough
+    on its own but resolving after it is."""
+    resolved = (dest_root / name).resolve()
+    if not _is_within(resolved, dest_root):
+        raise ArchiveError(f"refusing archive member with unsafe path: {name!r}")
+    return resolved
+
+
+def _check_symlink(member_path: Path, link_name: str, dest_root: Path) -> None:
+    """A symlink target resolves against the link's own directory unless it
+    is absolute — standard symlink semantics, and the semantics an attacker
+    is counting on."""
+    if PurePosixPath(link_name).is_absolute():
+        target = Path(link_name).resolve()
+    else:
+        target = (member_path.parent / link_name).resolve()
+    if not _is_within(target, dest_root):
+        raise ArchiveError(
+            f"refusing symlink member {member_path.name!r} whose target "
+            f"{link_name!r} escapes the destination"
+        )
+
+
+def _top_level(name: str) -> str | None:
+    """First path component of a member name, or None for a name that has
+    none (``.``/``./``, which GNU tar emits for the archive root). The
+    None case is not hypothetical: indexing ``parts[0]`` unguarded turns a
+    one-member archive into an IndexError instead of a clean refusal."""
+    parts = PurePosixPath(name).parts
+    return parts[0] if parts else None
+
+
+def _write_stream(src, out_path: Path, budget: list[int]) -> None:
+    """Copy a member's bytes in chunks, decrementing a shared budget as we
+    go. ``budget`` is a one-element list so the running total survives
+    across members without a class or a global."""
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "wb") as out:
+        while True:
+            chunk = src.read(_CHUNK_SIZE)
+            if not chunk:
+                return
+            budget[0] -= len(chunk)
+            if budget[0] < 0:
+                raise ArchiveError(
+                    "archive exceeds the uncompressed size cap during "
+                    "extraction — refusing the rest"
+                )
+            out.write(chunk)
+
+
+def _extract_tar(src: Path, dest_root: Path, budget: list[int], max_members: int) -> set[str]:
+    try:
+        tar = tarfile.open(src, mode="r:*")
+    except tarfile.TarError as exc:
+        raise ArchiveError(f"not a valid tar archive: {exc}") from exc
+
+    tops: set[str] = set()
+    seen = 0
+    with tar:
+        for member in tar:
+            seen += 1
+            if seen > max_members:
+                raise ArchiveError(
+                    f"archive has more than {max_members} members — refusing the rest"
+                )
+            top = _top_level(member.name)
+            if top is not None:
+                tops.add(top)
+            member_path = _safe_target(member.name, dest_root)
+
+            if member.issym():
+                _check_symlink(member_path, member.linkname, dest_root)
+                continue  # validated, deliberately not created — see docstring
+            if member.islnk():
+                raise ArchiveError(
+                    f"refusing hard-link member: {member.name!r}"
+                )
+            if member.isdir():
+                member_path.mkdir(parents=True, exist_ok=True)
+                continue
+            if not member.isfile():
+                # chr/blk/fifo — nothing a code archive needs.
+                raise ArchiveError(
+                    f"refusing archive member of unsupported type: {member.name!r}"
+                )
+
+            stream = tar.extractfile(member)
+            if stream is None:
+                continue
+            _write_stream(stream, member_path, budget)
+    return tops
+
+
+def _extract_zip(src: Path, dest_root: Path, budget: list[int], max_members: int) -> set[str]:
+    try:
+        zf = zipfile.ZipFile(src)
+    except zipfile.BadZipFile as exc:
+        raise ArchiveError(f"not a valid zip archive: {exc}") from exc
+
+    tops: set[str] = set()
+    with zf:
+        infos = zf.infolist()
+        if len(infos) > max_members:
+            raise ArchiveError(
+                f"archive has more than {max_members} members — refusing it"
+            )
+        for info in infos:
+            top = _top_level(info.filename)
+            if top is not None:
+                tops.add(top)
+            member_path = _safe_target(info.filename, dest_root)
+
+            # Zip stores unix mode in the high 16 bits of external_attr, but
+            # only when it was created on a unix host AND the writer bothered
+            # to set the file-type bits — plenty of legitimate zips carry
+            # bare permission bits with no S_IFREG. Treating a missing type
+            # as "unsupported" would refuse ordinary archives, so the type
+            # check only runs when there is a type to check.
+            mode = info.external_attr >> 16
+            if info.create_system == 3 and stat.S_IFMT(mode):
+                if stat.S_ISLNK(mode):
+                    # A zip symlink stores its target as the member's own
+                    # content, so reading it is a read of attacker bytes:
+                    # cap it rather than pulling an arbitrary "target" into
+                    # memory to validate a link we are not going to create.
+                    if info.file_size > 4096:
+                        raise ArchiveError(
+                            f"refusing symlink member {info.filename!r} with an "
+                            f"implausible {info.file_size}-byte target"
+                        )
+                    _check_symlink(
+                        member_path,
+                        zf.read(info).decode("utf-8", errors="replace"),
+                        dest_root,
+                    )
+                    continue
+                if not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
+                    raise ArchiveError(
+                        f"refusing archive member of unsupported type: "
+                        f"{info.filename!r}"
+                    )
+            if info.is_dir():
+                member_path.mkdir(parents=True, exist_ok=True)
+                continue
+            with zf.open(info) as stream:
+                _write_stream(stream, member_path, budget)
+    return tops
+
+
+def extract_archive_safely(
+    src: Path,
+    dest: Path,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    max_members: int = DEFAULT_MAX_MEMBERS,
+) -> Path:
+    """Unpack the archive at ``src`` under ``dest`` and return the path the
+    *content* actually lives at.
+
+    That return value is not always ``dest``: an archive whose members all
+    sit under a single top-level directory — which is exactly what GitHub's
+    ``codeload`` tarballs are, wrapping every repo in ``owner-name-<sha>/``
+    — has that wrapper as its content root. Returning it is what lets the
+    caller present the repo at the path the job's argv was compiled
+    against; returning ``dest`` blindly would leave every such job pointing
+    one directory too high, which is the same "file not found" the missing
+    unpacking caused in the first place.
+
+    The container format is detected from the bytes, never from the file
+    name: an attacker supplies the name, so trusting it would let them pick
+    which parser reads their bytes. Supported: tar with any of gzip/bzip2/xz
+    compression or none, and zip.
+
+    Raises ``ArchiveError`` for every refusal. On refusal ``dest`` is
+    removed entirely.
+    """
+    src = Path(src)
+    dest = Path(dest)
+    dest.mkdir(parents=True, exist_ok=True)
+    dest_root = dest.resolve()
+    budget = [int(max_bytes)]
+
+    try:
+        if zipfile.is_zipfile(src):
+            tops = _extract_zip(src, dest_root, budget, max_members)
+        else:
+            tops = _extract_tar(src, dest_root, budget, max_members)
+    except ArchiveError:
+        shutil.rmtree(dest_root, ignore_errors=True)
+        raise
+    except Exception as exc:
+        # Anything the stdlib raises from hostile bytes (an OSError on an
+        # absurd name, a zlib error mid-stream) becomes the same typed
+        # refusal: the caller must never have to catch the union of every
+        # decompressor's private exception tree.
+        shutil.rmtree(dest_root, ignore_errors=True)
+        raise ArchiveError(f"could not extract archive: {exc}") from exc
+
+    if len(tops) == 1:
+        only = dest_root / next(iter(tops))
+        if only.is_dir():
+            return only
+    return dest_root

flashnode/executor/argv_runner.py

@@ -0,0 +1,114 @@
+"""Tier-2 argv runner: run the user's own command inside a hardened container.
+
+The module runners execute an allowlisted `python -m <module>`. This runner
+executes whatever argv the job carried, which is what makes an arbitrary
+machine a useful compute resource — and is why it is container-only. There
+is no unsandboxed argv path here, by default or otherwise.
+
+The security control is the isolation tier plus the operator's IMAGE
+allowlist (what this volunteer consents to run), not a code allowlist: the
+user's code inside a permitted image is unrestricted.
+
+The task sees exactly one directory: its workdir bound at /work. Inputs are
+pre-staged by the agent at /work/inputs; outputs are written to /work/out.
+With --network none the job cannot fetch anything itself.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+
+from flashnode.executor.hardening import container_name, harden_args
+from flashnode.executor.images import DEFAULT_ALLOWED_IMAGE_PREFIXES, image_is_allowed
+from flashnode.executor.runner import TaskExecutionError
+
+_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+class ArgvDockerRunner:
+    def __init__(
+        self,
+        allowed_images: frozenset[str] = frozenset(),
+        cpus: float = 2.0,
+        memory_gb: float = 2.0,
+        timeout_seconds: float = 3600.0,
+        max_output_bytes: int = 2 * 1024**3,
+        allowed_image_prefixes: frozenset[str] = DEFAULT_ALLOWED_IMAGE_PREFIXES,
+    ):
+        self.allowed_images = allowed_images
+        self.allowed_image_prefixes = allowed_image_prefixes
+        self.cpus = cpus
+        self.memory_gb = memory_gb
+        self.timeout_seconds = timeout_seconds
+        self.max_output_bytes = max_output_bytes
+
+    def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
+        argv = payload.get("argv")
+        if not argv or not isinstance(argv, list) or not all(isinstance(t, str) for t in argv):
+            raise TaskExecutionError("payload 'argv' must be a non-empty list of strings")
+
+        # Checked BEFORE any subprocess call, so a hostile value such as
+        # "--privileged" can never reach docker's flag parser.
+        image = payload.get("image")
+        if not image or not image_is_allowed(
+            image, self.allowed_images, self.allowed_image_prefixes
+        ):
+            raise TaskExecutionError(f"image {image!r} is not allowlisted — refusing to run")
+
+        env_args: list[str] = []
+        for key, value in (payload.get("env") or {}).items():
+            if not _ENV_KEY.match(str(key)):
+                raise TaskExecutionError(f"illegal env key {key!r} — refusing to run")
+            env_args += ["--env", f"{key}={value}"]
+
+        workdir = Path(workdir)
+        outdir = workdir / "out"
+        outdir.mkdir(parents=True, exist_ok=True)
+
+        name = container_name(payload.get("task_id"))
+        command = [
+            "docker", "run", "--rm", "--name", name,
+            *harden_args(workdir, cpus=self.cpus, memory_gb=self.memory_gb),
+            *env_args,
+            image,          # argv follows the image, where docker treats it
+            *argv,          # as the container command: leading '-' is inert
+        ]
+        try:
+            proc = subprocess.run(
+                command, capture_output=True, timeout=self.timeout_seconds, check=False
+            )
+        except subprocess.TimeoutExpired:
+            # subprocess.run's timeout kills the docker CLIENT process, not
+            # the daemon-side container — it keeps running unless we kill it
+            # by name ourselves. Best-effort: the container may already be
+            # gone by the time we ask, and either way the wall-clock error
+            # below is what the caller needs to see, not a kill failure.
+            try:
+                subprocess.run(["docker", "kill", name], capture_output=True, timeout=10, check=False)
+            except Exception:
+                pass
+            raise TaskExecutionError(f"task exceeded {self.timeout_seconds}s wall clock")
+        except OSError as exc:
+            # `docker` missing/removed mid-run raises FileNotFoundError here
+            # (a subclass of OSError). Degrade to a failed task, not a dead
+            # agent — execute_one only catches TaskExecutionError/LeaseLost.
+            raise TaskExecutionError(f"docker is unavailable: {exc}") from exc
+        if proc.returncode != 0:
+            tail = proc.stderr.decode(errors="replace")[-800:]
+            raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
+
+        # metrics.json is load-bearing, not a preference: CommandRecipe sets
+        # commit_key to <prefix>/metrics.json and the coordinator validates
+        # the artifact at that key by sha256. Failing here turns an opaque
+        # commit rejection into a clear task error.
+        if not (outdir / "metrics.json").is_file():
+            raise TaskExecutionError("task produced no metrics.json — nothing to commit")
+
+        total = sum(p.stat().st_size for p in outdir.rglob("*") if p.is_file())
+        if total > self.max_output_bytes:
+            raise TaskExecutionError(
+                f"task output {total} B exceeds the {self.max_output_bytes} B cap"
+            )
+        return outdir

flashnode/executor/client.py

@@ -0,0 +1,201 @@
+"""Outbound-only HTTP client for the FlashRuntime coordinator.
+
+stdlib `urllib` on purpose: this code runs on strangers' machines, and every
+dependency is attack surface (AGENTS.md). All calls are *outbound* to the
+coordinator — the device never listens on anything.
+
+Two heartbeats, two endpoints, never merged:
+- node heartbeat   → /v1alpha1/nodes/{id}/heartbeat   ("this machine is up")
+- attempt heartbeat→ /v1alpha1/attempts/{id}/heartbeat ("this task is alive")
+A 410 on the attempt heartbeat means the lease is dead: the worker must stop
+work on that task (`LeaseLost`).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from flashruntime.protocol.v1alpha1 import Lease, NodeHeartbeat, NodeRegistration
+
+
+class LeaseLost(Exception):
+    """The coordinator refused the attempt heartbeat — stop working."""
+
+
+class CoordinatorClient:
+    def __init__(
+        self,
+        base_url: str,
+        timeout: float = 15.0,
+        join_code: str | None = None,
+        token: str | None = None,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.join_code = join_code  # sent on register when the pool requires one
+        self._token = token  # bearer credential for every authenticated call
+
+    # -- auth -----------------------------------------------------------------
+
+    def _headers(self) -> dict[str, str]:
+        """Auth header merged into every request this class makes.
+
+        Empty when no token is configured — correct for the self-hosted open
+        profile, where absence of a credential simply means no header.
+        """
+        if self._token:
+            return {"Authorization": f"Bearer {self._token}"}
+        return {}
+
+    # -- transport (patchable in tests) -------------------------------------
+
+    def _request(
+        self,
+        method: str,
+        path: str,
+        body: bytes | None = None,
+        content_type: str = "application/json",
+        headers: dict[str, str] | None = None,
+    ) -> tuple[int, bytes]:
+        all_headers = self._headers()
+        all_headers.update(headers or {})
+        if body is not None:
+            all_headers["Content-Type"] = content_type
+        req = urllib.request.Request(
+            f"{self.base_url}{path}",
+            data=body,
+            method=method,
+            headers=all_headers,
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                return resp.status, resp.read()
+        except urllib.error.HTTPError as exc:
+            return exc.code, exc.read()
+
+    def _json(self, method: str, path: str, payload: dict | str | None = None) -> tuple[int, dict]:
+        body = None
+        if payload is not None:
+            body = (payload if isinstance(payload, str) else json.dumps(payload)).encode()
+        status, raw = self._request(method, path, body)
+        return status, (json.loads(raw) if raw else {})
+
+    # -- node lifecycle -----------------------------------------------------
+
+    def register(self, registration: NodeRegistration) -> None:
+        headers = {"X-FlashML-Join-Code": self.join_code} if self.join_code else None
+        status, raw = self._request(
+            "POST",
+            "/v1alpha1/nodes/register",
+            registration.model_dump_json().encode(),
+            headers=headers,
+        )
+        if status != 200:
+            raise RuntimeError(f"registration failed ({status}): {raw.decode(errors='replace')}")
+
+    def node_heartbeat(self, node_id: str) -> bool:
+        status, _ = self._json(
+            "POST",
+            f"/v1alpha1/nodes/{node_id}/heartbeat",
+            NodeHeartbeat(node_id=node_id).model_dump_json(),
+        )
+        return status == 200
+
+    # -- lease lifecycle ----------------------------------------------------
+
+    def claim(self, node_id: str) -> Lease | None:
+        status, body = self._json("POST", "/v1alpha1/leases/claim", {"node_id": node_id})
+        if status == 204:
+            return None
+        if status != 200:
+            raise RuntimeError(f"claim failed ({status}): {body}")
+        return Lease.model_validate(body)
+
+    def attempt_heartbeat(self, lease_id: str) -> None:
+        status, body = self._json("POST", f"/v1alpha1/attempts/{lease_id}/heartbeat")
+        if status == 410:
+            raise LeaseLost(str(body))
+        if status != 200:
+            raise RuntimeError(f"attempt heartbeat failed ({status}): {body}")
+
+    def complete(self, lease_id: str, output_sha256: str) -> bool:
+        status, body = self._json(
+            "POST",
+            f"/v1alpha1/attempts/{lease_id}/complete",
+            {"output_sha256": output_sha256},
+        )
+        if status != 200:
+            raise RuntimeError(f"complete failed ({status}): {body}")
+        return bool(body.get("accepted"))
+
+    def fail(self, lease_id: str, reason: str) -> None:
+        self._json("POST", f"/v1alpha1/attempts/{lease_id}/fail", {"reason": reason})
+
+    # -- checkpoints (the agent is the courier; tasks stay offline) ----------
+
+    def checkpoint_latest(self, job_id: str, task_id: str) -> dict | None:
+        status, body = self._json(
+            "GET", f"/v1alpha1/jobs/{job_id}/tasks/{task_id}/checkpoints/latest"
+        )
+        if status == 404:
+            return None
+        if status != 200:
+            raise RuntimeError(f"checkpoint latest failed ({status}): {body}")
+        return body
+
+    def checkpoint_register_part(
+        self, job_id: str, task_id: str, attempt_id: str, step: int, part: dict
+    ) -> None:
+        status, body = self._json(
+            "POST",
+            f"/v1alpha1/jobs/{job_id}/tasks/{task_id}/checkpoints/parts",
+            {"attempt_id": attempt_id, "step": step, "part": part},
+        )
+        if status != 200:
+            raise RuntimeError(f"checkpoint part registration failed ({status}): {body}")
+
+    def checkpoint_commit(
+        self,
+        job_id: str,
+        task_id: str,
+        attempt_id: str,
+        step: int,
+        parts: list[dict],
+        storage_prefix: str,
+    ) -> dict:
+        status, body = self._json(
+            "POST",
+            f"/v1alpha1/jobs/{job_id}/tasks/{task_id}/checkpoints/commit",
+            {
+                "attempt_id": attempt_id,
+                "step": step,
+                "expected_parts": parts,
+                "storage_prefix": storage_prefix,
+            },
+        )
+        if status != 200:
+            raise RuntimeError(f"checkpoint commit refused ({status}): {body}")
+        return body
+
+    # -- artifacts (coordinator-hosted shared data) --------------------------
+
+    def download_artifact(self, key: str, destination: Path) -> Path:
+        status, raw = self._request("GET", f"/v1alpha1/artifacts/{key}")
+        if status != 200:
+            raise RuntimeError(f"artifact download failed ({status}): {key}")
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_bytes(raw)
+        return destination
+
+    def upload_artifact(self, local_path: Path, key: str) -> str:
+        data = Path(local_path).read_bytes()
+        status, _ = self._request(
+            "PUT", f"/v1alpha1/artifacts/{key}", data, content_type="application/octet-stream"
+        )
+        if status != 200:
+            raise RuntimeError(f"artifact upload failed ({status}): {key}")
+        return hashlib.sha256(data).hexdigest()