nkx-runtime 2.2.0__py3-none-any.whl

nkx_runtime/_core/crypto.py

@@ -0,0 +1,144 @@
+"""Envelope encryption primitives.
+
+Two-stage KEK derivation (v2 manifests):
+
+  passphrase ──scrypt──► base_kek
+                                    ──HKDF(info=context_bytes)──► final_kek
+  build_id + tenant_id + runtime_fp ────────────────────────────┘
+
+scrypt provides brute-force resistance on the passphrase.
+HKDF binds the key to a specific build/tenant/runtime context so a stolen
+artifact cannot be decrypted outside its intended environment.
+
+v1 manifests use scrypt only (no context binding).
+"""
+import base64
+import hashlib
+import os
+
+from cryptography.hazmat.primitives import hashes as _hashes
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
+
+_NONCE = 12
+_KEYLEN = 32
+
+# Suffix appended to a protected file's plaintext rel to get its ciphertext
+# blob on disk: pricing.py -> pricing.py.nkxc, and the mock stub
+# pricing.py.mock.py -> pricing.py.mock.py.nkxc. Appended, never substituted:
+# protect.py's _mock_plaintext_rel() and devmode's mock helpers all recover
+# the plaintext rel by stripping exactly this suffix, and entry["mock"] is
+# built assuming it starts with `rel`.
+#
+# It lives here, not in protect.py, because loader/materialize/rehook/devmode
+# all need it and `protect` is absent from the sdk-python sync allowlist
+# (Makefile RUNTIME_PY_MODULES) — importing it from there would break the
+# synced runtime slice. `crypto` is in all three allowlists and every one of
+# those modules already imports from it.
+#
+# NOT ".nkxd": that extension belongs to the NKXDGARD data/document format
+# (Projects 2/3, nakodax-data + nakodax-box), which is a self-describing
+# container — MAGIC, header-length prefix, header JSON, chunked ciphertext.
+# The blob written here is headerless (nonce || AES-GCM(ct||tag)) and is
+# meaningless without .nkx/manifest.json. Same suffix, two incompatible
+# layouts, across three repos.
+BLOB_EXT = ".nkxc"
+
+
+def b64(data: bytes) -> str:
+    return base64.b64encode(data).decode()
+
+
+def unb64(text: str) -> bytes:
+    return base64.b64decode(text)
+
+
+def sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def zeroize(buf: bytearray | None) -> None:
+    """Overwrite a mutable key buffer in place. A no-op on anything
+    immutable (bytes, str) — Python can't scrub those, which is exactly why
+    callers holding a decrypted key should keep it in a bytearray for as
+    long as they hold it, and call this the moment they're done."""
+    if not buf:
+        return
+    for i in range(len(buf)):
+        buf[i] = 0
+
+
+def new_kdf() -> dict:
+    # n=2**17 per current OWASP guidance for a passphrase-based KDF used as
+    # a brute-force barrier (audit finding B18) — 8x the prior 2**14.
+    return {"salt": b64(os.urandom(16)), "n": 2 ** 17, "r": 8, "p": 1}
+
+
+def derive_kek(passphrase: str, kdf: dict) -> bytes:
+    scrypt = Scrypt(
+        salt=unb64(kdf["salt"]),
+        length=_KEYLEN,
+        n=kdf["n"],
+        r=kdf["r"],
+        p=kdf["p"],
+    )
+    return scrypt.derive(passphrase.encode())
+
+
+def encrypt_blob(plaintext: bytes, dek: bytes, associated_data: bytes | None = None) -> bytes:
+    aes = AESGCM(dek)
+    nonce = os.urandom(_NONCE)
+    return nonce + aes.encrypt(nonce, plaintext, associated_data)
+
+
+def decrypt_blob(blob: bytes, dek: bytes, associated_data: bytes | None = None) -> bytes:
+    aes = AESGCM(dek)
+    nonce, ct = blob[:_NONCE], blob[_NONCE:]
+    return aes.decrypt(nonce, ct, associated_data)
+
+
+def wrap_dek(dek: bytes, kek: bytes, associated_data: bytes | None = None) -> bytes:
+    return encrypt_blob(dek, kek, associated_data)
+
+
+def unwrap_dek(wrapped: bytes, kek: bytes, associated_data: bytes | None = None) -> bytes:
+    return decrypt_blob(wrapped, kek, associated_data)
+
+
+def slot_aad(rel: str, kind: str) -> bytes:
+    """Associated data binding a piece of ciphertext to the manifest slot it
+    belongs to. AES-GCM's auth tag normally only proves "this ciphertext
+    decrypts validly under this key" — not which file it's *for* — so an
+    attacker with manifest write-access could copy one entry's
+    {ciphertext, wrapped_dek} pair into a different entry's slot and have
+    both decrypt "successfully", silently substituting the wrong content
+    under the wrong name. Binding each ciphertext to its own rel + kind
+    ("content"/"dek"/"mock-content"/"mock-dek") makes that substitution fail
+    to decrypt instead (audit finding B16).
+
+    kind distinguishes the four things that get encrypted per file so a
+    real-content blob can't be swapped for its own wrapped_dek's slot (or a
+    mock's), even though they'd otherwise share the same rel.
+    """
+    return f"{kind}:{rel}".encode()
+
+
+def bind_context_kek(base_kek: bytes, context_info: bytes) -> bytes:
+    """Bind a base KEK to a runtime context using HKDF.
+
+    context_info is the HKDF 'info' parameter — public and non-secret, but
+    cryptographically binding. A different environment produces different
+    context_info bytes, which produces a completely different final KEK, so
+    DEK unwrapping fails if the runtime context doesn't match exactly.
+    """
+    return HKDF(
+        algorithm=_hashes.SHA256(),
+        length=_KEYLEN,
+        salt=None,   # base_kek is already high-entropy; salt adds nothing here
+        info=context_info,
+    ).derive(base_kek)
+
+
+def new_dek() -> bytes:
+    return os.urandom(_KEYLEN)

nkx_runtime/_core/devmode.py

@@ -0,0 +1,223 @@
+"""Dev-mode import hook: serve mock stubs instead of `.nkxc` blobs.
+
+DevModeFinder      — sys.meta_path hook that loads stubs instead of .nkxc blobs
+DevModeLoader      — companion loader for DevModeFinder
+
+Activated by NKX_DEVMODE=1 when there's no backend to derive a real KEK
+from (NKX_KEK_URL absent) — works for v1 and v2/context-bound manifests
+alike, since plaintext stubs never touch the KEK at all.
+
+Stub *generation* lives in `mockgen.py`. This module must not import it: this
+one ships to the customer runtime wheel and that one does not, which is the
+entire point of the split.
+"""
+from __future__ import annotations
+
+import importlib.abc
+import importlib.util
+from pathlib import Path
+
+# ── dev-mode import hook ───────────────────────────────────────────────────────
+
+_ENCRYPTED_MOCK_ERROR = (
+    "nkx devmode: this mock is encrypted — a developer license and reachable "
+    "NKX_KEK_URL are required"
+)
+
+
+def _mock_enc_rel(entry: dict) -> str:
+    """Relative path of the encrypted mock blob for one manifest entry.
+
+    entry["mock"] is exactly the path _commit_stub_write() (protect.py)
+    actually wrote — the plaintext stub path during the dual-write window,
+    or the .nkxc path itself after the encrypted-only cutover. Both cases
+    are handled without a second manifest field: if it doesn't already end
+    in ".nkxc", the encrypted sibling is that same path with ".nkxc"
+    appended (exactly how the writer derives it), otherwise it already
+    *is* the encrypted path.
+    """
+    from .crypto import BLOB_EXT   # lazy, like every other import here
+    mock_rel = entry["mock"]
+    return mock_rel if mock_rel.endswith(BLOB_EXT) else mock_rel + BLOB_EXT
+
+
+def _mock_plaintext_rel(entry: dict) -> str:
+    """The plaintext-edit-path counterpart to _mock_enc_rel() above — same
+    rel + ".mock" + ext convention, just without the ".nkxc" suffix. Used to
+    find a developer's hand-edited plaintext during the MOCK_UNLOCKED
+    override window (see _DevModeLoader.exec_module)."""
+    from .crypto import BLOB_EXT   # lazy, like every other import here
+    enc_rel = _mock_enc_rel(entry)
+    return enc_rel[: -len(BLOB_EXT)] if enc_rel.endswith(BLOB_EXT) else enc_rel
+
+
+def _mock_unlocked_rels(root: Path) -> list[str]:
+    """Read protect.py's MOCK_UNLOCKED sentinel fresh on every call (no
+    caching) — self-service unlock-mock/reseal-mock (Phase 8) run in a
+    separate CLI process from whatever long-running process is importing
+    protected modules, so a cached read here could miss an edit made after
+    this process started.
+
+    Uses a private copy of the MOCK_UNLOCKED path rather than importing it
+    from protect.py — protect.py (encrypt/reseal/rekey) is off the
+    customer-facing packaging allowlist (see Makefile's RUNTIME_PY_MODULES),
+    but this function runs on every devmode-served import of an
+    encrypted-mock file, which does ship there. Importing protect.py here
+    made every such import crash with ModuleNotFoundError in the packaged
+    nkx-runtime slice (and, historically, the now-retired dev-cli slice) —
+    same class of gap materialize.py's _load_manifest() already sidesteps
+    for the same reason."""
+    import json
+    MOCK_UNLOCKED = ".nkx/MOCK_UNLOCKED"
+    sentinel = root / MOCK_UNLOCKED
+    return json.loads(sentinel.read_text()) if sentinel.exists() else []
+
+
+class _DevModeLoader(importlib.abc.Loader):
+    def __init__(self, finder: "DevModeFinder", fullname: str):
+        self.finder = finder
+        self.fullname = fullname
+
+    def create_module(self, spec):
+        return None
+
+    def exec_module(self, module):
+        rel, entry = self.finder.modules[self.fullname]
+        # MOCK_UNLOCKED override (Phase 8): while a file is listed there, a
+        # developer's live hand-edit on disk must win over decrypt-preference
+        # below — otherwise imports would keep serving stale ciphertext (from
+        # before the edit) or a freshly-derived placeholder, never what's
+        # actually sitting in the plaintext file they just changed. Per-file
+        # only: every other file in the same manifest stays decrypt-preferred.
+        if "wrapped_mock_dek" in entry and rel in _mock_unlocked_rels(self.finder.root):
+            plaintext_path = self.finder.root / _mock_plaintext_rel(entry)
+            if not plaintext_path.exists():
+                raise ImportError(
+                    f"nkx devmode: {rel!r} is listed in MOCK_UNLOCKED but its "
+                    f"plaintext is missing — run 'nkx-dev unlock-mock' again"
+                )
+            source = plaintext_path.read_bytes()
+            label = str(plaintext_path)
+        # Decrypt-preferred: a file with wrapped_mock_dek always attempts
+        # the encrypted read — the plaintext copy sitting next to it during
+        # the dual-write window is a fallback for *other, not-yet-upgraded*
+        # readers, not a same-process fallback for this one. Only entries
+        # with no wrapped_mock_dek at all (true legacy/plaintext-only
+        # files) read entry["mock"]'s plaintext path directly.
+        elif "wrapped_mock_dek" in entry:
+            source = self.finder._decrypt_mock(rel, entry)
+            label = str(self.finder.root / _mock_enc_rel(entry))
+        else:
+            mock_path = self.finder.root / entry.get("mock", rel + ".mock.py")
+            if not mock_path.exists():
+                raise ImportError(
+                    f"nkx devmode: no stub found for {self.fullname!r} — "
+                    f"run 'nkx mock <repo>' to generate stubs"
+                )
+            source = mock_path.read_bytes()
+            label = str(mock_path)
+        code = compile(source, label, "exec")
+        exec(code, module.__dict__)
+
+
+class DevModeFinder(importlib.abc.MetaPathFinder):
+    """sys.meta_path hook that serves .mock.py stubs instead of .nkxc blobs.
+
+    Encrypted mock content (Python hook-tier entries with wrapped_mock_dek)
+    is decrypted lazily via a mock-content KEK derived from build_id/kdf/
+    key_version + license_jwt — a manifest that's entirely legacy/plaintext
+    triggers zero network calls; only the first import of a file that
+    actually has wrapped_mock_dek derives (and reuses, via a single shared
+    RemoteKekManager, since the mock KEK is the same for every file in the
+    manifest — it never depends on which file is being imported).
+    """
+
+    def __init__(
+        self, root: str, manifest: dict, *,
+        build_id: str = "", kdf: dict | None = None,
+        key_version: int | None = None, license_jwt: str = "",
+        kek_manager=None,
+    ):
+        """`kek_manager`, when given, is a manager the caller already derived.
+
+        Spec 16 phase 3: _install_core asks the backend once, and the *answer*
+        is what selects this finder over ProtectedFinder — so the mock KEK is
+        already in hand by the time this is constructed, and deriving a second
+        one here would be a second round trip that could disagree with the
+        first. The lazy path below still exists for the case that has no
+        release row to route through /derive at all (a repo protected but
+        never published), and for a manifest that is entirely legacy plaintext
+        and makes no network call whatsoever.
+        """
+        self.root = Path(root)
+        self.modules = {
+            e["module"]: (rel, e)
+            for rel, e in manifest["files"].items()
+            if e.get("load_strategy", "hook") == "hook" and "module" in e
+        }
+        # Kept for binding_name(): the mock ciphertext is bound to the same
+        # repo-relative name as the real content, which in a per-root view is
+        # longer than the key this finder iterates. See manifest.binding_name.
+        self._manifest = manifest
+        self._build_id = build_id
+        self._kdf = kdf
+        self._key_version = key_version
+        self._license_jwt = license_jwt
+        self._mock_kek_manager = kek_manager  # lazy when None — see class docstring
+
+    def _decrypt_mock(self, rel: str, entry: dict) -> bytes:
+        from .crypto import decrypt_blob, slot_aad, unb64, unwrap_dek, zeroize
+        from .manifest import binding_name
+
+        name = binding_name(self._manifest, rel)
+
+        # A fresh, mutable copy from the manager (audit finding B12) —
+        # scrub it here rather than leave it sitting in memory until GC
+        # (which wouldn't scrub it anyway; bytes can't be).
+        mock_kek = self._get_mock_kek()
+        try:
+            enc_path = self.root / _mock_enc_rel(entry)
+            try:
+                blob = enc_path.read_bytes()
+            except FileNotFoundError:
+                # Must still be the distinct encrypted-mock ImportError, not a
+                # raw FileNotFoundError — a bare OSError subclass leaking out of
+                # exec_module() would violate the "distinct ImportError, never
+                # a generic/unrelated exception" contract this whole mechanism
+                # exists to provide (see _ENCRYPTED_MOCK_ERROR's module docstring).
+                raise ImportError(f"{_ENCRYPTED_MOCK_ERROR} (encrypted mock file missing on disk)") from None
+            try:
+                dek = unwrap_dek(unb64(entry["wrapped_mock_dek"]), mock_kek, slot_aad(name, "mock-dek"))
+                return decrypt_blob(blob, dek, slot_aad(name, "mock-content"))
+            except Exception:
+                raise ImportError(f"{_ENCRYPTED_MOCK_ERROR} (stale or revoked mock KEK)") from None
+        finally:
+            zeroize(mock_kek)
+
+    def _get_mock_kek(self) -> bytearray:
+        if self._mock_kek_manager is None:
+            if not self._license_jwt or self._kdf is None or self._key_version is None:
+                raise ImportError(_ENCRYPTED_MOCK_ERROR)
+            import functools
+            from . import remotekek
+            derive_fn = functools.partial(
+                remotekek.derive_mock_kek,
+                self._license_jwt, self._build_id, self._kdf, self._key_version,
+            )
+            try:
+                manager = remotekek.RemoteKekManager(derive_fn)
+            except Exception:
+                raise ImportError(_ENCRYPTED_MOCK_ERROR) from None
+            manager.start_refresh_thread()
+            self._mock_kek_manager = manager
+        try:
+            return self._mock_kek_manager.current_kek()
+        except Exception:
+            raise ImportError(_ENCRYPTED_MOCK_ERROR) from None
+
+    def find_spec(self, fullname, path, target=None):
+        if fullname in self.modules:
+            return importlib.util.spec_from_loader(
+                fullname, _DevModeLoader(self, fullname)
+            )
+        return None

nkx_runtime/_core/fingerprint.py

@@ -0,0 +1,158 @@
+"""Runtime instance fingerprinting for per-instance license anomaly detection.
+
+This is deliberately separate from context.py's HKDF/KEK-binding collectors:
+those bind artifact decryption to an environment (a security boundary — wrong
+environment, no decrypt). This module produces an anomaly *signal* only — the
+backend compares it against the last fingerprint seen for the same registered
+nonce and flags `shared_nonce_suspected` when they disagree, for a human to
+review. It is not, and does not need
+to be, tamper-proof: the actual access-control boundary remains the signed
+license JWT and its revoked/expired status, checked independently in
+context.py. Worst case if a fingerprint is spoofed or stripped is a missed
+flag, not a license bypass.
+
+Fingerprint = sha256(persisted_nonce + machine_signal).
+
+The persisted nonce alone proves nothing about machine identity — copying its
+file to a different machine carries the nonce along. What makes that copy
+visible as a *new* instance is the machine_signal half: it's gathered fresh
+from the running machine each time, so a copied nonce on a different host
+still produces a different combined fingerprint. The nonce's only job is
+continuity — recognizing "this is the same install restarting" across process
+restarts on the *same* machine, where machine_signal alone can be too noisy
+(e.g. a fresh container ID on every restart even on the same host).
+
+`.nkx/instance_nonce` used to hold a nonce generated locally by this
+module. It now holds the backend-issued registered-instance credential
+instead (instanceauth.ensure_instance_nonce(), the security boundary
+POST /internal/keys/derive checks) — this module just reads whatever is
+there. The two were unified deliberately: one file, one story, instead of a
+second nonce with a second lifecycle to reason about.
+"""
+from __future__ import annotations
+import hashlib
+import os
+from pathlib import Path
+
+from .context import _machine_id, credentials_dir
+
+# Core's phone-home cadence. backend/src/lib/runtimeInstances.ts documents the
+# matching staleness margin — the two are independent constants across the
+# Python/TS boundary and must be kept in sync by hand if tuned.
+HEARTBEAT_INTERVAL_SECONDS = 600  # 10 min
+
+
+def _persisted_nonce(repo_root: str | os.PathLike[str] | None = None) -> str:
+    """Read the instance nonce persisted by instanceauth.py.
+
+    Returns "" if it isn't there yet (e.g. registration hasn't run, or this
+    deployment predates the registered-instance exchange) — callers treat a
+    missing/empty fingerprint input as non-fatal (see
+    context._instance_fingerprint_header()), so this degrades to a slightly
+    noisier detection signal rather than raising.
+    """
+    nonce_file = credentials_dir(repo_root) / "instance_nonce"
+    if nonce_file.exists():
+        return nonce_file.read_text().strip()
+    return ""
+
+
+def _pod_uid() -> str:
+    """Kubernetes Downward API — requires the pod to expose POD_UID via
+    `fieldRef: metadata.uid` in its env. See core/README.md for the manifest
+    snippet operators need to add.
+    """
+    return os.environ.get("POD_UID", "")
+
+
+def _container_id() -> str:
+    """Docker container ID, parsed from cgroup info. Ephemeral — changes on
+    every container recreate, which is the correct behavior (a recreated
+    container is a new instance for concurrency-limit purposes).
+    """
+    for cgroup_path in ("/proc/self/cgroup", "/proc/1/cpuset"):
+        try:
+            text = Path(cgroup_path).read_text()
+        except Exception:
+            continue
+        for line in text.splitlines():
+            # Docker cgroup lines look like ".../docker/<64-char-id>"
+            parts = line.strip().split("/")
+            if parts and len(parts[-1]) == 64 and all(c in "0123456789abcdef" for c in parts[-1]):
+                return parts[-1]
+    return ""
+
+
+def _fallback_machine_id_file(repo_root: str | os.PathLike[str] | None = None) -> Path:
+    return credentials_dir(repo_root) / "machine_signal"
+
+
+def _nic_mac(repo_root: str | os.PathLike[str] | None = None) -> str:
+    """Last-resort machine signal, when no pod UID / container id / OS
+    machine-id is available.
+
+    uuid.getnode() is NOT stable here, despite this being the "last resort
+    but still stable" tier: per Python's own documented behavior, when no
+    real hardware MAC can be found — the common case for exactly the
+    containers/sandboxes that reach this fallback — it synthesizes a
+    random 48-bit value fresh on every call. Every restart then looks like
+    a brand-new machine, making concurrency-anomaly detection permanently
+    noisy for that deployment (audit finding B15). Persist a random id to
+    local disk the first time instead, so later restarts on the same
+    host/container read the same value back — mirrors instanceauth.py's
+    instance_nonce persistence, but best-effort only: this is a detection
+    signal, not a security boundary (see module docstring), so a
+    filesystem hiccup here just degrades to the old per-process-random
+    behavior for that one run rather than failing anything.
+    """
+    import uuid
+
+    path = _fallback_machine_id_file(repo_root)
+    try:
+        if path.exists():
+            existing = path.read_text().strip()
+            if existing:
+                return existing
+        value = uuid.uuid4().hex
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(value)
+        return value
+    except Exception:
+        return str(uuid.getnode())
+
+
+def _machine_signal(repo_root: str | os.PathLike[str] | None = None) -> str:
+    """Best-effort machine identity, in priority order: Kubernetes pod UID,
+    Docker container ID, OS machine-id (reused from context.py), NIC MAC as
+    last resort. Each is empty-string on failure; the first non-empty wins.
+    """
+    for collector in (_pod_uid, _container_id, _machine_id):
+        try:
+            value = collector()
+        except Exception:
+            value = ""
+        if value:
+            return value
+    # _nic_mac is called outside the loop, not wrapped in a lambda inside it,
+    # because it is the only collector that takes an argument — and the loop
+    # swallows every exception, so an arity mistake there would read as "this
+    # machine has no NIC" and silently degrade to "unknown" forever.
+    try:
+        return _nic_mac(repo_root) or "unknown"
+    except Exception:
+        return "unknown"
+
+
+def get_fingerprint(repo_root: str | os.PathLike[str] | None = None) -> str:
+    """Combined instance fingerprint sent on every validate call.
+
+    `repo_root` reaches the same credentials directory instanceauth.py wrote
+    the nonce to. Optional because this is a detection signal that degrades
+    rather than failing (see the module docstring), but a caller that has a
+    root should always pass it: reading the nonce from somewhere other than
+    where registration put it hashes an empty string and quietly makes every
+    restart look like a different machine.
+    """
+    nonce = _persisted_nonce(repo_root)
+    signal = _machine_signal(repo_root)
+    return hashlib.sha256(f"{nonce}:{signal}".encode()).hexdigest()