mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/hosted_crawler_container_smoke.py

@@ -0,0 +1,351 @@
+"""Linux image smoke for the real crawler seccomp, normalizer exec boundary, and decoder.
+
+Two things are checked here that cannot be checked anywhere else, because both are properties
+of the confined child rather than of the code that runs in it.
+
+The first is the crawler's own boundary: normalization happens in a child that can reach no
+network, and a deliberately malicious probe confirms every escape it might try returns EPERM.
+
+The second is the weather decoder. The filter installed in that child denies thread
+creation outright -- ``clone``, ``fork``, ``vfork`` and ``clone3`` are all in
+``_PROCESS_ESCAPE_SYSCALLS_X86_64`` -- so a compiled decoder that spawned a worker pool would not
+run slowly there, it would die. That makes this the one environment where "single-threaded" is
+enforced rather than merely expected, which is why the claim is tested here and not in a unit
+test that would pass on a decoder with a thread pool in it.
+
+The thread count is sampled **during** the decode, from the parent, while the child decodes in a
+loop. A library that spawned a pool for the duration of a call and joined it afterwards would
+pass a before-and-after check and die under the filter, so a before-and-after check would be the
+wrong check to write down as green.
+
+The decoded levels are compared against digests that were computed on both target platforms and
+found identical, so this doubles as the cross-platform determinism check for the decoder: a wrong
+answer here fails the image, exactly as a wrong warm-up answer halts a refresh.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import time
+from types import SimpleNamespace
+
+from mostlyright.data_harness.canonical import parse_canonical_json, sha256_bytes
+from mostlyright.data_harness.hosted_crawler import encode_normalization_input
+from mostlyright.data_harness.hosted_crawler_job import (
+    _disable_process_dumpability,
+    _install_networkless_seccomp,
+    _run_crawler_child,
+)
+from mostlyright.data_harness.hosted_crawler_protocol import (
+    OPENLIGADB_EGRESS_POLICY_ATTESTATION,
+    HostedCrawlerRequest,
+    parse_crawler_result,
+)
+from mostlyright.data_harness.readers.grib2 import MESSAGES_ROOT
+
+_RAW_FIXTURE = b"""[{
+"matchID":1,
+"matchDateTimeUTC":"2025-08-22T18:30:00Z",
+"lastUpdateDateTime":"2025-08-23T01:00:54.147",
+"leagueShortcut":"bl1",
+"leagueSeason":2025,
+"group":{"groupOrderID":1},
+"team1":{"teamId":40,"teamName":"FC Bayern Muenchen"},
+"team2":{"teamId":1635,"teamName":"RB Leipzig"},
+"matchIsFinished":true,
+"matchResults":[{
+"resultTypeID":2,
+"resultOrderID":1,
+"resultID":1,
+"pointsTeam1":6,
+"pointsTeam2":0
+}]
+}]"""
+
+_MALICIOUS_PROBE = r"""
+import ctypes
+import errno
+import os
+
+links = []
+for name in os.listdir("/proc/self/fd"):
+    try:
+        links.append(os.readlink("/proc/self/fd/" + name))
+    except OSError:
+        pass
+if any(link.startswith("socket:[") for link in links):
+    raise SystemExit(20)
+status = open("/proc/self/status", encoding="ascii").read()
+if "NoNewPrivs:\t1" not in status or "Seccomp:\t2" not in status:
+    raise SystemExit(21)
+try:
+    open(f"/proc/{os.getppid()}/environ", "rb").read()
+except PermissionError:
+    pass
+else:
+    raise SystemExit(22)
+libc = ctypes.CDLL(None, use_errno=True)
+for number, arguments in (
+    (41, (2, 1, 0)),
+    (42, (-1, 0, 0)),
+    (0x40000000 | 41, (2, 1, 0)),
+    (0x40000000 | 42, (-1, 0, 0)),
+    (56, (0, 0, 0, 0, 0)),
+    (101, (0, 0, 0, 0)),
+    (310, (os.getppid(), 0, 0, 0, 0, 0)),
+    (434, (os.getppid(), 0)),
+):
+    ctypes.set_errno(0)
+    outcome = libc.syscall(number, *arguments)
+    if outcome != -1 or ctypes.get_errno() != errno.EPERM:
+        raise SystemExit(23)
+print("networkless-seccomp-ok")
+"""
+
+
+# Every admitted template pair, with the point count and the digest of the levels the decoder
+# returns for it. The digests were computed on macOS arm64 and on Linux x86_64 and are identical:
+# the levels are exact integers recovered from a deterministic bit unpacking, so they have no
+# licence to differ, and pinning them here is what turns that from an expectation into a check.
+_DECODER_SAMPLES = (
+    (
+        "gfs_2m_temperature/gfs-2m-temperature.grib2",
+        0,
+        3,
+        65160,
+        "e6597dbf68fcf31ad983491122484bf6391d5ace74daf56cb5fa76d1c9b217db",
+    ),
+    (
+        "hrrr_2m_temperature/hrrr-2m-temperature.grib2",
+        30,
+        3,
+        1905141,
+        "2e2a2468d687c082c4ec00341e2dd0822d77b1be08246272279ad6ea7e8a7307",
+    ),
+    (
+        "hrrr_categorical_rain/hrrr-categorical-rain.grib2",
+        30,
+        0,
+        1905141,
+        "b0379dce013d12446af5c48bf41b14a3f7571c8c5e20cecacd48f2026881d61c",
+    ),
+)
+
+# Environment variables that a library with a thread pool would expect to be told to stand down
+# with. None of them is set here, and the probe refuses to run if one is. A decoder that is
+# single-threaded only when configured to be is single-threaded nowhere it is not configured, and
+# recording "single-threaded" on the strength of a variable somebody set would be recording the
+# variable rather than the property.
+_FORCING_VARIABLES = (
+    "RAYON_NUM_THREADS",
+    "OMP_NUM_THREADS",
+    "MKL_NUM_THREADS",
+    "OPENBLAS_NUM_THREADS",
+    "NUMEXPR_NUM_THREADS",
+    "TOKIO_WORKER_THREADS",
+    "MOSTLYRIGHT_GRIB_THREADS",
+)
+
+_DECODER_SECONDS = 1.5
+_DECODER_SAMPLE_INTERVAL = 0.005
+_DECODER_LEAST_SAMPLES = 20
+
+
+def _decoder_probe() -> str:
+    """The child's whole program, with the sample paths resolved by the parent.
+
+    The paths are resolved here rather than imported there on purpose, and the reason is a
+    measurement. Importing the decoder alone leaves the process on one thread. Importing this
+    harness package takes it to nine, because the analytics engines it loads start pools of their
+    own at import time -- nothing to do with the decoder, and pre-existing, but enough to drown
+    the signal this probe exists to read. So the child imports the decoder and the standard
+    library, and nothing else, and the thread count it reports is a statement about the decoder.
+    """
+
+    resolved = tuple(
+        (str((MESSAGES_ROOT / relative).resolve()), grid, packing, count, digest)
+        for relative, grid, packing, count, digest in _DECODER_SAMPLES
+    )
+    return f"""
+import hashlib
+import os
+import sys
+import time
+
+for name in {_FORCING_VARIABLES!r}:
+    if name in os.environ:
+        raise SystemExit(40)
+
+status = open("/proc/self/status", encoding="ascii").read()
+if "NoNewPrivs:\\t1" not in status or "Seccomp:\\t2" not in status:
+    raise SystemExit(41)
+
+
+def threads():
+    for line in open("/proc/self/status", encoding="ascii"):
+        if line.startswith("Threads:"):
+            return int(line.split()[1])
+    raise SystemExit(42)
+
+
+if threads() != 1:
+    raise SystemExit(43)
+
+import mostlyright_grib
+
+if threads() != 1:
+    raise SystemExit(44)
+
+samples = []
+for path, grid, packing, count, digest in {resolved!r}:
+    samples.append((open(path, "rb").read(), grid, packing, count, digest))
+
+if threads() != 1:
+    raise SystemExit(48)
+
+sys.stdout.write("decoder-ready\\n")
+sys.stdout.flush()
+
+rounds = 0
+deadline = time.monotonic() + {_DECODER_SECONDS!r}
+while True:
+    for content, grid, packing, count, digest in samples:
+        unpacked = mostlyright_grib.unpack(
+            content, grid_template=grid, packing_template=packing
+        )
+        if unpacked.point_count != count:
+            raise SystemExit(45)
+        if hashlib.sha256(memoryview(unpacked.levels).cast("B")).hexdigest() != digest:
+            raise SystemExit(46)
+    rounds += 1
+    if time.monotonic() >= deadline:
+        break
+
+if threads() != 1:
+    raise SystemExit(47)
+
+sys.stdout.write("decoder-ok %d %s\\n" % (rounds, mostlyright_grib.decoder_version()))
+"""
+
+
+def _thread_count(pid: int) -> int | None:
+    """How many threads the child has right now, read from outside it.
+
+    From outside, because from inside there is no way to look: the decode is one blocking call
+    into compiled code, a Python signal handler would not run until it returned, and a watcher
+    thread is exactly the thing the filter denies. The parent is not confined, so it can watch."""
+
+    try:
+        with open(f"/proc/{pid}/status", encoding="ascii") as handle:
+            for line in handle:
+                if line.startswith("Threads:"):
+                    return int(line.split()[1])
+    except (FileNotFoundError, ProcessLookupError):
+        return None
+    return None
+
+
+def _exercise_decoder_under_the_filter() -> None:
+    """Import the decoder and decode every admitted sample inside the confined child.
+
+    Succeeding at all is the strongest half of the claim, because the filter denies thread
+    creation: a decoder with a worker pool does not reach the second line. The sampling below is
+    the explicit half, so the count is asserted rather than inferred from a survival."""
+
+    probe = subprocess.Popen(
+        [sys.executable, "-c", _decoder_probe()],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env={},
+        close_fds=True,
+        preexec_fn=_install_networkless_seccomp,
+    )
+    assert probe.stdout is not None
+    if probe.stdout.readline() != b"decoder-ready\n":
+        probe.kill()
+        raise RuntimeError("weather decoder did not start under the crawler seccomp filter")
+
+    counts: list[int] = []
+    while probe.poll() is None:
+        observed = _thread_count(probe.pid)
+        if observed is None:
+            break
+        counts.append(observed)
+        time.sleep(_DECODER_SAMPLE_INTERVAL)
+
+    stdout, stderr = probe.communicate(timeout=30.0)
+    if probe.returncode != 0 or stderr:
+        raise RuntimeError(
+            f"weather decoder failed under the crawler seccomp filter with status "
+            f"{probe.returncode}"
+        )
+    if not stdout.startswith(b"decoder-ok "):
+        raise RuntimeError("weather decoder did not complete every admitted sample")
+    if len(counts) < _DECODER_LEAST_SAMPLES:
+        raise RuntimeError(
+            f"only {len(counts)} thread counts were taken while the decoder ran, which is too "
+            f"few to say anything about what it did during a decode"
+        )
+    if set(counts) != {1}:
+        raise RuntimeError(
+            f"the weather decoder ran on more than one thread during a decode: observed "
+            f"{sorted(set(counts))}"
+        )
+    print(stdout.decode("ascii").strip())
+
+
+def main() -> int:
+    """Exercise normalization plus a malicious network probe under the production launcher."""
+
+    _disable_process_dumpability()
+    request = HostedCrawlerRequest(
+        request_id="12121212-1212-4212-8212-121212121212",
+        source_id="openligadb",
+        adapter_id="external.openligadb",
+        adapter_version="1.0.0",
+        requested_at="2026-07-20T12:00:00Z",
+        query={"league_shortcut": "bl1", "league_season": 2025},
+        egress_policy_attestation=OPENLIGADB_EGRESS_POLICY_ATTESTATION,
+    )
+    source_url = "https://api.openligadb.de/getmatchdata/bl1/2025"
+    normalization_input = encode_normalization_input(
+        request,
+        SimpleNamespace(
+            source_url=source_url,
+            final_url=source_url,
+            media_type="application/json",
+            content=_RAW_FIXTURE,
+            content_sha256=sha256_bytes(_RAW_FIXTURE),
+            transport_evidence_digest="f" * 64,
+            hops=(SimpleNamespace(status=200, etag=None, last_modified=None),),
+        ),
+    )
+    result = parse_crawler_result(
+        parse_canonical_json(_run_crawler_child(request, normalization_input))
+    )
+    if result.request_digest != request.digest or b"FC Bayern Muenchen" not in (
+        result.normalized_content
+    ):
+        raise RuntimeError("sandboxed normalizer result binding failed")
+
+    probe = subprocess.run(
+        [sys.executable, "-c", _MALICIOUS_PROBE],
+        input=b"",
+        capture_output=True,
+        env={},
+        close_fds=True,
+        preexec_fn=_install_networkless_seccomp,
+        check=False,
+        timeout=10.0,
+    )
+    if probe.returncode != 0 or probe.stdout != b"networkless-seccomp-ok\n" or probe.stderr:
+        raise RuntimeError("crawler seccomp malicious-network probe failed")
+    print("networkless-seccomp-ok")
+    _exercise_decoder_under_the_filter()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())