chtypes 2.0.0.dev12__tar.gz → 2.0.0.dev13__tar.gz

{chtypes-2.0.0.dev12 → chtypes-2.0.0.dev13}/CHANGELOG.md

@@ -91,6 +91,7 @@ The four bindings in this repository are released together and give one answer,
 
 ### Fixed
 
+- Installing a pre-seeded cache entry from local blobs now checks the layer blob against the manifest's layer descriptor (size and sha256) before it decompresses it, and the manifest blob against its digest and its `index.json` descriptor's size before it parses it, as a downloaded blob already was and as the other three bindings already did. The install had checked only that the signed statement names the layer's digest, then unpacked the file stored under that name without hashing it, so a file replaced under a signed layer's name, holding the same library bytes plus other members, was installed with those members. Such an entry is now not installed, and the lookup is a miss. The installed library itself was always re-hashed against the signed predicate.
 - An offline lookup no longer reads every cached blob once per pre-seeded entry (#690). For each `index.json` entry with no record, the lookup read every blob of the layout, layers included, to find its signature, so entries it could not install (a signature not present, or one no trusted key verifies) cost entries × cache bytes on every lookup: about 4.9 s for 27 such entries over 1.27 GB (`measured`, darwin-arm64). The signature referrers are now read once per lookup, and never a layer, because a blob larger than `limits.manifest_bytes` cannot be a manifest. Installed entries were already skipped. The build that is installed is verified exactly as before: its manifest against its descriptor, its whole layer against its digest, and its signature, platform, request and fingerprint. A unit test counts the blob reads of one lookup that installs nothing and holds them to at most two per blob and five per entry, with no layer read, at 2 and at 20 entries alike.
 - Docs: `docs/guides/settings.md` names the ABI v2 admission time budget `chtypes_default_eval_cpu_nanos`, the evaluating thread's CPU time. Its examples used the 1.x name `chtypes_default_eval_wall_nanos`, which v2 refuses with `CHS_INVALID_ARGUMENT`.
 - A response body cut off part way (shorter than its Content-Length, a reset, a TLS close) is retried like a 5xx and, once the retry table is spent, is `CHTYPES_SOURCE_UNREACHABLE`; only an oversize body is `CHTYPES_ARTIFACT_CORRUPT`, and it is never retried (#641). Python took a body that ended before its Content-Length as the whole body and failed its digest check as corrupt without a retry; now a short body, or `http.client.IncompleteRead`, is retried and ends as unreachable. The rule is `docs/guides/fetch-v1.md` section 7, and the new `body-cut-off-then-ok` and `body-cut-off-exhausted` conformance cases hold every binding to it.

{chtypes-2.0.0.dev12 → chtypes-2.0.0.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: chtypes
-Version: 2.0.0.dev12
+Version: 2.0.0.dev13
 Summary: Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI
 Project-URL: Homepage, https://github.com/wave-rf/chtypes
 Project-URL: Repository, https://github.com/wave-rf/chtypes

{chtypes-2.0.0.dev12 → chtypes-2.0.0.dev13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "chtypes"
-version = "2.0.0.dev12"
+version = "2.0.0.dev13"
 description = "Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI"
 readme = "README.md"
 license = "Apache-2.0"

{chtypes-2.0.0.dev12 → chtypes-2.0.0.dev13}/src/chtypes/_ocifetch/_ensure.py

@@ -75,6 +75,7 @@ from chtypes._ocifetch._layout import (
 from chtypes._ocifetch._lock import Lock, LockPin, load_lock, new_lock, save_lock
 from chtypes._ocifetch._oci import (
     Descriptor,
+    copy_blob_verified,
     fetch_blob_bytes,
     fetch_blob_to_path,
     fetch_manifest_by_digest,
@@ -86,6 +87,8 @@ from chtypes._ocifetch._oci import (
     resolve_platform_manifest,
     spelling_components,
     validate_spelling,
+    verify_body_matches_digest,
+    verify_json_descriptor,
     version_within_request,
 )
 from chtypes._ocifetch._referrers import ReferrerListing, discover_referrers, read_listing
@@ -1346,6 +1349,7 @@ def _verify_and_install_from_local_blobs(
     trusted_keys: tuple[TrustedKey, ...],
     visible_only: bool = False,
     referrers: dict[str, list[tuple[str, dict]]] | None = None,
+    manifest_size: int | None = None,
 ) -> VerifiedRecord | None:
     """docs/guides/fetch-v1.md §1/§6: "A pre-seeded layout has entries in
     `index.json` with no corresponding `unpacked/` directory yet; the first
@@ -1357,7 +1361,14 @@ def _verify_and_install_from_local_blobs(
     `source_root/blobs/sha256/*`, never the network. Returns `None` — never
     raises — for a missing, unreadable or untrusted entry, the same
     tolerant shape `_find_verified_signature` uses for a network referrer:
-    a bad pre-seed is "nothing usable here," not a crash."""
+    a bad pre-seed is "nothing usable here," not a crash.
+
+    The manifest blob is checked against `manifest_digest` (and against
+    `manifest_size`, the index descriptor's size, when the caller has one)
+    before it is parsed, and the layer blob against the manifest's layer
+    descriptor (size and sha256) before it is decompressed: the same
+    checks a download gets (§7.3), so no byte the signature does not cover
+    reaches the install."""
     blobs_dir = source_root / "blobs" / "sha256"
     if not blobs_dir.is_dir():
         return None
@@ -1366,7 +1377,15 @@ def _verify_and_install_from_local_blobs(
     if not manifest_path.is_file():
         return None
     try:
-        manifest_doc = json.loads(manifest_path.read_bytes())
+        manifest_bytes = manifest_path.read_bytes()
+        verify_body_matches_digest(manifest_bytes, manifest_digest, what="pre-seeded manifest")
+        if manifest_size is not None:
+            verify_json_descriptor(
+                manifest_bytes,
+                Descriptor(media_type="", digest=manifest_digest, size=manifest_size),
+                what="pre-seeded manifest",
+            )
+        manifest_doc = json.loads(manifest_bytes)
         layer_desc = manifest_layer_descriptor(manifest_doc)
     except (ValueError, OSError, FetchError):
         return None
@@ -1415,9 +1434,18 @@ def _verify_and_install_from_local_blobs(
     scratch = _scratch_root(cache_write_root)
     tmp_dir = tempfile.mkdtemp(dir=str(scratch))
     try:
+        # Size and sha256 against the layer descriptor BEFORE decompressing
+        # (§7.3), on a copy in scratch, which is then the file unpacked.
+        verified_layer = os.path.join(tmp_dir, "layer")
+        copy_blob_verified(
+            str(layer_path),
+            layer_desc,
+            verified_layer,
+            what=f"pre-seeded layer {layer_desc.digest}",
+        )
         unpacked_dir = os.path.join(tmp_dir, "unpacked")
         unpack_tar_zst(
-            str(layer_path),
+            verified_layer,
             unpacked_dir,
             library_name=predicate["library"],
             library_sha256=predicate["library_sha256"],
@@ -1469,11 +1497,15 @@ def _verify_preseeded_entries(roots: Sequence[Path], trusted_keys: tuple[Trusted
         candidates: list[str] = []
         candidate_layers: set[str] = set()
         manifests_read: dict[str, bytes] = {}
+        sizes: dict[str, int] = {}
         for m in index_doc.get("manifests") or []:
             digest = m.get("digest")
             if not digest or digest in already_verified:
                 continue
             candidates.append(digest)
+            size = m.get("size")
+            if isinstance(size, int) and not isinstance(size, bool):
+                sizes[digest] = size
             try:
                 manifest_hex = parse_digest(digest)
                 manifest_bytes = (blobs_dir / manifest_hex).read_bytes()
@@ -1495,6 +1527,7 @@ def _verify_preseeded_entries(roots: Sequence[Path], trusted_keys: tuple[Trusted
                 trusted_keys=trusted_keys,
                 visible_only=True,
                 referrers=referrers,
+                manifest_size=sizes.get(digest),
             )
             if record is not None:
                 already_verified.add(digest)

{chtypes-2.0.0.dev12 → chtypes-2.0.0.dev13}/src/chtypes/_ocifetch/_oci.py

@@ -26,6 +26,7 @@ from chtypes._ocifetch._http import (
 
 __all__ = [
     "Descriptor",
+    "copy_blob_verified",
     "fetch_blob_bytes",
     "fetch_blob_to_path",
     "fetch_manifest_by_digest",
@@ -143,6 +144,46 @@ def verify_body_matches_digest(body: bytes, digest: str, *, what: str) -> None:
         )
 
 
+def copy_blob_verified(src_path: str, descriptor: Descriptor, dest_path: str, *, what: str) -> None:
+    """Copy a LOCAL blob file (a pre-seeded layout's `blobs/sha256/<hex>`)
+    to `dest_path`, streaming, and check the copy's size and sha256 against
+    `descriptor` BEFORE the caller does anything else with it — the local
+    counterpart of `fetch_blob_to_path`, with the same §7.3 rule ("check
+    size and sha256 before decompressing"). The caller unpacks the copy it
+    verified, never the source file, so the bytes checked are the bytes
+    used. Reads at most `descriptor.size + 1` bytes; removes `dest_path` on
+    any failure."""
+    expected = parse_digest(descriptor.digest)
+    hasher = hashlib.sha256()
+    size = 0
+    try:
+        with open(src_path, "rb") as src, open(dest_path, "wb") as dst:
+            while size <= descriptor.size:
+                chunk = src.read(min(1 << 20, descriptor.size + 1 - size))
+                if not chunk:
+                    break
+                hasher.update(chunk)
+                dst.write(chunk)
+                size += len(chunk)
+        if size > descriptor.size:
+            raise ArtifactCorruptError(
+                f"{what}: larger than the descriptor's size {descriptor.size}"
+            )
+        if size != descriptor.size:
+            raise ArtifactCorruptError(
+                f"{what}: size {size} disagreed with the descriptor's {descriptor.size}"
+            )
+        actual = hasher.hexdigest()
+        if actual != expected:
+            raise ArtifactCorruptError(f"{what}: sha256 {actual} disagreed with {expected}")
+    except BaseException:
+        try:
+            os.unlink(dest_path)
+        except OSError:
+            pass
+        raise
+
+
 def resolve_platform_manifest(index_doc: dict, platform_key: str) -> Descriptor:
     """Pick the one manifest descriptor in an OCI image index matching
     `platform_key` (e.g. "linux-arm64"), refusing a duplicate or absent

{chtypes-2.0.0.dev12 → chtypes-2.0.0.dev13}/tests/ocifetch/test_preseed_scan.py

@@ -8,8 +8,10 @@ The verdicts come from the blob reads the lookup itself made (every
 from __future__ import annotations
 
 import hashlib
+import io
 import json
 import shutil
+import tarfile
 from collections import Counter
 from collections.abc import Callable
 from pathlib import Path
@@ -159,3 +161,121 @@ def test_preseeded_install_still_verifies_the_layer(tmp_path: Path) -> None:
     layer.write_bytes(bytes(b))
     assert resolve_installed(Request("26.1"), "linux-arm64", _options(cache, "test-key")) is None
     assert not (cache / "unpacked" / "sha256" / digests[0].removeprefix("sha256:")).exists()
+
+
+def _tamper_layer(original: bytes, case: str) -> bytes:
+    """A different valid tar.zst carrying the signed library member's bytes
+    unchanged. "extra-member": every member of `original` plus `planted.txt`.
+    "manifest-json": the tarball's `manifest.json` altered — the signed
+    fixture layer has none (its members are `docs/` and the library), so its
+    altered form is a `manifest.json` the signed layer does not carry, beside
+    the same library bytes. Neither is covered by any signature."""
+    try:
+        from compression import zstd  # Python 3.14+
+    except ImportError:  # Python 3.11-3.13
+        from backports import zstd  # type: ignore[no-redef]
+
+    def add(dst: tarfile.TarFile, name: str, body: bytes) -> None:
+        info = tarfile.TarInfo(name)
+        info.size = len(body)
+        dst.addfile(info, io.BytesIO(body))
+
+    src = tarfile.open(fileobj=io.BytesIO(zstd.decompress(original)))
+    out = io.BytesIO()
+    with tarfile.open(fileobj=out, mode="w", format=tarfile.PAX_FORMAT) as dst:
+        for member in src.getmembers():
+            if member.name == "manifest.json":
+                continue
+            dst.addfile(member, src.extractfile(member) if member.isreg() else None)
+        if case == "extra-member":
+            add(dst, "planted.txt", b"bytes no signature covers\n")
+        else:
+            add(dst, "manifest.json", b'{"planted.txt": "bytes no signature covers"}\n')
+    return zstd.compress(out.getvalue())
+
+
+def _unpack_spy(monkeypatch: pytest.MonkeyPatch) -> list[str]:
+    """Every layer file the local install handed to `unpack_tar_zst`."""
+    import chtypes._ocifetch._ensure as ensure_mod
+
+    calls: list[str] = []
+    real = ensure_mod.unpack_tar_zst
+
+    def spy(path: str, *args: object, **kwargs: object) -> None:
+        calls.append(path)
+        real(path, *args, **kwargs)  # type: ignore[arg-type]
+
+    monkeypatch.setattr(ensure_mod, "unpack_tar_zst", spy)
+    return calls
+
+
+def _assert_nothing_installed(cache: Path, manifest: str) -> None:
+    leaked = [*cache.rglob("planted.txt"), *(cache / "unpacked").rglob("manifest.json")]
+    assert leaked == [], f"bytes outside the signature were installed: {leaked}"
+    assert not (cache / "unpacked" / "sha256" / manifest.removeprefix("sha256:")).exists()
+
+
+@pytest.mark.parametrize("case", ["extra-member", "manifest-json"])
+def test_preseeded_layer_swapped_under_its_signed_name_installs_nothing(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, case: str
+) -> None:
+    """The layer blob FILE under its signed digest name is replaced by a
+    different valid tar.zst carrying the same library bytes. The library
+    re-hash alone passes on those bytes; the layer descriptor check (size
+    and sha256) refuses them BEFORE any unpack. The lookup installs nothing
+    and is a miss."""
+    cache, digests = _seed_clones(tmp_path, 1)
+    layer = cache / "blobs" / "sha256" / BUILD_LAYER_HEX
+    layer.write_bytes(_tamper_layer(layer.read_bytes(), case))
+    assert hashlib.sha256(layer.read_bytes()).hexdigest() != BUILD_LAYER_HEX
+    unpacked = _unpack_spy(monkeypatch)
+    result = resolve_installed(Request("26.1"), "linux-arm64", _options(cache, "test-key"))
+    _assert_nothing_installed(cache, digests[0])
+    assert unpacked == [], f"{case}: the unverified layer reached unpack"
+    assert result is None
+
+
+def test_preseeded_manifest_swapped_under_its_digest_name_installs_nothing(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The manifest blob FILE under its index digest is replaced by other
+    bytes naming the same signed layer: the manifest no longer hashes to the
+    digest the signature names, and the lookup installs nothing."""
+    cache, digests = _seed_clones(tmp_path, 1)
+    path = cache / "blobs" / "sha256" / digests[0].removeprefix("sha256:")
+    manifest = json.loads(path.read_bytes())
+    manifest["annotations"] = {"test.swapped": "1"}
+    path.write_bytes(json.dumps(manifest).encode())
+    unpacked = _unpack_spy(monkeypatch)
+    assert resolve_installed(Request("26.1"), "linux-arm64", _options(cache, "test-key")) is None
+    assert unpacked == []
+    _assert_nothing_installed(cache, digests[0])
+
+
+def test_preseeded_manifest_size_disagreeing_with_index_installs_nothing(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The index descriptor's size disagrees with the manifest blob's: the
+    manifest is checked against its whole descriptor, as a download is."""
+    cache, digests = _seed_clones(tmp_path, 1)
+    index = json.loads((cache / "index.json").read_text())
+    index["manifests"][0]["size"] += 1
+    (cache / "index.json").write_text(json.dumps(index))
+    unpacked = _unpack_spy(monkeypatch)
+    assert resolve_installed(Request("26.1"), "linux-arm64", _options(cache, "test-key")) is None
+    assert unpacked == []
+    _assert_nothing_installed(cache, digests[0])
+
+
+def test_preseeded_untouched_layout_still_installs(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The control for the cases above: the same layout, untouched, reaches
+    unpack once, installs and answers."""
+    cache, digests = _seed_clones(tmp_path, 1)
+    unpacked = _unpack_spy(monkeypatch)
+    assert (
+        resolve_installed(Request("26.1"), "linux-arm64", _options(cache, "test-key")) is not None
+    )
+    assert len(unpacked) == 1
+    assert (cache / "unpacked" / "sha256" / digests[0].removeprefix("sha256:")).is_dir()

{chtypes-2.0.0.dev12 → chtypes-2.0.0.dev13}/uv.lock

@@ -74,7 +74,7 @@ wheels = [
 
 [[package]]
 name = "chtypes"
-version = "2.0.0.dev12"
+version = "2.0.0.dev13"
 source = { editable = "." }
 dependencies = [
     { name = "backports-zstd", marker = "python_full_version < '3.14'" },