chtypes 2.0.0.dev11__py3-none-any.whl → 2.0.0.dev13__py3-none-any.whl

chtypes/__init__.py

@@ -13,14 +13,15 @@ C ABI. Nothing here reimplements a coercion rule, which is why the answers are
 exact by construction rather than approximately right.
 
     import chtypes
-    from chtypes import Format, Registry
+    from chtypes import DocFlags, Format, Registry
 
     chtypes.setup(timezone="UTC")                # once, before the first open
     registry = Registry(autofetch=True)
     library = registry.for_version("26.8")       # a release line or an exact version
     ddl = b"CREATE TABLE t (ts DateTime, seq UInt8) ENGINE = Memory"
     with library.compile_table(ddl) as schema:
-        batch = schema.rows(Format.JSON_EACH_ROW, b'{"ts":"2026-01-15 10:30:00","seq":256}\\n')
+        body = b'{"ts":"2026-01-15 10:30:00","seq":256}\\n'
+        batch = schema.rows(Format.JSON_EACH_ROW, body, doc_flags=DocFlags.ALL)
 
     batch.outcome        # Outcome.ACCEPTED: the insert would succeed
     batch.transformed    # the changes ClickHouse makes and reports as success

chtypes/_ocifetch/_ensure.py

@@ -75,6 +75,7 @@ from chtypes._ocifetch._layout import (
 from chtypes._ocifetch._lock import Lock, LockPin, load_lock, new_lock, save_lock
 from chtypes._ocifetch._oci import (
     Descriptor,
+    copy_blob_verified,
     fetch_blob_bytes,
     fetch_blob_to_path,
     fetch_manifest_by_digest,
@@ -86,6 +87,8 @@ from chtypes._ocifetch._oci import (
     resolve_platform_manifest,
     spelling_components,
     validate_spelling,
+    verify_body_matches_digest,
+    verify_json_descriptor,
     version_within_request,
 )
 from chtypes._ocifetch._referrers import ReferrerListing, discover_referrers, read_listing
@@ -1296,6 +1299,48 @@ def _ensure(
         raise _translate_transport_error(e) from e
 
 
+def _local_bundle_referrers(
+    blobs_dir: Path, skip: Collection[str], known: dict[str, bytes] | None = None
+) -> dict[str, list[tuple[str, dict]]]:
+    """Each subject manifest digest's signature referrer manifests among
+    `blobs_dir`'s blobs, as (blob name, parsed manifest), in listing order.
+    A referrer need not be listed in `index.json`, so every blob that could
+    be one is read, once per lookup rather than once per entry. A blob named
+    in `skip` (the caller's candidate layers) and a blob larger than
+    `limits.manifest_bytes`, which cannot be a manifest, are never read, so
+    no layer is read here, and a blob in `known` (bytes the caller already
+    read, by name) is not read again (public issue #690)."""
+    out: dict[str, list[tuple[str, dict]]] = {}
+    try:
+        blob_paths = list(blobs_dir.iterdir())
+    except OSError:
+        return out
+    for blob_path in blob_paths:
+        if blob_path.name in skip:
+            continue
+        try:
+            body = known.get(blob_path.name) if known is not None else None
+            if body is None:
+                if blob_path.stat().st_size > C.MANIFEST_MAX_BYTES:
+                    continue
+                body = blob_path.read_bytes()
+            doc = json.loads(body)
+        except (ValueError, OSError, UnicodeDecodeError):
+            continue
+        if not isinstance(doc, dict) or doc.get("mediaType") != C.MEDIA_TYPE_MANIFEST:
+            continue
+        subject = doc.get("subject") or {}
+        subject_digest = subject.get("digest") if isinstance(subject, dict) else None
+        if not isinstance(subject_digest, str):
+            continue
+        # A signature referrer only: a revocation of the manifest carries the same
+        # bundle media type, and is not the signature of the artifact.
+        if doc.get("artifactType") != C.MEDIA_TYPE_BUNDLE:
+            continue
+        out.setdefault(subject_digest, []).append((blob_path.name, doc))
+    return out
+
+
 def _verify_and_install_from_local_blobs(
     *,
     source_root: Path,
@@ -1303,6 +1348,8 @@ def _verify_and_install_from_local_blobs(
     cache_write_root: Path,
     trusted_keys: tuple[TrustedKey, ...],
     visible_only: bool = False,
+    referrers: dict[str, list[tuple[str, dict]]] | None = None,
+    manifest_size: int | None = None,
 ) -> VerifiedRecord | None:
     """docs/guides/fetch-v1.md §1/§6: "A pre-seeded layout has entries in
     `index.json` with no corresponding `unpacked/` directory yet; the first
@@ -1314,7 +1361,14 @@ def _verify_and_install_from_local_blobs(
     `source_root/blobs/sha256/*`, never the network. Returns `None` — never
     raises — for a missing, unreadable or untrusted entry, the same
     tolerant shape `_find_verified_signature` uses for a network referrer:
-    a bad pre-seed is "nothing usable here," not a crash."""
+    a bad pre-seed is "nothing usable here," not a crash.
+
+    The manifest blob is checked against `manifest_digest` (and against
+    `manifest_size`, the index descriptor's size, when the caller has one)
+    before it is parsed, and the layer blob against the manifest's layer
+    descriptor (size and sha256) before it is decompressed: the same
+    checks a download gets (§7.3), so no byte the signature does not cover
+    reaches the install."""
     blobs_dir = source_root / "blobs" / "sha256"
     if not blobs_dir.is_dir():
         return None
@@ -1323,28 +1377,25 @@ def _verify_and_install_from_local_blobs(
     if not manifest_path.is_file():
         return None
     try:
-        manifest_doc = json.loads(manifest_path.read_bytes())
+        manifest_bytes = manifest_path.read_bytes()
+        verify_body_matches_digest(manifest_bytes, manifest_digest, what="pre-seeded manifest")
+        if manifest_size is not None:
+            verify_json_descriptor(
+                manifest_bytes,
+                Descriptor(media_type="", digest=manifest_digest, size=manifest_size),
+                what="pre-seeded manifest",
+            )
+        manifest_doc = json.loads(manifest_bytes)
         layer_desc = manifest_layer_descriptor(manifest_doc)
     except (ValueError, OSError, FetchError):
         return None
 
+    if referrers is None:
+        referrers = _local_bundle_referrers(blobs_dir, {parse_digest(layer_desc.digest)})
     verified = None
     bundle_digest = None
     bundle_manifest_digest = None
-    for blob_path in blobs_dir.iterdir():
-        try:
-            doc = json.loads(blob_path.read_bytes())
-        except (ValueError, OSError, UnicodeDecodeError):
-            continue
-        if not isinstance(doc, dict) or doc.get("mediaType") != C.MEDIA_TYPE_MANIFEST:
-            continue
-        subject = doc.get("subject") or {}
-        if subject.get("digest") != manifest_digest:
-            continue
-        # A signature referrer only: a revocation of the manifest carries the same
-        # bundle media type, and is not the signature of the artifact.
-        if doc.get("artifactType") != C.MEDIA_TYPE_BUNDLE:
-            continue
+    for referrer_name, doc in referrers.get(manifest_digest, ()):
         try:
             referrer_layer = manifest_single_layer(doc, expected_media_type=C.MEDIA_TYPE_BUNDLE)
             bundle_bytes = (blobs_dir / parse_digest(referrer_layer.digest)).read_bytes()
@@ -1354,7 +1405,7 @@ def _verify_and_install_from_local_blobs(
         if candidate is not None:
             verified = candidate
             bundle_digest = referrer_layer.digest
-            bundle_manifest_digest = "sha256:" + blob_path.name
+            bundle_manifest_digest = "sha256:" + referrer_name
             break
     if verified is None:
         return None
@@ -1383,9 +1434,18 @@ def _verify_and_install_from_local_blobs(
     scratch = _scratch_root(cache_write_root)
     tmp_dir = tempfile.mkdtemp(dir=str(scratch))
     try:
+        # Size and sha256 against the layer descriptor BEFORE decompressing
+        # (§7.3), on a copy in scratch, which is then the file unpacked.
+        verified_layer = os.path.join(tmp_dir, "layer")
+        copy_blob_verified(
+            str(layer_path),
+            layer_desc,
+            verified_layer,
+            what=f"pre-seeded layer {layer_desc.digest}",
+        )
         unpacked_dir = os.path.join(tmp_dir, "unpacked")
         unpack_tar_zst(
-            str(layer_path),
+            verified_layer,
             unpacked_dir,
             library_name=predicate["library"],
             library_sha256=predicate["library_sha256"],
@@ -1422,7 +1482,9 @@ def _verify_preseeded_entries(roots: Sequence[Path], trusted_keys: tuple[Trusted
     anywhere yet, and verifies+unpacks it from that root's OWN local blobs,
     writing the result into `roots[0]` (the writable cache) regardless of
     which root — including a read-only system directory — the blobs came
-    from."""
+    from. An entry that already has a record is passed over without reading
+    a blob, and each root's signature referrers are read once, never by
+    reading a candidate's layer (public issue #690)."""
     cache_write_root = roots[0]
     already_verified = {record.manifest for _dir, record in list_verified_records(roots)}
     for root in roots:
@@ -1431,16 +1493,41 @@ def _verify_preseeded_entries(roots: Sequence[Path], trusted_keys: tuple[Trusted
             index_doc = json.loads(index_path.read_text(encoding="utf-8"))
         except (OSError, ValueError):
             continue
+        blobs_dir = root / "blobs" / "sha256"
+        candidates: list[str] = []
+        candidate_layers: set[str] = set()
+        manifests_read: dict[str, bytes] = {}
+        sizes: dict[str, int] = {}
         for m in index_doc.get("manifests") or []:
             digest = m.get("digest")
             if not digest or digest in already_verified:
                 continue
+            candidates.append(digest)
+            size = m.get("size")
+            if isinstance(size, int) and not isinstance(size, bool):
+                sizes[digest] = size
+            try:
+                manifest_hex = parse_digest(digest)
+                manifest_bytes = (blobs_dir / manifest_hex).read_bytes()
+                manifests_read[manifest_hex] = manifest_bytes
+                layer = manifest_layer_descriptor(json.loads(manifest_bytes))
+                candidate_layers.add(parse_digest(layer.digest))
+            except (ValueError, OSError, FetchError):
+                continue
+        if not candidates:
+            continue
+        referrers = _local_bundle_referrers(blobs_dir, candidate_layers, manifests_read)
+        for digest in candidates:
+            if digest in already_verified:
+                continue
             record = _verify_and_install_from_local_blobs(
                 source_root=root,
                 manifest_digest=digest,
                 cache_write_root=cache_write_root,
                 trusted_keys=trusted_keys,
                 visible_only=True,
+                referrers=referrers,
+                manifest_size=sizes.get(digest),
             )
             if record is not None:
                 already_verified.add(digest)

chtypes/_ocifetch/_oci.py

@@ -26,6 +26,7 @@ from chtypes._ocifetch._http import (
 
 __all__ = [
     "Descriptor",
+    "copy_blob_verified",
     "fetch_blob_bytes",
     "fetch_blob_to_path",
     "fetch_manifest_by_digest",
@@ -143,6 +144,46 @@ def verify_body_matches_digest(body: bytes, digest: str, *, what: str) -> None:
         )
 
 
+def copy_blob_verified(src_path: str, descriptor: Descriptor, dest_path: str, *, what: str) -> None:
+    """Copy a LOCAL blob file (a pre-seeded layout's `blobs/sha256/<hex>`)
+    to `dest_path`, streaming, and check the copy's size and sha256 against
+    `descriptor` BEFORE the caller does anything else with it — the local
+    counterpart of `fetch_blob_to_path`, with the same §7.3 rule ("check
+    size and sha256 before decompressing"). The caller unpacks the copy it
+    verified, never the source file, so the bytes checked are the bytes
+    used. Reads at most `descriptor.size + 1` bytes; removes `dest_path` on
+    any failure."""
+    expected = parse_digest(descriptor.digest)
+    hasher = hashlib.sha256()
+    size = 0
+    try:
+        with open(src_path, "rb") as src, open(dest_path, "wb") as dst:
+            while size <= descriptor.size:
+                chunk = src.read(min(1 << 20, descriptor.size + 1 - size))
+                if not chunk:
+                    break
+                hasher.update(chunk)
+                dst.write(chunk)
+                size += len(chunk)
+        if size > descriptor.size:
+            raise ArtifactCorruptError(
+                f"{what}: larger than the descriptor's size {descriptor.size}"
+            )
+        if size != descriptor.size:
+            raise ArtifactCorruptError(
+                f"{what}: size {size} disagreed with the descriptor's {descriptor.size}"
+            )
+        actual = hasher.hexdigest()
+        if actual != expected:
+            raise ArtifactCorruptError(f"{what}: sha256 {actual} disagreed with {expected}")
+    except BaseException:
+        try:
+            os.unlink(dest_path)
+        except OSError:
+            pass
+        raise
+
+
 def resolve_platform_manifest(index_doc: dict, platform_key: str) -> Descriptor:
     """Pick the one manifest descriptor in an OCI image index matching
     `platform_key` (e.g. "linux-arm64"), refusing a duplicate or absent

chtypes/library.py

@@ -54,6 +54,8 @@ __all__ = ["Block", "Filter", "Library", "Schema", "Server", "open_unverified"]
 # One image per file, process-wide: keyed on the realpath's device and inode, so
 # two registries, two spellings or a hardlink of one artifact share one image
 # and one Library. Guarded by the setup lock, which an open holds throughout.
+# The batch preview's default `doc_flags`: 0, the lean result (no per-value documents).
+_LEAN_DOC_FLAGS = DocFlags(0)
 _IMAGES: dict[tuple[int, int], Library] = {}
 
 
@@ -212,7 +214,8 @@ class Filter(_Handle):
         insert_options_extra: Mapping[str, str] | None = None,
     ) -> FilterResult:
         """Evaluate over a body. `settings` and `session_timezone` are the body's
-        PARSE settings only: the filter's own zone was fixed when it compiled.
+        INSERT settings: they govern its parse and the expressions the INSERT
+        evaluates, never the WHERE, whose zone was fixed when the filter compiled.
         `columns` is the INSERT column list and the keywords after it the
         `insert_options` members, as for `Schema.rows`."""
         payload = body_bytes(body)
@@ -304,11 +307,14 @@ class Schema(_Handle):
         insert_options_extra: Mapping[str, str] | None = None,
         row_filter: Filter | None = None,
         export: Format | int | None = None,
-        doc_flags: DocFlags = DocFlags.ALL,
+        doc_flags: DocFlags = _LEAN_DOC_FLAGS,
     ) -> BatchResult:
         """A whole body. `export` asks for the accepted rows serialized in that
         format (`payload`, and `spans`); to insert a body whose rows carry a
-        `default_generated` column, insert `payload`, never the original input."""
+        `default_generated` column, insert `payload`, never the original input.
+
+        `doc_flags` chooses the document groups; absent means 0, the lean result
+        (no per-value documents). Pass `DocFlags.ALL` for every group."""
         payload = body_bytes(body)
         settings_bytes = settings_json(settings, session_timezone)
         columns_bytes = columns_json(columns)

{chtypes-2.0.0.dev11.dist-info → chtypes-2.0.0.dev13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: chtypes
-Version: 2.0.0.dev11
+Version: 2.0.0.dev13
 Summary: Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI
 Project-URL: Homepage, https://github.com/wave-rf/chtypes
 Project-URL: Repository, https://github.com/wave-rf/chtypes
@@ -43,7 +43,7 @@ uv add "chtypes>=2.0.0.dev0"            # or: pip install "chtypes>=2.0.0.dev0"
 
 ```python
 import chtypes
-from chtypes import Format, Registry
+from chtypes import DocFlags, Format, Registry
 
 chtypes.setup(timezone="UTC")  # once, before the first open (optional)
 registry = Registry(autofetch=True)
@@ -51,7 +51,7 @@ library = registry.for_version("26.8")  # a release line or an exact version
 
 ddl = b"CREATE TABLE t (x UInt8, ts DateTime DEFAULT now()) ENGINE = Memory"
 with library.compile_table(ddl) as schema:
-    batch = schema.rows(Format.JSON_EACH_ROW, b'{"x":256}\n')
+    batch = schema.rows(Format.JSON_EACH_ROW, b'{"x":256}\n', doc_flags=DocFlags.ALL)
 
 row = batch.rows[0]
 print(batch.outcome)  # accepted

{chtypes-2.0.0.dev11.dist-info → chtypes-2.0.0.dev13.dist-info}/RECORD

@@ -1,4 +1,4 @@
-chtypes/__init__.py,sha256=cmfI6SOZBF7nU7awAbwH1l0HriwA3wWj15NOOGvptfc,6682
+chtypes/__init__.py,sha256=8bwCCMfg7k233x9tK6Eg8lITeC8EDwSS5M8VCkxbuPY,6736
 chtypes/__main__.py,sha256=6qal_3mJBVfNc9LNHsSB0d9Sfurxix6QbNWG5TEa4L4,19392
 chtypes/_advisory.py,sha256=wTRNdwqLiOGsFKZQ7CYZHsfIhDrKc_arpaxEYZCUxV4,1212
 chtypes/_codes.py,sha256=mRhjP-7fOU6j15le7r1jHwLJkiClMIIIEtfOkriR7uE,1301
@@ -8,7 +8,7 @@ chtypes/_guard.py,sha256=kVgvxRWu7H-eba3hTqPBmpqjxOCWdQGwiFAHvSnl1OU,1996
 chtypes/_input.py,sha256=ZAUEs3r26_CsFiIvaXV9P-F_C9jlRRzMNKgG--G4HnQ,15157
 chtypes/_setup.py,sha256=7mXvPxLCHS9h1FRWNr5uKfCUg-Gk8VjmC-5KNW5HogI,6125
 chtypes/errors.py,sha256=-sCgdySXZn5HRL8Vkwbj9gSvhj0nvCCjwt3mFn6gKIc,10511
-chtypes/library.py,sha256=W7NywZEkogi5bcw8lGCEZZg3iwb6RF3e92BK5Y9yixg,21934
+chtypes/library.py,sha256=MYlk7ksWiq3OP2nJ4YoqRhP2rqLc_xdRktdNQAnwjm0,22290
 chtypes/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chtypes/registry.py,sha256=CQHnHr9T-2LyqRsYl5wdWKIVWCOiX9Vx4BLcZL2yCB0,23702
 chtypes/results.py,sha256=1HUdvhrjeEl9Fm1GOJUXVREtc7HN31s2yrQbm5_ZeKQ,15397
@@ -22,7 +22,7 @@ chtypes/_ocifetch/__init__.py,sha256=Nwmn1_JqkRSy-1q8182RT5m4HJfbdJgr54it7_TKvBg
 chtypes/_ocifetch/_channel.py,sha256=1MuIj5O8N1d8OdTN4Rx4zf3kc2Nh1hjx4o5BIE6JgmA,20484
 chtypes/_ocifetch/_constants.py,sha256=Gk-h6UkPg5dvNlhkr8z-UY4Y866unUPOSpE8ZwJvoJw,6287
 chtypes/_ocifetch/_dsse.py,sha256=nYkjmRefmVTonH1KUADNNUl0J9sh2C3fg-0WEyURjcc,9838
-chtypes/_ocifetch/_ensure.py,sha256=er9uP4MnyIfOZ3CJ2cyHolDdIw_rVYbPTZCod3RWmMw,77014
+chtypes/_ocifetch/_ensure.py,sha256=0v-tGzC4P0oLtADN-YBHzAMWaqOc2bYincxwUAhWtyg,81190
 chtypes/_ocifetch/_errors.py,sha256=SE0zcTUr1fSvSH7HiLGjAInYvD_im8_SyeZNf-VKzXI,7594
 chtypes/_ocifetch/_faults.py,sha256=HpoB9tJW7aLRiJdCFnspzkwhpFATZ6f6hGi49Mbfnhs,7609
 chtypes/_ocifetch/_goldens.py,sha256=8MicgeJSRd4WZkaEsdujRnOa7twQvbT-wfSMPZaD-ms,4191
@@ -30,15 +30,15 @@ chtypes/_ocifetch/_hold.py,sha256=AmXKLFWdPEmZ5fuxKbTKbzYn-Gue9AyMfYTquQ0qvOs,48
 chtypes/_ocifetch/_http.py,sha256=3D8WZXAObw-IiZqLSh9GEys2EKOEE2i9CVfUqjLh3E0,29013
 chtypes/_ocifetch/_layout.py,sha256=ezEoVrsSMhvDhQ4U04hsGdu1-_ISZQ4V6zg4gGIChTQ,21300
 chtypes/_ocifetch/_lock.py,sha256=14_uex9qXdOht-5FSrSvFvd99-gB7DmI3bDuxXs4YU8,5530
-chtypes/_ocifetch/_oci.py,sha256=UdA8QGTh1O27DBYRf5UxsevjgI52IecQlEf9IO6Y0A4,13920
+chtypes/_ocifetch/_oci.py,sha256=Xc0JEShGemdTlu2LviOBFSMTcjwCu9uP6jdG0qlKO80,15677
 chtypes/_ocifetch/_prune.py,sha256=gPToHNkOOVjCWA-lowJHnGpguqOFXklUOQs7PAGfhBQ,10203
 chtypes/_ocifetch/_referrers.py,sha256=SQ8BMVzzA1085azzfAA0Nw3GrHPjeu98smasKyS609A,10530
 chtypes/_ocifetch/_resolve.py,sha256=Qv8xFUdOR-Kb_JqmUeDiQ4_Xv2ETuNFpmoDckMp7eV8,11239
 chtypes/_ocifetch/_retired.py,sha256=mqnI9RA7UK0A5RthCeLlBeVAu5SHbtwKFwxsSZk5e6A,7018
 chtypes/_ocifetch/_revocation.py,sha256=DdKRjFp1eFAxVD7DZHK1bj3_sWVLlaEkVEVx6JyV2Zs,23710
 chtypes/_ocifetch/_unpack.py,sha256=Cn4-up63KL63L8pnkS5zwJ6VbbHPSigpCnmOdcHJ4RI,5609
-chtypes-2.0.0.dev11.dist-info/METADATA,sha256=yBHAW6_UeO3y8EAGTPXlNrIOvqgBJ0b8tTV5v73xiyk,11331
-chtypes-2.0.0.dev11.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-chtypes-2.0.0.dev11.dist-info/entry_points.txt,sha256=TdNcluWBR0VOt5mVFROsp-NiVPAdR3EHo0UsHlMC0r0,50
-chtypes-2.0.0.dev11.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-chtypes-2.0.0.dev11.dist-info/RECORD,,
+chtypes-2.0.0.dev13.dist-info/METADATA,sha256=W3xyv6dfQ9wI53cfWtQWioHnctgRuShzOSDZC5H3Jv4,11365
+chtypes-2.0.0.dev13.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+chtypes-2.0.0.dev13.dist-info/entry_points.txt,sha256=TdNcluWBR0VOt5mVFROsp-NiVPAdR3EHo0UsHlMC0r0,50
+chtypes-2.0.0.dev13.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+chtypes-2.0.0.dev13.dist-info/RECORD,,

{chtypes-2.0.0.dev11.dist-info → chtypes-2.0.0.dev13.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.32.4
+Generator: hatchling 1.33.0
 Root-Is-Purelib: true
 Tag: py3-none-any