x4d-devkit 0.46.0rc2__tar.gz → 0.47.0rc1__tar.gz

{x4d_devkit-0.46.0rc2/x4d_devkit.egg-info → x4d_devkit-0.47.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.46.0rc2
+Version: 0.47.0rc1
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0

{x4d_devkit-0.46.0rc2 → x4d_devkit-0.47.0rc1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.46.0rc2"
+version = "0.47.0rc1"
 description = "Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation"
 readme = "README.md"
 license = "Apache-2.0"

{x4d_devkit-0.46.0rc2 → x4d_devkit-0.47.0rc1}/tests/test_clean_basemap.py

@@ -3,6 +3,7 @@ import copy
 import numpy as np
 import pytest
 import zstandard
+import x4d_devkit.clean_basemap as clean_basemap
 
 from tests.semantic_scene_contract_fixtures import current_semantic_scene_component
 from x4d_devkit.clean_basemap import (
@@ -53,6 +54,7 @@ def fixture():
         raw_point_ordinals=[0, 1, 2],
         timestamps_ns=[100, 100, 100],
         decisions=decisions[:3],
+        intensities=[2.5, 19.0, 240.0],
     )
     profile = {
         "algorithm": {"id": "clean_static_basemap", "version": "0.6.0"},
@@ -101,6 +103,7 @@ def fixture():
                 "bounds": bounds,
                 "geometry": descriptor,
                 "mapping": mapping.descriptor("mapping"),
+                "intensity_range": [2.5, 240.0],
             }
         ],
         "diagnostics": {
@@ -138,6 +141,9 @@ def validate(manifest, assets):
         manifest,
         read_asset=assets.__getitem__,
         observation_to_world=lambda sweep, ordinals, times: original[ordinals],
+        observation_intensity=lambda sweep, ordinals, times: np.asarray(
+            [2.5, 19.0, 240.0], dtype=np.float32
+        )[ordinals],
     )
 
 
@@ -154,6 +160,7 @@ def remap(manifest, assets, **overrides):
         "raw_point_ordinals": mapping["raw_point_ordinal"],
         "timestamps_ns": mapping["timestamp_ns"],
         "decisions": mapping["decision"],
+        "intensities": mapping["intensity"],
     }
     encoded = encode_clean_basemap_mapping(
         geometry_raw_sha256=chunk["geometry"]["raw_sha256"], **(arguments | overrides)
@@ -189,6 +196,167 @@ def test_complete_scene_and_exact_original_observation_roundtrip():
         parsed.data["profile"]["voxel_size_m"] = 1
 
 
+def test_intensity_is_original_float32_and_does_not_change_geometry():
+    manifest, assets, _ = fixture()
+    before = copy.deepcopy(manifest["scene_source"])
+    assert MAPPING_DTYPE.itemsize == 24
+    remap(manifest, assets, intensities=[2.5, 20.0, 240.0])
+    assert manifest["scene_source"] == before
+    with pytest.raises(ValueError, match="intensity differs from its original"):
+        validate(manifest, assets)
+
+
+@pytest.mark.parametrize(
+    "values",
+    [
+        [1, 2],
+        [True, False, True],
+        [1, float("nan"), 3],
+        [1, float("inf"), 3],
+        [1, 1e100, 3],
+    ],
+)
+def test_invalid_intensity_cannot_be_encoded(values):
+    manifest, assets, _ = fixture()
+    with pytest.raises(ValueError, match="intensit"):
+        remap(manifest, assets, intensities=values)
+
+
+def test_intensity_range_cannot_hide_wrong_values():
+    manifest, assets, _ = fixture()
+    manifest["chunks"][0]["intensity_range"] = [0.0, 255.0]
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+    with pytest.raises(ValueError, match="intensity range"):
+        validate(manifest, assets)
+
+
+def test_old_manifest_and_mapping_are_rejected():
+    manifest, assets, _ = fixture()
+    manifest["contract_version"] = "x4d-clean-static-basemap-v4"
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+    with pytest.raises(ValueError):
+        parse_clean_basemap_manifest(manifest)
+    chunk = manifest["chunks"][0]
+    descriptor = chunk["mapping"]
+    raw = bytearray(zstandard.ZstdDecompressor().decompress(assets[descriptor["path"]]))
+    raw[:8] = b"X4DCBM2\0"
+    payload = zstandard.ZstdCompressor().compress(raw)
+    descriptor = dict(
+        descriptor,
+        path=f"generated/reconstruction/clean_basemap/mapping/{sha256_bytes(payload)[7:]}.zst",
+        sha256=sha256_bytes(payload),
+        raw_sha256=sha256_bytes(bytes(raw)),
+        size_bytes=len(payload),
+    )
+    with pytest.raises(ValueError):
+        decode_clean_basemap_mapping(
+            payload,
+            descriptor=descriptor,
+            geometry_raw_sha256=chunk["geometry"]["raw_sha256"],
+            point_count=chunk["point_count"],
+        )
+
+
+def test_decision_proof_reuse_is_within_one_validation_call(monkeypatch):
+    manifest, assets, _ = fixture()
+    decoded = []
+    original = clean_basemap.decode_clean_basemap_decisions
+
+    def count(*args, **kwargs):
+        result = original(*args, **kwargs)
+        assert not result.flags.writeable
+        decoded.append(result.nbytes)
+        return result
+
+    monkeypatch.setattr(clean_basemap, "decode_clean_basemap_decisions", count)
+    validate(manifest, assets)
+    assert decoded == [5]
+    validate(manifest, assets)
+    assert decoded == [5, 5]
+
+
+@pytest.mark.parametrize("changed", [b"tampered", bytearray, memoryview])
+def test_cached_decision_proof_still_checks_every_carrier_read(changed):
+    manifest, assets, _ = fixture()
+    original = original_points(manifest, assets)
+    path = manifest["sweep_sources"][0]["decisions"]["path"]
+    reads = 0
+
+    def read_asset(name):
+        nonlocal reads
+        body = assets[name]
+        if name == path:
+            reads += 1
+            if reads > 1:
+                return changed(body) if callable(changed) else changed
+        return body
+
+    with pytest.raises(ValueError, match="payload size/hash"):
+        validate_clean_basemap_resource(
+            manifest,
+            read_asset=read_asset,
+            observation_to_world=lambda sweep, ordinals, times: original[ordinals],
+            observation_intensity=lambda sweep, ordinals, times: np.asarray(
+                [2.5, 19.0, 240.0], dtype=np.float32
+            )[ordinals],
+        )
+
+
+def add_second_sweep(manifest, assets):
+    source = copy.deepcopy(manifest["sweep_sources"][0])
+    source.update(sample_data_token="sweep-1", asset_sha256="sha256:" + "c" * 64)
+    carrier = encode_clean_basemap_decisions(
+        asset_sha256=source["asset_sha256"], decisions=[1, 0, 3, 18, 33]
+    )
+    source["decisions"] = carrier.descriptor("decisions")
+    assets[source["decisions"]["path"]] = carrier.payload
+    manifest["sweep_sources"].append(source)
+    manifest["diagnostics"]["source_point_count"] *= 2
+    manifest["diagnostics"]["decision_counts"] = {
+        key: value * 2
+        for key, value in manifest["diagnostics"]["decision_counts"].items()
+    }
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+    return source
+
+
+@pytest.mark.parametrize("changed", ["source", "count", "descriptor"])
+def test_decision_cache_never_reuses_a_different_binding(changed):
+    manifest, assets, _ = fixture()
+    second = add_second_sweep(manifest, assets)
+    second["decisions"] = copy.deepcopy(manifest["sweep_sources"][0]["decisions"])
+    if changed != "source":
+        second["asset_sha256"] = manifest["sweep_sources"][0]["asset_sha256"]
+    if changed == "count":
+        second["point_count"] += 1
+        manifest["diagnostics"]["source_point_count"] += 1
+        manifest["diagnostics"]["decision_counts"]["33"] += 1
+    if changed == "descriptor":
+        second["decisions"]["size_bytes"] += 1
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+    with pytest.raises(ValueError):
+        validate(manifest, assets)
+
+
+@pytest.mark.parametrize("limit,expected_decodes", [(0, 3), (4, 3), (5, 3), (10, 2)])
+def test_decision_proof_cache_is_bounded_and_evicted_entries_are_revalidated(
+    monkeypatch, limit, expected_decodes
+):
+    manifest, assets, _ = fixture()
+    add_second_sweep(manifest, assets)
+    original = clean_basemap.decode_clean_basemap_decisions
+    calls = []
+
+    def count(*args, **kwargs):
+        calls.append(kwargs["asset_sha256"])
+        return original(*args, **kwargs)
+
+    monkeypatch.setattr(clean_basemap, "_DECISION_CACHE_BYTES", limit)
+    monkeypatch.setattr(clean_basemap, "decode_clean_basemap_decisions", count)
+    validate(manifest, assets)
+    assert len(calls) == expected_decodes
+
+
 @pytest.mark.parametrize(
     "overrides,message",
     [
@@ -221,6 +389,9 @@ def test_original_observation_projection_is_required(kind):
             manifest,
             read_asset=assets.__getitem__,
             observation_to_world=lambda *args: result,
+            observation_intensity=lambda sweep, ordinals, times: np.asarray(
+                [2.5, 19.0, 240.0], dtype=np.float32
+            )[ordinals],
         )
 
 
@@ -289,6 +460,7 @@ def test_every_byte_in_last_record_obeys_exact_decision_domain(family):
             raw_point_ordinals=np.arange(count),
             timestamps_ns=np.full(count, 100),
             decisions=np.full(count, KEEP_GROUND),
+            intensities=np.arange(count, dtype=np.float32),
         )
         domain, decode = KEEP_CODES, decode_clean_basemap_mapping
         arguments = dict(geometry_raw_sha256=anchor, point_count=count)

{x4d_devkit-0.46.0rc2 → x4d_devkit-0.47.0rc1}/tests/test_sdk_api.py

@@ -11,7 +11,7 @@ from x4d_devkit.eval.matching import MetricData
 
 
 def test_package_version_is_current_release():
-    assert x4d_devkit.__version__ == "0.46.0rc2"
+    assert x4d_devkit.__version__ == "0.47.0rc1"
 
 
 def test_loader_exposes_and_enforces_only_current_schema(tmp_path):

{x4d_devkit-0.46.0rc2 → x4d_devkit-0.47.0rc1}/x4d_devkit/clean_basemap.py

@@ -7,6 +7,7 @@ their native authorities; this module never guesses ownership from coordinates.
 from __future__ import annotations
 
 import copy
+from collections import OrderedDict
 from dataclasses import dataclass
 import math
 import re
@@ -35,10 +36,11 @@ from x4d_devkit.semantic_scene_contract import (
 )
 from x4d_devkit.time_contract import parse_time_ns
 
-CLEAN_BASEMAP_CONTRACT_VERSION = "x4d-clean-static-basemap-v4"
+CLEAN_BASEMAP_CONTRACT_VERSION = "x4d-clean-static-basemap-v5"
 CLEAN_BASEMAP_OPERATION = "reconstruction.prepare_clean_static_basemap"
-CLEAN_BASEMAP_MAPPING_FORMAT = "x4d-clean-static-basemap-mapping-v2"
+CLEAN_BASEMAP_MAPPING_FORMAT = "x4d-clean-static-basemap-mapping-v3"
 CLEAN_BASEMAP_DECISION_FORMAT = "x4d-clean-static-basemap-decisions-v3"
+_DECISION_CACHE_BYTES = 128 << 20
 CLEAN_BASEMAP_INPUT_FIELDS = frozenset(
     {"samples_revision", "sweeps_revision", "calibration_revision", "pose_revision"}
 )
@@ -53,7 +55,7 @@ DECISION_CODES = frozenset({0, 1, 3, 18, 32, 33, 34})
 KEEP_CODES = frozenset({0, 1, 3})
 _HASH = re.compile(r"^sha256:[0-9a-f]{64}$")
 _HEADER = struct.Struct("<8sI32s")
-_MAPPING_MAGIC = b"X4DCBM2\0"
+_MAPPING_MAGIC = b"X4DCBM3\0"
 _DECISION_MAGIC = b"X4DCBD3\0"
 MAPPING_DTYPE = np.dtype(
     [
@@ -61,7 +63,8 @@ MAPPING_DTYPE = np.dtype(
         ("raw_point_ordinal", "<u4"),
         ("timestamp_ns", "<u8"),
         ("decision", "u1"),
-        ("reserved", "u1", (7,)),
+        ("intensity", "<f4"),
+        ("reserved", "u1", (3,)),
     ]
 )
 _MAX_U32 = (1 << 32) - 1
@@ -106,6 +109,21 @@ def _bounds(value: Any, path: str) -> None:
         raise ValueError(f"{path} is inverted")
 
 
+def _intensity_range(value: Any, path: str) -> None:
+    if (
+        not isinstance(value, list)
+        or len(value) != 2
+        or any(
+            isinstance(v, bool)
+            or not isinstance(v, (int, float))
+            or not math.isfinite(v)
+            for v in value
+        )
+        or value[0] > value[1]
+    ):
+        raise ValueError(f"{path} must contain finite ordered intensity bounds")
+
+
 def _freeze(value: Any) -> Any:
     if isinstance(value, Mapping):
         return MappingProxyType({k: _freeze(v) for k, v in value.items()})
@@ -347,7 +365,16 @@ def parse_clean_basemap_manifest(
     for i, chunk in enumerate(chunks):
         path = f"chunks[{i}]"
         _object(
-            chunk, {"ordinal", "point_count", "bounds", "geometry", "mapping"}, path
+            chunk,
+            {
+                "ordinal",
+                "point_count",
+                "bounds",
+                "intensity_range",
+                "geometry",
+                "mapping",
+            },
+            path,
         )
         if chunk["ordinal"] != i or type(chunk["ordinal"]) is not int:
             raise ValueError("chunk ordinals must be contiguous in canonical order")
@@ -358,6 +385,7 @@ def parse_clean_basemap_manifest(
             f"{path}.point_count",
         )
         _bounds(chunk["bounds"], f"{path}.bounds")
+        _intensity_range(chunk["intensity_range"], f"{path}.intensity_range")
         bounds.append(chunk["bounds"])
         _descriptor(chunk["geometry"], "geometry", f"{path}.geometry")
         _descriptor(chunk["mapping"], "mapping", f"{path}.mapping")
@@ -465,6 +493,7 @@ def encode_clean_basemap_mapping(
     raw_point_ordinals: Any,
     timestamps_ns: Any,
     decisions: Any,
+    intensities: Any,
 ) -> CleanBasemapPayload:
     vectors = {
         "sweep_index": _vector(sweep_indices, _MAX_U32, "sweep_indices"),
@@ -485,6 +514,19 @@ def encode_clean_basemap_mapping(
     records = np.zeros(count, dtype=MAPPING_DTYPE)
     for key, vector in vectors.items():
         records[key] = vector
+    intensity = np.asarray(intensities)
+    if (
+        intensity.shape != (count,)
+        or intensity.dtype.kind not in "iuf"
+        or not np.isfinite(intensity).all()
+    ):
+        raise ValueError(
+            "intensities must contain one finite numeric value per geometry point"
+        )
+    with np.errstate(over="ignore", invalid="ignore"):
+        records["intensity"] = intensity
+    if not np.isfinite(records["intensity"]).all():
+        raise ValueError("intensities must be representable as finite float32")
     return _encode(_MAPPING_MAGIC, geometry_raw_sha256, records)
 
 
@@ -509,6 +551,7 @@ def decode_clean_basemap_mapping(
         np.any(records["reserved"] != 0)
         or np.any(~np.isin(records["decision"], tuple(KEEP_CODES)))
         or np.any(records["timestamp_ns"] > _MAX_I64)
+        or not np.isfinite(records["intensity"]).all()
     ):
         raise ValueError(
             "mapping records are noncanonical or contain removed observations"
@@ -602,6 +645,9 @@ def validate_clean_basemap_resource(
     observation_to_world: Callable[
         [Mapping[str, Any], np.ndarray, np.ndarray], np.ndarray
     ],
+    observation_intensity: Callable[
+        [Mapping[str, Any], np.ndarray, np.ndarray], np.ndarray
+    ],
 ) -> CleanBasemapManifest:
     """Verify exact original observations and byte-exact complete scene correspondence."""
     parsed = parse_clean_basemap_manifest(manifest)
@@ -611,13 +657,47 @@ def validate_clean_basemap_resource(
         for sweep in data["sweep_sources"]
     ]
     actual_counts = {str(code): 0 for code in DECISION_CODES}
-    for sweep in data["sweep_sources"]:
+    decisions_cache = OrderedDict()
+    decisions_bytes = 0
+
+    def read_decisions(sweep):
+        nonlocal decisions_bytes
+        descriptor = sweep["decisions"]
+        payload = read_asset(descriptor["path"])
+        # A cached proof never authorizes changed or non-byte carrier data.
+        if (
+            type(payload) is not bytes
+            or len(payload) != descriptor["size_bytes"]
+            or sha256_bytes(payload) != descriptor["sha256"]
+        ):
+            raise ValueError(
+                "clean basemap payload size/hash does not match descriptor"
+            )
+        key = (
+            tuple(sorted(descriptor.items())),
+            sweep["asset_sha256"],
+            sweep["point_count"],
+        )
+        values = decisions_cache.pop(key, None)
+        if values is not None:
+            decisions_cache[key] = values
+            return values
         values = decode_clean_basemap_decisions(
-            read_asset(sweep["decisions"]["path"]),
-            descriptor=sweep["decisions"],
+            payload,
+            descriptor=descriptor,
             asset_sha256=sweep["asset_sha256"],
             point_count=sweep["point_count"],
         )
+        if values.nbytes <= _DECISION_CACHE_BYTES:
+            while decisions_bytes + values.nbytes > _DECISION_CACHE_BYTES:
+                _, evicted = decisions_cache.popitem(last=False)
+                decisions_bytes -= evicted.nbytes
+            decisions_cache[key] = values
+            decisions_bytes += values.nbytes
+        return values
+
+    for sweep in data["sweep_sources"]:
+        values = read_decisions(sweep)
         counts = np.bincount(values, minlength=256)
         for code in DECISION_CODES:
             actual_counts[str(code)] += int(counts[code])
@@ -645,6 +725,11 @@ def validate_clean_basemap_resource(
             geometry_raw_sha256=descriptor["raw_sha256"],
             point_count=chunk["point_count"],
         )
+        if [
+            float(mapping["intensity"].min()),
+            float(mapping["intensity"].max()),
+        ] != chunk["intensity_range"]:
+            raise ValueError("basemap intensity range differs from its actual points")
         if np.any(mapping["sweep_index"] >= len(data["sweep_sources"])):
             raise ValueError("mapping source index is outside the pinned source tables")
         for index in np.unique(mapping["sweep_index"]):
@@ -661,12 +746,7 @@ def validate_clean_basemap_resource(
             if np.any(times < low) or np.any(times > high):
                 raise ValueError("mapping exact point time is outside its source sweep")
             _mark_unique(observation_seen[index], ordinals, "original observation")
-            values = decode_clean_basemap_decisions(
-                read_asset(sweep["decisions"]["path"]),
-                descriptor=sweep["decisions"],
-                asset_sha256=sweep["asset_sha256"],
-                point_count=sweep["point_count"],
-            )
+            values = read_decisions(sweep)
             if not np.array_equal(values[ordinals], mapping["decision"][selected]):
                 raise ValueError(
                     "mapping decision does not match its original observation"
@@ -684,6 +764,16 @@ def validate_clean_basemap_resource(
                 raise ValueError(
                     "basemap geometry differs from its original observations"
                 )
+            intensity = np.asarray(observation_intensity(sweep, ordinals, times))
+            if (
+                intensity.shape != ordinals.shape
+                or intensity.dtype.kind not in "iuf"
+                or not np.isfinite(intensity).all()
+                or not np.array_equal(intensity, mapping["intensity"][selected])
+            ):
+                raise ValueError(
+                    "basemap intensity differs from its original observations"
+                )
     return parsed
 
 

{x4d_devkit-0.46.0rc2 → x4d_devkit-0.47.0rc1}/x4d_devkit/client/reconstruction.py

@@ -307,7 +307,7 @@ class ReconstructionAPI:
                 "clean basemap chunk ordinal is outside the exact resource"
             )
         chunk = chunks[chunk_ordinal]
-        return decode_clean_basemap_mapping(
+        mapping = decode_clean_basemap_mapping(
             self._client.assets.fetch(
                 state.asset_urls[chunk["mapping"]["path"]]
             ).content,
@@ -315,6 +315,14 @@ class ReconstructionAPI:
             geometry_raw_sha256=chunk["geometry"]["raw_sha256"],
             point_count=chunk["point_count"],
         )
+        if [
+            float(mapping["intensity"].min()),
+            float(mapping["intensity"].max()),
+        ] != list(chunk["intensity_range"]):
+            raise ValueError(
+                "clean basemap intensity range differs from its descriptor"
+            )
+        return mapping
 
     def cancel_clean_basemap_task(
         self, *, project_id: int, task_id: str

{x4d_devkit-0.46.0rc2 → x4d_devkit-0.47.0rc1/x4d_devkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.46.0rc2
+Version: 0.47.0rc1
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0