x4d-devkit 0.42.0__tar.gz → 0.43.0rc1__tar.gz

{x4d_devkit-0.42.0 → x4d_devkit-0.43.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.42.0
+Version: 0.43.0rc1
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0

{x4d_devkit-0.42.0 → x4d_devkit-0.43.0rc1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.42.0"
+version = "0.43.0rc1"
 description = "Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation"
 readme = "README.md"
 license = "Apache-2.0"

{x4d_devkit-0.42.0 → x4d_devkit-0.43.0rc1}/tests/test_annotation_contract.py

@@ -31,6 +31,7 @@ def _annotation(
             "rotation": {"qx": 0.0, "qy": 0.0, "qz": 0.0, "qw": 1.0},
         },
         "velocity": None,
+        "motion": "auto",
         "attributes": {},
     }
 
@@ -125,6 +126,7 @@ def test_derive_annotation_views_uses_sample_time_not_input_order():
         {
             "instance_token": "instance-0",
             "category": "car",
+            "motion": "auto",
             "num_annotations": 2,
             "first_annotation_token": "annotation-z",
             "last_annotation_token": "annotation-a",
@@ -191,10 +193,53 @@ def test_annotation_documents_and_derived_views_are_recursively_immutable():
         ),
     ],
 )
-def test_derive_annotation_views_rejects_ambiguous_tracks(
-    annotations, samples, code
-):
+def test_derive_annotation_views_rejects_ambiguous_tracks(annotations, samples, code):
     with pytest.raises(AnnotationContractError) as exc:
         derive_annotation_views(copy.deepcopy(annotations), copy.deepcopy(samples))
 
     assert exc.value.code == code
+
+
+@pytest.mark.parametrize(
+    "motion", [None, True, 1, [], {}, "unknown", "Static", " static"]
+)
+def test_motion_is_explicit_and_canonical(motion):
+    row = _annotation("annotation-0", "sample-0")
+    row["motion"] = motion
+    with pytest.raises(AnnotationContractError) as exc:
+        build_clip_annotations("clip-0", "base_link", [row])
+    assert exc.value.code == "invalid_annotation_motion"
+
+
+def test_motion_changes_revision_and_is_shared_by_the_entire_instance():
+    first = _annotation("annotation-0", "sample-0")
+    second = _annotation("annotation-1", "sample-1")
+    automatic = build_clip_annotations("clip-0", "base_link", [first, second])
+    first["motion"] = "dynamic"
+    with pytest.raises(AnnotationContractError) as exc:
+        build_clip_annotations("clip-0", "base_link", [first, second])
+    assert exc.value.code == "instance_motion_mismatch"
+    second["motion"] = "dynamic"
+    dynamic = build_clip_annotations("clip-0", "base_link", [first, second])
+    assert dynamic.annotation_revision != automatic.annotation_revision
+    assert (
+        derive_annotation_views(
+            dynamic.to_dict()["annotations"], _samples()
+        ).instance_rows()[0]["motion"]
+        == "dynamic"
+    )
+
+
+def test_missing_motion_and_previous_annotation_version_are_rejected():
+    row = _annotation("annotation-0", "sample-0")
+    row.pop("motion")
+    with pytest.raises(AnnotationContractError) as exc:
+        build_clip_annotations("clip-0", "base_link", [row])
+    assert exc.value.code == "invalid_annotation_shape"
+    document = build_clip_annotations(
+        "clip-0", "base_link", [_annotation("annotation-0", "sample-0")]
+    ).to_dict()
+    document["contract_version"] = "x4d-clip-annotations-v3"
+    with pytest.raises(AnnotationContractError) as exc:
+        parse_clip_annotations(document)
+    assert exc.value.code == "invalid_annotation_contract_version"

{x4d_devkit-0.42.0 → x4d_devkit-0.43.0rc1}/tests/test_boundary.py

@@ -20,7 +20,9 @@ def test_devkit_does_not_import_backend_app_modules():
             if isinstance(node, ast.Import):
                 for alias in node.names:
                     if alias.name == "app" or alias.name.startswith("app."):
-                        offenders.append(f"{path.relative_to(ROOT)} imports {alias.name}")
+                        offenders.append(
+                            f"{path.relative_to(ROOT)} imports {alias.name}"
+                        )
             elif isinstance(node, ast.ImportFrom):
                 module = node.module or ""
                 if module == "app" or module.startswith("app."):
@@ -29,7 +31,9 @@ def test_devkit_does_not_import_backend_app_modules():
     assert offenders == []
 
 
-def _help_for(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], *args: str) -> str:
+def _help_for(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], *args: str
+) -> str:
     monkeypatch.setattr(sys, "argv", ["x4d-devkit", *args, "--help"])
     with pytest.raises(SystemExit) as exc:
         cli.main()
@@ -74,6 +78,8 @@ def test_top_level_exports_only_stable_sdk_surface():
     assert set(x4d_devkit.__all__) == {
         "__version__",
         "ANNOTATION_CONTRACT_VERSION",
+        "CLEAN_BASEMAP_CONTRACT_VERSION",
+        "CleanBasemapManifest",
         "Annotation",
         "AnnotationContractError",
         "AssetRef",
@@ -136,6 +142,7 @@ def test_top_level_exports_only_stable_sdk_surface():
         "load_semantic_scene_component",
         "parse_clip_work_session",
         "parse_clip_annotations",
+        "parse_clean_basemap_manifest",
         "parse_clip_label_schema_snapshot",
         "parse_semantic_scene_authoring_scene_seed_index",
         "parse_semantic_scene_authoring_submit",
@@ -146,6 +153,7 @@ def test_top_level_exports_only_stable_sdk_surface():
         "transform_concat_v1",
         "transform_concat_source_v1",
         "validate_clip",
+        "validate_clean_basemap_resource",
         "validate_semantic_scene_authoring_semantic_upload",
         "validate_platform_pose_tokens",
     }

x4d_devkit-0.43.0rc1/tests/test_clean_basemap.py

@@ -0,0 +1,391 @@
+import copy
+
+import numpy as np
+import pytest
+import zstandard
+
+from tests.semantic_scene_contract_fixtures import current_semantic_scene_component
+from x4d_devkit.annotation_contract import build_clip_annotations
+from x4d_devkit.clean_basemap import (
+    CLEAN_BASEMAP_CONTRACT_VERSION,
+    CLEAN_BASEMAP_INPUT_FIELDS,
+    DECISION_CODES,
+    KEEP_GROUND,
+    MAPPING_DTYPE,
+    clean_basemap_scene_source,
+    compute_clean_basemap_dependency_identity,
+    compute_clean_basemap_revision,
+    decode_clean_basemap_decisions,
+    decode_clean_basemap_mapping,
+    encode_clean_basemap_decisions,
+    encode_clean_basemap_mapping,
+    parse_clean_basemap_manifest,
+    validate_clean_basemap_annotation,
+    validate_clean_basemap_native_binding,
+    validate_clean_basemap_resource,
+)
+from x4d_devkit.semantic_scene_contract import (
+    compute_annotation_dependency_revisions,
+    decode_scene_geometry,
+    encode_scene_geometry,
+    sha256_bytes,
+)
+
+
+def fixture():
+    native = current_semantic_scene_component()
+    scene = clean_basemap_scene_source(native["scene_index"])
+    first = scene["chunks"][0]
+    geometry = decode_scene_geometry(
+        native["scene_geometry_payloads"][first["geometry"]],
+        point_count=3,
+        expected_payload_sha256=first["geometry_sha256"],
+        expected_raw_sha256=first["geometry_raw_sha256"],
+    )
+    xyz = np.vstack([geometry.xyz, [9.5, 0, 0]]).astype(np.float32)
+    rgb = np.vstack([geometry.rgb, [100, 110, 120]]).astype(np.uint8)
+    encoded = encode_scene_geometry(xyz, rgb)
+    descriptor = {
+        "path": f"generated/reconstruction/clean_basemap/geometry/{encoded.payload_sha256[7:]}.spc.zst",
+        "sha256": encoded.payload_sha256,
+        "raw_sha256": encoded.raw_sha256,
+        "size_bytes": len(encoded.payload),
+    }
+    decisions = [1, 0, 0, 2, 17]
+    source_hash = "sha256:" + "a" * 64
+    decision_payload = encode_clean_basemap_decisions(
+        asset_sha256=source_hash, decisions=decisions
+    )
+    mapping = encode_clean_basemap_mapping(
+        geometry_raw_sha256=encoded.raw_sha256,
+        sweep_indices=[0, 0, 0, 0],
+        raw_point_ordinals=[0, 1, 2, 3],
+        timestamps_ns=[100, 100, 100, 100],
+        owner_source_indices=[0, 0, 0, 1],
+        owner_point_ordinals=[0, 1, 2, 0],
+        decisions=decisions[:4],
+    )
+    instance_sources = [
+        {
+            key: row[key]
+            for key in (
+                "instance_token",
+                "point_count",
+                "geometry",
+                "geometry_sha256",
+                "geometry_raw_sha256",
+            )
+        }
+        | {"motion": "static"}
+        for row in native["instance_index"]["instances"]
+    ]
+    profile = {
+        "algorithm": {"id": "clean_static_basemap", "version": "1.0.0"},
+        "voxel_size_m": 0.03,
+        "min_range_m": 2.0,
+        "max_range_m": 60.0,
+        "lidar_channels": ["scanner-a"],
+        "unknown_policy": "retain",
+        "point_deskew": True,
+        "foreground_policy": "joint_temporal_box_ground_v1",
+        "occlusion_policy": "original_observations",
+        "policies": {
+            name: {"id": name, "version": "1.0.0", "parameters": {}}
+            for name in ("geometry", "ground", "motion", "rgb")
+        },
+    }
+    inputs = {key: "sha256:" + "b" * 64 for key in CLEAN_BASEMAP_INPUT_FIELDS}
+    dependencies = compute_annotation_dependency_revisions(native["annotation"])
+    for key in inputs:
+        if key in dependencies:
+            inputs[key] = dependencies[key]
+    manifest = {
+        "contract_version": CLEAN_BASEMAP_CONTRACT_VERSION,
+        "clip_id": native["annotation"]["clip_id"],
+        "frame_id": "clip_world",
+        "basemap_revision": "sha256:" + "0" * 64,
+        "dependency_identity": compute_clean_basemap_dependency_identity(
+            inputs, profile
+        ),
+        "input_identity": inputs,
+        "profile": profile,
+        "point_count": 4,
+        "bounds": [xyz.min(0).tolist(), xyz.max(0).tolist()],
+        "scene_source": scene,
+        "instance_sources": instance_sources,
+        "sweep_sources": [
+            {
+                "sample_data_token": "sweep-0",
+                "channel": "scanner-a",
+                "point_count": 5,
+                "asset_sha256": source_hash,
+                "time_bounds": [{"sec": 0, "nanosec": 100}, {"sec": 0, "nanosec": 100}],
+                "decisions": decision_payload.descriptor("decisions"),
+            }
+        ],
+        "chunks": [
+            {
+                "ordinal": 0,
+                "point_count": 4,
+                "bounds": [xyz.min(0).tolist(), xyz.max(0).tolist()],
+                "geometry": descriptor,
+                "mapping": mapping.descriptor("mapping"),
+            }
+        ],
+        "diagnostics": {
+            "source_point_count": 5,
+            "decision_counts": {
+                str(code): decisions.count(code) for code in DECISION_CODES
+            },
+        },
+    }
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+    assets = dict(native["scene_geometry_payloads"], **native["instance_payloads"])
+    assets.update(
+        {
+            descriptor["path"]: encoded.payload,
+            mapping.descriptor("mapping")["path"]: mapping.payload,
+            decision_payload.descriptor("decisions")["path"]: decision_payload.payload,
+        }
+    )
+    return manifest, assets, native
+
+
+def validate(manifest, assets):
+    geometry = manifest["chunks"][0]["geometry"]
+    original_xyz = decode_scene_geometry(
+        assets[geometry["path"]],
+        point_count=4,
+        expected_payload_sha256=geometry["sha256"],
+        expected_raw_sha256=geometry["raw_sha256"],
+    ).xyz
+    return validate_clean_basemap_resource(
+        manifest,
+        read_asset=assets.__getitem__,
+        observation_to_world=lambda sweep, ordinals, times: original_xyz[ordinals],
+        instance_to_world=lambda token, times, xyz: xyz + [10, 0, 0],
+    )
+
+
+def remap(manifest, assets, **overrides):
+    chunk = manifest["chunks"][0]
+    mapping = decode_clean_basemap_mapping(
+        assets[chunk["mapping"]["path"]],
+        descriptor=chunk["mapping"],
+        geometry_raw_sha256=chunk["geometry"]["raw_sha256"],
+        point_count=chunk["point_count"],
+    )
+    arguments = {
+        "sweep_indices": mapping["sweep_index"],
+        "raw_point_ordinals": mapping["raw_point_ordinal"],
+        "timestamps_ns": mapping["timestamp_ns"],
+        "owner_source_indices": mapping["owner_source_index"],
+        "owner_point_ordinals": mapping["owner_point_ordinal"],
+        "decisions": mapping["decision"],
+    }
+    payload = encode_clean_basemap_mapping(
+        geometry_raw_sha256=chunk["geometry"]["raw_sha256"], **(arguments | overrides)
+    )
+    chunk["mapping"] = payload.descriptor("mapping")
+    assets[chunk["mapping"]["path"]] = payload.payload
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+
+
+def test_complete_geometry_native_ownership_and_original_observation_roundtrip():
+    manifest, assets, native = fixture()
+    parsed = validate(manifest, assets)
+    assert parsed.to_dict() == manifest
+    validate_clean_basemap_annotation(parsed, native["annotation"])
+    validate_clean_basemap_native_binding(
+        parsed,
+        scene_index=native["scene_index"],
+        instance_index=native["instance_index"],
+    )
+    manifest["chunks"][0]["point_count"] = 9
+    assert parsed.data["chunks"][0]["point_count"] == 4
+    with pytest.raises(TypeError):
+        parsed.data["profile"]["voxel_size_m"] = 1
+
+
+@pytest.mark.parametrize(
+    "overrides, message",
+    [
+        ({"sweep_indices": [1, 0, 0, 0]}, "source index"),
+        ({"raw_point_ordinals": [5, 1, 2, 3]}, "original point ordinal"),
+        ({"raw_point_ordinals": [0, 0, 2, 3]}, "duplicate original observation"),
+        ({"raw_point_ordinals": [0, 1, 2, 4]}, "decision"),
+        ({"timestamps_ns": [101, 100, 100, 100]}, "exact point time"),
+        ({"owner_source_indices": [9, 0, 0, 1]}, "source index"),
+        ({"owner_point_ordinals": [0, 1, 3, 0]}, "native point ordinal"),
+        ({"owner_point_ordinals": [0, 1, 1, 0]}, "duplicate native scene"),
+        ({"owner_source_indices": [0, 0, 1, 1]}, "instance correspondence"),
+    ],
+)
+def test_valid_hash_cannot_hide_wrong_addresses_or_removed_sources(overrides, message):
+    manifest, assets, _ = fixture()
+    remap(manifest, assets, **overrides)
+    with pytest.raises(ValueError, match=message):
+        validate(manifest, assets)
+
+
+def test_dynamic_instance_support_is_not_static_display_evidence():
+    manifest, assets, _ = fixture()
+    manifest["instance_sources"][0]["motion"] = "dynamic"
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+    with pytest.raises(ValueError, match="dynamic instance"):
+        validate(manifest, assets)
+
+
+@pytest.mark.parametrize("result", [np.zeros((2, 3)), np.full((1, 3), np.nan)])
+def test_instance_transform_rejects_incomplete_or_nonfinite_results(result):
+    manifest, assets, _ = fixture()
+    descriptor = manifest["chunks"][0]["geometry"]
+    original_xyz = decode_scene_geometry(
+        assets[descriptor["path"]],
+        point_count=4,
+        expected_payload_sha256=descriptor["sha256"],
+        expected_raw_sha256=descriptor["raw_sha256"],
+    ).xyz
+    with pytest.raises(ValueError, match="pinned time-aligned track"):
+        validate_clean_basemap_resource(
+            manifest,
+            read_asset=assets.__getitem__,
+            observation_to_world=lambda sweep, ordinals, times: original_xyz[ordinals],
+            instance_to_world=lambda token, times, xyz: result,
+        )
+
+
+def test_native_geometry_agreement_cannot_hide_fabricated_original_observations():
+    manifest, assets, _ = fixture()
+    with pytest.raises(ValueError, match="differs from its original observations"):
+        validate_clean_basemap_resource(
+            manifest,
+            read_asset=assets.__getitem__,
+            observation_to_world=lambda sweep, ordinals, times: np.full(
+                (len(ordinals), 3), 999.0
+            ),
+            instance_to_world=lambda token, times, xyz: xyz + [10, 0, 0],
+        )
+
+
+def test_pose_coverage_rejections_are_accounted_and_cannot_be_mapped():
+    manifest, assets, _ = fixture()
+    sweep = manifest["sweep_sources"][0]
+    carrier = encode_clean_basemap_decisions(
+        asset_sha256=sweep["asset_sha256"], decisions=[1, 0, 0, 2, 34]
+    )
+    sweep["decisions"] = carrier.descriptor("decisions")
+    assets[sweep["decisions"]["path"]] = carrier.payload
+    manifest["diagnostics"]["decision_counts"].update({"17": 0, "34": 1})
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+    assert decode_clean_basemap_decisions(
+        carrier.payload,
+        descriptor=sweep["decisions"],
+        asset_sha256=sweep["asset_sha256"],
+        point_count=5,
+    ).tolist() == [1, 0, 0, 2, 34]
+    validate(manifest, assets)
+    remap(manifest, assets, raw_point_ordinals=[0, 1, 2, 4])
+    with pytest.raises(ValueError, match="decision"):
+        validate(manifest, assets)
+
+
+def test_auto_track_regrouping_invalidates_temporal_ownership():
+    _, _, native = fixture()
+    original = compute_annotation_dependency_revisions(native["annotation"])
+    rows = copy.deepcopy(native["annotation"]["annotations"])
+    rows[0]["instance_token"] = "regrouped-track"
+    edited = build_clip_annotations(
+        native["annotation"]["clip_id"],
+        native["annotation"]["annotation_frame_id"],
+        rows,
+    )
+    assert (
+        compute_annotation_dependency_revisions(edited)["scene_ownership_revision"]
+        != original["scene_ownership_revision"]
+    )
+
+
+def test_native_label_edit_keeps_geometry_binding_but_motion_or_source_edit_does_not():
+    manifest, _, native = fixture()
+    edited = copy.deepcopy(native["annotation"]["annotations"])
+    edited[0]["motion"] = "dynamic"
+    annotation = build_clip_annotations(
+        native["annotation"]["clip_id"],
+        native["annotation"]["annotation_frame_id"],
+        edited,
+    ).to_dict()
+    with pytest.raises(ValueError, match="dependencies"):
+        validate_clean_basemap_annotation(manifest, annotation)
+    scene = copy.deepcopy(native["scene_index"])
+    scene["chunks"][0]["semantic_histogram"] = [{"code": 0, "count": 3}]
+    # Existing native parsing enforces the independent label revision too.
+    from x4d_devkit.semantic_scene_contract import compute_scene_semantics_revision
+
+    scene["scene_semantics_revision"] = compute_scene_semantics_revision(scene)
+    assert clean_basemap_scene_source(scene) == manifest["scene_source"]
+
+
+@pytest.mark.parametrize(
+    "edit",
+    [
+        lambda m: m.update(extra=True),
+        lambda m: m.update(point_count=True),
+        lambda m: m["input_identity"].update(legacy="sha256:" + "f" * 64),
+        lambda m: m["chunks"][0]["geometry"].update(path="../other.spc.zst"),
+        lambda m: m["chunks"][0].update(ordinal=1),
+        lambda m: m["chunks"].append(copy.deepcopy(m["chunks"][0])),
+        lambda m: m["diagnostics"]["decision_counts"].update({"17": 0}),
+        lambda m: m["instance_sources"].reverse(),
+        lambda m: m["sweep_sources"].append(copy.deepcopy(m["sweep_sources"][0])),
+        lambda m: m["profile"].update(unknown_policy="delete"),
+        lambda m: m["profile"].update(point_deskew=False),
+    ],
+)
+def test_manifest_rejects_partial_unknown_and_noncanonical_shapes(edit):
+    manifest, _, _ = fixture()
+    edit(manifest)
+    manifest["basemap_revision"] = compute_clean_basemap_revision(manifest)
+    with pytest.raises(ValueError):
+        parse_clean_basemap_manifest(manifest)
+
+
+def test_decoder_rejects_cross_geometry_reserved_bytes_and_extra_frames():
+    manifest, assets, _ = fixture()
+    chunk = manifest["chunks"][0]
+    payload = assets[chunk["mapping"]["path"]]
+    with pytest.raises(ValueError, match="different source geometry"):
+        decode_clean_basemap_mapping(
+            payload,
+            descriptor=chunk["mapping"],
+            geometry_raw_sha256="sha256:" + "f" * 64,
+            point_count=4,
+        )
+    raw = bytearray(zstandard.ZstdDecompressor().decompress(payload))
+    raw[44 + MAPPING_DTYPE.fields["reserved"][1]] = 1
+    malformed = zstandard.ZstdCompressor().compress(raw)
+    for body in (malformed, payload + payload):
+        descriptor = {
+            "path": f"generated/reconstruction/clean_basemap/mapping/{sha256_bytes(body)[7:]}.zst",
+            "sha256": sha256_bytes(body),
+            "raw_sha256": sha256_bytes(bytes(raw)),
+            "size_bytes": len(body),
+        }
+        with pytest.raises(ValueError):
+            decode_clean_basemap_mapping(
+                body,
+                descriptor=descriptor,
+                geometry_raw_sha256=chunk["geometry"]["raw_sha256"],
+                point_count=4,
+            )
+
+
+def test_mapping_encoder_rejects_float_negative_and_removed_observations():
+    manifest, assets, _ = fixture()
+    for changes in (
+        {"raw_point_ordinals": [-1, 1, 2, 3]},
+        {"timestamps_ns": [100.0] * 4},
+        {"decisions": [KEEP_GROUND, 0, 0, 17]},
+    ):
+        with pytest.raises(ValueError):
+            remap(manifest, assets, **changes)