x4d-devkit 0.12.0__tar.gz → 0.14.2__tar.gz

{x4d_devkit-0.12.0/x4d_devkit.egg-info → x4d_devkit-0.14.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.12.0
+Version: 0.14.2
 Summary: X-4D dataset format SDK — load, validate, evaluate, and convert autonomous driving datasets
 Author: windzu
 License-Expression: Apache-2.0

{x4d_devkit-0.12.0 → x4d_devkit-0.14.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.12.0"
+version = "0.14.2"
 description = "X-4D dataset format SDK — load, validate, evaluate, and convert autonomous driving datasets"
 readme = "README.md"
 license = "Apache-2.0"

x4d_devkit-0.14.2/tests/test_loader.py

@@ -0,0 +1,253 @@
+"""Integration tests for ClipLoader — requires nuScenes mini data."""
+
+import os
+
+import numpy as np
+import pytest
+
+from x4d_devkit.core.loader import ClipLoader
+from x4d_devkit.core.models import (
+    ClipMeta, CalibratedSensor, EgoPose, Sample,
+    SampleData, Annotation, Instance,
+)
+
+NUSCENES_DATAROOT = "/data/PublicDatasets/nuscenes"
+NUSCENES_AVAILABLE = os.path.isdir(os.path.join(NUSCENES_DATAROOT, "v1.0-mini"))
+
+
+@pytest.fixture(scope="module")
+def clip_dir(tmp_path_factory):
+    """Convert scene-0061 once and return the clip directory path."""
+    if not NUSCENES_AVAILABLE:
+        pytest.skip("nuScenes data not found")
+    from x4d_devkit.converters.nuscenes import NuScenesConverter
+
+    out = tmp_path_factory.mktemp("loader_test")
+    conv = NuScenesConverter(
+        dataroot=NUSCENES_DATAROOT, version="v1.0-mini", output_dir=str(out),
+    )
+    scene = conv.nusc.scene[0]
+    clip_id = conv.convert_scene(scene["token"])
+    return out / "clips" / clip_id
+
+
+@pytest.mark.skipif(not NUSCENES_AVAILABLE, reason="nuScenes data not found")
+class TestClipLoader:
+    def test_load_meta(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        assert isinstance(loader.meta, ClipMeta)
+        assert loader.meta.clip_id.startswith("2018")
+        assert loader.meta.num_keyframes > 0
+
+    def test_samples(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        assert len(loader.samples) > 0
+        assert all(isinstance(s, Sample) for s in loader.samples)
+        timestamps = [s.timestamp_us for s in loader.samples]
+        assert timestamps == sorted(timestamps)
+
+    def test_ego_poses(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        assert len(loader.ego_poses) > 0
+        assert all(isinstance(ep, EgoPose) for ep in loader.ego_poses)
+
+    def test_calibrated_sensors(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        assert len(loader.calibrated_sensors) == 7
+        channels = {cs.channel for cs in loader.calibrated_sensors}
+        assert "CAM_FRONT" in channels
+        assert "LIDAR_TOP" in channels
+
+    def test_annotations_and_instances(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        assert len(loader.annotations) > 0
+        assert len(loader.instances) > 0
+        assert all(isinstance(a, Annotation) for a in loader.annotations)
+        assert all(isinstance(i, Instance) for i in loader.instances)
+
+    def test_get_by_token(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        sample = loader.samples[0]
+        found = loader.get(sample.token)
+        assert found is sample
+
+    def test_get_unknown_token_raises(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        with pytest.raises(KeyError):
+            loader.get("nonexistent_token")
+
+    def test_sample_data_for_channel(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        cam_data = loader.sample_data_for_channel("CAM_FRONT")
+        assert len(cam_data) > 0
+        assert all(sd.channel == "CAM_FRONT" for sd in cam_data)
+        timestamps = [sd.timestamp_us for sd in cam_data]
+        assert timestamps == sorted(timestamps)
+
+    def test_sample_data_for_sample(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        sample = loader.samples[0]
+        sds = loader.sample_data_for_sample(sample.token)
+        assert len(sds) > 0
+        assert all(sd.sample_token == sample.token for sd in sds)
+
+    def test_annotations_for_sample(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        for sample in loader.samples:
+            anns = loader.annotations_for_sample(sample.token)
+            if anns:
+                assert all(a.sample_token == sample.token for a in anns)
+                break
+
+    def test_load_point_cloud(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        lidar_data = loader.sample_data_for_channel("LIDAR_TOP")
+        kf = next(sd for sd in lidar_data if sd.is_keyframe)
+        pc = loader.load_point_cloud(kf)
+        assert isinstance(pc, np.ndarray)
+        assert pc.ndim == 2
+        assert pc.shape[1] == 5  # x, y, z, intensity, ring_index
+        assert pc.shape[0] > 100
+
+    def test_load_image_path(self, clip_dir):
+        loader = ClipLoader(clip_dir)
+        cam_data = loader.sample_data_for_channel("CAM_FRONT")
+        kf = next(sd for sd in cam_data if sd.is_keyframe)
+        path = loader.load_image_path(kf)
+        assert path.is_file()
+        assert path.suffix == ".jpg"
+
+
+# ---------------------------------------------------------------------------
+# Self-contained v0.9 fixture (does not require nuScenes data) — used to test
+# that ego_pose.json is optional in v0.9.
+# ---------------------------------------------------------------------------
+
+import json
+
+
+def _build_v09_clip(clip_dir, *, with_ego_pose: bool):
+    clip_dir.mkdir(parents=True, exist_ok=True)
+    (clip_dir / "samples" / "lidar_top").mkdir(parents=True)
+    ts = 1000000
+
+    (clip_dir / "meta.json").write_text(json.dumps({
+        "clip_id": clip_dir.name,
+        "schema_version": "0.9",
+        "session_id": "s",
+        "vehicle_id": "v01",
+        "capture_time_utc": "2024-01-01T00:00:00Z",
+        "duration_s": 1.0,
+        "keyframe_count": 1,
+        "sweep_count": 0,
+        "annotation_frame_id": "lidar_top",
+        "ego_pose_frame_id": "base_link",
+        "sensors": {
+            "lidar_top": {
+                "modality": "lidar",
+                "frame_id": "lidar_top",
+                "point_format": {
+                    "fields": ["x", "y", "z", "intensity", "ring_index"],
+                    "types": ["float32"] * 5,
+                    "bytes_per_point": 20,
+                },
+            }
+        },
+    }))
+
+    (clip_dir / "calibrated_sensor.json").write_text(json.dumps([
+        {
+            "calibrated_sensor_token": "cs_root",
+            "channel": "base_link",
+            "parent_frame": "base_link",
+            "translation": {"x": 0, "y": 0, "z": 0},
+            "rotation": {"qx": 0, "qy": 0, "qz": 0, "qw": 1},
+            "intrinsics": None,
+            "distortion": None,
+        },
+        {
+            "calibrated_sensor_token": "cs_lidar",
+            "channel": "lidar_top",
+            "parent_frame": "base_link",
+            "translation": {"x": 0, "y": 0, "z": 1.8},
+            "rotation": {"qx": 0, "qy": 0, "qz": 0, "qw": 1},
+            "intrinsics": None,
+            "distortion": None,
+        },
+    ]))
+
+    if with_ego_pose:
+        (clip_dir / "ego_pose.json").write_text(json.dumps([
+            {
+                "ego_pose_token": "ep_0",
+                "timestamp_us": ts,
+                "translation": {"x": 0, "y": 0, "z": 0},
+                "rotation": {"qx": 0, "qy": 0, "qz": 0, "qw": 1},
+            }
+        ]))
+
+    (clip_dir / "sample.json").write_text(json.dumps([
+        {
+            "sample_token": "s_0",
+            "clip_id": clip_dir.name,
+            "timestamp_us": ts,
+            "is_keyframe": True,
+            "prev": None,
+            "next": None,
+        }
+    ]))
+
+    (clip_dir / "sample_data.json").write_text(json.dumps([
+        {
+            "sample_data_token": "sd_0",
+            "sample_token": "s_0",
+            "channel": "lidar_top",
+            "timestamp_us": ts,
+            "file_path": "samples/lidar_top/1000000.bin",
+            "ego_pose_token": "ep_0",
+            "calibrated_sensor_token": "cs_lidar",
+            "is_keyframe": True,
+            "width": None,
+            "height": None,
+            "prev": None,
+            "next": None,
+        }
+    ]))
+
+    (clip_dir / "annotation.json").write_text("[]")
+    (clip_dir / "instance.json").write_text("[]")
+
+    pts = np.zeros((10, 5), dtype=np.float32)
+    pts.tofile(str(clip_dir / "samples" / "lidar_top" / "1000000.bin"))
+    return clip_dir
+
+
+class TestEgoPoseOptional:
+    """v0.9: ego_pose.json is optional (absent = SLAM not run). Self-contained
+    tests that do not require the nuScenes dataset."""
+
+    def test_has_ego_pose_true_when_file_exists(self, tmp_path):
+        clip = _build_v09_clip(tmp_path / "clip_with_pose", with_ego_pose=True)
+        loader = ClipLoader(clip)
+        assert loader.has_ego_pose is True
+        assert len(loader.ego_poses) == 1
+
+    def test_has_ego_pose_false_when_file_absent(self, tmp_path):
+        clip = _build_v09_clip(tmp_path / "clip_no_pose", with_ego_pose=False)
+        loader = ClipLoader(clip)
+        assert loader.has_ego_pose is False
+        assert loader.ego_poses == []
+
+    def test_clip_world_transform_raises_without_pose(self, tmp_path):
+        from x4d_devkit.core.loader import CLIP_WORLD_FRAME_ID
+        clip = _build_v09_clip(tmp_path / "clip_no_pose2", with_ego_pose=False)
+        loader = ClipLoader(clip)
+        sd = next(s for s in loader.sample_data_for_channel("lidar_top") if s.is_keyframe)
+        with pytest.raises(ValueError, match="has_ego_pose"):
+            loader.get_transform(loader.sensor_frame_id(sd), CLIP_WORLD_FRAME_ID, sd=sd)
+
+    def test_sensor_to_sensor_transform_works_without_pose(self, tmp_path):
+        clip = _build_v09_clip(tmp_path / "clip_no_pose3", with_ego_pose=False)
+        loader = ClipLoader(clip)
+        T = loader.get_transform("lidar_top", "base_link")
+        assert T.as_matrix.shape == (4, 4)

{x4d_devkit-0.12.0 → x4d_devkit-0.14.2}/tests/test_v06_schema.py

@@ -95,13 +95,13 @@ def test_sensor_info_no_frame_id_omits_key_in_to_dict():
     assert "frame_id" not in info.to_dict()
 
 
-def test_clip_meta_annotation_frame_id_default_none():
+def test_clip_meta_annotation_source_channel_default_none():
     meta = ClipMeta(
         clip_id="x", schema_version="0.5", session_id="s", vehicle_id="v",
         capture_time_utc="t", duration_s=0.0, num_keyframes=0, num_sweeps=0,
         sensors={},
     )
-    assert meta.annotation_frame_id is None
+    assert meta.annotation_source_channel is None
 
 
 # --- Loader: annotation_frame_id property ---

x4d_devkit-0.14.2/tests/test_v10_schema.py

@@ -0,0 +1,238 @@
+"""Tests for v0.10 schema: annotation_source_channel replaces annotation_frame_id.
+
+Covers:
+    - ClipMeta.annotation_source_channel round-trip
+    - ClipLoader.annotation_source_channel + derived annotation_frame_id
+    - Reverse-derivation back-compat for v0.6 – v0.9 clips
+    - Validator: required field, must index meta.sensors, must point at lidar
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from x4d_devkit import ClipLoader, validate_clip
+
+
+def _q_from_yaw(yaw: float) -> dict[str, float]:
+    return {"qx": 0.0, "qy": 0.0, "qz": math.sin(yaw / 2), "qw": math.cos(yaw / 2)}
+
+
+def _identity_calib(channel: str, modality: str = "lidar") -> dict:
+    return {
+        "calibrated_sensor_token": f"cs-{channel}",
+        "channel": channel,
+        "parent_frame": channel,
+        "translation": {"x": 0, "y": 0, "z": 0},
+        "rotation": {"qx": 0, "qy": 0, "qz": 0, "qw": 1},
+        "intrinsics": None,
+        "distortion": None,
+    }
+
+
+def _write_v10_clip(
+    clip_dir: Path,
+    *,
+    annotation_source_channel: str | None,
+    sensors_meta: dict,
+    calibrated: list[dict],
+    samples: list[dict] | None = None,
+    sample_data: list[dict] | None = None,
+    annotations: list[dict] | None = None,
+    extra_meta: dict | None = None,
+) -> None:
+    clip_dir.mkdir(parents=True, exist_ok=True)
+    meta: dict = {
+        "clip_id": "test-v10",
+        "schema_version": "0.10",
+        "session_id": "s",
+        "vehicle_id": "v",
+        "capture_time_utc": "2026-05-11T00:00:00Z",
+        "duration_s": 0.0,
+        "keyframe_count": len(samples or []) or 1,
+        "sweep_count": 0,
+        "sensors": sensors_meta,
+    }
+    if annotation_source_channel is not None:
+        meta["annotation_source_channel"] = annotation_source_channel
+    if extra_meta:
+        meta.update(extra_meta)
+    (clip_dir / "meta.json").write_text(json.dumps(meta))
+    (clip_dir / "calibrated_sensor.json").write_text(json.dumps(calibrated))
+    (clip_dir / "sample.json").write_text(json.dumps(samples or [{
+        "sample_token": "s-0", "timestamp_us": 0, "is_keyframe": True,
+        "prev": "", "next": "",
+    }]))
+    (clip_dir / "sample_data.json").write_text(json.dumps(sample_data or [{
+        "sample_data_token": "sd-0",
+        "sample_token": "s-0",
+        "channel": calibrated[0]["channel"],
+        "timestamp_us": 0,
+        "file_path": "samples/x/0.bin",
+        "ego_pose_token": "ep-0",
+        "calibrated_sensor_token": calibrated[0]["calibrated_sensor_token"],
+        "is_keyframe": True,
+        "width": None, "height": None,
+        "prev": "", "next": "",
+    }]))
+    (clip_dir / "annotation.json").write_text(json.dumps(annotations or []))
+    (clip_dir / "instance.json").write_text(json.dumps([]))
+
+
+# --- Model + loader: v0.10 round-trip ---
+
+def test_loader_annotation_source_channel_explicit(tmp_path):
+    _write_v10_clip(
+        tmp_path,
+        annotation_source_channel="lidar",
+        sensors_meta={"lidar": {"modality": "lidar", "frame_id": "base_link",
+                                 "point_format": {"fields": ["x", "y", "z"],
+                                                  "types": ["float32"] * 3,
+                                                  "bytes_per_point": 12}}},
+        calibrated=[_identity_calib("base_link")],
+    )
+    loader = ClipLoader(tmp_path)
+    assert loader.annotation_source_channel == "lidar"
+    assert loader.annotation_frame_id == "base_link"  # derived
+
+
+def test_loader_derived_frame_id_falls_back_to_channel_when_no_frame_id(tmp_path):
+    """Sensor without explicit frame_id falls back to channel name (v0.4 convention)."""
+    _write_v10_clip(
+        tmp_path,
+        annotation_source_channel="lidar_top",
+        sensors_meta={"lidar_top": {"modality": "lidar"}},  # no frame_id
+        calibrated=[_identity_calib("lidar_top")],
+    )
+    loader = ClipLoader(tmp_path)
+    assert loader.annotation_source_channel == "lidar_top"
+    assert loader.annotation_frame_id == "lidar_top"  # channel ≡ frame_id fallback
+
+
+# --- Back-compat: v0.6 – v0.9 clips reverse-derive annotation_source_channel ---
+
+def test_loader_back_compat_v09_derives_source_channel_from_frame_id(tmp_path):
+    """v0.9 clip with annotation_frame_id but no annotation_source_channel:
+    loader derives the channel by reverse-lookup on (modality=lidar, frame_id match).
+    """
+    (tmp_path / "meta.json").write_text(json.dumps({
+        "clip_id": "back-compat",
+        "schema_version": "0.9",
+        "session_id": "s", "vehicle_id": "v",
+        "capture_time_utc": "2026-05-01T00:00:00Z",
+        "duration_s": 0.0, "keyframe_count": 0, "sweep_count": 0,
+        "annotation_frame_id": "base_link",
+        "sensors": {"lidar": {"modality": "lidar", "frame_id": "base_link"}},
+    }))
+    (tmp_path / "calibrated_sensor.json").write_text(json.dumps([_identity_calib("base_link")]))
+    (tmp_path / "sample.json").write_text(json.dumps([]))
+    (tmp_path / "sample_data.json").write_text(json.dumps([]))
+    (tmp_path / "annotation.json").write_text(json.dumps([]))
+    (tmp_path / "instance.json").write_text(json.dumps([]))
+
+    loader = ClipLoader(tmp_path)
+    assert loader.annotation_source_channel == "lidar"
+    assert loader.annotation_frame_id == "base_link"
+
+
+def test_loader_back_compat_ambiguous_multi_lidar_leaves_source_channel_none(tmp_path):
+    """Two lidars share frame_id=base_link — reverse-lookup ambiguous, no derivation."""
+    (tmp_path / "meta.json").write_text(json.dumps({
+        "clip_id": "ambiguous",
+        "schema_version": "0.9",
+        "session_id": "s", "vehicle_id": "v",
+        "capture_time_utc": "2026-05-01T00:00:00Z",
+        "duration_s": 0.0, "keyframe_count": 0, "sweep_count": 0,
+        "annotation_frame_id": "base_link",
+        "sensors": {
+            "lidar":     {"modality": "lidar", "frame_id": "base_link"},
+            "lidar_raw": {"modality": "lidar", "frame_id": "base_link"},
+        },
+    }))
+    (tmp_path / "calibrated_sensor.json").write_text(json.dumps([_identity_calib("base_link")]))
+    (tmp_path / "sample.json").write_text(json.dumps([]))
+    (tmp_path / "sample_data.json").write_text(json.dumps([]))
+    (tmp_path / "annotation.json").write_text(json.dumps([]))
+    (tmp_path / "instance.json").write_text(json.dumps([]))
+
+    loader = ClipLoader(tmp_path)
+    assert loader.annotation_source_channel is None
+    assert loader.annotation_frame_id == "base_link"  # falls back to pre-v0.6 default
+
+
+# --- Validator: v0.10 invariants ---
+
+def test_validate_v10_missing_annotation_source_channel_errors(tmp_path):
+    _write_v10_clip(
+        tmp_path,
+        annotation_source_channel=None,  # missing on purpose
+        sensors_meta={"lidar": {"modality": "lidar", "frame_id": "base_link"}},
+        calibrated=[_identity_calib("base_link")],
+    )
+    report = validate_clip(tmp_path)
+    assert any(i.check == "annotation_source_channel" for i in report.errors)
+
+
+def test_validate_v10_unknown_source_channel_errors(tmp_path):
+    _write_v10_clip(
+        tmp_path,
+        annotation_source_channel="ghost",
+        sensors_meta={"lidar": {"modality": "lidar", "frame_id": "base_link"}},
+        calibrated=[_identity_calib("base_link")],
+    )
+    report = validate_clip(tmp_path)
+    msgs = [i.message for i in report.errors if i.check == "annotation_source_channel"]
+    assert any("not in meta.sensors" in m for m in msgs)
+
+
+def test_validate_v10_camera_source_channel_errors(tmp_path):
+    _write_v10_clip(
+        tmp_path,
+        annotation_source_channel="cam_front",
+        sensors_meta={"cam_front": {"modality": "camera", "frame_id": "cam_front"}},
+        calibrated=[_identity_calib("cam_front", modality="camera")],
+    )
+    report = validate_clip(tmp_path)
+    msgs = [i.message for i in report.errors if i.check == "annotation_source_channel"]
+    assert any("modality" in m for m in msgs)
+
+
+def test_validate_v10_legacy_annotation_frame_id_removed(tmp_path):
+    """Carrying meta.annotation_frame_id in a v0.10 clip is now an error
+    (the field was removed; use annotation_source_channel)."""
+    _write_v10_clip(
+        tmp_path,
+        annotation_source_channel="lidar",
+        sensors_meta={"lidar": {"modality": "lidar", "frame_id": "base_link"}},
+        calibrated=[_identity_calib("base_link")],
+        extra_meta={"annotation_frame_id": "base_link"},
+    )
+    report = validate_clip(tmp_path)
+    assert any(i.check == "annotation_frame_id_removed" for i in report.errors)
+
+
+def test_validate_v10_passes_minimal_valid(tmp_path):
+    _write_v10_clip(
+        tmp_path,
+        annotation_source_channel="lidar",
+        sensors_meta={"lidar": {"modality": "lidar", "frame_id": "base_link",
+                                 "point_format": {"fields": ["x", "y", "z", "intensity", "t"],
+                                                  "types": ["float32"] * 5,
+                                                  "bytes_per_point": 20}}},
+        calibrated=[_identity_calib("base_link")],
+    )
+    pcd_path = tmp_path / "samples/x/0.bin"
+    pcd_path.parent.mkdir(parents=True, exist_ok=True)
+    np.zeros(5, dtype=np.float32).tofile(pcd_path)
+
+    report = validate_clip(tmp_path)
+    schema_errors = [
+        i for i in report.errors
+        if i.check in {"annotation_source_channel", "annotation_frame_id_removed"}
+    ]
+    assert schema_errors == [], f"unexpected schema errors: {schema_errors}"