x4d-devkit 0.47.0__tar.gz → 0.48.0rc1__tar.gz

{x4d_devkit-0.47.0 → x4d_devkit-0.48.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.47.0
+Version: 0.48.0rc1
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0
@@ -549,6 +549,9 @@ boundary and complete workflow.
 
 ## Modules
 
+Native 2D image contracts and the read-only typed image client are documented in
+[Native image Clip v1](docs/image-clip-contract.md). Dataset 0.18 3D readers remain strict.
+
 | Module | Description |
 |--------|-------------|
 | `core` | Data models, token generation, coordinate transforms, clip loader |

{x4d_devkit-0.47.0 → x4d_devkit-0.48.0rc1}/README.md

@@ -506,6 +506,9 @@ boundary and complete workflow.
 
 ## Modules
 
+Native 2D image contracts and the read-only typed image client are documented in
+[Native image Clip v1](docs/image-clip-contract.md). Dataset 0.18 3D readers remain strict.
+
 | Module | Description |
 |--------|-------------|
 | `core` | Data models, token generation, coordinate transforms, clip loader |

{x4d_devkit-0.47.0 → x4d_devkit-0.48.0rc1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.47.0"
+version = "0.48.0rc1"
 description = "Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation"
 readme = "README.md"
 license = "Apache-2.0"

{x4d_devkit-0.47.0 → x4d_devkit-0.48.0rc1}/tests/test_boundary.py

@@ -76,6 +76,12 @@ def test_top_level_exports_only_stable_sdk_surface():
     import x4d_devkit
 
     assert set(x4d_devkit.__all__) == {
+        "IMAGE_GEOMETRY_CONTRACT_VERSION",
+        "ImageAttributeRanges",
+        "ImageGeometry",
+        "ImageGeometryError",
+        "parse_image_attribute_ranges",
+        "parse_image_geometry",
         "__version__",
         "VECTOR_MAP_CONTRACT_VERSION",
         "VectorMapDocument",

x4d_devkit-0.48.0rc1/tests/test_category_audit.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import copy
+import json
+from pathlib import Path
+import sys
+
+import pytest
+
+from tests.current_clip_fixtures import current_clip_tables
+from x4d_devkit import cli
+from x4d_devkit.annotation_contract import build_clip_annotations
+from x4d_devkit.audit import audit_category_mapping, collect_category_stats
+from x4d_devkit.training_identity import build_training_identity_manifest
+
+
+def _clip(path: Path, categories: list[tuple[int, str]]) -> Path:
+    """Write valid metadata only: sensor payloads deliberately do not exist."""
+    path.mkdir()
+    tables = current_clip_tables(clip_id=path.name, sample_count=2)
+    template = tables["annotation.json"]["annotations"][0]
+    rows = []
+    for index, (frame, category) in enumerate(categories):
+        row = copy.deepcopy(template)
+        row.update(
+            annotation_token=f"annotation-{index}", sample_token=f"sample-{frame}",
+            instance_token=f"instance-{index}", category=category,
+            attributes={} if category == "pedestrian" else {"vehicle.with_driver": True},
+        )
+        rows.append(row)
+    tables["annotation.json"] = build_clip_annotations(path.name, "base_link", rows).to_dict()
+    for name, content in tables.items():
+        (path / name).write_text(json.dumps(content))
+    return path
+
+
+@pytest.fixture
+def clips(tmp_path):
+    return [
+        _clip(tmp_path / "a", [(0, "car"), (0, "car"), (1, "car"), (0, "truck"), (1, "pedestrian")]),
+        _clip(tmp_path / "b", [(0, "car")]),
+    ]
+
+
+def _manifest(path, clips):
+    entries = []
+    for index, clip in enumerate(clips):
+        annotations = json.loads((clip / "annotation.json").read_text())
+        meta = json.loads((clip / "meta.json").read_text())
+        entries.append({
+            "project_id": 10 + index, "clip_id": clip.name, "source_revision": "source",
+            "samples_revision": "sha256:" + "1" * 64,
+            "sweeps_revision": "sha256:" + "2" * 64,
+            "calibration_revision": "calibration", "pose_revision": "pose",
+            "annotation_revision": annotations["annotation_revision"],
+            "semantic_scene_revision": None, "label_schema": meta["label_schema"],
+            "clip_schema_version": "0.18", "materialization_policy": "full",
+            "clip_dir": clip.name, "split": "train" if index == 0 else "validation",
+        })
+    value = build_training_identity_manifest(
+        projects=[{"project_id": 10 + index, "project_name": f"project-{index}"}
+                  for index in range(len(clips))],
+        clips=entries,
+    )
+    path.write_text(json.dumps(value))
+    return path
+
+
+def test_counts_use_occurrences_and_distinct_samples_without_media(clips):
+    report = collect_category_stats(clip_dirs=clips).to_dict()
+
+    assert report["summary"] == {"clip_count": 2, "sample_count": 4, "annotation_count": 6}
+    assert report["categories"] == [
+        {"category": "car", "annotation_count": 4, "sample_count": 3},
+        {"category": "pedestrian", "annotation_count": 1, "sample_count": 1},
+        {"category": "truck", "annotation_count": 1, "sample_count": 1},
+    ]
+    assert len(report["clips"]) == 2
+    assert report["clips"][1]["summary"]["sample_count"] == 2
+    source = report["clips"][0]["source"]
+    assert source["annotation_revision"].startswith("sha256:")
+    assert source["sample_index_revision"].startswith("sha256:")
+    assert source["sample_ids"] is None
+    assert source["label_schema"]["schema_id"] == "synthetic_road_scene"
+    assert not list(clips[0].rglob("*.bin"))
+
+
+def test_reports_are_deterministic_and_return_isolated_data(clips):
+    stats = collect_category_stats(clip_dirs=clips)
+    report = stats.to_dict()
+    assert report == collect_category_stats(clip_dirs=list(reversed(clips))).to_dict()
+    report["clips"][0]["source"]["label_schema"]["schema_id"] = "tampered"
+    assert stats.to_dict()["clips"][0]["source"]["label_schema"]["schema_id"] == "synthetic_road_scene"
+
+
+def test_many_to_one_mapping_unions_frames_and_separates_explicit_drops(clips):
+    stats = collect_category_stats(clip_dirs=clips)
+    result = audit_category_mapping(stats, {"truck": "Vehicle", "car": "Vehicle", "pedestrian": None, "unused": "Other"})
+
+    assert result["mapped_categories"] == [{
+        "category": "Vehicle", "annotation_count": 5, "sample_count": 3,
+        "raw_categories": ["car", "truck"],
+    }]
+    assert result["dropped_categories"] == [{"category": "pedestrian", "annotation_count": 1, "sample_count": 1}]
+    assert result["unmapped_categories"] == []
+    assert result["unused_mapping_keys"] == ["unused"]
+    assert result["summary"] == {
+        "mapped_annotation_count": 5, "dropped_annotation_count": 1, "unmapped_annotation_count": 0,
+    }
+
+
+def test_case_sensitive_mapping_does_not_infer_unknown_category_policy(clips):
+    stats = collect_category_stats(clip_dirs=clips)
+    result = audit_category_mapping(stats, {"Car": "Vehicle", "pedestrian": None})
+    assert result["summary"] == {
+        "mapped_annotation_count": 0, "dropped_annotation_count": 1, "unmapped_annotation_count": 5,
+    }
+    assert result["unused_mapping_keys"] == ["Car"]
+    assert [row["category"] for row in result["unmapped_categories"]] == ["car", "truck"]
+
+
+def test_empty_annotations_include_selected_empty_frames(tmp_path):
+    clip = _clip(tmp_path / "empty", [])
+    stats = collect_category_stats(clip_dirs=[clip])
+    assert stats.to_dict()["summary"] == {"clip_count": 1, "sample_count": 2, "annotation_count": 0}
+    assert stats.to_dict()["categories"] == []
+    assert audit_category_mapping(stats, {})["mapped_categories"] == []
+
+
+def test_sample_selection_is_exact_and_counts_empty_selected_frames(clips):
+    report = collect_category_stats(
+        clip_dirs=clips, sample_ids={"a": ["sample-1"], "b": ["sample-1"]}
+    ).to_dict()
+    assert report["summary"] == {"clip_count": 2, "sample_count": 2, "annotation_count": 2}
+    assert report["clips"][0]["source"]["sample_ids"] == ["sample-1"]
+    assert report["clips"][1]["summary"]["annotation_count"] == 0
+
+
+@pytest.mark.parametrize("selection, message", [
+    ({"a": ["sample-0"]}, "missing Clip"),
+    ({"a": ["sample-0"], "b": ["sample-0"], "extra": ["sample-0"]}, "outside"),
+    ({"a": ["absent"], "b": ["sample-0"]}, "unknown keyframe"),
+    ({"a": ["sample-0", "sample-0"], "b": ["sample-0"]}, "duplicate"),
+    ({"a": [], "b": ["sample-0"]}, "non-empty"),
+    ({"a": "sample-0", "b": ["sample-0"]}, "token list"),
+    ({"a": [False], "b": ["sample-0"]}, "strings"),
+])
+def test_invalid_sample_selection_fails_instead_of_partial_report(clips, selection, message):
+    with pytest.raises(ValueError, match=message):
+        collect_category_stats(clip_dirs=clips, sample_ids=selection)
+
+
+def test_manifest_resolves_relative_paths_and_exact_split_with_provenance(clips, tmp_path, monkeypatch):
+    manifest = _manifest(tmp_path / "training.json", clips)
+    monkeypatch.chdir("/tmp")
+    report = collect_category_stats(manifest=manifest, split="train").to_dict()
+    assert report["summary"] == {"clip_count": 1, "sample_count": 2, "annotation_count": 5}
+    assert report["manifest"]["path"] == str(manifest)
+    assert report["manifest"]["sha256"].startswith("sha256:")
+    assert report["groups"][0]["project_id"] == 10
+    assert report["groups"][0]["split"] == "train"
+    assert report["clips"][0]["source"]["clip_dir"] == str(clips[0])
+    with pytest.raises(ValueError, match="no Clips matched"):
+        collect_category_stats(manifest=manifest, split="Train")
+
+
+@pytest.mark.parametrize("field,value", [
+    ("annotation_revision", "sha256:" + "0" * 64),
+    ("clip_id", "different-clip"),
+])
+def test_stale_manifest_identity_fails(clips, tmp_path, field, value):
+    manifest = _manifest(tmp_path / "training.json", clips)
+    data = json.loads(manifest.read_text())
+    data["clips"][0][field] = value
+    manifest.write_text(json.dumps(data))
+    with pytest.raises(ValueError, match=field + " does not match"):
+        collect_category_stats(manifest=manifest)
+
+
+def test_stale_manifest_schema_identity_fails(clips, tmp_path):
+    manifest = _manifest(tmp_path / "training.json", clips)
+    data = json.loads(manifest.read_text())
+    data["clips"][0]["label_schema"]["fingerprint"] = "sha256:" + "0" * 64
+    manifest.write_text(json.dumps(data))
+    with pytest.raises(ValueError, match="label_schema does not match"):
+        collect_category_stats(manifest=manifest)
+
+
+@pytest.mark.parametrize("mutation,message", [
+    (lambda data: data.update(manifest_version="x4d-training-identity-v4"), "invalid"),
+    (lambda data: data["clips"][0].pop("clip_dir"), "requires clip_dir"),
+    (lambda data: data["clips"][0].update(ready=False), "not ready"),
+])
+def test_manifest_rejects_legacy_missing_and_unready_inputs(clips, tmp_path, mutation, message):
+    manifest = _manifest(tmp_path / "training.json", clips)
+    data = json.loads(manifest.read_text())
+    mutation(data)
+    manifest.write_text(json.dumps(data))
+    with pytest.raises(ValueError, match=message):
+        collect_category_stats(manifest=manifest)
+
+
+def test_invalid_duplicate_missing_and_legacy_clips_fail(clips, tmp_path):
+    with pytest.raises(ValueError, match="duplicate Clip ID"):
+        collect_category_stats(clip_dirs=[clips[0], clips[0]])
+    with pytest.raises(FileNotFoundError):
+        collect_category_stats(clip_dirs=[tmp_path / "absent"])
+    meta_path = clips[0] / "meta.json"
+    meta = json.loads(meta_path.read_text())
+    meta["schema_version"] = "0.17"
+    meta_path.write_text(json.dumps(meta))
+    with pytest.raises(ValueError, match="contract validation failed"):
+        collect_category_stats(clip_dirs=clips)
+
+
+def test_annotation_schema_validation_cannot_be_bypassed(clips):
+    path = clips[0] / "annotation.json"
+    data = json.loads(path.read_text())
+    data["annotations"][0]["attributes"] = {"not_a_declared_attribute": True}
+    path.write_text(json.dumps(build_clip_annotations("a", "base_link", data["annotations"]).to_dict()))
+    with pytest.raises(ValueError, match="contract validation failed"):
+        collect_category_stats(clip_dirs=clips)
+
+
+@pytest.mark.parametrize("mapping", [{"": "Vehicle"}, {"car ": "Vehicle"}, {"car": ""}, {"car": False}, {"car": 1}, {"car": ["Vehicle"]}, []])
+def test_mapping_rejects_coercions_and_ambiguous_strings(clips, mapping):
+    with pytest.raises((ValueError, TypeError)):
+        audit_category_mapping(collect_category_stats(clip_dirs=clips), mapping)
+
+
+def test_mapping_cannot_reconstruct_sample_overlap_from_aggregate_json(clips):
+    with pytest.raises(TypeError, match="result of collect_category_stats"):
+        audit_category_mapping(collect_category_stats(clip_dirs=clips).to_dict(), {})
+
+
+@pytest.mark.parametrize("kwargs", [{}, {"clip_dirs": []}, {"clip_dirs": "directory"}, {"clip_dirs": [], "manifest": "file"}, {"clip_dirs": [], "split": "train"}])
+def test_input_selection_is_explicit(kwargs):
+    with pytest.raises(ValueError):
+        collect_category_stats(**kwargs)
+
+
+def test_cli_matches_api_for_categories_and_mapping(clips, tmp_path, monkeypatch, capsys):
+    monkeypatch.setattr(sys, "argv", ["x4d-devkit", "dataset", "categories", "--clips", *map(str, clips), "--format", "json"])
+    cli.main()
+    stats = collect_category_stats(clip_dirs=clips)
+    assert json.loads(capsys.readouterr().out) == stats.to_dict()
+    mapping = {"car": "Vehicle", "truck": "Vehicle", "pedestrian": None}
+    path = tmp_path / "mapping.json"
+    path.write_text(json.dumps(mapping))
+    monkeypatch.setattr(sys, "argv", ["x4d-devkit", "dataset", "audit-mapping", "--clips", *map(str, clips), "--mapping", str(path), "--format", "json"])
+    cli.main()
+    assert json.loads(capsys.readouterr().out) == audit_category_mapping(stats, mapping)
+
+
+def test_cli_manifest_sample_selection_and_table(clips, tmp_path, monkeypatch, capsys):
+    manifest = _manifest(tmp_path / "training.json", clips)
+    samples = tmp_path / "samples.json"
+    samples.write_text(json.dumps({"a": ["sample-1"]}))
+    monkeypatch.setattr(sys, "argv", ["x4d-devkit", "dataset", "categories", "--manifest", str(manifest), "--split", "train", "--samples", str(samples)])
+    cli.main()
+    output = capsys.readouterr().out
+    assert "1 Clips, 1 samples, 2 annotations" in output
+    assert "pedestrian" in output
+
+
+def test_cli_rejects_duplicate_json_mapping_keys_without_partial_output(clips, tmp_path, monkeypatch, capsys):
+    path = tmp_path / "mapping.json"
+    path.write_text('{"car":"Vehicle","car":null}')
+    monkeypatch.setattr(sys, "argv", ["x4d-devkit", "dataset", "audit-mapping", "--clips", str(clips[0]), "--mapping", str(path), "--format", "json"])
+    with pytest.raises(SystemExit, match="duplicate JSON field"):
+        cli.main()
+    assert capsys.readouterr().out == ""