x4d-devkit 0.40.0__tar.gz → 0.41.0rc2__tar.gz

{x4d_devkit-0.40.0 → x4d_devkit-0.41.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.40.0
+Version: 0.41.0rc2
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0

{x4d_devkit-0.40.0 → x4d_devkit-0.41.0rc2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.40.0"
+version = "0.41.0rc2"
 description = "Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation"
 readme = "README.md"
 license = "Apache-2.0"

x4d_devkit-0.41.0rc2/tests/test_annotation_reference.py

@@ -0,0 +1,165 @@
+import copy
+import gzip
+import hashlib
+
+import httpx
+import numpy as np
+import pytest
+
+from x4d_devkit import X4DClient
+from x4d_devkit.annotation_reference import (
+    ANNOTATION_REFERENCE_OPERATION,
+    ANNOTATION_REFERENCE_SCHEMA,
+    GEOMETRY_IDENTITY_FIELDS,
+    decode_annotation_reference_chunk,
+    parse_annotation_reference_manifest,
+)
+
+
+def fixture():
+    xyz = np.array([[1.5, -2, 3.25], [4, 0, 5]], dtype="<f4")
+    payload = gzip.compress(xyz.tobytes(), mtime=0)
+    chunk = dict(
+        id="0",
+        uri="/api/v1/files/clips/c/generated/reconstruction/annotation_reference/a/0.bin.gz",
+        sha256="sha256:" + hashlib.sha256(payload).hexdigest(),
+        size_bytes=len(payload),
+        point_count=2,
+        bounds=[xyz.min(0).tolist(), xyz.max(0).tolist()],
+    )
+    overview = dict(
+        chunk, id="overview", uri=chunk["uri"].replace("0.bin", "overview.bin")
+    )
+    manifest = dict(
+        schema_version=ANNOTATION_REFERENCE_SCHEMA,
+        coordinate_frame="clip_world",
+        point_format="xyz-f32-le-gzip",
+        clip_id="c",
+        dependency_identity="sha256:" + "a" * 64,
+        input_identity={
+            name: "sha256:" + "b" * 64 for name in GEOMETRY_IDENTITY_FIELDS
+        },
+        profile=dict(
+            identity="annotation_reference@1",
+            algorithm="annotation_reference@1.0.0",
+            lidar_channels=["arbitrary-scanner"],
+            voxel_size_m=0.03,
+            overview_voxel_size_m=0.3,
+            min_range_m=2.0,
+            max_range_m=60.0,
+            point_deskew=False,
+            foreground_policy="retain_all",
+        ),
+        point_count=2,
+        bounds=chunk["bounds"],
+        chunks=[chunk],
+        overview=overview,
+    )
+    return manifest, payload, xyz.tobytes()
+
+
+def parse(payload, **kwargs):
+    return parse_annotation_reference_manifest(
+        payload, api_url="http://platform", expected_clip_id="c", **kwargs
+    )
+
+
+def test_exact_geometry_roundtrip_and_current_input_binding():
+    payload, compressed, points = fixture()
+    result = parse(payload, expected_input_identity=payload["input_identity"])
+    assert result.to_dict() == payload
+    assert decode_annotation_reference_chunk(result.chunks[0], compressed) == points
+    stale = dict(payload["input_identity"], pose_revision="sha256:" + "c" * 64)
+    with pytest.raises(ValueError, match="current geometry"):
+        parse(payload, expected_input_identity=stale)
+    with pytest.raises(ValueError, match="integrity"):
+        decode_annotation_reference_chunk(result.chunks[0], compressed[:-1] + b"x")
+
+
+@pytest.mark.parametrize(
+    "edit",
+    [
+        lambda m: m.update(point_count=3),
+        lambda m: m["chunks"].append(copy.deepcopy(m["chunks"][0])),
+        lambda m: m["chunks"][0].update(uri="http://untrusted/api/v1/files/x"),
+        lambda m: m["chunks"][0].update(
+            uri=m["chunks"][0]["uri"].replace("clips/c/", "clips/other/")
+        ),
+        lambda m: m["chunks"][0].update(point_count=1_000_001),
+        lambda m: m["chunks"][0].update(bounds=[[0, -2, 3.25], [4, 0, 5]]),
+        lambda m: m["input_identity"].update(annotation_revision="sha256:" + "a" * 64),
+        lambda m: m["profile"].update(foreground_policy="remove_all_boxes"),
+        lambda m: m["profile"].update(point_deskew=True),
+        lambda m: m["profile"].update(lidar_channels=["x", "x"]),
+        lambda m: m.update(coordinate_frame="sensor"),
+    ],
+)
+def test_incompatible_partial_or_cross_clip_reference_is_rejected(edit):
+    payload, _, _ = fixture()
+    edit(payload)
+    with pytest.raises(ValueError):
+        parse(payload)
+
+
+def test_valid_checksum_cannot_hide_extra_points_or_gzip_members():
+    payload, compressed, points = fixture()
+    for body in (gzip.compress(points + points), compressed + compressed):
+        payload["chunks"][0].update(
+            size_bytes=len(body), sha256="sha256:" + hashlib.sha256(body).hexdigest()
+        )
+        parsed = parse(payload)
+        with pytest.raises(ValueError, match="compression"):
+            decode_annotation_reference_chunk(parsed.chunks[0], body)
+
+
+def test_client_routes_and_validates_task_and_clip_binding():
+    manifest, _, _ = fixture()
+    state = dict(
+        operation=ANNOTATION_REFERENCE_OPERATION,
+        project_id=7,
+        clip_id="c",
+        dependency_identity=manifest["dependency_identity"],
+        status="completed",
+        task_id="task",
+        progress=100.0,
+        stage="ready",
+        error=None,
+        current=True,
+        manifest=manifest,
+    )
+    calls = []
+
+    def handler(request):
+        calls.append(request)
+        return httpx.Response(200, json=state)
+
+    client = X4DClient(api_url="http://platform", token="test")
+    client._http.close()
+    client._http = httpx.Client(
+        base_url="http://platform", transport=httpx.MockTransport(handler)
+    )
+    try:
+        assert client.reconstruction.prepare_annotation_reference(
+            project_id=7, clip_id="c"
+        ).current
+        assert (
+            client.reconstruction.get_current_annotation_reference(
+                project_id=7, clip_id="c"
+            ).manifest.point_count
+            == 2
+        )
+        assert (
+            client.reconstruction.get_annotation_reference_task(
+                project_id=7, task_id="task"
+            ).task_id
+            == "task"
+        )
+        assert calls[0].method == "POST"
+        assert calls[1].url.path.endswith("/annotation-reference/current")
+        state["project_id"] = 8
+        with pytest.raises(ValueError, match="project"):
+            client.reconstruction.get_current_annotation_reference(
+                project_id=7, clip_id="c"
+            )
+    finally:
+        client.close()

{x4d_devkit-0.40.0 → x4d_devkit-0.41.0rc2}/tests/test_preannotation_client.py

@@ -50,7 +50,7 @@ def _service() -> dict:
 
 def _run(*, status: str = "queued", reused: bool = False) -> dict:
     succeeded = status == "succeeded"
-    return {
+    result = {
         "run_id": "preann-run-1",
         "request_identity": _digest("a"),
         "project_id": 7,
@@ -76,7 +76,34 @@ def _run(*, status: str = "queued", reused: bool = False) -> dict:
         "created_at": "2026-08-18T08:00:00Z",
         "updated_at": "2026-08-18T08:02:00Z",
         "reused": reused,
+        "attempt_number": 1,
+        "attempt_count": 1,
+        "latest_attempt_id": "preann-run-1",
+        "is_latest_attempt": True,
+        "retry_of_run_id": None,
     }
+    result["attempts"] = [
+        {
+            key: copy.deepcopy(result[key])
+            for key in (
+                "run_id",
+                "attempt_number",
+                "retry_of_run_id",
+                "status",
+                "progress",
+                "error",
+                "candidate_id",
+                "temporal_workflow_id",
+                "actor_id",
+                "started_at",
+                "completed_at",
+                "cancellation_requested_at",
+                "created_at",
+                "updated_at",
+            )
+        }
+    ]
+    return result
 
 
 def _portable_candidate() -> dict:
@@ -144,7 +171,10 @@ def test_control_plane_uses_typed_service_run_cancel_and_candidate_contracts():
         requests.append(request)
         if request.url.path == "/api/v1/preannotation/services":
             return httpx.Response(200, json=[_service()])
-        if request.url.path == "/api/v1/preannotation/runs" and request.method == "POST":
+        if (
+            request.url.path == "/api/v1/preannotation/runs"
+            and request.method == "POST"
+        ):
             return httpx.Response(202, json=_run())
         if request.url.path == "/api/v1/preannotation/runs" and request.method == "GET":
             return httpx.Response(200, json=[_run(status="succeeded")])
@@ -172,9 +202,7 @@ def test_control_plane_uses_typed_service_run_cancel_and_candidate_contracts():
             profile_id="centerpoint-track-v1",
             config={"score_threshold": 0.4},
         )
-        runs = client.preannotation.list_runs(
-            project_id=7, clip_id="clip-1", limit=10
-        )
+        runs = client.preannotation.list_runs(project_id=7, clip_id="clip-1", limit=10)
         fetched = client.preannotation.get_run("preann-run-1")
         cancellation = client.preannotation.cancel_run("preann-run-1")
         candidate = client.preannotation.get_candidate(_digest("d"))
@@ -260,3 +288,72 @@ def test_create_run_rejects_non_i_json_config_before_request():
                 profile_id="centerpoint-track-v1",
                 config={"threshold": float("nan")},
             )
+
+
+def _retried_run():
+    previous = _run(status="failed")["attempts"][0]
+    result = _run(status="succeeded")
+    result.update(
+        run_id="preann-run-2",
+        attempt_number=2,
+        attempt_count=2,
+        latest_attempt_id="preann-run-2",
+        retry_of_run_id="preann-run-1",
+    )
+    result["attempts"] = [
+        previous,
+        {key: copy.deepcopy(result[key]) for key in previous},
+    ]
+    return result
+
+
+def test_retry_history_round_trip_and_historical_run():
+    payload = _retried_run()
+    parsed = PlatformPreannotationRun.model_validate(payload)
+    assert parsed.attempt_count == 2
+    assert parsed.attempts[0].status == PlatformPreannotationRunStatus.FAILED
+    assert parsed.attempts[1].candidate_id == parsed.candidate_id
+    assert (
+        PlatformPreannotationRun.model_validate_json(parsed.model_dump_json()) == parsed
+    )
+    payload.update(payload["attempts"][0])
+    payload["is_latest_attempt"] = False
+    old = PlatformPreannotationRun.model_validate(payload)
+    assert old.run_id == "preann-run-1" and old.latest_attempt_id == "preann-run-2"
+    payload["attempts"] = []
+    assert PlatformPreannotationRun.model_validate(payload).attempt_count == 2
+
+
+@pytest.mark.parametrize(
+    "field,value",
+    [
+        ("attempt_count", 1),
+        ("attempt_number", 1),
+        ("attempt_number", True),
+        ("latest_attempt_id", "preann-missing"),
+        ("is_latest_attempt", False),
+        ("is_latest_attempt", 1),
+        ("retry_of_run_id", None),
+        ("attempts", [_run(status="failed")["attempts"][0]]),
+    ],
+)
+def test_inconsistent_attempt_identity_is_rejected(field, value):
+    payload = _retried_run()
+    payload[field] = value
+    with pytest.raises(ValidationError):
+        PlatformPreannotationRun.model_validate(payload)
+
+
+@pytest.mark.parametrize("mutation", ["unknown", "link", "order", "state"])
+def test_malformed_attempt_history_is_rejected(mutation):
+    payload = _retried_run()
+    if mutation == "unknown":
+        payload["attempts"][0]["unknown_field"] = True
+    elif mutation == "link":
+        payload["attempts"][1]["retry_of_run_id"] = "another-run"
+    elif mutation == "order":
+        payload["attempts"].reverse()
+    else:
+        payload["attempts"][1]["progress"]["stage"] = "different-state"
+    with pytest.raises(ValidationError):
+        PlatformPreannotationRun.model_validate(payload)

{x4d_devkit-0.40.0 → x4d_devkit-0.41.0rc2}/tests/test_sdk_api.py

@@ -11,7 +11,7 @@ from x4d_devkit.eval.matching import MetricData
 
 
 def test_package_version_is_current_release():
-    assert x4d_devkit.__version__ == "0.40.0"
+    assert x4d_devkit.__version__ == "0.41.0rc2"
 
 
 def test_loader_exposes_and_enforces_only_current_schema(tmp_path):

x4d_devkit-0.41.0rc2/x4d_devkit/annotation_reference.py

@@ -0,0 +1,304 @@
+"""Pure point-cloud evidence bound to one Clip's exact geometry inputs.
+
+This optional generated evidence is not a Dataset annotation component.
+"""
+
+from __future__ import annotations
+
+import copy
+from dataclasses import dataclass
+import hashlib
+import math
+import re
+import zlib
+from typing import Any
+from urllib.parse import urlsplit
+
+from x4d_devkit.client.assets import _validated_file_uri
+from x4d_devkit.json_contract import ensure_i_json
+
+
+ANNOTATION_REFERENCE_SCHEMA = "x4d-annotation-reference-v1"
+ANNOTATION_REFERENCE_OPERATION = "reconstruction.prepare_annotation_reference"
+GEOMETRY_IDENTITY_FIELDS = (
+    "samples_revision",
+    "sweeps_revision",
+    "calibration_revision",
+    "pose_revision",
+)
+_HASH = re.compile(r"^sha256:[0-9a-f]{64}$")
+
+
+def _object(value, keys, path):
+    if not isinstance(value, dict) or set(value) != set(keys):
+        raise ValueError(f"{path} must contain exactly {sorted(keys)}")
+    return value
+
+
+def _text(value, path):
+    if not isinstance(value, str) or not value or value.strip() != value:
+        raise ValueError(f"{path} must be canonical text")
+    return value
+
+
+def _positive(value, path):
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        raise ValueError(f"{path} must be a positive integer")
+    return value
+
+
+def _hash(value, path):
+    if not isinstance(value, str) or not _HASH.fullmatch(value):
+        raise ValueError(f"{path} must be a SHA-256 identity")
+    return value
+
+
+def _bounds(value, path):
+    if not isinstance(value, list) or len(value) != 2:
+        raise ValueError(f"{path} must have lower and upper XYZ bounds")
+    for row in value:
+        if (
+            not isinstance(row, list)
+            or len(row) != 3
+            or any(
+                isinstance(v, bool)
+                or not isinstance(v, (int, float))
+                or not math.isfinite(v)
+                for v in row
+            )
+        ):
+            raise ValueError(f"{path} must contain finite XYZ coordinates")
+    if any(low > high for low, high in zip(*value)):
+        raise ValueError(f"{path} is inverted")
+
+
+@dataclass(frozen=True)
+class AnnotationReferenceChunk:
+    id: str
+    uri: str
+    sha256: str
+    size_bytes: int
+    point_count: int
+    bounds: tuple[tuple[float, ...], ...]
+
+    def to_dict(self):
+        return dict(
+            id=self.id,
+            uri=self.uri,
+            sha256=self.sha256,
+            size_bytes=self.size_bytes,
+            point_count=self.point_count,
+            bounds=[list(row) for row in self.bounds],
+        )
+
+
+@dataclass(frozen=True)
+class AnnotationReferenceManifest:
+    clip_id: str
+    dependency_identity: str
+    input_identity: dict[str, str]
+    profile: dict[str, Any]
+    point_count: int
+    bounds: tuple[tuple[float, ...], ...]
+    chunks: tuple[AnnotationReferenceChunk, ...]
+    overview: AnnotationReferenceChunk
+
+    def to_dict(self):
+        return dict(
+            schema_version=ANNOTATION_REFERENCE_SCHEMA,
+            coordinate_frame="clip_world",
+            point_format="xyz-f32-le-gzip",
+            clip_id=self.clip_id,
+            dependency_identity=self.dependency_identity,
+            input_identity=copy.deepcopy(self.input_identity),
+            profile=copy.deepcopy(self.profile),
+            point_count=self.point_count,
+            bounds=[list(row) for row in self.bounds],
+            chunks=[chunk.to_dict() for chunk in self.chunks],
+            overview=self.overview.to_dict(),
+        )
+
+
+def parse_annotation_reference_manifest(
+    payload: Any,
+    *,
+    api_url: str,
+    expected_clip_id: str,
+    expected_dependency_identity: str | None = None,
+    expected_input_identity: dict[str, str] | None = None,
+) -> AnnotationReferenceManifest:
+    value = _object(
+        ensure_i_json(payload),
+        {
+            "schema_version",
+            "coordinate_frame",
+            "point_format",
+            "clip_id",
+            "dependency_identity",
+            "input_identity",
+            "profile",
+            "point_count",
+            "bounds",
+            "chunks",
+            "overview",
+        },
+        "annotation reference",
+    )
+    if (
+        value["schema_version"] != ANNOTATION_REFERENCE_SCHEMA
+        or value["coordinate_frame"] != "clip_world"
+        or value["point_format"] != "xyz-f32-le-gzip"
+    ):
+        raise ValueError("Unsupported annotation reference contract")
+    if value["clip_id"] != expected_clip_id:
+        raise ValueError("Annotation reference belongs to another Clip")
+    dependency = _hash(value["dependency_identity"], "dependency_identity")
+    if (
+        expected_dependency_identity is not None
+        and dependency != expected_dependency_identity
+    ):
+        raise ValueError("Annotation reference dependency changed")
+    identity = _object(
+        value["input_identity"], GEOMETRY_IDENTITY_FIELDS, "input_identity"
+    )
+    for name, item in identity.items():
+        _hash(item, name)
+    if expected_input_identity is not None and identity != expected_input_identity:
+        raise ValueError("Annotation reference does not match the current geometry")
+    profile = _object(
+        value["profile"],
+        {
+            "identity",
+            "algorithm",
+            "lidar_channels",
+            "voxel_size_m",
+            "overview_voxel_size_m",
+            "min_range_m",
+            "max_range_m",
+            "point_deskew",
+            "foreground_policy",
+        },
+        "profile",
+    )
+    _text(profile["identity"], "profile.identity")
+    _text(profile["algorithm"], "profile.algorithm")
+    channels = profile["lidar_channels"]
+    if not isinstance(channels, list) or not channels:
+        raise ValueError("Reference profile requires LiDAR channels")
+    for channel in channels:
+        _text(channel, "lidar channel")
+    if len(set(channels)) != len(channels):
+        raise ValueError("Reference profile has duplicate LiDAR channels")
+    for key in ("voxel_size_m", "overview_voxel_size_m", "min_range_m", "max_range_m"):
+        item = profile[key]
+        if (
+            isinstance(item, bool)
+            or not isinstance(item, (int, float))
+            or not math.isfinite(item)
+            or item < 0
+        ):
+            raise ValueError(f"Invalid reference {key}")
+    if (
+        profile["voxel_size_m"] <= 0
+        or profile["overview_voxel_size_m"] < profile["voxel_size_m"]
+        or profile["max_range_m"] <= profile["min_range_m"]
+    ):
+        raise ValueError("Invalid reference voxel or range ordering")
+    if (
+        profile["point_deskew"] is not False
+        or profile["foreground_policy"] != "retain_all"
+    ):
+        raise ValueError(
+            "Reference v1 requires declared rigid scans and retained foreground"
+        )
+    count = _positive(value["point_count"], "point_count")
+    _bounds(value["bounds"], "bounds")
+
+    def chunk(item):
+        item = _object(
+            item,
+            {"id", "uri", "sha256", "size_bytes", "point_count", "bounds"},
+            "chunk",
+        )
+        _text(item["id"], "chunk.id")
+        uri = _validated_file_uri(item["uri"], api_url=api_url)
+        prefix = f"/api/v1/files/clips/{expected_clip_id}/generated/reconstruction/annotation_reference/"
+        if not urlsplit(uri).path.startswith(prefix):
+            raise ValueError("Reference chunk is outside its Clip evidence namespace")
+        _hash(item["sha256"], "chunk.sha256")
+        _positive(item["size_bytes"], "chunk.size_bytes")
+        n = _positive(item["point_count"], "chunk.point_count")
+        if n > 1_000_000 or item["size_bytes"] > n * 12 + 65536:
+            raise ValueError("Reference chunk exceeds bounded transfer size")
+        _bounds(item["bounds"], "chunk.bounds")
+        if any(
+            item["bounds"][0][a] < value["bounds"][0][a]
+            or item["bounds"][1][a] > value["bounds"][1][a]
+            for a in range(3)
+        ):
+            raise ValueError("Chunk lies outside reference bounds")
+        return AnnotationReferenceChunk(
+            item["id"],
+            uri,
+            item["sha256"],
+            item["size_bytes"],
+            n,
+            tuple(tuple(map(float, row)) for row in item["bounds"]),
+        )
+
+    if not isinstance(value["chunks"], list) or not value["chunks"]:
+        raise ValueError("Reference must declare its complete fine chunks")
+    chunks = tuple(chunk(item) for item in value["chunks"])
+    overview = chunk(value["overview"])
+    all_chunks = (*chunks, overview)
+    if (
+        len({c.id for c in all_chunks}) != len(all_chunks)
+        or len({c.uri for c in all_chunks}) != len(all_chunks)
+        or sum(c.point_count for c in chunks) != count
+        or overview.point_count > count
+    ):
+        raise ValueError(
+            "Reference chunk identities or complete point count are inconsistent"
+        )
+    return AnnotationReferenceManifest(
+        expected_clip_id,
+        dependency,
+        copy.deepcopy(identity),
+        copy.deepcopy(profile),
+        count,
+        tuple(tuple(map(float, row)) for row in value["bounds"]),
+        chunks,
+        overview,
+    )
+
+
+def decode_annotation_reference_chunk(
+    chunk: AnnotationReferenceChunk, payload: bytes
+) -> bytes:
+    """Verify compressed bytes and bound decompression before exposing XYZ."""
+    if (
+        len(payload) != chunk.size_bytes
+        or "sha256:" + hashlib.sha256(payload).hexdigest() != chunk.sha256
+    ):
+        raise ValueError("Annotation reference chunk integrity mismatch")
+    decoder = zlib.decompressobj(16 + zlib.MAX_WBITS)
+    points = decoder.decompress(payload, chunk.point_count * 12 + 1)
+    if (
+        len(points) != chunk.point_count * 12
+        or not decoder.eof
+        or decoder.unused_data
+        or decoder.unconsumed_tail
+    ):
+        raise ValueError(
+            "Annotation reference chunk point count or compression mismatch"
+        )
+    import numpy as np
+
+    xyz = np.frombuffer(points, dtype="<f4").reshape(-1, 3)
+    if (
+        not np.isfinite(xyz).all()
+        or not np.array_equal(xyz.min(axis=0), chunk.bounds[0])
+        or not np.array_equal(xyz.max(axis=0), chunk.bounds[1])
+    ):
+        raise ValueError("Annotation reference chunk geometry disagrees with manifest")
+    return points