x4d-devkit 0.40.0__tar.gz → 0.41.0rc1__tar.gz

{x4d_devkit-0.40.0 → x4d_devkit-0.41.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.40.0
+Version: 0.41.0rc1
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0

{x4d_devkit-0.40.0 → x4d_devkit-0.41.0rc1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.40.0"
+version = "0.41.0rc1"
 description = "Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation"
 readme = "README.md"
 license = "Apache-2.0"

x4d_devkit-0.41.0rc1/tests/test_annotation_reference.py

@@ -0,0 +1,165 @@
+import copy
+import gzip
+import hashlib
+
+import httpx
+import numpy as np
+import pytest
+
+from x4d_devkit import X4DClient
+from x4d_devkit.annotation_reference import (
+    ANNOTATION_REFERENCE_OPERATION,
+    ANNOTATION_REFERENCE_SCHEMA,
+    GEOMETRY_IDENTITY_FIELDS,
+    decode_annotation_reference_chunk,
+    parse_annotation_reference_manifest,
+)
+
+
+def fixture():
+    xyz = np.array([[1.5, -2, 3.25], [4, 0, 5]], dtype="<f4")
+    payload = gzip.compress(xyz.tobytes(), mtime=0)
+    chunk = dict(
+        id="0",
+        uri="/api/v1/files/clips/c/generated/reconstruction/annotation_reference/a/0.bin.gz",
+        sha256="sha256:" + hashlib.sha256(payload).hexdigest(),
+        size_bytes=len(payload),
+        point_count=2,
+        bounds=[xyz.min(0).tolist(), xyz.max(0).tolist()],
+    )
+    overview = dict(
+        chunk, id="overview", uri=chunk["uri"].replace("0.bin", "overview.bin")
+    )
+    manifest = dict(
+        schema_version=ANNOTATION_REFERENCE_SCHEMA,
+        coordinate_frame="clip_world",
+        point_format="xyz-f32-le-gzip",
+        clip_id="c",
+        dependency_identity="sha256:" + "a" * 64,
+        input_identity={
+            name: "sha256:" + "b" * 64 for name in GEOMETRY_IDENTITY_FIELDS
+        },
+        profile=dict(
+            identity="annotation_reference@1",
+            algorithm="annotation_reference@1.0.0",
+            lidar_channels=["arbitrary-scanner"],
+            voxel_size_m=0.03,
+            overview_voxel_size_m=0.3,
+            min_range_m=2.0,
+            max_range_m=60.0,
+            point_deskew=False,
+            foreground_policy="retain_all",
+        ),
+        point_count=2,
+        bounds=chunk["bounds"],
+        chunks=[chunk],
+        overview=overview,
+    )
+    return manifest, payload, xyz.tobytes()
+
+
+def parse(payload, **kwargs):
+    return parse_annotation_reference_manifest(
+        payload, api_url="http://platform", expected_clip_id="c", **kwargs
+    )
+
+
+def test_exact_geometry_roundtrip_and_current_input_binding():
+    payload, compressed, points = fixture()
+    result = parse(payload, expected_input_identity=payload["input_identity"])
+    assert result.to_dict() == payload
+    assert decode_annotation_reference_chunk(result.chunks[0], compressed) == points
+    stale = dict(payload["input_identity"], pose_revision="sha256:" + "c" * 64)
+    with pytest.raises(ValueError, match="current geometry"):
+        parse(payload, expected_input_identity=stale)
+    with pytest.raises(ValueError, match="integrity"):
+        decode_annotation_reference_chunk(result.chunks[0], compressed[:-1] + b"x")
+
+
+@pytest.mark.parametrize(
+    "edit",
+    [
+        lambda m: m.update(point_count=3),
+        lambda m: m["chunks"].append(copy.deepcopy(m["chunks"][0])),
+        lambda m: m["chunks"][0].update(uri="http://untrusted/api/v1/files/x"),
+        lambda m: m["chunks"][0].update(
+            uri=m["chunks"][0]["uri"].replace("clips/c/", "clips/other/")
+        ),
+        lambda m: m["chunks"][0].update(point_count=1_000_001),
+        lambda m: m["chunks"][0].update(bounds=[[0, -2, 3.25], [4, 0, 5]]),
+        lambda m: m["input_identity"].update(annotation_revision="sha256:" + "a" * 64),
+        lambda m: m["profile"].update(foreground_policy="remove_all_boxes"),
+        lambda m: m["profile"].update(point_deskew=True),
+        lambda m: m["profile"].update(lidar_channels=["x", "x"]),
+        lambda m: m.update(coordinate_frame="sensor"),
+    ],
+)
+def test_incompatible_partial_or_cross_clip_reference_is_rejected(edit):
+    payload, _, _ = fixture()
+    edit(payload)
+    with pytest.raises(ValueError):
+        parse(payload)
+
+
+def test_valid_checksum_cannot_hide_extra_points_or_gzip_members():
+    payload, compressed, points = fixture()
+    for body in (gzip.compress(points + points), compressed + compressed):
+        payload["chunks"][0].update(
+            size_bytes=len(body), sha256="sha256:" + hashlib.sha256(body).hexdigest()
+        )
+        parsed = parse(payload)
+        with pytest.raises(ValueError, match="compression"):
+            decode_annotation_reference_chunk(parsed.chunks[0], body)
+
+
+def test_client_routes_and_validates_task_and_clip_binding():
+    manifest, _, _ = fixture()
+    state = dict(
+        operation=ANNOTATION_REFERENCE_OPERATION,
+        project_id=7,
+        clip_id="c",
+        dependency_identity=manifest["dependency_identity"],
+        status="completed",
+        task_id="task",
+        progress=100.0,
+        stage="ready",
+        error=None,
+        current=True,
+        manifest=manifest,
+    )
+    calls = []
+
+    def handler(request):
+        calls.append(request)
+        return httpx.Response(200, json=state)
+
+    client = X4DClient(api_url="http://platform", token="test")
+    client._http.close()
+    client._http = httpx.Client(
+        base_url="http://platform", transport=httpx.MockTransport(handler)
+    )
+    try:
+        assert client.reconstruction.prepare_annotation_reference(
+            project_id=7, clip_id="c"
+        ).current
+        assert (
+            client.reconstruction.get_current_annotation_reference(
+                project_id=7, clip_id="c"
+            ).manifest.point_count
+            == 2
+        )
+        assert (
+            client.reconstruction.get_annotation_reference_task(
+                project_id=7, task_id="task"
+            ).task_id
+            == "task"
+        )
+        assert calls[0].method == "POST"
+        assert calls[1].url.path.endswith("/annotation-reference/current")
+        state["project_id"] = 8
+        with pytest.raises(ValueError, match="project"):
+            client.reconstruction.get_current_annotation_reference(
+                project_id=7, clip_id="c"
+            )
+    finally:
+        client.close()

{x4d_devkit-0.40.0 → x4d_devkit-0.41.0rc1}/tests/test_sdk_api.py

@@ -11,7 +11,7 @@ from x4d_devkit.eval.matching import MetricData
 
 
 def test_package_version_is_current_release():
-    assert x4d_devkit.__version__ == "0.40.0"
+    assert x4d_devkit.__version__ == "0.41.0rc1"
 
 
 def test_loader_exposes_and_enforces_only_current_schema(tmp_path):

x4d_devkit-0.41.0rc1/x4d_devkit/annotation_reference.py

@@ -0,0 +1,304 @@
+"""Pure point-cloud evidence bound to one Clip's exact geometry inputs.
+
+This optional generated evidence is not a Dataset annotation component.
+"""
+
+from __future__ import annotations
+
+import copy
+from dataclasses import dataclass
+import hashlib
+import math
+import re
+import zlib
+from typing import Any
+from urllib.parse import urlsplit
+
+from x4d_devkit.client.assets import _validated_file_uri
+from x4d_devkit.json_contract import ensure_i_json
+
+
+ANNOTATION_REFERENCE_SCHEMA = "x4d-annotation-reference-v1"
+ANNOTATION_REFERENCE_OPERATION = "reconstruction.prepare_annotation_reference"
+GEOMETRY_IDENTITY_FIELDS = (
+    "samples_revision",
+    "sweeps_revision",
+    "calibration_revision",
+    "pose_revision",
+)
+_HASH = re.compile(r"^sha256:[0-9a-f]{64}$")
+
+
+def _object(value, keys, path):
+    if not isinstance(value, dict) or set(value) != set(keys):
+        raise ValueError(f"{path} must contain exactly {sorted(keys)}")
+    return value
+
+
+def _text(value, path):
+    if not isinstance(value, str) or not value or value.strip() != value:
+        raise ValueError(f"{path} must be canonical text")
+    return value
+
+
+def _positive(value, path):
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        raise ValueError(f"{path} must be a positive integer")
+    return value
+
+
+def _hash(value, path):
+    if not isinstance(value, str) or not _HASH.fullmatch(value):
+        raise ValueError(f"{path} must be a SHA-256 identity")
+    return value
+
+
+def _bounds(value, path):
+    if not isinstance(value, list) or len(value) != 2:
+        raise ValueError(f"{path} must have lower and upper XYZ bounds")
+    for row in value:
+        if (
+            not isinstance(row, list)
+            or len(row) != 3
+            or any(
+                isinstance(v, bool)
+                or not isinstance(v, (int, float))
+                or not math.isfinite(v)
+                for v in row
+            )
+        ):
+            raise ValueError(f"{path} must contain finite XYZ coordinates")
+    if any(low > high for low, high in zip(*value)):
+        raise ValueError(f"{path} is inverted")
+
+
+@dataclass(frozen=True)
+class AnnotationReferenceChunk:
+    id: str
+    uri: str
+    sha256: str
+    size_bytes: int
+    point_count: int
+    bounds: tuple[tuple[float, ...], ...]
+
+    def to_dict(self):
+        return dict(
+            id=self.id,
+            uri=self.uri,
+            sha256=self.sha256,
+            size_bytes=self.size_bytes,
+            point_count=self.point_count,
+            bounds=[list(row) for row in self.bounds],
+        )
+
+
+@dataclass(frozen=True)
+class AnnotationReferenceManifest:
+    clip_id: str
+    dependency_identity: str
+    input_identity: dict[str, str]
+    profile: dict[str, Any]
+    point_count: int
+    bounds: tuple[tuple[float, ...], ...]
+    chunks: tuple[AnnotationReferenceChunk, ...]
+    overview: AnnotationReferenceChunk
+
+    def to_dict(self):
+        return dict(
+            schema_version=ANNOTATION_REFERENCE_SCHEMA,
+            coordinate_frame="clip_world",
+            point_format="xyz-f32-le-gzip",
+            clip_id=self.clip_id,
+            dependency_identity=self.dependency_identity,
+            input_identity=copy.deepcopy(self.input_identity),
+            profile=copy.deepcopy(self.profile),
+            point_count=self.point_count,
+            bounds=[list(row) for row in self.bounds],
+            chunks=[chunk.to_dict() for chunk in self.chunks],
+            overview=self.overview.to_dict(),
+        )
+
+
+def parse_annotation_reference_manifest(
+    payload: Any,
+    *,
+    api_url: str,
+    expected_clip_id: str,
+    expected_dependency_identity: str | None = None,
+    expected_input_identity: dict[str, str] | None = None,
+) -> AnnotationReferenceManifest:
+    value = _object(
+        ensure_i_json(payload),
+        {
+            "schema_version",
+            "coordinate_frame",
+            "point_format",
+            "clip_id",
+            "dependency_identity",
+            "input_identity",
+            "profile",
+            "point_count",
+            "bounds",
+            "chunks",
+            "overview",
+        },
+        "annotation reference",
+    )
+    if (
+        value["schema_version"] != ANNOTATION_REFERENCE_SCHEMA
+        or value["coordinate_frame"] != "clip_world"
+        or value["point_format"] != "xyz-f32-le-gzip"
+    ):
+        raise ValueError("Unsupported annotation reference contract")
+    if value["clip_id"] != expected_clip_id:
+        raise ValueError("Annotation reference belongs to another Clip")
+    dependency = _hash(value["dependency_identity"], "dependency_identity")
+    if (
+        expected_dependency_identity is not None
+        and dependency != expected_dependency_identity
+    ):
+        raise ValueError("Annotation reference dependency changed")
+    identity = _object(
+        value["input_identity"], GEOMETRY_IDENTITY_FIELDS, "input_identity"
+    )
+    for name, item in identity.items():
+        _hash(item, name)
+    if expected_input_identity is not None and identity != expected_input_identity:
+        raise ValueError("Annotation reference does not match the current geometry")
+    profile = _object(
+        value["profile"],
+        {
+            "identity",
+            "algorithm",
+            "lidar_channels",
+            "voxel_size_m",
+            "overview_voxel_size_m",
+            "min_range_m",
+            "max_range_m",
+            "point_deskew",
+            "foreground_policy",
+        },
+        "profile",
+    )
+    _text(profile["identity"], "profile.identity")
+    _text(profile["algorithm"], "profile.algorithm")
+    channels = profile["lidar_channels"]
+    if not isinstance(channels, list) or not channels:
+        raise ValueError("Reference profile requires LiDAR channels")
+    for channel in channels:
+        _text(channel, "lidar channel")
+    if len(set(channels)) != len(channels):
+        raise ValueError("Reference profile has duplicate LiDAR channels")
+    for key in ("voxel_size_m", "overview_voxel_size_m", "min_range_m", "max_range_m"):
+        item = profile[key]
+        if (
+            isinstance(item, bool)
+            or not isinstance(item, (int, float))
+            or not math.isfinite(item)
+            or item < 0
+        ):
+            raise ValueError(f"Invalid reference {key}")
+    if (
+        profile["voxel_size_m"] <= 0
+        or profile["overview_voxel_size_m"] < profile["voxel_size_m"]
+        or profile["max_range_m"] <= profile["min_range_m"]
+    ):
+        raise ValueError("Invalid reference voxel or range ordering")
+    if (
+        profile["point_deskew"] is not False
+        or profile["foreground_policy"] != "retain_all"
+    ):
+        raise ValueError(
+            "Reference v1 requires declared rigid scans and retained foreground"
+        )
+    count = _positive(value["point_count"], "point_count")
+    _bounds(value["bounds"], "bounds")
+
+    def chunk(item):
+        item = _object(
+            item,
+            {"id", "uri", "sha256", "size_bytes", "point_count", "bounds"},
+            "chunk",
+        )
+        _text(item["id"], "chunk.id")
+        uri = _validated_file_uri(item["uri"], api_url=api_url)
+        prefix = f"/api/v1/files/clips/{expected_clip_id}/generated/reconstruction/annotation_reference/"
+        if not urlsplit(uri).path.startswith(prefix):
+            raise ValueError("Reference chunk is outside its Clip evidence namespace")
+        _hash(item["sha256"], "chunk.sha256")
+        _positive(item["size_bytes"], "chunk.size_bytes")
+        n = _positive(item["point_count"], "chunk.point_count")
+        if n > 1_000_000 or item["size_bytes"] > n * 12 + 65536:
+            raise ValueError("Reference chunk exceeds bounded transfer size")
+        _bounds(item["bounds"], "chunk.bounds")
+        if any(
+            item["bounds"][0][a] < value["bounds"][0][a]
+            or item["bounds"][1][a] > value["bounds"][1][a]
+            for a in range(3)
+        ):
+            raise ValueError("Chunk lies outside reference bounds")
+        return AnnotationReferenceChunk(
+            item["id"],
+            uri,
+            item["sha256"],
+            item["size_bytes"],
+            n,
+            tuple(tuple(map(float, row)) for row in item["bounds"]),
+        )
+
+    if not isinstance(value["chunks"], list) or not value["chunks"]:
+        raise ValueError("Reference must declare its complete fine chunks")
+    chunks = tuple(chunk(item) for item in value["chunks"])
+    overview = chunk(value["overview"])
+    all_chunks = (*chunks, overview)
+    if (
+        len({c.id for c in all_chunks}) != len(all_chunks)
+        or len({c.uri for c in all_chunks}) != len(all_chunks)
+        or sum(c.point_count for c in chunks) != count
+        or overview.point_count > count
+    ):
+        raise ValueError(
+            "Reference chunk identities or complete point count are inconsistent"
+        )
+    return AnnotationReferenceManifest(
+        expected_clip_id,
+        dependency,
+        copy.deepcopy(identity),
+        copy.deepcopy(profile),
+        count,
+        tuple(tuple(map(float, row)) for row in value["bounds"]),
+        chunks,
+        overview,
+    )
+
+
+def decode_annotation_reference_chunk(
+    chunk: AnnotationReferenceChunk, payload: bytes
+) -> bytes:
+    """Verify compressed bytes and bound decompression before exposing XYZ."""
+    if (
+        len(payload) != chunk.size_bytes
+        or "sha256:" + hashlib.sha256(payload).hexdigest() != chunk.sha256
+    ):
+        raise ValueError("Annotation reference chunk integrity mismatch")
+    decoder = zlib.decompressobj(16 + zlib.MAX_WBITS)
+    points = decoder.decompress(payload, chunk.point_count * 12 + 1)
+    if (
+        len(points) != chunk.point_count * 12
+        or not decoder.eof
+        or decoder.unused_data
+        or decoder.unconsumed_tail
+    ):
+        raise ValueError(
+            "Annotation reference chunk point count or compression mismatch"
+        )
+    import numpy as np
+
+    xyz = np.frombuffer(points, dtype="<f4").reshape(-1, 3)
+    if (
+        not np.isfinite(xyz).all()
+        or not np.array_equal(xyz.min(axis=0), chunk.bounds[0])
+        or not np.array_equal(xyz.max(axis=0), chunk.bounds[1])
+    ):
+        raise ValueError("Annotation reference chunk geometry disagrees with manifest")
+    return points

x4d_devkit-0.41.0rc1/x4d_devkit/client/annotation_reference.py

@@ -0,0 +1,131 @@
+"""Typed reconstruction reference states without annotation write authority."""
+
+from __future__ import annotations
+
+import copy
+from dataclasses import dataclass
+from typing import Any
+
+from x4d_devkit.annotation_reference import (
+    ANNOTATION_REFERENCE_OPERATION,
+    AnnotationReferenceManifest,
+    _hash,
+    _object,
+    _positive,
+    _text,
+    parse_annotation_reference_manifest,
+)
+
+
+@dataclass(frozen=True)
+class AnnotationReferenceState:
+    project_id: int
+    clip_id: str
+    dependency_identity: str
+    status: str
+    task_id: str | None
+    progress: float
+    stage: str | None
+    error: dict[str, Any] | None
+    current: bool
+    manifest: AnnotationReferenceManifest | None
+
+    def to_dict(self):
+        return dict(
+            operation=ANNOTATION_REFERENCE_OPERATION,
+            project_id=self.project_id,
+            clip_id=self.clip_id,
+            dependency_identity=self.dependency_identity,
+            status=self.status,
+            task_id=self.task_id,
+            progress=self.progress,
+            stage=self.stage,
+            error=copy.deepcopy(self.error),
+            current=self.current,
+            manifest=self.manifest.to_dict() if self.manifest else None,
+        )
+
+
+def parse_annotation_reference_state(
+    payload, *, api_url, project_id, clip_id=None, task_id=None
+):
+    value = _object(
+        payload,
+        {
+            "operation",
+            "project_id",
+            "clip_id",
+            "dependency_identity",
+            "status",
+            "task_id",
+            "progress",
+            "stage",
+            "error",
+            "current",
+            "manifest",
+        },
+        "reference state",
+    )
+    if (
+        value["operation"] != ANNOTATION_REFERENCE_OPERATION
+        or value["project_id"] != project_id
+    ):
+        raise ValueError("Reference response belongs to another operation or project")
+    _positive(value["project_id"], "project_id")
+    _text(value["clip_id"], "clip_id")
+    _hash(value["dependency_identity"], "dependency_identity")
+    if clip_id is not None and value["clip_id"] != clip_id:
+        raise ValueError("Reference response belongs to another Clip")
+    if task_id is not None and value["task_id"] != task_id:
+        raise ValueError("Reference response belongs to another task")
+    if value["status"] not in {
+        "unavailable",
+        "queued",
+        "accepted",
+        "running",
+        "completed",
+        "failed",
+        "cancelled",
+    }:
+        raise ValueError("Unsupported reference task status")
+    if value["task_id"] is not None:
+        _text(value["task_id"], "task_id")
+    if value["stage"] is not None:
+        _text(value["stage"], "stage")
+    if type(value["current"]) is not bool:
+        raise ValueError("Reference current must be boolean")
+    progress = value["progress"]
+    if (
+        isinstance(progress, bool)
+        or not isinstance(progress, (int, float))
+        or not 0 <= progress <= 100
+    ):
+        raise ValueError("Invalid reference progress")
+    if value["error"] is not None and not isinstance(value["error"], dict):
+        raise ValueError("Reference error must be structured")
+    manifest = (
+        None
+        if value["manifest"] is None
+        else parse_annotation_reference_manifest(
+            value["manifest"],
+            api_url=api_url,
+            expected_clip_id=value["clip_id"],
+            expected_dependency_identity=value["dependency_identity"],
+        )
+    )
+    if value["current"] != (manifest is not None) or (
+        manifest and value["status"] != "completed"
+    ):
+        raise ValueError("Reference ready state is inconsistent")
+    return AnnotationReferenceState(
+        project_id,
+        value["clip_id"],
+        value["dependency_identity"],
+        value["status"],
+        value["task_id"],
+        float(progress),
+        value["stage"],
+        copy.deepcopy(value["error"]),
+        value["current"],
+        manifest,
+    )