x4d-devkit 0.51.0rc2__tar.gz → 0.51.0rc4__tar.gz

{x4d_devkit-0.51.0rc2 → x4d_devkit-0.51.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.51.0rc2
+Version: 0.51.0rc4
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0

{x4d_devkit-0.51.0rc2 → x4d_devkit-0.51.0rc4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.51.0rc2"
+version = "0.51.0rc4"
 description = "Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation"
 readme = "README.md"
 license = "Apache-2.0"

x4d_devkit-0.51.0rc4/tests/test_annotation_document_hash.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+import random
+
+import pytest
+import rfc8785
+
+from tests.test_annotation_contract import _annotation
+from x4d_devkit.annotation_contract import (
+    ANNOTATION_CONTRACT_VERSION,
+    AnnotationContractError,
+    ClipAnnotationDocument,
+    build_clip_annotations,
+    parse_clip_annotations,
+)
+
+_EDGE_FLOATS = (0.0, -0.0, 1e21, 1e-7, 5e-324, 1.7976931348623157e308, 0.1 + 0.2, -123.456)
+_EDGE_TEXT = ("", "é", "雪", "\U0001f697", "a\"b\\c", "
", "\x7f")
+
+
+def _sha(payload) -> str:
+    return f"sha256:{hashlib.sha256(rfc8785.dumps(payload)).hexdigest()}"
+
+
+def _random_rows(rng: random.Random, count: int) -> list[dict]:
+    rows = []
+    for index in range(count):
+        row = _annotation(
+            f"ann-{rng.randrange(10**9):09d}-{index}",
+            f"sample-{rng.randrange(5)}",
+            instance=f"instance-{rng.randrange(4)}",
+        )
+        row["bbox_3d"]["translation"]["x"] = rng.choice(_EDGE_FLOATS)
+        row["bbox_3d"]["size"]["width"] = rng.random() * 10
+        row["attributes"] = {rng.choice(_EDGE_TEXT) or "k": rng.choice(_EDGE_TEXT)}
+        if rng.random() < 0.5:
+            row["velocity"] = {"vx": rng.choice(_EDGE_FLOATS), "vy": 0.0, "vz": -1.5}
+        rows.append(row)
+    for instance in {row["instance_token"] for row in rows}:
+        motion = rng.choice(("auto", "static", "dynamic"))
+        for row in rows:
+            if row["instance_token"] == instance:
+                row["motion"] = motion
+    return rows
+
+
+def test_single_pass_digests_match_full_rfc8785_canonicalization():
+    rng = random.Random(8785)
+    for count in (0, 1, 2, 17, 120):
+        document = build_clip_annotations("clip-a", "base_link", _random_rows(rng, count))
+        payload = document.to_dict()
+        revision_payload = {key: value for key, value in payload.items() if key != "annotation_revision"}
+        assert document.annotation_revision == _sha(revision_payload)
+        assert document.document_hash() == _sha(payload)
+
+        parsed = parse_clip_annotations(payload)
+        assert parsed.annotation_revision == document.annotation_revision
+        assert parsed.document_hash() == _sha(payload)
+
+
+def test_document_hash_is_derived_for_directly_constructed_and_replaced_documents():
+    document = build_clip_annotations("clip-a", "base_link", _random_rows(random.Random(1), 9))
+    direct = ClipAnnotationDocument(
+        clip_id=document.clip_id,
+        annotation_frame_id=document.annotation_frame_id,
+        annotation_revision=document.annotation_revision,
+        annotations=document.annotations,
+    )
+    assert direct.document_hash() == document.document_hash()
+
+    replaced = dataclasses.replace(document, annotations=document.annotations[:3])
+    assert replaced.document_hash() == _sha(replaced.to_dict())
+    assert replaced.document_hash() != document.document_hash()
+
+
+def test_document_hash_is_not_part_of_document_equality():
+    rows = _random_rows(random.Random(2), 5)
+    built = build_clip_annotations("clip-a", "base_link", rows)
+    parsed = parse_clip_annotations(built.to_dict())
+    plain = ClipAnnotationDocument(
+        clip_id=built.clip_id,
+        annotation_frame_id=built.annotation_frame_id,
+        annotation_revision=built.annotation_revision,
+        annotations=built.annotations,
+    )
+    assert built == parsed == plain
+    assert "_document_hash" not in {field.name for field in dataclasses.fields(built)}
+    assert built.to_dict()["contract_version"] == ANNOTATION_CONTRACT_VERSION
+
+
+def test_parse_reuses_only_documents_this_module_verified():
+    built = build_clip_annotations("clip-a", "base_link", _random_rows(random.Random(3), 4))
+    assert parse_clip_annotations(built) is built
+    assert parse_clip_annotations(parse_clip_annotations(built.to_dict())) is not built
+
+    forged = ClipAnnotationDocument(
+        clip_id=built.clip_id,
+        annotation_frame_id=built.annotation_frame_id,
+        annotation_revision="sha256:" + "0" * 64,
+        annotations=built.annotations,
+    )
+    forged.document_hash()  # caching the hash must not mark the document verified
+    with pytest.raises(AnnotationContractError) as error:
+        parse_clip_annotations(forged)
+    assert error.value.code == "annotation_revision_mismatch"
+
+    replaced = dataclasses.replace(built, annotation_revision="sha256:" + "1" * 64)
+    with pytest.raises(AnnotationContractError):
+        parse_clip_annotations(replaced)

{x4d_devkit-0.51.0rc2 → x4d_devkit-0.51.0rc4}/tests/test_parse_equivalence.py

@@ -835,8 +835,12 @@ def test_annotation_entry_points_canonicalize_rows_only_once(monkeypatch, entry)
     else:
         assert annotation_revision(raw["clip_id"], raw["annotation_frame_id"],
                                    raw["annotations"]) == raw["annotation_revision"]
-    assert len(calls) == 1
-    assert calls[0]["annotations"] == raw["annotations"]
+    # Revision and document hash share one serialization of every row; no call
+    # re-serializes the row list or a complete payload. A document this module
+    # already verified is returned without serializing anything.
+    row_calls = [value for value in calls if isinstance(value, dict)]
+    assert row_calls == ([] if entry == "parsed-document" else raw["annotations"])
+    assert all(isinstance(value, str) for value in calls if not isinstance(value, dict))
 
 
 def test_parsed_annotation_documents_are_isolated_after_removing_document_deepcopy():

{x4d_devkit-0.51.0rc2 → x4d_devkit-0.51.0rc4}/tests/test_sdk_api.py

@@ -11,7 +11,7 @@ from x4d_devkit.eval.matching import MetricData
 
 
 def test_package_version_is_current_release():
-    assert x4d_devkit.__version__ == "0.51.0rc2"
+    assert x4d_devkit.__version__ == "0.51.0rc4"
 
 
 def test_loader_exposes_and_enforces_only_current_schema(tmp_path):

{x4d_devkit-0.51.0rc2 → x4d_devkit-0.51.0rc4}/x4d_devkit/annotation_contract.py

@@ -38,6 +38,13 @@ _DOCUMENT_FIELDS = {
     "annotations",
 }
 _REVISION_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
+# Set only by build/parse after canonical verification; never by document_hash().
+_VERIFIED_MARK = "_canonical_verified"
+
+
+def _mark_verified(document: "ClipAnnotationDocument", document_hash: str) -> None:
+    object.__setattr__(document, "_document_hash", document_hash)
+    object.__setattr__(document, _VERIFIED_MARK, True)
 _FRAME_ID_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
 
 
@@ -74,6 +81,24 @@ class ClipAnnotationDocument:
             tuple(_freeze_json(dict(row)) for row in self.annotations),
         )
 
+    def document_hash(self) -> str:
+        """Return ``sha256:`` of the RFC 8785 bytes of :meth:`to_dict`.
+
+        Documents built or parsed by this module carry it from the same single
+        canonicalization pass that produced ``annotation_revision``.
+        """
+
+        digest = self.__dict__.get("_document_hash")
+        if digest is None:
+            _, digest = _annotation_digests(
+                self.clip_id,
+                self.annotation_frame_id,
+                _thaw_json(self.annotations),
+                stored_revision=self.annotation_revision,
+            )
+            object.__setattr__(self, "_document_hash", digest)
+        return digest
+
     def to_dict(self) -> dict[str, Any]:
         return {
             "contract_version": ANNOTATION_CONTRACT_VERSION,
@@ -123,13 +148,15 @@ def build_clip_annotations(
         annotation_frame_id,
         path="annotation.annotation_frame_id",
     )
-    revision = _annotation_revision(parsed_clip_id, parsed_frame, canonical)
-    return ClipAnnotationDocument(
+    revision, document_hash = _annotation_digests(parsed_clip_id, parsed_frame, canonical)
+    document = ClipAnnotationDocument(
         clip_id=parsed_clip_id,
         annotation_frame_id=parsed_frame,
         annotation_revision=revision,
         annotations=tuple(canonical),
     )
+    _mark_verified(document, document_hash)
+    return document
 
 
 def parse_clip_annotations(
@@ -139,6 +166,9 @@ def parse_clip_annotations(
 
     i_json_checked = False
     if isinstance(value, ClipAnnotationDocument):
+        if value.__dict__.get(_VERIFIED_MARK) is True:
+            # Built or parsed by this module: already canonical and verified.
+            return value
         data = value.to_dict()
     elif isinstance(value, Mapping):
         data = dict(value)
@@ -205,7 +235,12 @@ def parse_clip_annotations(
             "annotation.annotation_revision",
             {"actual": revision},
         )
-    expected = _annotation_revision(clip_id, annotation_frame_id, annotations)
+    expected, document_hash = _annotation_digests(
+        clip_id,
+        annotation_frame_id,
+        annotations,
+        stored_revision=revision,
+    )
     if revision != expected:
         raise AnnotationContractError(
             "annotation_revision_mismatch",
@@ -213,12 +248,14 @@ def parse_clip_annotations(
             "annotation.annotation_revision",
             {"expected": expected, "actual": revision},
         )
-    return ClipAnnotationDocument(
+    document = ClipAnnotationDocument(
         clip_id=clip_id,
         annotation_frame_id=annotation_frame_id,
         annotation_revision=revision,
         annotations=tuple(annotations),
     )
+    _mark_verified(document, document_hash)
+    return document
 
 
 def annotation_revision(
@@ -244,14 +281,49 @@ def _annotation_revision(
 ) -> str:
     """Hash already checked rows with exactly one canonicalization pass."""
 
-    payload = {
-        "contract_version": ANNOTATION_CONTRACT_VERSION,
-        "clip_id": clip_id,
-        "annotation_frame_id": annotation_frame_id,
-        "annotations": canonical,
-    }
-    digest = hashlib.sha256(rfc8785.dumps(payload)).hexdigest()
-    return f"sha256:{digest}"
+    return _annotation_digests(clip_id, annotation_frame_id, canonical)[0]
+
+
+def _annotation_digests(
+    clip_id: str,
+    annotation_frame_id: str,
+    canonical: list[dict[str, Any]],
+    *,
+    stored_revision: str | None = None,
+) -> tuple[str, str]:
+    """Return ``(annotation_revision, document_hash)`` from one row pass.
+
+    RFC 8785 serializes an array as its comma-joined members and an object as
+    its members in UTF-16 key order without whitespace, so both canonical
+    payloads are assembled from the same row bytes. The document hash covers
+    ``stored_revision`` when given, otherwise the computed revision.
+    """
+
+    rows = b"[" + b",".join(rfc8785.dumps(row) for row in canonical) + b"]"
+    frame = rfc8785.dumps(annotation_frame_id)
+    tail = (
+        b',"clip_id":'
+        + rfc8785.dumps(clip_id)
+        + b',"contract_version":'
+        + rfc8785.dumps(ANNOTATION_CONTRACT_VERSION)
+        + b"}"
+    )
+    revision_digest = hashlib.sha256()
+    for part in (b'{"annotation_frame_id":', frame, b',"annotations":', rows, tail):
+        revision_digest.update(part)
+    revision = f"sha256:{revision_digest.hexdigest()}"
+    document_digest = hashlib.sha256()
+    for part in (
+        b'{"annotation_frame_id":',
+        frame,
+        b',"annotation_revision":',
+        rfc8785.dumps(stored_revision if stored_revision is not None else revision),
+        b',"annotations":',
+        rows,
+        tail,
+    ):
+        document_digest.update(part)
+    return revision, f"sha256:{document_digest.hexdigest()}"
 
 
 def derive_annotation_views(

{x4d_devkit-0.51.0rc2 → x4d_devkit-0.51.0rc4}/x4d_devkit/work_session.py

@@ -766,7 +766,8 @@ def parse_clip_work_session(
         "calibrated_sensor.json": tables["calibrated_sensor"],
         "sample.json": tables["samples"],
         "sample_data.json": tables["sample_data"],
-        "annotation.json": annotation_document.to_dict(),
+        # The validator accepts the verified document without re-canonicalizing it.
+        "annotation.json": annotation_document,
         "label_schema.json": label_schema.to_dict(),
     }
     if pose_revision is not None:

{x4d_devkit-0.51.0rc2 → x4d_devkit-0.51.0rc4}/x4d_devkit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.51.0rc2
+Version: 0.51.0rc4
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0

{x4d_devkit-0.51.0rc2 → x4d_devkit-0.51.0rc4}/x4d_devkit.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ README.md
 pyproject.toml
 tests/test_accounts.py
 tests/test_annotation_contract.py
+tests/test_annotation_document_hash.py
 tests/test_annotation_reference.py
 tests/test_assets_client.py
 tests/test_boundary.py