x4d-devkit 0.53.0rc3__tar.gz → 0.55.0__tar.gz

{x4d_devkit-0.53.0rc3/x4d_devkit.egg-info → x4d_devkit-0.55.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.53.0rc3
+Version: 0.55.0
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0
@@ -264,8 +264,8 @@ identity manifests, and update plans use one schema field only:
 `clip_label_schema` are not aliases and are rejected. A ready platform record
 without the complete identity is also rejected.
 
-`x4d-devkit dataset inspect` writes `x4d-remote-identity-v7`, and
-`x4d-devkit dataset diff` writes `x4d-update-plan-v9`. The v8 plan records the
+`x4d-devkit dataset inspect` writes `x4d-remote-identity-v8`, and
+`x4d-devkit dataset diff` writes `x4d-update-plan-v10`. The v10 plan records the
 requested materialization policy, each cache's reviewed samples/sweeps
 materialization revisions, and every Clip's reviewed target and local
 predecessor identities.

{x4d_devkit-0.53.0rc3 → x4d_devkit-0.55.0}/README.md

@@ -221,8 +221,8 @@ identity manifests, and update plans use one schema field only:
 `clip_label_schema` are not aliases and are rejected. A ready platform record
 without the complete identity is also rejected.
 
-`x4d-devkit dataset inspect` writes `x4d-remote-identity-v7`, and
-`x4d-devkit dataset diff` writes `x4d-update-plan-v9`. The v8 plan records the
+`x4d-devkit dataset inspect` writes `x4d-remote-identity-v8`, and
+`x4d-devkit dataset diff` writes `x4d-update-plan-v10`. The v10 plan records the
 requested materialization policy, each cache's reviewed samples/sweeps
 materialization revisions, and every Clip's reviewed target and local
 predecessor identities.

{x4d_devkit-0.53.0rc3 → x4d_devkit-0.55.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.53.0rc3"
+version = "0.55.0"
 description = "Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation"
 readme = "README.md"
 license = "Apache-2.0"

{x4d_devkit-0.53.0rc3 → x4d_devkit-0.55.0}/tests/test_dataset_identity.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import copy
+import hashlib
 import json
 
 import numpy as np
@@ -28,6 +29,11 @@ from x4d_devkit.dataset_identity import (
     update_plan_exit_code,
 )
 from x4d_devkit.label_schema import compute_label_schema_fingerprint
+from x4d_devkit.source_content import (
+    SOURCE_CONTENT_MANIFEST_CONTRACT,
+    load_local_source_content,
+    source_content_revisions,
+)
 from x4d_devkit.materialize import (
     load_materialized_cache_identity,
     materialize_work_session_clip,
@@ -42,6 +48,8 @@ def _identity(clip_id: str = "clip-a", **overrides):
         "source_revision": "sha256:context",
         "samples_revision": "sha256:" + "1" * 64,
         "sweeps_revision": "sha256:" + "2" * 64,
+        "samples_content_revision": "sha256:" + "7" * 64,
+        "sweeps_content_revision": "sha256:" + "8" * 64,
         "calibration_revision": "sha256:calibration",
         "pose_revision": "sha256:pose",
         "annotation_revision": "sha256:annotation",
@@ -392,32 +400,91 @@ class _FailedStream(_FakeStream):
         raise RuntimeError("fixture download failed")
 
 
+def _asset_bytes(path: str, overrides: dict[str, bytes] | None = None) -> bytes:
+    """Deterministic per-path fixture content; overrides model changed bytes."""
+    if overrides and path in overrides:
+        return overrides[path]
+    if path.endswith(".bin"):
+        seed = int(hashlib.sha256(path.encode()).hexdigest()[:4], 16) % 97
+        return np.array(
+            [(1.0, 2.0, 3.0, seed / 100.0, 1000)],
+            dtype=[
+                ("x", "<f4"),
+                ("y", "<f4"),
+                ("z", "<f4"),
+                ("intensity", "<f4"),
+                ("timestamp_ns", "<u8"),
+            ],
+        ).tobytes()
+    return b"jpeg:" + path.encode()
+
+
+def _bin_bytes(intensity: float) -> bytes:
+    return np.array(
+        [(1.0, 2.0, 3.0, intensity, 1000)],
+        dtype=[
+            ("x", "<f4"),
+            ("y", "<f4"),
+            ("z", "<f4"),
+            ("intensity", "<f4"),
+            ("timestamp_ns", "<u8"),
+        ],
+    ).tobytes()
+
+
+def _url_path(url) -> str:
+    text = str(url)
+    marker = "/api/v1/files/"
+    return text.split(marker, 1)[1] if marker in text else text
+
+
 class _FakeHTTP:
-    def __init__(self):
+    def __init__(self, overrides: dict[str, bytes] | None = None):
         self.urls = []
+        self.overrides = dict(overrides or {})
 
     def stream(self, method, url):
         self.urls.append((method, url))
-        payload = (
-            np.array(
-                [(1.0, 2.0, 3.0, 0.5, 1000)],
-                dtype=[
-                    ("x", "<f4"),
-                    ("y", "<f4"),
-                    ("z", "<f4"),
-                    ("intensity", "<f4"),
-                    ("timestamp_ns", "<u8"),
-                ],
-            ).tobytes()
-            if str(url).endswith(".bin")
-            else b"jpeg"
+        return _FakeStream(_asset_bytes(_url_path(url), self.overrides))
+
+
+def _source_manifest(payload, overrides=None):
+    rows = []
+    for row in payload["sample_data"]:
+        data = _asset_bytes(row["file_path"], overrides)
+        rows.append(
+            {
+                "path": row["file_path"],
+                "size": len(data),
+                "sha256": "sha256:" + hashlib.sha256(data).hexdigest(),
+            }
         )
-        return _FakeStream(payload)
+    rows.sort(key=lambda item: item["path"])
+    return {"contract_version": SOURCE_CONTENT_MANIFEST_CONTRACT, "objects": rows}
+
+
+class _DatasetIdentity:
+    def __init__(self, payload, overrides=None):
+        self.payload = payload
+        self.overrides = overrides
+        self.calls = 0
+
+    def source_content(self, *, project_id, clip_id):
+        self.calls += 1
+        manifest = _source_manifest(self.payload, self.overrides)
+        return {
+            "contract_version": "x4d-source-content-read-v1",
+            "project_id": project_id,
+            "clip_id": clip_id,
+            "source_revision": self.payload["source_revision"],
+            **source_content_revisions(manifest["objects"]),
+            "manifest": manifest,
+        }
 
 
 class _OneFailedHTTP(_FakeHTTP):
-    def __init__(self):
-        super().__init__()
+    def __init__(self, overrides=None):
+        super().__init__(overrides)
         self.failed = False
 
     def stream(self, method, url):
@@ -441,13 +508,15 @@ class _WorkSessions:
 
 
 class _Client:
-    def __init__(self, payload):
-        self._http = _FakeHTTP()
+    def __init__(self, payload, overrides=None):
+        self._http = _FakeHTTP(overrides)
         self.clip_work_sessions = _WorkSessions(payload)
+        self.dataset_identity = _DatasetIdentity(payload, overrides)
 
 
-def _local_from_work_session(payload):
+def _local_from_work_session(payload, overrides=None):
     identity = identity_from_work_session(payload).to_dict()
+    identity.update(source_content_revisions(_source_manifest(payload, overrides)["objects"]))
     return normalize_dataset_local_identity(
         {
             key: identity[key]
@@ -457,6 +526,8 @@ def _local_from_work_session(payload):
                 "source_revision",
                 "samples_revision",
                 "sweeps_revision",
+                "samples_content_revision",
+                "sweeps_content_revision",
                 "calibration_revision",
                 "pose_revision",
                 "annotation_revision",
@@ -527,6 +598,7 @@ def test_policy_upgrade_plan_reuses_keyframes_and_downloads_only_sweeps(tmp_path
     initial_client = _Client(payload)
     initial = materialize_work_session_clip(
         payload,
+        source_content=_source_manifest(payload),
         output_dir=clip_dir,
         client=initial_client,
         workers=1,
@@ -618,105 +690,169 @@ def test_policy_upgrade_plan_reuses_keyframes_and_downloads_only_sweeps(tmp_path
     assert client.clip_work_sessions.calls == 1
 
 
-def test_samples_change_refreshes_only_the_samples_group(tmp_path):
-    initial = _work_session_with_sweep()
+def _materialized_full(tmp_path, payload):
     clip_dir = tmp_path / "clips" / "clip-a"
-    materialize_work_session_clip(
-        initial,
+    result = materialize_work_session_clip(
+        payload,
+        source_content=_source_manifest(payload),
         output_dir=clip_dir,
-        client=_Client(initial),
+        client=_Client(payload),
         workers=1,
         materialization_policy="full",
     )
+    assert result["errors"] == []
+    return clip_dir
+
+
+def _group_paths(payload, group):
+    return sorted(
+        row["file_path"]
+        for row in payload["sample_data"]
+        if row["file_path"].startswith(group + "/")
+    )
+
+
+def _sync_one(tmp_path, *, initial, target_payload, overrides, changes):
     predecessor = _local_from_work_session(initial)
+    target = _local_from_work_session(target_payload, overrides)
+    plan = build_update_plan(
+        _response(
+            [_record(predecessor=predecessor, target=target, changes=changes)]
+        ),
+        materialization_policy="full",
+        local_materializations=_local_materializations(predecessor),
+    )
+    client = _Client(target_payload, overrides)
+    return plan, client, sync_update_plan(
+        client, plan=plan, data_root=tmp_path, workers=1
+    )
+
+
+def test_metadata_only_samples_change_transfers_no_bytes(tmp_path):
+    initial = _work_session_with_sweep()
+    clip_dir = _materialized_full(tmp_path, initial)
     target_payload = copy.deepcopy(initial)
     target_payload["source_revision"] = "source-samples-v2"
     target_payload["samples_revision"] = "sha256:" + "4" * 64
-    target = _local_from_work_session(target_payload)
-    record = _record(
-        predecessor=predecessor,
-        target=target,
+
+    plan, client, result = _sync_one(
+        tmp_path,
+        initial=initial,
+        target_payload=target_payload,
+        overrides=None,
         changes=("source_revision_changed", "samples_changed"),
     )
-    plan = build_update_plan(
-        _response([record]),
-        materialization_policy="full",
-        local_materializations=_local_materializations(predecessor),
-    )
+
     assert plan["actions"]["refresh_samples"] == ["clip-a"]
-    assert plan["actions"]["refresh_sweeps"] == []
+    assert result["failures"] == []
+    assert client._http.urls == []
+    assert result["source_transfer"]["downloaded_objects"] == 0
+    assert result["source_transfer"]["downloaded_bytes"] == 0
+    reused = result["executed_updates"][0]["result"]["reused_assets"]
+    assert {item["reuse"] for item in reused} == {"in_place"}
+    assert len(reused) == len(_group_paths(initial, "samples"))
+    cache_identity = load_materialized_cache_identity(clip_dir)
+    assert cache_identity["samples_revision"] == target_payload["samples_revision"]
+    assert validate_clip(clip_dir).ok
 
-    client = _Client(target_payload)
-    result = sync_update_plan(
-        client,
-        plan=plan,
-        data_root=tmp_path,
-        workers=1,
+
+def test_samples_content_change_downloads_only_changed_objects(tmp_path):
+    initial = _work_session_with_sweep()
+    clip_dir = _materialized_full(tmp_path, initial)
+    changed = next(p for p in _group_paths(initial, "samples") if p.endswith(".bin"))
+    overrides = {changed: _bin_bytes(0.99)}
+    target_payload = copy.deepcopy(initial)
+    target_payload["source_revision"] = "source-samples-v3"
+    target_payload["samples_revision"] = "sha256:" + "4" * 64
+
+    plan, client, result = _sync_one(
+        tmp_path,
+        initial=initial,
+        target_payload=target_payload,
+        overrides=overrides,
+        changes=(
+            "source_revision_changed",
+            "samples_changed",
+            "samples_content_changed",
+        ),
     )
 
     assert result["failures"] == []
-    assert client._http.urls
-    assert all("/samples/" in url for _, url in client._http.urls)
+    assert [_url_path(url) for _, url in client._http.urls] == [changed]
     transfer = result["source_transfer"]
-    assert transfer["accounting_complete"] is True
-    assert transfer["planned_objects"] == transfer["downloaded_objects"] == 2
-    assert transfer["planned_bytes"] == transfer["downloaded_bytes"] > 0
-    assert {
-        item["component"]
-        for item in result["executed_updates"][0]["result"]["downloaded_assets"]
-    } == {"samples"}
-    cache_identity = load_materialized_cache_identity(clip_dir)
-    assert cache_identity["samples_revision"] == target_payload["samples_revision"]
-    assert cache_identity["sweeps_revision"] == initial["sweeps_revision"]
+    assert transfer["planned_objects"] == transfer["downloaded_objects"] == 1
+    assert transfer["downloaded_bytes"] == len(overrides[changed])
+    assert (clip_dir / changed).read_bytes() == overrides[changed]
+    held = load_local_source_content(clip_dir)
+    assert held[changed]["sha256"] == (
+        "sha256:" + hashlib.sha256(overrides[changed]).hexdigest()
+    )
+    assert load_materialized_cache_identity(clip_dir)["samples_content_revision"] == (
+        _local_from_work_session(target_payload, overrides)["samples_content_revision"]
+    )
 
 
-def test_sweeps_change_refreshes_only_the_sweeps_group(tmp_path):
+def test_renamed_object_is_relocated_without_transfer(tmp_path):
     initial = _work_session_with_sweep()
-    clip_dir = tmp_path / "clips" / "clip-a"
-    materialize_work_session_clip(
-        initial,
-        output_dir=clip_dir,
-        client=_Client(initial),
-        workers=1,
-        materialization_policy="full",
+    clip_dir = _materialized_full(tmp_path, initial)
+    old_path = next(p for p in _group_paths(initial, "samples") if p.endswith(".bin"))
+    target_payload = copy.deepcopy(initial)
+    target_payload["source_revision"] = "source-renamed"
+    target_payload["samples_revision"] = "sha256:" + "9" * 64
+    for row in target_payload["sample_data"]:
+        if row["file_path"] == old_path:
+            # A timestamp correction re-places identical bytes at a new path.
+            row["timestamp"]["nanosec"] += 1
+            stamp = row["timestamp"]["sec"] * 1_000_000_000 + row["timestamp"]["nanosec"]
+            row["file_path"] = f"samples/{row['channel']}/{stamp}.bin"
+            new_path = row["file_path"]
+    target_payload["asset_urls"] = {
+        row["sample_data_token"]: f"/api/v1/files/{row['file_path']}"
+        for row in target_payload["sample_data"]
+    }
+    overrides = {new_path: _asset_bytes(old_path)}
+
+    plan, client, result = _sync_one(
+        tmp_path,
+        initial=initial,
+        target_payload=target_payload,
+        overrides=overrides,
+        changes=("source_revision_changed", "samples_changed"),
     )
-    predecessor = _local_from_work_session(initial)
+
+    assert result["failures"] == []
+    assert client._http.urls == []
+    assert not (clip_dir / old_path).exists()
+    assert (clip_dir / new_path).read_bytes() == _asset_bytes(old_path)
+    reused = result["executed_updates"][0]["result"]["reused_assets"]
+    assert [item["from"] for item in reused if item["reuse"] == "relocated"] == [old_path]
+
+
+def test_sweeps_content_change_downloads_only_changed_sweeps(tmp_path):
+    initial = _work_session_with_sweep()
+    clip_dir = _materialized_full(tmp_path, initial)
+    changed = _group_paths(initial, "sweeps")[0]
+    overrides = {changed: _bin_bytes(0.42) if changed.endswith(".bin") else b"jpeg-v2"}
     target_payload = copy.deepcopy(initial)
     target_payload["source_revision"] = "source-sweeps-v2"
     target_payload["sweeps_revision"] = "sha256:" + "5" * 64
-    target = _local_from_work_session(target_payload)
-    record = _record(
-        predecessor=predecessor,
-        target=target,
-        changes=("source_revision_changed", "sweeps_changed"),
-    )
-    plan = build_update_plan(
-        _response([record]),
-        materialization_policy="full",
-        local_materializations=_local_materializations(predecessor),
-    )
-    assert plan["actions"]["refresh_samples"] == []
-    assert plan["actions"]["refresh_sweeps"] == ["clip-a"]
 
-    client = _Client(target_payload)
-    result = sync_update_plan(
-        client,
-        plan=plan,
-        data_root=tmp_path,
-        workers=1,
+    plan, client, result = _sync_one(
+        tmp_path,
+        initial=initial,
+        target_payload=target_payload,
+        overrides=overrides,
+        changes=(
+            "source_revision_changed",
+            "sweeps_changed",
+            "sweeps_content_changed",
+        ),
     )
 
+    assert plan["actions"]["refresh_samples"] == []
+    assert plan["actions"]["refresh_sweeps"] == ["clip-a"]
     assert result["failures"] == []
-    assert client._http.urls
-    assert all("/sweeps/" in url for _, url in client._http.urls)
-    transfer = result["source_transfer"]
-    assert transfer["accounting_complete"] is True
-    assert transfer["planned_objects"] == transfer["downloaded_objects"] == 2
-    assert transfer["planned_bytes"] == transfer["downloaded_bytes"] > 0
-    assert {
-        item["component"]
-        for item in result["executed_updates"][0]["result"]["downloaded_assets"]
-    } == {"sweeps"}
+    assert [_url_path(url) for _, url in client._http.urls] == [changed]
     cache_identity = load_materialized_cache_identity(clip_dir)
     assert cache_identity["samples_revision"] == initial["samples_revision"]
     assert cache_identity["sweeps_revision"] == target_payload["sweeps_revision"]
@@ -724,55 +860,63 @@ def test_sweeps_change_refreshes_only_the_sweeps_group(tmp_path):
 
 def test_failed_source_refresh_retains_observed_transfer_accounting(tmp_path):
     initial = _work_session_with_sweep()
-    clip_dir = tmp_path / "clips" / "clip-a"
-    materialize_work_session_clip(
-        initial,
-        output_dir=clip_dir,
-        client=_Client(initial),
-        workers=1,
-        materialization_policy="full",
+    _materialized_full(tmp_path, initial)
+    changed = _group_paths(initial, "samples")
+    overrides = {path: b"changed:" + path.encode() for path in changed}
+    overrides.update(
+        {path: _bin_bytes(0.77) for path in changed if path.endswith(".bin")}
     )
-    predecessor = _local_from_work_session(initial)
     target_payload = copy.deepcopy(initial)
     target_payload["source_revision"] = "source-samples-failed"
     target_payload["samples_revision"] = "sha256:" + "6" * 64
-    target = _local_from_work_session(target_payload)
+    predecessor = _local_from_work_session(initial)
+    target = _local_from_work_session(target_payload, overrides)
     plan = build_update_plan(
         _response(
             [
                 _record(
                     predecessor=predecessor,
                     target=target,
-                    changes=("source_revision_changed", "samples_changed"),
+                    changes=(
+                        "source_revision_changed",
+                        "samples_changed",
+                        "samples_content_changed",
+                    ),
                 )
             ]
         ),
         materialization_policy="full",
         local_materializations=_local_materializations(predecessor),
     )
-    client = _Client(target_payload)
-    client._http = _OneFailedHTTP()
+    client = _Client(target_payload, overrides)
+    client._http = _OneFailedHTTP(overrides)
 
-    result = sync_update_plan(
-        client,
-        plan=plan,
-        data_root=tmp_path,
-        workers=1,
-    )
+    result = sync_update_plan(client, plan=plan, data_root=tmp_path, workers=1)
 
     assert result["executed_updates"] == []
     assert result["failures"][0]["code"] == "component_refresh_failed"
-    assert result["source_transfer"] == {
-        "planned_objects": 2,
-        "planned_bytes": 24,
-        "downloaded_objects": 1,
-        "downloaded_bytes": 24,
-        "accounting_complete": False,
-    }
-    assert (
-        result["failures"][0]["details"]["source_transfer"]
-        == result["source_transfer"]
+    transfer = result["source_transfer"]
+    assert transfer["planned_objects"] == len(changed)
+    assert transfer["downloaded_objects"] == len(changed) - 1
+    assert transfer["accounting_complete"] is False
+    assert result["failures"][0]["details"]["source_transfer"] == transfer
+
+
+def test_download_is_verified_against_content_identity(tmp_path):
+    initial = current_work_session_payload()
+    client = _Client(initial)
+    manifest = _source_manifest(initial)
+    manifest["objects"][0]["sha256"] = "sha256:" + "0" * 64
+    result = materialize_work_session_clip(
+        initial,
+        source_content=manifest,
+        output_dir=tmp_path / "clip",
+        client=client,
+        workers=1,
     )
+    assert [item["code"] for item in result["errors"]] == ["asset_download_failed"]
+    assert "content identity" in result["errors"][0]["message"]
+    assert load_materialized_cache_identity(tmp_path / "clip") is None
 
 
 def test_keyframes_cache_records_sweeps_change_without_asset_io(tmp_path):
@@ -780,6 +924,7 @@ def test_keyframes_cache_records_sweeps_change_without_asset_io(tmp_path):
     clip_dir = tmp_path / "clips" / "clip-a"
     materialize_work_session_clip(
         initial,
+        source_content=_source_manifest(initial),
         output_dir=clip_dir,
         client=_Client(initial),
         workers=1,
@@ -835,6 +980,7 @@ def test_sync_composes_schema_and_annotation_into_one_clip_update(tmp_path):
     clip_dir = tmp_path / "clips" / "clip-a"
     materialize_work_session_clip(
         initial,
+        source_content=_source_manifest(initial),
         output_dir=clip_dir,
         client=initial_client,
         workers=1,
@@ -858,7 +1004,7 @@ def test_sync_composes_schema_and_annotation_into_one_clip_update(tmp_path):
     result = sync_update_plan(client, plan=plan, data_root=tmp_path, workers=1)
 
     assert result["schema_version"] == SYNC_RESULT_SCHEMA_VERSION
-    assert result["schema_version"] == "x4d-sync-result-v9"
+    assert result["schema_version"] == "x4d-sync-result-v10"
     assert result["source_transfer"] == {
         "planned_objects": 0,
         "planned_bytes": 0,