x4d-devkit 0.23.0__tar.gz → 0.24.0rc1__tar.gz

{x4d_devkit-0.23.0 → x4d_devkit-0.24.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.23.0
+Version: 0.24.0rc1
 Summary: Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation
 Author: windzu
 License-Expression: Apache-2.0
@@ -66,7 +66,7 @@ Design decisions for this boundary are tracked in GitHub issues, starting with:
 ## Installation
 
 ```bash
-pip install x4d-devkit==0.23.0
+pip install x4d-devkit==0.24.0rc1
 ```
 
 Devkit is released independently as an immutable wheel. X-4D, XPoints, and
@@ -255,9 +255,11 @@ identity manifests, and update plans use one schema field only:
 without the complete identity is also rejected.
 
 `x4d-devkit dataset inspect` writes `x4d-remote-identity-v2`, and
-`x4d-devkit dataset diff` writes `x4d-update-plan-v2`. These artifacts contain
-normalized clip records once; they do not embed a duplicate raw platform
-response. V1 artifacts are rejected rather than migrated.
+`x4d-devkit dataset diff` writes `x4d-update-plan-v3`. The v3 plan records the
+exact materialization policy and each changed clip's reviewed target and local
+predecessor identities. Applying it is fail-closed and idempotent: completed
+clips are reported as already satisfied, while local or remote identity drift
+requires a new plan. Older plan contracts are rejected rather than migrated.
 
 ### Load a clip
 

{x4d_devkit-0.23.0 → x4d_devkit-0.24.0rc1}/README.md

@@ -23,7 +23,7 @@ Design decisions for this boundary are tracked in GitHub issues, starting with:
 ## Installation
 
 ```bash
-pip install x4d-devkit==0.23.0
+pip install x4d-devkit==0.24.0rc1
 ```
 
 Devkit is released independently as an immutable wheel. X-4D, XPoints, and
@@ -212,9 +212,11 @@ identity manifests, and update plans use one schema field only:
 without the complete identity is also rejected.
 
 `x4d-devkit dataset inspect` writes `x4d-remote-identity-v2`, and
-`x4d-devkit dataset diff` writes `x4d-update-plan-v2`. These artifacts contain
-normalized clip records once; they do not embed a duplicate raw platform
-response. V1 artifacts are rejected rather than migrated.
+`x4d-devkit dataset diff` writes `x4d-update-plan-v3`. The v3 plan records the
+exact materialization policy and each changed clip's reviewed target and local
+predecessor identities. Applying it is fail-closed and idempotent: completed
+clips are reported as already satisfied, while local or remote identity drift
+requires a new plan. Older plan contracts are rejected rather than migrated.
 
 ### Load a clip
 

{x4d_devkit-0.23.0 → x4d_devkit-0.24.0rc1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.23.0"
+version = "0.24.0rc1"
 description = "Official X-4D external SDK and CLI for datasets, platform APIs, training bridges, and automation"
 readme = "README.md"
 license = "Apache-2.0"

{x4d_devkit-0.23.0 → x4d_devkit-0.24.0rc1}/tests/test_dataset_identity.py

@@ -10,6 +10,7 @@ from tests.schema_contract_fixtures import schema_identity
 from x4d_devkit import cli
 from x4d_devkit.client.dataset_identity import DatasetIdentityAPI
 from x4d_devkit.consumer_identity import empty_consumer_identity_diff
+from x4d_devkit.current_state import identity_from_work_session
 from x4d_devkit.dataset_identity import (
     DATASET_IDENTITY_BUCKETS,
     build_remote_identity_manifest,
@@ -82,6 +83,10 @@ def _diff_record(bucket, **overrides):
                 pose_revision="sha256:pose",
             )
         )
+        if bucket != "new_clips":
+            record["local_identity"] = _local_identity(
+                clip_id=record["clip_id"],
+            )
     elif bucket == "removed_clips":
         record["local_identity"] = _local_identity(clip_id=record["clip_id"])
     elif bucket == "errors":
@@ -450,6 +455,7 @@ def test_null_remote_pose_revision_roundtrips_through_manifest_and_plan():
     plan = build_update_plan(
         diff,
         consumer_identity_diff=empty_consumer_identity_diff(),
+        materialization_policy="full",
     )
     assert plan["clips"][0]["pose_revision"] is None
     assert load_update_plan(plan)["clips"][0]["pose_revision"] is None
@@ -500,6 +506,7 @@ def test_build_update_plan_maps_actions_and_exit_code():
     plan = build_update_plan(
         diff,
         consumer_identity_diff=empty_consumer_identity_diff(),
+        materialization_policy="full",
         local_manifest="m.json",
     )
 
@@ -510,7 +517,8 @@ def test_build_update_plan_maps_actions_and_exit_code():
     assert plan["actions"]["reuse_source_cache"] == ["same", "ann"]
     assert plan["actions"]["rebuild_infos"] is True
     assert update_plan_exit_code(plan) == 13
-    assert plan["schema_version"] == "x4d-update-plan-v2"
+    assert plan["schema_version"] == "x4d-update-plan-v3"
+    assert plan["materialization_policy"] == "full"
     assert "platform_diff" not in plan
 
 
@@ -523,6 +531,7 @@ def test_build_update_plan_refreshes_schema_only_and_clip_format_changes():
     plan = build_update_plan(
         diff,
         consumer_identity_diff=empty_consumer_identity_diff(),
+        materialization_policy="full",
     )
 
     assert plan["actions"]["download_annotation_only"] == ["schema"]
@@ -553,7 +562,11 @@ def test_build_update_plan_includes_consumer_identity_changes():
         }],
     }
 
-    plan = build_update_plan(diff, consumer_identity_diff=consumer_diff)
+    plan = build_update_plan(
+        diff,
+        consumer_identity_diff=consumer_diff,
+        materialization_policy="full",
+    )
 
     assert plan["summary"]["consumer_changed"] is True
     assert plan["consumer_identity"]["changed_fields"] == ["split", "class_mapping"]
@@ -576,12 +589,13 @@ def test_build_update_plan_rejects_partial_consumer_identity_diff():
         build_update_plan(
             diff,
             consumer_identity_diff={"changed": True},
+            materialization_policy="full",
         )
 
 
 def test_build_update_plan_requires_explicit_consumer_identity_diff():
     with pytest.raises(TypeError, match="consumer_identity_diff"):
-        build_update_plan(_diff_response())
+        build_update_plan(_diff_response(), materialization_policy="full")
 
 
 def test_build_update_plan_distinguishes_pose_and_calibration_changes():
@@ -595,6 +609,7 @@ def test_build_update_plan_distinguishes_pose_and_calibration_changes():
     plan = build_update_plan(
         diff,
         consumer_identity_diff=empty_consumer_identity_diff(),
+        materialization_policy="full",
     )
 
     assert plan["summary"]["calibration_changed"] == 1
@@ -661,6 +676,7 @@ def test_dataset_diff_cli_writes_plan_and_exits(monkeypatch, tmp_path, capsys):
         project_name=None,
         clip_ids=None,
         include_asset_revision=False,
+        materialization_policy="full",
         out=str(out),
         format="table",
         api_url=None,
@@ -742,6 +758,7 @@ def test_dataset_diff_cli_compares_training_identity(monkeypatch, tmp_path, caps
         project_name=None,
         clip_ids=None,
         include_asset_revision=False,
+        materialization_policy="full",
         out=str(out),
         format="table",
         api_url=None,
@@ -820,10 +837,11 @@ def test_dataset_inspect_cli_writes_remote_manifest(monkeypatch, tmp_path, capsy
 
 def _plan(**overrides):
     data = {
-        "schema_version": "x4d-update-plan-v2",
+        "schema_version": "x4d-update-plan-v3",
         "generated_at": "2026-06-30T00:00:00Z",
         "x4d_devkit_version": "0.19.0",
         "local_manifest": None,
+        "materialization_policy": "full",
         "summary": {},
         "clips": [
             {
@@ -848,6 +866,7 @@ def _plan(**overrides):
         "actions": {
             "reuse_source_cache": ["clip-a"],
             "download_source": ["clip-b"],
+            "refresh_source_metadata": [],
             "download_annotation_only": ["clip-a"],
             "invalidate_mapped_cache": ["clip-a", "clip-b"],
             "rebuild_infos": True,
@@ -860,9 +879,88 @@ def _plan(**overrides):
     return data
 
 
-def test_load_update_plan_rejects_old_v1_contract():
-    with pytest.raises(ValueError, match="x4d-update-plan-v2"):
-        load_update_plan({**_plan(), "schema_version": "x4d-update-plan-v1"})
+def _portable_work_session_identity(payload):
+    identity = identity_from_work_session(payload).to_dict()
+    return {
+        key: identity[key]
+        for key in (
+            "project_id",
+            "clip_id",
+            "source_revision",
+            "calibration_revision",
+            "pose_revision",
+            "annotation_revision",
+            "label_schema",
+            "clip_schema_version",
+        )
+    }
+
+
+def _write_cache_marker(clip_dir, identity, *, policy="full"):
+    clip_dir.mkdir(parents=True, exist_ok=True)
+    (clip_dir / ".cache_identity.json").write_text(json.dumps({
+        **identity,
+        "ready": True,
+        "blockers": [],
+        "asset_revision": {},
+        "materialization_policy": policy,
+        "has_sweeps": policy == "full",
+        "x4d_devkit_version": "0.23.0",
+        "materialized_at": "2026-07-20T00:00:00Z",
+    }))
+
+
+class _FakeStream:
+    def __init__(self, payload=b"asset"):
+        self.payload = payload
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return None
+
+    def raise_for_status(self):
+        return None
+
+    def iter_bytes(self, chunk_size):
+        yield self.payload
+
+
+class _FakeHTTP:
+    def __init__(self):
+        self.urls = []
+
+    def stream(self, method, url):
+        self.urls.append((method, url))
+        return _FakeStream()
+
+
+def _sync_plan_for(
+    clips,
+    *,
+    download_source=(),
+    refresh_source_metadata=(),
+    download_annotation_only=(),
+    materialization_policy="full",
+):
+    return _plan(
+        clips=list(clips),
+        materialization_policy=materialization_policy,
+        actions={
+            **_plan()["actions"],
+            "reuse_source_cache": [],
+            "download_source": list(download_source),
+            "refresh_source_metadata": list(refresh_source_metadata),
+            "download_annotation_only": list(download_annotation_only),
+            "invalidate_mapped_cache": [],
+        },
+    )
+
+
+def test_load_update_plan_rejects_old_v2_contract():
+    with pytest.raises(ValueError, match="x4d-update-plan-v3"):
+        load_update_plan({**_plan(), "schema_version": "x4d-update-plan-v2"})
 
 
 def test_load_update_plan_rejects_old_consumer_identity_diff():
@@ -895,21 +993,30 @@ def test_load_update_plan_rejects_removed_schema_fields(field):
 
 
 def test_sync_update_plan_dry_run_reports_actions(tmp_path):
+    plan = _plan()
+    _write_cache_marker(
+        tmp_path / "cache" / "clips" / "clip-a",
+        plan["clips"][0]["local_identity"],
+    )
     result = sync_update_plan(
         None,
-        plan=_plan(),
+        plan=plan,
         data_root=tmp_path / "cache",
         dry_run=True,
         mapped_cache_dir=tmp_path / "mapped",
     )
 
     assert result["dry_run"] is True
-    assert result["downloaded_source"] == [{
-        "clip_id": "clip-b",
-        "path": str(tmp_path / "cache" / "clips" / "clip-b"),
-        "dry_run": True,
-    }]
-    assert result["downloaded_annotation_only"][0]["clip_id"] == "clip-a"
+    assert [item["clip_id"] for item in result["planned_actions"]] == [
+        "clip-b",
+        "clip-a",
+    ]
+    assert [item["clip_id"] for item in result["actions_to_execute"]] == [
+        "clip-b",
+        "clip-a",
+    ]
+    assert result["executed_actions"] == []
+    assert result["already_satisfied_actions"] == []
     assert result["invalidated_mapped_cache"] == [
         {"clip_id": "clip-a", "dry_run": True},
         {"clip_id": "clip-b", "dry_run": True},
@@ -919,48 +1026,62 @@ def test_sync_update_plan_dry_run_reports_actions(tmp_path):
 
 
 def test_sync_update_plan_blocks_not_ready_by_default(tmp_path):
+    plan = _sync_plan_for([
+        {
+            "bucket": "not_ready",
+            **_diff_record(
+                "not_ready",
+                project_id=1812,
+                clip_id="clip-a",
+            ),
+            "blockers": [{"code": "missing_ego_pose", "message": "ego pose is absent"}],
+        }
+    ])
     result = sync_update_plan(
         None,
-        plan=_plan(
-            clips=[
-                {
-                    "bucket": "not_ready",
-                    **_diff_record(
-                        "not_ready",
-                        project_id=1812,
-                        clip_id="clip-a",
-                    ),
-                    "blockers": [{"code": "missing_ego_pose", "message": "ego pose is absent"}],
-                }
-            ]
-        ),
+        plan=plan,
         data_root=tmp_path,
         dry_run=True,
     )
 
-    assert result["errors"] == [{
-        "clip_id": "clip-a",
-        "code": "not_ready",
-        "blockers": [{"code": "missing_ego_pose", "message": "ego pose is absent"}],
-    }]
+    assert result["failures"][0]["code"] == "not_ready"
+    assert result["failures"][0]["details"]["blockers"] == [
+        {"code": "missing_ego_pose", "message": "ego pose is absent"}
+    ]
     assert sync_result_exit_code(result) == 20
 
 
-def test_sync_update_plan_writes_annotation_only_tables_and_stale_markers(tmp_path):
+def test_sync_update_plan_annotation_only_first_apply_and_replay(tmp_path):
+    target_payload = current_work_session_payload(clip_id="clip-a")
+    target_identity = _portable_work_session_identity(target_payload)
+    predecessor = {
+        **target_identity,
+        "annotation_revision": "sha256:" + "1" * 64,
+    }
+    record = {
+        "bucket": "annotation_changed",
+        **_diff_record(
+            "annotation_changed",
+            **target_identity,
+            local_identity=predecessor,
+        ),
+    }
+    plan = _sync_plan_for(
+        [record],
+        download_annotation_only=["clip-a"],
+    )
     clip_dir = tmp_path / "cache" / "clips" / "clip-a"
-    clip_dir.mkdir(parents=True)
+    _write_cache_marker(clip_dir, predecessor)
     (clip_dir / "meta.json").write_text("{}\n")
 
     class WorkSessions:
+        def __init__(self):
+            self.calls = 0
+
         def open(self, project_id, clip_id):
             assert project_id == 1812
-            payload = current_work_session_payload(clip_id=clip_id)
-            payload.update(
-                source_revision="s2",
-                calibration_revision="c2",
-                pose_revision="p2",
-            )
-            return payload
+            self.calls += 1
+            return target_payload
 
     class Client:
         def __init__(self):
@@ -968,46 +1089,154 @@ def test_sync_update_plan_writes_annotation_only_tables_and_stale_markers(tmp_pa
             self._http = None
 
     client = Client()
-    result = sync_update_plan(
+    first = sync_update_plan(
         client,
-        plan=_plan(
-            clips=[
-                {
-                    "bucket": "annotation_changed",
-                    **_diff_record(
-                        "annotation_changed",
-                        project_id=1812,
-                        clip_id="clip-a",
-                    ),
-                }
-            ],
-            actions={
-                **_plan()["actions"],
-                "download_source": [],
-                "invalidate_mapped_cache": ["clip-a"],
-            },
-        ),
+        plan=plan,
         data_root=tmp_path / "cache",
-        mapped_cache_dir=tmp_path / "mapped",
     )
+    replay = sync_update_plan(client, plan=plan, data_root=tmp_path / "cache")
 
     annotation_document = json.loads((clip_dir / "annotation.json").read_text())
     assert annotation_document["annotations"][0]["annotation_token"] == "annotation-0"
     assert not (clip_dir / "instance.json").exists()
-    assert (
-        json.loads((clip_dir / ".x4d_identity.json").read_text())[
-            "annotation_revision"
-        ]
-        == annotation_document["annotation_revision"]
+    assert first["failures"] == []
+    assert first["executed_actions"][0]["action"] == "download_annotation_only"
+    assert replay["executed_actions"] == []
+    assert replay["already_satisfied_actions"][0]["clip_id"] == "clip-a"
+    assert client.clip_work_sessions.calls == 1
+
+
+def test_sync_update_plan_partial_retry_preserves_completed_clips(tmp_path):
+    payloads = {
+        clip_id: current_work_session_payload(clip_id=clip_id)
+        for clip_id in ("clip-a", "clip-b")
+    }
+    records = [
+        {
+            "bucket": "new_clips",
+            **_diff_record(
+                "new_clips",
+                **_portable_work_session_identity(payloads[clip_id]),
+            ),
+        }
+        for clip_id in ("clip-a", "clip-b")
+    ]
+    plan = _sync_plan_for(
+        records,
+        download_source=["clip-a", "clip-b"],
     )
-    assert (tmp_path / "mapped" / "clip-a" / ".x4d_stale.json").exists()
-    assert result["errors"] == []
+
+    class WorkSessions:
+        def __init__(self):
+            self.calls = []
+            self.fail_clip_b = True
+
+        def open(self, project_id, clip_id):
+            self.calls.append(clip_id)
+            if clip_id == "clip-b" and self.fail_clip_b:
+                raise RuntimeError("transient open failure")
+            return payloads[clip_id]
+
+    class Client:
+        def __init__(self):
+            self.clip_work_sessions = WorkSessions()
+            self._http = _FakeHTTP()
+
+    client = Client()
+    first = sync_update_plan(client, plan=plan, data_root=tmp_path / "cache", workers=1)
+    first_download_count = len(client._http.urls)
+    client.clip_work_sessions.fail_clip_b = False
+    retry = sync_update_plan(client, plan=plan, data_root=tmp_path / "cache", workers=1)
+    completed_download_count = len(client._http.urls)
+    replay = sync_update_plan(client, plan=plan, data_root=tmp_path / "cache", workers=1)
+
+    assert [item["clip_id"] for item in first["executed_actions"]] == ["clip-a"]
+    assert first["failures"][0]["clip_id"] == "clip-b"
+    assert [item["clip_id"] for item in retry["already_satisfied_actions"]] == ["clip-a"]
+    assert [item["clip_id"] for item in retry["executed_actions"]] == ["clip-b"]
+    assert len(client._http.urls) == first_download_count * 2
+    assert client.clip_work_sessions.calls.count("clip-a") == 1
+    assert replay["executed_actions"] == []
+    assert [item["clip_id"] for item in replay["already_satisfied_actions"]] == [
+        "clip-a",
+        "clip-b",
+    ]
+    assert len(client._http.urls) == completed_download_count
+
+
+def test_sync_update_plan_rejects_policy_and_local_target_conflicts(tmp_path):
+    plan = _plan()
+    clip_dir = tmp_path / "cache" / "clips" / "clip-a"
+    _write_cache_marker(
+        clip_dir,
+        plan["clips"][0]["local_identity"],
+        policy="keyframes_only",
+    )
+    policy_result = sync_update_plan(
+        None,
+        plan=plan,
+        data_root=tmp_path / "cache",
+        dry_run=True,
+    )
+    assert policy_result["failures"][0]["code"] == "materialization_policy_mismatch"
+
+    conflicting = {
+        **plan["clips"][0]["local_identity"],
+        "annotation_revision": "sha256:" + "2" * 64,
+    }
+    _write_cache_marker(clip_dir, conflicting)
+    conflict_result = sync_update_plan(
+        None,
+        plan=plan,
+        data_root=tmp_path / "cache",
+        dry_run=True,
+    )
+    assert conflict_result["failures"][0]["code"] == "update_plan_target_mismatch"
+
+
+def test_sync_update_plan_rejects_remote_state_moving_after_review(tmp_path):
+    target_payload = current_work_session_payload(clip_id="clip-a")
+    target_identity = _portable_work_session_identity(target_payload)
+    predecessor = {
+        **target_identity,
+        "annotation_revision": "sha256:" + "1" * 64,
+    }
+    plan = _sync_plan_for(
+        [{
+            "bucket": "annotation_changed",
+            **_diff_record(
+                "annotation_changed",
+                **target_identity,
+                local_identity=predecessor,
+            ),
+        }],
+        download_annotation_only=["clip-a"],
+    )
+    _write_cache_marker(tmp_path / "cache" / "clips" / "clip-a", predecessor)
+    moved = {**target_payload, "source_revision": "source-moved"}
+
+    class WorkSessions:
+        def open(self, project_id, clip_id):
+            return moved
+
+    class Client:
+        clip_work_sessions = WorkSessions()
+        _http = _FakeHTTP()
+
+    result = sync_update_plan(Client(), plan=plan, data_root=tmp_path / "cache")
+    assert result["executed_actions"] == []
+    assert result["failures"][0]["code"] == "remote_update_plan_target_mismatch"
 
 
 def test_dataset_sync_cli_writes_result_and_exits(monkeypatch, tmp_path, capsys):
+    plan_data = _plan()
     plan = tmp_path / "plan.json"
-    plan.write_text(json.dumps(_plan()))
+    plan.write_text(json.dumps(plan_data))
     out = tmp_path / "sync_result.json"
+    _write_cache_marker(
+        tmp_path / "cache" / "clips" / "clip-a",
+        plan_data["clips"][0]["local_identity"],
+    )
 
     args = argparse.Namespace(
         plan=str(plan),
@@ -1016,7 +1245,6 @@ def test_dataset_sync_cli_writes_result_and_exits(monkeypatch, tmp_path, capsys)
         dry_run=True,
         allow_not_ready_skip=False,
         mapped_cache_dir=None,
-        materialization_policy="full",
         out=str(out),
         format="table",
         api_url=None,
@@ -1028,4 +1256,4 @@ def test_dataset_sync_cli_writes_result_and_exits(monkeypatch, tmp_path, capsys)
 
     assert exc.value.code == 0
     assert "Dataset sync dry-run completed" in capsys.readouterr().out
-    assert json.loads(out.read_text())["schema_version"] == "x4d-sync-result-v1"
+    assert json.loads(out.read_text())["schema_version"] == "x4d-sync-result-v2"