x4d-devkit 0.2.0__tar.gz → 0.2.1__tar.gz

{x4d_devkit-0.2.0 → x4d_devkit-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x4d-devkit
-Version: 0.2.0
+Version: 0.2.1
 Summary: X-4D dataset format SDK — load, validate, evaluate, and convert autonomous driving datasets
 Author: windzu
 License-Expression: Apache-2.0
@@ -57,13 +57,33 @@ pip install x4d-devkit[client]
 ```python
 from x4d_devkit import ClipLoader
 
-clip = ClipLoader("/path/to/clip")
-print(clip.meta)
+loader = ClipLoader("/path/to/clip")
+print(loader.meta)
 
-for sample in clip.samples:
-    # Access sensor data
-    for sd in clip.sample_data_by_sample(sample.token):
-        print(sd.channel, sd.filename)
+for sample in loader.samples:
+    for sd in loader.sample_data_for_sample(sample.token):
+        print(sd.channel, sd.file_path)
+```
+
+### Coordinate frame transforms
+
+Point clouds and annotations can be loaded in different coordinate frames:
+
+```python
+loader = ClipLoader("/path/to/clip")
+sd = loader.sample_data_for_channel("LIDAR_TOP")[0]
+
+# Load point cloud in different frames
+pts_sensor = loader.load_point_cloud(sd, frame="sensor")  # raw (default)
+pts_ego = loader.load_point_cloud(sd, frame="ego")        # sensor → ego
+pts_world = loader.load_point_cloud(sd, frame="world")    # sensor → world
+
+# Get annotations in ego frame (for training)
+anns_ego = loader.annotations_for_sample(sample.token, frame="ego")
+
+# Get the transform matrix directly
+T = loader.get_transform(sd, from_frame="sensor", to_frame="world")
+pts_world = T.apply(pts_sensor[:, :3])  # or use T.as_matrix for 4x4
 ```
 
 ### Validate a clip

{x4d_devkit-0.2.0 → x4d_devkit-0.2.1}/README.md

@@ -25,13 +25,33 @@ pip install x4d-devkit[client]
 ```python
 from x4d_devkit import ClipLoader
 
-clip = ClipLoader("/path/to/clip")
-print(clip.meta)
+loader = ClipLoader("/path/to/clip")
+print(loader.meta)
 
-for sample in clip.samples:
-    # Access sensor data
-    for sd in clip.sample_data_by_sample(sample.token):
-        print(sd.channel, sd.filename)
+for sample in loader.samples:
+    for sd in loader.sample_data_for_sample(sample.token):
+        print(sd.channel, sd.file_path)
+```
+
+### Coordinate frame transforms
+
+Point clouds and annotations can be loaded in different coordinate frames:
+
+```python
+loader = ClipLoader("/path/to/clip")
+sd = loader.sample_data_for_channel("LIDAR_TOP")[0]
+
+# Load point cloud in different frames
+pts_sensor = loader.load_point_cloud(sd, frame="sensor")  # raw (default)
+pts_ego = loader.load_point_cloud(sd, frame="ego")        # sensor → ego
+pts_world = loader.load_point_cloud(sd, frame="world")    # sensor → world
+
+# Get annotations in ego frame (for training)
+anns_ego = loader.annotations_for_sample(sample.token, frame="ego")
+
+# Get the transform matrix directly
+T = loader.get_transform(sd, from_frame="sensor", to_frame="world")
+pts_world = T.apply(pts_sensor[:, :3])  # or use T.as_matrix for 4x4
 ```
 
 ### Validate a clip

{x4d_devkit-0.2.0 → x4d_devkit-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "x4d-devkit"
-version = "0.2.0"
+version = "0.2.1"
 description = "X-4D dataset format SDK — load, validate, evaluate, and convert autonomous driving datasets"
 readme = "README.md"
 license = "Apache-2.0"

{x4d_devkit-0.2.0 → x4d_devkit-0.2.1}/x4d_devkit/__init__.py

@@ -1,5 +1,6 @@
 """X-4D dataset format SDK."""
 
+from x4d_devkit.client import X4DClient
 from x4d_devkit.core import (
     Annotation,
     CalibratedSensor,
@@ -36,6 +37,7 @@ __all__ = [
     "Transform",
     "TransformChain",
     "ValidationReport",
+    "X4DClient",
     "generate_token",
     "nusc_detection_config",
     "validate_clip",

x4d_devkit-0.2.1/x4d_devkit/client/__init__.py

@@ -0,0 +1,13 @@
+from x4d_devkit.client.checkpoints import CheckpointsAPI
+from x4d_devkit.client.client import X4DClient
+from x4d_devkit.client.clips import ClipsAPI
+from x4d_devkit.client.projects import ProjectsAPI
+from x4d_devkit.client.test_sets import TestSetsAPI
+
+__all__ = [
+    "CheckpointsAPI",
+    "ClipsAPI",
+    "ProjectsAPI",
+    "TestSetsAPI",
+    "X4DClient",
+]

x4d_devkit-0.2.1/x4d_devkit/client/checkpoints.py

@@ -0,0 +1,64 @@
+"""Checkpoints API wrapper — training project registers its trained models
+so the platform's Model Compare view can run inference on them.
+
+This is the primary integration point between a training project and the
+platform: after every epoch the training CLI calls ``register(...)`` and
+the checkpoint immediately shows up in Compare's dropdown.
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from x4d_devkit.client.client import X4DClient
+
+
+class CheckpointsAPI:
+    def __init__(self, client: "X4DClient"):
+        self._client = client
+
+    def register(
+        self,
+        project_id: int,
+        name: str,
+        service_id: str,
+        ckpt_uri: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> dict:
+        """Register a trained checkpoint for inference.
+
+        Idempotent on (project_id, service_id, ckpt_uri) — re-calling with
+        the same arguments updates name/metadata instead of creating a dup.
+
+        Args:
+            project_id: X-4D project the checkpoint belongs to.
+            name: Human-readable name, e.g. ``"exp_hg1_v1 · epoch 5"``.
+            service_id: The ``service_id`` of the training/inference service
+                that owns the checkpoint file (must be registered with the
+                platform, otherwise inference dispatch will 404).
+            ckpt_uri: Local path to the .pth file *on the service container*.
+                Remote/S3 URIs are not yet supported.
+            metadata: Arbitrary JSONB bag. By convention keys used by the
+                inference backend are ``cfg_file``, ``epoch``, ``is_best``,
+                ``training_task_id``.
+        """
+        return self._client.post(
+            "/checkpoints",
+            json={
+                "project_id": project_id,
+                "name": name,
+                "service_id": service_id,
+                "ckpt_uri": ckpt_uri,
+                "metadata": metadata or {},
+            },
+        )
+
+    def list(self, project_id: int) -> list[dict]:
+        resp = self._client.get(f"/projects/{project_id}/checkpoints")
+        return resp["items"]
+
+    def get(self, checkpoint_id: int) -> dict:
+        return self._client.get(f"/checkpoints/{checkpoint_id}")
+
+    def delete(self, checkpoint_id: int) -> None:
+        self._client.delete(f"/checkpoints/{checkpoint_id}")

{x4d_devkit-0.2.0 → x4d_devkit-0.2.1}/x4d_devkit/client/client.py

@@ -1,19 +1,27 @@
-"""X-4D platform API client."""
+"""X-4D platform API client.
+
+Entry point for training projects to query the platform: list clips (with
+`exclude_test=True` default for safe-by-default training-data access),
+download archives via the file proxy, list test sets, and register trained
+checkpoints for inference.
+"""
 from __future__ import annotations
 
 import httpx
 
-from x4d_devkit.client.projects import ProjectsAPI
+from x4d_devkit.client.checkpoints import CheckpointsAPI
 from x4d_devkit.client.clips import ClipsAPI
+from x4d_devkit.client.projects import ProjectsAPI
+from x4d_devkit.client.test_sets import TestSetsAPI
 
 
 class X4DClient:
     """Client for the X-4D platform REST API.
 
     Args:
-        base_url: Platform API base URL (e.g. "http://localhost:8000")
+        base_url: Platform API base URL (e.g. "http://backend:8000")
         api_key: Optional API key for authentication (future use)
-        timeout: Request timeout in seconds
+        timeout: Request timeout in seconds (default 30; downloads bypass this)
     """
 
     def __init__(
@@ -32,20 +40,29 @@ class X4DClient:
         )
         self.projects = ProjectsAPI(self)
         self.clips = ClipsAPI(self)
+        self.test_sets = TestSetsAPI(self)
+        self.checkpoints = CheckpointsAPI(self)
 
     def _url(self, path: str) -> str:
         return f"{self._api_prefix}{path}"
 
-    def get(self, path: str, **kwargs) -> dict:
+    def get(self, path: str, **kwargs):
         resp = self._http.get(self._url(path), **kwargs)
         resp.raise_for_status()
         return resp.json()
 
-    def post(self, path: str, **kwargs) -> dict:
+    def post(self, path: str, **kwargs):
         resp = self._http.post(self._url(path), **kwargs)
         resp.raise_for_status()
         return resp.json()
 
+    def delete(self, path: str, **kwargs):
+        resp = self._http.delete(self._url(path), **kwargs)
+        resp.raise_for_status()
+        if resp.content:
+            return resp.json()
+        return None
+
     def close(self):
         self._http.close()
 

{x4d_devkit-0.2.0 → x4d_devkit-0.2.1}/x4d_devkit/client/clips.py

@@ -21,8 +21,15 @@ class ClipsAPI:
         page_size: int = 100,
         lifecycle: str | None = None,
         has_archive: bool | None = None,
+        exclude_test: bool = True,
     ) -> dict:
-        """List clips in a project with optional filters."""
+        """List clips in a project with optional filters.
+
+        ``exclude_test`` defaults to **True** so that training pipelines are
+        safe-by-default — a clip that belongs to any test set is hidden from
+        this call unless you opt in. Pass ``exclude_test=False`` for admin /
+        inspection flows that want the full project inventory.
+        """
         params: dict = {
             "project_id": project_id,
             "page": page,
@@ -32,10 +39,15 @@ class ClipsAPI:
             params["lifecycle"] = lifecycle
         if has_archive is not None:
             params["has_archive"] = str(has_archive).lower()
+        if exclude_test:
+            params["exclude_test"] = "true"
         return self._client.get("/clips", params=params)
 
     def list_all(self, project_id: int, **kwargs) -> list[dict]:
-        """List ALL clips in a project (handles pagination automatically)."""
+        """List ALL clips in a project (handles pagination automatically).
+
+        Inherits ``exclude_test=True`` default from :meth:`list`.
+        """
         all_items = []
         page = 1
         while True:
@@ -64,6 +76,11 @@ class ClipsAPI:
     ) -> Path:
         """Download and optionally extract a clip's tar archive.
 
+        Goes through the platform's ``/api/v1/files/{tar_path}`` proxy
+        rather than MinIO presigned URLs. This is the only reliable path
+        in multi-container deployments — presigned URLs point at the MinIO
+        hostname which is not routable from inside a training container.
+
         Args:
             clip_id: Clip ID to download
             output_dir: Directory to save/extract to
@@ -75,11 +92,16 @@ class ClipsAPI:
         output_dir = Path(output_dir)
         output_dir.mkdir(parents=True, exist_ok=True)
 
-        url = self.get_archive_download_url(clip_id)
+        # Look up the tar_path from the archive record (always relative,
+        # e.g. "clips/abc.tar") and stream through the file proxy.
+        info = self.get_archive_status(clip_id)
+        tar_path_remote = info.get("tar_path") or f"clips/{clip_id}.tar"
         tar_path = output_dir / f"{clip_id}.tar"
 
-        # Stream download
-        with self._client._http.stream("GET", url) as resp:
+        with self._client._http.stream(
+            "GET",
+            f"{self._client._api_prefix}/files/{tar_path_remote}",
+        ) as resp:
             resp.raise_for_status()
             with open(tar_path, "wb") as f:
                 for chunk in resp.iter_bytes(chunk_size=8 * 1024 * 1024):
@@ -87,10 +109,32 @@ class ClipsAPI:
 
         if extract:
             clip_dir = output_dir / clip_id
-            clip_dir.mkdir(parents=True, exist_ok=True)
+            # Extract to a temp dir first and find the real clip dir — tar
+            # may or may not wrap its contents in a top-level folder.
+            extract_tmp = output_dir / f"_extract_{clip_id}"
+            if extract_tmp.exists():
+                import shutil
+                shutil.rmtree(extract_tmp)
+            extract_tmp.mkdir(parents=True)
             with tarfile.open(tar_path, "r") as tar:
-                tar.extractall(path=clip_dir)
-            tar_path.unlink()
+                tar.extractall(path=extract_tmp)
+            tar_path.unlink(missing_ok=True)
+
+            meta_files = list(extract_tmp.rglob("meta.json"))
+            if meta_files:
+                actual = meta_files[0].parent
+                if clip_dir.exists():
+                    import shutil
+                    shutil.rmtree(clip_dir)
+                import shutil
+                shutil.move(str(actual), str(clip_dir))
+                if extract_tmp.exists() and extract_tmp != clip_dir:
+                    shutil.rmtree(extract_tmp, ignore_errors=True)
+            else:
+                import shutil
+                if clip_dir.exists():
+                    shutil.rmtree(clip_dir)
+                shutil.move(str(extract_tmp), str(clip_dir))
             return clip_dir
 
         return tar_path

{x4d_devkit-0.2.0 → x4d_devkit-0.2.1}/x4d_devkit/client/projects.py

@@ -23,3 +23,14 @@ class ProjectsAPI:
     def get(self, project_id: int) -> dict:
         """Get a project by ID."""
         return self._client.get(f"/projects/{project_id}")
+
+    def list_annotation_classes(self, project_id: int) -> list[dict]:
+        """List the annotation class definitions configured for a project.
+
+        Each item has ``{id, label, color, shortcut?, default_size?}``. The
+        platform returns raw labels as configured — no simplification, no
+        mapping. The training project owns its own `class_mapping:` in its
+        experiment yaml and maps these to its model's class space.
+        """
+        resp = self._client.get(f"/projects/{project_id}/annotation-classes")
+        return resp.get("classes", [])

x4d_devkit-0.2.1/x4d_devkit/client/test_sets.py

@@ -0,0 +1,63 @@
+"""Test sets API wrapper — benchmark clip selections managed by the platform.
+
+Test sets are the platform's way of claiming "these clips are the benchmark:
+don't train on them, don't let experiments move them". See the platform
+CLAUDE.md section on Benchmark Architecture for the design rationale.
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from x4d_devkit.client.client import X4DClient
+
+
+class TestSetsAPI:
+    def __init__(self, client: "X4DClient"):
+        self._client = client
+
+    def create(
+        self,
+        project_id: int,
+        name: str,
+        clip_ids: list[str],
+        description: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> dict:
+        """Create or replace a test set.
+
+        Idempotent on (project_id, name): re-calling with the same name
+        replaces the clip_ids list. Treat that as version replacement, not
+        in-place mutation — if you want to keep the old list, start a new
+        name (`v1` → `v2`).
+        """
+        return self._client.post(
+            "/test-sets",
+            json={
+                "project_id": project_id,
+                "name": name,
+                "description": description,
+                "clip_ids": clip_ids,
+                "metadata": metadata or {},
+            },
+        )
+
+    def list(self, project_id: int) -> list[dict]:
+        """List all test sets for a project."""
+        resp = self._client.get(f"/projects/{project_id}/test-sets")
+        return resp["items"]
+
+    def list_test_clip_ids(self, project_id: int) -> list[str]:
+        """Return the union of clip_ids across every test set in the project.
+
+        A clip is test-marked if it appears in at least one set. Training
+        pipelines use this (indirectly via ``clips.list_all(exclude_test=True)``)
+        to avoid training on benchmark data.
+        """
+        return self._client.get(f"/projects/{project_id}/test-clip-ids")
+
+    def get(self, test_set_id: int) -> dict:
+        return self._client.get(f"/test-sets/{test_set_id}")
+
+    def delete(self, test_set_id: int) -> None:
+        self._client.delete(f"/test-sets/{test_set_id}")

{x4d_devkit-0.2.0 → x4d_devkit-0.2.1}/x4d_devkit/core/loader.py

@@ -27,8 +27,26 @@ _VALID_FRAMES = {"sensor", "ego", "world"}
 class ClipLoader:
     """Load and index an X4D clip directory.
 
-    Eagerly parses all JSON metadata into frozen dataclasses.
+    Eagerly parses all JSON metadata into frozen dataclasses on construction.
     Binary sensor data (point clouds, images) is loaded on demand.
+
+    Args:
+        clip_dir: Path to a clip directory containing meta.json and other
+            X4D JSON/binary files.
+
+    Example::
+
+        loader = ClipLoader("/data/clips/my_clip")
+        print(loader.meta.clip_id)
+
+        # Iterate samples and load point clouds
+        for sample in loader.samples:
+            for sd in loader.sample_data_for_sample(sample.token):
+                if "LIDAR" in sd.channel:
+                    pts = loader.load_point_cloud(sd, frame="ego")
+
+        # Get annotations in ego frame for training
+        anns = loader.annotations_for_sample(sample.token, frame="ego")
     """
 
     def __init__(self, clip_dir: str | Path) -> None:
@@ -200,46 +218,78 @@ class ClipLoader:
 
     @property
     def clip_dir(self) -> Path:
+        """Absolute path to the clip directory."""
         return self._clip_dir
 
     @property
     def meta(self) -> ClipMeta:
+        """Clip metadata (from meta.json)."""
         return self._meta
 
     @property
     def samples(self) -> list[Sample]:
+        """All samples, sorted by timestamp."""
         return self._samples
 
     @property
     def sample_data(self) -> list[SampleData]:
+        """All sample_data records across all channels."""
         return self._sample_data
 
     @property
     def ego_poses(self) -> list[EgoPose]:
+        """All ego poses, sorted by timestamp."""
         return self._ego_poses
 
     @property
     def calibrated_sensors(self) -> list[CalibratedSensor]:
+        """All calibrated sensor records (one per channel)."""
         return self._calibrated_sensors
 
     @property
     def annotations(self) -> list[Annotation]:
+        """All annotations in world frame."""
         return self._annotations
 
     @property
     def instances(self) -> list[Instance]:
+        """All tracked object instances."""
         return self._instances
 
     def get(self, token: str) -> Any:
-        """O(1) lookup by token across all entity types. Raises KeyError if not found."""
+        """O(1) lookup by token across all entity types.
+
+        Args:
+            token: Any entity token (sample, sample_data, annotation, etc.).
+
+        Returns:
+            The entity object matching the token.
+
+        Raises:
+            KeyError: If the token is not found.
+        """
         return self._token_index[token]
 
     def sample_data_for_channel(self, channel: str) -> list[SampleData]:
-        """All sample_data for a channel, sorted by timestamp."""
+        """Get all sample_data for a sensor channel, sorted by timestamp.
+
+        Args:
+            channel: Sensor channel name (e.g. "LIDAR_TOP", "CAM_FRONT").
+
+        Returns:
+            List of SampleData records for this channel, time-ordered.
+        """
         return self._sd_by_channel.get(channel, [])
 
     def sample_data_for_sample(self, sample_token: str) -> list[SampleData]:
-        """All sample_data records for a given sample (all channels at that timestamp)."""
+        """Get all sample_data records for a sample (all channels at that timestamp).
+
+        Args:
+            sample_token: The sample token.
+
+        Returns:
+            List of SampleData records (one per channel) for this sample.
+        """
         return self._sd_by_sample.get(sample_token, [])
 
     def annotations_for_sample(
@@ -350,7 +400,16 @@ class ClipLoader:
             return (world_from_ego @ ego_from_sensor).inverse
 
     def load_image_path(self, sd: SampleData) -> Path:
-        """Return the absolute path to the image file (does not load pixels)."""
+        """Get the absolute path to an image file.
+
+        Does not load the image — returns the path for use with OpenCV, PIL, etc.
+
+        Args:
+            sd: The SampleData record for a camera channel.
+
+        Returns:
+            Absolute Path to the image file.
+        """
         return self._clip_dir / sd.file_path
 
     # --- Private helpers ---