unitlab 3.0.3__tar.gz → 3.0.5__tar.gz

{unitlab-3.0.3 → unitlab-3.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: unitlab
-Version: 3.0.3
+Version: 3.0.5
 Summary: Python SDK for the Unitlab.ai data annotation platform
 Project-URL: Homepage, https://unitlab.ai
 Project-URL: Documentation, https://docs.unitlab.ai
@@ -113,7 +113,8 @@ print(batch.uploaded, batch.batch_queue_id)
 
 The directory may mix images, video, audio, text, PDFs, DICOM, NIfTI, and NRRD.
 One call creates one Batch Queue. Call `batch.wait()` when the next operation
-depends on server-side processing.
+depends on server-side processing. Lidar scenes are uploaded with
+`generic_type="pointcloud"`; see [Lidar scenes](#lidar-scenes).
 
 ### 2. Curate in Assets, publish a dataset, and attach it
 
@@ -234,6 +235,76 @@ upload directly to storage in bounded parts. Tiled processing can take hours,
 so use a longer timeout such as `batch.wait(timeout=25200)` when the next
 operation depends on the generated COG/DZI derivative.
 
+Video, medical (NIfTI, NRRD, DICOM), audio and PDF files larger than 64 MiB
+also upload directly to storage, up to 500 MiB each; smaller ones keep the
+ordinary upload. A large medical file becomes one volume on its own and is not
+grouped with other files of the batch into a series.
+
+DICOM files named without a usable extension (`IM00001`, or a UID) are found by
+their content when a folder is uploaded and are sent as medical; nothing needs
+to be renamed.
+
+### Lidar scenes
+
+A lidar scene is one item whatever it is made of. It is uploaded only when the
+type is named; without it `.pcd`, `.ply`, `.bin` and scene `.zip` files are
+skipped. The workspace needs lidar scenes enabled.
+
+```python
+from unitlab import detect_scenes
+
+# Look first: what would be uploaded, nothing is sent.
+for scene in detect_scenes("./recordings").scenes:
+    print(scene.name, scene.frame_count, scene.point_sensors, scene.cameras)
+
+batch = project.upload("./recordings", generic_type="pointcloud")
+batch.wait()
+
+for item_id in batch.data_item_ids:
+    status = project.scene_status(item_id)
+    print(status.status, status.frame_count, status.error)
+
+project.retry_scene(item_id)  # a failed scene, about a minute after it failed
+```
+
+What `source` may be:
+
+- One file: a `.pcd`, `.ply` or `.bin` cloud, or a `.zip` that contains such
+  frames or a `scene.json`. Any other `.zip` is refused.
+- A folder with a `scene.json`, or with the point clouds of one sensor
+  (`drive_01/lidar/000.pcd`, ...). Image folders whose file names match the
+  frames become its cameras (`drive_01/cam_front/000.jpg`).
+- A folder with one subfolder per sensor that share file names. This is read as
+  one scene with several sensors; pass `split_sensor_folders=True` when each
+  folder is a recording of its own.
+- A folder that holds several such scenes: each one is uploaded as its own
+  item. Scene files left over in the folder are one scene each.
+- A nuScenes dataset (the folder with `samples/` and `v1.0-*/`): one scene per
+  recording, key frames only, with the calibration taken from its tables.
+
+Other files in the folder are not uploaded by this call; upload the folder once
+more without the type for ordinary media. A scene is limited to 20 GiB and
+1,000 frames, and one data unit is used per frame.
+
+`batch.wait()` covers scenes like any other upload. A failed scene is not in
+`project.data_units()`; find it in the Batch Queue (`queue.data()`) and ask
+`project.scene_status(item_id)` for the reason.
+
+Scenes go into Assets the same way, and the existing Asset methods apply:
+
+```python
+result = client.assets.upload(
+    "./recordings", generic_type="pointcloud", folder="Lidar"
+)
+asset = result.assets[0]
+status = asset.wait()          # status.frame_count, status.error
+asset.retry()                  # a failed scene
+asset = client.assets.asset("ASSET_ID")  # a handle by ID, for status and retry
+```
+
+`release.download_files()` writes a scene that was uploaded as a folder under
+`<item file name>/` with the paths it was uploaded with.
+
 ### Data Units and project lifecycle
 
 ```python
@@ -393,6 +464,18 @@ annotation_path = release.download(split="train", dest="./release-annotations")
 files_folder = release.download_files(dest="./release-files")
 ```
 
+`client.releases.create()` waits until the Release is ready. To return immediately
+and wait later:
+
+```python
+release = client.releases.create(project, wait=False)
+release.wait(timeout=7200)  # Wait before downloading.
+```
+
+Check progress with `release.refresh()`, then read `release.status` and
+`release.progress`. Lists show ready Releases; use `client.releases.get(release_id)`
+to check one still being prepared.
+
 Release item download tokens are persistent Unitlab URLs. For files stored in
 customer cloud storage, the URL redirects to a signed target URL; only that target
 URL is temporary.
@@ -411,6 +494,12 @@ unitlab project sources PROJECT_ID
 unitlab project detach-source PROJECT_ID SOURCE_LINK_ID --preview
 unitlab project upload PROJECT_ID --source ./data
 unitlab project upload PROJECT_ID --source ./maps
+unitlab project upload PROJECT_ID --source ./recordings --type pointcloud --dry-run
+unitlab project upload PROJECT_ID --source ./recordings --type pointcloud
+unitlab project upload PROJECT_ID --source ./rig --type pointcloud \
+  --split-sensor-folders
+unitlab project scene-status PROJECT_ID ITEM_ID
+unitlab project scene-retry PROJECT_ID ITEM_ID
 unitlab batch-queue list PROJECT_ID
 unitlab batch-queue detail PROJECT_ID BATCH_QUEUE_ID
 unitlab batch-queue status PROJECT_ID BATCH_QUEUE_ID
@@ -426,6 +515,9 @@ unitlab project attach PROJECT_ID --dataset-version DATASET_ID:2
 # Assets
 unitlab assets upload ./data --folder "Raw data" --tag incoming
 unitlab assets upload ./slides --folder "Slides"
+unitlab assets upload ./recordings --folder "Lidar" --type pointcloud
+unitlab assets status ASSET_ID
+unitlab assets retry ASSET_ID
 
 # Folders
 unitlab folders create "Raw data"
@@ -454,8 +546,9 @@ unitlab dataset unpublished-changes DATASET_ID
 unitlab dataset publish DATASET_ID --title "Initial snapshot"
 unitlab dataset versions DATASET_ID
 
-# Releases
+# Releases (create returns immediately; add --wait to wait until ready)
 unitlab release create PROJECT_ID --format UUEF --splits train=100
+unitlab release wait RELEASE_ID --timeout 7200
 unitlab release list
 unitlab release detail RELEASE_ID
 unitlab release download RELEASE_ID --split-type train --dest ./release-annotations

{unitlab-3.0.3 → unitlab-3.0.5}/README.md

@@ -80,7 +80,8 @@ print(batch.uploaded, batch.batch_queue_id)
 
 The directory may mix images, video, audio, text, PDFs, DICOM, NIfTI, and NRRD.
 One call creates one Batch Queue. Call `batch.wait()` when the next operation
-depends on server-side processing.
+depends on server-side processing. Lidar scenes are uploaded with
+`generic_type="pointcloud"`; see [Lidar scenes](#lidar-scenes).
 
 ### 2. Curate in Assets, publish a dataset, and attach it
 
@@ -201,6 +202,76 @@ upload directly to storage in bounded parts. Tiled processing can take hours,
 so use a longer timeout such as `batch.wait(timeout=25200)` when the next
 operation depends on the generated COG/DZI derivative.
 
+Video, medical (NIfTI, NRRD, DICOM), audio and PDF files larger than 64 MiB
+also upload directly to storage, up to 500 MiB each; smaller ones keep the
+ordinary upload. A large medical file becomes one volume on its own and is not
+grouped with other files of the batch into a series.
+
+DICOM files named without a usable extension (`IM00001`, or a UID) are found by
+their content when a folder is uploaded and are sent as medical; nothing needs
+to be renamed.
+
+### Lidar scenes
+
+A lidar scene is one item whatever it is made of. It is uploaded only when the
+type is named; without it `.pcd`, `.ply`, `.bin` and scene `.zip` files are
+skipped. The workspace needs lidar scenes enabled.
+
+```python
+from unitlab import detect_scenes
+
+# Look first: what would be uploaded, nothing is sent.
+for scene in detect_scenes("./recordings").scenes:
+    print(scene.name, scene.frame_count, scene.point_sensors, scene.cameras)
+
+batch = project.upload("./recordings", generic_type="pointcloud")
+batch.wait()
+
+for item_id in batch.data_item_ids:
+    status = project.scene_status(item_id)
+    print(status.status, status.frame_count, status.error)
+
+project.retry_scene(item_id)  # a failed scene, about a minute after it failed
+```
+
+What `source` may be:
+
+- One file: a `.pcd`, `.ply` or `.bin` cloud, or a `.zip` that contains such
+  frames or a `scene.json`. Any other `.zip` is refused.
+- A folder with a `scene.json`, or with the point clouds of one sensor
+  (`drive_01/lidar/000.pcd`, ...). Image folders whose file names match the
+  frames become its cameras (`drive_01/cam_front/000.jpg`).
+- A folder with one subfolder per sensor that share file names. This is read as
+  one scene with several sensors; pass `split_sensor_folders=True` when each
+  folder is a recording of its own.
+- A folder that holds several such scenes: each one is uploaded as its own
+  item. Scene files left over in the folder are one scene each.
+- A nuScenes dataset (the folder with `samples/` and `v1.0-*/`): one scene per
+  recording, key frames only, with the calibration taken from its tables.
+
+Other files in the folder are not uploaded by this call; upload the folder once
+more without the type for ordinary media. A scene is limited to 20 GiB and
+1,000 frames, and one data unit is used per frame.
+
+`batch.wait()` covers scenes like any other upload. A failed scene is not in
+`project.data_units()`; find it in the Batch Queue (`queue.data()`) and ask
+`project.scene_status(item_id)` for the reason.
+
+Scenes go into Assets the same way, and the existing Asset methods apply:
+
+```python
+result = client.assets.upload(
+    "./recordings", generic_type="pointcloud", folder="Lidar"
+)
+asset = result.assets[0]
+status = asset.wait()          # status.frame_count, status.error
+asset.retry()                  # a failed scene
+asset = client.assets.asset("ASSET_ID")  # a handle by ID, for status and retry
+```
+
+`release.download_files()` writes a scene that was uploaded as a folder under
+`<item file name>/` with the paths it was uploaded with.
+
 ### Data Units and project lifecycle
 
 ```python
@@ -360,6 +431,18 @@ annotation_path = release.download(split="train", dest="./release-annotations")
 files_folder = release.download_files(dest="./release-files")
 ```
 
+`client.releases.create()` waits until the Release is ready. To return immediately
+and wait later:
+
+```python
+release = client.releases.create(project, wait=False)
+release.wait(timeout=7200)  # Wait before downloading.
+```
+
+Check progress with `release.refresh()`, then read `release.status` and
+`release.progress`. Lists show ready Releases; use `client.releases.get(release_id)`
+to check one still being prepared.
+
 Release item download tokens are persistent Unitlab URLs. For files stored in
 customer cloud storage, the URL redirects to a signed target URL; only that target
 URL is temporary.
@@ -378,6 +461,12 @@ unitlab project sources PROJECT_ID
 unitlab project detach-source PROJECT_ID SOURCE_LINK_ID --preview
 unitlab project upload PROJECT_ID --source ./data
 unitlab project upload PROJECT_ID --source ./maps
+unitlab project upload PROJECT_ID --source ./recordings --type pointcloud --dry-run
+unitlab project upload PROJECT_ID --source ./recordings --type pointcloud
+unitlab project upload PROJECT_ID --source ./rig --type pointcloud \
+  --split-sensor-folders
+unitlab project scene-status PROJECT_ID ITEM_ID
+unitlab project scene-retry PROJECT_ID ITEM_ID
 unitlab batch-queue list PROJECT_ID
 unitlab batch-queue detail PROJECT_ID BATCH_QUEUE_ID
 unitlab batch-queue status PROJECT_ID BATCH_QUEUE_ID
@@ -393,6 +482,9 @@ unitlab project attach PROJECT_ID --dataset-version DATASET_ID:2
 # Assets
 unitlab assets upload ./data --folder "Raw data" --tag incoming
 unitlab assets upload ./slides --folder "Slides"
+unitlab assets upload ./recordings --folder "Lidar" --type pointcloud
+unitlab assets status ASSET_ID
+unitlab assets retry ASSET_ID
 
 # Folders
 unitlab folders create "Raw data"
@@ -421,8 +513,9 @@ unitlab dataset unpublished-changes DATASET_ID
 unitlab dataset publish DATASET_ID --title "Initial snapshot"
 unitlab dataset versions DATASET_ID
 
-# Releases
+# Releases (create returns immediately; add --wait to wait until ready)
 unitlab release create PROJECT_ID --format UUEF --splits train=100
+unitlab release wait RELEASE_ID --timeout 7200
 unitlab release list
 unitlab release detail RELEASE_ID
 unitlab release download RELEASE_ID --split-type train --dest ./release-annotations

{unitlab-3.0.3 → unitlab-3.0.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "unitlab"
-version = "3.0.3"
+version = "3.0.5"
 description = "Python SDK for the Unitlab.ai data annotation platform"
 readme = "README.md"
 license = "MIT"

{unitlab-3.0.3 → unitlab-3.0.5}/src/unitlab/__init__.py

@@ -1,10 +1,12 @@
 from importlib.metadata import version
 
 from ._grouping import tiles_from_template
+from ._scene_uploader import detect_scenes
 from .client import UnitlabClient
 from .exceptions import (
     AmbiguousUploadCompletionError,
     AuthenticationError,
+    ConflictError,
     NetworkError,
     NotFoundError,
     PermissionDeniedError,
@@ -67,6 +69,7 @@ __all__ = [
     "ChecklistAttribute",
     "CloudEntry",
     "CloudStorage",
+    "ConflictError",
     "DataUnit",
     "Dataset",
     "DatasetItem",
@@ -100,5 +103,6 @@ __all__ = [
     "WorkflowStage",
     "WorkflowTask",
     "__version__",
+    "detect_scenes",
     "tiles_from_template",
 ]

{unitlab-3.0.3 → unitlab-3.0.5}/src/unitlab/_http.py

@@ -7,6 +7,7 @@ import httpx
 
 from .exceptions import (
     AuthenticationError,
+    ConflictError,
     NetworkError,
     NotFoundError,
     PermissionDeniedError,
@@ -65,6 +66,10 @@ def raise_for_response(response: httpx.Response) -> None:
         if code == "permission_denied":
             raise PermissionDeniedError(message or "Forbidden", error, code)
         raise SubscriptionError(message or "Forbidden", error, code)
+    # Must stay above the message-based NotFound fallback below: a 409 detail
+    # can mention "not found" and still has to surface as a conflict.
+    if response.status_code == 409:
+        raise ConflictError(message, error, code)
     if response.status_code == 404 or "not found" in message.lower():
         raise NotFoundError(message, error, code)
     raise NetworkError(message, error, code)