unitlab 3.0.4__tar.gz → 3.1.0__tar.gz

{unitlab-3.0.4 → unitlab-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: unitlab
-Version: 3.0.4
+Version: 3.1.0
 Summary: Python SDK for the Unitlab.ai data annotation platform
 Project-URL: Homepage, https://unitlab.ai
 Project-URL: Documentation, https://docs.unitlab.ai
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
-Requires-Dist: httpx>=0.27
+Requires-Dist: httpx[socks]>=0.27
 Requires-Dist: tqdm>=4.60
 Requires-Dist: typer>=0.9
 Provides-Extra: dev
@@ -113,7 +113,8 @@ print(batch.uploaded, batch.batch_queue_id)
 
 The directory may mix images, video, audio, text, PDFs, DICOM, NIfTI, and NRRD.
 One call creates one Batch Queue. Call `batch.wait()` when the next operation
-depends on server-side processing.
+depends on server-side processing. Lidar scenes are uploaded with
+`generic_type="pointcloud"`; see [Lidar scenes](#lidar-scenes).
 
 ### 2. Curate in Assets, publish a dataset, and attach it
 
@@ -234,6 +235,76 @@ upload directly to storage in bounded parts. Tiled processing can take hours,
 so use a longer timeout such as `batch.wait(timeout=25200)` when the next
 operation depends on the generated COG/DZI derivative.
 
+Video, medical (NIfTI, NRRD, DICOM), audio and PDF files larger than 64 MiB
+also upload directly to storage, up to 500 MiB each; smaller ones keep the
+ordinary upload. A large medical file becomes one volume on its own and is not
+grouped with other files of the batch into a series.
+
+DICOM files named without a usable extension (`IM00001`, or a UID) are found by
+their content when a folder is uploaded and are sent as medical; nothing needs
+to be renamed.
+
+### Lidar scenes
+
+A lidar scene is one item whatever it is made of. It is uploaded only when the
+type is named; without it `.pcd`, `.ply`, `.bin` and scene `.zip` files are
+skipped. The workspace needs lidar scenes enabled.
+
+```python
+from unitlab import detect_scenes
+
+# Look first: what would be uploaded, nothing is sent.
+for scene in detect_scenes("./recordings").scenes:
+    print(scene.name, scene.frame_count, scene.point_sensors, scene.cameras)
+
+batch = project.upload("./recordings", generic_type="pointcloud")
+batch.wait()
+
+for item_id in batch.data_item_ids:
+    status = project.scene_status(item_id)
+    print(status.status, status.frame_count, status.error)
+
+project.retry_scene(item_id)  # a failed scene, about a minute after it failed
+```
+
+What `source` may be:
+
+- One file: a `.pcd`, `.ply` or `.bin` cloud, or a `.zip` that contains such
+  frames or a `scene.json`. Any other `.zip` is refused.
+- A folder with a `scene.json`, or with the point clouds of one sensor
+  (`drive_01/lidar/000.pcd`, ...). Image folders whose file names match the
+  frames become its cameras (`drive_01/cam_front/000.jpg`).
+- A folder with one subfolder per sensor that share file names. This is read as
+  one scene with several sensors; pass `split_sensor_folders=True` when each
+  folder is a recording of its own.
+- A folder that holds several such scenes: each one is uploaded as its own
+  item. Scene files left over in the folder are one scene each.
+- A nuScenes dataset (the folder with `samples/` and `v1.0-*/`): one scene per
+  recording, key frames only, with the calibration taken from its tables.
+
+Other files in the folder are not uploaded by this call; upload the folder once
+more without the type for ordinary media. A scene is limited to 20 GiB and
+1,000 frames, and one data unit is used per frame.
+
+`batch.wait()` covers scenes like any other upload. A failed scene is not in
+`project.data_units()`; find it in the Batch Queue (`queue.data()`) and ask
+`project.scene_status(item_id)` for the reason.
+
+Scenes go into Assets the same way, and the existing Asset methods apply:
+
+```python
+result = client.assets.upload(
+    "./recordings", generic_type="pointcloud", folder="Lidar"
+)
+asset = result.assets[0]
+status = asset.wait()          # status.frame_count, status.error
+asset.retry()                  # a failed scene
+asset = client.assets.asset("ASSET_ID")  # a handle by ID, for status and retry
+```
+
+`release.download_files()` writes a scene that was uploaded as a folder under
+`<item file name>/` with the paths it was uploaded with.
+
 ### Data Units and project lifecycle
 
 ```python
@@ -345,6 +416,72 @@ batch = project.import_cloud(
 status = batch.wait()
 ```
 
+### Import annotation JSON
+
+Media can come from S3 while the annotation JSON stays on your computer:
+
+```python
+project = client.projects.get("PROJECT_ID")
+media = project.import_cloud("CLOUD_STORAGE_ID", ["images/"])
+status = media.wait()
+if status.failed:
+    raise RuntimeError("Some media failed processing; inspect the Batch Queue.")
+
+batch = project.annotation_imports.create(
+    "./predictions.json",
+    source_name="my-model",  # Reuse this namespace for the same prediction source.
+)
+print(batch.preview["counts"], batch.preview["issues"])
+# Inspect the preview before saving labels. Approval is a separate review step.
+report = batch.commit()
+print(report)
+```
+
+Supported inputs are COCO JSON, Unitlab exports (UUEF item JSON, Audio JSON and
+GeoJSON, as `.json` or `.geojson`), `unitlab-image/1` and `unitlab-video/1`. They
+cover image, video, medical, document, audio, time series, text, HTML, geospatial
+and pathology items. A UUEF file holds one item: import each file as its own batch.
+Each JSON file is limited to 600 MiB. The backend matches original media paths/IDs,
+with a unique-filename fallback; ambiguous names require an explicit mapping.
+YOLO text/ZIP files and other tools' project files are not accepted by this importer.
+
+For COCO video, pass `frame_mapping_file="./frames.json"` in
+`unitlab-frame-map/1` format. Use `track_profile="attributes.track_id"` only for
+COCO exports with that explicit track-ID convention. Import video media at its
+original frame rate; annotations for frames omitted by sampling are rejected.
+
+`project.annotation_imports.sources()` lists saved prediction sources, and
+`project.annotation_imports.media(search="road.jpg")` finds media IDs for mappings.
+Pass `source_id=` instead of `source_name=` to reuse a source by ID. Set
+`options={"stable_ids": True}` only when your exporter keeps annotation IDs stable
+between prediction runs. Otherwise the default protects existing objects from
+unconfirmed ID reuse.
+
+After processing completes or you supply mappings, call `batch.refresh()` to
+rebuild the preview. `batch.refresh(options={...})` replaces all import options;
+omitting `options` preserves them. Supported options are `stable_ids`,
+`media_mapping`, `class_mapping`, `object_mapping`, and `review_mapping`.
+
+`batch.commit()` preserves conflicts and human deletions by default. To resolve
+a specific preview annotation, pass `decisions={annotation_key: "keep"}` (or
+`"incoming"` / `"skip"`). Invalid records prevent the whole commit unless you
+explicitly choose `allow_partial=True`. The report includes created, updated,
+kept, unchanged and skipped counts when applicable. A stale preview raises
+`ConflictError`; inspect a refreshed preview before committing again. If a
+commit response is lost, repeat the same batch's `commit()` with the same
+decisions; the server returns the saved report without duplicating labels.
+
+For annotations already in connected storage, omit the local file and pass
+`cloud_storage=storage`, `annotation_key="labels/predictions.json"`, and optionally
+`frame_mapping_key="labels/frames.json"`. Keys are relative to that connection's
+prefix. Media and annotations must resolve within the API key's workspace and
+the selected project.
+
+This requires a backend with the `/api/sdk/projects/{id}/annotation-imports/`
+routes and the annotation-import database migration installed. Commit requires
+a user-created API key so saved labels have an audit author. Legacy keys without
+a creator can preview but receive `PermissionDeniedError` when committing.
+
 ### Data Groups
 
 Let Unitlab suggest a filename pattern:
@@ -423,6 +560,12 @@ unitlab project sources PROJECT_ID
 unitlab project detach-source PROJECT_ID SOURCE_LINK_ID --preview
 unitlab project upload PROJECT_ID --source ./data
 unitlab project upload PROJECT_ID --source ./maps
+unitlab project upload PROJECT_ID --source ./recordings --type pointcloud --dry-run
+unitlab project upload PROJECT_ID --source ./recordings --type pointcloud
+unitlab project upload PROJECT_ID --source ./rig --type pointcloud \
+  --split-sensor-folders
+unitlab project scene-status PROJECT_ID ITEM_ID
+unitlab project scene-retry PROJECT_ID ITEM_ID
 unitlab batch-queue list PROJECT_ID
 unitlab batch-queue detail PROJECT_ID BATCH_QUEUE_ID
 unitlab batch-queue status PROJECT_ID BATCH_QUEUE_ID
@@ -438,6 +581,9 @@ unitlab project attach PROJECT_ID --dataset-version DATASET_ID:2
 # Assets
 unitlab assets upload ./data --folder "Raw data" --tag incoming
 unitlab assets upload ./slides --folder "Slides"
+unitlab assets upload ./recordings --folder "Lidar" --type pointcloud
+unitlab assets status ASSET_ID
+unitlab assets retry ASSET_ID
 
 # Folders
 unitlab folders create "Raw data"

{unitlab-3.0.4 → unitlab-3.1.0}/README.md

@@ -80,7 +80,8 @@ print(batch.uploaded, batch.batch_queue_id)
 
 The directory may mix images, video, audio, text, PDFs, DICOM, NIfTI, and NRRD.
 One call creates one Batch Queue. Call `batch.wait()` when the next operation
-depends on server-side processing.
+depends on server-side processing. Lidar scenes are uploaded with
+`generic_type="pointcloud"`; see [Lidar scenes](#lidar-scenes).
 
 ### 2. Curate in Assets, publish a dataset, and attach it
 
@@ -201,6 +202,76 @@ upload directly to storage in bounded parts. Tiled processing can take hours,
 so use a longer timeout such as `batch.wait(timeout=25200)` when the next
 operation depends on the generated COG/DZI derivative.
 
+Video, medical (NIfTI, NRRD, DICOM), audio and PDF files larger than 64 MiB
+also upload directly to storage, up to 500 MiB each; smaller ones keep the
+ordinary upload. A large medical file becomes one volume on its own and is not
+grouped with other files of the batch into a series.
+
+DICOM files named without a usable extension (`IM00001`, or a UID) are found by
+their content when a folder is uploaded and are sent as medical; nothing needs
+to be renamed.
+
+### Lidar scenes
+
+A lidar scene is one item whatever it is made of. It is uploaded only when the
+type is named; without it `.pcd`, `.ply`, `.bin` and scene `.zip` files are
+skipped. The workspace needs lidar scenes enabled.
+
+```python
+from unitlab import detect_scenes
+
+# Look first: what would be uploaded, nothing is sent.
+for scene in detect_scenes("./recordings").scenes:
+    print(scene.name, scene.frame_count, scene.point_sensors, scene.cameras)
+
+batch = project.upload("./recordings", generic_type="pointcloud")
+batch.wait()
+
+for item_id in batch.data_item_ids:
+    status = project.scene_status(item_id)
+    print(status.status, status.frame_count, status.error)
+
+project.retry_scene(item_id)  # a failed scene, about a minute after it failed
+```
+
+What `source` may be:
+
+- One file: a `.pcd`, `.ply` or `.bin` cloud, or a `.zip` that contains such
+  frames or a `scene.json`. Any other `.zip` is refused.
+- A folder with a `scene.json`, or with the point clouds of one sensor
+  (`drive_01/lidar/000.pcd`, ...). Image folders whose file names match the
+  frames become its cameras (`drive_01/cam_front/000.jpg`).
+- A folder with one subfolder per sensor that share file names. This is read as
+  one scene with several sensors; pass `split_sensor_folders=True` when each
+  folder is a recording of its own.
+- A folder that holds several such scenes: each one is uploaded as its own
+  item. Scene files left over in the folder are one scene each.
+- A nuScenes dataset (the folder with `samples/` and `v1.0-*/`): one scene per
+  recording, key frames only, with the calibration taken from its tables.
+
+Other files in the folder are not uploaded by this call; upload the folder once
+more without the type for ordinary media. A scene is limited to 20 GiB and
+1,000 frames, and one data unit is used per frame.
+
+`batch.wait()` covers scenes like any other upload. A failed scene is not in
+`project.data_units()`; find it in the Batch Queue (`queue.data()`) and ask
+`project.scene_status(item_id)` for the reason.
+
+Scenes go into Assets the same way, and the existing Asset methods apply:
+
+```python
+result = client.assets.upload(
+    "./recordings", generic_type="pointcloud", folder="Lidar"
+)
+asset = result.assets[0]
+status = asset.wait()          # status.frame_count, status.error
+asset.retry()                  # a failed scene
+asset = client.assets.asset("ASSET_ID")  # a handle by ID, for status and retry
+```
+
+`release.download_files()` writes a scene that was uploaded as a folder under
+`<item file name>/` with the paths it was uploaded with.
+
 ### Data Units and project lifecycle
 
 ```python
@@ -312,6 +383,72 @@ batch = project.import_cloud(
 status = batch.wait()
 ```
 
+### Import annotation JSON
+
+Media can come from S3 while the annotation JSON stays on your computer:
+
+```python
+project = client.projects.get("PROJECT_ID")
+media = project.import_cloud("CLOUD_STORAGE_ID", ["images/"])
+status = media.wait()
+if status.failed:
+    raise RuntimeError("Some media failed processing; inspect the Batch Queue.")
+
+batch = project.annotation_imports.create(
+    "./predictions.json",
+    source_name="my-model",  # Reuse this namespace for the same prediction source.
+)
+print(batch.preview["counts"], batch.preview["issues"])
+# Inspect the preview before saving labels. Approval is a separate review step.
+report = batch.commit()
+print(report)
+```
+
+Supported inputs are COCO JSON, Unitlab exports (UUEF item JSON, Audio JSON and
+GeoJSON, as `.json` or `.geojson`), `unitlab-image/1` and `unitlab-video/1`. They
+cover image, video, medical, document, audio, time series, text, HTML, geospatial
+and pathology items. A UUEF file holds one item: import each file as its own batch.
+Each JSON file is limited to 600 MiB. The backend matches original media paths/IDs,
+with a unique-filename fallback; ambiguous names require an explicit mapping.
+YOLO text/ZIP files and other tools' project files are not accepted by this importer.
+
+For COCO video, pass `frame_mapping_file="./frames.json"` in
+`unitlab-frame-map/1` format. Use `track_profile="attributes.track_id"` only for
+COCO exports with that explicit track-ID convention. Import video media at its
+original frame rate; annotations for frames omitted by sampling are rejected.
+
+`project.annotation_imports.sources()` lists saved prediction sources, and
+`project.annotation_imports.media(search="road.jpg")` finds media IDs for mappings.
+Pass `source_id=` instead of `source_name=` to reuse a source by ID. Set
+`options={"stable_ids": True}` only when your exporter keeps annotation IDs stable
+between prediction runs. Otherwise the default protects existing objects from
+unconfirmed ID reuse.
+
+After processing completes or you supply mappings, call `batch.refresh()` to
+rebuild the preview. `batch.refresh(options={...})` replaces all import options;
+omitting `options` preserves them. Supported options are `stable_ids`,
+`media_mapping`, `class_mapping`, `object_mapping`, and `review_mapping`.
+
+`batch.commit()` preserves conflicts and human deletions by default. To resolve
+a specific preview annotation, pass `decisions={annotation_key: "keep"}` (or
+`"incoming"` / `"skip"`). Invalid records prevent the whole commit unless you
+explicitly choose `allow_partial=True`. The report includes created, updated,
+kept, unchanged and skipped counts when applicable. A stale preview raises
+`ConflictError`; inspect a refreshed preview before committing again. If a
+commit response is lost, repeat the same batch's `commit()` with the same
+decisions; the server returns the saved report without duplicating labels.
+
+For annotations already in connected storage, omit the local file and pass
+`cloud_storage=storage`, `annotation_key="labels/predictions.json"`, and optionally
+`frame_mapping_key="labels/frames.json"`. Keys are relative to that connection's
+prefix. Media and annotations must resolve within the API key's workspace and
+the selected project.
+
+This requires a backend with the `/api/sdk/projects/{id}/annotation-imports/`
+routes and the annotation-import database migration installed. Commit requires
+a user-created API key so saved labels have an audit author. Legacy keys without
+a creator can preview but receive `PermissionDeniedError` when committing.
+
 ### Data Groups
 
 Let Unitlab suggest a filename pattern:
@@ -390,6 +527,12 @@ unitlab project sources PROJECT_ID
 unitlab project detach-source PROJECT_ID SOURCE_LINK_ID --preview
 unitlab project upload PROJECT_ID --source ./data
 unitlab project upload PROJECT_ID --source ./maps
+unitlab project upload PROJECT_ID --source ./recordings --type pointcloud --dry-run
+unitlab project upload PROJECT_ID --source ./recordings --type pointcloud
+unitlab project upload PROJECT_ID --source ./rig --type pointcloud \
+  --split-sensor-folders
+unitlab project scene-status PROJECT_ID ITEM_ID
+unitlab project scene-retry PROJECT_ID ITEM_ID
 unitlab batch-queue list PROJECT_ID
 unitlab batch-queue detail PROJECT_ID BATCH_QUEUE_ID
 unitlab batch-queue status PROJECT_ID BATCH_QUEUE_ID
@@ -405,6 +548,9 @@ unitlab project attach PROJECT_ID --dataset-version DATASET_ID:2
 # Assets
 unitlab assets upload ./data --folder "Raw data" --tag incoming
 unitlab assets upload ./slides --folder "Slides"
+unitlab assets upload ./recordings --folder "Lidar" --type pointcloud
+unitlab assets status ASSET_ID
+unitlab assets retry ASSET_ID
 
 # Folders
 unitlab folders create "Raw data"

{unitlab-3.0.4 → unitlab-3.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "unitlab"
-version = "3.0.4"
+version = "3.1.0"
 description = "Python SDK for the Unitlab.ai data annotation platform"
 readme = "README.md"
 license = "MIT"
@@ -23,7 +23,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
 dependencies = [
-    "httpx>=0.27",
+    "httpx[socks]>=0.27",
     "tqdm>=4.60",
     "typer>=0.9",
 ]

{unitlab-3.0.4 → unitlab-3.1.0}/src/unitlab/__init__.py

@@ -1,6 +1,7 @@
 from importlib.metadata import version
 
 from ._grouping import tiles_from_template
+from ._scene_uploader import detect_scenes
 from .client import UnitlabClient
 from .exceptions import (
     AmbiguousUploadCompletionError,
@@ -26,6 +27,7 @@ from .ontology import (
     TextAttribute,
 )
 from .resources import (
+    AnnotationImport,
     Asset,
     AttachedSource,
     BatchQueue,
@@ -58,6 +60,7 @@ __version__ = version("unitlab")
 
 __all__ = [
     "AmbiguousUploadCompletionError",
+    "AnnotationImport",
     "Asset",
     "AssetUploadResult",
     "AttachPreview",
@@ -102,5 +105,6 @@ __all__ = [
     "WorkflowStage",
     "WorkflowTask",
     "__version__",
+    "detect_scenes",
     "tiles_from_template",
 ]

{unitlab-3.0.4 → unitlab-3.1.0}/src/unitlab/_http.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import time
 from typing import Any
 
 import httpx
@@ -16,6 +17,11 @@ from .exceptions import (
     ValidationError,
 )
 
+# Match HTTPTransport(retries=3): one initial attempt, then three retries for
+# connection failures only. Leave transport construction to HTTPX so that its
+# standard HTTP_PROXY/HTTPS_PROXY/ALL_PROXY and NO_PROXY handling stays active.
+_CONNECT_RETRY_DELAYS = (0.0, 0.5, 1.0)
+
 
 def _extract_error_message(response: httpx.Response) -> str:
     try:
@@ -90,7 +96,6 @@ class HttpApi:
         self.client = httpx.Client(
             base_url=api_url,
             headers={"Authorization": f"Api-Key {api_key}"},
-            transport=httpx.HTTPTransport(retries=3),
             timeout=60.0,
         )
 
@@ -124,7 +129,14 @@ class HttpApi:
             }
             if timeout is not None:
                 kwargs["timeout"] = timeout
-            response = self.client.request(method, endpoint, **kwargs)
+            for attempt in range(len(_CONNECT_RETRY_DELAYS) + 1):
+                try:
+                    response = self.client.request(method, endpoint, **kwargs)
+                    break
+                except (httpx.ConnectError, httpx.ConnectTimeout):
+                    if attempt == len(_CONNECT_RETRY_DELAYS):
+                        raise
+                    time.sleep(_CONNECT_RETRY_DELAYS[attempt])
         except httpx.TimeoutException as exc:
             raise RequestTimeoutError(str(exc), exc) from exc
         except httpx.HTTPError as exc: