unitlab 3.0.5__tar.gz → 3.1.1__tar.gz

{unitlab-3.0.5 → unitlab-3.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: unitlab
-Version: 3.0.5
+Version: 3.1.1
 Summary: Python SDK for the Unitlab.ai data annotation platform
 Project-URL: Homepage, https://unitlab.ai
 Project-URL: Documentation, https://docs.unitlab.ai
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
-Requires-Dist: httpx>=0.27
+Requires-Dist: httpx[socks]>=0.27
 Requires-Dist: tqdm>=4.60
 Requires-Dist: typer>=0.9
 Provides-Extra: dev
@@ -416,6 +416,72 @@ batch = project.import_cloud(
 status = batch.wait()
 ```
 
+### Import annotation JSON
+
+Media can come from S3 while the annotation JSON stays on your computer:
+
+```python
+project = client.projects.get("PROJECT_ID")
+media = project.import_cloud("CLOUD_STORAGE_ID", ["images/"])
+status = media.wait()
+if status.failed:
+    raise RuntimeError("Some media failed processing; inspect the Batch Queue.")
+
+batch = project.annotation_imports.create(
+    "./predictions.json",
+    source_name="my-model",  # Reuse this namespace for the same prediction source.
+)
+print(batch.preview["counts"], batch.preview["issues"])
+# Inspect the preview before saving labels. Approval is a separate review step.
+report = batch.commit()
+print(report)
+```
+
+Supported inputs are COCO JSON, Unitlab exports (UUEF item JSON, Audio JSON and
+GeoJSON, as `.json` or `.geojson`), `unitlab-image/1` and `unitlab-video/1`. They
+cover image, video, medical, document, audio, time series, text, HTML, geospatial
+and pathology items. A UUEF file holds one item: import each file as its own batch.
+Each JSON file is limited to 600 MiB. The backend matches original media paths/IDs,
+with a unique-filename fallback; ambiguous names require an explicit mapping.
+YOLO text/ZIP files and other tools' project files are not accepted by this importer.
+
+For COCO video, pass `frame_mapping_file="./frames.json"` in
+`unitlab-frame-map/1` format. Use `track_profile="attributes.track_id"` only for
+COCO exports with that explicit track-ID convention. Import video media at its
+original frame rate; annotations for frames omitted by sampling are rejected.
+
+`project.annotation_imports.sources()` lists saved prediction sources, and
+`project.annotation_imports.media(search="road.jpg")` finds media IDs for mappings.
+Pass `source_id=` instead of `source_name=` to reuse a source by ID. Set
+`options={"stable_ids": True}` only when your exporter keeps annotation IDs stable
+between prediction runs. Otherwise the default protects existing objects from
+unconfirmed ID reuse.
+
+After processing completes or you supply mappings, call `batch.refresh()` to
+rebuild the preview. `batch.refresh(options={...})` replaces all import options;
+omitting `options` preserves them. Supported options are `stable_ids`,
+`media_mapping`, `class_mapping`, `object_mapping`, and `review_mapping`.
+
+`batch.commit()` preserves conflicts and human deletions by default. To resolve
+a specific preview annotation, pass `decisions={annotation_key: "keep"}` (or
+`"incoming"` / `"skip"`). Invalid records prevent the whole commit unless you
+explicitly choose `allow_partial=True`. The report includes created, updated,
+kept, unchanged and skipped counts when applicable. A stale preview raises
+`ConflictError`; inspect a refreshed preview before committing again. If a
+commit response is lost, repeat the same batch's `commit()` with the same
+decisions; the server returns the saved report without duplicating labels.
+
+For annotations already in connected storage, omit the local file and pass
+`cloud_storage=storage`, `annotation_key="labels/predictions.json"`, and optionally
+`frame_mapping_key="labels/frames.json"`. Keys are relative to that connection's
+prefix. Media and annotations must resolve within the API key's workspace and
+the selected project.
+
+This requires a backend with the `/api/sdk/projects/{id}/annotation-imports/`
+routes and the annotation-import database migration installed. Commit requires
+a user-created API key so saved labels have an audit author. Legacy keys without
+a creator can preview but receive `PermissionDeniedError` when committing.
+
 ### Data Groups
 
 Let Unitlab suggest a filename pattern:

{unitlab-3.0.5 → unitlab-3.1.1}/README.md

@@ -383,6 +383,72 @@ batch = project.import_cloud(
 status = batch.wait()
 ```
 
+### Import annotation JSON
+
+Media can come from S3 while the annotation JSON stays on your computer:
+
+```python
+project = client.projects.get("PROJECT_ID")
+media = project.import_cloud("CLOUD_STORAGE_ID", ["images/"])
+status = media.wait()
+if status.failed:
+    raise RuntimeError("Some media failed processing; inspect the Batch Queue.")
+
+batch = project.annotation_imports.create(
+    "./predictions.json",
+    source_name="my-model",  # Reuse this namespace for the same prediction source.
+)
+print(batch.preview["counts"], batch.preview["issues"])
+# Inspect the preview before saving labels. Approval is a separate review step.
+report = batch.commit()
+print(report)
+```
+
+Supported inputs are COCO JSON, Unitlab exports (UUEF item JSON, Audio JSON and
+GeoJSON, as `.json` or `.geojson`), `unitlab-image/1` and `unitlab-video/1`. They
+cover image, video, medical, document, audio, time series, text, HTML, geospatial
+and pathology items. A UUEF file holds one item: import each file as its own batch.
+Each JSON file is limited to 600 MiB. The backend matches original media paths/IDs,
+with a unique-filename fallback; ambiguous names require an explicit mapping.
+YOLO text/ZIP files and other tools' project files are not accepted by this importer.
+
+For COCO video, pass `frame_mapping_file="./frames.json"` in
+`unitlab-frame-map/1` format. Use `track_profile="attributes.track_id"` only for
+COCO exports with that explicit track-ID convention. Import video media at its
+original frame rate; annotations for frames omitted by sampling are rejected.
+
+`project.annotation_imports.sources()` lists saved prediction sources, and
+`project.annotation_imports.media(search="road.jpg")` finds media IDs for mappings.
+Pass `source_id=` instead of `source_name=` to reuse a source by ID. Set
+`options={"stable_ids": True}` only when your exporter keeps annotation IDs stable
+between prediction runs. Otherwise the default protects existing objects from
+unconfirmed ID reuse.
+
+After processing completes or you supply mappings, call `batch.refresh()` to
+rebuild the preview. `batch.refresh(options={...})` replaces all import options;
+omitting `options` preserves them. Supported options are `stable_ids`,
+`media_mapping`, `class_mapping`, `object_mapping`, and `review_mapping`.
+
+`batch.commit()` preserves conflicts and human deletions by default. To resolve
+a specific preview annotation, pass `decisions={annotation_key: "keep"}` (or
+`"incoming"` / `"skip"`). Invalid records prevent the whole commit unless you
+explicitly choose `allow_partial=True`. The report includes created, updated,
+kept, unchanged and skipped counts when applicable. A stale preview raises
+`ConflictError`; inspect a refreshed preview before committing again. If a
+commit response is lost, repeat the same batch's `commit()` with the same
+decisions; the server returns the saved report without duplicating labels.
+
+For annotations already in connected storage, omit the local file and pass
+`cloud_storage=storage`, `annotation_key="labels/predictions.json"`, and optionally
+`frame_mapping_key="labels/frames.json"`. Keys are relative to that connection's
+prefix. Media and annotations must resolve within the API key's workspace and
+the selected project.
+
+This requires a backend with the `/api/sdk/projects/{id}/annotation-imports/`
+routes and the annotation-import database migration installed. Commit requires
+a user-created API key so saved labels have an audit author. Legacy keys without
+a creator can preview but receive `PermissionDeniedError` when committing.
+
 ### Data Groups
 
 Let Unitlab suggest a filename pattern:

{unitlab-3.0.5 → unitlab-3.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "unitlab"
-version = "3.0.5"
+version = "3.1.1"
 description = "Python SDK for the Unitlab.ai data annotation platform"
 readme = "README.md"
 license = "MIT"
@@ -23,7 +23,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
 dependencies = [
-    "httpx>=0.27",
+    "httpx[socks]>=0.27",
     "tqdm>=4.60",
     "typer>=0.9",
 ]

{unitlab-3.0.5 → unitlab-3.1.1}/src/unitlab/__init__.py

@@ -5,6 +5,7 @@ from ._scene_uploader import detect_scenes
 from .client import UnitlabClient
 from .exceptions import (
     AmbiguousUploadCompletionError,
+    AnnotationImportError,
     AuthenticationError,
     ConflictError,
     NetworkError,
@@ -27,6 +28,7 @@ from .ontology import (
     TextAttribute,
 )
 from .resources import (
+    AnnotationImport,
     Asset,
     AttachedSource,
     BatchQueue,
@@ -59,6 +61,8 @@ __version__ = version("unitlab")
 
 __all__ = [
     "AmbiguousUploadCompletionError",
+    "AnnotationImport",
+    "AnnotationImportError",
     "Asset",
     "AssetUploadResult",
     "AttachPreview",

{unitlab-3.0.5 → unitlab-3.1.1}/src/unitlab/_downloader.py

@@ -8,7 +8,7 @@ from urllib.parse import urlparse
 import httpx
 import tqdm
 
-from ._http import _safe_path
+from ._http import _safe_path, error_text
 from ._uploader import run_sync
 from .exceptions import NetworkError
 
@@ -45,7 +45,9 @@ def download_annotation(
     except (httpx.HTTPError, OSError) as exc:
         if partial.exists():
             partial.unlink()
-        raise NetworkError(f"Failed to download release: {exc}", exc) from exc
+        raise NetworkError(
+            f"Failed to download release: {error_text(exc)}", exc
+        ) from exc
     return str(target.resolve())
 
 

{unitlab-3.0.5 → unitlab-3.1.1}/src/unitlab/_http.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import time
 from typing import Any
 
 import httpx
@@ -16,6 +17,11 @@ from .exceptions import (
     ValidationError,
 )
 
+# Match HTTPTransport(retries=3): one initial attempt, then three retries for
+# connection failures only. Leave transport construction to HTTPX so that its
+# standard HTTP_PROXY/HTTPS_PROXY/ALL_PROXY and NO_PROXY handling stays active.
+_CONNECT_RETRY_DELAYS = (0.0, 0.5, 1.0)
+
 
 def _extract_error_message(response: httpx.Response) -> str:
     try:
@@ -35,6 +41,9 @@ def _extract_error_message(response: httpx.Response) -> str:
             parts.append(f"{field}: {value}")
         if parts:
             return "; ".join(parts)
+    if isinstance(body, list):
+        # DRF serializes ValidationError("...") raised in a view as ["..."].
+        return "; ".join(str(item) for item in body)
     return str(body)
 
 
@@ -48,6 +57,21 @@ def _extract_error_code(response: httpx.Response) -> str:
     return ""
 
 
+def error_text(exc: BaseException) -> str:
+    """``str(exc)``, or a readable stand-in when it is empty.
+
+    httpx leaves ReadTimeout, ReadError, RemoteProtocolError and ConnectError
+    without a message when the server resets or never answers; an empty
+    string then reached users as a misleading follow-up error.
+    """
+    text = str(exc).strip()
+    if text:
+        return text
+    if isinstance(exc, httpx.TimeoutException):
+        return f"{type(exc).__name__}: no answer before the request timed out"
+    return f"{type(exc).__name__}: the connection closed without a response"
+
+
 def raise_for_response(response: httpx.Response) -> None:
     if response.is_success:
         return
@@ -90,7 +114,6 @@ class HttpApi:
         self.client = httpx.Client(
             base_url=api_url,
             headers={"Authorization": f"Api-Key {api_key}"},
-            transport=httpx.HTTPTransport(retries=3),
             timeout=60.0,
         )
 
@@ -124,11 +147,18 @@ class HttpApi:
             }
             if timeout is not None:
                 kwargs["timeout"] = timeout
-            response = self.client.request(method, endpoint, **kwargs)
+            for attempt in range(len(_CONNECT_RETRY_DELAYS) + 1):
+                try:
+                    response = self.client.request(method, endpoint, **kwargs)
+                    break
+                except (httpx.ConnectError, httpx.ConnectTimeout):
+                    if attempt == len(_CONNECT_RETRY_DELAYS):
+                        raise
+                    time.sleep(_CONNECT_RETRY_DELAYS[attempt])
         except httpx.TimeoutException as exc:
-            raise RequestTimeoutError(str(exc), exc) from exc
+            raise RequestTimeoutError(error_text(exc), exc) from exc
         except httpx.HTTPError as exc:
-            raise NetworkError(str(exc), exc) from exc
+            raise NetworkError(error_text(exc), exc) from exc
         raise_for_response(response)
         if not response.content:
             return None

{unitlab-3.0.5 → unitlab-3.1.1}/src/unitlab/_scene_uploader.py

@@ -43,7 +43,7 @@ import httpx
 import tqdm
 
 from . import _uploader
-from ._http import _extract_error_message, raise_for_response
+from ._http import _extract_error_message, error_text, raise_for_response
 from ._scenes import (
     Scene,
     SceneDetection,
@@ -221,7 +221,7 @@ async def _api_post(
     try:
         response = await client.post(url, json=body)
     except (httpx.HTTPError, httpx.TimeoutException) as exc:
-        raise _SceneRefusedError(str(exc)) from exc
+        raise _SceneRefusedError(error_text(exc)) from exc
     if response.status_code in (401, 403):
         raise_for_response(response)
     if response.status_code == 404:
@@ -306,7 +306,9 @@ async def _put_scene_file(
                 await asyncio.sleep(
                     _uploader.TILED_PART_RETRY_DELAY_SECONDS * (attempt + 1)
                 )
-    raise NetworkError(f"Upload of {file.path} failed: {last_error}", last_error)
+    raise NetworkError(
+        f"Upload of {file.path} failed: {error_text(last_error)}", last_error
+    )
 
 
 async def _upload_scene_folder(

{unitlab-3.0.5 → unitlab-3.1.1}/src/unitlab/_uploader.py

@@ -12,7 +12,7 @@ from typing import Any
 import httpx
 import tqdm
 
-from ._http import _extract_error_message, raise_for_response
+from ._http import _extract_error_message, error_text, raise_for_response
 from .exceptions import (
     AmbiguousUploadCompletionError,
     NetworkError,
@@ -200,7 +200,7 @@ async def _post_file(
     except SubscriptionError:
         raise
     except (httpx.TransportError, httpx.TimeoutException) as exc:
-        return None, str(exc)
+        return None, error_text(exc)
     except ValueError:
         return None, "Unexpected response format"
 
@@ -297,7 +297,9 @@ async def _put_tiled_part(
             last_error = exc
             if attempt + 1 < TILED_PART_MAX_ATTEMPTS:
                 await asyncio.sleep(TILED_PART_RETRY_DELAY_SECONDS * (attempt + 1))
-    raise NetworkError(f"Multipart part upload failed: {last_error}", last_error)
+    raise NetworkError(
+        f"Multipart part upload failed: {error_text(last_error)}", last_error
+    )
 
 
 async def _abort_tiled_upload(
@@ -332,7 +334,7 @@ async def _upload_tiled_file(
     try:
         response = await api_client.post(f"{endpoint}/initiate/", json=initiate)
     except (httpx.HTTPError, httpx.TimeoutException) as exc:
-        raise _TiledUploadRejectedError(str(exc)) from exc
+        raise _TiledUploadRejectedError(error_text(exc)) from exc
     if _can_fallback_from_initiate(response):
         raise _FallbackToSimpleUploadError(_extract_error_message(response))
     if response.status_code in (401, 403):
@@ -549,7 +551,7 @@ async def _finalize_medical_upload(client, endpoint: str, batch_queue_id: str) -
             raise_for_response(response)
             return
         except (httpx.TransportError, httpx.TimeoutException) as exc:
-            last_error = str(exc)
+            last_error = error_text(exc)
             if attempt < RETRY_COUNT:
                 await asyncio.sleep(RETRY_DELAY_SECONDS)
                 continue
@@ -620,12 +622,15 @@ async def _upload_project_async(
     primary_column: str | None,
     batch_size: int,
     show_progress: bool,
+    max_concurrent_uploads: int = UPLOAD_CONCURRENCY,
 ) -> UploadResult:
     failures: list[UploadFailure] = []
     responses: list[dict[str, Any]] = []
     successful_medical = False
     batch_error: Exception | None = None
-    chunk_size = max(1, min(UPLOAD_CONCURRENCY, batch_size))
+    # Files in flight at once; a slow server (one local web process) answers
+    # each upload in seconds, so fewer at a time avoids long queues.
+    chunk_size = max(1, min(max_concurrent_uploads, batch_size))
     progress = tqdm.tqdm(total=len(files), ncols=80, disable=not show_progress)
     try:
         part_semaphore = asyncio.Semaphore(TILED_PART_CONCURRENCY)
@@ -740,9 +745,12 @@ def upload_project(
     primary_column: str | None = None,
     batch_size: int = 100,
     show_progress: bool = True,
+    max_concurrent_uploads: int = UPLOAD_CONCURRENCY,
 ) -> tuple[UploadResult, str]:
     if batch_size < 1:
         raise ValueError("batch_size must be at least 1")
+    if max_concurrent_uploads < 1:
+        raise ValueError("max_concurrent_uploads must be at least 1")
     upload_info = api.get(f"/api/sdk/projects/{project_id}/upload-info/")
     files, preflight_failures = _project_files(source, upload_info, generic_type)
     if not files and not preflight_failures:
@@ -769,6 +777,7 @@ def upload_project(
             primary_column=primary_column,
             batch_size=batch_size,
             show_progress=show_progress,
+            max_concurrent_uploads=max_concurrent_uploads,
         )
     )
     result.total += len(preflight_failures)

{unitlab-3.0.5 → unitlab-3.1.1}/src/unitlab/_waiter.py

@@ -8,10 +8,8 @@ import tqdm
 
 from .exceptions import (
     NetworkError,
-    NotFoundError,
     ProcessingTimeoutError,
     RequestTimeoutError,
-    SubscriptionError,
 )
 from .types import ProcessingStatus
 
@@ -53,9 +51,13 @@ def wait_for_status(
                     progress.refresh()
                 if status.processing == 0:
                     return status
-            except (NotFoundError, SubscriptionError):
-                raise
-            except (NetworkError, RequestTimeoutError):
+            except (NetworkError, RequestTimeoutError) as exc:
+                # A NetworkError subclass is an answer (permission refused,
+                # invalid, not found, conflict, billing) and is raised at once,
+                # as in the uploaders. Only a lost answer (timeout, dropped
+                # connection) or a server error is asked again.
+                if isinstance(exc, NetworkError) and type(exc) is not NetworkError:
+                    raise
                 transient_errors += 1
                 if transient_errors > 3:
                     raise

{unitlab-3.0.5 → unitlab-3.1.1}/src/unitlab/exceptions.py

@@ -65,5 +65,43 @@ class ProcessingTimeoutError(TimeoutError):
         self.status = status
 
 
+class AnnotationImportError(UnitlabError):
+    """Raised when an annotation import cannot be written as previewed.
+
+    ``rows`` lists the blocked annotations (``key``, ``external_key``,
+    ``media_key``, ``reason``) and ``issues`` the file-level problems
+    (``row``, ``reason``) of the preview the server checked. ``batch_id``
+    is the import batch. Nothing was written when this is raised.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        detail: Exception | None = None,
+        code: str = "annotation_import_failed",
+        *,
+        rows: list | None = None,
+        issues: list | None = None,
+        batch_id: str | None = None,
+    ):
+        super().__init__(message, detail, code)
+        self.rows = rows or []
+        self.issues = issues or []
+        self.batch_id = batch_id
+
+    def __str__(self) -> str:
+        # Show the first reasons inline: the server's commit message alone
+        # ("Resolve blocked records ...") does not say which rows.
+        lines = [self.message]
+        reasons = [
+            f"{row.get('external_key') or row.get('key')}: {row.get('reason')}"
+            for row in self.rows
+        ] + [f"{issue.get('row')}: {issue.get('reason')}" for issue in self.issues]
+        lines += [f"  - {reason}" for reason in reasons[:5]]
+        if len(reasons) > 5:
+            lines.append(f"  ... and {len(reasons) - 5} more (see .rows / .issues)")
+        return "\n".join(lines)
+
+
 class AmbiguousUploadCompletionError(NetworkError):
     """Raised when multipart completion may have succeeded server-side."""

{unitlab-3.0.5 → unitlab-3.1.1}/src/unitlab/resources/__init__.py

@@ -1,3 +1,4 @@
+from .annotation_imports import AnnotationImport
 from .assets import Asset, Folder, GroupingResult
 from .cloud import CloudStorage
 from .datasets import Dataset, DatasetItem, DatasetVersion
@@ -8,6 +9,7 @@ from .releases import Release
 from .workflow import Workflow, WorkflowStage, WorkflowTask
 
 __all__ = [
+    "AnnotationImport",
     "Asset",
     "AttachedSource",
     "BatchQueue",