unitlab 3.1.0__tar.gz → 3.1.2__tar.gz

{unitlab-3.1.0 → unitlab-3.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: unitlab
-Version: 3.1.0
+Version: 3.1.2
 Summary: Python SDK for the Unitlab.ai data annotation platform
 Project-URL: Homepage, https://unitlab.ai
 Project-URL: Documentation, https://docs.unitlab.ai

{unitlab-3.1.0 → unitlab-3.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "unitlab"
-version = "3.1.0"
+version = "3.1.2"
 description = "Python SDK for the Unitlab.ai data annotation platform"
 readme = "README.md"
 license = "MIT"

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/__init__.py

@@ -5,6 +5,7 @@ from ._scene_uploader import detect_scenes
 from .client import UnitlabClient
 from .exceptions import (
     AmbiguousUploadCompletionError,
+    AnnotationImportError,
     AuthenticationError,
     ConflictError,
     NetworkError,
@@ -61,6 +62,7 @@ __version__ = version("unitlab")
 __all__ = [
     "AmbiguousUploadCompletionError",
     "AnnotationImport",
+    "AnnotationImportError",
     "Asset",
     "AssetUploadResult",
     "AttachPreview",

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/_downloader.py

@@ -8,7 +8,7 @@ from urllib.parse import urlparse
 import httpx
 import tqdm
 
-from ._http import _safe_path
+from ._http import _safe_path, error_text
 from ._uploader import run_sync
 from .exceptions import NetworkError
 
@@ -45,7 +45,9 @@ def download_annotation(
     except (httpx.HTTPError, OSError) as exc:
         if partial.exists():
             partial.unlink()
-        raise NetworkError(f"Failed to download release: {exc}", exc) from exc
+        raise NetworkError(
+            f"Failed to download release: {error_text(exc)}", exc
+        ) from exc
     return str(target.resolve())
 
 

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/_http.py

@@ -41,6 +41,9 @@ def _extract_error_message(response: httpx.Response) -> str:
             parts.append(f"{field}: {value}")
         if parts:
             return "; ".join(parts)
+    if isinstance(body, list):
+        # DRF serializes ValidationError("...") raised in a view as ["..."].
+        return "; ".join(str(item) for item in body)
     return str(body)
 
 
@@ -54,6 +57,21 @@ def _extract_error_code(response: httpx.Response) -> str:
     return ""
 
 
+def error_text(exc: BaseException) -> str:
+    """``str(exc)``, or a readable stand-in when it is empty.
+
+    httpx leaves ReadTimeout, ReadError, RemoteProtocolError and ConnectError
+    without a message when the server resets or never answers; an empty
+    string then reached users as a misleading follow-up error.
+    """
+    text = str(exc).strip()
+    if text:
+        return text
+    if isinstance(exc, httpx.TimeoutException):
+        return f"{type(exc).__name__}: no answer before the request timed out"
+    return f"{type(exc).__name__}: the connection closed without a response"
+
+
 def raise_for_response(response: httpx.Response) -> None:
     if response.is_success:
         return
@@ -138,9 +156,9 @@ class HttpApi:
                         raise
                     time.sleep(_CONNECT_RETRY_DELAYS[attempt])
         except httpx.TimeoutException as exc:
-            raise RequestTimeoutError(str(exc), exc) from exc
+            raise RequestTimeoutError(error_text(exc), exc) from exc
         except httpx.HTTPError as exc:
-            raise NetworkError(str(exc), exc) from exc
+            raise NetworkError(error_text(exc), exc) from exc
         raise_for_response(response)
         if not response.content:
             return None

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/_scene_uploader.py

@@ -43,7 +43,7 @@ import httpx
 import tqdm
 
 from . import _uploader
-from ._http import _extract_error_message, raise_for_response
+from ._http import _extract_error_message, error_text, raise_for_response
 from ._scenes import (
     Scene,
     SceneDetection,
@@ -221,7 +221,7 @@ async def _api_post(
     try:
         response = await client.post(url, json=body)
     except (httpx.HTTPError, httpx.TimeoutException) as exc:
-        raise _SceneRefusedError(str(exc)) from exc
+        raise _SceneRefusedError(error_text(exc)) from exc
     if response.status_code in (401, 403):
         raise_for_response(response)
     if response.status_code == 404:
@@ -306,7 +306,9 @@ async def _put_scene_file(
                 await asyncio.sleep(
                     _uploader.TILED_PART_RETRY_DELAY_SECONDS * (attempt + 1)
                 )
-    raise NetworkError(f"Upload of {file.path} failed: {last_error}", last_error)
+    raise NetworkError(
+        f"Upload of {file.path} failed: {error_text(last_error)}", last_error
+    )
 
 
 async def _upload_scene_folder(

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/_uploader.py

@@ -12,7 +12,7 @@ from typing import Any
 import httpx
 import tqdm
 
-from ._http import _extract_error_message, raise_for_response
+from ._http import _extract_error_message, error_text, raise_for_response
 from .exceptions import (
     AmbiguousUploadCompletionError,
     NetworkError,
@@ -200,7 +200,7 @@ async def _post_file(
     except SubscriptionError:
         raise
     except (httpx.TransportError, httpx.TimeoutException) as exc:
-        return None, str(exc)
+        return None, error_text(exc)
     except ValueError:
         return None, "Unexpected response format"
 
@@ -297,7 +297,9 @@ async def _put_tiled_part(
             last_error = exc
             if attempt + 1 < TILED_PART_MAX_ATTEMPTS:
                 await asyncio.sleep(TILED_PART_RETRY_DELAY_SECONDS * (attempt + 1))
-    raise NetworkError(f"Multipart part upload failed: {last_error}", last_error)
+    raise NetworkError(
+        f"Multipart part upload failed: {error_text(last_error)}", last_error
+    )
 
 
 async def _abort_tiled_upload(
@@ -332,7 +334,7 @@ async def _upload_tiled_file(
     try:
         response = await api_client.post(f"{endpoint}/initiate/", json=initiate)
     except (httpx.HTTPError, httpx.TimeoutException) as exc:
-        raise _TiledUploadRejectedError(str(exc)) from exc
+        raise _TiledUploadRejectedError(error_text(exc)) from exc
     if _can_fallback_from_initiate(response):
         raise _FallbackToSimpleUploadError(_extract_error_message(response))
     if response.status_code in (401, 403):
@@ -549,7 +551,7 @@ async def _finalize_medical_upload(client, endpoint: str, batch_queue_id: str) -
             raise_for_response(response)
             return
         except (httpx.TransportError, httpx.TimeoutException) as exc:
-            last_error = str(exc)
+            last_error = error_text(exc)
             if attempt < RETRY_COUNT:
                 await asyncio.sleep(RETRY_DELAY_SECONDS)
                 continue
@@ -620,12 +622,15 @@ async def _upload_project_async(
     primary_column: str | None,
     batch_size: int,
     show_progress: bool,
+    max_concurrent_uploads: int = UPLOAD_CONCURRENCY,
 ) -> UploadResult:
     failures: list[UploadFailure] = []
     responses: list[dict[str, Any]] = []
     successful_medical = False
     batch_error: Exception | None = None
-    chunk_size = max(1, min(UPLOAD_CONCURRENCY, batch_size))
+    # Files in flight at once; a slow server (one local web process) answers
+    # each upload in seconds, so fewer at a time avoids long queues.
+    chunk_size = max(1, min(max_concurrent_uploads, batch_size))
     progress = tqdm.tqdm(total=len(files), ncols=80, disable=not show_progress)
     try:
         part_semaphore = asyncio.Semaphore(TILED_PART_CONCURRENCY)
@@ -740,9 +745,12 @@ def upload_project(
     primary_column: str | None = None,
     batch_size: int = 100,
     show_progress: bool = True,
+    max_concurrent_uploads: int = UPLOAD_CONCURRENCY,
 ) -> tuple[UploadResult, str]:
     if batch_size < 1:
         raise ValueError("batch_size must be at least 1")
+    if max_concurrent_uploads < 1:
+        raise ValueError("max_concurrent_uploads must be at least 1")
     upload_info = api.get(f"/api/sdk/projects/{project_id}/upload-info/")
     files, preflight_failures = _project_files(source, upload_info, generic_type)
     if not files and not preflight_failures:
@@ -769,6 +777,7 @@ def upload_project(
             primary_column=primary_column,
             batch_size=batch_size,
             show_progress=show_progress,
+            max_concurrent_uploads=max_concurrent_uploads,
         )
     )
     result.total += len(preflight_failures)

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/_waiter.py

@@ -8,10 +8,8 @@ import tqdm
 
 from .exceptions import (
     NetworkError,
-    NotFoundError,
     ProcessingTimeoutError,
     RequestTimeoutError,
-    SubscriptionError,
 )
 from .types import ProcessingStatus
 
@@ -53,9 +51,13 @@ def wait_for_status(
                     progress.refresh()
                 if status.processing == 0:
                     return status
-            except (NotFoundError, SubscriptionError):
-                raise
-            except (NetworkError, RequestTimeoutError):
+            except (NetworkError, RequestTimeoutError) as exc:
+                # A NetworkError subclass is an answer (permission refused,
+                # invalid, not found, conflict, billing) and is raised at once,
+                # as in the uploaders. Only a lost answer (timeout, dropped
+                # connection) or a server error is asked again.
+                if isinstance(exc, NetworkError) and type(exc) is not NetworkError:
+                    raise
                 transient_errors += 1
                 if transient_errors > 3:
                     raise

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/exceptions.py

@@ -65,5 +65,43 @@ class ProcessingTimeoutError(TimeoutError):
         self.status = status
 
 
+class AnnotationImportError(UnitlabError):
+    """Raised when an annotation import cannot be written as previewed.
+
+    ``rows`` lists the blocked annotations (``key``, ``external_key``,
+    ``media_key``, ``reason``) and ``issues`` the file-level problems
+    (``row``, ``reason``) of the preview the server checked. ``batch_id``
+    is the import batch. Nothing was written when this is raised.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        detail: Exception | None = None,
+        code: str = "annotation_import_failed",
+        *,
+        rows: list | None = None,
+        issues: list | None = None,
+        batch_id: str | None = None,
+    ):
+        super().__init__(message, detail, code)
+        self.rows = rows or []
+        self.issues = issues or []
+        self.batch_id = batch_id
+
+    def __str__(self) -> str:
+        # Show the first reasons inline: the server's commit message alone
+        # ("Resolve blocked records ...") does not say which rows.
+        lines = [self.message]
+        reasons = [
+            f"{row.get('external_key') or row.get('key')}: {row.get('reason')}"
+            for row in self.rows
+        ] + [f"{issue.get('row')}: {issue.get('reason')}" for issue in self.issues]
+        lines += [f"  - {reason}" for reason in reasons[:5]]
+        if len(reasons) > 5:
+            lines.append(f"  ... and {len(reasons) - 5} more (see .rows / .issues)")
+        return "\n".join(lines)
+
+
 class AmbiguousUploadCompletionError(NetworkError):
     """Raised when multipart completion may have succeeded server-side."""

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/resources/annotation_imports.py

@@ -7,12 +7,16 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
 
+from ..exceptions import AnnotationImportError, ValidationError
 from ._base import Namespace, identifier
 
 if TYPE_CHECKING:
     from unitlab.client import UnitlabClient
 
 MAX_FILE_BYTES = 600 * 1024 * 1024
+# Preview and commit re-check every row and item on the server; live runs on
+# large files took over a minute, past the client's 60 s default.
+SERVER_WORK_TIMEOUT = 300
 
 
 class AnnotationImports(Namespace):
@@ -119,9 +123,16 @@ class AnnotationImports(Namespace):
                 files[name] = (path.name, stream, "application/json")
             if files:
                 data["options"] = json.dumps(import_options, allow_nan=False)
-                raw = self._api.post(self._endpoint, data=data, files=files)
+                raw = self._api.post(
+                    self._endpoint,
+                    data=data,
+                    files=files,
+                    timeout=SERVER_WORK_TIMEOUT,
+                )
             else:
-                raw = self._api.post(self._endpoint, json=data)
+                raw = self._api.post(
+                    self._endpoint, json=data, timeout=SERVER_WORK_TIMEOUT
+                )
 
         return AnnotationImport(
             id=str(raw["batch_id"]),
@@ -164,19 +175,48 @@ class AnnotationImport:
         *,
         decisions: dict[str, Literal["keep", "incoming", "skip"]] | None = None,
         allow_partial: bool = False,
+        timeout: float = SERVER_WORK_TIMEOUT,
     ) -> dict[str, Any]:
         """Save this preview as labels; approval remains a separate review step.
 
         Conflicts and human deletions default to ``keep``. Blocked records reject
         the entire commit unless ``allow_partial=True``. A stale preview raises
         ConflictError without refreshing or overwriting human work. Repeating a
-        successful commit returns its original report without duplicating labels.
+        successful commit returns its original report without duplicating labels,
+        so after a timeout or a dropped connection call ``commit`` again with
+        the same arguments. ``timeout`` is in seconds (the server checks every
+        item again first, which can take over a minute).
+
+        Raises:
+            AnnotationImportError: Blocked records or file issues and
+                ``allow_partial=False``; ``rows`` and ``issues`` list them from
+                this preview, and nothing was written.
         """
-        return self._client._api.post(
-            f"{self._endpoint}commit/",
-            json={
-                "token": self.preview["token"],
-                "decisions": decisions or {},
-                "allow_partial": allow_partial,
-            },
-        )
+        try:
+            return self._client._api.post(
+                f"{self._endpoint}commit/",
+                json={
+                    "token": self.preview["token"],
+                    "decisions": decisions or {},
+                    "allow_partial": allow_partial,
+                },
+                timeout=timeout,
+            )
+        except ValidationError as exc:
+            # The token matched, so the server judged exactly this preview:
+            # name its blocked rows instead of the server's one-line refusal.
+            rows = [
+                row
+                for row in self.preview.get("annotations") or []
+                if row.get("status") == "blocked"
+            ]
+            issues = self.preview.get("issues") or []
+            if not (rows or issues):
+                raise
+            raise AnnotationImportError(
+                f"{exc.message} Pass allow_partial=True to skip them.",
+                exc,
+                rows=rows,
+                issues=issues,
+                batch_id=self.id,
+            ) from exc

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/resources/projects.py

@@ -267,6 +267,7 @@ class Project:
         primary_column: str | None = None,
         batch_size: int = 100,
         split_sensor_folders: bool = False,
+        max_concurrent_uploads: int = _uploader.UPLOAD_CONCURRENCY,
     ) -> UploadBatch:
         """Upload local files into this Project.
 
@@ -285,6 +286,9 @@ class Project:
             split_sensor_folders: Lidar scenes only. Read sensor folders that
                 share file names as one scene each instead of one scene with
                 several sensors (see ``unitlab.detect_scenes``).
+            max_concurrent_uploads: Files sent at the same time (default 20).
+                Lower it for a slow or local server; a file whose request
+                times out is reported in ``failed``.
 
         Returns:
             Local upload results and the server-side Batch Queue ID. Server
@@ -316,6 +320,7 @@ class Project:
                 primary_column=primary_column,
                 batch_size=batch_size,
                 show_progress=sys.stderr.isatty(),
+                max_concurrent_uploads=max_concurrent_uploads,
             )
         batch = UploadBatch(
             project_id=self.id,

{unitlab-3.1.0 → unitlab-3.1.2}/src/unitlab/resources/releases.py

@@ -35,6 +35,7 @@ class ReleasesNamespace(Namespace):
         data_types=None,
         bundle_formats: dict[str, str] | None = None,
         license_id: str | None = None,
+        completed_only: bool = False,
         wait: bool = True,
         timeout: float = 7200,
     ) -> Release:
@@ -61,6 +62,11 @@ class ReleasesNamespace(Namespace):
                 includes every available concrete type.
             bundle_formats: Per-concrete-data-type formats for multimodal bundles.
             license_id: Optional Dataset license ID.
+            completed_only: Only items whose current workflow status is
+                Complete (a Data Group by its own status), with their latest
+                labels. Raises ``UnitlabError`` (code
+                ``completed_only_unsupported``) if the server does not know the
+                option, instead of returning a Release of every item.
             wait: Block until the Release is ready or has failed.
             timeout: Seconds to wait for preparation when ``wait`` is true.
 
@@ -98,6 +104,9 @@ class ReleasesNamespace(Namespace):
             }
         if license_id is not None:
             payload["license"] = identifier(license_id)
+        if completed_only:
+            # Sent only when asked, so every other request body stays as before.
+            payload["completed_only"] = True
         # The long request timeout stays: a backend that predates background
         # preparation still builds the whole Release inside this POST.
         raw = self._api.post(
@@ -106,6 +115,14 @@ class ReleasesNamespace(Namespace):
             timeout=600.0,
         )
         release = Release._from_raw(self._client, raw)
+        if completed_only and not release.completed_only:
+            # An older server drops the unknown field and releases every item;
+            # its answer then lacks the echo. Say so rather than hand that back.
+            raise UnitlabError(
+                "This Unitlab server does not support completed_only releases; "
+                f"Release {release.id} was created with every labelled item.",
+                code="completed_only_unsupported",
+            )
         return release.wait(timeout=timeout) if wait else release
 
 
@@ -124,6 +141,8 @@ class Release:
     status: str | None = None
     progress: int | None = None
     status_error: str | None = None
+    # True when the Release holds only items that were Complete.
+    completed_only: bool = False
 
     @classmethod
     def _from_raw(cls, client: UnitlabClient, raw: dict[str, Any]) -> Release:
@@ -140,6 +159,7 @@ class Release:
             status=raw.get("status"),
             progress=raw.get("progress"),
             status_error=raw.get("status_error"),
+            completed_only=bool(raw.get("completed_only", False)),
         )
 
     def refresh(self) -> Release:

{unitlab-3.1.0 → unitlab-3.1.2}/tests/test_annotation_imports.py

@@ -258,7 +258,7 @@ def test_file_size_limit_and_closed_streams_on_request_failure(
     project = project_factory(lambda request: httpx.Response(500))
     opened = []
 
-    def post(endpoint, *, data, files):
+    def post(endpoint, *, data, files, **kwargs):
         opened.extend(item[1] for item in files.values())
         raise RuntimeError("request failed")
 

{unitlab-3.1.0 → unitlab-3.1.2}/tests/test_client.py

@@ -34,6 +34,84 @@ def client_with_handler(handler):
     return client
 
 
+def test_step_by_step_commit_lists_blocked_rows(tmp_path):
+    from unitlab.exceptions import AnnotationImportError
+
+    source = tmp_path / "predictions.json"
+    source.write_text("{}")
+    blocked = {
+        "key": "3",
+        "external_key": "ann-3",
+        "status": "blocked",
+        "reason": "Geometry lies outside original media dimensions.",
+    }
+    timeouts = []
+
+    def handler(request):
+        timeouts.append(request.extensions["timeout"]["read"])
+        if request.url.path.endswith("/commit/"):
+            return httpx.Response(
+                400, json=["Resolve blocked records or explicitly choose to skip."]
+            )
+        return httpx.Response(
+            201,
+            json={
+                "batch_id": "b1",
+                "source_id": "s1",
+                "token": "t1",
+                "annotations": [blocked],
+            },
+        )
+
+    client = client_with_handler(handler)
+    project = Project(id="project-id", name="Images", raw={}, _client=client)
+    batch = project.annotation_imports.create(source, source_name="model-a")
+    with pytest.raises(AnnotationImportError) as error:
+        batch.commit()
+    assert error.value.rows == [blocked]
+    assert error.value.batch_id == "b1"
+    # The server's list body reads as text, not as "['...']".
+    assert str(error.value).startswith("Resolve blocked records")
+    assert "ann-3: Geometry lies outside" in str(error.value)
+    assert "allow_partial=True" in str(error.value)
+    # Preview and commit re-check every item on the server: past 60 s.
+    assert timeouts == [300, 300]
+    client.close()
+
+
+def test_wait_raises_refusals_at_once_and_retries_server_errors(monkeypatch):
+    from unitlab import _waiter
+    from unitlab.exceptions import PermissionDeniedError, ValidationError
+
+    monkeypatch.setattr(_waiter.time, "sleep", lambda _: None)
+    answers = []
+    done = {"total": 1, "completed": 1, "processing": 0, "failed": 0}
+
+    def handler(request):
+        return answers.pop(0)
+
+    client = client_with_handler(handler)
+    for refusal, error in (
+        (
+            httpx.Response(403, json={"detail": "No.", "code": "permission_denied"}),
+            PermissionDeniedError,
+        ),
+        (httpx.Response(400, json={"detail": "Bad."}), ValidationError),
+    ):
+        answers[:] = [refusal, httpx.Response(200, json=done)]
+        with pytest.raises(error):
+            _waiter.wait_for_status(client._api, "/status/", resource_name="Q")
+        assert len(answers) == 1  # asked once, not retried
+
+    answers[:] = [
+        httpx.Response(503, json={"detail": "Busy."}),
+        httpx.Response(200, json=done),
+    ]
+    status = _waiter.wait_for_status(client._api, "/status/", resource_name="Q")
+    assert status.completed == 1 and not answers
+    client.close()
+
+
 def test_duplicate_public_methods_are_removed():
     removed = (
         (

{unitlab-3.1.0 → unitlab-3.1.2}/tests/test_uploader.py

@@ -49,6 +49,65 @@ def configured_client(handler):
     return client
 
 
+def test_empty_httpx_errors_get_a_readable_message():
+    from unitlab._http import error_text
+
+    assert error_text(httpx.ReadTimeout("")) == (
+        "ReadTimeout: no answer before the request timed out"
+    )
+    assert error_text(httpx.RemoteProtocolError("")).startswith(
+        "RemoteProtocolError: the connection closed"
+    )
+    assert error_text(httpx.ConnectError("refused")) == "refused"
+
+
+def test_upload_reports_dropped_connections_and_caps_files_in_flight(tmp_path):
+    import asyncio
+
+    for index in range(5):
+        (tmp_path / f"img{index}.png").write_bytes(b"png")
+    in_flight = peak = 0
+
+    def info(request):
+        return httpx.Response(
+            200,
+            json={
+                "accepted_formats": ["png"],
+                "max_file_sizes": {"img": 1_000_000},
+                "max_file_size": 1_000_000,
+                "generic_type": None,
+            },
+        )
+
+    async def upload(request):
+        nonlocal in_flight, peak
+        in_flight += 1
+        peak = max(peak, in_flight)
+        await asyncio.sleep(0.01)
+        in_flight -= 1
+        if b'filename="img0.png"' in request.content:
+            # What a restarting server does: httpx raises with no message.
+            raise httpx.RemoteProtocolError("")
+        return httpx.Response(201, json={"datasource_id": "ds", "generic_type": "img"})
+
+    client = UnitlabClient(api_key="key", api_url="http://testserver")
+    client._api.client.close()
+    client._api.client = httpx.Client(
+        base_url="http://testserver", transport=httpx.MockTransport(info)
+    )
+    client._api.async_client = lambda *, timeout=600.0: httpx.AsyncClient(
+        base_url="http://testserver", transport=httpx.MockTransport(upload)
+    )
+    project = Project(id="project-id", name="Images", raw={}, _client=client)
+    batch = project.upload(tmp_path, max_concurrent_uploads=2)
+    assert peak == 2
+    assert batch.uploaded == 4
+    assert [failure.path.name for failure in batch.failed] == ["img0.png"]
+    assert "connection closed" in batch.failed[0].error
+    assert "missing datasource_id" not in batch.failed[0].error
+    client.close()
+
+
 def test_project_upload_uses_one_batch_queue_and_chunks(tmp_path):
     session_ids = []
     uploads = 0