unitlab 2.4.1__tar.gz → 2.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unitlab-2.4.1 → unitlab-2.4.3}/.claude/settings.local.json +8 -1
- {unitlab-2.4.1 → unitlab-2.4.3}/LICENSE.md +1 -1
- {unitlab-2.4.1 → unitlab-2.4.3}/PKG-INFO +3 -6
- {unitlab-2.4.1 → unitlab-2.4.3}/pyproject.toml +3 -6
- {unitlab-2.4.1 → unitlab-2.4.3}/src/unitlab/cli.py +6 -13
- {unitlab-2.4.1 → unitlab-2.4.3}/src/unitlab/client.py +103 -73
- {unitlab-2.4.1 → unitlab-2.4.3}/.gitignore +0 -0
- {unitlab-2.4.1 → unitlab-2.4.3}/README.md +0 -0
- {unitlab-2.4.1 → unitlab-2.4.3}/src/unitlab/__init__.py +0 -0
- {unitlab-2.4.1 → unitlab-2.4.3}/src/unitlab/__main__.py +0 -0
- {unitlab-2.4.1 → unitlab-2.4.3}/src/unitlab/_config.py +0 -0
- {unitlab-2.4.1 → unitlab-2.4.3}/src/unitlab/exceptions.py +0 -0
|
@@ -36,7 +36,14 @@
|
|
|
36
36
|
"Bash(git update-ref:*)",
|
|
37
37
|
"Bash(gh api:*)",
|
|
38
38
|
"Bash(for user in Shahrullo shohmax mrakhror maxunitlab)",
|
|
39
|
-
"Bash(do echo \"Removing $user...\")"
|
|
39
|
+
"Bash(do echo \"Removing $user...\")",
|
|
40
|
+
"WebFetch(domain:unitlab-storage.s3.us-east-2.amazonaws.com)",
|
|
41
|
+
"Bash(ps:*)",
|
|
42
|
+
"WebFetch(domain:127.0.0.1)",
|
|
43
|
+
"Bash(DATASET_ID=\"b727a6fd-7786-4d40-b75b-56e7f52f378b\")",
|
|
44
|
+
"Bash(pipenv run:*)",
|
|
45
|
+
"Bash(DS=\"b727a6fd-7786-4d40-b75b-56e7f52f378b\")",
|
|
46
|
+
"Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(''''OK'''' if ''''file'''' in d else d\\)\")"
|
|
40
47
|
]
|
|
41
48
|
}
|
|
42
49
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: unitlab
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.3
|
|
4
4
|
Summary: Python SDK for the Unitlab.ai data annotation platform
|
|
5
5
|
Project-URL: Homepage, https://unitlab.ai
|
|
6
6
|
Project-URL: Documentation, https://docs.unitlab.ai
|
|
@@ -19,18 +19,15 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.14
|
|
20
20
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
21
|
Requires-Python: >=3.10
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
Requires-Dist: aiohttp>=3.9
|
|
24
|
-
Requires-Dist: requests>=2.28
|
|
22
|
+
Requires-Dist: httpx>=0.27
|
|
25
23
|
Requires-Dist: tqdm>=4.60
|
|
26
24
|
Requires-Dist: typer>=0.9
|
|
27
25
|
Requires-Dist: validators>=0.20
|
|
28
26
|
Provides-Extra: dev
|
|
29
|
-
Requires-Dist: aioresponses>=0.7; extra == 'dev'
|
|
30
27
|
Requires-Dist: build>=1.0; extra == 'dev'
|
|
31
28
|
Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
|
|
32
29
|
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
33
|
-
Requires-Dist:
|
|
30
|
+
Requires-Dist: respx>=0.21; extra == 'dev'
|
|
34
31
|
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
35
32
|
Requires-Dist: twine>=5.0; extra == 'dev'
|
|
36
33
|
Description-Content-Type: text/markdown
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "unitlab"
|
|
7
|
-
version = "2.4.
|
|
7
|
+
version = "2.4.3"
|
|
8
8
|
description = "Python SDK for the Unitlab.ai data annotation platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -23,9 +23,7 @@ classifiers = [
|
|
|
23
23
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
24
24
|
]
|
|
25
25
|
dependencies = [
|
|
26
|
-
"
|
|
27
|
-
"aiofiles>=23.0",
|
|
28
|
-
"requests>=2.28",
|
|
26
|
+
"httpx>=0.27",
|
|
29
27
|
"tqdm>=4.60",
|
|
30
28
|
"typer>=0.9",
|
|
31
29
|
"validators>=0.20",
|
|
@@ -35,8 +33,7 @@ dependencies = [
|
|
|
35
33
|
dev = [
|
|
36
34
|
"pytest>=7.0",
|
|
37
35
|
"pytest-asyncio>=0.21",
|
|
38
|
-
"
|
|
39
|
-
"aioresponses>=0.7",
|
|
36
|
+
"respx>=0.21",
|
|
40
37
|
"ruff>=0.4",
|
|
41
38
|
"build>=1.0",
|
|
42
39
|
"twine>=5.0",
|
|
@@ -39,7 +39,7 @@ class DownloadType(str, Enum):
|
|
|
39
39
|
|
|
40
40
|
class DatasetSplit(str, Enum):
|
|
41
41
|
train = "train"
|
|
42
|
-
|
|
42
|
+
validation = "validation"
|
|
43
43
|
test = "test"
|
|
44
44
|
|
|
45
45
|
|
|
@@ -120,24 +120,17 @@ def dataset_download(
|
|
|
120
120
|
typer.Option(help="Download type (annotation, files)"),
|
|
121
121
|
] = DownloadType.annotation,
|
|
122
122
|
export_type: Annotated[
|
|
123
|
-
str,
|
|
123
|
+
str,
|
|
124
|
+
typer.Option(help="Export type (e.g. COCO, YOLOv8, YOLOv5, UUEF)"),
|
|
124
125
|
] = "COCO",
|
|
125
126
|
split_type: Annotated[
|
|
126
|
-
DatasetSplit, typer.Option(help="Dataset split (train,
|
|
127
|
+
DatasetSplit, typer.Option(help="Dataset split (train, validation, test)")
|
|
127
128
|
] = DatasetSplit.train,
|
|
128
129
|
):
|
|
129
130
|
if download_type == DownloadType.annotation:
|
|
130
|
-
|
|
131
|
-
raise typer.BadParameter(
|
|
132
|
-
"Export type is required when download type is annotation"
|
|
133
|
-
)
|
|
134
|
-
if not split_type:
|
|
135
|
-
raise typer.BadParameter(
|
|
136
|
-
"Split type is required when download type is annotation"
|
|
137
|
-
)
|
|
138
|
-
get_client(api_key).dataset_download(pk, export_type, split_type)
|
|
131
|
+
get_client(api_key).dataset_download(str(pk), export_type, split_type.value)
|
|
139
132
|
else:
|
|
140
|
-
typer.echo(get_client(api_key).dataset_download_files(pk))
|
|
133
|
+
typer.echo(get_client(api_key).dataset_download_files(str(pk)))
|
|
141
134
|
|
|
142
135
|
|
|
143
136
|
if __name__ == "__main__":
|
|
@@ -4,13 +4,12 @@ import asyncio
|
|
|
4
4
|
import functools
|
|
5
5
|
import logging
|
|
6
6
|
import os
|
|
7
|
+
import uuid
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from typing import Any
|
|
9
|
-
from urllib.parse import
|
|
10
|
+
from urllib.parse import urlparse
|
|
10
11
|
|
|
11
|
-
import
|
|
12
|
-
import aiohttp
|
|
13
|
-
import requests
|
|
12
|
+
import httpx
|
|
14
13
|
import tqdm
|
|
15
14
|
|
|
16
15
|
from ._config import get_api_key, get_api_url
|
|
@@ -28,6 +27,27 @@ _UPLOAD_CONCURRENCY = 20
|
|
|
28
27
|
_DOWNLOAD_CONCURRENCY = 50
|
|
29
28
|
|
|
30
29
|
|
|
30
|
+
def _extract_error_message(response: httpx.Response) -> str:
|
|
31
|
+
"""Extract a human-readable message from an error response."""
|
|
32
|
+
try:
|
|
33
|
+
body = response.json()
|
|
34
|
+
if isinstance(body, dict):
|
|
35
|
+
for key in ("detail", "message"):
|
|
36
|
+
if key in body:
|
|
37
|
+
return str(body[key])
|
|
38
|
+
parts = []
|
|
39
|
+
for field, errors in body.items():
|
|
40
|
+
if isinstance(errors, list):
|
|
41
|
+
parts.append(f"{field}: {', '.join(str(e) for e in errors)}")
|
|
42
|
+
else:
|
|
43
|
+
parts.append(f"{field}: {errors}")
|
|
44
|
+
if parts:
|
|
45
|
+
return "; ".join(parts)
|
|
46
|
+
except (ValueError, KeyError):
|
|
47
|
+
pass
|
|
48
|
+
return response.text
|
|
49
|
+
|
|
50
|
+
|
|
31
51
|
def handle_exceptions(f):
|
|
32
52
|
"""Catch exceptions and throw Unitlab exceptions."""
|
|
33
53
|
|
|
@@ -37,23 +57,23 @@ def handle_exceptions(f):
|
|
|
37
57
|
r = f(self, *args, **kwargs)
|
|
38
58
|
r.raise_for_status()
|
|
39
59
|
return r.json()
|
|
40
|
-
except
|
|
60
|
+
except httpx.TimeoutException as e:
|
|
41
61
|
raise TimeoutError(message=str(e), detail=e) from e
|
|
42
|
-
except
|
|
43
|
-
text = e.response
|
|
62
|
+
except httpx.HTTPStatusError as e:
|
|
63
|
+
text = _extract_error_message(e.response)
|
|
44
64
|
if e.response.status_code == 401:
|
|
45
65
|
raise AuthenticationError(
|
|
46
66
|
message="Authentication failed", detail=e
|
|
47
67
|
) from e
|
|
48
68
|
if e.response.status_code == 403:
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
) from e
|
|
53
|
-
if "not found" in text:
|
|
69
|
+
msg = f"Forbidden: {e.response.status_code} {e.response.reason_phrase}"
|
|
70
|
+
raise SubscriptionError(message=msg, detail=e) from e
|
|
71
|
+
if e.response.status_code == 404 or "not found" in text.lower():
|
|
54
72
|
raise NotFoundError(message=text, detail=e) from e
|
|
55
73
|
raise NetworkError(message=text, detail=e) from e
|
|
56
|
-
except
|
|
74
|
+
except ValueError as e:
|
|
75
|
+
raise NetworkError(message="Unexpected response format", detail=e) from e
|
|
76
|
+
except httpx.HTTPError as e:
|
|
57
77
|
raise NetworkError(message=str(e), detail=e) from e
|
|
58
78
|
|
|
59
79
|
return wrapper
|
|
@@ -123,11 +143,12 @@ class UnitlabClient:
|
|
|
123
143
|
self.api_url: str = (
|
|
124
144
|
api_url or os.environ.get("UNITLAB_API_URL") or get_api_url()
|
|
125
145
|
)
|
|
126
|
-
self.
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
146
|
+
self._client = httpx.Client(
|
|
147
|
+
base_url=self.api_url,
|
|
148
|
+
headers={"Authorization": f"Api-Key {self.api_key}"},
|
|
149
|
+
transport=httpx.HTTPTransport(retries=3),
|
|
150
|
+
timeout=60.0,
|
|
151
|
+
)
|
|
131
152
|
|
|
132
153
|
def close(self) -> None:
|
|
133
154
|
"""Close :class:`UnitlabClient` connections.
|
|
@@ -147,7 +168,7 @@ class UnitlabClient:
|
|
|
147
168
|
with UnitlabClient() as client:
|
|
148
169
|
client.projects()
|
|
149
170
|
"""
|
|
150
|
-
self.
|
|
171
|
+
self._client.close()
|
|
151
172
|
|
|
152
173
|
def __enter__(self) -> UnitlabClient:
|
|
153
174
|
return self
|
|
@@ -160,21 +181,15 @@ class UnitlabClient:
|
|
|
160
181
|
) -> None:
|
|
161
182
|
self.close()
|
|
162
183
|
|
|
163
|
-
def _get_headers(self) -> dict[str, str]:
|
|
164
|
-
return {"Authorization": f"Api-Key {self.api_key}"}
|
|
165
|
-
|
|
166
184
|
@handle_exceptions
|
|
167
|
-
def _get(self, endpoint: str) ->
|
|
168
|
-
return self.
|
|
185
|
+
def _get(self, endpoint: str) -> httpx.Response:
|
|
186
|
+
return self._client.get(endpoint)
|
|
169
187
|
|
|
170
188
|
@handle_exceptions
|
|
171
189
|
def _post(
|
|
172
190
|
self, endpoint: str, data: dict[str, Any] | None = None
|
|
173
|
-
) ->
|
|
174
|
-
return self.
|
|
175
|
-
urljoin(self.api_url, endpoint),
|
|
176
|
-
json=data if data is not None else {},
|
|
177
|
-
)
|
|
191
|
+
) -> httpx.Response:
|
|
192
|
+
return self._client.post(endpoint, json=data if data is not None else {})
|
|
178
193
|
|
|
179
194
|
def projects(self, pretty: int = 0) -> Any:
|
|
180
195
|
return self._get(f"/api/sdk/projects/?pretty={pretty}")
|
|
@@ -233,26 +248,28 @@ class UnitlabClient:
|
|
|
233
248
|
num_batches = (num_files + batch_size - 1) // batch_size
|
|
234
249
|
semaphore = asyncio.Semaphore(_UPLOAD_CONCURRENCY)
|
|
235
250
|
|
|
251
|
+
# Medical projects use session-based upload with finalize
|
|
252
|
+
session_id = str(uuid.uuid4()) if generic_type == "medical" else None
|
|
253
|
+
|
|
236
254
|
async def post_file(
|
|
237
|
-
|
|
255
|
+
client: httpx.AsyncClient, file: str, project_id: str
|
|
238
256
|
) -> int:
|
|
239
|
-
async with semaphore
|
|
240
|
-
|
|
241
|
-
form_data.add_field(
|
|
242
|
-
"file", await f.read(), filename=os.path.basename(file)
|
|
243
|
-
)
|
|
257
|
+
async with semaphore:
|
|
258
|
+
extra_data: dict[str, str] = {}
|
|
244
259
|
if generic_type == "text":
|
|
245
|
-
|
|
260
|
+
extra_data["sentences_per_chunk"] = str(sentences_per_chunk)
|
|
246
261
|
elif generic_type == "video":
|
|
247
|
-
|
|
262
|
+
extra_data["fps"] = str(fps)
|
|
263
|
+
if session_id:
|
|
264
|
+
extra_data["session_id"] = session_id
|
|
248
265
|
|
|
249
266
|
try:
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
267
|
+
with open(file, "rb") as f:
|
|
268
|
+
response = await client.post(
|
|
269
|
+
f"/api/sdk/projects/{project_id}/upload-data/",
|
|
270
|
+
files={"file": (os.path.basename(file), f)},
|
|
271
|
+
data=extra_data,
|
|
272
|
+
)
|
|
256
273
|
response.raise_for_status()
|
|
257
274
|
return 1
|
|
258
275
|
except Exception as e:
|
|
@@ -262,22 +279,30 @@ class UnitlabClient:
|
|
|
262
279
|
async def main() -> None:
|
|
263
280
|
logger.info(f"Uploading {num_files} files to project {project_id}")
|
|
264
281
|
with tqdm.tqdm(total=num_files, ncols=80) as pbar:
|
|
265
|
-
async with
|
|
266
|
-
|
|
267
|
-
|
|
282
|
+
async with httpx.AsyncClient(
|
|
283
|
+
base_url=self.api_url,
|
|
284
|
+
headers={"Authorization": f"Api-Key {self.api_key}"},
|
|
285
|
+
timeout=600.0,
|
|
286
|
+
) as client:
|
|
268
287
|
for i in range(num_batches):
|
|
269
|
-
tasks = [
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
session=session, file=file, project_id=project_id
|
|
276
|
-
)
|
|
277
|
-
)
|
|
288
|
+
tasks = [
|
|
289
|
+
post_file(client=client, file=file, project_id=project_id)
|
|
290
|
+
for file in filtered_files[
|
|
291
|
+
i * batch_size : min((i + 1) * batch_size, num_files)
|
|
292
|
+
]
|
|
293
|
+
]
|
|
278
294
|
for f in asyncio.as_completed(tasks):
|
|
279
295
|
pbar.update(await f)
|
|
280
296
|
|
|
297
|
+
# Finalize medical session after all uploads
|
|
298
|
+
if session_id:
|
|
299
|
+
response = await client.post(
|
|
300
|
+
f"/api/sdk/projects/{project_id}/"
|
|
301
|
+
f"medical-upload-sessions/{session_id}/finalize/",
|
|
302
|
+
)
|
|
303
|
+
response.raise_for_status()
|
|
304
|
+
logger.info("Medical session finalized")
|
|
305
|
+
|
|
281
306
|
asyncio.run(main())
|
|
282
307
|
|
|
283
308
|
def datasets(self, pretty: int = 0) -> Any:
|
|
@@ -295,17 +320,21 @@ class UnitlabClient:
|
|
|
295
320
|
},
|
|
296
321
|
)
|
|
297
322
|
|
|
298
|
-
|
|
299
|
-
|
|
323
|
+
file_url = response["file"]
|
|
324
|
+
try:
|
|
325
|
+
with httpx.stream("GET", file_url, timeout=300.0) as r:
|
|
326
|
+
r.raise_for_status()
|
|
300
327
|
|
|
301
|
-
|
|
302
|
-
|
|
328
|
+
parsed_url = urlparse(file_url)
|
|
329
|
+
filename = os.path.basename(parsed_url.path)
|
|
303
330
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
331
|
+
with open(filename, "wb") as f:
|
|
332
|
+
for chunk in r.iter_bytes():
|
|
333
|
+
f.write(chunk)
|
|
334
|
+
logger.info(f"File: {os.path.abspath(filename)}")
|
|
335
|
+
return os.path.abspath(filename)
|
|
336
|
+
except httpx.HTTPError as e:
|
|
337
|
+
raise NetworkError(message=f"Failed to download file: {e}", detail=e) from e
|
|
309
338
|
|
|
310
339
|
def dataset_download_files(self, dataset_id: str) -> str:
|
|
311
340
|
"""Download files from a dataset.
|
|
@@ -350,26 +379,27 @@ class UnitlabClient:
|
|
|
350
379
|
semaphore = asyncio.Semaphore(_DOWNLOAD_CONCURRENCY)
|
|
351
380
|
|
|
352
381
|
async def download_file(
|
|
353
|
-
|
|
382
|
+
client: httpx.AsyncClient, dataset_file: dict[str, str]
|
|
354
383
|
) -> int:
|
|
355
|
-
async with semaphore
|
|
384
|
+
async with semaphore:
|
|
356
385
|
try:
|
|
357
|
-
|
|
386
|
+
async with client.stream("GET", dataset_file["source"]) as r:
|
|
387
|
+
r.raise_for_status()
|
|
388
|
+
with open(dataset_file["file_path"], "wb") as f:
|
|
389
|
+
async for chunk in r.aiter_bytes():
|
|
390
|
+
f.write(chunk)
|
|
391
|
+
return 1
|
|
358
392
|
except Exception as e:
|
|
359
393
|
logger.error(
|
|
360
394
|
f"Error downloading file {dataset_file['file_name']} - {e}"
|
|
361
395
|
)
|
|
362
396
|
return 0
|
|
363
|
-
async with aiofiles.open(dataset_file["file_path"], "wb") as f:
|
|
364
|
-
async for chunk in r.content.iter_any():
|
|
365
|
-
await f.write(chunk)
|
|
366
|
-
return 1
|
|
367
397
|
|
|
368
398
|
async def main() -> None:
|
|
369
399
|
with tqdm.tqdm(total=len(files_to_download), ncols=80) as pbar:
|
|
370
|
-
async with
|
|
400
|
+
async with httpx.AsyncClient(timeout=600.0) as client:
|
|
371
401
|
tasks = [
|
|
372
|
-
download_file(
|
|
402
|
+
download_file(client=client, dataset_file=df)
|
|
373
403
|
for df in files_to_download
|
|
374
404
|
]
|
|
375
405
|
for f in asyncio.as_completed(tasks):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|