unitlab 3.0.0__tar.gz → 3.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unitlab-3.0.0 → unitlab-3.0.2}/PKG-INFO +23 -5
- {unitlab-3.0.0 → unitlab-3.0.2}/README.md +21 -3
- {unitlab-3.0.0 → unitlab-3.0.2}/pyproject.toml +1 -1
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/__init__.py +2 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/_downloader.py +21 -8
- unitlab-3.0.2/src/unitlab/_uploader.py +779 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/_waiter.py +25 -10
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/cli.py +13 -3
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/exceptions.py +5 -1
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/assets.py +39 -1
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/releases.py +17 -8
- {unitlab-3.0.0 → unitlab-3.0.2}/tests/test_cli.py +37 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/tests/test_client.py +4 -4
- {unitlab-3.0.0 → unitlab-3.0.2}/tests/test_downloader.py +87 -2
- {unitlab-3.0.0 → unitlab-3.0.2}/tests/test_http.py +7 -3
- unitlab-3.0.2/tests/test_tiled_uploads.py +510 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/tests/test_uploader.py +32 -0
- unitlab-3.0.0/src/unitlab/_uploader.py +0 -418
- {unitlab-3.0.0 → unitlab-3.0.2}/.gitignore +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/LICENSE.md +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/__main__.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/_config.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/_grouping.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/_http.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/client.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/ontology.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/py.typed +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/__init__.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/_base.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/cloud.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/datasets.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/embeddings.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/ontologies.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/projects.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/resources/workflow.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/src/unitlab/types.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.2}/tests/test_embeddings.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: unitlab
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.2
|
|
4
4
|
Summary: Python SDK for the Unitlab.ai data annotation platform
|
|
5
5
|
Project-URL: Homepage, https://unitlab.ai
|
|
6
6
|
Project-URL: Documentation, https://docs.unitlab.ai
|
|
@@ -214,6 +214,7 @@ project = client.projects.get("PROJECT_ID")
|
|
|
214
214
|
project = client.projects.create("Road scenes")
|
|
215
215
|
|
|
216
216
|
batch = project.upload("./data", fps=2.0)
|
|
217
|
+
tiled_batch = project.upload("./maps")
|
|
217
218
|
|
|
218
219
|
status = batch.status()
|
|
219
220
|
status = batch.wait(timeout=1800)
|
|
@@ -228,6 +229,11 @@ status = queue.wait()
|
|
|
228
229
|
Completion means `processing == 0`; a completed Batch Queue may still report
|
|
229
230
|
individual failures.
|
|
230
231
|
|
|
232
|
+
GeoTIFF and whole-slide files are detected automatically. Large tiled files
|
|
233
|
+
upload directly to storage in bounded parts. Tiled processing can take hours,
|
|
234
|
+
so use a longer timeout such as `batch.wait(timeout=25200)` when the next
|
|
235
|
+
operation depends on the generated COG/DZI derivative.
|
|
236
|
+
|
|
231
237
|
### Data Units and project lifecycle
|
|
232
238
|
|
|
233
239
|
```python
|
|
@@ -257,12 +263,18 @@ items = folder.list_items()
|
|
|
257
263
|
result = client.assets.upload("./images", folder_id=folder.id, tags=["train"])
|
|
258
264
|
asset = result.assets[0]
|
|
259
265
|
|
|
266
|
+
tiled = client.assets.upload(
|
|
267
|
+
"./slides",
|
|
268
|
+
folder_id=folder.id,
|
|
269
|
+
)
|
|
270
|
+
tiled.assets[0].wait()
|
|
271
|
+
|
|
260
272
|
cloud_folder = client.assets.create_cloud_folder(
|
|
261
273
|
"Incoming",
|
|
262
274
|
"CLOUD_STORAGE_ID",
|
|
263
275
|
prefix="incoming/",
|
|
264
276
|
)
|
|
265
|
-
cloud_folder.sync_cloud()
|
|
277
|
+
cloud_folder.sync_cloud() # Tiled files are registered as processing Assets.
|
|
266
278
|
```
|
|
267
279
|
|
|
268
280
|
### Datasets and published versions
|
|
@@ -377,10 +389,14 @@ release = client.releases.create(
|
|
|
377
389
|
releases = client.releases.list()
|
|
378
390
|
release = client.releases.get(release.id)
|
|
379
391
|
|
|
380
|
-
annotation_path = release.download(split="train")
|
|
392
|
+
annotation_path = release.download(split="train", dest="./release-annotations")
|
|
381
393
|
files_folder = release.download_files(dest="./release-files")
|
|
382
394
|
```
|
|
383
395
|
|
|
396
|
+
Release item download tokens are persistent Unitlab URLs. For files stored in
|
|
397
|
+
customer cloud storage, the URL redirects to a signed target URL; only that target
|
|
398
|
+
URL is temporary.
|
|
399
|
+
|
|
384
400
|
## CLI quick reference
|
|
385
401
|
|
|
386
402
|
```bash
|
|
@@ -394,6 +410,7 @@ unitlab project data-unit PROJECT_ID DATA_UNIT_ID
|
|
|
394
410
|
unitlab project sources PROJECT_ID
|
|
395
411
|
unitlab project detach-source PROJECT_ID SOURCE_LINK_ID --preview
|
|
396
412
|
unitlab project upload PROJECT_ID --source ./data
|
|
413
|
+
unitlab project upload PROJECT_ID --source ./maps
|
|
397
414
|
unitlab batch-queue list PROJECT_ID
|
|
398
415
|
unitlab batch-queue detail PROJECT_ID BATCH_QUEUE_ID
|
|
399
416
|
unitlab batch-queue status PROJECT_ID BATCH_QUEUE_ID
|
|
@@ -408,6 +425,7 @@ unitlab project attach PROJECT_ID --dataset-version DATASET_ID:2
|
|
|
408
425
|
|
|
409
426
|
# Assets
|
|
410
427
|
unitlab assets upload ./data --folder "Raw data" --tag incoming
|
|
428
|
+
unitlab assets upload ./slides --folder "Slides"
|
|
411
429
|
|
|
412
430
|
# Folders
|
|
413
431
|
unitlab folders create "Raw data"
|
|
@@ -440,7 +458,7 @@ unitlab dataset versions DATASET_ID
|
|
|
440
458
|
unitlab release create PROJECT_ID --format UUEF --splits train=100
|
|
441
459
|
unitlab release list
|
|
442
460
|
unitlab release detail RELEASE_ID
|
|
443
|
-
unitlab release download RELEASE_ID --split-type train
|
|
461
|
+
unitlab release download RELEASE_ID --split-type train --dest ./release-annotations
|
|
444
462
|
|
|
445
463
|
# Ontologies
|
|
446
464
|
unitlab ontology list
|
|
@@ -181,6 +181,7 @@ project = client.projects.get("PROJECT_ID")
|
|
|
181
181
|
project = client.projects.create("Road scenes")
|
|
182
182
|
|
|
183
183
|
batch = project.upload("./data", fps=2.0)
|
|
184
|
+
tiled_batch = project.upload("./maps")
|
|
184
185
|
|
|
185
186
|
status = batch.status()
|
|
186
187
|
status = batch.wait(timeout=1800)
|
|
@@ -195,6 +196,11 @@ status = queue.wait()
|
|
|
195
196
|
Completion means `processing == 0`; a completed Batch Queue may still report
|
|
196
197
|
individual failures.
|
|
197
198
|
|
|
199
|
+
GeoTIFF and whole-slide files are detected automatically. Large tiled files
|
|
200
|
+
upload directly to storage in bounded parts. Tiled processing can take hours,
|
|
201
|
+
so use a longer timeout such as `batch.wait(timeout=25200)` when the next
|
|
202
|
+
operation depends on the generated COG/DZI derivative.
|
|
203
|
+
|
|
198
204
|
### Data Units and project lifecycle
|
|
199
205
|
|
|
200
206
|
```python
|
|
@@ -224,12 +230,18 @@ items = folder.list_items()
|
|
|
224
230
|
result = client.assets.upload("./images", folder_id=folder.id, tags=["train"])
|
|
225
231
|
asset = result.assets[0]
|
|
226
232
|
|
|
233
|
+
tiled = client.assets.upload(
|
|
234
|
+
"./slides",
|
|
235
|
+
folder_id=folder.id,
|
|
236
|
+
)
|
|
237
|
+
tiled.assets[0].wait()
|
|
238
|
+
|
|
227
239
|
cloud_folder = client.assets.create_cloud_folder(
|
|
228
240
|
"Incoming",
|
|
229
241
|
"CLOUD_STORAGE_ID",
|
|
230
242
|
prefix="incoming/",
|
|
231
243
|
)
|
|
232
|
-
cloud_folder.sync_cloud()
|
|
244
|
+
cloud_folder.sync_cloud() # Tiled files are registered as processing Assets.
|
|
233
245
|
```
|
|
234
246
|
|
|
235
247
|
### Datasets and published versions
|
|
@@ -344,10 +356,14 @@ release = client.releases.create(
|
|
|
344
356
|
releases = client.releases.list()
|
|
345
357
|
release = client.releases.get(release.id)
|
|
346
358
|
|
|
347
|
-
annotation_path = release.download(split="train")
|
|
359
|
+
annotation_path = release.download(split="train", dest="./release-annotations")
|
|
348
360
|
files_folder = release.download_files(dest="./release-files")
|
|
349
361
|
```
|
|
350
362
|
|
|
363
|
+
Release item download tokens are persistent Unitlab URLs. For files stored in
|
|
364
|
+
customer cloud storage, the URL redirects to a signed target URL; only that target
|
|
365
|
+
URL is temporary.
|
|
366
|
+
|
|
351
367
|
## CLI quick reference
|
|
352
368
|
|
|
353
369
|
```bash
|
|
@@ -361,6 +377,7 @@ unitlab project data-unit PROJECT_ID DATA_UNIT_ID
|
|
|
361
377
|
unitlab project sources PROJECT_ID
|
|
362
378
|
unitlab project detach-source PROJECT_ID SOURCE_LINK_ID --preview
|
|
363
379
|
unitlab project upload PROJECT_ID --source ./data
|
|
380
|
+
unitlab project upload PROJECT_ID --source ./maps
|
|
364
381
|
unitlab batch-queue list PROJECT_ID
|
|
365
382
|
unitlab batch-queue detail PROJECT_ID BATCH_QUEUE_ID
|
|
366
383
|
unitlab batch-queue status PROJECT_ID BATCH_QUEUE_ID
|
|
@@ -375,6 +392,7 @@ unitlab project attach PROJECT_ID --dataset-version DATASET_ID:2
|
|
|
375
392
|
|
|
376
393
|
# Assets
|
|
377
394
|
unitlab assets upload ./data --folder "Raw data" --tag incoming
|
|
395
|
+
unitlab assets upload ./slides --folder "Slides"
|
|
378
396
|
|
|
379
397
|
# Folders
|
|
380
398
|
unitlab folders create "Raw data"
|
|
@@ -407,7 +425,7 @@ unitlab dataset versions DATASET_ID
|
|
|
407
425
|
unitlab release create PROJECT_ID --format UUEF --splits train=100
|
|
408
426
|
unitlab release list
|
|
409
427
|
unitlab release detail RELEASE_ID
|
|
410
|
-
unitlab release download RELEASE_ID --split-type train
|
|
428
|
+
unitlab release download RELEASE_ID --split-type train --dest ./release-annotations
|
|
411
429
|
|
|
412
430
|
# Ontologies
|
|
413
431
|
unitlab ontology list
|
|
@@ -3,6 +3,7 @@ from importlib.metadata import version
|
|
|
3
3
|
from ._grouping import tiles_from_template
|
|
4
4
|
from .client import UnitlabClient
|
|
5
5
|
from .exceptions import (
|
|
6
|
+
AmbiguousUploadCompletionError,
|
|
6
7
|
AuthenticationError,
|
|
7
8
|
NetworkError,
|
|
8
9
|
NotFoundError,
|
|
@@ -55,6 +56,7 @@ from .types import (
|
|
|
55
56
|
__version__ = version("unitlab")
|
|
56
57
|
|
|
57
58
|
__all__ = [
|
|
59
|
+
"AmbiguousUploadCompletionError",
|
|
58
60
|
"Asset",
|
|
59
61
|
"AssetUploadResult",
|
|
60
62
|
"AttachPreview",
|
|
@@ -15,32 +15,45 @@ from .exceptions import NetworkError
|
|
|
15
15
|
DOWNLOAD_CONCURRENCY = 50
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
def download_annotation(
|
|
18
|
+
def download_annotation(
|
|
19
|
+
api,
|
|
20
|
+
release_id: str,
|
|
21
|
+
split: str | None = None,
|
|
22
|
+
dest: str | Path | None = None,
|
|
23
|
+
) -> str:
|
|
19
24
|
payload = {"download_type": "annotation"}
|
|
20
25
|
if split is not None:
|
|
21
26
|
payload["split_type"] = split
|
|
22
|
-
response = api.post(
|
|
27
|
+
response = api.post(
|
|
28
|
+
f"/api/sdk/releases/{release_id}/",
|
|
29
|
+
json=payload,
|
|
30
|
+
timeout=600.0,
|
|
31
|
+
)
|
|
23
32
|
file_url = response["file"]
|
|
24
33
|
filename = os.path.basename(urlparse(file_url).path) or f"{release_id}.zip"
|
|
25
|
-
|
|
34
|
+
destination = Path(dest) if dest is not None else Path()
|
|
35
|
+
target = destination / filename
|
|
36
|
+
partial = target.with_name(f"{target.name}.part")
|
|
26
37
|
try:
|
|
38
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
27
39
|
with httpx.stream("GET", file_url, timeout=300.0) as stream:
|
|
28
40
|
stream.raise_for_status()
|
|
29
|
-
with open(
|
|
41
|
+
with partial.open("wb") as handle:
|
|
30
42
|
for chunk in stream.iter_bytes():
|
|
31
43
|
handle.write(chunk)
|
|
32
|
-
os.replace(partial,
|
|
44
|
+
os.replace(partial, target)
|
|
33
45
|
except (httpx.HTTPError, OSError) as exc:
|
|
34
|
-
if
|
|
35
|
-
|
|
46
|
+
if partial.exists():
|
|
47
|
+
partial.unlink()
|
|
36
48
|
raise NetworkError(f"Failed to download release: {exc}", exc) from exc
|
|
37
|
-
return
|
|
49
|
+
return str(target.resolve())
|
|
38
50
|
|
|
39
51
|
|
|
40
52
|
def download_files(api, release_id: str, dest: str | Path | None = None) -> str:
|
|
41
53
|
response = api.post(
|
|
42
54
|
f"/api/sdk/releases/{release_id}/",
|
|
43
55
|
json={"download_type": "files"},
|
|
56
|
+
timeout=600.0,
|
|
44
57
|
)
|
|
45
58
|
base_folder = str(Path(dest) if dest is not None else Path(release_id))
|
|
46
59
|
os.makedirs(base_folder, exist_ok=True)
|