unitlab 3.0.0__tar.gz → 3.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unitlab-3.0.0 → unitlab-3.0.1}/PKG-INFO +7 -3
- {unitlab-3.0.0 → unitlab-3.0.1}/README.md +6 -2
- {unitlab-3.0.0 → unitlab-3.0.1}/pyproject.toml +1 -1
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/_downloader.py +21 -8
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/cli.py +12 -3
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/releases.py +17 -3
- {unitlab-3.0.0 → unitlab-3.0.1}/tests/test_cli.py +35 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/tests/test_client.py +3 -2
- {unitlab-3.0.0 → unitlab-3.0.1}/tests/test_downloader.py +59 -2
- {unitlab-3.0.0 → unitlab-3.0.1}/tests/test_http.py +7 -3
- {unitlab-3.0.0 → unitlab-3.0.1}/.gitignore +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/LICENSE.md +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/__init__.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/__main__.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/_config.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/_grouping.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/_http.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/_uploader.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/_waiter.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/client.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/exceptions.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/ontology.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/py.typed +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/__init__.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/_base.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/assets.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/cloud.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/datasets.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/embeddings.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/ontologies.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/projects.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/resources/workflow.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/src/unitlab/types.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/tests/test_embeddings.py +0 -0
- {unitlab-3.0.0 → unitlab-3.0.1}/tests/test_uploader.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: unitlab
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.1
|
|
4
4
|
Summary: Python SDK for the Unitlab.ai data annotation platform
|
|
5
5
|
Project-URL: Homepage, https://unitlab.ai
|
|
6
6
|
Project-URL: Documentation, https://docs.unitlab.ai
|
|
@@ -377,10 +377,14 @@ release = client.releases.create(
|
|
|
377
377
|
releases = client.releases.list()
|
|
378
378
|
release = client.releases.get(release.id)
|
|
379
379
|
|
|
380
|
-
annotation_path = release.download(split="train")
|
|
380
|
+
annotation_path = release.download(split="train", dest="./release-annotations")
|
|
381
381
|
files_folder = release.download_files(dest="./release-files")
|
|
382
382
|
```
|
|
383
383
|
|
|
384
|
+
Release item download tokens are persistent Unitlab URLs. For files stored in
|
|
385
|
+
customer cloud storage, the URL redirects to a signed target URL; only that target
|
|
386
|
+
URL is temporary.
|
|
387
|
+
|
|
384
388
|
## CLI quick reference
|
|
385
389
|
|
|
386
390
|
```bash
|
|
@@ -440,7 +444,7 @@ unitlab dataset versions DATASET_ID
|
|
|
440
444
|
unitlab release create PROJECT_ID --format UUEF --splits train=100
|
|
441
445
|
unitlab release list
|
|
442
446
|
unitlab release detail RELEASE_ID
|
|
443
|
-
unitlab release download RELEASE_ID --split-type train
|
|
447
|
+
unitlab release download RELEASE_ID --split-type train --dest ./release-annotations
|
|
444
448
|
|
|
445
449
|
# Ontologies
|
|
446
450
|
unitlab ontology list
|
|
@@ -344,10 +344,14 @@ release = client.releases.create(
|
|
|
344
344
|
releases = client.releases.list()
|
|
345
345
|
release = client.releases.get(release.id)
|
|
346
346
|
|
|
347
|
-
annotation_path = release.download(split="train")
|
|
347
|
+
annotation_path = release.download(split="train", dest="./release-annotations")
|
|
348
348
|
files_folder = release.download_files(dest="./release-files")
|
|
349
349
|
```
|
|
350
350
|
|
|
351
|
+
Release item download tokens are persistent Unitlab URLs. For files stored in
|
|
352
|
+
customer cloud storage, the URL redirects to a signed target URL; only that target
|
|
353
|
+
URL is temporary.
|
|
354
|
+
|
|
351
355
|
## CLI quick reference
|
|
352
356
|
|
|
353
357
|
```bash
|
|
@@ -407,7 +411,7 @@ unitlab dataset versions DATASET_ID
|
|
|
407
411
|
unitlab release create PROJECT_ID --format UUEF --splits train=100
|
|
408
412
|
unitlab release list
|
|
409
413
|
unitlab release detail RELEASE_ID
|
|
410
|
-
unitlab release download RELEASE_ID --split-type train
|
|
414
|
+
unitlab release download RELEASE_ID --split-type train --dest ./release-annotations
|
|
411
415
|
|
|
412
416
|
# Ontologies
|
|
413
417
|
unitlab ontology list
|
|
@@ -15,32 +15,45 @@ from .exceptions import NetworkError
|
|
|
15
15
|
DOWNLOAD_CONCURRENCY = 50
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
def download_annotation(
|
|
18
|
+
def download_annotation(
|
|
19
|
+
api,
|
|
20
|
+
release_id: str,
|
|
21
|
+
split: str | None = None,
|
|
22
|
+
dest: str | Path | None = None,
|
|
23
|
+
) -> str:
|
|
19
24
|
payload = {"download_type": "annotation"}
|
|
20
25
|
if split is not None:
|
|
21
26
|
payload["split_type"] = split
|
|
22
|
-
response = api.post(
|
|
27
|
+
response = api.post(
|
|
28
|
+
f"/api/sdk/releases/{release_id}/",
|
|
29
|
+
json=payload,
|
|
30
|
+
timeout=600.0,
|
|
31
|
+
)
|
|
23
32
|
file_url = response["file"]
|
|
24
33
|
filename = os.path.basename(urlparse(file_url).path) or f"{release_id}.zip"
|
|
25
|
-
|
|
34
|
+
destination = Path(dest) if dest is not None else Path()
|
|
35
|
+
target = destination / filename
|
|
36
|
+
partial = target.with_name(f"{target.name}.part")
|
|
26
37
|
try:
|
|
38
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
27
39
|
with httpx.stream("GET", file_url, timeout=300.0) as stream:
|
|
28
40
|
stream.raise_for_status()
|
|
29
|
-
with open(
|
|
41
|
+
with partial.open("wb") as handle:
|
|
30
42
|
for chunk in stream.iter_bytes():
|
|
31
43
|
handle.write(chunk)
|
|
32
|
-
os.replace(partial,
|
|
44
|
+
os.replace(partial, target)
|
|
33
45
|
except (httpx.HTTPError, OSError) as exc:
|
|
34
|
-
if
|
|
35
|
-
|
|
46
|
+
if partial.exists():
|
|
47
|
+
partial.unlink()
|
|
36
48
|
raise NetworkError(f"Failed to download release: {exc}", exc) from exc
|
|
37
|
-
return
|
|
49
|
+
return str(target.resolve())
|
|
38
50
|
|
|
39
51
|
|
|
40
52
|
def download_files(api, release_id: str, dest: str | Path | None = None) -> str:
|
|
41
53
|
response = api.post(
|
|
42
54
|
f"/api/sdk/releases/{release_id}/",
|
|
43
55
|
json={"download_type": "files"},
|
|
56
|
+
timeout=600.0,
|
|
44
57
|
)
|
|
45
58
|
base_folder = str(Path(dest) if dest is not None else Path(release_id))
|
|
46
59
|
os.makedirs(base_folder, exist_ok=True)
|
|
@@ -1210,12 +1210,15 @@ def release_download(
|
|
|
1210
1210
|
pk: UUID,
|
|
1211
1211
|
download_type: DownloadType = typer.Option(DownloadType.annotation),
|
|
1212
1212
|
split_type: str | None = typer.Option(None),
|
|
1213
|
-
dest: Path | None = typer.Option(
|
|
1213
|
+
dest: Path | None = typer.Option(
|
|
1214
|
+
None,
|
|
1215
|
+
help="Destination directory for annotations or source files.",
|
|
1216
|
+
),
|
|
1214
1217
|
api_key: API_KEY = None,
|
|
1215
1218
|
):
|
|
1216
1219
|
release = get_client(api_key).releases.get(str(pk))
|
|
1217
1220
|
result = (
|
|
1218
|
-
release.download(split_type)
|
|
1221
|
+
release.download(split_type, dest=dest)
|
|
1219
1222
|
if download_type == DownloadType.annotation
|
|
1220
1223
|
else release.download_files(dest=dest)
|
|
1221
1224
|
)
|
|
@@ -1237,7 +1240,13 @@ def release_create(
|
|
|
1237
1240
|
export_type: str = typer.Option("UUEF", "--format"),
|
|
1238
1241
|
splits: str = typer.Option("train=100"),
|
|
1239
1242
|
data_type: list[str] | None = typer.Option(None, "--data-type"),
|
|
1240
|
-
include_download_tokens: bool = typer.Option(
|
|
1243
|
+
include_download_tokens: bool = typer.Option(
|
|
1244
|
+
False,
|
|
1245
|
+
help=(
|
|
1246
|
+
"Include persistent Unitlab item download tokens; only a "
|
|
1247
|
+
"customer-cloud signed target URL is temporary."
|
|
1248
|
+
),
|
|
1249
|
+
),
|
|
1241
1250
|
license_id: UUID | None = typer.Option(None, "--license"),
|
|
1242
1251
|
api_key: API_KEY = None,
|
|
1243
1252
|
json_output: JSON_OUTPUT = False,
|
|
@@ -39,7 +39,9 @@ class ReleasesNamespace(Namespace):
|
|
|
39
39
|
project: Project handle or ID.
|
|
40
40
|
export_type: Primary annotation export format.
|
|
41
41
|
split_ratios: Percentage assigned to each output split.
|
|
42
|
-
include_download_tokens: Include
|
|
42
|
+
include_download_tokens: Include persistent Unitlab item download tokens.
|
|
43
|
+
The Unitlab token URL remains valid; a customer-cloud signed
|
|
44
|
+
target URL returned by it is temporary.
|
|
43
45
|
upload_sessions: Optional Batch Queue handles or IDs to include.
|
|
44
46
|
data_types: Optional public data types to include. ``multimodal``
|
|
45
47
|
includes every available concrete type.
|
|
@@ -83,6 +85,7 @@ class ReleasesNamespace(Namespace):
|
|
|
83
85
|
raw = self._api.post(
|
|
84
86
|
f"/api/sdk/projects/{identifier(project)}/releases/",
|
|
85
87
|
json=payload,
|
|
88
|
+
timeout=600.0,
|
|
86
89
|
)
|
|
87
90
|
return Release._from_raw(self._client, raw)
|
|
88
91
|
|
|
@@ -116,16 +119,27 @@ class Release:
|
|
|
116
119
|
is_public=bool(raw.get("is_public", False)),
|
|
117
120
|
)
|
|
118
121
|
|
|
119
|
-
def download(
|
|
122
|
+
def download(
|
|
123
|
+
self,
|
|
124
|
+
split: str | None = None,
|
|
125
|
+
*,
|
|
126
|
+
dest: str | Path | None = None,
|
|
127
|
+
) -> str:
|
|
120
128
|
"""Download annotations for one split or the combined Release.
|
|
121
129
|
|
|
122
130
|
Args:
|
|
123
131
|
split: Optional split name such as ``train`` or ``test``.
|
|
132
|
+
dest: Destination directory; defaults to the current directory.
|
|
124
133
|
|
|
125
134
|
Returns:
|
|
126
135
|
Absolute path to the downloaded archive.
|
|
127
136
|
"""
|
|
128
|
-
return _downloader.download_annotation(
|
|
137
|
+
return _downloader.download_annotation(
|
|
138
|
+
self._client._api,
|
|
139
|
+
self.id,
|
|
140
|
+
split,
|
|
141
|
+
dest,
|
|
142
|
+
)
|
|
129
143
|
|
|
130
144
|
def download_files(self, dest: str | Path | None = None) -> str:
|
|
131
145
|
"""Download the source files represented by this Release.
|
|
@@ -546,6 +546,41 @@ def test_folder_list_defaults_to_all_and_parent_scopes(monkeypatch):
|
|
|
546
546
|
assert "folder" not in assets_help.stdout.lower()
|
|
547
547
|
|
|
548
548
|
|
|
549
|
+
def test_release_annotation_download_passes_destination(monkeypatch, tmp_path):
|
|
550
|
+
calls = []
|
|
551
|
+
|
|
552
|
+
class Release:
|
|
553
|
+
@staticmethod
|
|
554
|
+
def download(split, *, dest):
|
|
555
|
+
calls.append((split, dest))
|
|
556
|
+
return str(dest / "release.zip")
|
|
557
|
+
|
|
558
|
+
releases = SimpleNamespace(get=lambda _release_id: Release())
|
|
559
|
+
monkeypatch.setattr(
|
|
560
|
+
"unitlab.cli.get_client",
|
|
561
|
+
lambda _api_key: SimpleNamespace(releases=releases),
|
|
562
|
+
)
|
|
563
|
+
release_id = "00000000-0000-0000-0000-000000000001"
|
|
564
|
+
destination = tmp_path / "annotations"
|
|
565
|
+
|
|
566
|
+
result = runner.invoke(
|
|
567
|
+
app,
|
|
568
|
+
[
|
|
569
|
+
"release",
|
|
570
|
+
"download",
|
|
571
|
+
release_id,
|
|
572
|
+
"--split-type",
|
|
573
|
+
"train",
|
|
574
|
+
"--dest",
|
|
575
|
+
str(destination),
|
|
576
|
+
],
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
assert result.exit_code == 0
|
|
580
|
+
assert calls == [("train", destination)]
|
|
581
|
+
assert result.stdout.strip() == str(destination / "release.zip")
|
|
582
|
+
|
|
583
|
+
|
|
549
584
|
def test_console_entrypoint_prints_concise_sdk_errors(monkeypatch, capsys):
|
|
550
585
|
def fail():
|
|
551
586
|
raise UnitlabError("Actionable failure")
|
|
@@ -278,14 +278,15 @@ def test_release_detail_parses_and_downloads(monkeypatch):
|
|
|
278
278
|
|
|
279
279
|
monkeypatch.setattr(
|
|
280
280
|
"unitlab.resources.releases._downloader.download_annotation",
|
|
281
|
-
lambda api, release_id, split: f"{release_id}-{split}.zip",
|
|
281
|
+
lambda api, release_id, split, dest: f"{dest}/{release_id}-{split}.zip",
|
|
282
282
|
)
|
|
283
|
-
assert release.download("train") == "r1-train.zip"
|
|
283
|
+
assert release.download("train", dest="exports") == "exports/r1-train.zip"
|
|
284
284
|
client.close()
|
|
285
285
|
|
|
286
286
|
|
|
287
287
|
def test_multimodal_release_uses_all_available_types():
|
|
288
288
|
def handler(request):
|
|
289
|
+
assert set(request.extensions["timeout"].values()) == {600.0}
|
|
289
290
|
payload = json.loads(request.content)
|
|
290
291
|
assert "generic_types" not in payload
|
|
291
292
|
return httpx.Response(
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
from contextlib import contextmanager
|
|
2
3
|
|
|
3
4
|
import httpx
|
|
4
5
|
import pytest
|
|
5
6
|
|
|
6
7
|
from unitlab import NetworkError
|
|
7
|
-
from unitlab._downloader import download_files
|
|
8
|
+
from unitlab._downloader import download_annotation, download_files
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class BrokenStream(httpx.AsyncByteStream):
|
|
@@ -13,13 +14,68 @@ class BrokenStream(httpx.AsyncByteStream):
|
|
|
13
14
|
raise httpx.ReadError("connection lost")
|
|
14
15
|
|
|
15
16
|
|
|
17
|
+
def test_annotation_download_uses_destination_and_long_manifest_timeout(
|
|
18
|
+
monkeypatch,
|
|
19
|
+
tmp_path,
|
|
20
|
+
):
|
|
21
|
+
posts = []
|
|
22
|
+
streams = []
|
|
23
|
+
file_url = "https://files.test/release-train.zip?signature=temporary"
|
|
24
|
+
|
|
25
|
+
class Api:
|
|
26
|
+
@staticmethod
|
|
27
|
+
def post(*args, **kwargs):
|
|
28
|
+
posts.append((args, kwargs))
|
|
29
|
+
return {"file": file_url}
|
|
30
|
+
|
|
31
|
+
response = httpx.Response(
|
|
32
|
+
200,
|
|
33
|
+
content=b"archive",
|
|
34
|
+
request=httpx.Request("GET", file_url),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
@contextmanager
|
|
38
|
+
def stream(*args, **kwargs):
|
|
39
|
+
streams.append((args, kwargs))
|
|
40
|
+
yield response
|
|
41
|
+
|
|
42
|
+
monkeypatch.setattr(
|
|
43
|
+
"unitlab._downloader.httpx.stream",
|
|
44
|
+
stream,
|
|
45
|
+
)
|
|
46
|
+
destination = tmp_path / "annotations"
|
|
47
|
+
|
|
48
|
+
result = download_annotation(Api(), "release-1", "train", destination)
|
|
49
|
+
|
|
50
|
+
final = destination / "release-train.zip"
|
|
51
|
+
assert result == str(final.resolve())
|
|
52
|
+
assert final.read_bytes() == b"archive"
|
|
53
|
+
assert not (destination / "release-train.zip.part").exists()
|
|
54
|
+
assert streams == [(("GET", file_url), {"timeout": 300.0})]
|
|
55
|
+
assert posts == [
|
|
56
|
+
(
|
|
57
|
+
("/api/sdk/releases/release-1/",),
|
|
58
|
+
{
|
|
59
|
+
"json": {
|
|
60
|
+
"download_type": "annotation",
|
|
61
|
+
"split_type": "train",
|
|
62
|
+
},
|
|
63
|
+
"timeout": 600.0,
|
|
64
|
+
},
|
|
65
|
+
)
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
|
|
16
69
|
def test_failed_download_does_not_leave_a_file_that_next_run_skips(
|
|
17
70
|
monkeypatch,
|
|
18
71
|
tmp_path,
|
|
19
72
|
):
|
|
73
|
+
posts = []
|
|
74
|
+
|
|
20
75
|
class Api:
|
|
21
76
|
@staticmethod
|
|
22
|
-
def post(*
|
|
77
|
+
def post(*args, **kwargs):
|
|
78
|
+
posts.append((args, kwargs))
|
|
23
79
|
return [
|
|
24
80
|
{
|
|
25
81
|
"file_name": "images/scan.png",
|
|
@@ -56,6 +112,7 @@ def test_failed_download_does_not_leave_a_file_that_next_run_skips(
|
|
|
56
112
|
|
|
57
113
|
download_files(Api(), "release-1", destination)
|
|
58
114
|
assert final.read_bytes() == b"complete"
|
|
115
|
+
assert all(kwargs["timeout"] == 600.0 for _args, kwargs in posts)
|
|
59
116
|
|
|
60
117
|
|
|
61
118
|
def test_download_files_works_inside_a_running_event_loop(monkeypatch, tmp_path):
|
|
@@ -53,17 +53,21 @@ def test_forbidden_responses_distinguish_permission_and_subscription_errors():
|
|
|
53
53
|
assert exc.value.code == ""
|
|
54
54
|
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
@pytest.mark.parametrize(
|
|
57
|
+
"code",
|
|
58
|
+
["workspace_read_only", "workspace_license_required"],
|
|
59
|
+
)
|
|
60
|
+
def test_commercial_error_code_is_preserved(code):
|
|
57
61
|
response = httpx.Response(
|
|
58
62
|
403,
|
|
59
63
|
request=httpx.Request("POST", "http://testserver/action"),
|
|
60
64
|
json={
|
|
61
65
|
"detail": "Workspace is read-only.",
|
|
62
|
-
"code":
|
|
66
|
+
"code": code,
|
|
63
67
|
},
|
|
64
68
|
)
|
|
65
69
|
|
|
66
70
|
with pytest.raises(SubscriptionError) as exc:
|
|
67
71
|
raise_for_response(response)
|
|
68
72
|
|
|
69
|
-
assert exc.value.code ==
|
|
73
|
+
assert exc.value.code == code
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|