wandb 0.18.1__py3-none-win32.whl → 0.18.2__py3-none-win32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +3 -3
- wandb/__init__.pyi +67 -12
- wandb/apis/internal.py +3 -0
- wandb/apis/public/api.py +128 -2
- wandb/apis/public/artifacts.py +11 -7
- wandb/apis/public/jobs.py +8 -0
- wandb/apis/public/runs.py +16 -5
- wandb/bin/wandb-core +0 -0
- wandb/cli/cli.py +0 -3
- wandb/errors/__init__.py +11 -40
- wandb/errors/errors.py +37 -0
- wandb/errors/warnings.py +2 -0
- wandb/integration/tensorboard/log.py +1 -1
- wandb/old/core.py +2 -80
- wandb/plot/bar.py +7 -4
- wandb/plot/confusion_matrix.py +5 -4
- wandb/plot/histogram.py +7 -4
- wandb/plot/line.py +7 -4
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/sdk/artifacts/_validators.py +48 -3
- wandb/sdk/artifacts/artifact.py +157 -183
- wandb/sdk/artifacts/artifact_file_cache.py +13 -11
- wandb/sdk/artifacts/artifact_instance_cache.py +4 -2
- wandb/sdk/artifacts/artifact_manifest.py +13 -11
- wandb/sdk/artifacts/artifact_manifest_entry.py +24 -22
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +9 -7
- wandb/sdk/artifacts/artifact_saver.py +27 -25
- wandb/sdk/artifacts/exceptions.py +26 -25
- wandb/sdk/artifacts/storage_handler.py +11 -9
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +16 -14
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +15 -13
- wandb/sdk/artifacts/storage_handlers/http_handler.py +15 -14
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +10 -8
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +14 -12
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +19 -19
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +10 -8
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +12 -10
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +9 -7
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +31 -29
- wandb/sdk/artifacts/storage_policy.py +20 -20
- wandb/sdk/backend/backend.py +8 -26
- wandb/sdk/data_types/base_types/wb_value.py +1 -3
- wandb/sdk/data_types/video.py +2 -2
- wandb/sdk/interface/interface.py +0 -24
- wandb/sdk/interface/interface_shared.py +0 -12
- wandb/sdk/internal/handler.py +0 -10
- wandb/sdk/internal/internal_api.py +71 -0
- wandb/sdk/internal/sender.py +0 -43
- wandb/sdk/internal/tb_watcher.py +1 -1
- wandb/sdk/lib/_settings_toposort_generated.py +1 -0
- wandb/sdk/lib/hashutil.py +34 -12
- wandb/sdk/lib/service_connection.py +216 -0
- wandb/sdk/lib/service_token.py +94 -0
- wandb/sdk/lib/sock_client.py +7 -3
- wandb/sdk/service/server.py +2 -5
- wandb/sdk/service/service.py +0 -22
- wandb/sdk/wandb_init.py +32 -22
- wandb/sdk/wandb_run.py +12 -7
- wandb/sdk/wandb_settings.py +2 -0
- wandb/sdk/wandb_setup.py +25 -16
- wandb/sdk/wandb_sync.py +9 -3
- wandb/sdk/wandb_watch.py +31 -15
- wandb/util.py +8 -1
- {wandb-0.18.1.dist-info → wandb-0.18.2.dist-info}/METADATA +2 -1
- {wandb-0.18.1.dist-info → wandb-0.18.2.dist-info}/RECORD +71 -71
- wandb/sdk/internal/update.py +0 -113
- wandb/sdk/service/service_base.py +0 -50
- wandb/sdk/service/service_sock.py +0 -70
- wandb/sdk/wandb_manager.py +0 -232
- /wandb/{sdk/lib → plot}/viz.py +0 -0
- {wandb-0.18.1.dist-info → wandb-0.18.2.dist-info}/WHEEL +0 -0
- {wandb-0.18.1.dist-info → wandb-0.18.2.dist-info}/entry_points.txt +0 -0
- {wandb-0.18.1.dist-info → wandb-0.18.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,8 +1,10 @@
|
|
1
1
|
"""GCS storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import time
|
4
6
|
from pathlib import PurePosixPath
|
5
|
-
from typing import TYPE_CHECKING,
|
7
|
+
from typing import TYPE_CHECKING, Sequence
|
6
8
|
from urllib.parse import ParseResult, urlparse
|
7
9
|
|
8
10
|
from wandb import util
|
@@ -26,17 +28,17 @@ class _GCSIsADirectoryError(Exception):
|
|
26
28
|
|
27
29
|
|
28
30
|
class GCSHandler(StorageHandler):
|
29
|
-
_client:
|
31
|
+
_client: gcs_module.client.Client | None
|
30
32
|
|
31
|
-
def __init__(self, scheme:
|
33
|
+
def __init__(self, scheme: str | None = None) -> None:
|
32
34
|
self._scheme = scheme or "gs"
|
33
35
|
self._client = None
|
34
36
|
self._cache = get_artifact_file_cache()
|
35
37
|
|
36
|
-
def can_handle(self, parsed_url:
|
38
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
37
39
|
return parsed_url.scheme == self._scheme
|
38
40
|
|
39
|
-
def init_gcs(self) ->
|
41
|
+
def init_gcs(self) -> gcs_module.client.Client:
|
40
42
|
if self._client is not None:
|
41
43
|
return self._client
|
42
44
|
storage = util.get_module(
|
@@ -46,7 +48,7 @@ class GCSHandler(StorageHandler):
|
|
46
48
|
self._client = storage.Client()
|
47
49
|
return self._client
|
48
50
|
|
49
|
-
def _parse_uri(self, uri: str) ->
|
51
|
+
def _parse_uri(self, uri: str) -> tuple[str, str, str | None]:
|
50
52
|
url = urlparse(uri)
|
51
53
|
bucket = url.netloc
|
52
54
|
key = url.path[1:]
|
@@ -57,7 +59,7 @@ class GCSHandler(StorageHandler):
|
|
57
59
|
self,
|
58
60
|
manifest_entry: ArtifactManifestEntry,
|
59
61
|
local: bool = False,
|
60
|
-
) ->
|
62
|
+
) -> URIStr | FilePathStr:
|
61
63
|
assert manifest_entry.ref is not None
|
62
64
|
if not local:
|
63
65
|
return manifest_entry.ref
|
@@ -106,11 +108,11 @@ class GCSHandler(StorageHandler):
|
|
106
108
|
|
107
109
|
def store_path(
|
108
110
|
self,
|
109
|
-
artifact:
|
110
|
-
path:
|
111
|
-
name:
|
111
|
+
artifact: Artifact,
|
112
|
+
path: URIStr | FilePathStr,
|
113
|
+
name: StrPath | None = None,
|
112
114
|
checksum: bool = True,
|
113
|
-
max_objects:
|
115
|
+
max_objects: int | None = None,
|
114
116
|
) -> Sequence[ArtifactManifestEntry]:
|
115
117
|
self.init_gcs()
|
116
118
|
assert self._client is not None # mypy: unwraps optionality
|
@@ -159,9 +161,9 @@ class GCSHandler(StorageHandler):
|
|
159
161
|
|
160
162
|
def _entry_from_obj(
|
161
163
|
self,
|
162
|
-
obj:
|
164
|
+
obj: gcs_module.blob.Blob,
|
163
165
|
path: str,
|
164
|
-
name:
|
166
|
+
name: StrPath | None = None,
|
165
167
|
prefix: str = "",
|
166
168
|
multi: bool = False,
|
167
169
|
) -> ArtifactManifestEntry:
|
@@ -1,7 +1,9 @@
|
|
1
1
|
"""HTTP storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
|
-
from typing import TYPE_CHECKING,
|
6
|
+
from typing import TYPE_CHECKING, Sequence
|
5
7
|
from urllib.parse import ParseResult
|
6
8
|
|
7
9
|
from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
|
@@ -13,26 +15,25 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
|
13
15
|
|
14
16
|
if TYPE_CHECKING:
|
15
17
|
import requests
|
18
|
+
from requests.structures import CaseInsensitiveDict
|
16
19
|
|
17
20
|
from wandb.sdk.artifacts.artifact import Artifact
|
18
21
|
|
19
22
|
|
20
23
|
class HTTPHandler(StorageHandler):
|
21
|
-
def __init__(
|
22
|
-
self, session: "requests.Session", scheme: Optional[str] = None
|
23
|
-
) -> None:
|
24
|
+
def __init__(self, session: requests.Session, scheme: str | None = None) -> None:
|
24
25
|
self._scheme = scheme or "http"
|
25
26
|
self._cache = get_artifact_file_cache()
|
26
27
|
self._session = session
|
27
28
|
|
28
|
-
def can_handle(self, parsed_url:
|
29
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
29
30
|
return parsed_url.scheme == self._scheme
|
30
31
|
|
31
32
|
def load_path(
|
32
33
|
self,
|
33
34
|
manifest_entry: ArtifactManifestEntry,
|
34
35
|
local: bool = False,
|
35
|
-
) ->
|
36
|
+
) -> URIStr | FilePathStr:
|
36
37
|
if not local:
|
37
38
|
assert manifest_entry.ref is not None
|
38
39
|
return manifest_entry.ref
|
@@ -55,7 +56,7 @@ class HTTPHandler(StorageHandler):
|
|
55
56
|
)
|
56
57
|
response.raise_for_status()
|
57
58
|
|
58
|
-
digest:
|
59
|
+
digest: ETag | FilePathStr | URIStr | None
|
59
60
|
digest, size, extra = self._entry_from_headers(response.headers)
|
60
61
|
digest = digest or manifest_entry.ref
|
61
62
|
if manifest_entry.digest != digest:
|
@@ -70,11 +71,11 @@ class HTTPHandler(StorageHandler):
|
|
70
71
|
|
71
72
|
def store_path(
|
72
73
|
self,
|
73
|
-
artifact:
|
74
|
-
path:
|
75
|
-
name:
|
74
|
+
artifact: Artifact,
|
75
|
+
path: URIStr | FilePathStr,
|
76
|
+
name: StrPath | None = None,
|
76
77
|
checksum: bool = True,
|
77
|
-
max_objects:
|
78
|
+
max_objects: int | None = None,
|
78
79
|
) -> Sequence[ArtifactManifestEntry]:
|
79
80
|
name = name or os.path.basename(path)
|
80
81
|
if not checksum:
|
@@ -87,7 +88,7 @@ class HTTPHandler(StorageHandler):
|
|
87
88
|
headers=_thread_local_api_settings.headers,
|
88
89
|
) as response:
|
89
90
|
response.raise_for_status()
|
90
|
-
digest:
|
91
|
+
digest: ETag | FilePathStr | URIStr | None
|
91
92
|
digest, size, extra = self._entry_from_headers(response.headers)
|
92
93
|
digest = digest or path
|
93
94
|
return [
|
@@ -97,8 +98,8 @@ class HTTPHandler(StorageHandler):
|
|
97
98
|
]
|
98
99
|
|
99
100
|
def _entry_from_headers(
|
100
|
-
self, headers:
|
101
|
-
) ->
|
101
|
+
self, headers: CaseInsensitiveDict
|
102
|
+
) -> tuple[ETag | None, int | None, dict[str, str]]:
|
102
103
|
response_headers = {k.lower(): v for k, v in headers.items()}
|
103
104
|
size = None
|
104
105
|
if response_headers.get("content-length", None):
|
@@ -1,9 +1,11 @@
|
|
1
1
|
"""Local file storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
6
|
import shutil
|
5
7
|
import time
|
6
|
-
from typing import TYPE_CHECKING,
|
8
|
+
from typing import TYPE_CHECKING, Sequence
|
7
9
|
from urllib.parse import ParseResult
|
8
10
|
|
9
11
|
from wandb import util
|
@@ -22,7 +24,7 @@ if TYPE_CHECKING:
|
|
22
24
|
class LocalFileHandler(StorageHandler):
|
23
25
|
"""Handles file:// references."""
|
24
26
|
|
25
|
-
def __init__(self, scheme:
|
27
|
+
def __init__(self, scheme: str | None = None) -> None:
|
26
28
|
"""Track files or directories on a local filesystem.
|
27
29
|
|
28
30
|
Expand directories to create an entry for each file contained.
|
@@ -30,14 +32,14 @@ class LocalFileHandler(StorageHandler):
|
|
30
32
|
self._scheme = scheme or "file"
|
31
33
|
self._cache = get_artifact_file_cache()
|
32
34
|
|
33
|
-
def can_handle(self, parsed_url:
|
35
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
34
36
|
return parsed_url.scheme == self._scheme
|
35
37
|
|
36
38
|
def load_path(
|
37
39
|
self,
|
38
40
|
manifest_entry: ArtifactManifestEntry,
|
39
41
|
local: bool = False,
|
40
|
-
) ->
|
42
|
+
) -> URIStr | FilePathStr:
|
41
43
|
if manifest_entry.ref is None:
|
42
44
|
raise ValueError(f"Cannot add path with no ref: {manifest_entry.path}")
|
43
45
|
local_path = util.local_file_uri_to_path(str(manifest_entry.ref))
|
@@ -69,11 +71,11 @@ class LocalFileHandler(StorageHandler):
|
|
69
71
|
|
70
72
|
def store_path(
|
71
73
|
self,
|
72
|
-
artifact:
|
73
|
-
path:
|
74
|
-
name:
|
74
|
+
artifact: Artifact,
|
75
|
+
path: URIStr | FilePathStr,
|
76
|
+
name: StrPath | None = None,
|
75
77
|
checksum: bool = True,
|
76
|
-
max_objects:
|
78
|
+
max_objects: int | None = None,
|
77
79
|
) -> Sequence[ArtifactManifestEntry]:
|
78
80
|
local_path = util.local_file_uri_to_path(path)
|
79
81
|
max_objects = max_objects or DEFAULT_MAX_OBJECTS
|
@@ -1,6 +1,8 @@
|
|
1
1
|
"""Multi storage handler."""
|
2
2
|
|
3
|
-
from
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from typing import TYPE_CHECKING, Sequence
|
4
6
|
from urllib.parse import urlparse
|
5
7
|
|
6
8
|
from wandb.sdk.artifacts.storage_handler import StorageHandler
|
@@ -12,17 +14,17 @@ if TYPE_CHECKING:
|
|
12
14
|
|
13
15
|
|
14
16
|
class MultiHandler(StorageHandler):
|
15
|
-
_handlers:
|
17
|
+
_handlers: list[StorageHandler]
|
16
18
|
|
17
19
|
def __init__(
|
18
20
|
self,
|
19
|
-
handlers:
|
20
|
-
default_handler:
|
21
|
+
handlers: list[StorageHandler] | None = None,
|
22
|
+
default_handler: StorageHandler | None = None,
|
21
23
|
) -> None:
|
22
24
|
self._handlers = handlers or []
|
23
25
|
self._default_handler = default_handler
|
24
26
|
|
25
|
-
def _get_handler(self, url:
|
27
|
+
def _get_handler(self, url: FilePathStr | URIStr) -> StorageHandler:
|
26
28
|
parsed_url = urlparse(url)
|
27
29
|
for handler in self._handlers:
|
28
30
|
if handler.can_handle(parsed_url):
|
@@ -33,21 +35,21 @@ class MultiHandler(StorageHandler):
|
|
33
35
|
|
34
36
|
def load_path(
|
35
37
|
self,
|
36
|
-
manifest_entry:
|
38
|
+
manifest_entry: ArtifactManifestEntry,
|
37
39
|
local: bool = False,
|
38
|
-
) ->
|
40
|
+
) -> URIStr | FilePathStr:
|
39
41
|
assert manifest_entry.ref is not None
|
40
42
|
handler = self._get_handler(manifest_entry.ref)
|
41
43
|
return handler.load_path(manifest_entry, local=local)
|
42
44
|
|
43
45
|
def store_path(
|
44
46
|
self,
|
45
|
-
artifact:
|
46
|
-
path:
|
47
|
-
name:
|
47
|
+
artifact: Artifact,
|
48
|
+
path: URIStr | FilePathStr,
|
49
|
+
name: str | None = None,
|
48
50
|
checksum: bool = True,
|
49
|
-
max_objects:
|
50
|
-
) -> Sequence[
|
51
|
+
max_objects: int | None = None,
|
52
|
+
) -> Sequence[ArtifactManifestEntry]:
|
51
53
|
handler = self._get_handler(path)
|
52
54
|
return handler.store_path(
|
53
55
|
artifact, path, name=name, checksum=checksum, max_objects=max_objects
|
@@ -1,9 +1,11 @@
|
|
1
1
|
"""S3 storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
6
|
import time
|
5
7
|
from pathlib import PurePosixPath
|
6
|
-
from typing import TYPE_CHECKING,
|
8
|
+
from typing import TYPE_CHECKING, Sequence
|
7
9
|
from urllib.parse import parse_qsl, urlparse
|
8
10
|
|
9
11
|
from wandb import util
|
@@ -31,18 +33,18 @@ if TYPE_CHECKING:
|
|
31
33
|
|
32
34
|
|
33
35
|
class S3Handler(StorageHandler):
|
34
|
-
_s3:
|
36
|
+
_s3: boto3.resources.base.ServiceResource | None
|
35
37
|
_scheme: str
|
36
38
|
|
37
|
-
def __init__(self, scheme:
|
39
|
+
def __init__(self, scheme: str | None = None) -> None:
|
38
40
|
self._scheme = scheme or "s3"
|
39
41
|
self._s3 = None
|
40
42
|
self._cache = get_artifact_file_cache()
|
41
43
|
|
42
|
-
def can_handle(self, parsed_url:
|
44
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
43
45
|
return parsed_url.scheme == self._scheme
|
44
46
|
|
45
|
-
def init_boto(self) ->
|
47
|
+
def init_boto(self) -> boto3.resources.base.ServiceResource:
|
46
48
|
if self._s3 is not None:
|
47
49
|
return self._s3
|
48
50
|
boto: boto3 = util.get_module(
|
@@ -58,7 +60,7 @@ class S3Handler(StorageHandler):
|
|
58
60
|
self._botocore = util.get_module("botocore")
|
59
61
|
return self._s3
|
60
62
|
|
61
|
-
def _parse_uri(self, uri: str) ->
|
63
|
+
def _parse_uri(self, uri: str) -> tuple[str, str, str | None]:
|
62
64
|
url = urlparse(uri)
|
63
65
|
query = dict(parse_qsl(url.query))
|
64
66
|
|
@@ -72,7 +74,7 @@ class S3Handler(StorageHandler):
|
|
72
74
|
self,
|
73
75
|
manifest_entry: ArtifactManifestEntry,
|
74
76
|
local: bool = False,
|
75
|
-
) ->
|
77
|
+
) -> URIStr | FilePathStr:
|
76
78
|
if not local:
|
77
79
|
assert manifest_entry.ref is not None
|
78
80
|
return manifest_entry.ref
|
@@ -141,11 +143,11 @@ class S3Handler(StorageHandler):
|
|
141
143
|
|
142
144
|
def store_path(
|
143
145
|
self,
|
144
|
-
artifact:
|
145
|
-
path:
|
146
|
-
name:
|
146
|
+
artifact: Artifact,
|
147
|
+
path: URIStr | FilePathStr,
|
148
|
+
name: StrPath | None = None,
|
147
149
|
checksum: bool = True,
|
148
|
-
max_objects:
|
150
|
+
max_objects: int | None = None,
|
149
151
|
) -> Sequence[ArtifactManifestEntry]:
|
150
152
|
self.init_boto()
|
151
153
|
assert self._s3 is not None # mypy: unwraps optionality
|
@@ -222,9 +224,7 @@ class S3Handler(StorageHandler):
|
|
222
224
|
)
|
223
225
|
return entries
|
224
226
|
|
225
|
-
def _size_from_obj(
|
226
|
-
self, obj: Union["boto3.s3.Object", "boto3.s3.ObjectSummary"]
|
227
|
-
) -> int:
|
227
|
+
def _size_from_obj(self, obj: boto3.s3.Object | boto3.s3.ObjectSummary) -> int:
|
228
228
|
# ObjectSummary has size, Object has content_length
|
229
229
|
size: int
|
230
230
|
if hasattr(obj, "size"):
|
@@ -235,9 +235,9 @@ class S3Handler(StorageHandler):
|
|
235
235
|
|
236
236
|
def _entry_from_obj(
|
237
237
|
self,
|
238
|
-
obj:
|
238
|
+
obj: boto3.s3.Object | boto3.s3.ObjectSummary,
|
239
239
|
path: str,
|
240
|
-
name:
|
240
|
+
name: StrPath | None = None,
|
241
241
|
prefix: str = "",
|
242
242
|
multi: bool = False,
|
243
243
|
) -> ArtifactManifestEntry:
|
@@ -281,14 +281,14 @@ class S3Handler(StorageHandler):
|
|
281
281
|
)
|
282
282
|
|
283
283
|
@staticmethod
|
284
|
-
def _etag_from_obj(obj:
|
284
|
+
def _etag_from_obj(obj: boto3.s3.Object | boto3.s3.ObjectSummary) -> ETag:
|
285
285
|
etag: ETag
|
286
286
|
etag = obj.e_tag[1:-1] # escape leading and trailing quote
|
287
287
|
return etag
|
288
288
|
|
289
289
|
def _extra_from_obj(
|
290
|
-
self, obj:
|
291
|
-
) ->
|
290
|
+
self, obj: boto3.s3.Object | boto3.s3.ObjectSummary
|
291
|
+
) -> dict[str, str]:
|
292
292
|
extra = {
|
293
293
|
"etag": obj.e_tag[1:-1], # escape leading and trailing quote
|
294
294
|
}
|
@@ -1,6 +1,8 @@
|
|
1
1
|
"""Tracking storage handler."""
|
2
2
|
|
3
|
-
from
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from typing import TYPE_CHECKING, Sequence
|
4
6
|
from urllib.parse import urlparse
|
5
7
|
|
6
8
|
from wandb.errors.term import termwarn
|
@@ -15,7 +17,7 @@ if TYPE_CHECKING:
|
|
15
17
|
|
16
18
|
|
17
19
|
class TrackingHandler(StorageHandler):
|
18
|
-
def __init__(self, scheme:
|
20
|
+
def __init__(self, scheme: str | None = None) -> None:
|
19
21
|
"""Track paths with no modification or special processing.
|
20
22
|
|
21
23
|
Useful when paths being tracked are on file systems mounted at a standardized
|
@@ -26,14 +28,14 @@ class TrackingHandler(StorageHandler):
|
|
26
28
|
"""
|
27
29
|
self._scheme = scheme or ""
|
28
30
|
|
29
|
-
def can_handle(self, parsed_url:
|
31
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
30
32
|
return parsed_url.scheme == self._scheme
|
31
33
|
|
32
34
|
def load_path(
|
33
35
|
self,
|
34
36
|
manifest_entry: ArtifactManifestEntry,
|
35
37
|
local: bool = False,
|
36
|
-
) ->
|
38
|
+
) -> URIStr | FilePathStr:
|
37
39
|
if local:
|
38
40
|
# Likely a user error. The tracking handler is
|
39
41
|
# oblivious to the underlying paths, so it has
|
@@ -48,11 +50,11 @@ class TrackingHandler(StorageHandler):
|
|
48
50
|
|
49
51
|
def store_path(
|
50
52
|
self,
|
51
|
-
artifact:
|
52
|
-
path:
|
53
|
-
name:
|
53
|
+
artifact: Artifact,
|
54
|
+
path: URIStr | FilePathStr,
|
55
|
+
name: StrPath | None = None,
|
54
56
|
checksum: bool = True,
|
55
|
-
max_objects:
|
57
|
+
max_objects: int | None = None,
|
56
58
|
) -> Sequence[ArtifactManifestEntry]:
|
57
59
|
url = urlparse(path)
|
58
60
|
if name is None:
|
@@ -1,7 +1,9 @@
|
|
1
1
|
"""WB artifact storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
|
-
from typing import TYPE_CHECKING,
|
6
|
+
from typing import TYPE_CHECKING, Sequence
|
5
7
|
from urllib.parse import urlparse
|
6
8
|
|
7
9
|
import wandb
|
@@ -22,14 +24,14 @@ if TYPE_CHECKING:
|
|
22
24
|
class WBArtifactHandler(StorageHandler):
|
23
25
|
"""Handles loading and storing Artifact reference-type files."""
|
24
26
|
|
25
|
-
_client:
|
27
|
+
_client: PublicApi | None
|
26
28
|
|
27
29
|
def __init__(self) -> None:
|
28
30
|
self._scheme = "wandb-artifact"
|
29
31
|
self._cache = get_artifact_file_cache()
|
30
32
|
self._client = None
|
31
33
|
|
32
|
-
def can_handle(self, parsed_url:
|
34
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
33
35
|
return parsed_url.scheme == self._scheme
|
34
36
|
|
35
37
|
@property
|
@@ -42,7 +44,7 @@ class WBArtifactHandler(StorageHandler):
|
|
42
44
|
self,
|
43
45
|
manifest_entry: ArtifactManifestEntry,
|
44
46
|
local: bool = False,
|
45
|
-
) ->
|
47
|
+
) -> URIStr | FilePathStr:
|
46
48
|
"""Load the file in the specified artifact given its corresponding entry.
|
47
49
|
|
48
50
|
Download the referenced artifact; create and return a new symlink to the caller.
|
@@ -67,7 +69,7 @@ class WBArtifactHandler(StorageHandler):
|
|
67
69
|
hex_to_b64_id(artifact_id), self.client.client
|
68
70
|
)
|
69
71
|
assert dep_artifact is not None
|
70
|
-
link_target_path:
|
72
|
+
link_target_path: URIStr | FilePathStr
|
71
73
|
if local:
|
72
74
|
link_target_path = dep_artifact.get_entry(artifact_file_path).download()
|
73
75
|
else:
|
@@ -77,11 +79,11 @@ class WBArtifactHandler(StorageHandler):
|
|
77
79
|
|
78
80
|
def store_path(
|
79
81
|
self,
|
80
|
-
artifact:
|
81
|
-
path:
|
82
|
-
name:
|
82
|
+
artifact: Artifact,
|
83
|
+
path: URIStr | FilePathStr,
|
84
|
+
name: StrPath | None = None,
|
83
85
|
checksum: bool = True,
|
84
|
-
max_objects:
|
86
|
+
max_objects: int | None = None,
|
85
87
|
) -> Sequence[ArtifactManifestEntry]:
|
86
88
|
"""Store the file or directory at the given path into the specified artifact.
|
87
89
|
|
@@ -97,7 +99,7 @@ class WBArtifactHandler(StorageHandler):
|
|
97
99
|
"""
|
98
100
|
# Recursively resolve the reference until a concrete asset is found
|
99
101
|
# TODO: Consider resolving server-side for performance improvements.
|
100
|
-
iter_path:
|
102
|
+
iter_path: URIStr | FilePathStr | None = path
|
101
103
|
while iter_path is not None and urlparse(iter_path).scheme == self._scheme:
|
102
104
|
artifact_id = util.host_from_path(iter_path)
|
103
105
|
artifact_file_path = util.uri_from_path(iter_path)
|
@@ -1,7 +1,9 @@
|
|
1
1
|
"""WB local artifact storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
|
-
from typing import TYPE_CHECKING,
|
6
|
+
from typing import TYPE_CHECKING, Sequence
|
5
7
|
|
6
8
|
import wandb
|
7
9
|
from wandb import util
|
@@ -22,25 +24,25 @@ class WBLocalArtifactHandler(StorageHandler):
|
|
22
24
|
def __init__(self) -> None:
|
23
25
|
self._scheme = "wandb-client-artifact"
|
24
26
|
|
25
|
-
def can_handle(self, parsed_url:
|
27
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
26
28
|
return parsed_url.scheme == self._scheme
|
27
29
|
|
28
30
|
def load_path(
|
29
31
|
self,
|
30
32
|
manifest_entry: ArtifactManifestEntry,
|
31
33
|
local: bool = False,
|
32
|
-
) ->
|
34
|
+
) -> URIStr | FilePathStr:
|
33
35
|
raise NotImplementedError(
|
34
36
|
"Should not be loading a path for an artifact entry with unresolved client id."
|
35
37
|
)
|
36
38
|
|
37
39
|
def store_path(
|
38
40
|
self,
|
39
|
-
artifact:
|
40
|
-
path:
|
41
|
-
name:
|
41
|
+
artifact: Artifact,
|
42
|
+
path: URIStr | FilePathStr,
|
43
|
+
name: StrPath | None = None,
|
42
44
|
checksum: bool = True,
|
43
|
-
max_objects:
|
45
|
+
max_objects: int | None = None,
|
44
46
|
) -> Sequence[ArtifactManifestEntry]:
|
45
47
|
"""Store the file or directory at the given path within the specified artifact.
|
46
48
|
|