wandb 0.18.0rc1__py3-none-macosx_11_0_arm64.whl → 0.18.2__py3-none-macosx_11_0_arm64.whl
Sign up to get free protection for your applications and to get access to all the features.
- wandb/__init__.py +4 -4
- wandb/__init__.pyi +67 -12
- wandb/apis/internal.py +3 -0
- wandb/apis/public/api.py +128 -2
- wandb/apis/public/artifacts.py +11 -7
- wandb/apis/public/jobs.py +8 -0
- wandb/apis/public/runs.py +18 -5
- wandb/bin/apple_gpu_stats +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/cli.py +0 -5
- wandb/data_types.py +9 -2019
- wandb/env.py +0 -5
- wandb/errors/__init__.py +11 -40
- wandb/errors/errors.py +37 -0
- wandb/errors/warnings.py +2 -0
- wandb/{sklearn → integration/sklearn}/calculate/calibration_curves.py +7 -7
- wandb/{sklearn → integration/sklearn}/calculate/class_proportions.py +1 -1
- wandb/{sklearn → integration/sklearn}/calculate/confusion_matrix.py +3 -2
- wandb/{sklearn → integration/sklearn}/calculate/elbow_curve.py +6 -6
- wandb/{sklearn → integration/sklearn}/calculate/learning_curve.py +2 -2
- wandb/{sklearn → integration/sklearn}/calculate/outlier_candidates.py +2 -2
- wandb/{sklearn → integration/sklearn}/calculate/residuals.py +8 -8
- wandb/{sklearn → integration/sklearn}/calculate/silhouette.py +2 -2
- wandb/{sklearn → integration/sklearn}/calculate/summary_metrics.py +2 -2
- wandb/{sklearn → integration/sklearn}/plot/classifier.py +5 -5
- wandb/{sklearn → integration/sklearn}/plot/clusterer.py +10 -6
- wandb/{sklearn → integration/sklearn}/plot/regressor.py +5 -5
- wandb/{sklearn → integration/sklearn}/plot/shared.py +3 -3
- wandb/{sklearn → integration/sklearn}/utils.py +8 -8
- wandb/integration/tensorboard/log.py +1 -1
- wandb/{wandb_torch.py → integration/torch/wandb_torch.py} +36 -32
- wandb/old/core.py +2 -80
- wandb/plot/bar.py +7 -4
- wandb/plot/confusion_matrix.py +5 -4
- wandb/plot/histogram.py +7 -4
- wandb/plot/line.py +7 -4
- wandb/proto/v3/wandb_base_pb2.py +2 -1
- wandb/proto/v3/wandb_internal_pb2.py +2 -1
- wandb/proto/v3/wandb_server_pb2.py +2 -1
- wandb/proto/v3/wandb_settings_pb2.py +3 -2
- wandb/proto/v3/wandb_telemetry_pb2.py +2 -1
- wandb/proto/v4/wandb_base_pb2.py +2 -1
- wandb/proto/v4/wandb_internal_pb2.py +2 -1
- wandb/proto/v4/wandb_server_pb2.py +2 -1
- wandb/proto/v4/wandb_settings_pb2.py +3 -2
- wandb/proto/v4/wandb_telemetry_pb2.py +2 -1
- wandb/proto/v5/wandb_base_pb2.py +3 -2
- wandb/proto/v5/wandb_internal_pb2.py +3 -2
- wandb/proto/v5/wandb_server_pb2.py +3 -2
- wandb/proto/v5/wandb_settings_pb2.py +4 -3
- wandb/proto/v5/wandb_telemetry_pb2.py +3 -2
- wandb/sdk/artifacts/_validators.py +48 -3
- wandb/sdk/artifacts/artifact.py +157 -183
- wandb/sdk/artifacts/artifact_file_cache.py +13 -11
- wandb/sdk/artifacts/artifact_instance_cache.py +4 -2
- wandb/sdk/artifacts/artifact_manifest.py +13 -11
- wandb/sdk/artifacts/artifact_manifest_entry.py +24 -22
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +9 -7
- wandb/sdk/artifacts/artifact_saver.py +27 -25
- wandb/sdk/artifacts/exceptions.py +26 -25
- wandb/sdk/artifacts/storage_handler.py +11 -9
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +16 -14
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +15 -13
- wandb/sdk/artifacts/storage_handlers/http_handler.py +15 -14
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +10 -8
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +14 -12
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +19 -19
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +10 -8
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +12 -10
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +9 -7
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +31 -29
- wandb/sdk/artifacts/storage_policy.py +20 -20
- wandb/sdk/backend/backend.py +8 -26
- wandb/sdk/data_types/audio.py +165 -0
- wandb/sdk/data_types/base_types/wb_value.py +1 -3
- wandb/sdk/data_types/bokeh.py +70 -0
- wandb/sdk/data_types/graph.py +405 -0
- wandb/sdk/data_types/image.py +156 -0
- wandb/sdk/data_types/table.py +1204 -0
- wandb/sdk/data_types/trace_tree.py +2 -2
- wandb/sdk/data_types/utils.py +49 -0
- wandb/sdk/data_types/video.py +2 -2
- wandb/sdk/interface/interface.py +0 -24
- wandb/sdk/interface/interface_shared.py +0 -12
- wandb/sdk/internal/handler.py +0 -10
- wandb/sdk/internal/internal_api.py +71 -0
- wandb/sdk/internal/sender.py +0 -43
- wandb/sdk/internal/tb_watcher.py +1 -1
- wandb/sdk/lib/_settings_toposort_generated.py +1 -0
- wandb/sdk/lib/hashutil.py +34 -12
- wandb/sdk/lib/service_connection.py +216 -0
- wandb/sdk/lib/service_token.py +94 -0
- wandb/sdk/lib/sock_client.py +7 -3
- wandb/sdk/service/server.py +2 -5
- wandb/sdk/service/service.py +2 -31
- wandb/sdk/service/streams.py +0 -7
- wandb/sdk/wandb_init.py +42 -25
- wandb/sdk/wandb_run.py +18 -159
- wandb/sdk/wandb_settings.py +2 -0
- wandb/sdk/wandb_setup.py +25 -16
- wandb/sdk/wandb_sync.py +9 -3
- wandb/sdk/wandb_watch.py +31 -15
- wandb/sklearn.py +35 -0
- wandb/util.py +14 -3
- {wandb-0.18.0rc1.dist-info → wandb-0.18.2.dist-info}/METADATA +6 -5
- {wandb-0.18.0rc1.dist-info → wandb-0.18.2.dist-info}/RECORD +115 -111
- wandb/sdk/internal/update.py +0 -113
- wandb/sdk/lib/console.py +0 -39
- wandb/sdk/service/service_base.py +0 -50
- wandb/sdk/service/service_sock.py +0 -70
- wandb/sdk/wandb_manager.py +0 -232
- /wandb/{sklearn → integration/sklearn}/__init__.py +0 -0
- /wandb/{sklearn → integration/sklearn}/calculate/__init__.py +0 -0
- /wandb/{sklearn → integration/sklearn}/calculate/decision_boundaries.py +0 -0
- /wandb/{sklearn → integration/sklearn}/calculate/feature_importances.py +0 -0
- /wandb/{sklearn → integration/sklearn}/plot/__init__.py +0 -0
- /wandb/{sdk/lib → plot}/viz.py +0 -0
- {wandb-0.18.0rc1.dist-info → wandb-0.18.2.dist-info}/WHEEL +0 -0
- {wandb-0.18.0rc1.dist-info → wandb-0.18.2.dist-info}/entry_points.txt +0 -0
- {wandb-0.18.0rc1.dist-info → wandb-0.18.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,8 +1,10 @@
|
|
1
1
|
"""GCS storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import time
|
4
6
|
from pathlib import PurePosixPath
|
5
|
-
from typing import TYPE_CHECKING,
|
7
|
+
from typing import TYPE_CHECKING, Sequence
|
6
8
|
from urllib.parse import ParseResult, urlparse
|
7
9
|
|
8
10
|
from wandb import util
|
@@ -26,17 +28,17 @@ class _GCSIsADirectoryError(Exception):
|
|
26
28
|
|
27
29
|
|
28
30
|
class GCSHandler(StorageHandler):
|
29
|
-
_client:
|
31
|
+
_client: gcs_module.client.Client | None
|
30
32
|
|
31
|
-
def __init__(self, scheme:
|
33
|
+
def __init__(self, scheme: str | None = None) -> None:
|
32
34
|
self._scheme = scheme or "gs"
|
33
35
|
self._client = None
|
34
36
|
self._cache = get_artifact_file_cache()
|
35
37
|
|
36
|
-
def can_handle(self, parsed_url:
|
38
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
37
39
|
return parsed_url.scheme == self._scheme
|
38
40
|
|
39
|
-
def init_gcs(self) ->
|
41
|
+
def init_gcs(self) -> gcs_module.client.Client:
|
40
42
|
if self._client is not None:
|
41
43
|
return self._client
|
42
44
|
storage = util.get_module(
|
@@ -46,7 +48,7 @@ class GCSHandler(StorageHandler):
|
|
46
48
|
self._client = storage.Client()
|
47
49
|
return self._client
|
48
50
|
|
49
|
-
def _parse_uri(self, uri: str) ->
|
51
|
+
def _parse_uri(self, uri: str) -> tuple[str, str, str | None]:
|
50
52
|
url = urlparse(uri)
|
51
53
|
bucket = url.netloc
|
52
54
|
key = url.path[1:]
|
@@ -57,7 +59,7 @@ class GCSHandler(StorageHandler):
|
|
57
59
|
self,
|
58
60
|
manifest_entry: ArtifactManifestEntry,
|
59
61
|
local: bool = False,
|
60
|
-
) ->
|
62
|
+
) -> URIStr | FilePathStr:
|
61
63
|
assert manifest_entry.ref is not None
|
62
64
|
if not local:
|
63
65
|
return manifest_entry.ref
|
@@ -106,11 +108,11 @@ class GCSHandler(StorageHandler):
|
|
106
108
|
|
107
109
|
def store_path(
|
108
110
|
self,
|
109
|
-
artifact:
|
110
|
-
path:
|
111
|
-
name:
|
111
|
+
artifact: Artifact,
|
112
|
+
path: URIStr | FilePathStr,
|
113
|
+
name: StrPath | None = None,
|
112
114
|
checksum: bool = True,
|
113
|
-
max_objects:
|
115
|
+
max_objects: int | None = None,
|
114
116
|
) -> Sequence[ArtifactManifestEntry]:
|
115
117
|
self.init_gcs()
|
116
118
|
assert self._client is not None # mypy: unwraps optionality
|
@@ -159,9 +161,9 @@ class GCSHandler(StorageHandler):
|
|
159
161
|
|
160
162
|
def _entry_from_obj(
|
161
163
|
self,
|
162
|
-
obj:
|
164
|
+
obj: gcs_module.blob.Blob,
|
163
165
|
path: str,
|
164
|
-
name:
|
166
|
+
name: StrPath | None = None,
|
165
167
|
prefix: str = "",
|
166
168
|
multi: bool = False,
|
167
169
|
) -> ArtifactManifestEntry:
|
@@ -1,7 +1,9 @@
|
|
1
1
|
"""HTTP storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
|
-
from typing import TYPE_CHECKING,
|
6
|
+
from typing import TYPE_CHECKING, Sequence
|
5
7
|
from urllib.parse import ParseResult
|
6
8
|
|
7
9
|
from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
|
@@ -13,26 +15,25 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
|
13
15
|
|
14
16
|
if TYPE_CHECKING:
|
15
17
|
import requests
|
18
|
+
from requests.structures import CaseInsensitiveDict
|
16
19
|
|
17
20
|
from wandb.sdk.artifacts.artifact import Artifact
|
18
21
|
|
19
22
|
|
20
23
|
class HTTPHandler(StorageHandler):
|
21
|
-
def __init__(
|
22
|
-
self, session: "requests.Session", scheme: Optional[str] = None
|
23
|
-
) -> None:
|
24
|
+
def __init__(self, session: requests.Session, scheme: str | None = None) -> None:
|
24
25
|
self._scheme = scheme or "http"
|
25
26
|
self._cache = get_artifact_file_cache()
|
26
27
|
self._session = session
|
27
28
|
|
28
|
-
def can_handle(self, parsed_url:
|
29
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
29
30
|
return parsed_url.scheme == self._scheme
|
30
31
|
|
31
32
|
def load_path(
|
32
33
|
self,
|
33
34
|
manifest_entry: ArtifactManifestEntry,
|
34
35
|
local: bool = False,
|
35
|
-
) ->
|
36
|
+
) -> URIStr | FilePathStr:
|
36
37
|
if not local:
|
37
38
|
assert manifest_entry.ref is not None
|
38
39
|
return manifest_entry.ref
|
@@ -55,7 +56,7 @@ class HTTPHandler(StorageHandler):
|
|
55
56
|
)
|
56
57
|
response.raise_for_status()
|
57
58
|
|
58
|
-
digest:
|
59
|
+
digest: ETag | FilePathStr | URIStr | None
|
59
60
|
digest, size, extra = self._entry_from_headers(response.headers)
|
60
61
|
digest = digest or manifest_entry.ref
|
61
62
|
if manifest_entry.digest != digest:
|
@@ -70,11 +71,11 @@ class HTTPHandler(StorageHandler):
|
|
70
71
|
|
71
72
|
def store_path(
|
72
73
|
self,
|
73
|
-
artifact:
|
74
|
-
path:
|
75
|
-
name:
|
74
|
+
artifact: Artifact,
|
75
|
+
path: URIStr | FilePathStr,
|
76
|
+
name: StrPath | None = None,
|
76
77
|
checksum: bool = True,
|
77
|
-
max_objects:
|
78
|
+
max_objects: int | None = None,
|
78
79
|
) -> Sequence[ArtifactManifestEntry]:
|
79
80
|
name = name or os.path.basename(path)
|
80
81
|
if not checksum:
|
@@ -87,7 +88,7 @@ class HTTPHandler(StorageHandler):
|
|
87
88
|
headers=_thread_local_api_settings.headers,
|
88
89
|
) as response:
|
89
90
|
response.raise_for_status()
|
90
|
-
digest:
|
91
|
+
digest: ETag | FilePathStr | URIStr | None
|
91
92
|
digest, size, extra = self._entry_from_headers(response.headers)
|
92
93
|
digest = digest or path
|
93
94
|
return [
|
@@ -97,8 +98,8 @@ class HTTPHandler(StorageHandler):
|
|
97
98
|
]
|
98
99
|
|
99
100
|
def _entry_from_headers(
|
100
|
-
self, headers:
|
101
|
-
) ->
|
101
|
+
self, headers: CaseInsensitiveDict
|
102
|
+
) -> tuple[ETag | None, int | None, dict[str, str]]:
|
102
103
|
response_headers = {k.lower(): v for k, v in headers.items()}
|
103
104
|
size = None
|
104
105
|
if response_headers.get("content-length", None):
|
@@ -1,9 +1,11 @@
|
|
1
1
|
"""Local file storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
6
|
import shutil
|
5
7
|
import time
|
6
|
-
from typing import TYPE_CHECKING,
|
8
|
+
from typing import TYPE_CHECKING, Sequence
|
7
9
|
from urllib.parse import ParseResult
|
8
10
|
|
9
11
|
from wandb import util
|
@@ -22,7 +24,7 @@ if TYPE_CHECKING:
|
|
22
24
|
class LocalFileHandler(StorageHandler):
|
23
25
|
"""Handles file:// references."""
|
24
26
|
|
25
|
-
def __init__(self, scheme:
|
27
|
+
def __init__(self, scheme: str | None = None) -> None:
|
26
28
|
"""Track files or directories on a local filesystem.
|
27
29
|
|
28
30
|
Expand directories to create an entry for each file contained.
|
@@ -30,14 +32,14 @@ class LocalFileHandler(StorageHandler):
|
|
30
32
|
self._scheme = scheme or "file"
|
31
33
|
self._cache = get_artifact_file_cache()
|
32
34
|
|
33
|
-
def can_handle(self, parsed_url:
|
35
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
34
36
|
return parsed_url.scheme == self._scheme
|
35
37
|
|
36
38
|
def load_path(
|
37
39
|
self,
|
38
40
|
manifest_entry: ArtifactManifestEntry,
|
39
41
|
local: bool = False,
|
40
|
-
) ->
|
42
|
+
) -> URIStr | FilePathStr:
|
41
43
|
if manifest_entry.ref is None:
|
42
44
|
raise ValueError(f"Cannot add path with no ref: {manifest_entry.path}")
|
43
45
|
local_path = util.local_file_uri_to_path(str(manifest_entry.ref))
|
@@ -69,11 +71,11 @@ class LocalFileHandler(StorageHandler):
|
|
69
71
|
|
70
72
|
def store_path(
|
71
73
|
self,
|
72
|
-
artifact:
|
73
|
-
path:
|
74
|
-
name:
|
74
|
+
artifact: Artifact,
|
75
|
+
path: URIStr | FilePathStr,
|
76
|
+
name: StrPath | None = None,
|
75
77
|
checksum: bool = True,
|
76
|
-
max_objects:
|
78
|
+
max_objects: int | None = None,
|
77
79
|
) -> Sequence[ArtifactManifestEntry]:
|
78
80
|
local_path = util.local_file_uri_to_path(path)
|
79
81
|
max_objects = max_objects or DEFAULT_MAX_OBJECTS
|
@@ -1,6 +1,8 @@
|
|
1
1
|
"""Multi storage handler."""
|
2
2
|
|
3
|
-
from
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from typing import TYPE_CHECKING, Sequence
|
4
6
|
from urllib.parse import urlparse
|
5
7
|
|
6
8
|
from wandb.sdk.artifacts.storage_handler import StorageHandler
|
@@ -12,17 +14,17 @@ if TYPE_CHECKING:
|
|
12
14
|
|
13
15
|
|
14
16
|
class MultiHandler(StorageHandler):
|
15
|
-
_handlers:
|
17
|
+
_handlers: list[StorageHandler]
|
16
18
|
|
17
19
|
def __init__(
|
18
20
|
self,
|
19
|
-
handlers:
|
20
|
-
default_handler:
|
21
|
+
handlers: list[StorageHandler] | None = None,
|
22
|
+
default_handler: StorageHandler | None = None,
|
21
23
|
) -> None:
|
22
24
|
self._handlers = handlers or []
|
23
25
|
self._default_handler = default_handler
|
24
26
|
|
25
|
-
def _get_handler(self, url:
|
27
|
+
def _get_handler(self, url: FilePathStr | URIStr) -> StorageHandler:
|
26
28
|
parsed_url = urlparse(url)
|
27
29
|
for handler in self._handlers:
|
28
30
|
if handler.can_handle(parsed_url):
|
@@ -33,21 +35,21 @@ class MultiHandler(StorageHandler):
|
|
33
35
|
|
34
36
|
def load_path(
|
35
37
|
self,
|
36
|
-
manifest_entry:
|
38
|
+
manifest_entry: ArtifactManifestEntry,
|
37
39
|
local: bool = False,
|
38
|
-
) ->
|
40
|
+
) -> URIStr | FilePathStr:
|
39
41
|
assert manifest_entry.ref is not None
|
40
42
|
handler = self._get_handler(manifest_entry.ref)
|
41
43
|
return handler.load_path(manifest_entry, local=local)
|
42
44
|
|
43
45
|
def store_path(
|
44
46
|
self,
|
45
|
-
artifact:
|
46
|
-
path:
|
47
|
-
name:
|
47
|
+
artifact: Artifact,
|
48
|
+
path: URIStr | FilePathStr,
|
49
|
+
name: str | None = None,
|
48
50
|
checksum: bool = True,
|
49
|
-
max_objects:
|
50
|
-
) -> Sequence[
|
51
|
+
max_objects: int | None = None,
|
52
|
+
) -> Sequence[ArtifactManifestEntry]:
|
51
53
|
handler = self._get_handler(path)
|
52
54
|
return handler.store_path(
|
53
55
|
artifact, path, name=name, checksum=checksum, max_objects=max_objects
|
@@ -1,9 +1,11 @@
|
|
1
1
|
"""S3 storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
6
|
import time
|
5
7
|
from pathlib import PurePosixPath
|
6
|
-
from typing import TYPE_CHECKING,
|
8
|
+
from typing import TYPE_CHECKING, Sequence
|
7
9
|
from urllib.parse import parse_qsl, urlparse
|
8
10
|
|
9
11
|
from wandb import util
|
@@ -31,18 +33,18 @@ if TYPE_CHECKING:
|
|
31
33
|
|
32
34
|
|
33
35
|
class S3Handler(StorageHandler):
|
34
|
-
_s3:
|
36
|
+
_s3: boto3.resources.base.ServiceResource | None
|
35
37
|
_scheme: str
|
36
38
|
|
37
|
-
def __init__(self, scheme:
|
39
|
+
def __init__(self, scheme: str | None = None) -> None:
|
38
40
|
self._scheme = scheme or "s3"
|
39
41
|
self._s3 = None
|
40
42
|
self._cache = get_artifact_file_cache()
|
41
43
|
|
42
|
-
def can_handle(self, parsed_url:
|
44
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
43
45
|
return parsed_url.scheme == self._scheme
|
44
46
|
|
45
|
-
def init_boto(self) ->
|
47
|
+
def init_boto(self) -> boto3.resources.base.ServiceResource:
|
46
48
|
if self._s3 is not None:
|
47
49
|
return self._s3
|
48
50
|
boto: boto3 = util.get_module(
|
@@ -58,7 +60,7 @@ class S3Handler(StorageHandler):
|
|
58
60
|
self._botocore = util.get_module("botocore")
|
59
61
|
return self._s3
|
60
62
|
|
61
|
-
def _parse_uri(self, uri: str) ->
|
63
|
+
def _parse_uri(self, uri: str) -> tuple[str, str, str | None]:
|
62
64
|
url = urlparse(uri)
|
63
65
|
query = dict(parse_qsl(url.query))
|
64
66
|
|
@@ -72,7 +74,7 @@ class S3Handler(StorageHandler):
|
|
72
74
|
self,
|
73
75
|
manifest_entry: ArtifactManifestEntry,
|
74
76
|
local: bool = False,
|
75
|
-
) ->
|
77
|
+
) -> URIStr | FilePathStr:
|
76
78
|
if not local:
|
77
79
|
assert manifest_entry.ref is not None
|
78
80
|
return manifest_entry.ref
|
@@ -141,11 +143,11 @@ class S3Handler(StorageHandler):
|
|
141
143
|
|
142
144
|
def store_path(
|
143
145
|
self,
|
144
|
-
artifact:
|
145
|
-
path:
|
146
|
-
name:
|
146
|
+
artifact: Artifact,
|
147
|
+
path: URIStr | FilePathStr,
|
148
|
+
name: StrPath | None = None,
|
147
149
|
checksum: bool = True,
|
148
|
-
max_objects:
|
150
|
+
max_objects: int | None = None,
|
149
151
|
) -> Sequence[ArtifactManifestEntry]:
|
150
152
|
self.init_boto()
|
151
153
|
assert self._s3 is not None # mypy: unwraps optionality
|
@@ -222,9 +224,7 @@ class S3Handler(StorageHandler):
|
|
222
224
|
)
|
223
225
|
return entries
|
224
226
|
|
225
|
-
def _size_from_obj(
|
226
|
-
self, obj: Union["boto3.s3.Object", "boto3.s3.ObjectSummary"]
|
227
|
-
) -> int:
|
227
|
+
def _size_from_obj(self, obj: boto3.s3.Object | boto3.s3.ObjectSummary) -> int:
|
228
228
|
# ObjectSummary has size, Object has content_length
|
229
229
|
size: int
|
230
230
|
if hasattr(obj, "size"):
|
@@ -235,9 +235,9 @@ class S3Handler(StorageHandler):
|
|
235
235
|
|
236
236
|
def _entry_from_obj(
|
237
237
|
self,
|
238
|
-
obj:
|
238
|
+
obj: boto3.s3.Object | boto3.s3.ObjectSummary,
|
239
239
|
path: str,
|
240
|
-
name:
|
240
|
+
name: StrPath | None = None,
|
241
241
|
prefix: str = "",
|
242
242
|
multi: bool = False,
|
243
243
|
) -> ArtifactManifestEntry:
|
@@ -281,14 +281,14 @@ class S3Handler(StorageHandler):
|
|
281
281
|
)
|
282
282
|
|
283
283
|
@staticmethod
|
284
|
-
def _etag_from_obj(obj:
|
284
|
+
def _etag_from_obj(obj: boto3.s3.Object | boto3.s3.ObjectSummary) -> ETag:
|
285
285
|
etag: ETag
|
286
286
|
etag = obj.e_tag[1:-1] # escape leading and trailing quote
|
287
287
|
return etag
|
288
288
|
|
289
289
|
def _extra_from_obj(
|
290
|
-
self, obj:
|
291
|
-
) ->
|
290
|
+
self, obj: boto3.s3.Object | boto3.s3.ObjectSummary
|
291
|
+
) -> dict[str, str]:
|
292
292
|
extra = {
|
293
293
|
"etag": obj.e_tag[1:-1], # escape leading and trailing quote
|
294
294
|
}
|
@@ -1,6 +1,8 @@
|
|
1
1
|
"""Tracking storage handler."""
|
2
2
|
|
3
|
-
from
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from typing import TYPE_CHECKING, Sequence
|
4
6
|
from urllib.parse import urlparse
|
5
7
|
|
6
8
|
from wandb.errors.term import termwarn
|
@@ -15,7 +17,7 @@ if TYPE_CHECKING:
|
|
15
17
|
|
16
18
|
|
17
19
|
class TrackingHandler(StorageHandler):
|
18
|
-
def __init__(self, scheme:
|
20
|
+
def __init__(self, scheme: str | None = None) -> None:
|
19
21
|
"""Track paths with no modification or special processing.
|
20
22
|
|
21
23
|
Useful when paths being tracked are on file systems mounted at a standardized
|
@@ -26,14 +28,14 @@ class TrackingHandler(StorageHandler):
|
|
26
28
|
"""
|
27
29
|
self._scheme = scheme or ""
|
28
30
|
|
29
|
-
def can_handle(self, parsed_url:
|
31
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
30
32
|
return parsed_url.scheme == self._scheme
|
31
33
|
|
32
34
|
def load_path(
|
33
35
|
self,
|
34
36
|
manifest_entry: ArtifactManifestEntry,
|
35
37
|
local: bool = False,
|
36
|
-
) ->
|
38
|
+
) -> URIStr | FilePathStr:
|
37
39
|
if local:
|
38
40
|
# Likely a user error. The tracking handler is
|
39
41
|
# oblivious to the underlying paths, so it has
|
@@ -48,11 +50,11 @@ class TrackingHandler(StorageHandler):
|
|
48
50
|
|
49
51
|
def store_path(
|
50
52
|
self,
|
51
|
-
artifact:
|
52
|
-
path:
|
53
|
-
name:
|
53
|
+
artifact: Artifact,
|
54
|
+
path: URIStr | FilePathStr,
|
55
|
+
name: StrPath | None = None,
|
54
56
|
checksum: bool = True,
|
55
|
-
max_objects:
|
57
|
+
max_objects: int | None = None,
|
56
58
|
) -> Sequence[ArtifactManifestEntry]:
|
57
59
|
url = urlparse(path)
|
58
60
|
if name is None:
|
@@ -1,7 +1,9 @@
|
|
1
1
|
"""WB artifact storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
|
-
from typing import TYPE_CHECKING,
|
6
|
+
from typing import TYPE_CHECKING, Sequence
|
5
7
|
from urllib.parse import urlparse
|
6
8
|
|
7
9
|
import wandb
|
@@ -22,14 +24,14 @@ if TYPE_CHECKING:
|
|
22
24
|
class WBArtifactHandler(StorageHandler):
|
23
25
|
"""Handles loading and storing Artifact reference-type files."""
|
24
26
|
|
25
|
-
_client:
|
27
|
+
_client: PublicApi | None
|
26
28
|
|
27
29
|
def __init__(self) -> None:
|
28
30
|
self._scheme = "wandb-artifact"
|
29
31
|
self._cache = get_artifact_file_cache()
|
30
32
|
self._client = None
|
31
33
|
|
32
|
-
def can_handle(self, parsed_url:
|
34
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
33
35
|
return parsed_url.scheme == self._scheme
|
34
36
|
|
35
37
|
@property
|
@@ -42,7 +44,7 @@ class WBArtifactHandler(StorageHandler):
|
|
42
44
|
self,
|
43
45
|
manifest_entry: ArtifactManifestEntry,
|
44
46
|
local: bool = False,
|
45
|
-
) ->
|
47
|
+
) -> URIStr | FilePathStr:
|
46
48
|
"""Load the file in the specified artifact given its corresponding entry.
|
47
49
|
|
48
50
|
Download the referenced artifact; create and return a new symlink to the caller.
|
@@ -67,7 +69,7 @@ class WBArtifactHandler(StorageHandler):
|
|
67
69
|
hex_to_b64_id(artifact_id), self.client.client
|
68
70
|
)
|
69
71
|
assert dep_artifact is not None
|
70
|
-
link_target_path:
|
72
|
+
link_target_path: URIStr | FilePathStr
|
71
73
|
if local:
|
72
74
|
link_target_path = dep_artifact.get_entry(artifact_file_path).download()
|
73
75
|
else:
|
@@ -77,11 +79,11 @@ class WBArtifactHandler(StorageHandler):
|
|
77
79
|
|
78
80
|
def store_path(
|
79
81
|
self,
|
80
|
-
artifact:
|
81
|
-
path:
|
82
|
-
name:
|
82
|
+
artifact: Artifact,
|
83
|
+
path: URIStr | FilePathStr,
|
84
|
+
name: StrPath | None = None,
|
83
85
|
checksum: bool = True,
|
84
|
-
max_objects:
|
86
|
+
max_objects: int | None = None,
|
85
87
|
) -> Sequence[ArtifactManifestEntry]:
|
86
88
|
"""Store the file or directory at the given path into the specified artifact.
|
87
89
|
|
@@ -97,7 +99,7 @@ class WBArtifactHandler(StorageHandler):
|
|
97
99
|
"""
|
98
100
|
# Recursively resolve the reference until a concrete asset is found
|
99
101
|
# TODO: Consider resolving server-side for performance improvements.
|
100
|
-
iter_path:
|
102
|
+
iter_path: URIStr | FilePathStr | None = path
|
101
103
|
while iter_path is not None and urlparse(iter_path).scheme == self._scheme:
|
102
104
|
artifact_id = util.host_from_path(iter_path)
|
103
105
|
artifact_file_path = util.uri_from_path(iter_path)
|
@@ -1,7 +1,9 @@
|
|
1
1
|
"""WB local artifact storage handler."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import os
|
4
|
-
from typing import TYPE_CHECKING,
|
6
|
+
from typing import TYPE_CHECKING, Sequence
|
5
7
|
|
6
8
|
import wandb
|
7
9
|
from wandb import util
|
@@ -22,25 +24,25 @@ class WBLocalArtifactHandler(StorageHandler):
|
|
22
24
|
def __init__(self) -> None:
|
23
25
|
self._scheme = "wandb-client-artifact"
|
24
26
|
|
25
|
-
def can_handle(self, parsed_url:
|
27
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
26
28
|
return parsed_url.scheme == self._scheme
|
27
29
|
|
28
30
|
def load_path(
|
29
31
|
self,
|
30
32
|
manifest_entry: ArtifactManifestEntry,
|
31
33
|
local: bool = False,
|
32
|
-
) ->
|
34
|
+
) -> URIStr | FilePathStr:
|
33
35
|
raise NotImplementedError(
|
34
36
|
"Should not be loading a path for an artifact entry with unresolved client id."
|
35
37
|
)
|
36
38
|
|
37
39
|
def store_path(
|
38
40
|
self,
|
39
|
-
artifact:
|
40
|
-
path:
|
41
|
-
name:
|
41
|
+
artifact: Artifact,
|
42
|
+
path: URIStr | FilePathStr,
|
43
|
+
name: StrPath | None = None,
|
42
44
|
checksum: bool = True,
|
43
|
-
max_objects:
|
45
|
+
max_objects: int | None = None,
|
44
46
|
) -> Sequence[ArtifactManifestEntry]:
|
45
47
|
"""Store the file or directory at the given path within the specified artifact.
|
46
48
|
|