wandb 0.15.4__py3-none-any.whl → 0.15.5__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- wandb/__init__.py +1 -1
- wandb/analytics/sentry.py +1 -0
- wandb/apis/internal.py +3 -0
- wandb/apis/public.py +18 -20
- wandb/beta/workflows.py +5 -6
- wandb/cli/cli.py +27 -27
- wandb/data_types.py +2 -0
- wandb/integration/langchain/wandb_tracer.py +16 -179
- wandb/integration/sagemaker/config.py +2 -2
- wandb/integration/tensorboard/log.py +4 -4
- wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
- wandb/proto/wandb_deprecated.py +3 -1
- wandb/sdk/__init__.py +1 -4
- wandb/sdk/artifacts/__init__.py +0 -14
- wandb/sdk/artifacts/artifact.py +1757 -277
- wandb/sdk/artifacts/artifact_manifest_entry.py +26 -6
- wandb/sdk/artifacts/artifact_state.py +10 -0
- wandb/sdk/artifacts/artifacts_cache.py +7 -8
- wandb/sdk/artifacts/exceptions.py +4 -4
- wandb/sdk/artifacts/storage_handler.py +2 -2
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +16 -6
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +2 -2
- wandb/sdk/artifacts/storage_handlers/http_handler.py +2 -2
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +2 -2
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +2 -2
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +35 -32
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +2 -2
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +5 -9
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +2 -2
- wandb/sdk/artifacts/storage_policies/s3_bucket_policy.py +2 -2
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +24 -16
- wandb/sdk/artifacts/storage_policy.py +3 -3
- wandb/sdk/data_types/_dtypes.py +7 -12
- wandb/sdk/data_types/base_types/json_metadata.py +2 -2
- wandb/sdk/data_types/base_types/media.py +5 -6
- wandb/sdk/data_types/base_types/wb_value.py +12 -13
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +4 -5
- wandb/sdk/data_types/helper_types/classes.py +5 -8
- wandb/sdk/data_types/helper_types/image_mask.py +4 -5
- wandb/sdk/data_types/histogram.py +3 -3
- wandb/sdk/data_types/html.py +3 -4
- wandb/sdk/data_types/image.py +4 -5
- wandb/sdk/data_types/molecule.py +2 -2
- wandb/sdk/data_types/object_3d.py +3 -3
- wandb/sdk/data_types/plotly.py +2 -2
- wandb/sdk/data_types/saved_model.py +7 -8
- wandb/sdk/data_types/trace_tree.py +4 -4
- wandb/sdk/data_types/video.py +4 -4
- wandb/sdk/interface/interface.py +8 -10
- wandb/sdk/internal/file_stream.py +2 -3
- wandb/sdk/internal/internal_api.py +99 -4
- wandb/sdk/internal/job_builder.py +15 -7
- wandb/sdk/internal/sender.py +4 -0
- wandb/sdk/internal/settings_static.py +1 -0
- wandb/sdk/launch/_project_spec.py +9 -7
- wandb/sdk/launch/agent/agent.py +115 -58
- wandb/sdk/launch/agent/job_status_tracker.py +34 -0
- wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
- wandb/sdk/launch/builder/abstract.py +5 -1
- wandb/sdk/launch/builder/build.py +16 -10
- wandb/sdk/launch/builder/docker_builder.py +9 -2
- wandb/sdk/launch/builder/kaniko_builder.py +108 -22
- wandb/sdk/launch/builder/noop.py +3 -1
- wandb/sdk/launch/environment/aws_environment.py +2 -1
- wandb/sdk/launch/environment/azure_environment.py +124 -0
- wandb/sdk/launch/github_reference.py +30 -18
- wandb/sdk/launch/launch.py +1 -1
- wandb/sdk/launch/loader.py +15 -0
- wandb/sdk/launch/registry/azure_container_registry.py +132 -0
- wandb/sdk/launch/registry/elastic_container_registry.py +38 -4
- wandb/sdk/launch/registry/google_artifact_registry.py +46 -7
- wandb/sdk/launch/runner/abstract.py +19 -3
- wandb/sdk/launch/runner/kubernetes_runner.py +111 -47
- wandb/sdk/launch/runner/local_container.py +101 -48
- wandb/sdk/launch/runner/sagemaker_runner.py +59 -9
- wandb/sdk/launch/runner/vertex_runner.py +8 -4
- wandb/sdk/launch/sweeps/scheduler.py +102 -27
- wandb/sdk/launch/sweeps/utils.py +21 -0
- wandb/sdk/launch/utils.py +19 -7
- wandb/sdk/lib/_settings_toposort_generated.py +3 -0
- wandb/sdk/service/server.py +22 -9
- wandb/sdk/service/service.py +27 -8
- wandb/sdk/verify/verify.py +6 -9
- wandb/sdk/wandb_config.py +2 -4
- wandb/sdk/wandb_init.py +2 -0
- wandb/sdk/wandb_require.py +7 -0
- wandb/sdk/wandb_run.py +32 -35
- wandb/sdk/wandb_settings.py +10 -3
- wandb/testing/relay.py +15 -2
- wandb/util.py +55 -23
- {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/METADATA +11 -8
- {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/RECORD +97 -97
- wandb/integration/langchain/util.py +0 -191
- wandb/sdk/artifacts/invalid_artifact.py +0 -23
- wandb/sdk/artifacts/lazy_artifact.py +0 -162
- wandb/sdk/artifacts/local_artifact.py +0 -719
- wandb/sdk/artifacts/public_artifact.py +0 -1188
- {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/LICENSE +0 -0
- {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/WHEEL +0 -0
- {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/entry_points.txt +0 -0
- {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/top_level.txt +0 -0
@@ -2,14 +2,24 @@
|
|
2
2
|
import os
|
3
3
|
from pathlib import Path
|
4
4
|
from typing import TYPE_CHECKING, Dict, Optional, Union
|
5
|
+
from urllib.parse import urlparse
|
5
6
|
|
7
|
+
import wandb
|
8
|
+
from wandb import util
|
6
9
|
from wandb.errors.term import termwarn
|
7
10
|
from wandb.sdk.lib import filesystem
|
8
|
-
from wandb.sdk.lib.hashutil import
|
11
|
+
from wandb.sdk.lib.hashutil import (
|
12
|
+
B64MD5,
|
13
|
+
ETag,
|
14
|
+
b64_to_hex_id,
|
15
|
+
hex_to_b64_id,
|
16
|
+
md5_file_b64,
|
17
|
+
)
|
9
18
|
from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
|
10
19
|
|
11
20
|
if TYPE_CHECKING:
|
12
|
-
from wandb.
|
21
|
+
from wandb.apis.public import RetryingClient
|
22
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
13
23
|
|
14
24
|
|
15
25
|
class ArtifactManifestEntry:
|
@@ -23,7 +33,8 @@ class ArtifactManifestEntry:
|
|
23
33
|
extra: Dict
|
24
34
|
local_path: Optional[str]
|
25
35
|
|
26
|
-
_parent_artifact: Optional["
|
36
|
+
_parent_artifact: Optional["Artifact"] = None
|
37
|
+
_download_url: Optional[str] = None
|
27
38
|
|
28
39
|
def __init__(
|
29
40
|
self,
|
@@ -51,7 +62,7 @@ class ArtifactManifestEntry:
|
|
51
62
|
termwarn("ArtifactManifestEntry.name is deprecated, use .path instead")
|
52
63
|
return self.path
|
53
64
|
|
54
|
-
def parent_artifact(self) -> "
|
65
|
+
def parent_artifact(self) -> "Artifact":
|
55
66
|
"""Get the artifact to which this artifact entry belongs.
|
56
67
|
|
57
68
|
Returns:
|
@@ -109,8 +120,7 @@ class ArtifactManifestEntry:
|
|
109
120
|
if self._parent_artifact is None:
|
110
121
|
return self.ref
|
111
122
|
return self._parent_artifact.manifest.storage_policy.load_reference(
|
112
|
-
self._parent_artifact.manifest.entries[self.path],
|
113
|
-
local=False,
|
123
|
+
self._parent_artifact.manifest.entries[self.path], local=False
|
114
124
|
)
|
115
125
|
|
116
126
|
def ref_url(self) -> str:
|
@@ -137,3 +147,13 @@ class ArtifactManifestEntry:
|
|
137
147
|
+ "/"
|
138
148
|
+ self.path
|
139
149
|
)
|
150
|
+
|
151
|
+
def _is_artifact_reference(self) -> bool:
|
152
|
+
return self.ref is not None and urlparse(self.ref).scheme == "wandb-artifact"
|
153
|
+
|
154
|
+
def _get_referenced_artifact(self, client: "RetryingClient") -> "Artifact":
|
155
|
+
artifact: "Artifact" = wandb.Artifact._from_id(
|
156
|
+
hex_to_b64_id(util.host_from_path(self.ref)), client
|
157
|
+
)
|
158
|
+
assert artifact is not None
|
159
|
+
return artifact
|
@@ -15,8 +15,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
|
15
15
|
if TYPE_CHECKING:
|
16
16
|
import sys
|
17
17
|
|
18
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
19
|
-
from wandb.sdk.artifacts.local_artifact import Artifact as LocalArtifact
|
18
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
20
19
|
|
21
20
|
if sys.version_info >= (3, 8):
|
22
21
|
from typing import Protocol
|
@@ -36,8 +35,8 @@ class ArtifactsCache:
|
|
36
35
|
mkdir_exists_ok(self._cache_dir)
|
37
36
|
self._md5_obj_dir = os.path.join(self._cache_dir, "obj", "md5")
|
38
37
|
self._etag_obj_dir = os.path.join(self._cache_dir, "obj", "etag")
|
39
|
-
self._artifacts_by_id: Dict[str, "
|
40
|
-
self._artifacts_by_client_id: Dict[str, "
|
38
|
+
self._artifacts_by_id: Dict[str, "Artifact"] = CappedDict()
|
39
|
+
self._artifacts_by_client_id: Dict[str, "Artifact"] = CappedDict()
|
41
40
|
|
42
41
|
def check_md5_obj_path(
|
43
42
|
self, b64_md5: B64MD5, size: int
|
@@ -69,18 +68,18 @@ class ArtifactsCache:
|
|
69
68
|
mkdir_exists_ok(os.path.dirname(path))
|
70
69
|
return FilePathStr(path), False, opener
|
71
70
|
|
72
|
-
def get_artifact(self, artifact_id: str) -> Optional["
|
71
|
+
def get_artifact(self, artifact_id: str) -> Optional["Artifact"]:
|
73
72
|
return self._artifacts_by_id.get(artifact_id)
|
74
73
|
|
75
|
-
def store_artifact(self, artifact: "
|
74
|
+
def store_artifact(self, artifact: "Artifact") -> None:
|
76
75
|
if not artifact.id:
|
77
76
|
raise ArtifactNotLoggedError(artifact, "store_artifact")
|
78
77
|
self._artifacts_by_id[artifact.id] = artifact
|
79
78
|
|
80
|
-
def get_client_artifact(self, client_id: str) -> Optional["
|
79
|
+
def get_client_artifact(self, client_id: str) -> Optional["Artifact"]:
|
81
80
|
return self._artifacts_by_client_id.get(client_id)
|
82
81
|
|
83
|
-
def store_client_artifact(self, artifact: "
|
82
|
+
def store_client_artifact(self, artifact: "Artifact") -> None:
|
84
83
|
self._artifacts_by_client_id[artifact._client_id] = artifact
|
85
84
|
|
86
85
|
def cleanup(self, target_size: int, remove_temp: bool = False) -> int:
|
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Optional
|
|
4
4
|
from wandb import errors
|
5
5
|
|
6
6
|
if TYPE_CHECKING:
|
7
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
7
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
8
8
|
|
9
9
|
|
10
10
|
class ArtifactStatusError(AttributeError):
|
@@ -12,7 +12,7 @@ class ArtifactStatusError(AttributeError):
|
|
12
12
|
|
13
13
|
def __init__(
|
14
14
|
self,
|
15
|
-
artifact: Optional["
|
15
|
+
artifact: Optional["Artifact"] = None,
|
16
16
|
attr: Optional[str] = None,
|
17
17
|
msg: str = "Artifact is in an invalid state for the requested operation.",
|
18
18
|
):
|
@@ -28,7 +28,7 @@ class ArtifactNotLoggedError(ArtifactStatusError):
|
|
28
28
|
"""Raised for Artifact methods or attributes only available after logging."""
|
29
29
|
|
30
30
|
def __init__(
|
31
|
-
self, artifact: Optional["
|
31
|
+
self, artifact: Optional["Artifact"] = None, attr: Optional[str] = None
|
32
32
|
):
|
33
33
|
super().__init__(
|
34
34
|
artifact,
|
@@ -42,7 +42,7 @@ class ArtifactFinalizedError(ArtifactStatusError):
|
|
42
42
|
"""Raised for Artifact methods or attributes that can't be changed after logging."""
|
43
43
|
|
44
44
|
def __init__(
|
45
|
-
self, artifact: Optional["
|
45
|
+
self, artifact: Optional["Artifact"] = None, attr: Optional[str] = None
|
46
46
|
):
|
47
47
|
super().__init__(
|
48
48
|
artifact,
|
@@ -6,7 +6,7 @@ from wandb.sdk.lib.paths import FilePathStr, URIStr
|
|
6
6
|
if TYPE_CHECKING:
|
7
7
|
from urllib.parse import ParseResult
|
8
8
|
|
9
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
9
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
10
10
|
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
11
11
|
|
12
12
|
DEFAULT_MAX_OBJECTS = 10000
|
@@ -39,7 +39,7 @@ class StorageHandler:
|
|
39
39
|
|
40
40
|
def store_path(
|
41
41
|
self,
|
42
|
-
artifact: "
|
42
|
+
artifact: "Artifact",
|
43
43
|
path: Union[URIStr, FilePathStr],
|
44
44
|
name: Optional[str] = None,
|
45
45
|
checksum: bool = True,
|
@@ -4,6 +4,7 @@ from types import ModuleType
|
|
4
4
|
from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union
|
5
5
|
from urllib.parse import ParseResult, parse_qsl, urlparse
|
6
6
|
|
7
|
+
import wandb
|
7
8
|
from wandb import util
|
8
9
|
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
9
10
|
from wandb.sdk.artifacts.artifacts_cache import get_artifacts_cache
|
@@ -12,9 +13,10 @@ from wandb.sdk.lib.hashutil import ETag
|
|
12
13
|
from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
|
13
14
|
|
14
15
|
if TYPE_CHECKING:
|
16
|
+
import azure.identity # type: ignore
|
15
17
|
import azure.storage.blob # type: ignore
|
16
18
|
|
17
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
19
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
18
20
|
|
19
21
|
|
20
22
|
class AzureHandler(StorageHandler):
|
@@ -45,8 +47,7 @@ class AzureHandler(StorageHandler):
|
|
45
47
|
)
|
46
48
|
version_id = manifest_entry.extra.get("versionID")
|
47
49
|
blob_service_client = self._get_module("azure.storage.blob").BlobServiceClient(
|
48
|
-
account_url,
|
49
|
-
credential=self._get_module("azure.identity").DefaultAzureCredential(),
|
50
|
+
account_url, credential=self._get_credential(account_url)
|
50
51
|
)
|
51
52
|
blob_client = blob_service_client.get_blob_client(
|
52
53
|
container=container_name, blob=blob_name
|
@@ -89,7 +90,7 @@ class AzureHandler(StorageHandler):
|
|
89
90
|
|
90
91
|
def store_path(
|
91
92
|
self,
|
92
|
-
artifact: "
|
93
|
+
artifact: "Artifact",
|
93
94
|
path: Union[URIStr, FilePathStr],
|
94
95
|
name: Optional[StrPath] = None,
|
95
96
|
checksum: bool = True,
|
@@ -104,8 +105,7 @@ class AzureHandler(StorageHandler):
|
|
104
105
|
]
|
105
106
|
|
106
107
|
blob_service_client = self._get_module("azure.storage.blob").BlobServiceClient(
|
107
|
-
account_url,
|
108
|
-
credential=self._get_module("azure.identity").DefaultAzureCredential(),
|
108
|
+
account_url, credential=self._get_credential(account_url)
|
109
109
|
)
|
110
110
|
blob_client = blob_service_client.get_blob_client(
|
111
111
|
container=container_name, blob=blob_name
|
@@ -157,6 +157,16 @@ class AzureHandler(StorageHandler):
|
|
157
157
|
assert isinstance(module, ModuleType)
|
158
158
|
return module
|
159
159
|
|
160
|
+
def _get_credential(
|
161
|
+
self, account_url: str
|
162
|
+
) -> Union["azure.identity.DefaultAzureCredential", str]:
|
163
|
+
if (
|
164
|
+
wandb.run
|
165
|
+
and account_url in wandb.run.settings.azure_account_url_to_access_key
|
166
|
+
):
|
167
|
+
return wandb.run.settings.azure_account_url_to_access_key[account_url]
|
168
|
+
return self._get_module("azure.identity").DefaultAzureCredential()
|
169
|
+
|
160
170
|
def _parse_uri(self, uri: str) -> Tuple[str, str, str, Dict[str, str]]:
|
161
171
|
parsed_url = urlparse(uri)
|
162
172
|
query = dict(parse_qsl(parsed_url.query))
|
@@ -16,7 +16,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
|
16
16
|
if TYPE_CHECKING:
|
17
17
|
import google.cloud.storage as gcs_module # type: ignore
|
18
18
|
|
19
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
19
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
20
20
|
|
21
21
|
|
22
22
|
class GCSHandler(StorageHandler):
|
@@ -106,7 +106,7 @@ class GCSHandler(StorageHandler):
|
|
106
106
|
|
107
107
|
def store_path(
|
108
108
|
self,
|
109
|
-
artifact: "
|
109
|
+
artifact: "Artifact",
|
110
110
|
path: Union[URIStr, FilePathStr],
|
111
111
|
name: Optional[StrPath] = None,
|
112
112
|
checksum: bool = True,
|
@@ -13,7 +13,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
|
13
13
|
if TYPE_CHECKING:
|
14
14
|
import requests
|
15
15
|
|
16
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
16
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
17
17
|
|
18
18
|
|
19
19
|
class HTTPHandler(StorageHandler):
|
@@ -69,7 +69,7 @@ class HTTPHandler(StorageHandler):
|
|
69
69
|
|
70
70
|
def store_path(
|
71
71
|
self,
|
72
|
-
artifact: "
|
72
|
+
artifact: "Artifact",
|
73
73
|
path: Union[URIStr, FilePathStr],
|
74
74
|
name: Optional[StrPath] = None,
|
75
75
|
checksum: bool = True,
|
@@ -15,7 +15,7 @@ from wandb.sdk.lib.hashutil import B64MD5, md5_file_b64, md5_string
|
|
15
15
|
from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
16
16
|
|
17
17
|
if TYPE_CHECKING:
|
18
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
18
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
19
19
|
|
20
20
|
|
21
21
|
class LocalFileHandler(StorageHandler):
|
@@ -66,7 +66,7 @@ class LocalFileHandler(StorageHandler):
|
|
66
66
|
|
67
67
|
def store_path(
|
68
68
|
self,
|
69
|
-
artifact: "
|
69
|
+
artifact: "Artifact",
|
70
70
|
path: Union[URIStr, FilePathStr],
|
71
71
|
name: Optional[StrPath] = None,
|
72
72
|
checksum: bool = True,
|
@@ -6,7 +6,7 @@ from wandb.sdk.artifacts.storage_handler import StorageHandler
|
|
6
6
|
from wandb.sdk.lib.paths import FilePathStr, URIStr
|
7
7
|
|
8
8
|
if TYPE_CHECKING:
|
9
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
9
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
10
10
|
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
11
11
|
|
12
12
|
|
@@ -41,7 +41,7 @@ class MultiHandler(StorageHandler):
|
|
41
41
|
|
42
42
|
def store_path(
|
43
43
|
self,
|
44
|
-
artifact: "
|
44
|
+
artifact: "Artifact",
|
45
45
|
path: Union[URIStr, FilePathStr],
|
46
46
|
name: Optional[str] = None,
|
47
47
|
checksum: bool = True,
|
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
|
|
27
27
|
import boto3.s3 # type: ignore
|
28
28
|
import boto3.session # type: ignore
|
29
29
|
|
30
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
30
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
31
31
|
|
32
32
|
|
33
33
|
class S3Handler(StorageHandler):
|
@@ -104,39 +104,42 @@ class S3Handler(StorageHandler):
|
|
104
104
|
version = manifest_entry.extra.get("versionID")
|
105
105
|
|
106
106
|
extra_args = {}
|
107
|
-
if version
|
108
|
-
|
109
|
-
|
107
|
+
if version:
|
108
|
+
obj = self._s3.ObjectVersion(bucket, key, version).Object()
|
109
|
+
extra_args["VersionId"] = version
|
110
|
+
else:
|
110
111
|
obj = self._s3.Object(bucket, key)
|
112
|
+
|
113
|
+
try:
|
111
114
|
etag = self._etag_from_obj(obj)
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
115
|
+
except self._botocore.exceptions.ClientError as e:
|
116
|
+
if e.response["Error"]["Code"] == "404":
|
117
|
+
raise FileNotFoundError(
|
118
|
+
f"Unable to find {manifest_entry.path} at s3://{bucket}/{key}"
|
119
|
+
) from e
|
120
|
+
raise
|
121
|
+
|
122
|
+
if etag != manifest_entry.digest:
|
123
|
+
# Try to match the etag with some other version.
|
124
|
+
if version or not self.versioning_enabled(bucket):
|
125
|
+
raise ValueError(
|
126
|
+
f"Digest mismatch for object {manifest_entry.ref}: expected {manifest_entry.digest} but found {etag}"
|
127
|
+
)
|
128
|
+
obj = None
|
129
|
+
object_versions = self._s3.Bucket(bucket).object_versions.filter(Prefix=key)
|
130
|
+
for object_version in object_versions:
|
131
|
+
if manifest_entry.extra.get("etag") == self._etag_from_obj(
|
132
|
+
object_version
|
133
|
+
):
|
134
|
+
obj = object_version.Object()
|
135
|
+
extra_args["VersionId"] = object_version.version_id
|
136
|
+
break
|
137
|
+
if obj is None:
|
138
|
+
raise FileNotFoundError(
|
139
|
+
"Couldn't find object version for {}/{} matching etag {}".format(
|
140
|
+
bucket, key, manifest_entry.extra.get("etag")
|
136
141
|
)
|
137
|
-
|
138
|
-
obj = self._s3.ObjectVersion(bucket, key, version).Object()
|
139
|
-
extra_args["VersionId"] = version
|
142
|
+
)
|
140
143
|
|
141
144
|
with cache_open(mode="wb") as f:
|
142
145
|
obj.download_fileobj(f, ExtraArgs=extra_args)
|
@@ -144,7 +147,7 @@ class S3Handler(StorageHandler):
|
|
144
147
|
|
145
148
|
def store_path(
|
146
149
|
self,
|
147
|
-
artifact: "
|
150
|
+
artifact: "Artifact",
|
148
151
|
path: Union[URIStr, FilePathStr],
|
149
152
|
name: Optional[StrPath] = None,
|
150
153
|
checksum: bool = True,
|
@@ -10,7 +10,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
|
10
10
|
if TYPE_CHECKING:
|
11
11
|
from urllib.parse import ParseResult
|
12
12
|
|
13
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
13
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
14
14
|
|
15
15
|
|
16
16
|
class TrackingHandler(StorageHandler):
|
@@ -47,7 +47,7 @@ class TrackingHandler(StorageHandler):
|
|
47
47
|
|
48
48
|
def store_path(
|
49
49
|
self,
|
50
|
-
artifact: "
|
50
|
+
artifact: "Artifact",
|
51
51
|
path: Union[URIStr, FilePathStr],
|
52
52
|
name: Optional[StrPath] = None,
|
53
53
|
checksum: bool = True,
|
@@ -3,11 +3,11 @@ import os
|
|
3
3
|
from typing import TYPE_CHECKING, Optional, Sequence, Union
|
4
4
|
from urllib.parse import urlparse
|
5
5
|
|
6
|
+
import wandb
|
6
7
|
from wandb import util
|
7
8
|
from wandb.apis import PublicApi
|
8
9
|
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
9
10
|
from wandb.sdk.artifacts.artifacts_cache import get_artifacts_cache
|
10
|
-
from wandb.sdk.artifacts.public_artifact import Artifact as PublicArtifact
|
11
11
|
from wandb.sdk.artifacts.storage_handler import StorageHandler
|
12
12
|
from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, hex_to_b64_id
|
13
13
|
from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
@@ -15,7 +15,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
|
15
15
|
if TYPE_CHECKING:
|
16
16
|
from urllib.parse import ParseResult
|
17
17
|
|
18
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
18
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
19
19
|
|
20
20
|
|
21
21
|
class WBArtifactHandler(StorageHandler):
|
@@ -62,7 +62,7 @@ class WBArtifactHandler(StorageHandler):
|
|
62
62
|
artifact_id = util.host_from_path(manifest_entry.ref)
|
63
63
|
artifact_file_path = util.uri_from_path(manifest_entry.ref)
|
64
64
|
|
65
|
-
dep_artifact =
|
65
|
+
dep_artifact = wandb.Artifact._from_id(
|
66
66
|
hex_to_b64_id(artifact_id), self.client.client
|
67
67
|
)
|
68
68
|
assert dep_artifact is not None
|
@@ -76,7 +76,7 @@ class WBArtifactHandler(StorageHandler):
|
|
76
76
|
|
77
77
|
def store_path(
|
78
78
|
self,
|
79
|
-
artifact: "
|
79
|
+
artifact: "Artifact",
|
80
80
|
path: Union[URIStr, FilePathStr],
|
81
81
|
name: Optional[StrPath] = None,
|
82
82
|
checksum: bool = True,
|
@@ -100,15 +100,11 @@ class WBArtifactHandler(StorageHandler):
|
|
100
100
|
while iter_path is not None and urlparse(iter_path).scheme == self._scheme:
|
101
101
|
artifact_id = util.host_from_path(iter_path)
|
102
102
|
artifact_file_path = util.uri_from_path(iter_path)
|
103
|
-
target_artifact =
|
103
|
+
target_artifact = wandb.Artifact._from_id(
|
104
104
|
hex_to_b64_id(artifact_id), self.client.client
|
105
105
|
)
|
106
106
|
assert target_artifact is not None
|
107
107
|
|
108
|
-
# this should only have an effect if the user added the reference by url
|
109
|
-
# string directly (in other words they did not already load the artifact into ram.)
|
110
|
-
target_artifact._load_manifest()
|
111
|
-
|
112
108
|
entry = target_artifact.manifest.get_entry_by_path(artifact_file_path)
|
113
109
|
assert entry is not None
|
114
110
|
iter_path = entry.ref
|
@@ -12,7 +12,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
|
12
12
|
if TYPE_CHECKING:
|
13
13
|
from urllib.parse import ParseResult
|
14
14
|
|
15
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
15
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
16
16
|
|
17
17
|
|
18
18
|
class WBLocalArtifactHandler(StorageHandler):
|
@@ -36,7 +36,7 @@ class WBLocalArtifactHandler(StorageHandler):
|
|
36
36
|
|
37
37
|
def store_path(
|
38
38
|
self,
|
39
|
-
artifact: "
|
39
|
+
artifact: "Artifact",
|
40
40
|
path: Union[URIStr, FilePathStr],
|
41
41
|
name: Optional[StrPath] = None,
|
42
42
|
checksum: bool = True,
|
@@ -9,7 +9,7 @@ from wandb.sdk.artifacts.storage_handlers.tracking_handler import TrackingHandle
|
|
9
9
|
from wandb.sdk.artifacts.storage_policy import StoragePolicy
|
10
10
|
|
11
11
|
if TYPE_CHECKING:
|
12
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
12
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
13
13
|
from wandb.sdk.lib.paths import FilePathStr, URIStr
|
14
14
|
|
15
15
|
|
@@ -50,7 +50,7 @@ class __S3BucketPolicy(StoragePolicy): # noqa: N801
|
|
50
50
|
|
51
51
|
def store_path(
|
52
52
|
self,
|
53
|
-
artifact: "
|
53
|
+
artifact: "Artifact",
|
54
54
|
path: Union[URIStr, FilePathStr],
|
55
55
|
name: Optional[str] = None,
|
56
56
|
checksum: bool = True,
|
@@ -30,12 +30,12 @@ from wandb.sdk.lib.paths import FilePathStr, URIStr
|
|
30
30
|
|
31
31
|
if TYPE_CHECKING:
|
32
32
|
from wandb.filesync.step_prepare import StepPrepare
|
33
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
33
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
34
34
|
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
35
35
|
from wandb.sdk.internal import progress
|
36
36
|
|
37
|
-
#
|
38
|
-
#
|
37
|
+
# Sleep length: 0, 2, 4, 8, 16, 32, 64, 120, 120, 120, 120, 120, 120, 120, 120, 120
|
38
|
+
# seconds, i.e. a total of 20min 6s.
|
39
39
|
_REQUEST_RETRY_STRATEGY = urllib3.util.retry.Retry(
|
40
40
|
backoff_factor=1,
|
41
41
|
total=16,
|
@@ -105,7 +105,7 @@ class WandbStoragePolicy(StoragePolicy):
|
|
105
105
|
|
106
106
|
def load_file(
|
107
107
|
self,
|
108
|
-
artifact: "
|
108
|
+
artifact: "Artifact",
|
109
109
|
manifest_entry: "ArtifactManifestEntry",
|
110
110
|
) -> FilePathStr:
|
111
111
|
path, hit, cache_open = self._cache.check_md5_obj_path(
|
@@ -115,17 +115,25 @@ class WandbStoragePolicy(StoragePolicy):
|
|
115
115
|
if hit:
|
116
116
|
return path
|
117
117
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
118
|
+
if manifest_entry._download_url is not None:
|
119
|
+
response = self._session.get(manifest_entry._download_url, stream=True)
|
120
|
+
try:
|
121
|
+
response.raise_for_status()
|
122
|
+
except Exception:
|
123
|
+
# Signed URL might have expired, fall back to fetching it one by one.
|
124
|
+
manifest_entry._download_url = None
|
125
|
+
if manifest_entry._download_url is None:
|
126
|
+
auth = None
|
127
|
+
if not _thread_local_api_settings.cookies:
|
128
|
+
auth = ("api", self._api.api_key)
|
129
|
+
response = self._session.get(
|
130
|
+
self._file_url(self._api, artifact.entity, manifest_entry),
|
131
|
+
auth=auth,
|
132
|
+
cookies=_thread_local_api_settings.cookies,
|
133
|
+
headers=_thread_local_api_settings.headers,
|
134
|
+
stream=True,
|
135
|
+
)
|
136
|
+
response.raise_for_status()
|
129
137
|
|
130
138
|
with cache_open(mode="wb") as file:
|
131
139
|
for data in response.iter_content(chunk_size=16 * 1024):
|
@@ -134,7 +142,7 @@ class WandbStoragePolicy(StoragePolicy):
|
|
134
142
|
|
135
143
|
def store_reference(
|
136
144
|
self,
|
137
|
-
artifact: "
|
145
|
+
artifact: "Artifact",
|
138
146
|
path: Union[URIStr, FilePathStr],
|
139
147
|
name: Optional[str] = None,
|
140
148
|
checksum: bool = True,
|
@@ -5,7 +5,7 @@ from wandb.sdk.lib.paths import FilePathStr, URIStr
|
|
5
5
|
|
6
6
|
if TYPE_CHECKING:
|
7
7
|
from wandb.filesync.step_prepare import StepPrepare
|
8
|
-
from wandb.sdk.artifacts.artifact import Artifact
|
8
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
9
9
|
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
10
10
|
from wandb.sdk.internal.progress import ProgressFn
|
11
11
|
|
@@ -30,7 +30,7 @@ class StoragePolicy:
|
|
30
30
|
raise NotImplementedError
|
31
31
|
|
32
32
|
def load_file(
|
33
|
-
self, artifact: "
|
33
|
+
self, artifact: "Artifact", manifest_entry: "ArtifactManifestEntry"
|
34
34
|
) -> FilePathStr:
|
35
35
|
raise NotImplementedError
|
36
36
|
|
@@ -57,7 +57,7 @@ class StoragePolicy:
|
|
57
57
|
|
58
58
|
def store_reference(
|
59
59
|
self,
|
60
|
-
artifact: "
|
60
|
+
artifact: "Artifact",
|
61
61
|
path: Union[URIStr, FilePathStr],
|
62
62
|
name: Optional[str] = None,
|
63
63
|
checksum: bool = True,
|