wandb 0.15.4__py3-none-any.whl → 0.15.5__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. wandb/__init__.py +1 -1
  2. wandb/analytics/sentry.py +1 -0
  3. wandb/apis/internal.py +3 -0
  4. wandb/apis/public.py +18 -20
  5. wandb/beta/workflows.py +5 -6
  6. wandb/cli/cli.py +27 -27
  7. wandb/data_types.py +2 -0
  8. wandb/integration/langchain/wandb_tracer.py +16 -179
  9. wandb/integration/sagemaker/config.py +2 -2
  10. wandb/integration/tensorboard/log.py +4 -4
  11. wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
  12. wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
  13. wandb/proto/wandb_deprecated.py +3 -1
  14. wandb/sdk/__init__.py +1 -4
  15. wandb/sdk/artifacts/__init__.py +0 -14
  16. wandb/sdk/artifacts/artifact.py +1757 -277
  17. wandb/sdk/artifacts/artifact_manifest_entry.py +26 -6
  18. wandb/sdk/artifacts/artifact_state.py +10 -0
  19. wandb/sdk/artifacts/artifacts_cache.py +7 -8
  20. wandb/sdk/artifacts/exceptions.py +4 -4
  21. wandb/sdk/artifacts/storage_handler.py +2 -2
  22. wandb/sdk/artifacts/storage_handlers/azure_handler.py +16 -6
  23. wandb/sdk/artifacts/storage_handlers/gcs_handler.py +2 -2
  24. wandb/sdk/artifacts/storage_handlers/http_handler.py +2 -2
  25. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +2 -2
  26. wandb/sdk/artifacts/storage_handlers/multi_handler.py +2 -2
  27. wandb/sdk/artifacts/storage_handlers/s3_handler.py +35 -32
  28. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +2 -2
  29. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +5 -9
  30. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +2 -2
  31. wandb/sdk/artifacts/storage_policies/s3_bucket_policy.py +2 -2
  32. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +24 -16
  33. wandb/sdk/artifacts/storage_policy.py +3 -3
  34. wandb/sdk/data_types/_dtypes.py +7 -12
  35. wandb/sdk/data_types/base_types/json_metadata.py +2 -2
  36. wandb/sdk/data_types/base_types/media.py +5 -6
  37. wandb/sdk/data_types/base_types/wb_value.py +12 -13
  38. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +4 -5
  39. wandb/sdk/data_types/helper_types/classes.py +5 -8
  40. wandb/sdk/data_types/helper_types/image_mask.py +4 -5
  41. wandb/sdk/data_types/histogram.py +3 -3
  42. wandb/sdk/data_types/html.py +3 -4
  43. wandb/sdk/data_types/image.py +4 -5
  44. wandb/sdk/data_types/molecule.py +2 -2
  45. wandb/sdk/data_types/object_3d.py +3 -3
  46. wandb/sdk/data_types/plotly.py +2 -2
  47. wandb/sdk/data_types/saved_model.py +7 -8
  48. wandb/sdk/data_types/trace_tree.py +4 -4
  49. wandb/sdk/data_types/video.py +4 -4
  50. wandb/sdk/interface/interface.py +8 -10
  51. wandb/sdk/internal/file_stream.py +2 -3
  52. wandb/sdk/internal/internal_api.py +99 -4
  53. wandb/sdk/internal/job_builder.py +15 -7
  54. wandb/sdk/internal/sender.py +4 -0
  55. wandb/sdk/internal/settings_static.py +1 -0
  56. wandb/sdk/launch/_project_spec.py +9 -7
  57. wandb/sdk/launch/agent/agent.py +115 -58
  58. wandb/sdk/launch/agent/job_status_tracker.py +34 -0
  59. wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
  60. wandb/sdk/launch/builder/abstract.py +5 -1
  61. wandb/sdk/launch/builder/build.py +16 -10
  62. wandb/sdk/launch/builder/docker_builder.py +9 -2
  63. wandb/sdk/launch/builder/kaniko_builder.py +108 -22
  64. wandb/sdk/launch/builder/noop.py +3 -1
  65. wandb/sdk/launch/environment/aws_environment.py +2 -1
  66. wandb/sdk/launch/environment/azure_environment.py +124 -0
  67. wandb/sdk/launch/github_reference.py +30 -18
  68. wandb/sdk/launch/launch.py +1 -1
  69. wandb/sdk/launch/loader.py +15 -0
  70. wandb/sdk/launch/registry/azure_container_registry.py +132 -0
  71. wandb/sdk/launch/registry/elastic_container_registry.py +38 -4
  72. wandb/sdk/launch/registry/google_artifact_registry.py +46 -7
  73. wandb/sdk/launch/runner/abstract.py +19 -3
  74. wandb/sdk/launch/runner/kubernetes_runner.py +111 -47
  75. wandb/sdk/launch/runner/local_container.py +101 -48
  76. wandb/sdk/launch/runner/sagemaker_runner.py +59 -9
  77. wandb/sdk/launch/runner/vertex_runner.py +8 -4
  78. wandb/sdk/launch/sweeps/scheduler.py +102 -27
  79. wandb/sdk/launch/sweeps/utils.py +21 -0
  80. wandb/sdk/launch/utils.py +19 -7
  81. wandb/sdk/lib/_settings_toposort_generated.py +3 -0
  82. wandb/sdk/service/server.py +22 -9
  83. wandb/sdk/service/service.py +27 -8
  84. wandb/sdk/verify/verify.py +6 -9
  85. wandb/sdk/wandb_config.py +2 -4
  86. wandb/sdk/wandb_init.py +2 -0
  87. wandb/sdk/wandb_require.py +7 -0
  88. wandb/sdk/wandb_run.py +32 -35
  89. wandb/sdk/wandb_settings.py +10 -3
  90. wandb/testing/relay.py +15 -2
  91. wandb/util.py +55 -23
  92. {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/METADATA +11 -8
  93. {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/RECORD +97 -97
  94. wandb/integration/langchain/util.py +0 -191
  95. wandb/sdk/artifacts/invalid_artifact.py +0 -23
  96. wandb/sdk/artifacts/lazy_artifact.py +0 -162
  97. wandb/sdk/artifacts/local_artifact.py +0 -719
  98. wandb/sdk/artifacts/public_artifact.py +0 -1188
  99. {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/LICENSE +0 -0
  100. {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/WHEEL +0 -0
  101. {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/entry_points.txt +0 -0
  102. {wandb-0.15.4.dist-info → wandb-0.15.5.dist-info}/top_level.txt +0 -0
@@ -2,14 +2,24 @@
2
2
  import os
3
3
  from pathlib import Path
4
4
  from typing import TYPE_CHECKING, Dict, Optional, Union
5
+ from urllib.parse import urlparse
5
6
 
7
+ import wandb
8
+ from wandb import util
6
9
  from wandb.errors.term import termwarn
7
10
  from wandb.sdk.lib import filesystem
8
- from wandb.sdk.lib.hashutil import B64MD5, ETag, b64_to_hex_id, md5_file_b64
11
+ from wandb.sdk.lib.hashutil import (
12
+ B64MD5,
13
+ ETag,
14
+ b64_to_hex_id,
15
+ hex_to_b64_id,
16
+ md5_file_b64,
17
+ )
9
18
  from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
10
19
 
11
20
  if TYPE_CHECKING:
12
- from wandb.sdk.artifacts.public_artifact import Artifact as PublicArtifact
21
+ from wandb.apis.public import RetryingClient
22
+ from wandb.sdk.artifacts.artifact import Artifact
13
23
 
14
24
 
15
25
  class ArtifactManifestEntry:
@@ -23,7 +33,8 @@ class ArtifactManifestEntry:
23
33
  extra: Dict
24
34
  local_path: Optional[str]
25
35
 
26
- _parent_artifact: Optional["PublicArtifact"] = None
36
+ _parent_artifact: Optional["Artifact"] = None
37
+ _download_url: Optional[str] = None
27
38
 
28
39
  def __init__(
29
40
  self,
@@ -51,7 +62,7 @@ class ArtifactManifestEntry:
51
62
  termwarn("ArtifactManifestEntry.name is deprecated, use .path instead")
52
63
  return self.path
53
64
 
54
- def parent_artifact(self) -> "PublicArtifact":
65
+ def parent_artifact(self) -> "Artifact":
55
66
  """Get the artifact to which this artifact entry belongs.
56
67
 
57
68
  Returns:
@@ -109,8 +120,7 @@ class ArtifactManifestEntry:
109
120
  if self._parent_artifact is None:
110
121
  return self.ref
111
122
  return self._parent_artifact.manifest.storage_policy.load_reference(
112
- self._parent_artifact.manifest.entries[self.path],
113
- local=False,
123
+ self._parent_artifact.manifest.entries[self.path], local=False
114
124
  )
115
125
 
116
126
  def ref_url(self) -> str:
@@ -137,3 +147,13 @@ class ArtifactManifestEntry:
137
147
  + "/"
138
148
  + self.path
139
149
  )
150
+
151
+ def _is_artifact_reference(self) -> bool:
152
+ return self.ref is not None and urlparse(self.ref).scheme == "wandb-artifact"
153
+
154
+ def _get_referenced_artifact(self, client: "RetryingClient") -> "Artifact":
155
+ artifact: "Artifact" = wandb.Artifact._from_id(
156
+ hex_to_b64_id(util.host_from_path(self.ref)), client
157
+ )
158
+ assert artifact is not None
159
+ return artifact
@@ -0,0 +1,10 @@
1
+ """Artifact state."""
2
+ from enum import Enum
3
+
4
+
5
+ class ArtifactState(Enum):
6
+ PENDING = "PENDING"
7
+ COMMITTED = "COMMITTED"
8
+ DELETED = "DELETED"
9
+ GARBAGE_COLLECTED = "GARBAGE_COLLECTED"
10
+ PENDING_DELETION = "PENDING_DELETION"
@@ -15,8 +15,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
15
15
  if TYPE_CHECKING:
16
16
  import sys
17
17
 
18
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
19
- from wandb.sdk.artifacts.local_artifact import Artifact as LocalArtifact
18
+ from wandb.sdk.artifacts.artifact import Artifact
20
19
 
21
20
  if sys.version_info >= (3, 8):
22
21
  from typing import Protocol
@@ -36,8 +35,8 @@ class ArtifactsCache:
36
35
  mkdir_exists_ok(self._cache_dir)
37
36
  self._md5_obj_dir = os.path.join(self._cache_dir, "obj", "md5")
38
37
  self._etag_obj_dir = os.path.join(self._cache_dir, "obj", "etag")
39
- self._artifacts_by_id: Dict[str, "ArtifactInterface"] = CappedDict()
40
- self._artifacts_by_client_id: Dict[str, "LocalArtifact"] = CappedDict()
38
+ self._artifacts_by_id: Dict[str, "Artifact"] = CappedDict()
39
+ self._artifacts_by_client_id: Dict[str, "Artifact"] = CappedDict()
41
40
 
42
41
  def check_md5_obj_path(
43
42
  self, b64_md5: B64MD5, size: int
@@ -69,18 +68,18 @@ class ArtifactsCache:
69
68
  mkdir_exists_ok(os.path.dirname(path))
70
69
  return FilePathStr(path), False, opener
71
70
 
72
- def get_artifact(self, artifact_id: str) -> Optional["ArtifactInterface"]:
71
+ def get_artifact(self, artifact_id: str) -> Optional["Artifact"]:
73
72
  return self._artifacts_by_id.get(artifact_id)
74
73
 
75
- def store_artifact(self, artifact: "ArtifactInterface") -> None:
74
+ def store_artifact(self, artifact: "Artifact") -> None:
76
75
  if not artifact.id:
77
76
  raise ArtifactNotLoggedError(artifact, "store_artifact")
78
77
  self._artifacts_by_id[artifact.id] = artifact
79
78
 
80
- def get_client_artifact(self, client_id: str) -> Optional["LocalArtifact"]:
79
+ def get_client_artifact(self, client_id: str) -> Optional["Artifact"]:
81
80
  return self._artifacts_by_client_id.get(client_id)
82
81
 
83
- def store_client_artifact(self, artifact: "LocalArtifact") -> None:
82
+ def store_client_artifact(self, artifact: "Artifact") -> None:
84
83
  self._artifacts_by_client_id[artifact._client_id] = artifact
85
84
 
86
85
  def cleanup(self, target_size: int, remove_temp: bool = False) -> int:
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Optional
4
4
  from wandb import errors
5
5
 
6
6
  if TYPE_CHECKING:
7
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
7
+ from wandb.sdk.artifacts.artifact import Artifact
8
8
 
9
9
 
10
10
  class ArtifactStatusError(AttributeError):
@@ -12,7 +12,7 @@ class ArtifactStatusError(AttributeError):
12
12
 
13
13
  def __init__(
14
14
  self,
15
- artifact: Optional["ArtifactInterface"] = None,
15
+ artifact: Optional["Artifact"] = None,
16
16
  attr: Optional[str] = None,
17
17
  msg: str = "Artifact is in an invalid state for the requested operation.",
18
18
  ):
@@ -28,7 +28,7 @@ class ArtifactNotLoggedError(ArtifactStatusError):
28
28
  """Raised for Artifact methods or attributes only available after logging."""
29
29
 
30
30
  def __init__(
31
- self, artifact: Optional["ArtifactInterface"] = None, attr: Optional[str] = None
31
+ self, artifact: Optional["Artifact"] = None, attr: Optional[str] = None
32
32
  ):
33
33
  super().__init__(
34
34
  artifact,
@@ -42,7 +42,7 @@ class ArtifactFinalizedError(ArtifactStatusError):
42
42
  """Raised for Artifact methods or attributes that can't be changed after logging."""
43
43
 
44
44
  def __init__(
45
- self, artifact: Optional["ArtifactInterface"] = None, attr: Optional[str] = None
45
+ self, artifact: Optional["Artifact"] = None, attr: Optional[str] = None
46
46
  ):
47
47
  super().__init__(
48
48
  artifact,
@@ -6,7 +6,7 @@ from wandb.sdk.lib.paths import FilePathStr, URIStr
6
6
  if TYPE_CHECKING:
7
7
  from urllib.parse import ParseResult
8
8
 
9
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
9
+ from wandb.sdk.artifacts.artifact import Artifact
10
10
  from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
11
11
 
12
12
  DEFAULT_MAX_OBJECTS = 10000
@@ -39,7 +39,7 @@ class StorageHandler:
39
39
 
40
40
  def store_path(
41
41
  self,
42
- artifact: "ArtifactInterface",
42
+ artifact: "Artifact",
43
43
  path: Union[URIStr, FilePathStr],
44
44
  name: Optional[str] = None,
45
45
  checksum: bool = True,
@@ -4,6 +4,7 @@ from types import ModuleType
4
4
  from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union
5
5
  from urllib.parse import ParseResult, parse_qsl, urlparse
6
6
 
7
+ import wandb
7
8
  from wandb import util
8
9
  from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
9
10
  from wandb.sdk.artifacts.artifacts_cache import get_artifacts_cache
@@ -12,9 +13,10 @@ from wandb.sdk.lib.hashutil import ETag
12
13
  from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
13
14
 
14
15
  if TYPE_CHECKING:
16
+ import azure.identity # type: ignore
15
17
  import azure.storage.blob # type: ignore
16
18
 
17
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
19
+ from wandb.sdk.artifacts.artifact import Artifact
18
20
 
19
21
 
20
22
  class AzureHandler(StorageHandler):
@@ -45,8 +47,7 @@ class AzureHandler(StorageHandler):
45
47
  )
46
48
  version_id = manifest_entry.extra.get("versionID")
47
49
  blob_service_client = self._get_module("azure.storage.blob").BlobServiceClient(
48
- account_url,
49
- credential=self._get_module("azure.identity").DefaultAzureCredential(),
50
+ account_url, credential=self._get_credential(account_url)
50
51
  )
51
52
  blob_client = blob_service_client.get_blob_client(
52
53
  container=container_name, blob=blob_name
@@ -89,7 +90,7 @@ class AzureHandler(StorageHandler):
89
90
 
90
91
  def store_path(
91
92
  self,
92
- artifact: "ArtifactInterface",
93
+ artifact: "Artifact",
93
94
  path: Union[URIStr, FilePathStr],
94
95
  name: Optional[StrPath] = None,
95
96
  checksum: bool = True,
@@ -104,8 +105,7 @@ class AzureHandler(StorageHandler):
104
105
  ]
105
106
 
106
107
  blob_service_client = self._get_module("azure.storage.blob").BlobServiceClient(
107
- account_url,
108
- credential=self._get_module("azure.identity").DefaultAzureCredential(),
108
+ account_url, credential=self._get_credential(account_url)
109
109
  )
110
110
  blob_client = blob_service_client.get_blob_client(
111
111
  container=container_name, blob=blob_name
@@ -157,6 +157,16 @@ class AzureHandler(StorageHandler):
157
157
  assert isinstance(module, ModuleType)
158
158
  return module
159
159
 
160
+ def _get_credential(
161
+ self, account_url: str
162
+ ) -> Union["azure.identity.DefaultAzureCredential", str]:
163
+ if (
164
+ wandb.run
165
+ and account_url in wandb.run.settings.azure_account_url_to_access_key
166
+ ):
167
+ return wandb.run.settings.azure_account_url_to_access_key[account_url]
168
+ return self._get_module("azure.identity").DefaultAzureCredential()
169
+
160
170
  def _parse_uri(self, uri: str) -> Tuple[str, str, str, Dict[str, str]]:
161
171
  parsed_url = urlparse(uri)
162
172
  query = dict(parse_qsl(parsed_url.query))
@@ -16,7 +16,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
16
16
  if TYPE_CHECKING:
17
17
  import google.cloud.storage as gcs_module # type: ignore
18
18
 
19
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
19
+ from wandb.sdk.artifacts.artifact import Artifact
20
20
 
21
21
 
22
22
  class GCSHandler(StorageHandler):
@@ -106,7 +106,7 @@ class GCSHandler(StorageHandler):
106
106
 
107
107
  def store_path(
108
108
  self,
109
- artifact: "ArtifactInterface",
109
+ artifact: "Artifact",
110
110
  path: Union[URIStr, FilePathStr],
111
111
  name: Optional[StrPath] = None,
112
112
  checksum: bool = True,
@@ -13,7 +13,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
13
13
  if TYPE_CHECKING:
14
14
  import requests
15
15
 
16
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
16
+ from wandb.sdk.artifacts.artifact import Artifact
17
17
 
18
18
 
19
19
  class HTTPHandler(StorageHandler):
@@ -69,7 +69,7 @@ class HTTPHandler(StorageHandler):
69
69
 
70
70
  def store_path(
71
71
  self,
72
- artifact: "ArtifactInterface",
72
+ artifact: "Artifact",
73
73
  path: Union[URIStr, FilePathStr],
74
74
  name: Optional[StrPath] = None,
75
75
  checksum: bool = True,
@@ -15,7 +15,7 @@ from wandb.sdk.lib.hashutil import B64MD5, md5_file_b64, md5_string
15
15
  from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
16
16
 
17
17
  if TYPE_CHECKING:
18
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
18
+ from wandb.sdk.artifacts.artifact import Artifact
19
19
 
20
20
 
21
21
  class LocalFileHandler(StorageHandler):
@@ -66,7 +66,7 @@ class LocalFileHandler(StorageHandler):
66
66
 
67
67
  def store_path(
68
68
  self,
69
- artifact: "ArtifactInterface",
69
+ artifact: "Artifact",
70
70
  path: Union[URIStr, FilePathStr],
71
71
  name: Optional[StrPath] = None,
72
72
  checksum: bool = True,
@@ -6,7 +6,7 @@ from wandb.sdk.artifacts.storage_handler import StorageHandler
6
6
  from wandb.sdk.lib.paths import FilePathStr, URIStr
7
7
 
8
8
  if TYPE_CHECKING:
9
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
9
+ from wandb.sdk.artifacts.artifact import Artifact
10
10
  from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
11
11
 
12
12
 
@@ -41,7 +41,7 @@ class MultiHandler(StorageHandler):
41
41
 
42
42
  def store_path(
43
43
  self,
44
- artifact: "ArtifactInterface",
44
+ artifact: "Artifact",
45
45
  path: Union[URIStr, FilePathStr],
46
46
  name: Optional[str] = None,
47
47
  checksum: bool = True,
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
27
27
  import boto3.s3 # type: ignore
28
28
  import boto3.session # type: ignore
29
29
 
30
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
30
+ from wandb.sdk.artifacts.artifact import Artifact
31
31
 
32
32
 
33
33
  class S3Handler(StorageHandler):
@@ -104,39 +104,42 @@ class S3Handler(StorageHandler):
104
104
  version = manifest_entry.extra.get("versionID")
105
105
 
106
106
  extra_args = {}
107
- if version is None:
108
- # We don't have version information so just get the latest version
109
- # and fallback to listing all versions if we don't have a match.
107
+ if version:
108
+ obj = self._s3.ObjectVersion(bucket, key, version).Object()
109
+ extra_args["VersionId"] = version
110
+ else:
110
111
  obj = self._s3.Object(bucket, key)
112
+
113
+ try:
111
114
  etag = self._etag_from_obj(obj)
112
- if etag != manifest_entry.digest:
113
- if self.versioning_enabled(bucket):
114
- # Fallback to listing versions
115
- obj = None
116
- object_versions = self._s3.Bucket(bucket).object_versions.filter(
117
- Prefix=key
118
- )
119
- for object_version in object_versions:
120
- if (
121
- manifest_entry.extra.get("etag")
122
- == object_version.e_tag[1:-1]
123
- ):
124
- obj = object_version.Object()
125
- extra_args["VersionId"] = object_version.version_id
126
- break
127
- if obj is None:
128
- raise ValueError(
129
- "Couldn't find object version for {}/{} matching etag {}".format(
130
- bucket, key, manifest_entry.extra.get("etag")
131
- )
132
- )
133
- else:
134
- raise ValueError(
135
- f"Digest mismatch for object {manifest_entry.ref}: expected {manifest_entry.digest} but found {etag}"
115
+ except self._botocore.exceptions.ClientError as e:
116
+ if e.response["Error"]["Code"] == "404":
117
+ raise FileNotFoundError(
118
+ f"Unable to find {manifest_entry.path} at s3://{bucket}/{key}"
119
+ ) from e
120
+ raise
121
+
122
+ if etag != manifest_entry.digest:
123
+ # Try to match the etag with some other version.
124
+ if version or not self.versioning_enabled(bucket):
125
+ raise ValueError(
126
+ f"Digest mismatch for object {manifest_entry.ref}: expected {manifest_entry.digest} but found {etag}"
127
+ )
128
+ obj = None
129
+ object_versions = self._s3.Bucket(bucket).object_versions.filter(Prefix=key)
130
+ for object_version in object_versions:
131
+ if manifest_entry.extra.get("etag") == self._etag_from_obj(
132
+ object_version
133
+ ):
134
+ obj = object_version.Object()
135
+ extra_args["VersionId"] = object_version.version_id
136
+ break
137
+ if obj is None:
138
+ raise FileNotFoundError(
139
+ "Couldn't find object version for {}/{} matching etag {}".format(
140
+ bucket, key, manifest_entry.extra.get("etag")
136
141
  )
137
- else:
138
- obj = self._s3.ObjectVersion(bucket, key, version).Object()
139
- extra_args["VersionId"] = version
142
+ )
140
143
 
141
144
  with cache_open(mode="wb") as f:
142
145
  obj.download_fileobj(f, ExtraArgs=extra_args)
@@ -144,7 +147,7 @@ class S3Handler(StorageHandler):
144
147
 
145
148
  def store_path(
146
149
  self,
147
- artifact: "ArtifactInterface",
150
+ artifact: "Artifact",
148
151
  path: Union[URIStr, FilePathStr],
149
152
  name: Optional[StrPath] = None,
150
153
  checksum: bool = True,
@@ -10,7 +10,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
10
10
  if TYPE_CHECKING:
11
11
  from urllib.parse import ParseResult
12
12
 
13
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
13
+ from wandb.sdk.artifacts.artifact import Artifact
14
14
 
15
15
 
16
16
  class TrackingHandler(StorageHandler):
@@ -47,7 +47,7 @@ class TrackingHandler(StorageHandler):
47
47
 
48
48
  def store_path(
49
49
  self,
50
- artifact: "ArtifactInterface",
50
+ artifact: "Artifact",
51
51
  path: Union[URIStr, FilePathStr],
52
52
  name: Optional[StrPath] = None,
53
53
  checksum: bool = True,
@@ -3,11 +3,11 @@ import os
3
3
  from typing import TYPE_CHECKING, Optional, Sequence, Union
4
4
  from urllib.parse import urlparse
5
5
 
6
+ import wandb
6
7
  from wandb import util
7
8
  from wandb.apis import PublicApi
8
9
  from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
9
10
  from wandb.sdk.artifacts.artifacts_cache import get_artifacts_cache
10
- from wandb.sdk.artifacts.public_artifact import Artifact as PublicArtifact
11
11
  from wandb.sdk.artifacts.storage_handler import StorageHandler
12
12
  from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, hex_to_b64_id
13
13
  from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
@@ -15,7 +15,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
15
15
  if TYPE_CHECKING:
16
16
  from urllib.parse import ParseResult
17
17
 
18
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
18
+ from wandb.sdk.artifacts.artifact import Artifact
19
19
 
20
20
 
21
21
  class WBArtifactHandler(StorageHandler):
@@ -62,7 +62,7 @@ class WBArtifactHandler(StorageHandler):
62
62
  artifact_id = util.host_from_path(manifest_entry.ref)
63
63
  artifact_file_path = util.uri_from_path(manifest_entry.ref)
64
64
 
65
- dep_artifact = PublicArtifact.from_id(
65
+ dep_artifact = wandb.Artifact._from_id(
66
66
  hex_to_b64_id(artifact_id), self.client.client
67
67
  )
68
68
  assert dep_artifact is not None
@@ -76,7 +76,7 @@ class WBArtifactHandler(StorageHandler):
76
76
 
77
77
  def store_path(
78
78
  self,
79
- artifact: "ArtifactInterface",
79
+ artifact: "Artifact",
80
80
  path: Union[URIStr, FilePathStr],
81
81
  name: Optional[StrPath] = None,
82
82
  checksum: bool = True,
@@ -100,15 +100,11 @@ class WBArtifactHandler(StorageHandler):
100
100
  while iter_path is not None and urlparse(iter_path).scheme == self._scheme:
101
101
  artifact_id = util.host_from_path(iter_path)
102
102
  artifact_file_path = util.uri_from_path(iter_path)
103
- target_artifact = PublicArtifact.from_id(
103
+ target_artifact = wandb.Artifact._from_id(
104
104
  hex_to_b64_id(artifact_id), self.client.client
105
105
  )
106
106
  assert target_artifact is not None
107
107
 
108
- # this should only have an effect if the user added the reference by url
109
- # string directly (in other words they did not already load the artifact into ram.)
110
- target_artifact._load_manifest()
111
-
112
108
  entry = target_artifact.manifest.get_entry_by_path(artifact_file_path)
113
109
  assert entry is not None
114
110
  iter_path = entry.ref
@@ -12,7 +12,7 @@ from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
12
12
  if TYPE_CHECKING:
13
13
  from urllib.parse import ParseResult
14
14
 
15
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
15
+ from wandb.sdk.artifacts.artifact import Artifact
16
16
 
17
17
 
18
18
  class WBLocalArtifactHandler(StorageHandler):
@@ -36,7 +36,7 @@ class WBLocalArtifactHandler(StorageHandler):
36
36
 
37
37
  def store_path(
38
38
  self,
39
- artifact: "ArtifactInterface",
39
+ artifact: "Artifact",
40
40
  path: Union[URIStr, FilePathStr],
41
41
  name: Optional[StrPath] = None,
42
42
  checksum: bool = True,
@@ -9,7 +9,7 @@ from wandb.sdk.artifacts.storage_handlers.tracking_handler import TrackingHandle
9
9
  from wandb.sdk.artifacts.storage_policy import StoragePolicy
10
10
 
11
11
  if TYPE_CHECKING:
12
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
12
+ from wandb.sdk.artifacts.artifact import Artifact
13
13
  from wandb.sdk.lib.paths import FilePathStr, URIStr
14
14
 
15
15
 
@@ -50,7 +50,7 @@ class __S3BucketPolicy(StoragePolicy): # noqa: N801
50
50
 
51
51
  def store_path(
52
52
  self,
53
- artifact: "ArtifactInterface",
53
+ artifact: "Artifact",
54
54
  path: Union[URIStr, FilePathStr],
55
55
  name: Optional[str] = None,
56
56
  checksum: bool = True,
@@ -30,12 +30,12 @@ from wandb.sdk.lib.paths import FilePathStr, URIStr
30
30
 
31
31
  if TYPE_CHECKING:
32
32
  from wandb.filesync.step_prepare import StepPrepare
33
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
33
+ from wandb.sdk.artifacts.artifact import Artifact
34
34
  from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
35
35
  from wandb.sdk.internal import progress
36
36
 
37
- # This makes the first sleep 1s, and then doubles it up to total times,
38
- # which makes for ~18 hours.
37
+ # Sleep length: 0, 2, 4, 8, 16, 32, 64, 120, 120, 120, 120, 120, 120, 120, 120, 120
38
+ # seconds, i.e. a total of 20min 6s.
39
39
  _REQUEST_RETRY_STRATEGY = urllib3.util.retry.Retry(
40
40
  backoff_factor=1,
41
41
  total=16,
@@ -105,7 +105,7 @@ class WandbStoragePolicy(StoragePolicy):
105
105
 
106
106
  def load_file(
107
107
  self,
108
- artifact: "ArtifactInterface",
108
+ artifact: "Artifact",
109
109
  manifest_entry: "ArtifactManifestEntry",
110
110
  ) -> FilePathStr:
111
111
  path, hit, cache_open = self._cache.check_md5_obj_path(
@@ -115,17 +115,25 @@ class WandbStoragePolicy(StoragePolicy):
115
115
  if hit:
116
116
  return path
117
117
 
118
- auth = None
119
- if not _thread_local_api_settings.cookies:
120
- auth = ("api", self._api.api_key)
121
- response = self._session.get(
122
- self._file_url(self._api, artifact.entity, manifest_entry),
123
- auth=auth,
124
- cookies=_thread_local_api_settings.cookies,
125
- headers=_thread_local_api_settings.headers,
126
- stream=True,
127
- )
128
- response.raise_for_status()
118
+ if manifest_entry._download_url is not None:
119
+ response = self._session.get(manifest_entry._download_url, stream=True)
120
+ try:
121
+ response.raise_for_status()
122
+ except Exception:
123
+ # Signed URL might have expired, fall back to fetching it one by one.
124
+ manifest_entry._download_url = None
125
+ if manifest_entry._download_url is None:
126
+ auth = None
127
+ if not _thread_local_api_settings.cookies:
128
+ auth = ("api", self._api.api_key)
129
+ response = self._session.get(
130
+ self._file_url(self._api, artifact.entity, manifest_entry),
131
+ auth=auth,
132
+ cookies=_thread_local_api_settings.cookies,
133
+ headers=_thread_local_api_settings.headers,
134
+ stream=True,
135
+ )
136
+ response.raise_for_status()
129
137
 
130
138
  with cache_open(mode="wb") as file:
131
139
  for data in response.iter_content(chunk_size=16 * 1024):
@@ -134,7 +142,7 @@ class WandbStoragePolicy(StoragePolicy):
134
142
 
135
143
  def store_reference(
136
144
  self,
137
- artifact: "ArtifactInterface",
145
+ artifact: "Artifact",
138
146
  path: Union[URIStr, FilePathStr],
139
147
  name: Optional[str] = None,
140
148
  checksum: bool = True,
@@ -5,7 +5,7 @@ from wandb.sdk.lib.paths import FilePathStr, URIStr
5
5
 
6
6
  if TYPE_CHECKING:
7
7
  from wandb.filesync.step_prepare import StepPrepare
8
- from wandb.sdk.artifacts.artifact import Artifact as ArtifactInterface
8
+ from wandb.sdk.artifacts.artifact import Artifact
9
9
  from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
10
10
  from wandb.sdk.internal.progress import ProgressFn
11
11
 
@@ -30,7 +30,7 @@ class StoragePolicy:
30
30
  raise NotImplementedError
31
31
 
32
32
  def load_file(
33
- self, artifact: "ArtifactInterface", manifest_entry: "ArtifactManifestEntry"
33
+ self, artifact: "Artifact", manifest_entry: "ArtifactManifestEntry"
34
34
  ) -> FilePathStr:
35
35
  raise NotImplementedError
36
36
 
@@ -57,7 +57,7 @@ class StoragePolicy:
57
57
 
58
58
  def store_reference(
59
59
  self,
60
- artifact: "ArtifactInterface",
60
+ artifact: "Artifact",
61
61
  path: Union[URIStr, FilePathStr],
62
62
  name: Optional[str] = None,
63
63
  checksum: bool = True,