wandb 0.19.12rc1__py3-none-win32.whl → 0.20.1__py3-none-win32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. wandb/__init__.py +1 -2
  2. wandb/__init__.pyi +3 -6
  3. wandb/_iterutils.py +26 -7
  4. wandb/_pydantic/__init__.py +2 -1
  5. wandb/_pydantic/utils.py +7 -0
  6. wandb/agents/pyagent.py +9 -15
  7. wandb/analytics/sentry.py +1 -2
  8. wandb/apis/attrs.py +3 -4
  9. wandb/apis/importers/internals/util.py +1 -1
  10. wandb/apis/importers/validation.py +2 -2
  11. wandb/apis/importers/wandb.py +30 -25
  12. wandb/apis/normalize.py +2 -2
  13. wandb/apis/public/__init__.py +1 -0
  14. wandb/apis/public/api.py +37 -33
  15. wandb/apis/public/artifacts.py +103 -72
  16. wandb/apis/public/jobs.py +3 -2
  17. wandb/apis/public/registries/registries_search.py +4 -2
  18. wandb/apis/public/registries/registry.py +1 -1
  19. wandb/apis/public/registries/utils.py +9 -9
  20. wandb/apis/public/runs.py +18 -6
  21. wandb/automations/_filters/expressions.py +1 -1
  22. wandb/automations/_filters/operators.py +1 -1
  23. wandb/automations/_filters/run_metrics.py +1 -1
  24. wandb/beta/workflows.py +6 -5
  25. wandb/bin/gpu_stats.exe +0 -0
  26. wandb/bin/wandb-core +0 -0
  27. wandb/cli/cli.py +54 -73
  28. wandb/docker/__init__.py +21 -74
  29. wandb/docker/names.py +40 -0
  30. wandb/env.py +0 -1
  31. wandb/errors/util.py +1 -1
  32. wandb/filesync/step_checksum.py +1 -1
  33. wandb/filesync/step_upload.py +1 -1
  34. wandb/integration/diffusers/resolvers/multimodal.py +1 -2
  35. wandb/integration/gym/__init__.py +5 -6
  36. wandb/integration/keras/callbacks/model_checkpoint.py +2 -2
  37. wandb/integration/keras/keras.py +13 -19
  38. wandb/integration/kfp/kfp_patch.py +2 -3
  39. wandb/integration/langchain/wandb_tracer.py +1 -1
  40. wandb/integration/metaflow/metaflow.py +13 -13
  41. wandb/integration/openai/fine_tuning.py +3 -2
  42. wandb/integration/sagemaker/auth.py +2 -1
  43. wandb/integration/sklearn/utils.py +2 -1
  44. wandb/integration/tensorboard/__init__.py +1 -1
  45. wandb/integration/tensorboard/log.py +2 -5
  46. wandb/integration/tensorflow/__init__.py +2 -2
  47. wandb/jupyter.py +20 -17
  48. wandb/plot/confusion_matrix.py +1 -1
  49. wandb/plot/utils.py +8 -7
  50. wandb/proto/v3/wandb_internal_pb2.py +355 -335
  51. wandb/proto/v3/wandb_settings_pb2.py +2 -2
  52. wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
  53. wandb/proto/v4/wandb_internal_pb2.py +339 -335
  54. wandb/proto/v4/wandb_settings_pb2.py +2 -2
  55. wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
  56. wandb/proto/v5/wandb_internal_pb2.py +339 -335
  57. wandb/proto/v5/wandb_settings_pb2.py +2 -2
  58. wandb/proto/v5/wandb_telemetry_pb2.py +12 -12
  59. wandb/proto/v6/wandb_internal_pb2.py +339 -335
  60. wandb/proto/v6/wandb_settings_pb2.py +2 -2
  61. wandb/proto/v6/wandb_telemetry_pb2.py +12 -12
  62. wandb/proto/wandb_deprecated.py +6 -8
  63. wandb/sdk/artifacts/_internal_artifact.py +43 -0
  64. wandb/sdk/artifacts/_validators.py +55 -35
  65. wandb/sdk/artifacts/artifact.py +117 -115
  66. wandb/sdk/artifacts/artifact_download_logger.py +2 -0
  67. wandb/sdk/artifacts/artifact_saver.py +1 -3
  68. wandb/sdk/artifacts/artifact_state.py +2 -0
  69. wandb/sdk/artifacts/artifact_ttl.py +2 -0
  70. wandb/sdk/artifacts/exceptions.py +14 -0
  71. wandb/sdk/artifacts/staging.py +2 -0
  72. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +2 -6
  73. wandb/sdk/artifacts/storage_handlers/multi_handler.py +1 -1
  74. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +2 -6
  75. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +1 -5
  76. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +1 -1
  77. wandb/sdk/artifacts/storage_layout.py +2 -0
  78. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +3 -3
  79. wandb/sdk/backend/backend.py +11 -182
  80. wandb/sdk/data_types/_dtypes.py +2 -6
  81. wandb/sdk/data_types/audio.py +20 -3
  82. wandb/sdk/data_types/base_types/media.py +12 -7
  83. wandb/sdk/data_types/base_types/wb_value.py +8 -18
  84. wandb/sdk/data_types/bokeh.py +19 -2
  85. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +17 -1
  86. wandb/sdk/data_types/helper_types/image_mask.py +7 -1
  87. wandb/sdk/data_types/html.py +4 -4
  88. wandb/sdk/data_types/image.py +178 -103
  89. wandb/sdk/data_types/molecule.py +6 -6
  90. wandb/sdk/data_types/object_3d.py +10 -5
  91. wandb/sdk/data_types/saved_model.py +11 -6
  92. wandb/sdk/data_types/table.py +313 -83
  93. wandb/sdk/data_types/table_decorators.py +108 -0
  94. wandb/sdk/data_types/utils.py +43 -7
  95. wandb/sdk/data_types/video.py +21 -3
  96. wandb/sdk/interface/interface.py +10 -0
  97. wandb/sdk/internal/datastore.py +2 -6
  98. wandb/sdk/internal/file_pusher.py +1 -5
  99. wandb/sdk/internal/file_stream.py +8 -17
  100. wandb/sdk/internal/handler.py +2 -2
  101. wandb/sdk/internal/incremental_table_util.py +53 -0
  102. wandb/sdk/internal/internal.py +3 -5
  103. wandb/sdk/internal/internal_api.py +66 -89
  104. wandb/sdk/internal/job_builder.py +2 -7
  105. wandb/sdk/internal/profiler.py +2 -2
  106. wandb/sdk/internal/progress.py +1 -3
  107. wandb/sdk/internal/run.py +1 -6
  108. wandb/sdk/internal/sender.py +24 -36
  109. wandb/sdk/internal/system/assets/aggregators.py +1 -7
  110. wandb/sdk/internal/system/assets/disk.py +3 -3
  111. wandb/sdk/internal/system/assets/gpu.py +4 -4
  112. wandb/sdk/internal/system/assets/gpu_amd.py +4 -4
  113. wandb/sdk/internal/system/assets/interfaces.py +6 -6
  114. wandb/sdk/internal/system/assets/tpu.py +1 -1
  115. wandb/sdk/internal/system/assets/trainium.py +6 -6
  116. wandb/sdk/internal/system/system_info.py +5 -7
  117. wandb/sdk/internal/system/system_monitor.py +4 -4
  118. wandb/sdk/internal/tb_watcher.py +5 -7
  119. wandb/sdk/launch/_launch.py +1 -1
  120. wandb/sdk/launch/_project_spec.py +19 -20
  121. wandb/sdk/launch/agent/agent.py +3 -3
  122. wandb/sdk/launch/agent/config.py +1 -1
  123. wandb/sdk/launch/agent/job_status_tracker.py +2 -2
  124. wandb/sdk/launch/builder/build.py +2 -3
  125. wandb/sdk/launch/builder/kaniko_builder.py +5 -4
  126. wandb/sdk/launch/environment/gcp_environment.py +1 -2
  127. wandb/sdk/launch/registry/azure_container_registry.py +2 -2
  128. wandb/sdk/launch/registry/elastic_container_registry.py +2 -2
  129. wandb/sdk/launch/registry/google_artifact_registry.py +3 -3
  130. wandb/sdk/launch/runner/abstract.py +5 -5
  131. wandb/sdk/launch/runner/kubernetes_monitor.py +2 -2
  132. wandb/sdk/launch/runner/kubernetes_runner.py +1 -1
  133. wandb/sdk/launch/runner/sagemaker_runner.py +2 -4
  134. wandb/sdk/launch/runner/vertex_runner.py +2 -7
  135. wandb/sdk/launch/sweeps/__init__.py +1 -1
  136. wandb/sdk/launch/sweeps/scheduler.py +2 -2
  137. wandb/sdk/launch/sweeps/utils.py +3 -3
  138. wandb/sdk/launch/utils.py +3 -4
  139. wandb/sdk/lib/apikey.py +5 -8
  140. wandb/sdk/lib/config_util.py +3 -3
  141. wandb/sdk/lib/fsm.py +3 -18
  142. wandb/sdk/lib/gitlib.py +6 -5
  143. wandb/sdk/lib/ipython.py +2 -2
  144. wandb/sdk/lib/json_util.py +9 -14
  145. wandb/sdk/lib/printer.py +3 -8
  146. wandb/sdk/lib/redirect.py +1 -1
  147. wandb/sdk/lib/retry.py +3 -7
  148. wandb/sdk/lib/run_moment.py +2 -2
  149. wandb/sdk/lib/service_connection.py +3 -1
  150. wandb/sdk/lib/service_token.py +1 -2
  151. wandb/sdk/mailbox/mailbox_handle.py +3 -7
  152. wandb/sdk/mailbox/response_handle.py +2 -6
  153. wandb/sdk/service/streams.py +3 -7
  154. wandb/sdk/verify/verify.py +5 -6
  155. wandb/sdk/wandb_config.py +1 -1
  156. wandb/sdk/wandb_init.py +38 -106
  157. wandb/sdk/wandb_login.py +7 -6
  158. wandb/sdk/wandb_run.py +52 -240
  159. wandb/sdk/wandb_settings.py +71 -60
  160. wandb/sdk/wandb_setup.py +40 -14
  161. wandb/sdk/wandb_watch.py +5 -7
  162. wandb/sync/__init__.py +1 -1
  163. wandb/sync/sync.py +13 -13
  164. wandb/util.py +17 -35
  165. wandb/wandb_agent.py +8 -11
  166. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/METADATA +5 -5
  167. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/RECORD +170 -168
  168. wandb/docker/auth.py +0 -435
  169. wandb/docker/www_authenticate.py +0 -94
  170. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/WHEEL +0 -0
  171. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/entry_points.txt +0 -0
  172. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/licenses/LICENSE +0 -0
@@ -19,16 +19,18 @@ from copy import copy
19
19
  from dataclasses import dataclass
20
20
  from datetime import datetime, timedelta
21
21
  from functools import partial
22
+ from itertools import filterfalse
22
23
  from pathlib import PurePosixPath
23
- from typing import IO, Any, Dict, Iterator, Literal, Sequence, Type, cast, final
24
+ from typing import IO, TYPE_CHECKING, Any, Iterator, Literal, Sequence, Type, final
24
25
  from urllib.parse import quote, urljoin, urlparse
25
26
 
26
27
  import requests
27
28
 
28
29
  import wandb
29
- from wandb import data_types, env, util
30
+ from wandb import data_types, env
31
+ from wandb._iterutils import one
30
32
  from wandb.apis.normalize import normalize_exceptions
31
- from wandb.apis.public import ArtifactCollection, ArtifactFiles, RetryingClient, Run
33
+ from wandb.apis.public import ArtifactCollection, ArtifactFiles, Run
32
34
  from wandb.apis.public.utils import gql_compat
33
35
  from wandb.data_types import WBValue
34
36
  from wandb.errors import CommError
@@ -36,37 +38,6 @@ from wandb.errors.term import termerror, termlog, termwarn
36
38
  from wandb.proto import wandb_internal_pb2 as pb
37
39
  from wandb.proto.wandb_deprecated import Deprecated
38
40
  from wandb.sdk import wandb_setup
39
- from wandb.sdk.artifacts._generated.fetch_linked_artifacts import FetchLinkedArtifacts
40
- from wandb.sdk.artifacts._generated.operations import FETCH_LINKED_ARTIFACTS_GQL
41
- from wandb.sdk.artifacts._graphql_fragments import (
42
- _gql_artifact_fragment,
43
- omit_artifact_fields,
44
- )
45
- from wandb.sdk.artifacts._validators import (
46
- LINKED_ARTIFACT_COLLECTION_TYPE,
47
- _LinkArtifactFields,
48
- ensure_logged,
49
- ensure_not_finalized,
50
- is_artifact_registry_project,
51
- validate_aliases,
52
- validate_artifact_name,
53
- validate_tags,
54
- )
55
- from wandb.sdk.artifacts.artifact_download_logger import ArtifactDownloadLogger
56
- from wandb.sdk.artifacts.artifact_instance_cache import artifact_instance_cache
57
- from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
58
- from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
59
- from wandb.sdk.artifacts.artifact_manifests.artifact_manifest_v1 import (
60
- ArtifactManifestV1,
61
- )
62
- from wandb.sdk.artifacts.artifact_state import ArtifactState
63
- from wandb.sdk.artifacts.artifact_ttl import ArtifactTTL
64
- from wandb.sdk.artifacts.exceptions import ArtifactNotLoggedError, WaitTimeoutError
65
- from wandb.sdk.artifacts.staging import get_staging_dir
66
- from wandb.sdk.artifacts.storage_handlers.gcs_handler import _GCSIsADirectoryError
67
- from wandb.sdk.artifacts.storage_layout import StorageLayout
68
- from wandb.sdk.artifacts.storage_policies import WANDB_STORAGE_POLICY
69
- from wandb.sdk.artifacts.storage_policy import StoragePolicy
70
41
  from wandb.sdk.data_types._dtypes import Type as WBType
71
42
  from wandb.sdk.data_types._dtypes import TypeRegistry
72
43
  from wandb.sdk.internal.internal_api import Api as InternalApi
@@ -77,23 +48,68 @@ from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, md5_file_b64
77
48
  from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
78
49
  from wandb.sdk.lib.runid import generate_id
79
50
  from wandb.sdk.mailbox import MailboxHandle
51
+ from wandb.util import (
52
+ alias_is_version_index,
53
+ artifact_to_json,
54
+ fsync_open,
55
+ json_dumps_safer,
56
+ uri_from_path,
57
+ vendor_setup,
58
+ )
80
59
 
81
60
  from ._generated import (
82
61
  ADD_ALIASES_GQL,
83
62
  DELETE_ALIASES_GQL,
63
+ FETCH_LINKED_ARTIFACTS_GQL,
84
64
  UPDATE_ARTIFACT_GQL,
85
65
  ArtifactAliasInput,
86
66
  ArtifactCollectionAliasInput,
67
+ FetchLinkedArtifacts,
87
68
  TagInput,
88
69
  UpdateArtifact,
89
70
  )
71
+ from ._graphql_fragments import _gql_artifact_fragment, omit_artifact_fields
72
+ from ._validators import (
73
+ LINKED_ARTIFACT_COLLECTION_TYPE,
74
+ _LinkArtifactFields,
75
+ ensure_logged,
76
+ ensure_not_finalized,
77
+ is_artifact_registry_project,
78
+ validate_aliases,
79
+ validate_artifact_name,
80
+ validate_artifact_type,
81
+ validate_metadata,
82
+ validate_tags,
83
+ validate_ttl_duration_seconds,
84
+ )
85
+ from .artifact_download_logger import ArtifactDownloadLogger
86
+ from .artifact_instance_cache import artifact_instance_cache
87
+ from .artifact_manifest import ArtifactManifest
88
+ from .artifact_manifest_entry import ArtifactManifestEntry
89
+ from .artifact_manifests.artifact_manifest_v1 import ArtifactManifestV1
90
+ from .artifact_state import ArtifactState
91
+ from .artifact_ttl import ArtifactTTL
92
+ from .exceptions import (
93
+ ArtifactNotLoggedError,
94
+ TooFewItemsError,
95
+ TooManyItemsError,
96
+ WaitTimeoutError,
97
+ )
98
+ from .staging import get_staging_dir
99
+ from .storage_handlers.gcs_handler import _GCSIsADirectoryError
100
+ from .storage_layout import StorageLayout
101
+ from .storage_policies import WANDB_STORAGE_POLICY
102
+ from .storage_policy import StoragePolicy
90
103
 
91
- reset_path = util.vendor_setup()
104
+ reset_path = vendor_setup()
92
105
 
93
106
  from wandb_gql import gql # noqa: E402
94
107
 
95
108
  reset_path()
96
109
 
110
+ if TYPE_CHECKING:
111
+ from wandb.apis.public import RetryingClient
112
+
97
113
  logger = logging.getLogger(__name__)
98
114
 
99
115
 
@@ -121,8 +137,8 @@ class Artifact:
121
137
  type: The artifact's type. Use the type of an artifact to both organize
122
138
  and differentiate artifacts. You can use any string that contains letters,
123
139
  numbers, underscores, hyphens, and dots. Common types include `dataset` or `model`.
124
- Include `model` within your type string if you want to link the artifact
125
- to the W&B Model Registry.
140
+ Note: Some types are reserved for internal use and cannot be set by users.
141
+ Such types include `job` and types that start with `wandb-`.
126
142
  description: A description of the artifact. For Model or Dataset Artifacts,
127
143
  add documentation for your standardized team model or dataset card. View
128
144
  an artifact's description programmatically with the `Artifact.description`
@@ -132,7 +148,7 @@ class Artifact:
132
148
  dictionary of key-value pairs. You can specify no more than 100 total keys.
133
149
  incremental: Use `Artifact.new_draft()` method instead to modify an
134
150
  existing artifact.
135
- use_as: W&B Launch specific parameter. Not recommended for general use.
151
+ use_as: Deprecated.
136
152
  is_link: Boolean indication of if the artifact is a linked artifact(`True`) or source artifact(`False`).
137
153
 
138
154
  Returns:
@@ -156,12 +172,10 @@ class Artifact:
156
172
  f"Artifact name may only contain alphanumeric characters, dashes, "
157
173
  f"underscores, and dots. Invalid name: {name}"
158
174
  )
159
- if type == "job" or type.startswith("wandb-"):
160
- raise ValueError(
161
- "Artifact types 'job' and 'wandb-*' are reserved for internal use. "
162
- "Please use a different type."
163
- )
164
- if incremental:
175
+
176
+ from wandb.sdk.artifacts._internal_artifact import InternalArtifact
177
+
178
+ if incremental and not isinstance(self, InternalArtifact):
165
179
  termwarn("Using experimental arg `incremental`")
166
180
 
167
181
  # Internal.
@@ -193,9 +207,9 @@ class Artifact:
193
207
  self._source_version: str | None = None
194
208
  self._source_artifact: Artifact | None = None
195
209
  self._is_link: bool = False
196
- self._type: str = type
210
+ self._type: str = validate_artifact_type(type, name)
197
211
  self._description: str | None = description
198
- self._metadata: dict = self._normalize_metadata(metadata)
212
+ self._metadata: dict[str, Any] = validate_metadata(metadata)
199
213
  self._ttl_duration_seconds: int | None = None
200
214
  self._ttl_is_inherited: bool = True
201
215
  self._ttl_changed: bool = False
@@ -205,7 +219,14 @@ class Artifact:
205
219
  self._saved_tags: list[str] = []
206
220
  self._distributed_id: str | None = None
207
221
  self._incremental: bool = incremental
208
- self._use_as: str | None = use_as
222
+ if use_as is not None:
223
+ deprecate(
224
+ field_name=Deprecated.artifact__init_use_as,
225
+ warning_message=(
226
+ "`use_as` argument is deprecated and does not affect the behaviour of `wandb.Artifact()`"
227
+ ),
228
+ )
229
+ self._use_as: str | None = None
209
230
  self._state: ArtifactState = ArtifactState.PENDING
210
231
  self._manifest: ArtifactManifest | _DeferredArtifactManifest | None = (
211
232
  ArtifactManifestV1(self._storage_policy)
@@ -226,8 +247,7 @@ class Artifact:
226
247
 
227
248
  @classmethod
228
249
  def _from_id(cls, artifact_id: str, client: RetryingClient) -> Artifact | None:
229
- artifact = artifact_instance_cache.get(artifact_id)
230
- if artifact is not None:
250
+ if (artifact := artifact_instance_cache.get(artifact_id)) is not None:
231
251
  return artifact
232
252
 
233
253
  query = gql(
@@ -398,23 +418,18 @@ class Artifact:
398
418
  and obj["artifactCollection"]["name"] == collection
399
419
  ]
400
420
 
401
- version_aliases = [
402
- alias for alias in processed_aliases if util.alias_is_version_index(alias)
403
- ]
404
- other_aliases = [
405
- alias
406
- for alias in processed_aliases
407
- if not util.alias_is_version_index(alias)
408
- ]
409
- if version_aliases:
410
- try:
411
- [version] = version_aliases
412
- except ValueError:
413
- raise ValueError(
414
- f"Expected at most one version alias, got {len(version_aliases)}: {version_aliases!r}"
415
- )
416
- else:
417
- version = src_version
421
+ version_aliases = list(filter(alias_is_version_index, processed_aliases))
422
+ other_aliases = list(filterfalse(alias_is_version_index, processed_aliases))
423
+
424
+ try:
425
+ version = one(
426
+ version_aliases, too_short=TooFewItemsError, too_long=TooManyItemsError
427
+ )
428
+ except TooFewItemsError:
429
+ version = src_version # default to the source version
430
+ except TooManyItemsError:
431
+ msg = f"Expected at most one version alias, got {len(version_aliases)}: {version_aliases!r}"
432
+ raise ValueError(msg) from None
418
433
 
419
434
  self._version = version
420
435
 
@@ -429,11 +444,11 @@ class Artifact:
429
444
  self._saved_tags = copy(tags)
430
445
 
431
446
  metadata_str = attrs["metadata"]
432
- self._metadata = self._normalize_metadata(
447
+ self._metadata = validate_metadata(
433
448
  json.loads(metadata_str) if metadata_str else {}
434
449
  )
435
450
 
436
- self._ttl_duration_seconds = _ttl_duration_seconds_from_gql(
451
+ self._ttl_duration_seconds = validate_ttl_duration_seconds(
437
452
  attrs.get("ttlDurationSeconds")
438
453
  )
439
454
  self._ttl_is_inherited = (
@@ -643,9 +658,10 @@ class Artifact:
643
658
  if not self.is_link:
644
659
  return self
645
660
  if self._source_artifact is None:
661
+ if self._client is None:
662
+ raise ValueError("Client is not initialized")
663
+
646
664
  try:
647
- if self._client is None:
648
- raise ValueError("Client is not initialized")
649
665
  artifact = self._from_name(
650
666
  entity=self.source_entity,
651
667
  project=self.source_project,
@@ -798,7 +814,7 @@ class Artifact:
798
814
  wandb.termwarn(
799
815
  "Editing the metadata of this linked artifact will edit the metadata for the source artifact and it's linked artifacts as well."
800
816
  )
801
- self._metadata = self._normalize_metadata(metadata)
817
+ self._metadata = validate_metadata(metadata)
802
818
 
803
819
  @property
804
820
  def ttl(self) -> timedelta | None:
@@ -912,6 +928,11 @@ class Artifact:
912
928
 
913
929
  @property
914
930
  def use_as(self) -> str | None:
931
+ """Deprecated."""
932
+ deprecate(
933
+ field_name=Deprecated.artifact__use_as,
934
+ warning_message=("The use_as property of Artifact is deprecated."),
935
+ )
915
936
  return self._use_as
916
937
 
917
938
  @property
@@ -1081,9 +1102,7 @@ class Artifact:
1081
1102
  with telemetry.context() as tel:
1082
1103
  tel.feature.artifact_incremental = True
1083
1104
 
1084
- singleton = wandb_setup._setup(start_service=False)
1085
-
1086
- if run := singleton.most_recent_active_run:
1105
+ if run := wandb_setup.singleton().most_recent_active_run:
1087
1106
  # TODO: Deprecate and encourage explicit log_artifact().
1088
1107
  run.log_artifact(self)
1089
1108
  else:
@@ -1276,7 +1295,7 @@ class Artifact:
1276
1295
  gql_vars = {
1277
1296
  "artifactID": self.id,
1278
1297
  "description": self.description,
1279
- "metadata": util.json_dumps_safer(self.metadata),
1298
+ "metadata": json_dumps_safer(self.metadata),
1280
1299
  "ttlDurationSeconds": self._ttl_duration_seconds_to_gql(),
1281
1300
  "aliases": aliases,
1282
1301
  "tagsToAdd": [TagInput(tag_name=t).model_dump() for t in tags_to_add],
@@ -1352,7 +1371,7 @@ class Artifact:
1352
1371
 
1353
1372
  filesystem.mkdir_exists_ok(os.path.dirname(path))
1354
1373
  try:
1355
- with util.fsync_open(path, mode, encoding) as f:
1374
+ with fsync_open(path, mode, encoding) as f:
1356
1375
  yield f
1357
1376
  except FileExistsError:
1358
1377
  raise ValueError(f"File with name {name!r} already exists at {path!r}")
@@ -1361,7 +1380,7 @@ class Artifact:
1361
1380
  f"Failed to open the provided file ({type(e).__name__}: {e}). Please "
1362
1381
  f"provide the proper encoding."
1363
1382
  )
1364
- raise e
1383
+ raise
1365
1384
 
1366
1385
  self.add_file(
1367
1386
  path, name=name, policy="immutable", skip_cache=True, overwrite=overwrite
@@ -1427,6 +1446,7 @@ class Artifact:
1427
1446
  name: str | None = None,
1428
1447
  skip_cache: bool | None = False,
1429
1448
  policy: Literal["mutable", "immutable"] | None = "mutable",
1449
+ merge: bool = False,
1430
1450
  ) -> None:
1431
1451
  """Add a local directory to the artifact.
1432
1452
 
@@ -1439,6 +1459,10 @@ class Artifact:
1439
1459
  policy: "mutable" | "immutable". By default, "mutable"
1440
1460
  "mutable": Create a temporary copy of the file to prevent corruption during upload.
1441
1461
  "immutable": Disable protection, rely on the user not to delete or change the file.
1462
+ merge: If `False` (default), throws ValueError if a file was already added in a previous add_dir call
1463
+ and its content has changed. If `True`, overwrites existing files with changed content.
1464
+ Always adds new files and never removes files. To replace an entire directory, pass a name when adding the directory
1465
+ using `add_dir(local_path, name=my_prefix)` and call `remove(my_prefix)` to remove the directory, then add it again.
1442
1466
 
1443
1467
  Raises:
1444
1468
  ArtifactFinalizedError: You cannot make changes to the current artifact
@@ -1446,7 +1470,7 @@ class Artifact:
1446
1470
  ValueError: Policy must be "mutable" or "immutable"
1447
1471
  """
1448
1472
  if not os.path.isdir(local_path):
1449
- raise ValueError("Path is not a directory: {}".format(local_path))
1473
+ raise ValueError(f"Path is not a directory: {local_path}")
1450
1474
 
1451
1475
  termlog(
1452
1476
  "Adding directory to artifact ({})... ".format(
@@ -1472,6 +1496,7 @@ class Artifact:
1472
1496
  path=physical_path,
1473
1497
  skip_cache=skip_cache,
1474
1498
  policy=policy,
1499
+ overwrite=merge,
1475
1500
  )
1476
1501
 
1477
1502
  num_threads = 8
@@ -1612,8 +1637,9 @@ class Artifact:
1612
1637
  data_types._SavedModel,
1613
1638
  )
1614
1639
  if not isinstance(obj, allowed_types):
1615
- raise ValueError(
1616
- f"Found object of type {obj.__class__}, expected one of: {allowed_types}"
1640
+ raise TypeError(
1641
+ f"Found object of type {obj.__class__}, expected one of:"
1642
+ f" {allowed_types}"
1617
1643
  )
1618
1644
 
1619
1645
  obj_id = id(obj)
@@ -1747,7 +1773,7 @@ class Artifact:
1747
1773
  name = LogicalPath(name)
1748
1774
  entry = self.manifest.entries.get(name) or self._get_obj_entry(name)[0]
1749
1775
  if entry is None:
1750
- raise KeyError("Path not contained in artifact: {}".format(name))
1776
+ raise KeyError(f"Path not contained in artifact: {name}")
1751
1777
  entry._parent_artifact = self
1752
1778
  return entry
1753
1779
 
@@ -1775,7 +1801,7 @@ class Artifact:
1775
1801
  assert self._client is not None
1776
1802
  artifact = self._from_id(referenced_id, client=self._client)
1777
1803
  assert artifact is not None
1778
- return artifact.get(util.uri_from_path(entry.ref))
1804
+ return artifact.get(uri_from_path(entry.ref))
1779
1805
 
1780
1806
  # Special case for wandb.Table. This is intended to be a short term
1781
1807
  # optimization. Since tables are likely to download many other assets in
@@ -1901,7 +1927,7 @@ class Artifact:
1901
1927
 
1902
1928
  # TODO: Create a special stream instead of relying on an existing run.
1903
1929
  if wandb.run is None:
1904
- wl = wandb.setup()
1930
+ wl = wandb_setup.singleton()
1905
1931
 
1906
1932
  stream_id = generate_id()
1907
1933
 
@@ -1963,9 +1989,7 @@ class Artifact:
1963
1989
  if nfiles > 5000 or size > 50 * 1024 * 1024:
1964
1990
  log = True
1965
1991
  termlog(
1966
- "Downloading large artifact {}, {:.2f}MB. {} files... ".format(
1967
- self.name, size / (1024 * 1024), nfiles
1968
- ),
1992
+ f"Downloading large artifact {self.name}, {size / (1024 * 1024):.2f}MB. {nfiles} files... ",
1969
1993
  )
1970
1994
  start_time = datetime.now()
1971
1995
  download_logger = ArtifactDownloadLogger(nfiles=nfiles)
@@ -2053,8 +2077,8 @@ class Artifact:
2053
2077
  retryable_exceptions=(requests.RequestException),
2054
2078
  )
2055
2079
  def _fetch_file_urls(self, cursor: str | None, per_page: int | None = 5000) -> Any:
2056
- if InternalApi()._check_server_feature_with_fallback(
2057
- pb.ServerFeature.ARTIFACT_COLLECTION_MEMBERSHIP_FILES # type: ignore
2080
+ if InternalApi()._server_supports(
2081
+ pb.ServerFeature.ARTIFACT_COLLECTION_MEMBERSHIP_FILES
2058
2082
  ):
2059
2083
  query = gql(
2060
2084
  """
@@ -2165,7 +2189,7 @@ class Artifact:
2165
2189
 
2166
2190
  Args:
2167
2191
  root: The directory to verify. If None artifact will be downloaded to
2168
- './artifacts/self.name/'
2192
+ './artifacts/self.name/'.
2169
2193
 
2170
2194
  Raises:
2171
2195
  ArtifactNotLoggedError: If the artifact is not logged.
@@ -2181,16 +2205,14 @@ class Artifact:
2181
2205
  self.get_entry(artifact_path)
2182
2206
  except KeyError:
2183
2207
  raise ValueError(
2184
- "Found file {} which is not a member of artifact {}".format(
2185
- full_path, self.name
2186
- )
2208
+ f"Found file {full_path} which is not a member of artifact {self.name}"
2187
2209
  )
2188
2210
 
2189
2211
  ref_count = 0
2190
2212
  for entry in self.manifest.entries.values():
2191
2213
  if entry.ref is None:
2192
2214
  if md5_file_b64(os.path.join(root, entry.path)) != entry.digest:
2193
- raise ValueError("Digest mismatch for file: {}".format(entry.path))
2215
+ raise ValueError(f"Digest mismatch for file: {entry.path}")
2194
2216
  else:
2195
2217
  ref_count += 1
2196
2218
  if ref_count > 0:
@@ -2344,9 +2366,7 @@ class Artifact:
2344
2366
  "Linking to a link artifact will result in directly linking to the source artifact of that link artifact."
2345
2367
  )
2346
2368
 
2347
- singleton = wandb_setup._setup(start_service=False)
2348
-
2349
- if run := singleton.most_recent_active_run:
2369
+ if run := wandb_setup.singleton().most_recent_active_run:
2350
2370
  # TODO: Deprecate and encourage explicit link_artifact().
2351
2371
  return run.link_artifact(self, target_path, aliases)
2352
2372
 
@@ -2504,7 +2524,7 @@ class Artifact:
2504
2524
  Returns:
2505
2525
  A `dict` with `string` keys representing attributes of the artifact.
2506
2526
  """
2507
- return util.artifact_to_json(self)
2527
+ return artifact_to_json(self)
2508
2528
 
2509
2529
  @staticmethod
2510
2530
  def _expected_type(
@@ -2543,16 +2563,6 @@ class Artifact:
2543
2563
  or {}
2544
2564
  ).get("name")
2545
2565
 
2546
- @staticmethod
2547
- def _normalize_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
2548
- if metadata is None:
2549
- return {}
2550
- if not isinstance(metadata, dict):
2551
- raise TypeError(f"metadata must be dict, not {type(metadata)}")
2552
- return cast(
2553
- Dict[str, Any], json.loads(json.dumps(util.json_friendly_val(metadata)))
2554
- )
2555
-
2556
2566
  def _load_manifest(self, url: str) -> ArtifactManifest:
2557
2567
  with requests.get(url) as response:
2558
2568
  response.raise_for_status()
@@ -2648,14 +2658,6 @@ class Artifact:
2648
2658
  return linked_artifact
2649
2659
 
2650
2660
 
2651
- def _ttl_duration_seconds_from_gql(gql_ttl_duration_seconds: int | None) -> int | None:
2652
- # If gql_ttl_duration_seconds is not positive, its indicating that TTL is DISABLED(-2)
2653
- # gql_ttl_duration_seconds only returns None if the server is not compatible with setting Artifact TTLs
2654
- if gql_ttl_duration_seconds and gql_ttl_duration_seconds > 0:
2655
- return gql_ttl_duration_seconds
2656
- return None
2657
-
2658
-
2659
2661
  class _ArtifactVersionType(WBType):
2660
2662
  name = "artifactVersion"
2661
2663
  types = [Artifact]
@@ -1,5 +1,7 @@
1
1
  """Artifact download logger."""
2
2
 
3
+ from __future__ import annotations
4
+
3
5
  import multiprocessing.dummy
4
6
  import time
5
7
  from typing import Callable
@@ -273,7 +273,5 @@ class ArtifactSaver:
273
273
  if artifact_id is None:
274
274
  raise RuntimeError(f"Could not resolve client id {client_id}")
275
275
  entry.ref = URIStr(
276
- "wandb-artifact://{}/{}".format(
277
- b64_to_hex_id(B64MD5(artifact_id)), artifact_file_path
278
- )
276
+ f"wandb-artifact://{b64_to_hex_id(B64MD5(artifact_id))}/{artifact_file_path}"
279
277
  )
@@ -1,5 +1,7 @@
1
1
  """Artifact state."""
2
2
 
3
+ from __future__ import annotations
4
+
3
5
  from enum import Enum
4
6
 
5
7
 
@@ -1,5 +1,7 @@
1
1
  """Artifact TTL."""
2
2
 
3
+ from __future__ import annotations
4
+
3
5
  from enum import Enum
4
6
 
5
7
 
@@ -55,3 +55,17 @@ class ArtifactFinalizedError(ArtifactStatusError):
55
55
 
56
56
  class WaitTimeoutError(errors.Error):
57
57
  """Raised when wait() timeout occurs before process is finished."""
58
+
59
+
60
+ class TooFewItemsError(ValueError):
61
+ """Raised when there are fewer items than expected in a collection.
62
+
63
+ Intended for internal use only.
64
+ """
65
+
66
+
67
+ class TooManyItemsError(ValueError):
68
+ """Raised when there are more items than expected in a collection.
69
+
70
+ Intended for internal use only.
71
+ """
@@ -5,6 +5,8 @@ in order to avoid file changes corrupting the artifact. Once the upload is compl
5
5
  file should be moved to the artifact cache.
6
6
  """
7
7
 
8
+ from __future__ import annotations
9
+
8
10
  import os
9
11
 
10
12
  from wandb import env
@@ -46,9 +46,7 @@ class LocalFileHandler(StorageHandler):
46
46
  local_path = util.local_file_uri_to_path(str(manifest_entry.ref))
47
47
  if not os.path.exists(local_path):
48
48
  raise ValueError(
49
- "Local file reference: Failed to find file at path {}".format(
50
- local_path
51
- )
49
+ f"Local file reference: Failed to find file at path {local_path}"
52
50
  )
53
51
 
54
52
  path, hit, cache_open = self._cache.check_md5_obj_path(
@@ -140,7 +138,5 @@ class LocalFileHandler(StorageHandler):
140
138
  entries.append(entry)
141
139
  else:
142
140
  # TODO: update error message if we don't allow directories.
143
- raise ValueError(
144
- 'Path "{}" must be a valid file or directory path'.format(path)
145
- )
141
+ raise ValueError(f'Path "{path}" must be a valid file or directory path')
146
142
  return entries
@@ -31,7 +31,7 @@ class MultiHandler(StorageHandler):
31
31
  return handler
32
32
  if self._default_handler is not None:
33
33
  return self._default_handler
34
- raise ValueError('No storage handler registered for url "{}"'.format(str(url)))
34
+ raise ValueError(f'No storage handler registered for url "{url!s}"')
35
35
 
36
36
  def load_path(
37
37
  self,
@@ -59,14 +59,10 @@ class TrackingHandler(StorageHandler):
59
59
  url = urlparse(path)
60
60
  if name is None:
61
61
  raise ValueError(
62
- 'You must pass name="<entry_name>" when tracking references with unknown schemes. ref: {}'.format(
63
- path
64
- )
62
+ f'You must pass name="<entry_name>" when tracking references with unknown schemes. ref: {path}'
65
63
  )
66
64
  termwarn(
67
- "Artifact references with unsupported schemes cannot be checksummed: {}".format(
68
- path
69
- )
65
+ f"Artifact references with unsupported schemes cannot be checksummed: {path}"
70
66
  )
71
67
  name = name or url.path[1:] # strip leading slash
72
68
  return [ArtifactManifestEntry(path=name, ref=path, digest=path)]
@@ -116,11 +116,7 @@ class WBArtifactHandler(StorageHandler):
116
116
  assert target_artifact is not None
117
117
  assert target_artifact.id is not None
118
118
  path = URIStr(
119
- "{}://{}/{}".format(
120
- self._scheme,
121
- b64_to_hex_id(B64MD5(target_artifact.id)),
122
- artifact_file_path,
123
- )
119
+ f"{self._scheme}://{b64_to_hex_id(B64MD5(target_artifact.id))}/{artifact_file_path}"
124
120
  )
125
121
 
126
122
  # Return the new entry
@@ -58,7 +58,7 @@ class WBLocalArtifactHandler(StorageHandler):
58
58
  target_path = util.uri_from_path(path)
59
59
  target_artifact = artifact_instance_cache.get(client_id)
60
60
  if not isinstance(target_artifact, wandb.Artifact):
61
- raise RuntimeError("Local Artifact not found - invalid reference")
61
+ raise TypeError("Artifact passed to store_path() must be a wandb.Artifact.")
62
62
  target_entry = target_artifact.manifest.entries[target_path] # type: ignore
63
63
  if target_entry is None:
64
64
  raise RuntimeError("Local entry not found - invalid reference")
@@ -1,5 +1,7 @@
1
1
  """Storage layout."""
2
2
 
3
+ from __future__ import annotations
4
+
3
5
 
4
6
  class StorageLayout:
5
7
  V1 = "V1"
@@ -254,7 +254,7 @@ class WandbStoragePolicy(StoragePolicy):
254
254
  if env.is_debug():
255
255
  logger.debug(f"Error writing chunk to file: {e}")
256
256
  download_has_error.set()
257
- raise e
257
+ raise
258
258
  else:
259
259
  raise ValueError(f"Unknown queue item type: {type(item)}")
260
260
 
@@ -340,7 +340,7 @@ class WandbStoragePolicy(StoragePolicy):
340
340
  if env.is_debug():
341
341
  logger.debug(f"Error downloading file: {e}")
342
342
  download_has_error.set()
343
- raise e
343
+ raise
344
344
  finally:
345
345
  # Always signal the writer to stop
346
346
  q.put(_CHUNK_QUEUE_SENTINEL)
@@ -387,7 +387,7 @@ class WandbStoragePolicy(StoragePolicy):
387
387
  api.settings("base_url"), entity_name, md5_hex
388
388
  )
389
389
  elif storage_layout == StorageLayout.V2:
390
- if api._check_server_feature_with_fallback(
390
+ if api._server_supports(
391
391
  ServerFeature.ARTIFACT_COLLECTION_MEMBERSHIP_FILE_DOWNLOAD_HANDLER # type: ignore
392
392
  ):
393
393
  return "{}/artifactsV2/{}/{}/{}/{}/{}/{}/{}".format(