wandb 0.19.12rc1__py3-none-win32.whl → 0.20.1__py3-none-win32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +1 -2
- wandb/__init__.pyi +3 -6
- wandb/_iterutils.py +26 -7
- wandb/_pydantic/__init__.py +2 -1
- wandb/_pydantic/utils.py +7 -0
- wandb/agents/pyagent.py +9 -15
- wandb/analytics/sentry.py +1 -2
- wandb/apis/attrs.py +3 -4
- wandb/apis/importers/internals/util.py +1 -1
- wandb/apis/importers/validation.py +2 -2
- wandb/apis/importers/wandb.py +30 -25
- wandb/apis/normalize.py +2 -2
- wandb/apis/public/__init__.py +1 -0
- wandb/apis/public/api.py +37 -33
- wandb/apis/public/artifacts.py +103 -72
- wandb/apis/public/jobs.py +3 -2
- wandb/apis/public/registries/registries_search.py +4 -2
- wandb/apis/public/registries/registry.py +1 -1
- wandb/apis/public/registries/utils.py +9 -9
- wandb/apis/public/runs.py +18 -6
- wandb/automations/_filters/expressions.py +1 -1
- wandb/automations/_filters/operators.py +1 -1
- wandb/automations/_filters/run_metrics.py +1 -1
- wandb/beta/workflows.py +6 -5
- wandb/bin/gpu_stats.exe +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/cli.py +54 -73
- wandb/docker/__init__.py +21 -74
- wandb/docker/names.py +40 -0
- wandb/env.py +0 -1
- wandb/errors/util.py +1 -1
- wandb/filesync/step_checksum.py +1 -1
- wandb/filesync/step_upload.py +1 -1
- wandb/integration/diffusers/resolvers/multimodal.py +1 -2
- wandb/integration/gym/__init__.py +5 -6
- wandb/integration/keras/callbacks/model_checkpoint.py +2 -2
- wandb/integration/keras/keras.py +13 -19
- wandb/integration/kfp/kfp_patch.py +2 -3
- wandb/integration/langchain/wandb_tracer.py +1 -1
- wandb/integration/metaflow/metaflow.py +13 -13
- wandb/integration/openai/fine_tuning.py +3 -2
- wandb/integration/sagemaker/auth.py +2 -1
- wandb/integration/sklearn/utils.py +2 -1
- wandb/integration/tensorboard/__init__.py +1 -1
- wandb/integration/tensorboard/log.py +2 -5
- wandb/integration/tensorflow/__init__.py +2 -2
- wandb/jupyter.py +20 -17
- wandb/plot/confusion_matrix.py +1 -1
- wandb/plot/utils.py +8 -7
- wandb/proto/v3/wandb_internal_pb2.py +355 -335
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v4/wandb_internal_pb2.py +339 -335
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v5/wandb_internal_pb2.py +339 -335
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v6/wandb_internal_pb2.py +339 -335
- wandb/proto/v6/wandb_settings_pb2.py +2 -2
- wandb/proto/v6/wandb_telemetry_pb2.py +12 -12
- wandb/proto/wandb_deprecated.py +6 -8
- wandb/sdk/artifacts/_internal_artifact.py +43 -0
- wandb/sdk/artifacts/_validators.py +55 -35
- wandb/sdk/artifacts/artifact.py +117 -115
- wandb/sdk/artifacts/artifact_download_logger.py +2 -0
- wandb/sdk/artifacts/artifact_saver.py +1 -3
- wandb/sdk/artifacts/artifact_state.py +2 -0
- wandb/sdk/artifacts/artifact_ttl.py +2 -0
- wandb/sdk/artifacts/exceptions.py +14 -0
- wandb/sdk/artifacts/staging.py +2 -0
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +2 -6
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +1 -1
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +2 -6
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +1 -5
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +1 -1
- wandb/sdk/artifacts/storage_layout.py +2 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +3 -3
- wandb/sdk/backend/backend.py +11 -182
- wandb/sdk/data_types/_dtypes.py +2 -6
- wandb/sdk/data_types/audio.py +20 -3
- wandb/sdk/data_types/base_types/media.py +12 -7
- wandb/sdk/data_types/base_types/wb_value.py +8 -18
- wandb/sdk/data_types/bokeh.py +19 -2
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +17 -1
- wandb/sdk/data_types/helper_types/image_mask.py +7 -1
- wandb/sdk/data_types/html.py +4 -4
- wandb/sdk/data_types/image.py +178 -103
- wandb/sdk/data_types/molecule.py +6 -6
- wandb/sdk/data_types/object_3d.py +10 -5
- wandb/sdk/data_types/saved_model.py +11 -6
- wandb/sdk/data_types/table.py +313 -83
- wandb/sdk/data_types/table_decorators.py +108 -0
- wandb/sdk/data_types/utils.py +43 -7
- wandb/sdk/data_types/video.py +21 -3
- wandb/sdk/interface/interface.py +10 -0
- wandb/sdk/internal/datastore.py +2 -6
- wandb/sdk/internal/file_pusher.py +1 -5
- wandb/sdk/internal/file_stream.py +8 -17
- wandb/sdk/internal/handler.py +2 -2
- wandb/sdk/internal/incremental_table_util.py +53 -0
- wandb/sdk/internal/internal.py +3 -5
- wandb/sdk/internal/internal_api.py +66 -89
- wandb/sdk/internal/job_builder.py +2 -7
- wandb/sdk/internal/profiler.py +2 -2
- wandb/sdk/internal/progress.py +1 -3
- wandb/sdk/internal/run.py +1 -6
- wandb/sdk/internal/sender.py +24 -36
- wandb/sdk/internal/system/assets/aggregators.py +1 -7
- wandb/sdk/internal/system/assets/disk.py +3 -3
- wandb/sdk/internal/system/assets/gpu.py +4 -4
- wandb/sdk/internal/system/assets/gpu_amd.py +4 -4
- wandb/sdk/internal/system/assets/interfaces.py +6 -6
- wandb/sdk/internal/system/assets/tpu.py +1 -1
- wandb/sdk/internal/system/assets/trainium.py +6 -6
- wandb/sdk/internal/system/system_info.py +5 -7
- wandb/sdk/internal/system/system_monitor.py +4 -4
- wandb/sdk/internal/tb_watcher.py +5 -7
- wandb/sdk/launch/_launch.py +1 -1
- wandb/sdk/launch/_project_spec.py +19 -20
- wandb/sdk/launch/agent/agent.py +3 -3
- wandb/sdk/launch/agent/config.py +1 -1
- wandb/sdk/launch/agent/job_status_tracker.py +2 -2
- wandb/sdk/launch/builder/build.py +2 -3
- wandb/sdk/launch/builder/kaniko_builder.py +5 -4
- wandb/sdk/launch/environment/gcp_environment.py +1 -2
- wandb/sdk/launch/registry/azure_container_registry.py +2 -2
- wandb/sdk/launch/registry/elastic_container_registry.py +2 -2
- wandb/sdk/launch/registry/google_artifact_registry.py +3 -3
- wandb/sdk/launch/runner/abstract.py +5 -5
- wandb/sdk/launch/runner/kubernetes_monitor.py +2 -2
- wandb/sdk/launch/runner/kubernetes_runner.py +1 -1
- wandb/sdk/launch/runner/sagemaker_runner.py +2 -4
- wandb/sdk/launch/runner/vertex_runner.py +2 -7
- wandb/sdk/launch/sweeps/__init__.py +1 -1
- wandb/sdk/launch/sweeps/scheduler.py +2 -2
- wandb/sdk/launch/sweeps/utils.py +3 -3
- wandb/sdk/launch/utils.py +3 -4
- wandb/sdk/lib/apikey.py +5 -8
- wandb/sdk/lib/config_util.py +3 -3
- wandb/sdk/lib/fsm.py +3 -18
- wandb/sdk/lib/gitlib.py +6 -5
- wandb/sdk/lib/ipython.py +2 -2
- wandb/sdk/lib/json_util.py +9 -14
- wandb/sdk/lib/printer.py +3 -8
- wandb/sdk/lib/redirect.py +1 -1
- wandb/sdk/lib/retry.py +3 -7
- wandb/sdk/lib/run_moment.py +2 -2
- wandb/sdk/lib/service_connection.py +3 -1
- wandb/sdk/lib/service_token.py +1 -2
- wandb/sdk/mailbox/mailbox_handle.py +3 -7
- wandb/sdk/mailbox/response_handle.py +2 -6
- wandb/sdk/service/streams.py +3 -7
- wandb/sdk/verify/verify.py +5 -6
- wandb/sdk/wandb_config.py +1 -1
- wandb/sdk/wandb_init.py +38 -106
- wandb/sdk/wandb_login.py +7 -6
- wandb/sdk/wandb_run.py +52 -240
- wandb/sdk/wandb_settings.py +71 -60
- wandb/sdk/wandb_setup.py +40 -14
- wandb/sdk/wandb_watch.py +5 -7
- wandb/sync/__init__.py +1 -1
- wandb/sync/sync.py +13 -13
- wandb/util.py +17 -35
- wandb/wandb_agent.py +8 -11
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/METADATA +5 -5
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/RECORD +170 -168
- wandb/docker/auth.py +0 -435
- wandb/docker/www_authenticate.py +0 -94
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/WHEEL +0 -0
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/entry_points.txt +0 -0
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/licenses/LICENSE +0 -0
wandb/sdk/artifacts/artifact.py
CHANGED
@@ -19,16 +19,18 @@ from copy import copy
|
|
19
19
|
from dataclasses import dataclass
|
20
20
|
from datetime import datetime, timedelta
|
21
21
|
from functools import partial
|
22
|
+
from itertools import filterfalse
|
22
23
|
from pathlib import PurePosixPath
|
23
|
-
from typing import IO,
|
24
|
+
from typing import IO, TYPE_CHECKING, Any, Iterator, Literal, Sequence, Type, final
|
24
25
|
from urllib.parse import quote, urljoin, urlparse
|
25
26
|
|
26
27
|
import requests
|
27
28
|
|
28
29
|
import wandb
|
29
|
-
from wandb import data_types, env
|
30
|
+
from wandb import data_types, env
|
31
|
+
from wandb._iterutils import one
|
30
32
|
from wandb.apis.normalize import normalize_exceptions
|
31
|
-
from wandb.apis.public import ArtifactCollection, ArtifactFiles,
|
33
|
+
from wandb.apis.public import ArtifactCollection, ArtifactFiles, Run
|
32
34
|
from wandb.apis.public.utils import gql_compat
|
33
35
|
from wandb.data_types import WBValue
|
34
36
|
from wandb.errors import CommError
|
@@ -36,37 +38,6 @@ from wandb.errors.term import termerror, termlog, termwarn
|
|
36
38
|
from wandb.proto import wandb_internal_pb2 as pb
|
37
39
|
from wandb.proto.wandb_deprecated import Deprecated
|
38
40
|
from wandb.sdk import wandb_setup
|
39
|
-
from wandb.sdk.artifacts._generated.fetch_linked_artifacts import FetchLinkedArtifacts
|
40
|
-
from wandb.sdk.artifacts._generated.operations import FETCH_LINKED_ARTIFACTS_GQL
|
41
|
-
from wandb.sdk.artifacts._graphql_fragments import (
|
42
|
-
_gql_artifact_fragment,
|
43
|
-
omit_artifact_fields,
|
44
|
-
)
|
45
|
-
from wandb.sdk.artifacts._validators import (
|
46
|
-
LINKED_ARTIFACT_COLLECTION_TYPE,
|
47
|
-
_LinkArtifactFields,
|
48
|
-
ensure_logged,
|
49
|
-
ensure_not_finalized,
|
50
|
-
is_artifact_registry_project,
|
51
|
-
validate_aliases,
|
52
|
-
validate_artifact_name,
|
53
|
-
validate_tags,
|
54
|
-
)
|
55
|
-
from wandb.sdk.artifacts.artifact_download_logger import ArtifactDownloadLogger
|
56
|
-
from wandb.sdk.artifacts.artifact_instance_cache import artifact_instance_cache
|
57
|
-
from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
|
58
|
-
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
59
|
-
from wandb.sdk.artifacts.artifact_manifests.artifact_manifest_v1 import (
|
60
|
-
ArtifactManifestV1,
|
61
|
-
)
|
62
|
-
from wandb.sdk.artifacts.artifact_state import ArtifactState
|
63
|
-
from wandb.sdk.artifacts.artifact_ttl import ArtifactTTL
|
64
|
-
from wandb.sdk.artifacts.exceptions import ArtifactNotLoggedError, WaitTimeoutError
|
65
|
-
from wandb.sdk.artifacts.staging import get_staging_dir
|
66
|
-
from wandb.sdk.artifacts.storage_handlers.gcs_handler import _GCSIsADirectoryError
|
67
|
-
from wandb.sdk.artifacts.storage_layout import StorageLayout
|
68
|
-
from wandb.sdk.artifacts.storage_policies import WANDB_STORAGE_POLICY
|
69
|
-
from wandb.sdk.artifacts.storage_policy import StoragePolicy
|
70
41
|
from wandb.sdk.data_types._dtypes import Type as WBType
|
71
42
|
from wandb.sdk.data_types._dtypes import TypeRegistry
|
72
43
|
from wandb.sdk.internal.internal_api import Api as InternalApi
|
@@ -77,23 +48,68 @@ from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, md5_file_b64
|
|
77
48
|
from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
|
78
49
|
from wandb.sdk.lib.runid import generate_id
|
79
50
|
from wandb.sdk.mailbox import MailboxHandle
|
51
|
+
from wandb.util import (
|
52
|
+
alias_is_version_index,
|
53
|
+
artifact_to_json,
|
54
|
+
fsync_open,
|
55
|
+
json_dumps_safer,
|
56
|
+
uri_from_path,
|
57
|
+
vendor_setup,
|
58
|
+
)
|
80
59
|
|
81
60
|
from ._generated import (
|
82
61
|
ADD_ALIASES_GQL,
|
83
62
|
DELETE_ALIASES_GQL,
|
63
|
+
FETCH_LINKED_ARTIFACTS_GQL,
|
84
64
|
UPDATE_ARTIFACT_GQL,
|
85
65
|
ArtifactAliasInput,
|
86
66
|
ArtifactCollectionAliasInput,
|
67
|
+
FetchLinkedArtifacts,
|
87
68
|
TagInput,
|
88
69
|
UpdateArtifact,
|
89
70
|
)
|
71
|
+
from ._graphql_fragments import _gql_artifact_fragment, omit_artifact_fields
|
72
|
+
from ._validators import (
|
73
|
+
LINKED_ARTIFACT_COLLECTION_TYPE,
|
74
|
+
_LinkArtifactFields,
|
75
|
+
ensure_logged,
|
76
|
+
ensure_not_finalized,
|
77
|
+
is_artifact_registry_project,
|
78
|
+
validate_aliases,
|
79
|
+
validate_artifact_name,
|
80
|
+
validate_artifact_type,
|
81
|
+
validate_metadata,
|
82
|
+
validate_tags,
|
83
|
+
validate_ttl_duration_seconds,
|
84
|
+
)
|
85
|
+
from .artifact_download_logger import ArtifactDownloadLogger
|
86
|
+
from .artifact_instance_cache import artifact_instance_cache
|
87
|
+
from .artifact_manifest import ArtifactManifest
|
88
|
+
from .artifact_manifest_entry import ArtifactManifestEntry
|
89
|
+
from .artifact_manifests.artifact_manifest_v1 import ArtifactManifestV1
|
90
|
+
from .artifact_state import ArtifactState
|
91
|
+
from .artifact_ttl import ArtifactTTL
|
92
|
+
from .exceptions import (
|
93
|
+
ArtifactNotLoggedError,
|
94
|
+
TooFewItemsError,
|
95
|
+
TooManyItemsError,
|
96
|
+
WaitTimeoutError,
|
97
|
+
)
|
98
|
+
from .staging import get_staging_dir
|
99
|
+
from .storage_handlers.gcs_handler import _GCSIsADirectoryError
|
100
|
+
from .storage_layout import StorageLayout
|
101
|
+
from .storage_policies import WANDB_STORAGE_POLICY
|
102
|
+
from .storage_policy import StoragePolicy
|
90
103
|
|
91
|
-
reset_path =
|
104
|
+
reset_path = vendor_setup()
|
92
105
|
|
93
106
|
from wandb_gql import gql # noqa: E402
|
94
107
|
|
95
108
|
reset_path()
|
96
109
|
|
110
|
+
if TYPE_CHECKING:
|
111
|
+
from wandb.apis.public import RetryingClient
|
112
|
+
|
97
113
|
logger = logging.getLogger(__name__)
|
98
114
|
|
99
115
|
|
@@ -121,8 +137,8 @@ class Artifact:
|
|
121
137
|
type: The artifact's type. Use the type of an artifact to both organize
|
122
138
|
and differentiate artifacts. You can use any string that contains letters,
|
123
139
|
numbers, underscores, hyphens, and dots. Common types include `dataset` or `model`.
|
124
|
-
|
125
|
-
|
140
|
+
Note: Some types are reserved for internal use and cannot be set by users.
|
141
|
+
Such types include `job` and types that start with `wandb-`.
|
126
142
|
description: A description of the artifact. For Model or Dataset Artifacts,
|
127
143
|
add documentation for your standardized team model or dataset card. View
|
128
144
|
an artifact's description programmatically with the `Artifact.description`
|
@@ -132,7 +148,7 @@ class Artifact:
|
|
132
148
|
dictionary of key-value pairs. You can specify no more than 100 total keys.
|
133
149
|
incremental: Use `Artifact.new_draft()` method instead to modify an
|
134
150
|
existing artifact.
|
135
|
-
use_as:
|
151
|
+
use_as: Deprecated.
|
136
152
|
is_link: Boolean indication of if the artifact is a linked artifact(`True`) or source artifact(`False`).
|
137
153
|
|
138
154
|
Returns:
|
@@ -156,12 +172,10 @@ class Artifact:
|
|
156
172
|
f"Artifact name may only contain alphanumeric characters, dashes, "
|
157
173
|
f"underscores, and dots. Invalid name: {name}"
|
158
174
|
)
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
)
|
164
|
-
if incremental:
|
175
|
+
|
176
|
+
from wandb.sdk.artifacts._internal_artifact import InternalArtifact
|
177
|
+
|
178
|
+
if incremental and not isinstance(self, InternalArtifact):
|
165
179
|
termwarn("Using experimental arg `incremental`")
|
166
180
|
|
167
181
|
# Internal.
|
@@ -193,9 +207,9 @@ class Artifact:
|
|
193
207
|
self._source_version: str | None = None
|
194
208
|
self._source_artifact: Artifact | None = None
|
195
209
|
self._is_link: bool = False
|
196
|
-
self._type: str = type
|
210
|
+
self._type: str = validate_artifact_type(type, name)
|
197
211
|
self._description: str | None = description
|
198
|
-
self._metadata: dict =
|
212
|
+
self._metadata: dict[str, Any] = validate_metadata(metadata)
|
199
213
|
self._ttl_duration_seconds: int | None = None
|
200
214
|
self._ttl_is_inherited: bool = True
|
201
215
|
self._ttl_changed: bool = False
|
@@ -205,7 +219,14 @@ class Artifact:
|
|
205
219
|
self._saved_tags: list[str] = []
|
206
220
|
self._distributed_id: str | None = None
|
207
221
|
self._incremental: bool = incremental
|
208
|
-
|
222
|
+
if use_as is not None:
|
223
|
+
deprecate(
|
224
|
+
field_name=Deprecated.artifact__init_use_as,
|
225
|
+
warning_message=(
|
226
|
+
"`use_as` argument is deprecated and does not affect the behaviour of `wandb.Artifact()`"
|
227
|
+
),
|
228
|
+
)
|
229
|
+
self._use_as: str | None = None
|
209
230
|
self._state: ArtifactState = ArtifactState.PENDING
|
210
231
|
self._manifest: ArtifactManifest | _DeferredArtifactManifest | None = (
|
211
232
|
ArtifactManifestV1(self._storage_policy)
|
@@ -226,8 +247,7 @@ class Artifact:
|
|
226
247
|
|
227
248
|
@classmethod
|
228
249
|
def _from_id(cls, artifact_id: str, client: RetryingClient) -> Artifact | None:
|
229
|
-
artifact
|
230
|
-
if artifact is not None:
|
250
|
+
if (artifact := artifact_instance_cache.get(artifact_id)) is not None:
|
231
251
|
return artifact
|
232
252
|
|
233
253
|
query = gql(
|
@@ -398,23 +418,18 @@ class Artifact:
|
|
398
418
|
and obj["artifactCollection"]["name"] == collection
|
399
419
|
]
|
400
420
|
|
401
|
-
version_aliases =
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
raise ValueError(
|
414
|
-
f"Expected at most one version alias, got {len(version_aliases)}: {version_aliases!r}"
|
415
|
-
)
|
416
|
-
else:
|
417
|
-
version = src_version
|
421
|
+
version_aliases = list(filter(alias_is_version_index, processed_aliases))
|
422
|
+
other_aliases = list(filterfalse(alias_is_version_index, processed_aliases))
|
423
|
+
|
424
|
+
try:
|
425
|
+
version = one(
|
426
|
+
version_aliases, too_short=TooFewItemsError, too_long=TooManyItemsError
|
427
|
+
)
|
428
|
+
except TooFewItemsError:
|
429
|
+
version = src_version # default to the source version
|
430
|
+
except TooManyItemsError:
|
431
|
+
msg = f"Expected at most one version alias, got {len(version_aliases)}: {version_aliases!r}"
|
432
|
+
raise ValueError(msg) from None
|
418
433
|
|
419
434
|
self._version = version
|
420
435
|
|
@@ -429,11 +444,11 @@ class Artifact:
|
|
429
444
|
self._saved_tags = copy(tags)
|
430
445
|
|
431
446
|
metadata_str = attrs["metadata"]
|
432
|
-
self._metadata =
|
447
|
+
self._metadata = validate_metadata(
|
433
448
|
json.loads(metadata_str) if metadata_str else {}
|
434
449
|
)
|
435
450
|
|
436
|
-
self._ttl_duration_seconds =
|
451
|
+
self._ttl_duration_seconds = validate_ttl_duration_seconds(
|
437
452
|
attrs.get("ttlDurationSeconds")
|
438
453
|
)
|
439
454
|
self._ttl_is_inherited = (
|
@@ -643,9 +658,10 @@ class Artifact:
|
|
643
658
|
if not self.is_link:
|
644
659
|
return self
|
645
660
|
if self._source_artifact is None:
|
661
|
+
if self._client is None:
|
662
|
+
raise ValueError("Client is not initialized")
|
663
|
+
|
646
664
|
try:
|
647
|
-
if self._client is None:
|
648
|
-
raise ValueError("Client is not initialized")
|
649
665
|
artifact = self._from_name(
|
650
666
|
entity=self.source_entity,
|
651
667
|
project=self.source_project,
|
@@ -798,7 +814,7 @@ class Artifact:
|
|
798
814
|
wandb.termwarn(
|
799
815
|
"Editing the metadata of this linked artifact will edit the metadata for the source artifact and it's linked artifacts as well."
|
800
816
|
)
|
801
|
-
self._metadata =
|
817
|
+
self._metadata = validate_metadata(metadata)
|
802
818
|
|
803
819
|
@property
|
804
820
|
def ttl(self) -> timedelta | None:
|
@@ -912,6 +928,11 @@ class Artifact:
|
|
912
928
|
|
913
929
|
@property
|
914
930
|
def use_as(self) -> str | None:
|
931
|
+
"""Deprecated."""
|
932
|
+
deprecate(
|
933
|
+
field_name=Deprecated.artifact__use_as,
|
934
|
+
warning_message=("The use_as property of Artifact is deprecated."),
|
935
|
+
)
|
915
936
|
return self._use_as
|
916
937
|
|
917
938
|
@property
|
@@ -1081,9 +1102,7 @@ class Artifact:
|
|
1081
1102
|
with telemetry.context() as tel:
|
1082
1103
|
tel.feature.artifact_incremental = True
|
1083
1104
|
|
1084
|
-
|
1085
|
-
|
1086
|
-
if run := singleton.most_recent_active_run:
|
1105
|
+
if run := wandb_setup.singleton().most_recent_active_run:
|
1087
1106
|
# TODO: Deprecate and encourage explicit log_artifact().
|
1088
1107
|
run.log_artifact(self)
|
1089
1108
|
else:
|
@@ -1276,7 +1295,7 @@ class Artifact:
|
|
1276
1295
|
gql_vars = {
|
1277
1296
|
"artifactID": self.id,
|
1278
1297
|
"description": self.description,
|
1279
|
-
"metadata":
|
1298
|
+
"metadata": json_dumps_safer(self.metadata),
|
1280
1299
|
"ttlDurationSeconds": self._ttl_duration_seconds_to_gql(),
|
1281
1300
|
"aliases": aliases,
|
1282
1301
|
"tagsToAdd": [TagInput(tag_name=t).model_dump() for t in tags_to_add],
|
@@ -1352,7 +1371,7 @@ class Artifact:
|
|
1352
1371
|
|
1353
1372
|
filesystem.mkdir_exists_ok(os.path.dirname(path))
|
1354
1373
|
try:
|
1355
|
-
with
|
1374
|
+
with fsync_open(path, mode, encoding) as f:
|
1356
1375
|
yield f
|
1357
1376
|
except FileExistsError:
|
1358
1377
|
raise ValueError(f"File with name {name!r} already exists at {path!r}")
|
@@ -1361,7 +1380,7 @@ class Artifact:
|
|
1361
1380
|
f"Failed to open the provided file ({type(e).__name__}: {e}). Please "
|
1362
1381
|
f"provide the proper encoding."
|
1363
1382
|
)
|
1364
|
-
raise
|
1383
|
+
raise
|
1365
1384
|
|
1366
1385
|
self.add_file(
|
1367
1386
|
path, name=name, policy="immutable", skip_cache=True, overwrite=overwrite
|
@@ -1427,6 +1446,7 @@ class Artifact:
|
|
1427
1446
|
name: str | None = None,
|
1428
1447
|
skip_cache: bool | None = False,
|
1429
1448
|
policy: Literal["mutable", "immutable"] | None = "mutable",
|
1449
|
+
merge: bool = False,
|
1430
1450
|
) -> None:
|
1431
1451
|
"""Add a local directory to the artifact.
|
1432
1452
|
|
@@ -1439,6 +1459,10 @@ class Artifact:
|
|
1439
1459
|
policy: "mutable" | "immutable". By default, "mutable"
|
1440
1460
|
"mutable": Create a temporary copy of the file to prevent corruption during upload.
|
1441
1461
|
"immutable": Disable protection, rely on the user not to delete or change the file.
|
1462
|
+
merge: If `False` (default), throws ValueError if a file was already added in a previous add_dir call
|
1463
|
+
and its content has changed. If `True`, overwrites existing files with changed content.
|
1464
|
+
Always adds new files and never removes files. To replace an entire directory, pass a name when adding the directory
|
1465
|
+
using `add_dir(local_path, name=my_prefix)` and call `remove(my_prefix)` to remove the directory, then add it again.
|
1442
1466
|
|
1443
1467
|
Raises:
|
1444
1468
|
ArtifactFinalizedError: You cannot make changes to the current artifact
|
@@ -1446,7 +1470,7 @@ class Artifact:
|
|
1446
1470
|
ValueError: Policy must be "mutable" or "immutable"
|
1447
1471
|
"""
|
1448
1472
|
if not os.path.isdir(local_path):
|
1449
|
-
raise ValueError("Path is not a directory: {}"
|
1473
|
+
raise ValueError(f"Path is not a directory: {local_path}")
|
1450
1474
|
|
1451
1475
|
termlog(
|
1452
1476
|
"Adding directory to artifact ({})... ".format(
|
@@ -1472,6 +1496,7 @@ class Artifact:
|
|
1472
1496
|
path=physical_path,
|
1473
1497
|
skip_cache=skip_cache,
|
1474
1498
|
policy=policy,
|
1499
|
+
overwrite=merge,
|
1475
1500
|
)
|
1476
1501
|
|
1477
1502
|
num_threads = 8
|
@@ -1612,8 +1637,9 @@ class Artifact:
|
|
1612
1637
|
data_types._SavedModel,
|
1613
1638
|
)
|
1614
1639
|
if not isinstance(obj, allowed_types):
|
1615
|
-
raise
|
1616
|
-
f"Found object of type {obj.__class__}, expected one of:
|
1640
|
+
raise TypeError(
|
1641
|
+
f"Found object of type {obj.__class__}, expected one of:"
|
1642
|
+
f" {allowed_types}"
|
1617
1643
|
)
|
1618
1644
|
|
1619
1645
|
obj_id = id(obj)
|
@@ -1747,7 +1773,7 @@ class Artifact:
|
|
1747
1773
|
name = LogicalPath(name)
|
1748
1774
|
entry = self.manifest.entries.get(name) or self._get_obj_entry(name)[0]
|
1749
1775
|
if entry is None:
|
1750
|
-
raise KeyError("Path not contained in artifact: {}"
|
1776
|
+
raise KeyError(f"Path not contained in artifact: {name}")
|
1751
1777
|
entry._parent_artifact = self
|
1752
1778
|
return entry
|
1753
1779
|
|
@@ -1775,7 +1801,7 @@ class Artifact:
|
|
1775
1801
|
assert self._client is not None
|
1776
1802
|
artifact = self._from_id(referenced_id, client=self._client)
|
1777
1803
|
assert artifact is not None
|
1778
|
-
return artifact.get(
|
1804
|
+
return artifact.get(uri_from_path(entry.ref))
|
1779
1805
|
|
1780
1806
|
# Special case for wandb.Table. This is intended to be a short term
|
1781
1807
|
# optimization. Since tables are likely to download many other assets in
|
@@ -1901,7 +1927,7 @@ class Artifact:
|
|
1901
1927
|
|
1902
1928
|
# TODO: Create a special stream instead of relying on an existing run.
|
1903
1929
|
if wandb.run is None:
|
1904
|
-
wl =
|
1930
|
+
wl = wandb_setup.singleton()
|
1905
1931
|
|
1906
1932
|
stream_id = generate_id()
|
1907
1933
|
|
@@ -1963,9 +1989,7 @@ class Artifact:
|
|
1963
1989
|
if nfiles > 5000 or size > 50 * 1024 * 1024:
|
1964
1990
|
log = True
|
1965
1991
|
termlog(
|
1966
|
-
"Downloading large artifact {}, {:.2f}MB. {} files... "
|
1967
|
-
self.name, size / (1024 * 1024), nfiles
|
1968
|
-
),
|
1992
|
+
f"Downloading large artifact {self.name}, {size / (1024 * 1024):.2f}MB. {nfiles} files... ",
|
1969
1993
|
)
|
1970
1994
|
start_time = datetime.now()
|
1971
1995
|
download_logger = ArtifactDownloadLogger(nfiles=nfiles)
|
@@ -2053,8 +2077,8 @@ class Artifact:
|
|
2053
2077
|
retryable_exceptions=(requests.RequestException),
|
2054
2078
|
)
|
2055
2079
|
def _fetch_file_urls(self, cursor: str | None, per_page: int | None = 5000) -> Any:
|
2056
|
-
if InternalApi().
|
2057
|
-
pb.ServerFeature.ARTIFACT_COLLECTION_MEMBERSHIP_FILES
|
2080
|
+
if InternalApi()._server_supports(
|
2081
|
+
pb.ServerFeature.ARTIFACT_COLLECTION_MEMBERSHIP_FILES
|
2058
2082
|
):
|
2059
2083
|
query = gql(
|
2060
2084
|
"""
|
@@ -2165,7 +2189,7 @@ class Artifact:
|
|
2165
2189
|
|
2166
2190
|
Args:
|
2167
2191
|
root: The directory to verify. If None artifact will be downloaded to
|
2168
|
-
'./artifacts/self.name/'
|
2192
|
+
'./artifacts/self.name/'.
|
2169
2193
|
|
2170
2194
|
Raises:
|
2171
2195
|
ArtifactNotLoggedError: If the artifact is not logged.
|
@@ -2181,16 +2205,14 @@ class Artifact:
|
|
2181
2205
|
self.get_entry(artifact_path)
|
2182
2206
|
except KeyError:
|
2183
2207
|
raise ValueError(
|
2184
|
-
"Found file {} which is not a member of artifact {}"
|
2185
|
-
full_path, self.name
|
2186
|
-
)
|
2208
|
+
f"Found file {full_path} which is not a member of artifact {self.name}"
|
2187
2209
|
)
|
2188
2210
|
|
2189
2211
|
ref_count = 0
|
2190
2212
|
for entry in self.manifest.entries.values():
|
2191
2213
|
if entry.ref is None:
|
2192
2214
|
if md5_file_b64(os.path.join(root, entry.path)) != entry.digest:
|
2193
|
-
raise ValueError("Digest mismatch for file: {
|
2215
|
+
raise ValueError(f"Digest mismatch for file: {entry.path}")
|
2194
2216
|
else:
|
2195
2217
|
ref_count += 1
|
2196
2218
|
if ref_count > 0:
|
@@ -2344,9 +2366,7 @@ class Artifact:
|
|
2344
2366
|
"Linking to a link artifact will result in directly linking to the source artifact of that link artifact."
|
2345
2367
|
)
|
2346
2368
|
|
2347
|
-
|
2348
|
-
|
2349
|
-
if run := singleton.most_recent_active_run:
|
2369
|
+
if run := wandb_setup.singleton().most_recent_active_run:
|
2350
2370
|
# TODO: Deprecate and encourage explicit link_artifact().
|
2351
2371
|
return run.link_artifact(self, target_path, aliases)
|
2352
2372
|
|
@@ -2504,7 +2524,7 @@ class Artifact:
|
|
2504
2524
|
Returns:
|
2505
2525
|
A `dict` with `string` keys representing attributes of the artifact.
|
2506
2526
|
"""
|
2507
|
-
return
|
2527
|
+
return artifact_to_json(self)
|
2508
2528
|
|
2509
2529
|
@staticmethod
|
2510
2530
|
def _expected_type(
|
@@ -2543,16 +2563,6 @@ class Artifact:
|
|
2543
2563
|
or {}
|
2544
2564
|
).get("name")
|
2545
2565
|
|
2546
|
-
@staticmethod
|
2547
|
-
def _normalize_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
|
2548
|
-
if metadata is None:
|
2549
|
-
return {}
|
2550
|
-
if not isinstance(metadata, dict):
|
2551
|
-
raise TypeError(f"metadata must be dict, not {type(metadata)}")
|
2552
|
-
return cast(
|
2553
|
-
Dict[str, Any], json.loads(json.dumps(util.json_friendly_val(metadata)))
|
2554
|
-
)
|
2555
|
-
|
2556
2566
|
def _load_manifest(self, url: str) -> ArtifactManifest:
|
2557
2567
|
with requests.get(url) as response:
|
2558
2568
|
response.raise_for_status()
|
@@ -2648,14 +2658,6 @@ class Artifact:
|
|
2648
2658
|
return linked_artifact
|
2649
2659
|
|
2650
2660
|
|
2651
|
-
def _ttl_duration_seconds_from_gql(gql_ttl_duration_seconds: int | None) -> int | None:
|
2652
|
-
# If gql_ttl_duration_seconds is not positive, its indicating that TTL is DISABLED(-2)
|
2653
|
-
# gql_ttl_duration_seconds only returns None if the server is not compatible with setting Artifact TTLs
|
2654
|
-
if gql_ttl_duration_seconds and gql_ttl_duration_seconds > 0:
|
2655
|
-
return gql_ttl_duration_seconds
|
2656
|
-
return None
|
2657
|
-
|
2658
|
-
|
2659
2661
|
class _ArtifactVersionType(WBType):
|
2660
2662
|
name = "artifactVersion"
|
2661
2663
|
types = [Artifact]
|
@@ -273,7 +273,5 @@ class ArtifactSaver:
|
|
273
273
|
if artifact_id is None:
|
274
274
|
raise RuntimeError(f"Could not resolve client id {client_id}")
|
275
275
|
entry.ref = URIStr(
|
276
|
-
"wandb-artifact://{}/{}"
|
277
|
-
b64_to_hex_id(B64MD5(artifact_id)), artifact_file_path
|
278
|
-
)
|
276
|
+
f"wandb-artifact://{b64_to_hex_id(B64MD5(artifact_id))}/{artifact_file_path}"
|
279
277
|
)
|
@@ -55,3 +55,17 @@ class ArtifactFinalizedError(ArtifactStatusError):
|
|
55
55
|
|
56
56
|
class WaitTimeoutError(errors.Error):
|
57
57
|
"""Raised when wait() timeout occurs before process is finished."""
|
58
|
+
|
59
|
+
|
60
|
+
class TooFewItemsError(ValueError):
|
61
|
+
"""Raised when there are fewer items than expected in a collection.
|
62
|
+
|
63
|
+
Intended for internal use only.
|
64
|
+
"""
|
65
|
+
|
66
|
+
|
67
|
+
class TooManyItemsError(ValueError):
|
68
|
+
"""Raised when there are more items than expected in a collection.
|
69
|
+
|
70
|
+
Intended for internal use only.
|
71
|
+
"""
|
wandb/sdk/artifacts/staging.py
CHANGED
@@ -46,9 +46,7 @@ class LocalFileHandler(StorageHandler):
|
|
46
46
|
local_path = util.local_file_uri_to_path(str(manifest_entry.ref))
|
47
47
|
if not os.path.exists(local_path):
|
48
48
|
raise ValueError(
|
49
|
-
"Local file reference: Failed to find file at path {}"
|
50
|
-
local_path
|
51
|
-
)
|
49
|
+
f"Local file reference: Failed to find file at path {local_path}"
|
52
50
|
)
|
53
51
|
|
54
52
|
path, hit, cache_open = self._cache.check_md5_obj_path(
|
@@ -140,7 +138,5 @@ class LocalFileHandler(StorageHandler):
|
|
140
138
|
entries.append(entry)
|
141
139
|
else:
|
142
140
|
# TODO: update error message if we don't allow directories.
|
143
|
-
raise ValueError(
|
144
|
-
'Path "{}" must be a valid file or directory path'.format(path)
|
145
|
-
)
|
141
|
+
raise ValueError(f'Path "{path}" must be a valid file or directory path')
|
146
142
|
return entries
|
@@ -31,7 +31,7 @@ class MultiHandler(StorageHandler):
|
|
31
31
|
return handler
|
32
32
|
if self._default_handler is not None:
|
33
33
|
return self._default_handler
|
34
|
-
raise ValueError('No storage handler registered for url "{}"'
|
34
|
+
raise ValueError(f'No storage handler registered for url "{url!s}"')
|
35
35
|
|
36
36
|
def load_path(
|
37
37
|
self,
|
@@ -59,14 +59,10 @@ class TrackingHandler(StorageHandler):
|
|
59
59
|
url = urlparse(path)
|
60
60
|
if name is None:
|
61
61
|
raise ValueError(
|
62
|
-
'You must pass name="<entry_name>" when tracking references with unknown schemes. ref: {}'
|
63
|
-
path
|
64
|
-
)
|
62
|
+
f'You must pass name="<entry_name>" when tracking references with unknown schemes. ref: {path}'
|
65
63
|
)
|
66
64
|
termwarn(
|
67
|
-
"Artifact references with unsupported schemes cannot be checksummed: {}"
|
68
|
-
path
|
69
|
-
)
|
65
|
+
f"Artifact references with unsupported schemes cannot be checksummed: {path}"
|
70
66
|
)
|
71
67
|
name = name or url.path[1:] # strip leading slash
|
72
68
|
return [ArtifactManifestEntry(path=name, ref=path, digest=path)]
|
@@ -116,11 +116,7 @@ class WBArtifactHandler(StorageHandler):
|
|
116
116
|
assert target_artifact is not None
|
117
117
|
assert target_artifact.id is not None
|
118
118
|
path = URIStr(
|
119
|
-
"{}://{}/{}"
|
120
|
-
self._scheme,
|
121
|
-
b64_to_hex_id(B64MD5(target_artifact.id)),
|
122
|
-
artifact_file_path,
|
123
|
-
)
|
119
|
+
f"{self._scheme}://{b64_to_hex_id(B64MD5(target_artifact.id))}/{artifact_file_path}"
|
124
120
|
)
|
125
121
|
|
126
122
|
# Return the new entry
|
@@ -58,7 +58,7 @@ class WBLocalArtifactHandler(StorageHandler):
|
|
58
58
|
target_path = util.uri_from_path(path)
|
59
59
|
target_artifact = artifact_instance_cache.get(client_id)
|
60
60
|
if not isinstance(target_artifact, wandb.Artifact):
|
61
|
-
raise
|
61
|
+
raise TypeError("Artifact passed to store_path() must be a wandb.Artifact.")
|
62
62
|
target_entry = target_artifact.manifest.entries[target_path] # type: ignore
|
63
63
|
if target_entry is None:
|
64
64
|
raise RuntimeError("Local entry not found - invalid reference")
|
@@ -254,7 +254,7 @@ class WandbStoragePolicy(StoragePolicy):
|
|
254
254
|
if env.is_debug():
|
255
255
|
logger.debug(f"Error writing chunk to file: {e}")
|
256
256
|
download_has_error.set()
|
257
|
-
raise
|
257
|
+
raise
|
258
258
|
else:
|
259
259
|
raise ValueError(f"Unknown queue item type: {type(item)}")
|
260
260
|
|
@@ -340,7 +340,7 @@ class WandbStoragePolicy(StoragePolicy):
|
|
340
340
|
if env.is_debug():
|
341
341
|
logger.debug(f"Error downloading file: {e}")
|
342
342
|
download_has_error.set()
|
343
|
-
raise
|
343
|
+
raise
|
344
344
|
finally:
|
345
345
|
# Always signal the writer to stop
|
346
346
|
q.put(_CHUNK_QUEUE_SENTINEL)
|
@@ -387,7 +387,7 @@ class WandbStoragePolicy(StoragePolicy):
|
|
387
387
|
api.settings("base_url"), entity_name, md5_hex
|
388
388
|
)
|
389
389
|
elif storage_layout == StorageLayout.V2:
|
390
|
-
if api.
|
390
|
+
if api._server_supports(
|
391
391
|
ServerFeature.ARTIFACT_COLLECTION_MEMBERSHIP_FILE_DOWNLOAD_HANDLER # type: ignore
|
392
392
|
):
|
393
393
|
return "{}/artifactsV2/{}/{}/{}/{}/{}/{}/{}".format(
|