wandb 0.15.3__py3-none-any.whl → 0.15.5__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- wandb/__init__.py +1 -1
- wandb/analytics/sentry.py +1 -0
- wandb/apis/importers/base.py +20 -5
- wandb/apis/importers/mlflow.py +7 -1
- wandb/apis/internal.py +12 -0
- wandb/apis/public.py +247 -1387
- wandb/apis/reports/_panels.py +58 -35
- wandb/beta/workflows.py +6 -7
- wandb/cli/cli.py +130 -60
- wandb/data_types.py +3 -1
- wandb/filesync/dir_watcher.py +21 -27
- wandb/filesync/step_checksum.py +8 -8
- wandb/filesync/step_prepare.py +23 -10
- wandb/filesync/step_upload.py +13 -13
- wandb/filesync/upload_job.py +4 -8
- wandb/integration/cohere/__init__.py +3 -0
- wandb/integration/cohere/cohere.py +21 -0
- wandb/integration/cohere/resolver.py +347 -0
- wandb/integration/gym/__init__.py +4 -6
- wandb/integration/huggingface/__init__.py +3 -0
- wandb/integration/huggingface/huggingface.py +18 -0
- wandb/integration/huggingface/resolver.py +213 -0
- wandb/integration/langchain/wandb_tracer.py +16 -179
- wandb/integration/openai/__init__.py +1 -3
- wandb/integration/openai/openai.py +11 -143
- wandb/integration/openai/resolver.py +111 -38
- wandb/integration/sagemaker/config.py +2 -2
- wandb/integration/tensorboard/log.py +4 -4
- wandb/old/settings.py +24 -7
- wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
- wandb/proto/wandb_deprecated.py +3 -1
- wandb/sdk/__init__.py +1 -1
- wandb/sdk/artifacts/__init__.py +0 -0
- wandb/sdk/artifacts/artifact.py +2101 -0
- wandb/sdk/artifacts/artifact_download_logger.py +42 -0
- wandb/sdk/artifacts/artifact_manifest.py +67 -0
- wandb/sdk/artifacts/artifact_manifest_entry.py +159 -0
- wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +91 -0
- wandb/sdk/{internal → artifacts}/artifact_saver.py +6 -5
- wandb/sdk/artifacts/artifact_state.py +10 -0
- wandb/sdk/{interface/artifacts/artifact_cache.py → artifacts/artifacts_cache.py} +22 -12
- wandb/sdk/artifacts/exceptions.py +55 -0
- wandb/sdk/artifacts/storage_handler.py +59 -0
- wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +192 -0
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +224 -0
- wandb/sdk/artifacts/storage_handlers/http_handler.py +112 -0
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +134 -0
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +53 -0
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +301 -0
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +67 -0
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +132 -0
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +72 -0
- wandb/sdk/artifacts/storage_layout.py +6 -0
- wandb/sdk/artifacts/storage_policies/__init__.py +0 -0
- wandb/sdk/artifacts/storage_policies/s3_bucket_policy.py +61 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +386 -0
- wandb/sdk/{interface/artifacts/artifact_storage.py → artifacts/storage_policy.py} +5 -57
- wandb/sdk/data_types/_dtypes.py +7 -12
- wandb/sdk/data_types/base_types/json_metadata.py +3 -2
- wandb/sdk/data_types/base_types/media.py +8 -8
- wandb/sdk/data_types/base_types/wb_value.py +12 -13
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +5 -6
- wandb/sdk/data_types/helper_types/classes.py +6 -8
- wandb/sdk/data_types/helper_types/image_mask.py +5 -6
- wandb/sdk/data_types/histogram.py +4 -3
- wandb/sdk/data_types/html.py +3 -4
- wandb/sdk/data_types/image.py +11 -9
- wandb/sdk/data_types/molecule.py +5 -3
- wandb/sdk/data_types/object_3d.py +7 -5
- wandb/sdk/data_types/plotly.py +3 -2
- wandb/sdk/data_types/saved_model.py +11 -11
- wandb/sdk/data_types/trace_tree.py +5 -4
- wandb/sdk/data_types/utils.py +3 -5
- wandb/sdk/data_types/video.py +5 -4
- wandb/sdk/integration_utils/auto_logging.py +215 -0
- wandb/sdk/interface/interface.py +15 -15
- wandb/sdk/internal/file_pusher.py +8 -16
- wandb/sdk/internal/file_stream.py +5 -11
- wandb/sdk/internal/handler.py +13 -1
- wandb/sdk/internal/internal_api.py +287 -13
- wandb/sdk/internal/job_builder.py +119 -30
- wandb/sdk/internal/sender.py +6 -26
- wandb/sdk/internal/settings_static.py +2 -0
- wandb/sdk/internal/system/assets/__init__.py +2 -0
- wandb/sdk/internal/system/assets/gpu.py +42 -0
- wandb/sdk/internal/system/assets/gpu_amd.py +216 -0
- wandb/sdk/internal/system/env_probe_helpers.py +13 -0
- wandb/sdk/internal/system/system_info.py +3 -3
- wandb/sdk/internal/tb_watcher.py +32 -22
- wandb/sdk/internal/thread_local_settings.py +18 -0
- wandb/sdk/launch/_project_spec.py +57 -11
- wandb/sdk/launch/agent/agent.py +147 -65
- wandb/sdk/launch/agent/job_status_tracker.py +34 -0
- wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
- wandb/sdk/launch/builder/abstract.py +5 -1
- wandb/sdk/launch/builder/build.py +21 -18
- wandb/sdk/launch/builder/docker_builder.py +10 -4
- wandb/sdk/launch/builder/kaniko_builder.py +113 -23
- wandb/sdk/launch/builder/noop.py +6 -3
- wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +46 -14
- wandb/sdk/launch/environment/aws_environment.py +3 -2
- wandb/sdk/launch/environment/azure_environment.py +124 -0
- wandb/sdk/launch/environment/gcp_environment.py +2 -4
- wandb/sdk/launch/environment/local_environment.py +1 -1
- wandb/sdk/launch/errors.py +19 -0
- wandb/sdk/launch/github_reference.py +32 -19
- wandb/sdk/launch/launch.py +3 -8
- wandb/sdk/launch/launch_add.py +6 -2
- wandb/sdk/launch/loader.py +21 -2
- wandb/sdk/launch/registry/azure_container_registry.py +132 -0
- wandb/sdk/launch/registry/elastic_container_registry.py +39 -5
- wandb/sdk/launch/registry/google_artifact_registry.py +68 -26
- wandb/sdk/launch/registry/local_registry.py +2 -1
- wandb/sdk/launch/runner/abstract.py +24 -3
- wandb/sdk/launch/runner/kubernetes_runner.py +479 -26
- wandb/sdk/launch/runner/local_container.py +103 -51
- wandb/sdk/launch/runner/local_process.py +1 -1
- wandb/sdk/launch/runner/sagemaker_runner.py +60 -10
- wandb/sdk/launch/runner/vertex_runner.py +10 -5
- wandb/sdk/launch/sweeps/__init__.py +7 -9
- wandb/sdk/launch/sweeps/scheduler.py +307 -77
- wandb/sdk/launch/sweeps/scheduler_sweep.py +2 -1
- wandb/sdk/launch/sweeps/utils.py +82 -35
- wandb/sdk/launch/utils.py +89 -75
- wandb/sdk/lib/_settings_toposort_generated.py +7 -0
- wandb/sdk/lib/capped_dict.py +26 -0
- wandb/sdk/lib/{git.py → gitlib.py} +76 -59
- wandb/sdk/lib/hashutil.py +12 -4
- wandb/sdk/lib/paths.py +96 -8
- wandb/sdk/lib/sock_client.py +2 -2
- wandb/sdk/lib/timer.py +1 -0
- wandb/sdk/service/server.py +22 -9
- wandb/sdk/service/server_sock.py +1 -1
- wandb/sdk/service/service.py +27 -8
- wandb/sdk/verify/verify.py +4 -7
- wandb/sdk/wandb_config.py +2 -6
- wandb/sdk/wandb_init.py +57 -53
- wandb/sdk/wandb_require.py +7 -0
- wandb/sdk/wandb_run.py +61 -223
- wandb/sdk/wandb_settings.py +28 -4
- wandb/testing/relay.py +15 -2
- wandb/util.py +74 -36
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/METADATA +15 -9
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/RECORD +151 -116
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/entry_points.txt +1 -0
- wandb/integration/langchain/util.py +0 -191
- wandb/sdk/interface/artifacts/__init__.py +0 -33
- wandb/sdk/interface/artifacts/artifact.py +0 -615
- wandb/sdk/interface/artifacts/artifact_manifest.py +0 -131
- wandb/sdk/wandb_artifacts.py +0 -2226
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/LICENSE +0 -0
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/WHEEL +0 -0
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/top_level.txt +0 -0
@@ -3,9 +3,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Uni
|
|
3
3
|
from wandb import util
|
4
4
|
|
5
5
|
if TYPE_CHECKING: # pragma: no cover
|
6
|
-
from wandb.
|
6
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
7
7
|
|
8
|
-
from ...wandb_artifacts import Artifact as LocalArtifact
|
9
8
|
from ...wandb_run import Run as LocalRun
|
10
9
|
|
11
10
|
TypeMappingType = Dict[str, Type["WBValue"]]
|
@@ -36,19 +35,19 @@ def _server_accepts_client_ids() -> bool:
|
|
36
35
|
|
37
36
|
|
38
37
|
class _WBValueArtifactSource:
|
39
|
-
artifact: "
|
38
|
+
artifact: "Artifact"
|
40
39
|
name: Optional[str]
|
41
40
|
|
42
|
-
def __init__(self, artifact: "
|
41
|
+
def __init__(self, artifact: "Artifact", name: Optional[str] = None) -> None:
|
43
42
|
self.artifact = artifact
|
44
43
|
self.name = name
|
45
44
|
|
46
45
|
|
47
46
|
class _WBValueArtifactTarget:
|
48
|
-
artifact: "
|
47
|
+
artifact: "Artifact"
|
49
48
|
name: Optional[str]
|
50
49
|
|
51
|
-
def __init__(self, artifact: "
|
50
|
+
def __init__(self, artifact: "Artifact", name: Optional[str] = None) -> None:
|
52
51
|
self.artifact = artifact
|
53
52
|
self.name = name
|
54
53
|
|
@@ -73,7 +72,7 @@ class WBValue:
|
|
73
72
|
self._artifact_source = None
|
74
73
|
self._artifact_target = None
|
75
74
|
|
76
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
75
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
77
76
|
"""Serialize the object into a JSON blob.
|
78
77
|
|
79
78
|
Uses current run or artifact to store additional data.
|
@@ -90,7 +89,7 @@ class WBValue:
|
|
90
89
|
|
91
90
|
@classmethod
|
92
91
|
def from_json(
|
93
|
-
cls: Type["WBValue"], json_obj: dict, source_artifact: "
|
92
|
+
cls: Type["WBValue"], json_obj: dict, source_artifact: "Artifact"
|
94
93
|
) -> "WBValue":
|
95
94
|
"""Deserialize a `json_obj` into it's class representation.
|
96
95
|
|
@@ -126,7 +125,7 @@ class WBValue:
|
|
126
125
|
|
127
126
|
@staticmethod
|
128
127
|
def init_from_json(
|
129
|
-
json_obj: dict, source_artifact: "
|
128
|
+
json_obj: dict, source_artifact: "Artifact"
|
130
129
|
) -> Optional["WBValue"]:
|
131
130
|
"""Initialize a `WBValue` from a JSON blob based on the class that creatd it.
|
132
131
|
|
@@ -186,7 +185,7 @@ class WBValue:
|
|
186
185
|
raise NotImplementedError
|
187
186
|
|
188
187
|
def _set_artifact_source(
|
189
|
-
self, artifact: "
|
188
|
+
self, artifact: "Artifact", name: Optional[str] = None
|
190
189
|
) -> None:
|
191
190
|
assert (
|
192
191
|
self._artifact_source is None
|
@@ -196,7 +195,7 @@ class WBValue:
|
|
196
195
|
self._artifact_source = _WBValueArtifactSource(artifact, name)
|
197
196
|
|
198
197
|
def _set_artifact_target(
|
199
|
-
self, artifact: "
|
198
|
+
self, artifact: "Artifact", name: Optional[str] = None
|
200
199
|
) -> None:
|
201
200
|
assert (
|
202
201
|
self._artifact_target is None
|
@@ -232,7 +231,7 @@ class WBValue:
|
|
232
231
|
elif (
|
233
232
|
self._artifact_target
|
234
233
|
and self._artifact_target.name
|
235
|
-
and self._artifact_target.artifact.
|
234
|
+
and self._artifact_target.artifact._is_draft_save_started()
|
236
235
|
and not util._is_offline()
|
237
236
|
and not _server_accepts_client_ids()
|
238
237
|
):
|
@@ -263,7 +262,7 @@ class WBValue:
|
|
263
262
|
elif (
|
264
263
|
self._artifact_target
|
265
264
|
and self._artifact_target.name
|
266
|
-
and self._artifact_target.artifact.
|
265
|
+
and self._artifact_target.artifact._is_draft_save_started()
|
267
266
|
and not util._is_offline()
|
268
267
|
and not _server_accepts_client_ids()
|
269
268
|
):
|
@@ -8,9 +8,8 @@ from wandb.util import has_num
|
|
8
8
|
from ..base_types.json_metadata import JSONMetadata
|
9
9
|
|
10
10
|
if TYPE_CHECKING: # pragma: no cover
|
11
|
-
from wandb.
|
11
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
12
12
|
|
13
|
-
from ...wandb_artifacts import Artifact as LocalArtifact
|
14
13
|
from ...wandb_run import Run as LocalRun
|
15
14
|
|
16
15
|
|
@@ -274,19 +273,19 @@ class BoundingBoxes2D(JSONMetadata):
|
|
274
273
|
raise TypeError("A box's caption must be a string")
|
275
274
|
return True
|
276
275
|
|
277
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
276
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
278
277
|
if isinstance(run_or_artifact, wandb.wandb_sdk.wandb_run.Run):
|
279
278
|
return super().to_json(run_or_artifact)
|
280
|
-
elif isinstance(run_or_artifact, wandb.
|
279
|
+
elif isinstance(run_or_artifact, wandb.Artifact):
|
281
280
|
# TODO (tim): I would like to log out a proper dictionary representing this object, but don't
|
282
281
|
# want to mess with the visualizations that are currently available in the UI. This really should output
|
283
282
|
# an object with a _type key. Will need to push this change to the UI first to ensure backwards compat
|
284
283
|
return self._val
|
285
284
|
else:
|
286
|
-
raise ValueError("to_json accepts wandb_run.Run or
|
285
|
+
raise ValueError("to_json accepts wandb_run.Run or wandb.Artifact")
|
287
286
|
|
288
287
|
@classmethod
|
289
288
|
def from_json(
|
290
|
-
cls: Type["BoundingBoxes2D"], json_obj: dict, source_artifact: "
|
289
|
+
cls: Type["BoundingBoxes2D"], json_obj: dict, source_artifact: "Artifact"
|
291
290
|
) -> "BoundingBoxes2D":
|
292
291
|
return cls({"box_data": json_obj}, "")
|
@@ -5,9 +5,8 @@ from .. import _dtypes
|
|
5
5
|
from ..base_types.media import Media
|
6
6
|
|
7
7
|
if TYPE_CHECKING: # pragma: no cover
|
8
|
-
from wandb.
|
8
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
9
9
|
|
10
|
-
from ...wandb_artifacts import Artifact as LocalArtifact
|
11
10
|
from ...wandb_run import Run as LocalRun
|
12
11
|
|
13
12
|
|
@@ -31,13 +30,11 @@ class Classes(Media):
|
|
31
30
|
def from_json(
|
32
31
|
cls: Type["Classes"],
|
33
32
|
json_obj: dict,
|
34
|
-
source_artifact: Optional["
|
33
|
+
source_artifact: Optional["Artifact"],
|
35
34
|
) -> "Classes":
|
36
35
|
return cls(json_obj.get("class_set")) # type: ignore
|
37
36
|
|
38
|
-
def to_json(
|
39
|
-
self, run_or_artifact: Optional[Union["LocalRun", "LocalArtifact"]]
|
40
|
-
) -> dict:
|
37
|
+
def to_json(self, run_or_artifact: Optional[Union["LocalRun", "Artifact"]]) -> dict:
|
41
38
|
json_obj = {}
|
42
39
|
# This is a bit of a hack to allow _ClassesIdType to
|
43
40
|
# be able to operate fully without an artifact in play.
|
@@ -116,7 +113,7 @@ class _ClassesIdType(_dtypes.Type):
|
|
116
113
|
def from_obj(cls, py_obj: Optional[Any] = None) -> "_dtypes.Type":
|
117
114
|
return cls(py_obj)
|
118
115
|
|
119
|
-
def to_json(self, artifact: Optional["
|
116
|
+
def to_json(self, artifact: Optional["Artifact"] = None) -> Dict[str, Any]:
|
120
117
|
cl_dict = super().to_json(artifact)
|
121
118
|
# TODO (tss): Refactor this block with the similar one in wandb.Image.
|
122
119
|
# This is a bit of a smell that the classes object does not follow
|
@@ -137,7 +134,7 @@ class _ClassesIdType(_dtypes.Type):
|
|
137
134
|
def from_json(
|
138
135
|
cls,
|
139
136
|
json_dict: Dict[str, Any],
|
140
|
-
artifact: Optional["
|
137
|
+
artifact: Optional["Artifact"] = None,
|
141
138
|
) -> "_dtypes.Type":
|
142
139
|
classes_obj = None
|
143
140
|
if (
|
@@ -148,6 +145,7 @@ class _ClassesIdType(_dtypes.Type):
|
|
148
145
|
classes_obj = artifact.get(
|
149
146
|
json_dict.get("params", {}).get("classes_obj", {}).get("path")
|
150
147
|
)
|
148
|
+
assert classes_obj is None or isinstance(classes_obj, Classes)
|
151
149
|
else:
|
152
150
|
raise RuntimeError("Expected artifact to be non-null.")
|
153
151
|
else:
|
@@ -10,9 +10,8 @@ from .._private import MEDIA_TMP
|
|
10
10
|
from ..base_types.media import Media
|
11
11
|
|
12
12
|
if TYPE_CHECKING: # pragma: no cover
|
13
|
-
from wandb.
|
13
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
14
14
|
|
15
|
-
from ...wandb_artifacts import Artifact as LocalArtifact
|
16
15
|
from ...wandb_run import Run as LocalRun
|
17
16
|
|
18
17
|
|
@@ -184,24 +183,24 @@ class ImageMask(Media):
|
|
184
183
|
|
185
184
|
@classmethod
|
186
185
|
def from_json(
|
187
|
-
cls: Type["ImageMask"], json_obj: dict, source_artifact: "
|
186
|
+
cls: Type["ImageMask"], json_obj: dict, source_artifact: "Artifact"
|
188
187
|
) -> "ImageMask":
|
189
188
|
return cls(
|
190
189
|
{"path": source_artifact.get_path(json_obj["path"]).download()},
|
191
190
|
key="",
|
192
191
|
)
|
193
192
|
|
194
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
193
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
195
194
|
json_dict = super().to_json(run_or_artifact)
|
196
195
|
|
197
196
|
if isinstance(run_or_artifact, wandb.wandb_sdk.wandb_run.Run):
|
198
197
|
json_dict["_type"] = self.type_name()
|
199
198
|
return json_dict
|
200
|
-
elif isinstance(run_or_artifact, wandb.
|
199
|
+
elif isinstance(run_or_artifact, wandb.Artifact):
|
201
200
|
# Nothing special to add (used to add "digest", but no longer used.)
|
202
201
|
return json_dict
|
203
202
|
else:
|
204
|
-
raise ValueError("to_json accepts wandb_run.Run or
|
203
|
+
raise ValueError("to_json accepts wandb_run.Run or wandb.Artifact")
|
205
204
|
|
206
205
|
@classmethod
|
207
206
|
def type_name(cls: Type["ImageMask"]) -> str:
|
@@ -6,9 +6,10 @@ from wandb import util
|
|
6
6
|
from .base_types.wb_value import WBValue
|
7
7
|
|
8
8
|
if TYPE_CHECKING: # pragma: no cover
|
9
|
-
import numpy as np
|
9
|
+
import numpy as np
|
10
|
+
|
11
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
10
12
|
|
11
|
-
from ..wandb_artifacts import Artifact as LocalArtifact
|
12
13
|
from ..wandb_run import Run as LocalRun
|
13
14
|
|
14
15
|
NumpyHistogram = Tuple[np.ndarray, np.ndarray]
|
@@ -83,7 +84,7 @@ class Histogram(WBValue):
|
|
83
84
|
if len(self.histogram) + 1 != len(self.bins):
|
84
85
|
raise ValueError("len(bins) must be len(histogram) + 1")
|
85
86
|
|
86
|
-
def to_json(self, run: Optional[Union["LocalRun", "
|
87
|
+
def to_json(self, run: Optional[Union["LocalRun", "Artifact"]] = None) -> dict:
|
87
88
|
return {"_type": self._log_type, "values": self.histogram, "bins": self.bins}
|
88
89
|
|
89
90
|
def __sizeof__(self) -> int:
|
wandb/sdk/data_types/html.py
CHANGED
@@ -10,9 +10,8 @@ from .base_types.media import BatchableMedia
|
|
10
10
|
if TYPE_CHECKING: # pragma: no cover
|
11
11
|
from typing import TextIO
|
12
12
|
|
13
|
-
from wandb.
|
13
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
14
14
|
|
15
|
-
from ..wandb_artifacts import Artifact as LocalArtifact
|
16
15
|
from ..wandb_run import Run as LocalRun
|
17
16
|
|
18
17
|
|
@@ -78,14 +77,14 @@ class Html(BatchableMedia):
|
|
78
77
|
def get_media_subdir(cls: Type["Html"]) -> str:
|
79
78
|
return os.path.join("media", "html")
|
80
79
|
|
81
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
80
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
82
81
|
json_dict = super().to_json(run_or_artifact)
|
83
82
|
json_dict["_type"] = self._log_type
|
84
83
|
return json_dict
|
85
84
|
|
86
85
|
@classmethod
|
87
86
|
def from_json(
|
88
|
-
cls: Type["Html"], json_obj: dict, source_artifact: "
|
87
|
+
cls: Type["Html"], json_obj: dict, source_artifact: "Artifact"
|
89
88
|
) -> "Html":
|
90
89
|
return cls(source_artifact.get_path(json_obj["path"]).download(), inject=False)
|
91
90
|
|
wandb/sdk/data_types/image.py
CHANGED
@@ -8,6 +8,7 @@ from urllib import parse
|
|
8
8
|
import wandb
|
9
9
|
from wandb import util
|
10
10
|
from wandb.sdk.lib import hashutil, runid
|
11
|
+
from wandb.sdk.lib.paths import LogicalPath
|
11
12
|
|
12
13
|
from ._private import MEDIA_TMP
|
13
14
|
from .base_types.media import BatchableMedia, Media
|
@@ -17,13 +18,12 @@ from .helper_types.image_mask import ImageMask
|
|
17
18
|
|
18
19
|
if TYPE_CHECKING: # pragma: no cover
|
19
20
|
import matplotlib # type: ignore
|
20
|
-
import numpy as np
|
21
|
+
import numpy as np
|
21
22
|
import torch # type: ignore
|
22
23
|
from PIL.Image import Image as PILImage
|
23
24
|
|
24
|
-
from wandb.
|
25
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
25
26
|
|
26
|
-
from ..wandb_artifacts import Artifact as LocalArtifact
|
27
27
|
from ..wandb_run import Run as LocalRun
|
28
28
|
|
29
29
|
ImageDataType = Union[
|
@@ -308,11 +308,13 @@ class Image(BatchableMedia):
|
|
308
308
|
|
309
309
|
@classmethod
|
310
310
|
def from_json(
|
311
|
-
cls: Type["Image"], json_obj: dict, source_artifact: "
|
311
|
+
cls: Type["Image"], json_obj: dict, source_artifact: "Artifact"
|
312
312
|
) -> "Image":
|
313
|
-
classes = None
|
313
|
+
classes: Optional[Classes] = None
|
314
314
|
if json_obj.get("classes") is not None:
|
315
|
-
|
315
|
+
value = source_artifact.get(json_obj["classes"]["path"])
|
316
|
+
assert isinstance(value, (type(None), Classes))
|
317
|
+
classes = value
|
316
318
|
|
317
319
|
masks = json_obj.get("masks")
|
318
320
|
_masks: Optional[Dict[str, ImageMask]] = None
|
@@ -385,7 +387,7 @@ class Image(BatchableMedia):
|
|
385
387
|
run, key, step, id_, ignore_copy_err=ignore_copy_err
|
386
388
|
)
|
387
389
|
|
388
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
390
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
389
391
|
json_dict = super().to_json(run_or_artifact)
|
390
392
|
json_dict["_type"] = Image._log_type
|
391
393
|
json_dict["format"] = self.format
|
@@ -399,7 +401,7 @@ class Image(BatchableMedia):
|
|
399
401
|
if self._caption:
|
400
402
|
json_dict["caption"] = self._caption
|
401
403
|
|
402
|
-
if isinstance(run_or_artifact, wandb.
|
404
|
+
if isinstance(run_or_artifact, wandb.Artifact):
|
403
405
|
artifact = run_or_artifact
|
404
406
|
if (
|
405
407
|
self._masks is not None or self._boxes is not None
|
@@ -493,7 +495,7 @@ class Image(BatchableMedia):
|
|
493
495
|
media_dir = cls.get_media_subdir()
|
494
496
|
|
495
497
|
for obj in jsons:
|
496
|
-
expected =
|
498
|
+
expected = LogicalPath(media_dir)
|
497
499
|
if "path" in obj and not obj["path"].startswith(expected):
|
498
500
|
raise ValueError(
|
499
501
|
"Files in an array of Image's must be in the {} directory, not {}".format(
|
wandb/sdk/data_types/molecule.py
CHANGED
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Optional, Sequence, Type, Union
|
|
5
5
|
|
6
6
|
from wandb import util
|
7
7
|
from wandb.sdk.lib import runid
|
8
|
+
from wandb.sdk.lib.paths import LogicalPath
|
8
9
|
|
9
10
|
from ._private import MEDIA_TMP
|
10
11
|
from .base_types.media import BatchableMedia, Media
|
@@ -14,7 +15,8 @@ if TYPE_CHECKING: # pragma: no cover
|
|
14
15
|
|
15
16
|
import rdkit.Chem # type: ignore
|
16
17
|
|
17
|
-
from
|
18
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
19
|
+
|
18
20
|
from ..wandb_run import Run as LocalRun
|
19
21
|
|
20
22
|
RDKitDataType = Union[str, "rdkit.Chem.rdchem.Mol"]
|
@@ -203,7 +205,7 @@ class Molecule(BatchableMedia):
|
|
203
205
|
def get_media_subdir(cls: Type["Molecule"]) -> str:
|
204
206
|
return os.path.join("media", "molecule")
|
205
207
|
|
206
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
208
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
207
209
|
json_dict = super().to_json(run_or_artifact)
|
208
210
|
json_dict["_type"] = self._log_type
|
209
211
|
if self._caption:
|
@@ -223,7 +225,7 @@ class Molecule(BatchableMedia):
|
|
223
225
|
jsons = [obj.to_json(run) for obj in seq]
|
224
226
|
|
225
227
|
for obj in jsons:
|
226
|
-
expected =
|
228
|
+
expected = LogicalPath(cls.get_media_subdir())
|
227
229
|
if not obj["path"].startswith(expected):
|
228
230
|
raise ValueError(
|
229
231
|
"Files in an array of Molecule's must be in the {} directory, not {}".format(
|
@@ -23,15 +23,17 @@ else:
|
|
23
23
|
import wandb
|
24
24
|
from wandb import util
|
25
25
|
from wandb.sdk.lib import runid
|
26
|
+
from wandb.sdk.lib.paths import LogicalPath
|
26
27
|
|
27
28
|
from . import _dtypes
|
28
29
|
from ._private import MEDIA_TMP
|
29
30
|
from .base_types.media import BatchableMedia
|
30
31
|
|
31
32
|
if TYPE_CHECKING: # pragma: no cover
|
32
|
-
import numpy as np
|
33
|
+
import numpy as np
|
34
|
+
|
35
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
33
36
|
|
34
|
-
from ..wandb_artifacts import Artifact as LocalArtifact
|
35
37
|
from ..wandb_run import Run as LocalRun
|
36
38
|
|
37
39
|
numeric = Union[int, float, np.integer, np.float_]
|
@@ -310,11 +312,11 @@ class Object3D(BatchableMedia):
|
|
310
312
|
def get_media_subdir(cls: Type["Object3D"]) -> str:
|
311
313
|
return os.path.join("media", "object3D")
|
312
314
|
|
313
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
315
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
314
316
|
json_dict = super().to_json(run_or_artifact)
|
315
317
|
json_dict["_type"] = Object3D._log_type
|
316
318
|
|
317
|
-
if isinstance(run_or_artifact, wandb.
|
319
|
+
if isinstance(run_or_artifact, wandb.Artifact):
|
318
320
|
if self._path is None or not self._path.endswith(".pts.json"):
|
319
321
|
raise ValueError(
|
320
322
|
"Non-point cloud 3D objects are not yet supported with Artifacts"
|
@@ -335,7 +337,7 @@ class Object3D(BatchableMedia):
|
|
335
337
|
jsons = [obj.to_json(run) for obj in seq]
|
336
338
|
|
337
339
|
for obj in jsons:
|
338
|
-
expected =
|
340
|
+
expected = LogicalPath(cls.get_media_subdir())
|
339
341
|
if not obj["path"].startswith(expected):
|
340
342
|
raise ValueError(
|
341
343
|
"Files in an array of Object3D's must be in the {} directory, not {}".format(
|
wandb/sdk/data_types/plotly.py
CHANGED
@@ -15,7 +15,8 @@ if TYPE_CHECKING: # pragma: no cover
|
|
15
15
|
import pandas as pd
|
16
16
|
import plotly # type: ignore
|
17
17
|
|
18
|
-
from
|
18
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
19
|
+
|
19
20
|
from ..wandb_run import Run as LocalRun
|
20
21
|
|
21
22
|
ValToJsonType = Union[
|
@@ -75,7 +76,7 @@ class Plotly(Media):
|
|
75
76
|
def get_media_subdir(cls: Type["Plotly"]) -> str:
|
76
77
|
return os.path.join("media", "plotly")
|
77
78
|
|
78
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
79
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
79
80
|
json_dict = super().to_json(run_or_artifact)
|
80
81
|
json_dict["_type"] = self._log_type
|
81
82
|
return json_dict
|
@@ -18,6 +18,7 @@ import wandb
|
|
18
18
|
from wandb import util
|
19
19
|
from wandb.sdk.lib import runid
|
20
20
|
from wandb.sdk.lib.hashutil import md5_file_hex
|
21
|
+
from wandb.sdk.lib.paths import LogicalPath
|
21
22
|
|
22
23
|
from ._private import MEDIA_TMP
|
23
24
|
from .base_types.wb_value import WBValue
|
@@ -28,9 +29,8 @@ if TYPE_CHECKING: # pragma: no cover
|
|
28
29
|
import tensorflow # type: ignore
|
29
30
|
import torch # type: ignore
|
30
31
|
|
31
|
-
from wandb.
|
32
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
32
33
|
|
33
|
-
from ..wandb_artifacts import Artifact as LocalArtifact
|
34
34
|
from ..wandb_run import Run as LocalRun
|
35
35
|
|
36
36
|
|
@@ -38,19 +38,19 @@ DEBUG_MODE = False
|
|
38
38
|
|
39
39
|
|
40
40
|
def _add_deterministic_dir_to_artifact(
|
41
|
-
artifact: "
|
41
|
+
artifact: "Artifact", dir_name: str, target_dir_root: str
|
42
42
|
) -> str:
|
43
43
|
file_paths = []
|
44
44
|
for dirpath, _, filenames in os.walk(dir_name, topdown=True):
|
45
45
|
for fn in filenames:
|
46
46
|
file_paths.append(os.path.join(dirpath, fn))
|
47
47
|
dirname = md5_file_hex(*file_paths)[:20]
|
48
|
-
target_path =
|
48
|
+
target_path = LogicalPath(os.path.join(target_dir_root, dirname))
|
49
49
|
artifact.add_dir(dir_name, target_path)
|
50
50
|
return target_path
|
51
51
|
|
52
52
|
|
53
|
-
def _load_dir_from_artifact(source_artifact: "
|
53
|
+
def _load_dir_from_artifact(source_artifact: "Artifact", path: str) -> str:
|
54
54
|
dl_path = None
|
55
55
|
|
56
56
|
# Look through the entire manifest to find all of the files in the directory.
|
@@ -125,14 +125,14 @@ class _SavedModel(WBValue, Generic[SavedModelObjType]):
|
|
125
125
|
|
126
126
|
@classmethod
|
127
127
|
def from_json(
|
128
|
-
cls: Type["_SavedModel"], json_obj: dict, source_artifact: "
|
128
|
+
cls: Type["_SavedModel"], json_obj: dict, source_artifact: "Artifact"
|
129
129
|
) -> "_SavedModel":
|
130
130
|
path = json_obj["path"]
|
131
131
|
|
132
132
|
# First, if the entry is a file, the download it.
|
133
133
|
entry = source_artifact.manifest.entries.get(path)
|
134
134
|
if entry is not None:
|
135
|
-
dl_path = source_artifact.get_path(path).download()
|
135
|
+
dl_path = str(source_artifact.get_path(path).download())
|
136
136
|
else:
|
137
137
|
# If not, assume it is directory.
|
138
138
|
# FUTURE: Add this functionality to the artifact loader
|
@@ -143,7 +143,7 @@ class _SavedModel(WBValue, Generic[SavedModelObjType]):
|
|
143
143
|
# and specified adapter.
|
144
144
|
return cls(dl_path)
|
145
145
|
|
146
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
146
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
147
147
|
# Unlike other data types, we do not allow adding to a Run directly. There is a
|
148
148
|
# bit of tech debt in the other data types which requires the input to `to_json`
|
149
149
|
# to accept a Run or Artifact. However, Run additions should be deprecated in the future.
|
@@ -319,7 +319,7 @@ class _PicklingSavedModel(_SavedModel[SavedModelObjType]):
|
|
319
319
|
|
320
320
|
@classmethod
|
321
321
|
def from_json(
|
322
|
-
cls: Type["_SavedModel"], json_obj: dict, source_artifact: "
|
322
|
+
cls: Type["_SavedModel"], json_obj: dict, source_artifact: "Artifact"
|
323
323
|
) -> "_PicklingSavedModel":
|
324
324
|
backup_path = [p for p in sys.path]
|
325
325
|
if (
|
@@ -336,9 +336,9 @@ class _PicklingSavedModel(_SavedModel[SavedModelObjType]):
|
|
336
336
|
|
337
337
|
return inst # type: ignore
|
338
338
|
|
339
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
339
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
340
340
|
json_obj = super().to_json(run_or_artifact)
|
341
|
-
assert isinstance(run_or_artifact, wandb.
|
341
|
+
assert isinstance(run_or_artifact, wandb.Artifact)
|
342
342
|
if self._dep_py_files_path is not None:
|
343
343
|
json_obj["dep_py_files_path"] = _add_deterministic_dir_to_artifact(
|
344
344
|
run_or_artifact,
|
@@ -13,12 +13,13 @@ from dataclasses import dataclass, field
|
|
13
13
|
from enum import Enum
|
14
14
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
15
15
|
|
16
|
-
|
16
|
+
import wandb.data_types
|
17
17
|
from wandb.sdk.data_types import _dtypes
|
18
18
|
from wandb.sdk.data_types.base_types.media import Media
|
19
19
|
|
20
20
|
if TYPE_CHECKING: # pragma: no cover
|
21
|
-
from
|
21
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
22
|
+
|
22
23
|
from ..wandb_run import Run as LocalRun
|
23
24
|
|
24
25
|
|
@@ -102,7 +103,7 @@ class WBTraceTree(Media):
|
|
102
103
|
def get_media_subdir(cls) -> str:
|
103
104
|
return "media/wb_trace_tree"
|
104
105
|
|
105
|
-
def to_json(self, run: Optional[Union["LocalRun", "
|
106
|
+
def to_json(self, run: Optional[Union["LocalRun", "Artifact"]]) -> dict:
|
106
107
|
res = {}
|
107
108
|
res["_type"] = self._log_type
|
108
109
|
# Here we use `dumps` to put things into string format. This is because
|
@@ -141,7 +142,7 @@ def _fallback_serialize(obj: Any) -> str:
|
|
141
142
|
def _safe_serialize(obj: dict) -> str:
|
142
143
|
try:
|
143
144
|
return json.dumps(
|
144
|
-
_json_helper(obj, None),
|
145
|
+
wandb.data_types._json_helper(obj, None),
|
145
146
|
skipkeys=True,
|
146
147
|
default=_fallback_serialize,
|
147
148
|
)
|
wandb/sdk/data_types/utils.py
CHANGED
@@ -124,7 +124,7 @@ def val_to_json(
|
|
124
124
|
# the array index?
|
125
125
|
# There is a bug here: if this array contains two arrays of the same type of
|
126
126
|
# anonymous media objects, their eventual names will collide.
|
127
|
-
# This used to happen. The frontend doesn't handle
|
127
|
+
# This used to happen. The frontend doesn't handle heterogeneous arrays
|
128
128
|
# raise ValueError(
|
129
129
|
# "Mixed media types in the same list aren't supported")
|
130
130
|
return [
|
@@ -145,12 +145,10 @@ def val_to_json(
|
|
145
145
|
# Special conditional to log tables as artifact entries as well.
|
146
146
|
# I suspect we will generalize this as we transition to storing all
|
147
147
|
# files in an artifact
|
148
|
-
# we sanitize the key to meet the constraints
|
148
|
+
# we sanitize the key to meet the constraints
|
149
149
|
# in this case, leaving only alpha numerics or underscores.
|
150
150
|
sanitized_key = re.sub(r"[^a-zA-Z0-9_]+", "", key)
|
151
|
-
art = wandb.
|
152
|
-
f"run-{run.id}-{sanitized_key}", "run_table"
|
153
|
-
)
|
151
|
+
art = wandb.Artifact(f"run-{run.id}-{sanitized_key}", "run_table")
|
154
152
|
art.add(val, key)
|
155
153
|
run.log_artifact(art)
|
156
154
|
|
wandb/sdk/data_types/video.py
CHANGED
@@ -13,9 +13,10 @@ from .base_types.media import BatchableMedia
|
|
13
13
|
if TYPE_CHECKING: # pragma: no cover
|
14
14
|
from typing import TextIO
|
15
15
|
|
16
|
-
import numpy as np
|
16
|
+
import numpy as np
|
17
|
+
|
18
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
17
19
|
|
18
|
-
from ..wandb_artifacts import Artifact as LocalArtifact
|
19
20
|
from ..wandb_run import Run as LocalRun
|
20
21
|
|
21
22
|
|
@@ -132,7 +133,7 @@ class Video(BatchableMedia):
|
|
132
133
|
required='wandb.Video requires moviepy and imageio when passing raw data. Install with "pip install moviepy imageio"',
|
133
134
|
)
|
134
135
|
tensor = self._prepare_video(self.data)
|
135
|
-
_, self._height, self._width, self._channels = tensor.shape
|
136
|
+
_, self._height, self._width, self._channels = tensor.shape # type: ignore
|
136
137
|
|
137
138
|
# encode sequence of images into gif string
|
138
139
|
clip = mpy.ImageSequenceClip(list(tensor), fps=self._fps)
|
@@ -169,7 +170,7 @@ class Video(BatchableMedia):
|
|
169
170
|
def get_media_subdir(cls: Type["Video"]) -> str:
|
170
171
|
return os.path.join("media", "videos")
|
171
172
|
|
172
|
-
def to_json(self, run_or_artifact: Union["LocalRun", "
|
173
|
+
def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
|
173
174
|
json_dict = super().to_json(run_or_artifact)
|
174
175
|
json_dict["_type"] = self._log_type
|
175
176
|
|