wandb 0.15.3__py3-none-any.whl → 0.15.5__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- wandb/__init__.py +1 -1
- wandb/analytics/sentry.py +1 -0
- wandb/apis/importers/base.py +20 -5
- wandb/apis/importers/mlflow.py +7 -1
- wandb/apis/internal.py +12 -0
- wandb/apis/public.py +247 -1387
- wandb/apis/reports/_panels.py +58 -35
- wandb/beta/workflows.py +6 -7
- wandb/cli/cli.py +130 -60
- wandb/data_types.py +3 -1
- wandb/filesync/dir_watcher.py +21 -27
- wandb/filesync/step_checksum.py +8 -8
- wandb/filesync/step_prepare.py +23 -10
- wandb/filesync/step_upload.py +13 -13
- wandb/filesync/upload_job.py +4 -8
- wandb/integration/cohere/__init__.py +3 -0
- wandb/integration/cohere/cohere.py +21 -0
- wandb/integration/cohere/resolver.py +347 -0
- wandb/integration/gym/__init__.py +4 -6
- wandb/integration/huggingface/__init__.py +3 -0
- wandb/integration/huggingface/huggingface.py +18 -0
- wandb/integration/huggingface/resolver.py +213 -0
- wandb/integration/langchain/wandb_tracer.py +16 -179
- wandb/integration/openai/__init__.py +1 -3
- wandb/integration/openai/openai.py +11 -143
- wandb/integration/openai/resolver.py +111 -38
- wandb/integration/sagemaker/config.py +2 -2
- wandb/integration/tensorboard/log.py +4 -4
- wandb/old/settings.py +24 -7
- wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
- wandb/proto/wandb_deprecated.py +3 -1
- wandb/sdk/__init__.py +1 -1
- wandb/sdk/artifacts/__init__.py +0 -0
- wandb/sdk/artifacts/artifact.py +2101 -0
- wandb/sdk/artifacts/artifact_download_logger.py +42 -0
- wandb/sdk/artifacts/artifact_manifest.py +67 -0
- wandb/sdk/artifacts/artifact_manifest_entry.py +159 -0
- wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +91 -0
- wandb/sdk/{internal → artifacts}/artifact_saver.py +6 -5
- wandb/sdk/artifacts/artifact_state.py +10 -0
- wandb/sdk/{interface/artifacts/artifact_cache.py → artifacts/artifacts_cache.py} +22 -12
- wandb/sdk/artifacts/exceptions.py +55 -0
- wandb/sdk/artifacts/storage_handler.py +59 -0
- wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +192 -0
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +224 -0
- wandb/sdk/artifacts/storage_handlers/http_handler.py +112 -0
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +134 -0
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +53 -0
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +301 -0
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +67 -0
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +132 -0
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +72 -0
- wandb/sdk/artifacts/storage_layout.py +6 -0
- wandb/sdk/artifacts/storage_policies/__init__.py +0 -0
- wandb/sdk/artifacts/storage_policies/s3_bucket_policy.py +61 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +386 -0
- wandb/sdk/{interface/artifacts/artifact_storage.py → artifacts/storage_policy.py} +5 -57
- wandb/sdk/data_types/_dtypes.py +7 -12
- wandb/sdk/data_types/base_types/json_metadata.py +3 -2
- wandb/sdk/data_types/base_types/media.py +8 -8
- wandb/sdk/data_types/base_types/wb_value.py +12 -13
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +5 -6
- wandb/sdk/data_types/helper_types/classes.py +6 -8
- wandb/sdk/data_types/helper_types/image_mask.py +5 -6
- wandb/sdk/data_types/histogram.py +4 -3
- wandb/sdk/data_types/html.py +3 -4
- wandb/sdk/data_types/image.py +11 -9
- wandb/sdk/data_types/molecule.py +5 -3
- wandb/sdk/data_types/object_3d.py +7 -5
- wandb/sdk/data_types/plotly.py +3 -2
- wandb/sdk/data_types/saved_model.py +11 -11
- wandb/sdk/data_types/trace_tree.py +5 -4
- wandb/sdk/data_types/utils.py +3 -5
- wandb/sdk/data_types/video.py +5 -4
- wandb/sdk/integration_utils/auto_logging.py +215 -0
- wandb/sdk/interface/interface.py +15 -15
- wandb/sdk/internal/file_pusher.py +8 -16
- wandb/sdk/internal/file_stream.py +5 -11
- wandb/sdk/internal/handler.py +13 -1
- wandb/sdk/internal/internal_api.py +287 -13
- wandb/sdk/internal/job_builder.py +119 -30
- wandb/sdk/internal/sender.py +6 -26
- wandb/sdk/internal/settings_static.py +2 -0
- wandb/sdk/internal/system/assets/__init__.py +2 -0
- wandb/sdk/internal/system/assets/gpu.py +42 -0
- wandb/sdk/internal/system/assets/gpu_amd.py +216 -0
- wandb/sdk/internal/system/env_probe_helpers.py +13 -0
- wandb/sdk/internal/system/system_info.py +3 -3
- wandb/sdk/internal/tb_watcher.py +32 -22
- wandb/sdk/internal/thread_local_settings.py +18 -0
- wandb/sdk/launch/_project_spec.py +57 -11
- wandb/sdk/launch/agent/agent.py +147 -65
- wandb/sdk/launch/agent/job_status_tracker.py +34 -0
- wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
- wandb/sdk/launch/builder/abstract.py +5 -1
- wandb/sdk/launch/builder/build.py +21 -18
- wandb/sdk/launch/builder/docker_builder.py +10 -4
- wandb/sdk/launch/builder/kaniko_builder.py +113 -23
- wandb/sdk/launch/builder/noop.py +6 -3
- wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +46 -14
- wandb/sdk/launch/environment/aws_environment.py +3 -2
- wandb/sdk/launch/environment/azure_environment.py +124 -0
- wandb/sdk/launch/environment/gcp_environment.py +2 -4
- wandb/sdk/launch/environment/local_environment.py +1 -1
- wandb/sdk/launch/errors.py +19 -0
- wandb/sdk/launch/github_reference.py +32 -19
- wandb/sdk/launch/launch.py +3 -8
- wandb/sdk/launch/launch_add.py +6 -2
- wandb/sdk/launch/loader.py +21 -2
- wandb/sdk/launch/registry/azure_container_registry.py +132 -0
- wandb/sdk/launch/registry/elastic_container_registry.py +39 -5
- wandb/sdk/launch/registry/google_artifact_registry.py +68 -26
- wandb/sdk/launch/registry/local_registry.py +2 -1
- wandb/sdk/launch/runner/abstract.py +24 -3
- wandb/sdk/launch/runner/kubernetes_runner.py +479 -26
- wandb/sdk/launch/runner/local_container.py +103 -51
- wandb/sdk/launch/runner/local_process.py +1 -1
- wandb/sdk/launch/runner/sagemaker_runner.py +60 -10
- wandb/sdk/launch/runner/vertex_runner.py +10 -5
- wandb/sdk/launch/sweeps/__init__.py +7 -9
- wandb/sdk/launch/sweeps/scheduler.py +307 -77
- wandb/sdk/launch/sweeps/scheduler_sweep.py +2 -1
- wandb/sdk/launch/sweeps/utils.py +82 -35
- wandb/sdk/launch/utils.py +89 -75
- wandb/sdk/lib/_settings_toposort_generated.py +7 -0
- wandb/sdk/lib/capped_dict.py +26 -0
- wandb/sdk/lib/{git.py → gitlib.py} +76 -59
- wandb/sdk/lib/hashutil.py +12 -4
- wandb/sdk/lib/paths.py +96 -8
- wandb/sdk/lib/sock_client.py +2 -2
- wandb/sdk/lib/timer.py +1 -0
- wandb/sdk/service/server.py +22 -9
- wandb/sdk/service/server_sock.py +1 -1
- wandb/sdk/service/service.py +27 -8
- wandb/sdk/verify/verify.py +4 -7
- wandb/sdk/wandb_config.py +2 -6
- wandb/sdk/wandb_init.py +57 -53
- wandb/sdk/wandb_require.py +7 -0
- wandb/sdk/wandb_run.py +61 -223
- wandb/sdk/wandb_settings.py +28 -4
- wandb/testing/relay.py +15 -2
- wandb/util.py +74 -36
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/METADATA +15 -9
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/RECORD +151 -116
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/entry_points.txt +1 -0
- wandb/integration/langchain/util.py +0 -191
- wandb/sdk/interface/artifacts/__init__.py +0 -33
- wandb/sdk/interface/artifacts/artifact.py +0 -615
- wandb/sdk/interface/artifacts/artifact_manifest.py +0 -131
- wandb/sdk/wandb_artifacts.py +0 -2226
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/LICENSE +0 -0
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/WHEEL +0 -0
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/top_level.txt +0 -0
wandb/sdk/wandb_run.py
CHANGED
@@ -47,12 +47,13 @@ from wandb.proto.wandb_internal_pb2 import (
|
|
47
47
|
RunRecord,
|
48
48
|
ServerInfoResponse,
|
49
49
|
)
|
50
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
50
51
|
from wandb.sdk.internal import job_builder
|
51
52
|
from wandb.sdk.lib.import_hooks import (
|
52
53
|
register_post_import_hook,
|
53
54
|
unregister_post_import_hook,
|
54
55
|
)
|
55
|
-
from wandb.sdk.lib.paths import FilePathStr, StrPath
|
56
|
+
from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath
|
56
57
|
from wandb.util import (
|
57
58
|
_is_artifact_object,
|
58
59
|
_is_artifact_string,
|
@@ -61,14 +62,11 @@ from wandb.util import (
|
|
61
62
|
_resolve_aliases,
|
62
63
|
add_import_hook,
|
63
64
|
parse_artifact_string,
|
64
|
-
to_forward_slash_path,
|
65
65
|
)
|
66
66
|
from wandb.viz import CustomChart, Visualize, custom_chart
|
67
67
|
|
68
|
-
from . import
|
68
|
+
from . import wandb_config, wandb_metric, wandb_summary
|
69
69
|
from .data_types._dtypes import TypeRegistry
|
70
|
-
from .interface.artifacts import Artifact as ArtifactInterface
|
71
|
-
from .interface.artifacts import ArtifactNotLoggedError
|
72
70
|
from .interface.interface import GlobStr, InterfaceBase
|
73
71
|
from .interface.summary_record import SummaryRecord
|
74
72
|
from .lib import (
|
@@ -83,13 +81,12 @@ from .lib import (
|
|
83
81
|
telemetry,
|
84
82
|
)
|
85
83
|
from .lib.exit_hooks import ExitHooks
|
86
|
-
from .lib.
|
84
|
+
from .lib.gitlib import GitRepo
|
87
85
|
from .lib.mailbox import MailboxError, MailboxHandle, MailboxProbe, MailboxProgress
|
88
86
|
from .lib.printer import get_printer
|
89
87
|
from .lib.proto_util import message_to_dict
|
90
88
|
from .lib.reporting import Reporter
|
91
89
|
from .lib.wburls import wburls
|
92
|
-
from .wandb_artifacts import Artifact
|
93
90
|
from .wandb_settings import Settings, SettingsConsole
|
94
91
|
from .wandb_setup import _WandbSetup
|
95
92
|
|
@@ -109,8 +106,6 @@ if TYPE_CHECKING:
|
|
109
106
|
SampledHistoryResponse,
|
110
107
|
)
|
111
108
|
|
112
|
-
from .data_types.base_types.wb_value import WBValue
|
113
|
-
from .interface.artifacts import ArtifactManifest, ArtifactManifestEntry
|
114
109
|
from .interface.interface import FilesDict, PolicyName
|
115
110
|
from .lib.printer import PrinterJupyter, PrinterTerm
|
116
111
|
from .wandb_alerts import AlertLevel
|
@@ -481,7 +476,6 @@ class Run:
|
|
481
476
|
_notes: Optional[str]
|
482
477
|
|
483
478
|
_run_obj: Optional[RunRecord]
|
484
|
-
_run_obj_offline: Optional[RunRecord]
|
485
479
|
# Use string literal annotation because of type reference loop
|
486
480
|
_backend: Optional["wandb.sdk.backend.backend.Backend"]
|
487
481
|
_internal_run_interface: Optional[
|
@@ -604,7 +598,6 @@ class Run:
|
|
604
598
|
|
605
599
|
# Returned from backend request_run(), set from wandb_init?
|
606
600
|
self._run_obj = None
|
607
|
-
self._run_obj_offline = None
|
608
601
|
|
609
602
|
# Created when the run "starts".
|
610
603
|
self._run_status_checker = None
|
@@ -641,7 +634,7 @@ class Run:
|
|
641
634
|
self._launch_artifact_mapping: Dict[str, Any] = {}
|
642
635
|
self._unique_launch_artifact_sequence_names: Dict[str, Any] = {}
|
643
636
|
if self._settings.save_code and self._settings.program_relpath:
|
644
|
-
config[wandb_key]["code_path"] =
|
637
|
+
config[wandb_key]["code_path"] = LogicalPath(
|
645
638
|
os.path.join("code", self._settings.program_relpath)
|
646
639
|
)
|
647
640
|
if sweep_config:
|
@@ -883,8 +876,7 @@ class Run:
|
|
883
876
|
tel.feature.set_run_name = True
|
884
877
|
self._name = name
|
885
878
|
if self._backend and self._backend.interface:
|
886
|
-
|
887
|
-
self._backend.interface.publish_run(run)
|
879
|
+
self._backend.interface.publish_run(self)
|
888
880
|
|
889
881
|
@property
|
890
882
|
@_run_decorator._attach
|
@@ -905,8 +897,7 @@ class Run:
|
|
905
897
|
def notes(self, notes: str) -> None:
|
906
898
|
self._notes = notes
|
907
899
|
if self._backend and self._backend.interface:
|
908
|
-
|
909
|
-
self._backend.interface.publish_run(run)
|
900
|
+
self._backend.interface.publish_run(self)
|
910
901
|
|
911
902
|
@property
|
912
903
|
@_run_decorator._attach
|
@@ -914,9 +905,8 @@ class Run:
|
|
914
905
|
"""Tags associated with the run, if there are any."""
|
915
906
|
if self._tags:
|
916
907
|
return self._tags
|
917
|
-
|
918
|
-
|
919
|
-
return tuple(run_obj.tags)
|
908
|
+
if self._run_obj:
|
909
|
+
return tuple(self._run_obj.tags)
|
920
910
|
return None
|
921
911
|
|
922
912
|
@tags.setter
|
@@ -926,8 +916,7 @@ class Run:
|
|
926
916
|
tel.feature.set_run_tags = True
|
927
917
|
self._tags = tuple(tags)
|
928
918
|
if self._backend and self._backend.interface:
|
929
|
-
|
930
|
-
self._backend.interface.publish_run(run)
|
919
|
+
self._backend.interface.publish_run(self)
|
931
920
|
|
932
921
|
@property
|
933
922
|
@_run_decorator._attach
|
@@ -999,8 +988,7 @@ class Run:
|
|
999
988
|
return self._step
|
1000
989
|
|
1001
990
|
def project_name(self) -> str:
|
1002
|
-
|
1003
|
-
return run_obj.project if run_obj else ""
|
991
|
+
return self._run_obj.project if self._run_obj else ""
|
1004
992
|
|
1005
993
|
@property
|
1006
994
|
@_run_decorator._attach
|
@@ -1026,8 +1014,7 @@ class Run:
|
|
1026
1014
|
return self._settings._noop
|
1027
1015
|
|
1028
1016
|
def _get_group(self) -> str:
|
1029
|
-
|
1030
|
-
return run_obj.run_group if run_obj else ""
|
1017
|
+
return self._run_obj.run_group if self._run_obj else ""
|
1031
1018
|
|
1032
1019
|
@property
|
1033
1020
|
@_run_decorator._attach
|
@@ -1038,7 +1025,7 @@ class Run:
|
|
1038
1025
|
|
1039
1026
|
If you are doing a distributed training you should give all of the
|
1040
1027
|
runs in the training the same group.
|
1041
|
-
If you are doing
|
1028
|
+
If you are doing cross-validation you should give all the cross-validation
|
1042
1029
|
folds the same group.
|
1043
1030
|
"""
|
1044
1031
|
return self._get_group()
|
@@ -1046,8 +1033,7 @@ class Run:
|
|
1046
1033
|
@property
|
1047
1034
|
@_run_decorator._attach
|
1048
1035
|
def job_type(self) -> str:
|
1049
|
-
|
1050
|
-
return run_obj.job_type if run_obj else ""
|
1036
|
+
return self._run_obj.job_type if self._run_obj else ""
|
1051
1037
|
|
1052
1038
|
@property
|
1053
1039
|
@_run_decorator._attach
|
@@ -1096,10 +1082,19 @@ class Run:
|
|
1096
1082
|
An `Artifact` object if code was logged
|
1097
1083
|
"""
|
1098
1084
|
if name is None:
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1102
|
-
|
1085
|
+
if self.settings._jupyter:
|
1086
|
+
notebook_name = None
|
1087
|
+
if self.settings.notebook_name:
|
1088
|
+
notebook_name = self.settings.notebook_name
|
1089
|
+
elif self.settings._jupyter_path:
|
1090
|
+
if self.settings._jupyter_path.startswith("fileId="):
|
1091
|
+
notebook_name = self.settings._jupyter_name
|
1092
|
+
else:
|
1093
|
+
notebook_name = self.settings._jupyter_path
|
1094
|
+
name_string = f"{self._project}-{notebook_name}"
|
1095
|
+
else:
|
1096
|
+
name_string = f"{self._project}-{self._settings.program_relpath}"
|
1097
|
+
name = wandb.util.make_artifact_name_safe(f"source-{name_string}")
|
1103
1098
|
art = wandb.Artifact(name, "code")
|
1104
1099
|
files_added = False
|
1105
1100
|
if root is not None:
|
@@ -1289,14 +1284,14 @@ class Run:
|
|
1289
1284
|
|
1290
1285
|
def _config_artifact_callback(
|
1291
1286
|
self, key: str, val: Union[str, Artifact, dict]
|
1292
|
-
) ->
|
1287
|
+
) -> Artifact:
|
1293
1288
|
# artifacts can look like dicts as they are passed into the run config
|
1294
1289
|
# since the run config stores them on the backend as a dict with fields shown
|
1295
1290
|
# in wandb.util.artifact_to_json
|
1296
1291
|
if _is_artifact_version_weave_dict(val):
|
1297
1292
|
assert isinstance(val, dict)
|
1298
1293
|
public_api = self._public_api()
|
1299
|
-
artifact =
|
1294
|
+
artifact = Artifact._from_id(val["id"], public_api.client)
|
1300
1295
|
return self.use_artifact(artifact, use_as=key)
|
1301
1296
|
elif _is_artifact_string(val):
|
1302
1297
|
# this will never fail, but is required to make mypy happy
|
@@ -1309,12 +1304,11 @@ class Run:
|
|
1309
1304
|
else:
|
1310
1305
|
public_api = self._public_api()
|
1311
1306
|
if is_id:
|
1312
|
-
artifact =
|
1307
|
+
artifact = Artifact._from_id(artifact_string, public_api._client)
|
1313
1308
|
else:
|
1314
1309
|
artifact = public_api.artifact(name=artifact_string)
|
1315
1310
|
# in the future we'll need to support using artifacts from
|
1316
|
-
# different instances of wandb.
|
1317
|
-
# likely to convert the retrieved public.Artifact to a wandb.Artifact
|
1311
|
+
# different instances of wandb.
|
1318
1312
|
|
1319
1313
|
return self.use_artifact(artifact, use_as=key)
|
1320
1314
|
elif _is_artifact_object(val):
|
@@ -1457,6 +1451,9 @@ class Run:
|
|
1457
1451
|
|
1458
1452
|
def _set_run_obj(self, run_obj: RunRecord) -> None:
|
1459
1453
|
self._run_obj = run_obj
|
1454
|
+
if self.settings._offline:
|
1455
|
+
return
|
1456
|
+
|
1460
1457
|
self._entity = run_obj.entity
|
1461
1458
|
self._project = run_obj.project
|
1462
1459
|
|
@@ -1485,9 +1482,6 @@ class Run:
|
|
1485
1482
|
settings=self._settings,
|
1486
1483
|
)
|
1487
1484
|
|
1488
|
-
def _set_run_obj_offline(self, run_obj: RunRecord) -> None:
|
1489
|
-
self._run_obj_offline = run_obj
|
1490
|
-
|
1491
1485
|
def _add_singleton(
|
1492
1486
|
self, data_type: str, key: str, value: Dict[Union[int, str], str]
|
1493
1487
|
) -> None:
|
@@ -2173,6 +2167,8 @@ class Run:
|
|
2173
2167
|
self._output_writer = None
|
2174
2168
|
|
2175
2169
|
def _on_init(self) -> None:
|
2170
|
+
if self._settings._offline:
|
2171
|
+
return
|
2176
2172
|
if self._backend and self._backend.interface:
|
2177
2173
|
logger.info("communicating current version")
|
2178
2174
|
version_handle = self._backend.interface.deliver_check_version(
|
@@ -2591,7 +2587,7 @@ class Run:
|
|
2591
2587
|
@_run_decorator._attach
|
2592
2588
|
def link_artifact(
|
2593
2589
|
self,
|
2594
|
-
artifact:
|
2590
|
+
artifact: Artifact,
|
2595
2591
|
target_path: str,
|
2596
2592
|
aliases: Optional[List[str]] = None,
|
2597
2593
|
) -> None:
|
@@ -2616,7 +2612,7 @@ class Run:
|
|
2616
2612
|
aliases = []
|
2617
2613
|
|
2618
2614
|
if self._backend and self._backend.interface:
|
2619
|
-
if
|
2615
|
+
if artifact.is_draft() and not artifact._is_draft_save_started():
|
2620
2616
|
artifact = self._log_artifact(artifact)
|
2621
2617
|
if not self._settings._offline:
|
2622
2618
|
self._backend.interface.publish_link_artifact(
|
@@ -2635,11 +2631,11 @@ class Run:
|
|
2635
2631
|
@_run_decorator._attach
|
2636
2632
|
def use_artifact(
|
2637
2633
|
self,
|
2638
|
-
artifact_or_name: Union[str,
|
2634
|
+
artifact_or_name: Union[str, Artifact],
|
2639
2635
|
type: Optional[str] = None,
|
2640
2636
|
aliases: Optional[List[str]] = None,
|
2641
2637
|
use_as: Optional[str] = None,
|
2642
|
-
) ->
|
2638
|
+
) -> Artifact:
|
2643
2639
|
"""Declare an artifact as an input to a run.
|
2644
2640
|
|
2645
2641
|
Call `download` or `file` on the returned object to get the contents locally.
|
@@ -2704,10 +2700,10 @@ class Run:
|
|
2704
2700
|
aliases = []
|
2705
2701
|
elif isinstance(aliases, str):
|
2706
2702
|
aliases = [aliases]
|
2707
|
-
if isinstance(artifact_or_name,
|
2703
|
+
if isinstance(artifact_or_name, Artifact) and artifact.is_draft():
|
2708
2704
|
if use_as is not None:
|
2709
2705
|
wandb.termwarn(
|
2710
|
-
"Indicating use_as is not supported when using
|
2706
|
+
"Indicating use_as is not supported when using a draft artifact"
|
2711
2707
|
)
|
2712
2708
|
self._log_artifact(
|
2713
2709
|
artifact,
|
@@ -2717,13 +2713,13 @@ class Run:
|
|
2717
2713
|
)
|
2718
2714
|
artifact.wait()
|
2719
2715
|
artifact._use_as = use_as or artifact.name
|
2720
|
-
elif isinstance(artifact,
|
2716
|
+
elif isinstance(artifact, Artifact) and not artifact.is_draft():
|
2721
2717
|
if (
|
2722
2718
|
self._launch_artifact_mapping
|
2723
2719
|
and artifact.name in self._launch_artifact_mapping.keys()
|
2724
2720
|
):
|
2725
2721
|
wandb.termwarn(
|
2726
|
-
"Swapping artifacts is not supported when using
|
2722
|
+
"Swapping artifacts is not supported when using a non-draft artifact. "
|
2727
2723
|
f"Using {artifact.name}."
|
2728
2724
|
)
|
2729
2725
|
artifact._use_as = use_as or artifact.name
|
@@ -2743,11 +2739,11 @@ class Run:
|
|
2743
2739
|
@_run_decorator._attach
|
2744
2740
|
def log_artifact(
|
2745
2741
|
self,
|
2746
|
-
artifact_or_path: Union[
|
2742
|
+
artifact_or_path: Union[Artifact, StrPath],
|
2747
2743
|
name: Optional[str] = None,
|
2748
2744
|
type: Optional[str] = None,
|
2749
2745
|
aliases: Optional[List[str]] = None,
|
2750
|
-
) ->
|
2746
|
+
) -> Artifact:
|
2751
2747
|
"""Declare an artifact as an output of a run.
|
2752
2748
|
|
2753
2749
|
Arguments:
|
@@ -2780,12 +2776,12 @@ class Run:
|
|
2780
2776
|
@_run_decorator._attach
|
2781
2777
|
def upsert_artifact(
|
2782
2778
|
self,
|
2783
|
-
artifact_or_path: Union[
|
2779
|
+
artifact_or_path: Union[Artifact, str],
|
2784
2780
|
name: Optional[str] = None,
|
2785
2781
|
type: Optional[str] = None,
|
2786
2782
|
aliases: Optional[List[str]] = None,
|
2787
2783
|
distributed_id: Optional[str] = None,
|
2788
|
-
) ->
|
2784
|
+
) -> Artifact:
|
2789
2785
|
"""Declare (or append to) a non-finalized artifact as output of a run.
|
2790
2786
|
|
2791
2787
|
Note that you must call run.finish_artifact() to finalize the artifact.
|
@@ -2834,12 +2830,12 @@ class Run:
|
|
2834
2830
|
@_run_decorator._attach
|
2835
2831
|
def finish_artifact(
|
2836
2832
|
self,
|
2837
|
-
artifact_or_path: Union[
|
2833
|
+
artifact_or_path: Union[Artifact, str],
|
2838
2834
|
name: Optional[str] = None,
|
2839
2835
|
type: Optional[str] = None,
|
2840
2836
|
aliases: Optional[List[str]] = None,
|
2841
2837
|
distributed_id: Optional[str] = None,
|
2842
|
-
) ->
|
2838
|
+
) -> Artifact:
|
2843
2839
|
"""Finishes a non-finalized artifact as output of a run.
|
2844
2840
|
|
2845
2841
|
Subsequent "upserts" with the same distributed ID will result in a new version.
|
@@ -2886,7 +2882,7 @@ class Run:
|
|
2886
2882
|
|
2887
2883
|
def _log_artifact(
|
2888
2884
|
self,
|
2889
|
-
artifact_or_path: Union[
|
2885
|
+
artifact_or_path: Union[Artifact, StrPath],
|
2890
2886
|
name: Optional[str] = None,
|
2891
2887
|
type: Optional[str] = None,
|
2892
2888
|
aliases: Optional[List[str]] = None,
|
@@ -2894,7 +2890,7 @@ class Run:
|
|
2894
2890
|
finalize: bool = True,
|
2895
2891
|
is_user_created: bool = False,
|
2896
2892
|
use_after_commit: bool = False,
|
2897
|
-
) ->
|
2893
|
+
) -> Artifact:
|
2898
2894
|
api = internal.Api()
|
2899
2895
|
if api.settings().get("anonymous") == "true":
|
2900
2896
|
wandb.termwarn(
|
@@ -2923,7 +2919,7 @@ class Run:
|
|
2923
2919
|
is_user_created=is_user_created,
|
2924
2920
|
use_after_commit=use_after_commit,
|
2925
2921
|
)
|
2926
|
-
artifact.
|
2922
|
+
artifact._set_save_future(future, self._public_api().client)
|
2927
2923
|
else:
|
2928
2924
|
self._backend.interface.publish_artifact(
|
2929
2925
|
self,
|
@@ -2946,10 +2942,9 @@ class Run:
|
|
2946
2942
|
|
2947
2943
|
def _public_api(self, overrides: Optional[Dict[str, str]] = None) -> PublicApi:
|
2948
2944
|
overrides = {"run": self._run_id}
|
2949
|
-
|
2950
|
-
|
2951
|
-
overrides["
|
2952
|
-
overrides["project"] = run_obj.project
|
2945
|
+
if not (self._settings._offline or self._run_obj is None):
|
2946
|
+
overrides["entity"] = self._run_obj.entity
|
2947
|
+
overrides["project"] = self._run_obj.project
|
2953
2948
|
return public.Api(overrides)
|
2954
2949
|
|
2955
2950
|
# TODO(jhr): annotate this
|
@@ -2957,11 +2952,11 @@ class Run:
|
|
2957
2952
|
if not self._settings._offline:
|
2958
2953
|
try:
|
2959
2954
|
public_api = self._public_api()
|
2960
|
-
expected_type =
|
2961
|
-
public_api.client,
|
2962
|
-
artifact.name,
|
2955
|
+
expected_type = Artifact._expected_type(
|
2963
2956
|
public_api.settings["entity"],
|
2964
2957
|
public_api.settings["project"],
|
2958
|
+
artifact.name,
|
2959
|
+
public_api.client,
|
2965
2960
|
)
|
2966
2961
|
except requests.exceptions.RequestException:
|
2967
2962
|
# Just return early if there is a network error. This is
|
@@ -2976,11 +2971,11 @@ class Run:
|
|
2976
2971
|
|
2977
2972
|
def _prepare_artifact(
|
2978
2973
|
self,
|
2979
|
-
artifact_or_path: Union[
|
2974
|
+
artifact_or_path: Union[Artifact, StrPath],
|
2980
2975
|
name: Optional[str] = None,
|
2981
2976
|
type: Optional[str] = None,
|
2982
2977
|
aliases: Optional[List[str]] = None,
|
2983
|
-
) -> Tuple[
|
2978
|
+
) -> Tuple[Artifact, List[str]]:
|
2984
2979
|
if isinstance(artifact_or_path, (str, os.PathLike)):
|
2985
2980
|
name = name or f"run-{self._run_id}-{os.path.basename(artifact_or_path)}"
|
2986
2981
|
artifact = wandb.Artifact(name, type or "unspecified")
|
@@ -3703,160 +3698,3 @@ def finish(exit_code: Optional[int] = None, quiet: Optional[bool] = None) -> Non
|
|
3703
3698
|
"""
|
3704
3699
|
if wandb.run:
|
3705
3700
|
wandb.run.finish(exit_code=exit_code, quiet=quiet)
|
3706
|
-
|
3707
|
-
|
3708
|
-
class InvalidArtifact:
|
3709
|
-
"""An "artifact" that raises an error when any properties are accessed."""
|
3710
|
-
|
3711
|
-
def __init__(self, base_artifact: "ArtifactInterface"):
|
3712
|
-
super().__setattr__("base_artifact", base_artifact)
|
3713
|
-
|
3714
|
-
def __getattr__(self, __name: str) -> Any:
|
3715
|
-
raise ArtifactNotLoggedError(artifact=self.base_artifact, attr=__name)
|
3716
|
-
|
3717
|
-
def __setattr__(self, __name: str, __value: Any) -> None:
|
3718
|
-
raise ArtifactNotLoggedError(artifact=self.base_artifact, attr=__name)
|
3719
|
-
|
3720
|
-
def __bool__(self) -> bool:
|
3721
|
-
return False
|
3722
|
-
|
3723
|
-
|
3724
|
-
class WaitTimeoutError(errors.Error):
|
3725
|
-
"""Raised when wait() timeout occurs before process is finished."""
|
3726
|
-
|
3727
|
-
|
3728
|
-
class _LazyArtifact(ArtifactInterface):
|
3729
|
-
_api: PublicApi
|
3730
|
-
_instance: Union[ArtifactInterface, InvalidArtifact]
|
3731
|
-
_future: Any
|
3732
|
-
|
3733
|
-
def __init__(self, api: PublicApi, future: Any):
|
3734
|
-
self._api = api
|
3735
|
-
self._instance = InvalidArtifact(self)
|
3736
|
-
self._future = future
|
3737
|
-
|
3738
|
-
def __getattr__(self, item: str) -> Any:
|
3739
|
-
return getattr(self._instance, item)
|
3740
|
-
|
3741
|
-
def wait(self, timeout: Optional[int] = None) -> ArtifactInterface:
|
3742
|
-
if not self._instance:
|
3743
|
-
future_get = self._future.get(timeout)
|
3744
|
-
if not future_get:
|
3745
|
-
raise WaitTimeoutError(
|
3746
|
-
"Artifact upload wait timed out, failed to fetch Artifact response"
|
3747
|
-
)
|
3748
|
-
resp = future_get.response.log_artifact_response
|
3749
|
-
if resp.error_message:
|
3750
|
-
raise ValueError(resp.error_message)
|
3751
|
-
self._instance = public.Artifact.from_id(resp.artifact_id, self._api.client)
|
3752
|
-
assert isinstance(
|
3753
|
-
self._instance, ArtifactInterface
|
3754
|
-
), "Insufficient permissions to fetch Artifact with id {} from {}".format(
|
3755
|
-
resp.artifact_id, self._api.client.app_url
|
3756
|
-
)
|
3757
|
-
return self._instance
|
3758
|
-
|
3759
|
-
@property
|
3760
|
-
def id(self) -> Optional[str]:
|
3761
|
-
return self._instance.id
|
3762
|
-
|
3763
|
-
@property
|
3764
|
-
def source_version(self) -> Optional[str]:
|
3765
|
-
return self._instance.source_version
|
3766
|
-
|
3767
|
-
@property
|
3768
|
-
def version(self) -> str:
|
3769
|
-
return self._instance.version
|
3770
|
-
|
3771
|
-
@property
|
3772
|
-
def name(self) -> str:
|
3773
|
-
return self._instance.name
|
3774
|
-
|
3775
|
-
@property
|
3776
|
-
def full_name(self) -> str:
|
3777
|
-
return self._instance.full_name
|
3778
|
-
|
3779
|
-
@property
|
3780
|
-
def type(self) -> str:
|
3781
|
-
return self._instance.type
|
3782
|
-
|
3783
|
-
@property
|
3784
|
-
def entity(self) -> str:
|
3785
|
-
return self._instance.entity
|
3786
|
-
|
3787
|
-
@property
|
3788
|
-
def project(self) -> str:
|
3789
|
-
return self._instance.project
|
3790
|
-
|
3791
|
-
@property
|
3792
|
-
def manifest(self) -> "ArtifactManifest":
|
3793
|
-
return self._instance.manifest
|
3794
|
-
|
3795
|
-
@property
|
3796
|
-
def digest(self) -> str:
|
3797
|
-
return self._instance.digest
|
3798
|
-
|
3799
|
-
@property
|
3800
|
-
def state(self) -> str:
|
3801
|
-
return self._instance.state
|
3802
|
-
|
3803
|
-
@property
|
3804
|
-
def size(self) -> int:
|
3805
|
-
return self._instance.size
|
3806
|
-
|
3807
|
-
@property
|
3808
|
-
def commit_hash(self) -> str:
|
3809
|
-
return self._instance.commit_hash
|
3810
|
-
|
3811
|
-
@property
|
3812
|
-
def description(self) -> Optional[str]:
|
3813
|
-
return self._instance.description
|
3814
|
-
|
3815
|
-
@description.setter
|
3816
|
-
def description(self, desc: Optional[str]) -> None:
|
3817
|
-
self._instance.description = desc
|
3818
|
-
|
3819
|
-
@property
|
3820
|
-
def metadata(self) -> dict:
|
3821
|
-
return self._instance.metadata
|
3822
|
-
|
3823
|
-
@metadata.setter
|
3824
|
-
def metadata(self, metadata: dict) -> None:
|
3825
|
-
self._instance.metadata = metadata
|
3826
|
-
|
3827
|
-
@property
|
3828
|
-
def aliases(self) -> List[str]:
|
3829
|
-
return self._instance.aliases
|
3830
|
-
|
3831
|
-
@aliases.setter
|
3832
|
-
def aliases(self, aliases: List[str]) -> None:
|
3833
|
-
self._instance.aliases = aliases
|
3834
|
-
|
3835
|
-
def used_by(self) -> List["wandb.apis.public.Run"]:
|
3836
|
-
return self._instance.used_by()
|
3837
|
-
|
3838
|
-
def logged_by(self) -> "wandb.apis.public.Run":
|
3839
|
-
return self._instance.logged_by()
|
3840
|
-
|
3841
|
-
def get_path(self, name: str) -> "ArtifactManifestEntry":
|
3842
|
-
return self._instance.get_path(name)
|
3843
|
-
|
3844
|
-
def get(self, name: str) -> "WBValue":
|
3845
|
-
return self._instance.get(name)
|
3846
|
-
|
3847
|
-
def download(
|
3848
|
-
self, root: Optional[str] = None, recursive: bool = False
|
3849
|
-
) -> FilePathStr:
|
3850
|
-
return self._instance.download(root, recursive)
|
3851
|
-
|
3852
|
-
def checkout(self, root: Optional[str] = None) -> str:
|
3853
|
-
return self._instance.checkout(root)
|
3854
|
-
|
3855
|
-
def verify(self, root: Optional[str] = None) -> Any:
|
3856
|
-
return self._instance.verify(root)
|
3857
|
-
|
3858
|
-
def save(self) -> None:
|
3859
|
-
self._instance.save()
|
3860
|
-
|
3861
|
-
def delete(self) -> None:
|
3862
|
-
self._instance.delete()
|
wandb/sdk/wandb_settings.py
CHANGED
@@ -37,13 +37,14 @@ import wandb.env
|
|
37
37
|
from wandb import util
|
38
38
|
from wandb.apis.internal import Api
|
39
39
|
from wandb.errors import UsageError
|
40
|
+
from wandb.sdk.internal.system.env_probe_helpers import is_aws_lambda
|
40
41
|
from wandb.sdk.lib import filesystem
|
41
42
|
from wandb.sdk.lib._settings_toposort_generated import SETTINGS_TOPOLOGICALLY_SORTED
|
42
43
|
from wandb.sdk.wandb_config import Config
|
43
44
|
from wandb.sdk.wandb_setup import _EarlyLogger
|
44
45
|
|
45
46
|
from .lib import apikey
|
46
|
-
from .lib.
|
47
|
+
from .lib.gitlib import GitRepo
|
47
48
|
from .lib.ipython import _get_python_type
|
48
49
|
from .lib.runid import generate_id
|
49
50
|
|
@@ -449,16 +450,18 @@ class Settings:
|
|
449
450
|
# settings are declared as class attributes for static type checking purposes
|
450
451
|
# and to help with IDE autocomplete.
|
451
452
|
_args: Sequence[str]
|
453
|
+
_aws_lambda: bool
|
452
454
|
_async_upload_concurrency_limit: int
|
453
455
|
_cli_only_mode: bool # Avoid running any code specific for runs
|
454
456
|
_colab: bool
|
455
457
|
_config_dict: Config
|
456
458
|
_console: SettingsConsole
|
457
459
|
_cuda: str
|
458
|
-
_disable_meta: bool
|
459
|
-
_disable_service: bool
|
460
|
-
_disable_stats: bool
|
460
|
+
_disable_meta: bool # Do not collect system metadata
|
461
|
+
_disable_service: bool # Disable wandb-service, spin up internal process the old way
|
462
|
+
_disable_stats: bool # Do not collect system metrics
|
461
463
|
_disable_viewer: bool # Prevent early viewer query
|
464
|
+
_disable_setproctitle: bool # Do not use setproctitle on internal process
|
462
465
|
_except_exit: bool
|
463
466
|
_executable: str
|
464
467
|
_extra_http_headers: Mapping[str, str]
|
@@ -484,6 +487,7 @@ class Settings:
|
|
484
487
|
_platform: str
|
485
488
|
_python: str
|
486
489
|
_runqueue_item_id: str
|
490
|
+
_require_nexus: bool
|
487
491
|
_save_requirements: bool
|
488
492
|
_service_transport: str
|
489
493
|
_service_wait: float
|
@@ -506,6 +510,7 @@ class Settings:
|
|
506
510
|
allow_val_change: bool
|
507
511
|
anonymous: str
|
508
512
|
api_key: str
|
513
|
+
azure_account_url_to_access_key: Dict[str, str]
|
509
514
|
base_url: str # The base url for the wandb api
|
510
515
|
code_dir: str
|
511
516
|
config_paths: Sequence[str]
|
@@ -530,6 +535,7 @@ class Settings:
|
|
530
535
|
ignore_globs: Tuple[str]
|
531
536
|
init_timeout: float
|
532
537
|
is_local: bool
|
538
|
+
job_source: str
|
533
539
|
label_disable: bool
|
534
540
|
launch: bool
|
535
541
|
launch_config_path: str
|
@@ -599,6 +605,10 @@ class Settings:
|
|
599
605
|
Note that key names must be the same as the class attribute names.
|
600
606
|
"""
|
601
607
|
props: Dict[str, Dict[str, Any]] = dict(
|
608
|
+
_aws_lambda={
|
609
|
+
"hook": lambda _: is_aws_lambda(),
|
610
|
+
"auto_hook": True,
|
611
|
+
},
|
602
612
|
_async_upload_concurrency_limit={
|
603
613
|
"preprocessor": int,
|
604
614
|
"validator": self._validate__async_upload_concurrency_limit,
|
@@ -609,6 +619,7 @@ class Settings:
|
|
609
619
|
"preprocessor": _str_as_bool,
|
610
620
|
"is_policy": True,
|
611
621
|
},
|
622
|
+
_disable_setproctitle={"value": False, "preprocessor": _str_as_bool},
|
612
623
|
_disable_stats={"preprocessor": _str_as_bool},
|
613
624
|
_disable_viewer={"preprocessor": _str_as_bool},
|
614
625
|
_extra_http_headers={"preprocessor": _str_as_json},
|
@@ -655,6 +666,7 @@ class Settings:
|
|
655
666
|
},
|
656
667
|
_sync={"value": False},
|
657
668
|
_platform={"value": util.get_platform_name()},
|
669
|
+
_require_nexus={"value": False, "preprocessor": _str_as_bool},
|
658
670
|
_save_requirements={"value": True, "preprocessor": _str_as_bool},
|
659
671
|
_service_wait={
|
660
672
|
"value": 30,
|
@@ -693,6 +705,7 @@ class Settings:
|
|
693
705
|
},
|
694
706
|
anonymous={"validator": self._validate_anonymous},
|
695
707
|
api_key={"validator": self._validate_api_key},
|
708
|
+
azure_account_url_to_access_key={"value": {}},
|
696
709
|
base_url={
|
697
710
|
"value": "https://api.wandb.ai",
|
698
711
|
"preprocessor": lambda x: str(x).strip().rstrip("/"),
|
@@ -731,6 +744,7 @@ class Settings:
|
|
731
744
|
),
|
732
745
|
"auto_hook": True,
|
733
746
|
},
|
747
|
+
job_source={"validator": self._validate_job_source},
|
734
748
|
label_disable={"preprocessor": _str_as_bool},
|
735
749
|
launch={"preprocessor": _str_as_bool},
|
736
750
|
log_dir={
|
@@ -1113,6 +1127,15 @@ class Settings:
|
|
1113
1127
|
|
1114
1128
|
return True
|
1115
1129
|
|
1130
|
+
@staticmethod
|
1131
|
+
def _validate_job_source(value: str) -> bool:
|
1132
|
+
valid_sources = ["repo", "artifact", "image"]
|
1133
|
+
if value not in valid_sources:
|
1134
|
+
raise UsageError(
|
1135
|
+
f"Settings field `job_source`: {value!r} not in {valid_sources}"
|
1136
|
+
)
|
1137
|
+
return True
|
1138
|
+
|
1116
1139
|
# other helper methods
|
1117
1140
|
@staticmethod
|
1118
1141
|
def _path_convert(*args: str) -> str:
|
@@ -1542,6 +1565,7 @@ class Settings:
|
|
1542
1565
|
"WANDB_TRACELOG": "_tracelog",
|
1543
1566
|
"WANDB_DISABLE_SERVICE": "_disable_service",
|
1544
1567
|
"WANDB_SERVICE_TRANSPORT": "_service_transport",
|
1568
|
+
"WANDB_REQUIRE_NEXUS": "_require_nexus",
|
1545
1569
|
"WANDB_DIR": "root_dir",
|
1546
1570
|
"WANDB_NAME": "run_name",
|
1547
1571
|
"WANDB_NOTES": "run_notes",
|