wandb 0.15.3__py3-none-any.whl → 0.15.5__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (156) hide show
  1. wandb/__init__.py +1 -1
  2. wandb/analytics/sentry.py +1 -0
  3. wandb/apis/importers/base.py +20 -5
  4. wandb/apis/importers/mlflow.py +7 -1
  5. wandb/apis/internal.py +12 -0
  6. wandb/apis/public.py +247 -1387
  7. wandb/apis/reports/_panels.py +58 -35
  8. wandb/beta/workflows.py +6 -7
  9. wandb/cli/cli.py +130 -60
  10. wandb/data_types.py +3 -1
  11. wandb/filesync/dir_watcher.py +21 -27
  12. wandb/filesync/step_checksum.py +8 -8
  13. wandb/filesync/step_prepare.py +23 -10
  14. wandb/filesync/step_upload.py +13 -13
  15. wandb/filesync/upload_job.py +4 -8
  16. wandb/integration/cohere/__init__.py +3 -0
  17. wandb/integration/cohere/cohere.py +21 -0
  18. wandb/integration/cohere/resolver.py +347 -0
  19. wandb/integration/gym/__init__.py +4 -6
  20. wandb/integration/huggingface/__init__.py +3 -0
  21. wandb/integration/huggingface/huggingface.py +18 -0
  22. wandb/integration/huggingface/resolver.py +213 -0
  23. wandb/integration/langchain/wandb_tracer.py +16 -179
  24. wandb/integration/openai/__init__.py +1 -3
  25. wandb/integration/openai/openai.py +11 -143
  26. wandb/integration/openai/resolver.py +111 -38
  27. wandb/integration/sagemaker/config.py +2 -2
  28. wandb/integration/tensorboard/log.py +4 -4
  29. wandb/old/settings.py +24 -7
  30. wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
  31. wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
  32. wandb/proto/wandb_deprecated.py +3 -1
  33. wandb/sdk/__init__.py +1 -1
  34. wandb/sdk/artifacts/__init__.py +0 -0
  35. wandb/sdk/artifacts/artifact.py +2101 -0
  36. wandb/sdk/artifacts/artifact_download_logger.py +42 -0
  37. wandb/sdk/artifacts/artifact_manifest.py +67 -0
  38. wandb/sdk/artifacts/artifact_manifest_entry.py +159 -0
  39. wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
  40. wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +91 -0
  41. wandb/sdk/{internal → artifacts}/artifact_saver.py +6 -5
  42. wandb/sdk/artifacts/artifact_state.py +10 -0
  43. wandb/sdk/{interface/artifacts/artifact_cache.py → artifacts/artifacts_cache.py} +22 -12
  44. wandb/sdk/artifacts/exceptions.py +55 -0
  45. wandb/sdk/artifacts/storage_handler.py +59 -0
  46. wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
  47. wandb/sdk/artifacts/storage_handlers/azure_handler.py +192 -0
  48. wandb/sdk/artifacts/storage_handlers/gcs_handler.py +224 -0
  49. wandb/sdk/artifacts/storage_handlers/http_handler.py +112 -0
  50. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +134 -0
  51. wandb/sdk/artifacts/storage_handlers/multi_handler.py +53 -0
  52. wandb/sdk/artifacts/storage_handlers/s3_handler.py +301 -0
  53. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +67 -0
  54. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +132 -0
  55. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +72 -0
  56. wandb/sdk/artifacts/storage_layout.py +6 -0
  57. wandb/sdk/artifacts/storage_policies/__init__.py +0 -0
  58. wandb/sdk/artifacts/storage_policies/s3_bucket_policy.py +61 -0
  59. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +386 -0
  60. wandb/sdk/{interface/artifacts/artifact_storage.py → artifacts/storage_policy.py} +5 -57
  61. wandb/sdk/data_types/_dtypes.py +7 -12
  62. wandb/sdk/data_types/base_types/json_metadata.py +3 -2
  63. wandb/sdk/data_types/base_types/media.py +8 -8
  64. wandb/sdk/data_types/base_types/wb_value.py +12 -13
  65. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +5 -6
  66. wandb/sdk/data_types/helper_types/classes.py +6 -8
  67. wandb/sdk/data_types/helper_types/image_mask.py +5 -6
  68. wandb/sdk/data_types/histogram.py +4 -3
  69. wandb/sdk/data_types/html.py +3 -4
  70. wandb/sdk/data_types/image.py +11 -9
  71. wandb/sdk/data_types/molecule.py +5 -3
  72. wandb/sdk/data_types/object_3d.py +7 -5
  73. wandb/sdk/data_types/plotly.py +3 -2
  74. wandb/sdk/data_types/saved_model.py +11 -11
  75. wandb/sdk/data_types/trace_tree.py +5 -4
  76. wandb/sdk/data_types/utils.py +3 -5
  77. wandb/sdk/data_types/video.py +5 -4
  78. wandb/sdk/integration_utils/auto_logging.py +215 -0
  79. wandb/sdk/interface/interface.py +15 -15
  80. wandb/sdk/internal/file_pusher.py +8 -16
  81. wandb/sdk/internal/file_stream.py +5 -11
  82. wandb/sdk/internal/handler.py +13 -1
  83. wandb/sdk/internal/internal_api.py +287 -13
  84. wandb/sdk/internal/job_builder.py +119 -30
  85. wandb/sdk/internal/sender.py +6 -26
  86. wandb/sdk/internal/settings_static.py +2 -0
  87. wandb/sdk/internal/system/assets/__init__.py +2 -0
  88. wandb/sdk/internal/system/assets/gpu.py +42 -0
  89. wandb/sdk/internal/system/assets/gpu_amd.py +216 -0
  90. wandb/sdk/internal/system/env_probe_helpers.py +13 -0
  91. wandb/sdk/internal/system/system_info.py +3 -3
  92. wandb/sdk/internal/tb_watcher.py +32 -22
  93. wandb/sdk/internal/thread_local_settings.py +18 -0
  94. wandb/sdk/launch/_project_spec.py +57 -11
  95. wandb/sdk/launch/agent/agent.py +147 -65
  96. wandb/sdk/launch/agent/job_status_tracker.py +34 -0
  97. wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
  98. wandb/sdk/launch/builder/abstract.py +5 -1
  99. wandb/sdk/launch/builder/build.py +21 -18
  100. wandb/sdk/launch/builder/docker_builder.py +10 -4
  101. wandb/sdk/launch/builder/kaniko_builder.py +113 -23
  102. wandb/sdk/launch/builder/noop.py +6 -3
  103. wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +46 -14
  104. wandb/sdk/launch/environment/aws_environment.py +3 -2
  105. wandb/sdk/launch/environment/azure_environment.py +124 -0
  106. wandb/sdk/launch/environment/gcp_environment.py +2 -4
  107. wandb/sdk/launch/environment/local_environment.py +1 -1
  108. wandb/sdk/launch/errors.py +19 -0
  109. wandb/sdk/launch/github_reference.py +32 -19
  110. wandb/sdk/launch/launch.py +3 -8
  111. wandb/sdk/launch/launch_add.py +6 -2
  112. wandb/sdk/launch/loader.py +21 -2
  113. wandb/sdk/launch/registry/azure_container_registry.py +132 -0
  114. wandb/sdk/launch/registry/elastic_container_registry.py +39 -5
  115. wandb/sdk/launch/registry/google_artifact_registry.py +68 -26
  116. wandb/sdk/launch/registry/local_registry.py +2 -1
  117. wandb/sdk/launch/runner/abstract.py +24 -3
  118. wandb/sdk/launch/runner/kubernetes_runner.py +479 -26
  119. wandb/sdk/launch/runner/local_container.py +103 -51
  120. wandb/sdk/launch/runner/local_process.py +1 -1
  121. wandb/sdk/launch/runner/sagemaker_runner.py +60 -10
  122. wandb/sdk/launch/runner/vertex_runner.py +10 -5
  123. wandb/sdk/launch/sweeps/__init__.py +7 -9
  124. wandb/sdk/launch/sweeps/scheduler.py +307 -77
  125. wandb/sdk/launch/sweeps/scheduler_sweep.py +2 -1
  126. wandb/sdk/launch/sweeps/utils.py +82 -35
  127. wandb/sdk/launch/utils.py +89 -75
  128. wandb/sdk/lib/_settings_toposort_generated.py +7 -0
  129. wandb/sdk/lib/capped_dict.py +26 -0
  130. wandb/sdk/lib/{git.py → gitlib.py} +76 -59
  131. wandb/sdk/lib/hashutil.py +12 -4
  132. wandb/sdk/lib/paths.py +96 -8
  133. wandb/sdk/lib/sock_client.py +2 -2
  134. wandb/sdk/lib/timer.py +1 -0
  135. wandb/sdk/service/server.py +22 -9
  136. wandb/sdk/service/server_sock.py +1 -1
  137. wandb/sdk/service/service.py +27 -8
  138. wandb/sdk/verify/verify.py +4 -7
  139. wandb/sdk/wandb_config.py +2 -6
  140. wandb/sdk/wandb_init.py +57 -53
  141. wandb/sdk/wandb_require.py +7 -0
  142. wandb/sdk/wandb_run.py +61 -223
  143. wandb/sdk/wandb_settings.py +28 -4
  144. wandb/testing/relay.py +15 -2
  145. wandb/util.py +74 -36
  146. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/METADATA +15 -9
  147. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/RECORD +151 -116
  148. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/entry_points.txt +1 -0
  149. wandb/integration/langchain/util.py +0 -191
  150. wandb/sdk/interface/artifacts/__init__.py +0 -33
  151. wandb/sdk/interface/artifacts/artifact.py +0 -615
  152. wandb/sdk/interface/artifacts/artifact_manifest.py +0 -131
  153. wandb/sdk/wandb_artifacts.py +0 -2226
  154. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/LICENSE +0 -0
  155. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/WHEEL +0 -0
  156. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/top_level.txt +0 -0
wandb/sdk/wandb_run.py CHANGED
@@ -47,12 +47,13 @@ from wandb.proto.wandb_internal_pb2 import (
47
47
  RunRecord,
48
48
  ServerInfoResponse,
49
49
  )
50
+ from wandb.sdk.artifacts.artifact import Artifact
50
51
  from wandb.sdk.internal import job_builder
51
52
  from wandb.sdk.lib.import_hooks import (
52
53
  register_post_import_hook,
53
54
  unregister_post_import_hook,
54
55
  )
55
- from wandb.sdk.lib.paths import FilePathStr, StrPath
56
+ from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath
56
57
  from wandb.util import (
57
58
  _is_artifact_object,
58
59
  _is_artifact_string,
@@ -61,14 +62,11 @@ from wandb.util import (
61
62
  _resolve_aliases,
62
63
  add_import_hook,
63
64
  parse_artifact_string,
64
- to_forward_slash_path,
65
65
  )
66
66
  from wandb.viz import CustomChart, Visualize, custom_chart
67
67
 
68
- from . import wandb_artifacts, wandb_config, wandb_metric, wandb_summary
68
+ from . import wandb_config, wandb_metric, wandb_summary
69
69
  from .data_types._dtypes import TypeRegistry
70
- from .interface.artifacts import Artifact as ArtifactInterface
71
- from .interface.artifacts import ArtifactNotLoggedError
72
70
  from .interface.interface import GlobStr, InterfaceBase
73
71
  from .interface.summary_record import SummaryRecord
74
72
  from .lib import (
@@ -83,13 +81,12 @@ from .lib import (
83
81
  telemetry,
84
82
  )
85
83
  from .lib.exit_hooks import ExitHooks
86
- from .lib.git import GitRepo
84
+ from .lib.gitlib import GitRepo
87
85
  from .lib.mailbox import MailboxError, MailboxHandle, MailboxProbe, MailboxProgress
88
86
  from .lib.printer import get_printer
89
87
  from .lib.proto_util import message_to_dict
90
88
  from .lib.reporting import Reporter
91
89
  from .lib.wburls import wburls
92
- from .wandb_artifacts import Artifact
93
90
  from .wandb_settings import Settings, SettingsConsole
94
91
  from .wandb_setup import _WandbSetup
95
92
 
@@ -109,8 +106,6 @@ if TYPE_CHECKING:
109
106
  SampledHistoryResponse,
110
107
  )
111
108
 
112
- from .data_types.base_types.wb_value import WBValue
113
- from .interface.artifacts import ArtifactManifest, ArtifactManifestEntry
114
109
  from .interface.interface import FilesDict, PolicyName
115
110
  from .lib.printer import PrinterJupyter, PrinterTerm
116
111
  from .wandb_alerts import AlertLevel
@@ -481,7 +476,6 @@ class Run:
481
476
  _notes: Optional[str]
482
477
 
483
478
  _run_obj: Optional[RunRecord]
484
- _run_obj_offline: Optional[RunRecord]
485
479
  # Use string literal annotation because of type reference loop
486
480
  _backend: Optional["wandb.sdk.backend.backend.Backend"]
487
481
  _internal_run_interface: Optional[
@@ -604,7 +598,6 @@ class Run:
604
598
 
605
599
  # Returned from backend request_run(), set from wandb_init?
606
600
  self._run_obj = None
607
- self._run_obj_offline = None
608
601
 
609
602
  # Created when the run "starts".
610
603
  self._run_status_checker = None
@@ -641,7 +634,7 @@ class Run:
641
634
  self._launch_artifact_mapping: Dict[str, Any] = {}
642
635
  self._unique_launch_artifact_sequence_names: Dict[str, Any] = {}
643
636
  if self._settings.save_code and self._settings.program_relpath:
644
- config[wandb_key]["code_path"] = to_forward_slash_path(
637
+ config[wandb_key]["code_path"] = LogicalPath(
645
638
  os.path.join("code", self._settings.program_relpath)
646
639
  )
647
640
  if sweep_config:
@@ -883,8 +876,7 @@ class Run:
883
876
  tel.feature.set_run_name = True
884
877
  self._name = name
885
878
  if self._backend and self._backend.interface:
886
- run = self._backend.interface._make_run(self)
887
- self._backend.interface.publish_run(run)
879
+ self._backend.interface.publish_run(self)
888
880
 
889
881
  @property
890
882
  @_run_decorator._attach
@@ -905,8 +897,7 @@ class Run:
905
897
  def notes(self, notes: str) -> None:
906
898
  self._notes = notes
907
899
  if self._backend and self._backend.interface:
908
- run = self._backend.interface._make_run(self)
909
- self._backend.interface.publish_run(run)
900
+ self._backend.interface.publish_run(self)
910
901
 
911
902
  @property
912
903
  @_run_decorator._attach
@@ -914,9 +905,8 @@ class Run:
914
905
  """Tags associated with the run, if there are any."""
915
906
  if self._tags:
916
907
  return self._tags
917
- run_obj = self._run_obj or self._run_obj_offline
918
- if run_obj:
919
- return tuple(run_obj.tags)
908
+ if self._run_obj:
909
+ return tuple(self._run_obj.tags)
920
910
  return None
921
911
 
922
912
  @tags.setter
@@ -926,8 +916,7 @@ class Run:
926
916
  tel.feature.set_run_tags = True
927
917
  self._tags = tuple(tags)
928
918
  if self._backend and self._backend.interface:
929
- run = self._backend.interface._make_run(self)
930
- self._backend.interface.publish_run(run)
919
+ self._backend.interface.publish_run(self)
931
920
 
932
921
  @property
933
922
  @_run_decorator._attach
@@ -999,8 +988,7 @@ class Run:
999
988
  return self._step
1000
989
 
1001
990
  def project_name(self) -> str:
1002
- run_obj = self._run_obj or self._run_obj_offline
1003
- return run_obj.project if run_obj else ""
991
+ return self._run_obj.project if self._run_obj else ""
1004
992
 
1005
993
  @property
1006
994
  @_run_decorator._attach
@@ -1026,8 +1014,7 @@ class Run:
1026
1014
  return self._settings._noop
1027
1015
 
1028
1016
  def _get_group(self) -> str:
1029
- run_obj = self._run_obj or self._run_obj_offline
1030
- return run_obj.run_group if run_obj else ""
1017
+ return self._run_obj.run_group if self._run_obj else ""
1031
1018
 
1032
1019
  @property
1033
1020
  @_run_decorator._attach
@@ -1038,7 +1025,7 @@ class Run:
1038
1025
 
1039
1026
  If you are doing a distributed training you should give all of the
1040
1027
  runs in the training the same group.
1041
- If you are doing crossvalidation you should give all the crossvalidation
1028
+ If you are doing cross-validation you should give all the cross-validation
1042
1029
  folds the same group.
1043
1030
  """
1044
1031
  return self._get_group()
@@ -1046,8 +1033,7 @@ class Run:
1046
1033
  @property
1047
1034
  @_run_decorator._attach
1048
1035
  def job_type(self) -> str:
1049
- run_obj = self._run_obj or self._run_obj_offline
1050
- return run_obj.job_type if run_obj else ""
1036
+ return self._run_obj.job_type if self._run_obj else ""
1051
1037
 
1052
1038
  @property
1053
1039
  @_run_decorator._attach
@@ -1096,10 +1082,19 @@ class Run:
1096
1082
  An `Artifact` object if code was logged
1097
1083
  """
1098
1084
  if name is None:
1099
- name_string = wandb.util.make_artifact_name_safe(
1100
- f"{self._project}-{self._settings.program_relpath}"
1101
- )
1102
- name = f"source-{name_string}"
1085
+ if self.settings._jupyter:
1086
+ notebook_name = None
1087
+ if self.settings.notebook_name:
1088
+ notebook_name = self.settings.notebook_name
1089
+ elif self.settings._jupyter_path:
1090
+ if self.settings._jupyter_path.startswith("fileId="):
1091
+ notebook_name = self.settings._jupyter_name
1092
+ else:
1093
+ notebook_name = self.settings._jupyter_path
1094
+ name_string = f"{self._project}-{notebook_name}"
1095
+ else:
1096
+ name_string = f"{self._project}-{self._settings.program_relpath}"
1097
+ name = wandb.util.make_artifact_name_safe(f"source-{name_string}")
1103
1098
  art = wandb.Artifact(name, "code")
1104
1099
  files_added = False
1105
1100
  if root is not None:
@@ -1289,14 +1284,14 @@ class Run:
1289
1284
 
1290
1285
  def _config_artifact_callback(
1291
1286
  self, key: str, val: Union[str, Artifact, dict]
1292
- ) -> Union[Artifact, public.Artifact]:
1287
+ ) -> Artifact:
1293
1288
  # artifacts can look like dicts as they are passed into the run config
1294
1289
  # since the run config stores them on the backend as a dict with fields shown
1295
1290
  # in wandb.util.artifact_to_json
1296
1291
  if _is_artifact_version_weave_dict(val):
1297
1292
  assert isinstance(val, dict)
1298
1293
  public_api = self._public_api()
1299
- artifact = public.Artifact.from_id(val["id"], public_api.client)
1294
+ artifact = Artifact._from_id(val["id"], public_api.client)
1300
1295
  return self.use_artifact(artifact, use_as=key)
1301
1296
  elif _is_artifact_string(val):
1302
1297
  # this will never fail, but is required to make mypy happy
@@ -1309,12 +1304,11 @@ class Run:
1309
1304
  else:
1310
1305
  public_api = self._public_api()
1311
1306
  if is_id:
1312
- artifact = public.Artifact.from_id(artifact_string, public_api._client)
1307
+ artifact = Artifact._from_id(artifact_string, public_api._client)
1313
1308
  else:
1314
1309
  artifact = public_api.artifact(name=artifact_string)
1315
1310
  # in the future we'll need to support using artifacts from
1316
- # different instances of wandb. simplest way to do that is
1317
- # likely to convert the retrieved public.Artifact to a wandb.Artifact
1311
+ # different instances of wandb.
1318
1312
 
1319
1313
  return self.use_artifact(artifact, use_as=key)
1320
1314
  elif _is_artifact_object(val):
@@ -1457,6 +1451,9 @@ class Run:
1457
1451
 
1458
1452
  def _set_run_obj(self, run_obj: RunRecord) -> None:
1459
1453
  self._run_obj = run_obj
1454
+ if self.settings._offline:
1455
+ return
1456
+
1460
1457
  self._entity = run_obj.entity
1461
1458
  self._project = run_obj.project
1462
1459
 
@@ -1485,9 +1482,6 @@ class Run:
1485
1482
  settings=self._settings,
1486
1483
  )
1487
1484
 
1488
- def _set_run_obj_offline(self, run_obj: RunRecord) -> None:
1489
- self._run_obj_offline = run_obj
1490
-
1491
1485
  def _add_singleton(
1492
1486
  self, data_type: str, key: str, value: Dict[Union[int, str], str]
1493
1487
  ) -> None:
@@ -2173,6 +2167,8 @@ class Run:
2173
2167
  self._output_writer = None
2174
2168
 
2175
2169
  def _on_init(self) -> None:
2170
+ if self._settings._offline:
2171
+ return
2176
2172
  if self._backend and self._backend.interface:
2177
2173
  logger.info("communicating current version")
2178
2174
  version_handle = self._backend.interface.deliver_check_version(
@@ -2591,7 +2587,7 @@ class Run:
2591
2587
  @_run_decorator._attach
2592
2588
  def link_artifact(
2593
2589
  self,
2594
- artifact: Union[public.Artifact, Artifact],
2590
+ artifact: Artifact,
2595
2591
  target_path: str,
2596
2592
  aliases: Optional[List[str]] = None,
2597
2593
  ) -> None:
@@ -2616,7 +2612,7 @@ class Run:
2616
2612
  aliases = []
2617
2613
 
2618
2614
  if self._backend and self._backend.interface:
2619
- if isinstance(artifact, Artifact) and not artifact._logged_artifact:
2615
+ if artifact.is_draft() and not artifact._is_draft_save_started():
2620
2616
  artifact = self._log_artifact(artifact)
2621
2617
  if not self._settings._offline:
2622
2618
  self._backend.interface.publish_link_artifact(
@@ -2635,11 +2631,11 @@ class Run:
2635
2631
  @_run_decorator._attach
2636
2632
  def use_artifact(
2637
2633
  self,
2638
- artifact_or_name: Union[str, public.Artifact, Artifact],
2634
+ artifact_or_name: Union[str, Artifact],
2639
2635
  type: Optional[str] = None,
2640
2636
  aliases: Optional[List[str]] = None,
2641
2637
  use_as: Optional[str] = None,
2642
- ) -> Union[public.Artifact, Artifact]:
2638
+ ) -> Artifact:
2643
2639
  """Declare an artifact as an input to a run.
2644
2640
 
2645
2641
  Call `download` or `file` on the returned object to get the contents locally.
@@ -2704,10 +2700,10 @@ class Run:
2704
2700
  aliases = []
2705
2701
  elif isinstance(aliases, str):
2706
2702
  aliases = [aliases]
2707
- if isinstance(artifact_or_name, wandb.Artifact):
2703
+ if isinstance(artifact_or_name, Artifact) and artifact.is_draft():
2708
2704
  if use_as is not None:
2709
2705
  wandb.termwarn(
2710
- "Indicating use_as is not supported when using an artifact with an instance of `wandb.Artifact`"
2706
+ "Indicating use_as is not supported when using a draft artifact"
2711
2707
  )
2712
2708
  self._log_artifact(
2713
2709
  artifact,
@@ -2717,13 +2713,13 @@ class Run:
2717
2713
  )
2718
2714
  artifact.wait()
2719
2715
  artifact._use_as = use_as or artifact.name
2720
- elif isinstance(artifact, public.Artifact):
2716
+ elif isinstance(artifact, Artifact) and not artifact.is_draft():
2721
2717
  if (
2722
2718
  self._launch_artifact_mapping
2723
2719
  and artifact.name in self._launch_artifact_mapping.keys()
2724
2720
  ):
2725
2721
  wandb.termwarn(
2726
- "Swapping artifacts is not supported when using an instance of `public.Artifact`. "
2722
+ "Swapping artifacts is not supported when using a non-draft artifact. "
2727
2723
  f"Using {artifact.name}."
2728
2724
  )
2729
2725
  artifact._use_as = use_as or artifact.name
@@ -2743,11 +2739,11 @@ class Run:
2743
2739
  @_run_decorator._attach
2744
2740
  def log_artifact(
2745
2741
  self,
2746
- artifact_or_path: Union[wandb_artifacts.Artifact, StrPath],
2742
+ artifact_or_path: Union[Artifact, StrPath],
2747
2743
  name: Optional[str] = None,
2748
2744
  type: Optional[str] = None,
2749
2745
  aliases: Optional[List[str]] = None,
2750
- ) -> wandb_artifacts.Artifact:
2746
+ ) -> Artifact:
2751
2747
  """Declare an artifact as an output of a run.
2752
2748
 
2753
2749
  Arguments:
@@ -2780,12 +2776,12 @@ class Run:
2780
2776
  @_run_decorator._attach
2781
2777
  def upsert_artifact(
2782
2778
  self,
2783
- artifact_or_path: Union[wandb_artifacts.Artifact, str],
2779
+ artifact_or_path: Union[Artifact, str],
2784
2780
  name: Optional[str] = None,
2785
2781
  type: Optional[str] = None,
2786
2782
  aliases: Optional[List[str]] = None,
2787
2783
  distributed_id: Optional[str] = None,
2788
- ) -> wandb_artifacts.Artifact:
2784
+ ) -> Artifact:
2789
2785
  """Declare (or append to) a non-finalized artifact as output of a run.
2790
2786
 
2791
2787
  Note that you must call run.finish_artifact() to finalize the artifact.
@@ -2834,12 +2830,12 @@ class Run:
2834
2830
  @_run_decorator._attach
2835
2831
  def finish_artifact(
2836
2832
  self,
2837
- artifact_or_path: Union[wandb_artifacts.Artifact, str],
2833
+ artifact_or_path: Union[Artifact, str],
2838
2834
  name: Optional[str] = None,
2839
2835
  type: Optional[str] = None,
2840
2836
  aliases: Optional[List[str]] = None,
2841
2837
  distributed_id: Optional[str] = None,
2842
- ) -> wandb_artifacts.Artifact:
2838
+ ) -> Artifact:
2843
2839
  """Finishes a non-finalized artifact as output of a run.
2844
2840
 
2845
2841
  Subsequent "upserts" with the same distributed ID will result in a new version.
@@ -2886,7 +2882,7 @@ class Run:
2886
2882
 
2887
2883
  def _log_artifact(
2888
2884
  self,
2889
- artifact_or_path: Union[wandb_artifacts.Artifact, StrPath],
2885
+ artifact_or_path: Union[Artifact, StrPath],
2890
2886
  name: Optional[str] = None,
2891
2887
  type: Optional[str] = None,
2892
2888
  aliases: Optional[List[str]] = None,
@@ -2894,7 +2890,7 @@ class Run:
2894
2890
  finalize: bool = True,
2895
2891
  is_user_created: bool = False,
2896
2892
  use_after_commit: bool = False,
2897
- ) -> wandb_artifacts.Artifact:
2893
+ ) -> Artifact:
2898
2894
  api = internal.Api()
2899
2895
  if api.settings().get("anonymous") == "true":
2900
2896
  wandb.termwarn(
@@ -2923,7 +2919,7 @@ class Run:
2923
2919
  is_user_created=is_user_created,
2924
2920
  use_after_commit=use_after_commit,
2925
2921
  )
2926
- artifact._logged_artifact = _LazyArtifact(self._public_api(), future)
2922
+ artifact._set_save_future(future, self._public_api().client)
2927
2923
  else:
2928
2924
  self._backend.interface.publish_artifact(
2929
2925
  self,
@@ -2946,10 +2942,9 @@ class Run:
2946
2942
 
2947
2943
  def _public_api(self, overrides: Optional[Dict[str, str]] = None) -> PublicApi:
2948
2944
  overrides = {"run": self._run_id}
2949
- run_obj = self._run_obj
2950
- if run_obj is not None:
2951
- overrides["entity"] = run_obj.entity
2952
- overrides["project"] = run_obj.project
2945
+ if not (self._settings._offline or self._run_obj is None):
2946
+ overrides["entity"] = self._run_obj.entity
2947
+ overrides["project"] = self._run_obj.project
2953
2948
  return public.Api(overrides)
2954
2949
 
2955
2950
  # TODO(jhr): annotate this
@@ -2957,11 +2952,11 @@ class Run:
2957
2952
  if not self._settings._offline:
2958
2953
  try:
2959
2954
  public_api = self._public_api()
2960
- expected_type = public.Artifact.expected_type(
2961
- public_api.client,
2962
- artifact.name,
2955
+ expected_type = Artifact._expected_type(
2963
2956
  public_api.settings["entity"],
2964
2957
  public_api.settings["project"],
2958
+ artifact.name,
2959
+ public_api.client,
2965
2960
  )
2966
2961
  except requests.exceptions.RequestException:
2967
2962
  # Just return early if there is a network error. This is
@@ -2976,11 +2971,11 @@ class Run:
2976
2971
 
2977
2972
  def _prepare_artifact(
2978
2973
  self,
2979
- artifact_or_path: Union[wandb_artifacts.Artifact, StrPath],
2974
+ artifact_or_path: Union[Artifact, StrPath],
2980
2975
  name: Optional[str] = None,
2981
2976
  type: Optional[str] = None,
2982
2977
  aliases: Optional[List[str]] = None,
2983
- ) -> Tuple[wandb_artifacts.Artifact, List[str]]:
2978
+ ) -> Tuple[Artifact, List[str]]:
2984
2979
  if isinstance(artifact_or_path, (str, os.PathLike)):
2985
2980
  name = name or f"run-{self._run_id}-{os.path.basename(artifact_or_path)}"
2986
2981
  artifact = wandb.Artifact(name, type or "unspecified")
@@ -3703,160 +3698,3 @@ def finish(exit_code: Optional[int] = None, quiet: Optional[bool] = None) -> Non
3703
3698
  """
3704
3699
  if wandb.run:
3705
3700
  wandb.run.finish(exit_code=exit_code, quiet=quiet)
3706
-
3707
-
3708
- class InvalidArtifact:
3709
- """An "artifact" that raises an error when any properties are accessed."""
3710
-
3711
- def __init__(self, base_artifact: "ArtifactInterface"):
3712
- super().__setattr__("base_artifact", base_artifact)
3713
-
3714
- def __getattr__(self, __name: str) -> Any:
3715
- raise ArtifactNotLoggedError(artifact=self.base_artifact, attr=__name)
3716
-
3717
- def __setattr__(self, __name: str, __value: Any) -> None:
3718
- raise ArtifactNotLoggedError(artifact=self.base_artifact, attr=__name)
3719
-
3720
- def __bool__(self) -> bool:
3721
- return False
3722
-
3723
-
3724
- class WaitTimeoutError(errors.Error):
3725
- """Raised when wait() timeout occurs before process is finished."""
3726
-
3727
-
3728
- class _LazyArtifact(ArtifactInterface):
3729
- _api: PublicApi
3730
- _instance: Union[ArtifactInterface, InvalidArtifact]
3731
- _future: Any
3732
-
3733
- def __init__(self, api: PublicApi, future: Any):
3734
- self._api = api
3735
- self._instance = InvalidArtifact(self)
3736
- self._future = future
3737
-
3738
- def __getattr__(self, item: str) -> Any:
3739
- return getattr(self._instance, item)
3740
-
3741
- def wait(self, timeout: Optional[int] = None) -> ArtifactInterface:
3742
- if not self._instance:
3743
- future_get = self._future.get(timeout)
3744
- if not future_get:
3745
- raise WaitTimeoutError(
3746
- "Artifact upload wait timed out, failed to fetch Artifact response"
3747
- )
3748
- resp = future_get.response.log_artifact_response
3749
- if resp.error_message:
3750
- raise ValueError(resp.error_message)
3751
- self._instance = public.Artifact.from_id(resp.artifact_id, self._api.client)
3752
- assert isinstance(
3753
- self._instance, ArtifactInterface
3754
- ), "Insufficient permissions to fetch Artifact with id {} from {}".format(
3755
- resp.artifact_id, self._api.client.app_url
3756
- )
3757
- return self._instance
3758
-
3759
- @property
3760
- def id(self) -> Optional[str]:
3761
- return self._instance.id
3762
-
3763
- @property
3764
- def source_version(self) -> Optional[str]:
3765
- return self._instance.source_version
3766
-
3767
- @property
3768
- def version(self) -> str:
3769
- return self._instance.version
3770
-
3771
- @property
3772
- def name(self) -> str:
3773
- return self._instance.name
3774
-
3775
- @property
3776
- def full_name(self) -> str:
3777
- return self._instance.full_name
3778
-
3779
- @property
3780
- def type(self) -> str:
3781
- return self._instance.type
3782
-
3783
- @property
3784
- def entity(self) -> str:
3785
- return self._instance.entity
3786
-
3787
- @property
3788
- def project(self) -> str:
3789
- return self._instance.project
3790
-
3791
- @property
3792
- def manifest(self) -> "ArtifactManifest":
3793
- return self._instance.manifest
3794
-
3795
- @property
3796
- def digest(self) -> str:
3797
- return self._instance.digest
3798
-
3799
- @property
3800
- def state(self) -> str:
3801
- return self._instance.state
3802
-
3803
- @property
3804
- def size(self) -> int:
3805
- return self._instance.size
3806
-
3807
- @property
3808
- def commit_hash(self) -> str:
3809
- return self._instance.commit_hash
3810
-
3811
- @property
3812
- def description(self) -> Optional[str]:
3813
- return self._instance.description
3814
-
3815
- @description.setter
3816
- def description(self, desc: Optional[str]) -> None:
3817
- self._instance.description = desc
3818
-
3819
- @property
3820
- def metadata(self) -> dict:
3821
- return self._instance.metadata
3822
-
3823
- @metadata.setter
3824
- def metadata(self, metadata: dict) -> None:
3825
- self._instance.metadata = metadata
3826
-
3827
- @property
3828
- def aliases(self) -> List[str]:
3829
- return self._instance.aliases
3830
-
3831
- @aliases.setter
3832
- def aliases(self, aliases: List[str]) -> None:
3833
- self._instance.aliases = aliases
3834
-
3835
- def used_by(self) -> List["wandb.apis.public.Run"]:
3836
- return self._instance.used_by()
3837
-
3838
- def logged_by(self) -> "wandb.apis.public.Run":
3839
- return self._instance.logged_by()
3840
-
3841
- def get_path(self, name: str) -> "ArtifactManifestEntry":
3842
- return self._instance.get_path(name)
3843
-
3844
- def get(self, name: str) -> "WBValue":
3845
- return self._instance.get(name)
3846
-
3847
- def download(
3848
- self, root: Optional[str] = None, recursive: bool = False
3849
- ) -> FilePathStr:
3850
- return self._instance.download(root, recursive)
3851
-
3852
- def checkout(self, root: Optional[str] = None) -> str:
3853
- return self._instance.checkout(root)
3854
-
3855
- def verify(self, root: Optional[str] = None) -> Any:
3856
- return self._instance.verify(root)
3857
-
3858
- def save(self) -> None:
3859
- self._instance.save()
3860
-
3861
- def delete(self) -> None:
3862
- self._instance.delete()
@@ -37,13 +37,14 @@ import wandb.env
37
37
  from wandb import util
38
38
  from wandb.apis.internal import Api
39
39
  from wandb.errors import UsageError
40
+ from wandb.sdk.internal.system.env_probe_helpers import is_aws_lambda
40
41
  from wandb.sdk.lib import filesystem
41
42
  from wandb.sdk.lib._settings_toposort_generated import SETTINGS_TOPOLOGICALLY_SORTED
42
43
  from wandb.sdk.wandb_config import Config
43
44
  from wandb.sdk.wandb_setup import _EarlyLogger
44
45
 
45
46
  from .lib import apikey
46
- from .lib.git import GitRepo
47
+ from .lib.gitlib import GitRepo
47
48
  from .lib.ipython import _get_python_type
48
49
  from .lib.runid import generate_id
49
50
 
@@ -449,16 +450,18 @@ class Settings:
449
450
  # settings are declared as class attributes for static type checking purposes
450
451
  # and to help with IDE autocomplete.
451
452
  _args: Sequence[str]
453
+ _aws_lambda: bool
452
454
  _async_upload_concurrency_limit: int
453
455
  _cli_only_mode: bool # Avoid running any code specific for runs
454
456
  _colab: bool
455
457
  _config_dict: Config
456
458
  _console: SettingsConsole
457
459
  _cuda: str
458
- _disable_meta: bool
459
- _disable_service: bool
460
- _disable_stats: bool
460
+ _disable_meta: bool # Do not collect system metadata
461
+ _disable_service: bool # Disable wandb-service, spin up internal process the old way
462
+ _disable_stats: bool # Do not collect system metrics
461
463
  _disable_viewer: bool # Prevent early viewer query
464
+ _disable_setproctitle: bool # Do not use setproctitle on internal process
462
465
  _except_exit: bool
463
466
  _executable: str
464
467
  _extra_http_headers: Mapping[str, str]
@@ -484,6 +487,7 @@ class Settings:
484
487
  _platform: str
485
488
  _python: str
486
489
  _runqueue_item_id: str
490
+ _require_nexus: bool
487
491
  _save_requirements: bool
488
492
  _service_transport: str
489
493
  _service_wait: float
@@ -506,6 +510,7 @@ class Settings:
506
510
  allow_val_change: bool
507
511
  anonymous: str
508
512
  api_key: str
513
+ azure_account_url_to_access_key: Dict[str, str]
509
514
  base_url: str # The base url for the wandb api
510
515
  code_dir: str
511
516
  config_paths: Sequence[str]
@@ -530,6 +535,7 @@ class Settings:
530
535
  ignore_globs: Tuple[str]
531
536
  init_timeout: float
532
537
  is_local: bool
538
+ job_source: str
533
539
  label_disable: bool
534
540
  launch: bool
535
541
  launch_config_path: str
@@ -599,6 +605,10 @@ class Settings:
599
605
  Note that key names must be the same as the class attribute names.
600
606
  """
601
607
  props: Dict[str, Dict[str, Any]] = dict(
608
+ _aws_lambda={
609
+ "hook": lambda _: is_aws_lambda(),
610
+ "auto_hook": True,
611
+ },
602
612
  _async_upload_concurrency_limit={
603
613
  "preprocessor": int,
604
614
  "validator": self._validate__async_upload_concurrency_limit,
@@ -609,6 +619,7 @@ class Settings:
609
619
  "preprocessor": _str_as_bool,
610
620
  "is_policy": True,
611
621
  },
622
+ _disable_setproctitle={"value": False, "preprocessor": _str_as_bool},
612
623
  _disable_stats={"preprocessor": _str_as_bool},
613
624
  _disable_viewer={"preprocessor": _str_as_bool},
614
625
  _extra_http_headers={"preprocessor": _str_as_json},
@@ -655,6 +666,7 @@ class Settings:
655
666
  },
656
667
  _sync={"value": False},
657
668
  _platform={"value": util.get_platform_name()},
669
+ _require_nexus={"value": False, "preprocessor": _str_as_bool},
658
670
  _save_requirements={"value": True, "preprocessor": _str_as_bool},
659
671
  _service_wait={
660
672
  "value": 30,
@@ -693,6 +705,7 @@ class Settings:
693
705
  },
694
706
  anonymous={"validator": self._validate_anonymous},
695
707
  api_key={"validator": self._validate_api_key},
708
+ azure_account_url_to_access_key={"value": {}},
696
709
  base_url={
697
710
  "value": "https://api.wandb.ai",
698
711
  "preprocessor": lambda x: str(x).strip().rstrip("/"),
@@ -731,6 +744,7 @@ class Settings:
731
744
  ),
732
745
  "auto_hook": True,
733
746
  },
747
+ job_source={"validator": self._validate_job_source},
734
748
  label_disable={"preprocessor": _str_as_bool},
735
749
  launch={"preprocessor": _str_as_bool},
736
750
  log_dir={
@@ -1113,6 +1127,15 @@ class Settings:
1113
1127
 
1114
1128
  return True
1115
1129
 
1130
+ @staticmethod
1131
+ def _validate_job_source(value: str) -> bool:
1132
+ valid_sources = ["repo", "artifact", "image"]
1133
+ if value not in valid_sources:
1134
+ raise UsageError(
1135
+ f"Settings field `job_source`: {value!r} not in {valid_sources}"
1136
+ )
1137
+ return True
1138
+
1116
1139
  # other helper methods
1117
1140
  @staticmethod
1118
1141
  def _path_convert(*args: str) -> str:
@@ -1542,6 +1565,7 @@ class Settings:
1542
1565
  "WANDB_TRACELOG": "_tracelog",
1543
1566
  "WANDB_DISABLE_SERVICE": "_disable_service",
1544
1567
  "WANDB_SERVICE_TRANSPORT": "_service_transport",
1568
+ "WANDB_REQUIRE_NEXUS": "_require_nexus",
1545
1569
  "WANDB_DIR": "root_dir",
1546
1570
  "WANDB_NAME": "run_name",
1547
1571
  "WANDB_NOTES": "run_notes",