wandb 0.15.3__py3-none-any.whl → 0.15.5__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (156) hide show
  1. wandb/__init__.py +1 -1
  2. wandb/analytics/sentry.py +1 -0
  3. wandb/apis/importers/base.py +20 -5
  4. wandb/apis/importers/mlflow.py +7 -1
  5. wandb/apis/internal.py +12 -0
  6. wandb/apis/public.py +247 -1387
  7. wandb/apis/reports/_panels.py +58 -35
  8. wandb/beta/workflows.py +6 -7
  9. wandb/cli/cli.py +130 -60
  10. wandb/data_types.py +3 -1
  11. wandb/filesync/dir_watcher.py +21 -27
  12. wandb/filesync/step_checksum.py +8 -8
  13. wandb/filesync/step_prepare.py +23 -10
  14. wandb/filesync/step_upload.py +13 -13
  15. wandb/filesync/upload_job.py +4 -8
  16. wandb/integration/cohere/__init__.py +3 -0
  17. wandb/integration/cohere/cohere.py +21 -0
  18. wandb/integration/cohere/resolver.py +347 -0
  19. wandb/integration/gym/__init__.py +4 -6
  20. wandb/integration/huggingface/__init__.py +3 -0
  21. wandb/integration/huggingface/huggingface.py +18 -0
  22. wandb/integration/huggingface/resolver.py +213 -0
  23. wandb/integration/langchain/wandb_tracer.py +16 -179
  24. wandb/integration/openai/__init__.py +1 -3
  25. wandb/integration/openai/openai.py +11 -143
  26. wandb/integration/openai/resolver.py +111 -38
  27. wandb/integration/sagemaker/config.py +2 -2
  28. wandb/integration/tensorboard/log.py +4 -4
  29. wandb/old/settings.py +24 -7
  30. wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
  31. wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
  32. wandb/proto/wandb_deprecated.py +3 -1
  33. wandb/sdk/__init__.py +1 -1
  34. wandb/sdk/artifacts/__init__.py +0 -0
  35. wandb/sdk/artifacts/artifact.py +2101 -0
  36. wandb/sdk/artifacts/artifact_download_logger.py +42 -0
  37. wandb/sdk/artifacts/artifact_manifest.py +67 -0
  38. wandb/sdk/artifacts/artifact_manifest_entry.py +159 -0
  39. wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
  40. wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +91 -0
  41. wandb/sdk/{internal → artifacts}/artifact_saver.py +6 -5
  42. wandb/sdk/artifacts/artifact_state.py +10 -0
  43. wandb/sdk/{interface/artifacts/artifact_cache.py → artifacts/artifacts_cache.py} +22 -12
  44. wandb/sdk/artifacts/exceptions.py +55 -0
  45. wandb/sdk/artifacts/storage_handler.py +59 -0
  46. wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
  47. wandb/sdk/artifacts/storage_handlers/azure_handler.py +192 -0
  48. wandb/sdk/artifacts/storage_handlers/gcs_handler.py +224 -0
  49. wandb/sdk/artifacts/storage_handlers/http_handler.py +112 -0
  50. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +134 -0
  51. wandb/sdk/artifacts/storage_handlers/multi_handler.py +53 -0
  52. wandb/sdk/artifacts/storage_handlers/s3_handler.py +301 -0
  53. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +67 -0
  54. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +132 -0
  55. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +72 -0
  56. wandb/sdk/artifacts/storage_layout.py +6 -0
  57. wandb/sdk/artifacts/storage_policies/__init__.py +0 -0
  58. wandb/sdk/artifacts/storage_policies/s3_bucket_policy.py +61 -0
  59. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +386 -0
  60. wandb/sdk/{interface/artifacts/artifact_storage.py → artifacts/storage_policy.py} +5 -57
  61. wandb/sdk/data_types/_dtypes.py +7 -12
  62. wandb/sdk/data_types/base_types/json_metadata.py +3 -2
  63. wandb/sdk/data_types/base_types/media.py +8 -8
  64. wandb/sdk/data_types/base_types/wb_value.py +12 -13
  65. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +5 -6
  66. wandb/sdk/data_types/helper_types/classes.py +6 -8
  67. wandb/sdk/data_types/helper_types/image_mask.py +5 -6
  68. wandb/sdk/data_types/histogram.py +4 -3
  69. wandb/sdk/data_types/html.py +3 -4
  70. wandb/sdk/data_types/image.py +11 -9
  71. wandb/sdk/data_types/molecule.py +5 -3
  72. wandb/sdk/data_types/object_3d.py +7 -5
  73. wandb/sdk/data_types/plotly.py +3 -2
  74. wandb/sdk/data_types/saved_model.py +11 -11
  75. wandb/sdk/data_types/trace_tree.py +5 -4
  76. wandb/sdk/data_types/utils.py +3 -5
  77. wandb/sdk/data_types/video.py +5 -4
  78. wandb/sdk/integration_utils/auto_logging.py +215 -0
  79. wandb/sdk/interface/interface.py +15 -15
  80. wandb/sdk/internal/file_pusher.py +8 -16
  81. wandb/sdk/internal/file_stream.py +5 -11
  82. wandb/sdk/internal/handler.py +13 -1
  83. wandb/sdk/internal/internal_api.py +287 -13
  84. wandb/sdk/internal/job_builder.py +119 -30
  85. wandb/sdk/internal/sender.py +6 -26
  86. wandb/sdk/internal/settings_static.py +2 -0
  87. wandb/sdk/internal/system/assets/__init__.py +2 -0
  88. wandb/sdk/internal/system/assets/gpu.py +42 -0
  89. wandb/sdk/internal/system/assets/gpu_amd.py +216 -0
  90. wandb/sdk/internal/system/env_probe_helpers.py +13 -0
  91. wandb/sdk/internal/system/system_info.py +3 -3
  92. wandb/sdk/internal/tb_watcher.py +32 -22
  93. wandb/sdk/internal/thread_local_settings.py +18 -0
  94. wandb/sdk/launch/_project_spec.py +57 -11
  95. wandb/sdk/launch/agent/agent.py +147 -65
  96. wandb/sdk/launch/agent/job_status_tracker.py +34 -0
  97. wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
  98. wandb/sdk/launch/builder/abstract.py +5 -1
  99. wandb/sdk/launch/builder/build.py +21 -18
  100. wandb/sdk/launch/builder/docker_builder.py +10 -4
  101. wandb/sdk/launch/builder/kaniko_builder.py +113 -23
  102. wandb/sdk/launch/builder/noop.py +6 -3
  103. wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +46 -14
  104. wandb/sdk/launch/environment/aws_environment.py +3 -2
  105. wandb/sdk/launch/environment/azure_environment.py +124 -0
  106. wandb/sdk/launch/environment/gcp_environment.py +2 -4
  107. wandb/sdk/launch/environment/local_environment.py +1 -1
  108. wandb/sdk/launch/errors.py +19 -0
  109. wandb/sdk/launch/github_reference.py +32 -19
  110. wandb/sdk/launch/launch.py +3 -8
  111. wandb/sdk/launch/launch_add.py +6 -2
  112. wandb/sdk/launch/loader.py +21 -2
  113. wandb/sdk/launch/registry/azure_container_registry.py +132 -0
  114. wandb/sdk/launch/registry/elastic_container_registry.py +39 -5
  115. wandb/sdk/launch/registry/google_artifact_registry.py +68 -26
  116. wandb/sdk/launch/registry/local_registry.py +2 -1
  117. wandb/sdk/launch/runner/abstract.py +24 -3
  118. wandb/sdk/launch/runner/kubernetes_runner.py +479 -26
  119. wandb/sdk/launch/runner/local_container.py +103 -51
  120. wandb/sdk/launch/runner/local_process.py +1 -1
  121. wandb/sdk/launch/runner/sagemaker_runner.py +60 -10
  122. wandb/sdk/launch/runner/vertex_runner.py +10 -5
  123. wandb/sdk/launch/sweeps/__init__.py +7 -9
  124. wandb/sdk/launch/sweeps/scheduler.py +307 -77
  125. wandb/sdk/launch/sweeps/scheduler_sweep.py +2 -1
  126. wandb/sdk/launch/sweeps/utils.py +82 -35
  127. wandb/sdk/launch/utils.py +89 -75
  128. wandb/sdk/lib/_settings_toposort_generated.py +7 -0
  129. wandb/sdk/lib/capped_dict.py +26 -0
  130. wandb/sdk/lib/{git.py → gitlib.py} +76 -59
  131. wandb/sdk/lib/hashutil.py +12 -4
  132. wandb/sdk/lib/paths.py +96 -8
  133. wandb/sdk/lib/sock_client.py +2 -2
  134. wandb/sdk/lib/timer.py +1 -0
  135. wandb/sdk/service/server.py +22 -9
  136. wandb/sdk/service/server_sock.py +1 -1
  137. wandb/sdk/service/service.py +27 -8
  138. wandb/sdk/verify/verify.py +4 -7
  139. wandb/sdk/wandb_config.py +2 -6
  140. wandb/sdk/wandb_init.py +57 -53
  141. wandb/sdk/wandb_require.py +7 -0
  142. wandb/sdk/wandb_run.py +61 -223
  143. wandb/sdk/wandb_settings.py +28 -4
  144. wandb/testing/relay.py +15 -2
  145. wandb/util.py +74 -36
  146. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/METADATA +15 -9
  147. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/RECORD +151 -116
  148. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/entry_points.txt +1 -0
  149. wandb/integration/langchain/util.py +0 -191
  150. wandb/sdk/interface/artifacts/__init__.py +0 -33
  151. wandb/sdk/interface/artifacts/artifact.py +0 -615
  152. wandb/sdk/interface/artifacts/artifact_manifest.py +0 -131
  153. wandb/sdk/wandb_artifacts.py +0 -2226
  154. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/LICENSE +0 -0
  155. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/WHEEL +0 -0
  156. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/top_level.txt +0 -0
wandb/sdk/wandb_init.py CHANGED
@@ -663,6 +663,9 @@ class _WandbInit:
663
663
  if os.environ.get("PEX"):
664
664
  tel.env.pex = True
665
665
 
666
+ if self.settings._aws_lambda:
667
+ tel.env.aws_lambda = True
668
+
666
669
  if os.environ.get(wandb.env._DISABLE_SERVICE):
667
670
  tel.feature.service_disabled = True
668
671
 
@@ -672,6 +675,8 @@ class _WandbInit:
672
675
  tel.feature.flow_control_disabled = True
673
676
  if self.settings._flow_control_custom:
674
677
  tel.feature.flow_control_custom = True
678
+ if self.settings._require_nexus:
679
+ tel.feature.nexus = True
675
680
 
676
681
  tel.env.maybe_mp = _maybe_mp_process(backend)
677
682
 
@@ -715,86 +720,82 @@ class _WandbInit:
715
720
  if not self.settings.disable_git:
716
721
  run._populate_git_info()
717
722
 
718
- run_proto = backend.interface._make_run(run)
719
723
  run_result: Optional["pb.RunUpdateResult"] = None
720
724
 
721
725
  if self.settings._offline:
722
726
  with telemetry.context(run=run) as tel:
723
727
  tel.feature.offline = True
724
728
 
725
- backend.interface.publish_run(run_proto)
726
- run._set_run_obj_offline(run_proto)
727
729
  if self.settings.resume:
728
730
  wandb.termwarn(
729
731
  "`resume` will be ignored since W&B syncing is set to `offline`. "
730
732
  f"Starting a new run with run id {run.id}."
731
733
  )
732
- else:
733
- error: Optional["wandb.errors.Error"] = None
734
+ error: Optional["wandb.errors.Error"] = None
734
735
 
735
- timeout = self.settings.init_timeout
736
+ timeout = self.settings.init_timeout
736
737
 
737
- logger.info(f"communicating run to backend with {timeout} second timeout")
738
+ logger.info(f"communicating run to backend with {timeout} second timeout")
738
739
 
739
- run_init_handle = backend.interface.deliver_run(run_proto)
740
- result = run_init_handle.wait(
741
- timeout=timeout,
742
- on_progress=self._on_progress_init,
743
- cancel=True,
740
+ run_init_handle = backend.interface.deliver_run(run)
741
+ result = run_init_handle.wait(
742
+ timeout=timeout,
743
+ on_progress=self._on_progress_init,
744
+ cancel=True,
745
+ )
746
+ if result:
747
+ run_result = result.run_result
748
+
749
+ if run_result is None:
750
+ error_message = (
751
+ f"Run initialization has timed out after {timeout} sec. "
752
+ f"\nPlease refer to the documentation for additional information: {wburls.get('doc_start_err')}"
753
+ )
754
+ # We're not certain whether the error we encountered is due to an issue
755
+ # with the server (a "CommError") or if it's a problem within the SDK (an "Error").
756
+ # This means that the error could be a result of the server being unresponsive,
757
+ # or it could be because we were unable to communicate with the wandb service.
758
+ error = CommError(error_message)
759
+ run_init_handle._cancel()
760
+ elif run_result.HasField("error"):
761
+ error = ProtobufErrorHandler.to_exception(run_result.error)
762
+
763
+ if error is not None:
764
+ logger.error(f"encountered error: {error}")
765
+ if not manager:
766
+ # Shutdown the backend and get rid of the logger
767
+ # we don't need to do console cleanup at this point
768
+ backend.cleanup()
769
+ self.teardown()
770
+ raise error
771
+
772
+ assert run_result is not None # for mypy
773
+
774
+ if not run_result.HasField("run"):
775
+ raise Error(
776
+ "It appears that something have gone wrong during the program execution as an unexpected missing field was encountered. "
777
+ "(run_result is missing the 'run' field)"
744
778
  )
745
- if result:
746
- run_result = result.run_result
747
779
 
748
- if run_result is None:
749
- error_message = (
750
- f"Run initialization has timed out after {timeout} sec. "
751
- f"\nPlease refer to the documentation for additional information: {wburls.get('doc_start_err')}"
752
- )
753
- # We're not certain whether the error we encountered is due to an issue
754
- # with the server (a "CommError") or if it's a problem within the SDK (an "Error").
755
- # This means that the error could be a result of the server being unresponsive,
756
- # or it could be because we were unable to communicate with the wandb service.
757
- error = CommError(error_message)
758
- run_init_handle._cancel()
759
- elif run_result.HasField("error"):
760
- error = ProtobufErrorHandler.to_exception(run_result.error)
761
-
762
- if error is not None:
763
- logger.error(f"encountered error: {error}")
764
- if not manager:
765
- # Shutdown the backend and get rid of the logger
766
- # we don't need to do console cleanup at this point
767
- backend.cleanup()
768
- self.teardown()
769
- raise error
770
-
771
- assert run_result is not None # for mypy
772
-
773
- if not run_result.HasField("run"):
774
- raise Error(
775
- "It appears that something have gone wrong during the program execution as an unexpected missing field was encountered. "
776
- "(run_result is missing the 'run' field)"
777
- )
780
+ if run_result.run.resumed:
781
+ logger.info("run resumed")
782
+ with telemetry.context(run=run) as tel:
783
+ tel.feature.resumed = run_result.run.resumed
778
784
 
779
- if run_result.run.resumed:
780
- logger.info("run resumed")
781
- with telemetry.context(run=run) as tel:
782
- tel.feature.resumed = run_result.run.resumed
785
+ run._set_run_obj(run_result.run)
783
786
 
784
- run._set_run_obj(run_result.run)
785
- run._on_init()
787
+ run._on_init()
786
788
 
787
789
  logger.info("starting run threads in backend")
788
790
  # initiate run (stats and metadata probing)
789
- run_obj = run._run_obj or run._run_obj_offline
790
791
 
791
792
  if manager:
792
793
  manager._inform_start(settings=self.settings, run_id=self.settings.run_id)
793
794
 
794
795
  assert backend.interface
795
- assert run_obj
796
+ assert run._run_obj
796
797
 
797
- run_start_handle = backend.interface.deliver_run_start(run_obj)
798
+ run_start_handle = backend.interface.deliver_run_start(run._run_obj)
798
799
  # TODO: add progress to let user know we are doing something
799
800
  run_start_result = run_start_handle.wait(timeout=30)
800
801
  if run_start_result is None:
@@ -881,6 +882,7 @@ def _attach(
881
882
  "run_id": attach_id,
882
883
  "_start_time": response["_start_time"],
883
884
  "_start_datetime": response["_start_datetime"],
885
+ "_offline": response["_offline"],
884
886
  },
885
887
  source=Source.INIT,
886
888
  )
@@ -901,6 +903,7 @@ def _attach(
901
903
  assert backend.interface
902
904
 
903
905
  mailbox.enable_keepalive()
906
+
904
907
  attach_handle = backend.interface.deliver_attach(attach_id)
905
908
  # TODO: add progress to let user know we are doing something
906
909
  attach_result = attach_handle.wait(timeout=30)
@@ -910,6 +913,7 @@ def _attach(
910
913
  attach_response = attach_result.response.attach_response
911
914
  if attach_response.error and attach_response.error.message:
912
915
  raise UsageError(f"Failed to attach to run: {attach_response.error.message}")
916
+
913
917
  run._set_run_obj(attach_response.run)
914
918
  run._on_attach()
915
919
  return run
@@ -9,6 +9,7 @@ Example:
9
9
  wandb.require("incremental-artifacts@beta")
10
10
  """
11
11
 
12
+ import os
12
13
  from typing import Optional, Sequence, Union
13
14
 
14
15
  import wandb
@@ -38,6 +39,12 @@ class _Requires:
38
39
  def require_service(self) -> None:
39
40
  self._require_service()
40
41
 
42
+ def _require_nexus(self) -> None:
43
+ os.environ["WANDB_REQUIRE_NEXUS"] = "True"
44
+
45
+ def require_nexus(self) -> None:
46
+ self._require_nexus()
47
+
41
48
  def apply(self) -> None:
42
49
  """Call require_* method for supported features."""
43
50
  last_message: str = ""