wandb 0.15.3__py3-none-any.whl → 0.15.5__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- wandb/__init__.py +1 -1
- wandb/analytics/sentry.py +1 -0
- wandb/apis/importers/base.py +20 -5
- wandb/apis/importers/mlflow.py +7 -1
- wandb/apis/internal.py +12 -0
- wandb/apis/public.py +247 -1387
- wandb/apis/reports/_panels.py +58 -35
- wandb/beta/workflows.py +6 -7
- wandb/cli/cli.py +130 -60
- wandb/data_types.py +3 -1
- wandb/filesync/dir_watcher.py +21 -27
- wandb/filesync/step_checksum.py +8 -8
- wandb/filesync/step_prepare.py +23 -10
- wandb/filesync/step_upload.py +13 -13
- wandb/filesync/upload_job.py +4 -8
- wandb/integration/cohere/__init__.py +3 -0
- wandb/integration/cohere/cohere.py +21 -0
- wandb/integration/cohere/resolver.py +347 -0
- wandb/integration/gym/__init__.py +4 -6
- wandb/integration/huggingface/__init__.py +3 -0
- wandb/integration/huggingface/huggingface.py +18 -0
- wandb/integration/huggingface/resolver.py +213 -0
- wandb/integration/langchain/wandb_tracer.py +16 -179
- wandb/integration/openai/__init__.py +1 -3
- wandb/integration/openai/openai.py +11 -143
- wandb/integration/openai/resolver.py +111 -38
- wandb/integration/sagemaker/config.py +2 -2
- wandb/integration/tensorboard/log.py +4 -4
- wandb/old/settings.py +24 -7
- wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
- wandb/proto/wandb_deprecated.py +3 -1
- wandb/sdk/__init__.py +1 -1
- wandb/sdk/artifacts/__init__.py +0 -0
- wandb/sdk/artifacts/artifact.py +2101 -0
- wandb/sdk/artifacts/artifact_download_logger.py +42 -0
- wandb/sdk/artifacts/artifact_manifest.py +67 -0
- wandb/sdk/artifacts/artifact_manifest_entry.py +159 -0
- wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +91 -0
- wandb/sdk/{internal → artifacts}/artifact_saver.py +6 -5
- wandb/sdk/artifacts/artifact_state.py +10 -0
- wandb/sdk/{interface/artifacts/artifact_cache.py → artifacts/artifacts_cache.py} +22 -12
- wandb/sdk/artifacts/exceptions.py +55 -0
- wandb/sdk/artifacts/storage_handler.py +59 -0
- wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +192 -0
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +224 -0
- wandb/sdk/artifacts/storage_handlers/http_handler.py +112 -0
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +134 -0
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +53 -0
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +301 -0
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +67 -0
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +132 -0
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +72 -0
- wandb/sdk/artifacts/storage_layout.py +6 -0
- wandb/sdk/artifacts/storage_policies/__init__.py +0 -0
- wandb/sdk/artifacts/storage_policies/s3_bucket_policy.py +61 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +386 -0
- wandb/sdk/{interface/artifacts/artifact_storage.py → artifacts/storage_policy.py} +5 -57
- wandb/sdk/data_types/_dtypes.py +7 -12
- wandb/sdk/data_types/base_types/json_metadata.py +3 -2
- wandb/sdk/data_types/base_types/media.py +8 -8
- wandb/sdk/data_types/base_types/wb_value.py +12 -13
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +5 -6
- wandb/sdk/data_types/helper_types/classes.py +6 -8
- wandb/sdk/data_types/helper_types/image_mask.py +5 -6
- wandb/sdk/data_types/histogram.py +4 -3
- wandb/sdk/data_types/html.py +3 -4
- wandb/sdk/data_types/image.py +11 -9
- wandb/sdk/data_types/molecule.py +5 -3
- wandb/sdk/data_types/object_3d.py +7 -5
- wandb/sdk/data_types/plotly.py +3 -2
- wandb/sdk/data_types/saved_model.py +11 -11
- wandb/sdk/data_types/trace_tree.py +5 -4
- wandb/sdk/data_types/utils.py +3 -5
- wandb/sdk/data_types/video.py +5 -4
- wandb/sdk/integration_utils/auto_logging.py +215 -0
- wandb/sdk/interface/interface.py +15 -15
- wandb/sdk/internal/file_pusher.py +8 -16
- wandb/sdk/internal/file_stream.py +5 -11
- wandb/sdk/internal/handler.py +13 -1
- wandb/sdk/internal/internal_api.py +287 -13
- wandb/sdk/internal/job_builder.py +119 -30
- wandb/sdk/internal/sender.py +6 -26
- wandb/sdk/internal/settings_static.py +2 -0
- wandb/sdk/internal/system/assets/__init__.py +2 -0
- wandb/sdk/internal/system/assets/gpu.py +42 -0
- wandb/sdk/internal/system/assets/gpu_amd.py +216 -0
- wandb/sdk/internal/system/env_probe_helpers.py +13 -0
- wandb/sdk/internal/system/system_info.py +3 -3
- wandb/sdk/internal/tb_watcher.py +32 -22
- wandb/sdk/internal/thread_local_settings.py +18 -0
- wandb/sdk/launch/_project_spec.py +57 -11
- wandb/sdk/launch/agent/agent.py +147 -65
- wandb/sdk/launch/agent/job_status_tracker.py +34 -0
- wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
- wandb/sdk/launch/builder/abstract.py +5 -1
- wandb/sdk/launch/builder/build.py +21 -18
- wandb/sdk/launch/builder/docker_builder.py +10 -4
- wandb/sdk/launch/builder/kaniko_builder.py +113 -23
- wandb/sdk/launch/builder/noop.py +6 -3
- wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +46 -14
- wandb/sdk/launch/environment/aws_environment.py +3 -2
- wandb/sdk/launch/environment/azure_environment.py +124 -0
- wandb/sdk/launch/environment/gcp_environment.py +2 -4
- wandb/sdk/launch/environment/local_environment.py +1 -1
- wandb/sdk/launch/errors.py +19 -0
- wandb/sdk/launch/github_reference.py +32 -19
- wandb/sdk/launch/launch.py +3 -8
- wandb/sdk/launch/launch_add.py +6 -2
- wandb/sdk/launch/loader.py +21 -2
- wandb/sdk/launch/registry/azure_container_registry.py +132 -0
- wandb/sdk/launch/registry/elastic_container_registry.py +39 -5
- wandb/sdk/launch/registry/google_artifact_registry.py +68 -26
- wandb/sdk/launch/registry/local_registry.py +2 -1
- wandb/sdk/launch/runner/abstract.py +24 -3
- wandb/sdk/launch/runner/kubernetes_runner.py +479 -26
- wandb/sdk/launch/runner/local_container.py +103 -51
- wandb/sdk/launch/runner/local_process.py +1 -1
- wandb/sdk/launch/runner/sagemaker_runner.py +60 -10
- wandb/sdk/launch/runner/vertex_runner.py +10 -5
- wandb/sdk/launch/sweeps/__init__.py +7 -9
- wandb/sdk/launch/sweeps/scheduler.py +307 -77
- wandb/sdk/launch/sweeps/scheduler_sweep.py +2 -1
- wandb/sdk/launch/sweeps/utils.py +82 -35
- wandb/sdk/launch/utils.py +89 -75
- wandb/sdk/lib/_settings_toposort_generated.py +7 -0
- wandb/sdk/lib/capped_dict.py +26 -0
- wandb/sdk/lib/{git.py → gitlib.py} +76 -59
- wandb/sdk/lib/hashutil.py +12 -4
- wandb/sdk/lib/paths.py +96 -8
- wandb/sdk/lib/sock_client.py +2 -2
- wandb/sdk/lib/timer.py +1 -0
- wandb/sdk/service/server.py +22 -9
- wandb/sdk/service/server_sock.py +1 -1
- wandb/sdk/service/service.py +27 -8
- wandb/sdk/verify/verify.py +4 -7
- wandb/sdk/wandb_config.py +2 -6
- wandb/sdk/wandb_init.py +57 -53
- wandb/sdk/wandb_require.py +7 -0
- wandb/sdk/wandb_run.py +61 -223
- wandb/sdk/wandb_settings.py +28 -4
- wandb/testing/relay.py +15 -2
- wandb/util.py +74 -36
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/METADATA +15 -9
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/RECORD +151 -116
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/entry_points.txt +1 -0
- wandb/integration/langchain/util.py +0 -191
- wandb/sdk/interface/artifacts/__init__.py +0 -33
- wandb/sdk/interface/artifacts/artifact.py +0 -615
- wandb/sdk/interface/artifacts/artifact_manifest.py +0 -131
- wandb/sdk/wandb_artifacts.py +0 -2226
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/LICENSE +0 -0
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/WHEEL +0 -0
- {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/top_level.txt +0 -0
wandb/sdk/wandb_init.py
CHANGED
@@ -663,6 +663,9 @@ class _WandbInit:
|
|
663
663
|
if os.environ.get("PEX"):
|
664
664
|
tel.env.pex = True
|
665
665
|
|
666
|
+
if self.settings._aws_lambda:
|
667
|
+
tel.env.aws_lambda = True
|
668
|
+
|
666
669
|
if os.environ.get(wandb.env._DISABLE_SERVICE):
|
667
670
|
tel.feature.service_disabled = True
|
668
671
|
|
@@ -672,6 +675,8 @@ class _WandbInit:
|
|
672
675
|
tel.feature.flow_control_disabled = True
|
673
676
|
if self.settings._flow_control_custom:
|
674
677
|
tel.feature.flow_control_custom = True
|
678
|
+
if self.settings._require_nexus:
|
679
|
+
tel.feature.nexus = True
|
675
680
|
|
676
681
|
tel.env.maybe_mp = _maybe_mp_process(backend)
|
677
682
|
|
@@ -715,86 +720,82 @@ class _WandbInit:
|
|
715
720
|
if not self.settings.disable_git:
|
716
721
|
run._populate_git_info()
|
717
722
|
|
718
|
-
run_proto = backend.interface._make_run(run)
|
719
723
|
run_result: Optional["pb.RunUpdateResult"] = None
|
720
724
|
|
721
725
|
if self.settings._offline:
|
722
726
|
with telemetry.context(run=run) as tel:
|
723
727
|
tel.feature.offline = True
|
724
728
|
|
725
|
-
backend.interface.publish_run(run_proto)
|
726
|
-
run._set_run_obj_offline(run_proto)
|
727
729
|
if self.settings.resume:
|
728
730
|
wandb.termwarn(
|
729
731
|
"`resume` will be ignored since W&B syncing is set to `offline`. "
|
730
732
|
f"Starting a new run with run id {run.id}."
|
731
733
|
)
|
732
|
-
|
733
|
-
error: Optional["wandb.errors.Error"] = None
|
734
|
+
error: Optional["wandb.errors.Error"] = None
|
734
735
|
|
735
|
-
|
736
|
+
timeout = self.settings.init_timeout
|
736
737
|
|
737
|
-
|
738
|
+
logger.info(f"communicating run to backend with {timeout} second timeout")
|
738
739
|
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
|
743
|
-
|
740
|
+
run_init_handle = backend.interface.deliver_run(run)
|
741
|
+
result = run_init_handle.wait(
|
742
|
+
timeout=timeout,
|
743
|
+
on_progress=self._on_progress_init,
|
744
|
+
cancel=True,
|
745
|
+
)
|
746
|
+
if result:
|
747
|
+
run_result = result.run_result
|
748
|
+
|
749
|
+
if run_result is None:
|
750
|
+
error_message = (
|
751
|
+
f"Run initialization has timed out after {timeout} sec. "
|
752
|
+
f"\nPlease refer to the documentation for additional information: {wburls.get('doc_start_err')}"
|
753
|
+
)
|
754
|
+
# We're not certain whether the error we encountered is due to an issue
|
755
|
+
# with the server (a "CommError") or if it's a problem within the SDK (an "Error").
|
756
|
+
# This means that the error could be a result of the server being unresponsive,
|
757
|
+
# or it could be because we were unable to communicate with the wandb service.
|
758
|
+
error = CommError(error_message)
|
759
|
+
run_init_handle._cancel()
|
760
|
+
elif run_result.HasField("error"):
|
761
|
+
error = ProtobufErrorHandler.to_exception(run_result.error)
|
762
|
+
|
763
|
+
if error is not None:
|
764
|
+
logger.error(f"encountered error: {error}")
|
765
|
+
if not manager:
|
766
|
+
# Shutdown the backend and get rid of the logger
|
767
|
+
# we don't need to do console cleanup at this point
|
768
|
+
backend.cleanup()
|
769
|
+
self.teardown()
|
770
|
+
raise error
|
771
|
+
|
772
|
+
assert run_result is not None # for mypy
|
773
|
+
|
774
|
+
if not run_result.HasField("run"):
|
775
|
+
raise Error(
|
776
|
+
"It appears that something have gone wrong during the program execution as an unexpected missing field was encountered. "
|
777
|
+
"(run_result is missing the 'run' field)"
|
744
778
|
)
|
745
|
-
if result:
|
746
|
-
run_result = result.run_result
|
747
779
|
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
)
|
753
|
-
# We're not certain whether the error we encountered is due to an issue
|
754
|
-
# with the server (a "CommError") or if it's a problem within the SDK (an "Error").
|
755
|
-
# This means that the error could be a result of the server being unresponsive,
|
756
|
-
# or it could be because we were unable to communicate with the wandb service.
|
757
|
-
error = CommError(error_message)
|
758
|
-
run_init_handle._cancel()
|
759
|
-
elif run_result.HasField("error"):
|
760
|
-
error = ProtobufErrorHandler.to_exception(run_result.error)
|
761
|
-
|
762
|
-
if error is not None:
|
763
|
-
logger.error(f"encountered error: {error}")
|
764
|
-
if not manager:
|
765
|
-
# Shutdown the backend and get rid of the logger
|
766
|
-
# we don't need to do console cleanup at this point
|
767
|
-
backend.cleanup()
|
768
|
-
self.teardown()
|
769
|
-
raise error
|
770
|
-
|
771
|
-
assert run_result is not None # for mypy
|
772
|
-
|
773
|
-
if not run_result.HasField("run"):
|
774
|
-
raise Error(
|
775
|
-
"It appears that something have gone wrong during the program execution as an unexpected missing field was encountered. "
|
776
|
-
"(run_result is missing the 'run' field)"
|
777
|
-
)
|
780
|
+
if run_result.run.resumed:
|
781
|
+
logger.info("run resumed")
|
782
|
+
with telemetry.context(run=run) as tel:
|
783
|
+
tel.feature.resumed = run_result.run.resumed
|
778
784
|
|
779
|
-
|
780
|
-
logger.info("run resumed")
|
781
|
-
with telemetry.context(run=run) as tel:
|
782
|
-
tel.feature.resumed = run_result.run.resumed
|
785
|
+
run._set_run_obj(run_result.run)
|
783
786
|
|
784
|
-
|
785
|
-
run._on_init()
|
787
|
+
run._on_init()
|
786
788
|
|
787
789
|
logger.info("starting run threads in backend")
|
788
790
|
# initiate run (stats and metadata probing)
|
789
|
-
run_obj = run._run_obj or run._run_obj_offline
|
790
791
|
|
791
792
|
if manager:
|
792
793
|
manager._inform_start(settings=self.settings, run_id=self.settings.run_id)
|
793
794
|
|
794
795
|
assert backend.interface
|
795
|
-
assert
|
796
|
+
assert run._run_obj
|
796
797
|
|
797
|
-
run_start_handle = backend.interface.deliver_run_start(
|
798
|
+
run_start_handle = backend.interface.deliver_run_start(run._run_obj)
|
798
799
|
# TODO: add progress to let user know we are doing something
|
799
800
|
run_start_result = run_start_handle.wait(timeout=30)
|
800
801
|
if run_start_result is None:
|
@@ -881,6 +882,7 @@ def _attach(
|
|
881
882
|
"run_id": attach_id,
|
882
883
|
"_start_time": response["_start_time"],
|
883
884
|
"_start_datetime": response["_start_datetime"],
|
885
|
+
"_offline": response["_offline"],
|
884
886
|
},
|
885
887
|
source=Source.INIT,
|
886
888
|
)
|
@@ -901,6 +903,7 @@ def _attach(
|
|
901
903
|
assert backend.interface
|
902
904
|
|
903
905
|
mailbox.enable_keepalive()
|
906
|
+
|
904
907
|
attach_handle = backend.interface.deliver_attach(attach_id)
|
905
908
|
# TODO: add progress to let user know we are doing something
|
906
909
|
attach_result = attach_handle.wait(timeout=30)
|
@@ -910,6 +913,7 @@ def _attach(
|
|
910
913
|
attach_response = attach_result.response.attach_response
|
911
914
|
if attach_response.error and attach_response.error.message:
|
912
915
|
raise UsageError(f"Failed to attach to run: {attach_response.error.message}")
|
916
|
+
|
913
917
|
run._set_run_obj(attach_response.run)
|
914
918
|
run._on_attach()
|
915
919
|
return run
|
wandb/sdk/wandb_require.py
CHANGED
@@ -9,6 +9,7 @@ Example:
|
|
9
9
|
wandb.require("incremental-artifacts@beta")
|
10
10
|
"""
|
11
11
|
|
12
|
+
import os
|
12
13
|
from typing import Optional, Sequence, Union
|
13
14
|
|
14
15
|
import wandb
|
@@ -38,6 +39,12 @@ class _Requires:
|
|
38
39
|
def require_service(self) -> None:
|
39
40
|
self._require_service()
|
40
41
|
|
42
|
+
def _require_nexus(self) -> None:
|
43
|
+
os.environ["WANDB_REQUIRE_NEXUS"] = "True"
|
44
|
+
|
45
|
+
def require_nexus(self) -> None:
|
46
|
+
self._require_nexus()
|
47
|
+
|
41
48
|
def apply(self) -> None:
|
42
49
|
"""Call require_* method for supported features."""
|
43
50
|
last_message: str = ""
|