wandb 0.19.12rc1__py3-none-win32.whl → 0.20.1__py3-none-win32.whl

wandb/sdk/backend/backend.py

@@ -4,206 +4,45 @@ Manage backend.
 
 """
 
-import importlib.machinery
 import logging
-import multiprocessing
-import os
-import queue
-import sys
-import threading
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union
-
-import wandb
+from typing import TYPE_CHECKING, Optional
+
 from wandb.sdk.interface.interface import InterfaceBase
-from wandb.sdk.interface.interface_queue import InterfaceQueue
-from wandb.sdk.interface.router_queue import MessageQueueRouter
-from wandb.sdk.internal.internal import wandb_internal
-from wandb.sdk.internal.settings_static import SettingsStatic
-from wandb.sdk.mailbox import Mailbox
 from wandb.sdk.wandb_settings import Settings
 
 if TYPE_CHECKING:
-    from wandb.proto.wandb_internal_pb2 import Record, Result
     from wandb.sdk.lib import service_connection
 
-    RecordQueue = Union["queue.Queue[Record]", multiprocessing.Queue[Record]]
-    ResultQueue = Union["queue.Queue[Result]", multiprocessing.Queue[Result]]
-
 logger = logging.getLogger("wandb")
 
 
-class BackendThread(threading.Thread):
-    """Class to running internal process as a thread."""
-
-    def __init__(self, target: Callable, kwargs: Dict[str, Any]) -> None:
-        threading.Thread.__init__(self)
-        self.name = "BackendThr"
-        self._target = target
-        self._kwargs = kwargs
-        self.daemon = True
-        self.pid = 0
-
-    def run(self) -> None:
-        self._target(**self._kwargs)
-
-
 class Backend:
-    # multiprocessing context or module
-    _multiprocessing: multiprocessing.context.BaseContext
-
     interface: Optional[InterfaceBase]
-    _router: Optional[MessageQueueRouter]
 
-    _internal_pid: Optional[int]
-    wandb_process: Optional[multiprocessing.process.BaseProcess]
     _settings: Settings
-    record_q: Optional["RecordQueue"]
-    result_q: Optional["ResultQueue"]
+
+    _done: bool
+
+    _service: Optional["service_connection.ServiceConnection"]
 
     def __init__(
         self,
         settings: Settings,
-        log_level: Optional[int] = None,
-        service: "Optional[service_connection.ServiceConnection]" = None,
+        service: Optional["service_connection.ServiceConnection"] = None,
     ) -> None:
         self._done = False
-        self.record_q = None
-        self.result_q = None
-        self.wandb_process = None
 
         self.interface = None
-        self._router = None
 
-        self._internal_pid = None
         self._settings = settings
-        self._log_level = log_level
         self._service = service
 
-        self._multiprocessing = multiprocessing  # type: ignore
-        self._multiprocessing_setup()
-
-        # for _module_main_* methods
-        self._save_mod_path: Optional[str] = None
-        self._save_mod_spec = None
-
-    def _multiprocessing_setup(self) -> None:
-        if self._settings.start_method == "thread":
-            return
-
-        # defaulting to spawn for now, fork needs more testing
-        start_method = self._settings.start_method or "spawn"
-
-        # TODO: use fork context if unix and frozen?
-        # if py34+, else fall back
-        if not hasattr(multiprocessing, "get_context"):
-            return
-        all_methods = multiprocessing.get_all_start_methods()
-        logger.info(
-            "multiprocessing start_methods={}, using: {}".format(
-                ",".join(all_methods), start_method
-            )
-        )
-        ctx = multiprocessing.get_context(start_method)
-        self._multiprocessing = ctx
-
-    def _module_main_install(self) -> None:
-        # Support running code without a: __name__ == "__main__"
-        main_module = sys.modules["__main__"]
-        main_mod_spec = getattr(main_module, "__spec__", None)
-        main_mod_path = getattr(main_module, "__file__", None)
-        if main_mod_spec is None:  # hack for pdb
-            # Note: typing has trouble with BuiltinImporter
-            loader: Loader = importlib.machinery.BuiltinImporter  # type: ignore # noqa: F821
-            main_mod_spec = importlib.machinery.ModuleSpec(
-                name="wandb.mpmain", loader=loader
-            )
-            main_module.__spec__ = main_mod_spec
-        else:
-            self._save_mod_spec = main_mod_spec
-
-        if main_mod_path is not None:
-            self._save_mod_path = main_module.__file__
-            fname = os.path.join(
-                os.path.dirname(wandb.__file__), "mpmain", "__main__.py"
-            )
-            main_module.__file__ = fname
-
-    def _module_main_uninstall(self) -> None:
-        main_module = sys.modules["__main__"]
-        # Undo temporary changes from: __name__ == "__main__"
-        main_module.__spec__ = self._save_mod_spec
-        if self._save_mod_path:
-            main_module.__file__ = self._save_mod_path
-
     def ensure_launched(self) -> None:
         """Launch backend worker if not running."""
-        if self._service:
-            assert self._settings.run_id
-            self.interface = self._service.make_interface(
-                stream_id=self._settings.run_id,
-            )
-            return
-
-        settings = self._settings.model_copy()
-        settings.x_log_level = self._log_level or logging.DEBUG
-
-        start_method = settings.start_method
-
-        settings_static = SettingsStatic(settings.to_proto())
-        user_pid = os.getpid()
-
-        if start_method == "thread":
-            self.record_q = queue.Queue()
-            self.result_q = queue.Queue()
-            wandb._set_internal_process(disable=True)  # type: ignore
-            wandb_thread = BackendThread(
-                target=wandb_internal,
-                kwargs=dict(
-                    settings=settings_static,
-                    record_q=self.record_q,
-                    result_q=self.result_q,
-                    user_pid=user_pid,
-                ),
-            )
-            # TODO: risky cast, assumes BackendThread Process duck typing
-            self.wandb_process = wandb_thread  # type: ignore
-        else:
-            self.record_q = self._multiprocessing.Queue()
-            self.result_q = self._multiprocessing.Queue()
-            self.wandb_process = self._multiprocessing.Process(  # type: ignore
-                target=wandb_internal,
-                kwargs=dict(
-                    settings=settings_static,
-                    record_q=self.record_q,
-                    result_q=self.result_q,
-                    user_pid=user_pid,
-                ),
-            )
-            assert self.wandb_process
-            self.wandb_process.name = "wandb_internal"
-
-        self._module_main_install()
-
-        logger.info("starting backend process...")
-        # Start the process with __name__ == "__main__" workarounds
-        assert self.wandb_process
-        self.wandb_process.start()
-        self._internal_pid = self.wandb_process.pid
-        logger.info(f"started backend process with pid: {self.wandb_process.pid}")
-
-        self._module_main_uninstall()
-
-        mailbox = Mailbox()
-        self.interface = InterfaceQueue(
-            process=self.wandb_process,
-            record_q=self.record_q,  # type: ignore
-            result_q=self.result_q,  # type: ignore
-            mailbox=mailbox,
-        )
-        self._router = MessageQueueRouter(
-            request_queue=self.record_q,  # type: ignore
-            response_queue=self.result_q,  # type: ignore
-            mailbox=mailbox,
+        assert self._settings.run_id
+        assert self._service
+        self.interface = self._service.make_interface(
+            stream_id=self._settings.run_id,
         )
 
     def server_status(self) -> None:
@@ -216,13 +55,3 @@ class Backend:
         self._done = True
         if self.interface:
             self.interface.join()
-        if self._router:
-            self._router.join()
-        if self.wandb_process:
-            self.wandb_process.join()
-
-        if self.record_q and hasattr(self.record_q, "close"):
-            self.record_q.close()
-        if self.result_q and hasattr(self.result_q, "close"):
-            self.result_q.close()
-        # No printing allowed from here until redirect restore!!!

wandb/sdk/data_types/_dtypes.py

@@ -686,9 +686,7 @@ class ListType(Type):
             for ndx, obj in enumerate(list(other)):
                 _new_element_type = new_element_type.assign(obj)
                 if isinstance(_new_element_type, InvalidType):
-                    exp += "\n{}Index {}:\n{}".format(
-                        gap, ndx, new_element_type.explain(obj, depth + 1)
-                    )
+                    exp += f"\n{gap}Index {ndx}:\n{new_element_type.explain(obj, depth + 1)}"
                     break
                 new_element_type = _new_element_type
         return exp
@@ -727,9 +725,7 @@ class NDArrayType(Type):
             return cls(shape)
         else:
             raise TypeError(
-                "NDArrayType.from_obj expects py_obj to be ndarray or list, found {}".format(
-                    py_obj.__class__
-                )
+                f"NDArrayType.from_obj expects py_obj to be ndarray or list, found {py_obj.__class__}"
             )
 
     def assign_type(self, wb_type: "Type") -> t.Union["NDArrayType", InvalidType]:

wandb/sdk/data_types/audio.py

@@ -1,6 +1,7 @@
 import hashlib
 import os
-from typing import Optional
+import pathlib
+from typing import TYPE_CHECKING, Optional, Union
 
 from wandb import util
 from wandb.sdk.lib import filesystem, runid
@@ -9,6 +10,9 @@ from . import _dtypes
 from ._private import MEDIA_TMP
 from .base_types.media import BatchableMedia
 
+if TYPE_CHECKING:
+    import numpy as np
+
 
 class Audio(BatchableMedia):
     """Wandb class for audio clips.
@@ -23,13 +27,25 @@ class Audio(BatchableMedia):
 
     _log_type = "audio-file"
 
-    def __init__(self, data_or_path, sample_rate=None, caption=None):
+    def __init__(
+        self,
+        data_or_path: Union[
+            str,
+            pathlib.Path,
+            list,
+            "np.ndarray",
+        ],
+        sample_rate: Optional[int] = None,
+        caption: Optional[str] = None,
+    ):
         """Accept a path to an audio file or a numpy array of audio data."""
         super().__init__(caption=caption)
         self._duration = None
         self._sample_rate = sample_rate
 
-        if isinstance(data_or_path, str):
+        if isinstance(data_or_path, (str, pathlib.Path)):
+            data_or_path = str(data_or_path)
+
             if self.path_is_reference(data_or_path):
                 self._path = data_or_path
                 self._sha256 = hashlib.sha256(data_or_path.encode("utf-8")).hexdigest()
@@ -48,6 +64,7 @@ class Audio(BatchableMedia):
             )
 
             tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".wav")
+
             soundfile.write(tmp_path, data_or_path, sample_rate)
             self._duration = len(data_or_path) / float(sample_rate)
 

wandb/sdk/data_types/base_types/media.py

@@ -1,5 +1,6 @@
 import hashlib
 import os
+import pathlib
 import platform
 import re
 import shutil
@@ -118,7 +119,10 @@ class Media(WBValue):
         self._caption = caption
 
     def _set_file(
-        self, path: str, is_tmp: bool = False, extension: Optional[str] = None
+        self,
+        path: str,
+        is_tmp: bool = False,
+        extension: Optional[str] = None,
     ) -> None:
         self._path = path
         self._is_tmp = is_tmp
@@ -195,9 +199,9 @@ class Media(WBValue):
         else:
             try:
                 shutil.copy(self._path, new_path)
-            except shutil.SameFileError as e:
+            except shutil.SameFileError:
                 if not ignore_copy_err:
-                    raise e
+                    raise
             self._path = new_path
             run._publish_file(media_path)
 
@@ -243,9 +247,7 @@ class Media(WBValue):
                 json_obj["_latest_artifact_path"] = artifact_entry_latest_url
 
             if artifact_entry_url is None or self.is_bound():
-                assert self.is_bound(), "Value of type {} must be bound to a run with bind_to_run() before being serialized to JSON.".format(
-                    type(self).__name__
-                )
+                assert self.is_bound(), f"Value of type {type(self).__name__} must be bound to a run with bind_to_run() before being serialized to JSON."
 
                 assert (
                     self._run is run
@@ -329,7 +331,10 @@ class Media(WBValue):
         )
 
     @staticmethod
-    def path_is_reference(path: Optional[str]) -> bool:
+    def path_is_reference(path: Optional[Union[str, pathlib.Path]]) -> bool:
+        if path is None or isinstance(path, pathlib.Path):
+            return False
+
         return bool(path and re.match(r"^(gs|s3|https?)://", path))
 
 

wandb/sdk/data_types/base_types/wb_value.py

@@ -22,7 +22,7 @@ def _is_maybe_offline() -> bool:
     Returns:
         Whether the user likely configured wandb to be offline.
     """
-    singleton = wandb_setup._setup(start_service=False)
+    singleton = wandb_setup.singleton()
 
     # First check: if there's a run, check if it is offline.
     #
@@ -37,7 +37,7 @@ def _is_maybe_offline() -> bool:
 
 
 def _server_accepts_client_ids() -> bool:
-    from wandb.util import parse_version
+    from packaging.version import parse
 
     # There are versions of W&B Server that cannot accept client IDs. Those versions of
     # the backend have a max_cli_version of less than "0.11.0." If the backend cannot
@@ -51,7 +51,7 @@ def _server_accepts_client_ids() -> bool:
     # client IDs.
 
     if _is_maybe_offline():
-        singleton = wandb_setup._setup(start_service=False)
+        singleton = wandb_setup.singleton()
 
         if run := singleton.most_recent_active_run:
             return run._settings.allow_offline_artifacts
@@ -63,7 +63,7 @@ def _server_accepts_client_ids() -> bool:
     max_cli_version = util._get_max_cli_version()
     if max_cli_version is None:
         return False
-    accepts_client_ids: bool = parse_version("0.11.0") <= parse_version(max_cli_version)
+    accepts_client_ids: bool = parse(max_cli_version) >= parse("0.11.0")
     return accepts_client_ids
 
 
@@ -220,9 +220,7 @@ class WBValue:
     ) -> None:
         assert (
             self._artifact_source is None
-        ), "Cannot update artifact_source. Existing source: {}/{}".format(
-            self._artifact_source.artifact, self._artifact_source.name
-        )
+        ), f"Cannot update artifact_source. Existing source: {self._artifact_source.artifact}/{self._artifact_source.name}"
         self._artifact_source = _WBValueArtifactSource(artifact, name)
 
     def _set_artifact_target(
@@ -230,9 +228,7 @@ class WBValue:
     ) -> None:
         assert (
             self._artifact_target is None
-        ), "Cannot update artifact_target. Existing target: {}/{}".format(
-            self._artifact_target.artifact, self._artifact_target.name
-        )
+        ), f"Cannot update artifact_target. Existing target: {self._artifact_target.artifact}/{self._artifact_target.name}"
         self._artifact_target = _WBValueArtifactTarget(artifact, name)
 
     def _get_artifact_entry_ref_url(self) -> Optional[str]:
@@ -250,10 +246,7 @@ class WBValue:
             and self._artifact_target.artifact._final
             and _server_accepts_client_ids()
         ):
-            return "wandb-client-artifact://{}/{}".format(
-                self._artifact_target.artifact._client_id,
-                type(self).with_suffix(self._artifact_target.name),
-            )
+            return f"wandb-client-artifact://{self._artifact_target.artifact._client_id}/{type(self).with_suffix(self._artifact_target.name)}"
         # Else if we do not support client IDs, but online, then block on upload
         # Note: this is old behavior just to stay backwards compatible
         # with older server versions. This code path should be removed
@@ -281,10 +274,7 @@ class WBValue:
             and self._artifact_target.artifact._final
             and _server_accepts_client_ids()
         ):
-            return "wandb-client-artifact://{}:latest/{}".format(
-                self._artifact_target.artifact._sequence_client_id,
-                type(self).with_suffix(self._artifact_target.name),
-            )
+            return f"wandb-client-artifact://{self._artifact_target.artifact._sequence_client_id}:latest/{type(self).with_suffix(self._artifact_target.name)}"
         # Else if we do not support client IDs, then block on upload
         # Note: this is old behavior just to stay backwards compatible
         # with older server versions. This code path should be removed

wandb/sdk/data_types/bokeh.py

@@ -1,6 +1,8 @@
 import codecs
 import json
 import os
+import pathlib
+from typing import TYPE_CHECKING, Union
 
 from wandb import util
 from wandb.sdk.lib import runid
@@ -9,6 +11,9 @@ from . import _dtypes
 from ._private import MEDIA_TMP
 from .base_types.media import Media
 
+if TYPE_CHECKING:
+    from bokeh import document, model
+
 
 class Bokeh(Media):
     """Wandb class for Bokeh plots.
@@ -19,10 +24,22 @@ class Bokeh(Media):
 
     _log_type = "bokeh-file"
 
-    def __init__(self, data_or_path):
+    def __init__(
+        self,
+        data_or_path: Union[
+            str,
+            pathlib.Path,
+            "document.Document",
+            "model.Model",
+        ],
+    ):
         super().__init__()
         bokeh = util.get_module("bokeh", required=True)
-        if isinstance(data_or_path, str) and os.path.exists(data_or_path):
+        if isinstance(data_or_path, (str, pathlib.Path)) and os.path.exists(
+            data_or_path
+        ):
+            data_or_path = str(data_or_path)
+
             with open(data_or_path) as file:
                 b_json = json.load(file)
             self.b_obj = bokeh.document.Document.from_json(b_json)

wandb/sdk/data_types/helper_types/bounding_boxes_2d.py

@@ -13,6 +13,18 @@ if TYPE_CHECKING:  # pragma: no cover
     from ...wandb_run import Run as LocalRun
 
 
+def _convert_pytorch_tensor_to_list(box_data):
+    for box in box_data:
+        if (
+            "position" in box
+            and "middle" in box["position"]
+            and util.is_pytorch_tensor_typename(
+                util.get_full_typename(box["position"]["middle"])
+            )
+        ):
+            box["position"]["middle"] = box["position"]["middle"].tolist()
+
+
 class BoundingBoxes2D(JSONMetadata):
     """Format images with 2D bounding box overlays for logging to W&B.
 
@@ -195,7 +207,11 @@ class BoundingBoxes2D(JSONMetadata):
             key: (string) The readable name or id for this set of bounding boxes (e.g.
                 predictions, ground_truth)
         """
+        # Pytorch tensors are not serializable to json,
+        # so we convert them to lists to avoid errors later on.
+        _convert_pytorch_tensor_to_list(val.get("box_data", []))
         super().__init__(val)
+
         self._val = val["box_data"]
         self._key = key
         # Add default class mapping
@@ -304,7 +320,7 @@ class BoundingBoxes2D(JSONMetadata):
             # an object with a _type key. Will need to push this change to the UI first to ensure backwards compat
             return self._val
         else:
-            raise ValueError("to_json accepts wandb_run.Run or wandb.Artifact")
+            raise TypeError("to_json accepts wandb_run.Run or wandb.Artifact")
 
     @classmethod
     def from_json(

wandb/sdk/data_types/helper_types/image_mask.py

@@ -147,6 +147,12 @@ class ImageMask(Media):
             self._set_file(val["path"])
         else:
             np = util.get_module("numpy", required="Image mask support requires numpy")
+
+            if util.is_pytorch_tensor_typename(
+                util.get_full_typename(val["mask_data"])
+            ):
+                val["mask_data"] = val["mask_data"].cpu().numpy()
+
             # Add default class mapping
             if "class_labels" not in val:
                 classes = np.unique(val["mask_data"]).astype(np.int32).tolist()
@@ -214,7 +220,7 @@ class ImageMask(Media):
             # Nothing special to add (used to add "digest", but no longer used.)
             return json_dict
         else:
-            raise ValueError("to_json accepts wandb_run.Run or wandb.Artifact")
+            raise TypeError("to_json accepts wandb_run.Run or wandb.Artifact")
 
     @classmethod
     def type_name(cls: Type["ImageMask"]) -> str:

wandb/sdk/data_types/html.py

@@ -1,4 +1,5 @@
 import os
+import pathlib
 from typing import TYPE_CHECKING, Sequence, Type, Union
 
 from wandb.sdk.lib import filesystem, runid
@@ -22,7 +23,7 @@ class Html(BatchableMedia):
 
     def __init__(
         self,
-        data: Union[str, "TextIO"],
+        data: Union[str, pathlib.Path, "TextIO"],
         inject: bool = True,
         data_is_not_path: bool = False,
     ) -> None:
@@ -52,14 +53,13 @@ class Html(BatchableMedia):
         """
         super().__init__()
         data_is_path = (
-            isinstance(data, str)
+            isinstance(data, (str, pathlib.Path))
             and os.path.isfile(data)
             and os.path.splitext(data)[1] == ".html"
         ) and not data_is_not_path
         data_path = ""
         if data_is_path:
-            assert isinstance(data, str)
-            data_path = data
+            data_path = str(data)
             with open(data_path, encoding="utf-8") as file:
                 self.html = file.read()
         elif isinstance(data, str):