wandb 0.21.2__py3-none-macosx_12_0_arm64.whl

wandb/sdk/lib/import_hooks.py

@@ -0,0 +1,275 @@
+"""Implements a post-import hook mechanism.
+
+Styled as per PEP-369. Note that it doesn't cope with modules being reloaded.
+
+Note: This file is based on
+https://github.com/GrahamDumpleton/wrapt/blob/1.12.1/src/wrapt/importer.py
+and manual backports of later patches up to 1.15.0 in the wrapt repository
+(with slight modifications).
+"""
+
+import sys
+import threading
+from importlib.util import find_spec
+from typing import Any, Callable, Dict, Optional, Union
+
+# The dictionary registering any post import hooks to be triggered once
+# the target module has been imported. Once a module has been imported
+# and the hooks fired, the list of hooks recorded against the target
+# module will be truncated but the list left in the dictionary. This
+# acts as a flag to indicate that the module had already been imported.
+
+_post_import_hooks: Dict = {}
+_post_import_hooks_init: bool = False
+_post_import_hooks_lock = threading.RLock()
+
+# Register a new post import hook for the target module name. This
+# differs from the PEP-369 implementation in that it also allows the
+# hook function to be specified as a string consisting of the name of
+# the callback in the form 'module:function'. This will result in a
+# proxy callback being registered which will defer loading of the
+# specified module containing the callback function until required.
+
+
+def _create_import_hook_from_string(name: str) -> Callable:
+    def import_hook(module: Any) -> Callable:
+        module_name, function = name.split(":")
+        attrs = function.split(".")
+        __import__(module_name)
+        callback = sys.modules[module_name]
+        for attr in attrs:
+            callback = getattr(callback, attr)
+        return callback(module)  # type: ignore
+
+    return import_hook
+
+
+def register_post_import_hook(
+    hook: Union[str, Callable], hook_id: str, name: str
+) -> None:
+    # Create a deferred import hook if hook is a string name rather than
+    # a callable function.
+
+    if isinstance(hook, (str,)):
+        hook = _create_import_hook_from_string(hook)
+
+    # Automatically install the import hook finder if it has not already
+    # been installed.
+
+    with _post_import_hooks_lock:
+        global _post_import_hooks_init
+
+        if not _post_import_hooks_init:
+            _post_import_hooks_init = True
+            sys.meta_path.insert(0, ImportHookFinder())  # type: ignore
+
+        # Check if the module is already imported. If not, register the hook
+        # to be called after import.
+
+        module = sys.modules.get(name, None)
+
+        if module is None:
+            _post_import_hooks.setdefault(name, {}).update({hook_id: hook})
+
+    # If the module is already imported, we fire the hook right away. Note that
+    # the hook is called outside of the lock to avoid deadlocks if code run as a
+    # consequence of calling the module import hook in turn triggers a separate
+    # thread which tries to register an import hook.
+
+    if module is not None:
+        hook(module)
+
+
+def unregister_post_import_hook(name: str, hook_id: Optional[str]) -> None:
+    # Remove the import hook if it has been registered.
+    with _post_import_hooks_lock:
+        hooks = _post_import_hooks.get(name)
+
+        if hooks is not None:
+            if hook_id is not None:
+                hooks.pop(hook_id, None)
+
+                if not hooks:
+                    del _post_import_hooks[name]
+            else:
+                del _post_import_hooks[name]
+
+
+def unregister_all_post_import_hooks() -> None:
+    with _post_import_hooks_lock:
+        _post_import_hooks.clear()
+
+
+# Indicate that a module has been loaded. Any post import hooks which
+# were registered against the target module will be invoked. If an
+# exception is raised in any of the post import hooks, that will cause
+# the import of the target module to fail.
+
+
+def notify_module_loaded(module: Any) -> None:
+    name = getattr(module, "__name__", None)
+
+    with _post_import_hooks_lock:
+        hooks = _post_import_hooks.pop(name, {})
+
+    # Note that the hook is called outside of the lock to avoid deadlocks if
+    # code run as a consequence of calling the module import hook in turn
+    # triggers a separate thread which tries to register an import hook.
+    for hook in hooks.values():
+        if hook:
+            hook(module)
+
+
+# A custom module import finder. This intercepts attempts to import
+# modules and watches out for attempts to import target modules of
+# interest. When a module of interest is imported, then any post import
+# hooks which are registered will be invoked.
+
+
+class _ImportHookChainedLoader:
+    def __init__(self, loader: Any) -> None:
+        self.loader = loader
+
+        if hasattr(loader, "load_module"):
+            self.load_module = self._load_module
+        if hasattr(loader, "create_module"):
+            self.create_module = self._create_module
+        if hasattr(loader, "exec_module"):
+            self.exec_module = self._exec_module
+
+    def _set_loader(self, module: Any) -> None:
+        # Set module's loader to self.loader unless it's already set to
+        # something else. Import machinery will set it to spec.loader if it is
+        # None, so handle None as well. The module may not support attribute
+        # assignment, in which case we simply skip it. Note that we also deal
+        # with __loader__ not existing at all. This is to future proof things
+        # due to proposal to remove the attribute as described in the GitHub
+        # issue at https://github.com/python/cpython/issues/77458. Also prior
+        # to Python 3.3, the __loader__ attribute was only set if a custom
+        # module loader was used. It isn't clear whether the attribute still
+        # existed in that case or was set to None.
+
+        class UNDEFINED:
+            pass
+
+        if getattr(module, "__loader__", UNDEFINED) in (None, self):
+            try:
+                module.__loader__ = self.loader
+            except AttributeError:
+                pass
+
+        if (
+            getattr(module, "__spec__", None) is not None
+            and getattr(module.__spec__, "loader", None) is self
+        ):
+            module.__spec__.loader = self.loader
+
+    def _load_module(self, fullname: str) -> Any:
+        module = self.loader.load_module(fullname)
+        self._set_loader(module)
+        notify_module_loaded(module)
+
+        return module
+
+    # Python 3.4 introduced create_module() and exec_module() instead of
+    # load_module() alone. Splitting the two steps.
+
+    def _create_module(self, spec: Any) -> Any:
+        return self.loader.create_module(spec)
+
+    def _exec_module(self, module: Any) -> None:
+        self._set_loader(module)
+        self.loader.exec_module(module)
+        notify_module_loaded(module)
+
+
+class ImportHookFinder:
+    def __init__(self) -> None:
+        self.in_progress: Dict = {}
+
+    def find_module(  # type: ignore
+        self,
+        fullname: str,
+        path: Optional[str] = None,
+    ) -> Optional["_ImportHookChainedLoader"]:
+        # If the module being imported is not one we have registered
+        # post import hooks for, we can return immediately. We will
+        # take no further part in the importing of this module.
+
+        with _post_import_hooks_lock:
+            if fullname not in _post_import_hooks:
+                return None
+
+        # When we are interested in a specific module, we will call back
+        # into the import system a second time to defer to the import
+        # finder that is supposed to handle the importing of the module.
+        # We set an in progress flag for the target module so that on
+        # the second time through we don't trigger another call back
+        # into the import system and cause a infinite loop.
+
+        if fullname in self.in_progress:
+            return None
+
+        self.in_progress[fullname] = True
+
+        # Now call back into the import system again.
+
+        try:
+            # For Python 3 we need to use find_spec().loader
+            # from the importlib.util module. It doesn't actually
+            # import the target module and only finds the
+            # loader. If a loader is found, we need to return
+            # our own loader which will then in turn call the
+            # real loader to import the module and invoke the
+            # post import hooks.
+            loader = getattr(find_spec(fullname), "loader", None)
+
+            if loader and not isinstance(loader, _ImportHookChainedLoader):
+                return _ImportHookChainedLoader(loader)
+
+        finally:
+            del self.in_progress[fullname]
+
+    def find_spec(
+        self, fullname: str, path: Optional[str] = None, target: Any = None
+    ) -> Any:
+        # Since Python 3.4, you are meant to implement find_spec() method
+        # instead of find_module() and since Python 3.10 you get deprecation
+        # warnings if you don't define find_spec().
+
+        # If the module being imported is not one we have registered
+        # post import hooks for, we can return immediately. We will
+        # take no further part in the importing of this module.
+
+        with _post_import_hooks_lock:
+            if fullname not in _post_import_hooks:
+                return None
+
+        # When we are interested in a specific module, we will call back
+        # into the import system a second time to defer to the import
+        # finder that is supposed to handle the importing of the module.
+        # We set an in progress flag for the target module so that on
+        # the second time through we don't trigger another call back
+        # into the import system and cause a infinite loop.
+
+        if fullname in self.in_progress:
+            return None
+
+        self.in_progress[fullname] = True
+
+        # Now call back into the import system again.
+
+        try:
+            # This should only be Python 3 so find_spec() should always
+            # exist so don't need to check.
+            spec = find_spec(fullname)
+            loader = getattr(spec, "loader", None)
+
+            if loader and not isinstance(loader, _ImportHookChainedLoader):
+                assert spec is not None
+                spec.loader = _ImportHookChainedLoader(loader)  # type: ignore
+
+            return spec
+
+        finally:
+            del self.in_progress[fullname]

wandb/sdk/lib/interrupt.py

@@ -0,0 +1,37 @@
+"""Utility to send an interrupt (Ctrl+C) signal to the main thread.
+
+This is necessary because Windows and POSIX use different models for Ctrl+C
+interrupts.
+"""
+
+import platform
+import signal
+import threading
+
+
+def interrupt_main():
+    """Interrupt the main Python thread with a SIGINT signal.
+
+    In POSIX, signal.pthread_kill() is the most reliable way to send a signal
+    to the main thread.
+
+    os.kill() is often recommended, but it isn't guaranteed to deliver the
+    signal to the main OS thread. Likewise, signal.raise_signal() delivers
+    the signal to the current thread in POSIX. The issue is that if any other
+    thread receives the signal, Python will set an internal flag and process it
+    on the main thread at the next opportunity. If the main thread is executing
+    C code or is blocked on a syscall (e.g. time.sleep(999999)) the signal
+    handler won't execute until that's done---i.e. Python won't preempt the OS
+    thread on its own.
+
+    On Windows, pthread_kill is not available and os.kill() ignores its
+    second argument and always kills the process. However,
+    signal.raise_signal() does the right thing.
+    """
+    if platform.system() == "Windows":
+        signal.raise_signal(signal.SIGINT)
+    else:
+        signal.pthread_kill(
+            threading.main_thread().ident,
+            signal.SIGINT,
+        )

wandb/sdk/lib/ipython.py

@@ -0,0 +1,126 @@
+import logging
+import sys
+import warnings
+from typing import Literal, Optional
+
+import wandb
+
+PythonType = Literal["python", "ipython", "jupyter"]
+
+logger = logging.getLogger(__name__)
+
+
+def toggle_button(what="run"):
+    """Returns the HTML for a button used to reveal the element following it.
+
+    The element immediately after the button must have `display: none`.
+    """
+    return (
+        "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">"
+        f"Display W&B {what}"
+        "</button>"
+    )
+
+
+def _get_python_type() -> PythonType:
+    if "IPython" not in sys.modules:
+        return "python"
+
+    try:
+        from IPython import get_ipython  # type: ignore
+
+        # Calling get_ipython can cause an ImportError
+        if get_ipython() is None:
+            return "python"
+    except ImportError:
+        return "python"
+
+    # jupyter-based environments (e.g. jupyter itself, colab, kaggle, etc) have a connection file
+    ip_kernel_app_connection_file = (
+        (get_ipython().config.get("IPKernelApp", {}) or {})
+        .get("connection_file", "")
+        .lower()
+    ) or (
+        (get_ipython().config.get("ColabKernelApp", {}) or {})
+        .get("connection_file", "")
+        .lower()
+    )
+
+    if (
+        ("terminal" in get_ipython().__module__)
+        or ("jupyter" not in ip_kernel_app_connection_file)
+        or ("spyder" in sys.modules)
+    ):
+        return "ipython"
+    else:
+        return "jupyter"
+
+
+def in_jupyter() -> bool:
+    """Returns True if we're in a Jupyter notebook."""
+    return _get_python_type() == "jupyter"
+
+
+def in_ipython() -> bool:
+    """Returns True if we're running in IPython in the terminal."""
+    return _get_python_type() == "ipython"
+
+
+def in_notebook() -> bool:
+    """Returns True if we're running in Jupyter or IPython."""
+    return _get_python_type() != "python"
+
+
+class ProgressWidget:
+    """A simple wrapper to render a nice progress bar with a label."""
+
+    def __init__(self, widgets, min, max):
+        from IPython import display
+
+        self._ipython_display = display
+
+        self.widgets = widgets
+        self._progress = widgets.FloatProgress(min=min, max=max)
+        self._label = widgets.Label()
+        self._widget = self.widgets.VBox([self._label, self._progress])
+        self._displayed = False
+        self._disabled = False
+
+    def update(self, value: float, label: str) -> None:
+        if self._disabled:
+            return
+        try:
+            self._progress.value = value
+            self._label.value = label
+            if not self._displayed:
+                self._displayed = True
+                self._ipython_display.display(self._widget)
+        except Exception:
+            logger.exception("Error in ProgressWidget.update()")
+            self._disabled = True
+            wandb.termwarn(
+                "Unable to render progress bar, see the user log for details"
+            )
+
+    def close(self) -> None:
+        if self._disabled or not self._displayed:
+            return
+        self._widget.close()
+
+
+def jupyter_progress_bar(min: float = 0, max: float = 1.0) -> Optional[ProgressWidget]:
+    """Return an ipywidget progress bar or None if we can't import it."""
+    widgets = wandb.util.get_module("ipywidgets")
+    try:
+        if widgets is None:
+            # TODO: this currently works in iPython but it's deprecated since 4.0
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                from IPython.html import widgets  # type: ignore
+
+        assert hasattr(widgets, "VBox")
+        assert hasattr(widgets, "Label")
+        assert hasattr(widgets, "FloatProgress")
+        return ProgressWidget(widgets, min=min, max=max)
+    except (ImportError, AssertionError):
+        return None

wandb/sdk/lib/json_util.py

@@ -0,0 +1,75 @@
+import json
+import logging
+import os
+from typing import Any, Union
+
+logger = logging.getLogger(__name__)
+
+
+try:
+    import orjson  # type: ignore
+
+    # todo: orjson complies with the json standard and does not support
+    #  NaN, Infinity, and -Infinity. Should be fixed in the future.
+
+    # additional safeguard for now
+    if os.environ.get("_WANDB_ORJSON"):
+
+        def dumps(obj: Any, **kwargs: Any) -> str:
+            """Wrapper for <json|orjson>.dumps."""
+            cls = kwargs.pop("cls", None)
+            try:
+                _kwargs = kwargs.copy()
+                if cls:
+                    _kwargs["default"] = cls.default
+                encoded = orjson.dumps(
+                    obj, option=orjson.OPT_NON_STR_KEYS, **_kwargs
+                ).decode()
+            except Exception:
+                logger.exception("Error using orjson.dumps")
+                if cls:
+                    kwargs["cls"] = cls
+                encoded = json.dumps(obj, **kwargs)
+
+            return encoded  # type: ignore[no-any-return]
+
+        def dump(obj: Any, fp: Any, **kwargs: Any) -> None:
+            """Wrapper for <json|orjson>.dump."""
+            cls = kwargs.pop("cls", None)
+            try:
+                _kwargs = kwargs.copy()
+                if cls:
+                    _kwargs["default"] = cls.default
+                encoded = orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS, **_kwargs)
+                fp.write(encoded)
+            except Exception:
+                logger.exception("Error using orjson.dump")
+                if cls:
+                    kwargs["cls"] = cls
+                json.dump(obj, fp, **kwargs)
+
+        def loads(obj: Union[str, bytes]) -> Any:
+            """Wrapper for orjson.loads."""
+            try:
+                decoded = orjson.loads(obj)
+            except Exception:
+                logger.exception("Error using orjson.loads")
+                decoded = json.loads(obj)
+
+            return decoded
+
+        def load(fp: Any) -> Any:
+            """Wrapper for orjson.load."""
+            try:
+                decoded = orjson.loads(fp.read())
+            except Exception:
+                logger.exception("Error using orjson.load")
+                decoded = json.load(fp)
+
+            return decoded
+
+    else:
+        from json import dump, dumps, load, loads  # type: ignore[assignment]
+
+except ImportError:
+    from json import dump, dumps, load, loads  # type: ignore[assignment] # noqa: F401

wandb/sdk/lib/lazyloader.py

@@ -0,0 +1,63 @@
+"""module lazyloader."""
+
+import importlib
+import sys
+import types
+
+
+class LazyLoader(types.ModuleType):
+    """Lazily import a module, mainly to avoid pulling in large dependencies.
+
+    We use this for tensorflow and other optional libraries primarily at the
+    top module level.
+    """
+
+    # The lint error here is incorrect.
+    def __init__(
+        self,
+        local_name,  # pylint: disable=super-on-old-class
+        parent_module_globals,
+        name,
+        warning=None,
+    ):
+        self._local_name = local_name
+        self._parent_module_globals = parent_module_globals
+        self._warning = warning
+
+        super().__init__(str(name))
+
+    def _load(self):
+        """Load the module and insert it into the parent's globals."""
+        # Import the target module and insert it into the parent's namespace
+        module = importlib.import_module(self.__name__)
+        self._parent_module_globals[self._local_name] = module
+        # print("import", self.__name__)
+        # print("Set global", self._local_name)
+        # print("mod", module)
+        sys.modules[self._local_name] = module
+
+        # Emit a warning if one was specified
+        if self._warning:
+            print(self._warning)  # noqa: T201
+            # Make sure to only warn once.
+            self._warning = None
+
+        # Update this object's dict so that if someone keeps a reference to the
+        #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
+        #   that fail).
+        self.__dict__.update(module.__dict__)
+
+        return module
+
+    # def __getattribute__(self, item):
+    #     print("getattribute", item)
+
+    def __getattr__(self, item):
+        # print("getattr", item)
+        module = self._load()
+        return getattr(module, item)
+
+    def __dir__(self):
+        # print("dir")
+        module = self._load()
+        return dir(module)

wandb/sdk/lib/module.py

@@ -0,0 +1,72 @@
+#
+import wandb
+
+from . import preinit
+
+
+def set_global(
+    run=None,
+    config=None,
+    log=None,
+    summary=None,
+    save=None,
+    use_artifact=None,
+    log_artifact=None,
+    define_metric=None,
+    alert=None,
+    mark_preempting=None,
+    log_model=None,
+    use_model=None,
+    link_model=None,
+    watch=None,
+    unwatch=None,
+):
+    if run:
+        wandb.run = run
+    if config is not None:
+        wandb.config = config
+    if log:
+        wandb.log = log
+    if summary is not None:
+        wandb.summary = summary
+    if save:
+        wandb.save = save
+    if use_artifact:
+        wandb.use_artifact = use_artifact
+    if log_artifact:
+        wandb.log_artifact = log_artifact
+    if define_metric:
+        wandb.define_metric = define_metric
+    if alert:
+        wandb.alert = alert
+    if mark_preempting:
+        wandb.mark_preempting = mark_preempting
+    if log_model:
+        wandb.log_model = log_model
+    if use_model:
+        wandb.use_model = use_model
+    if link_model:
+        wandb.link_model = link_model
+    if watch:
+        wandb.watch = watch
+    if unwatch:
+        wandb.unwatch = unwatch
+
+
+def unset_globals():
+    wandb.run = None
+    wandb.config = preinit.PreInitObject("wandb.config")
+    wandb.summary = preinit.PreInitObject("wandb.summary")
+    wandb.log = preinit.PreInitCallable("wandb.log", wandb.Run.log)
+    wandb.watch = preinit.PreInitCallable("wandb.watch", wandb.Run.watch)
+    wandb.unwatch = preinit.PreInitCallable("wandb.unwatch", wandb.Run.unwatch)
+    wandb.save = preinit.PreInitCallable("wandb.save", wandb.Run.save)
+    wandb.use_artifact = preinit.PreInitCallable(
+        "wandb.use_artifact", wandb.Run.use_artifact
+    )
+    wandb.log_artifact = preinit.PreInitCallable(
+        "wandb.log_artifact", wandb.Run.log_artifact
+    )
+    wandb.define_metric = preinit.PreInitCallable(
+        "wandb.define_metric", wandb.Run.define_metric
+    )