wandb 0.15.9__py3-none-any.whl → 0.15.11__py3-none-any.whl

wandb/util.py

@@ -4,6 +4,7 @@ import functools
 import gzip
 import importlib
 import importlib.util
+import itertools
 import json
 import logging
 import math
@@ -13,8 +14,10 @@ import platform
 import queue
 import random
 import re
+import secrets
 import shlex
 import socket
+import string
 import sys
 import tarfile
 import tempfile
@@ -42,6 +45,7 @@ from typing import (
     Set,
     TextIO,
     Tuple,
+    TypeVar,
     Union,
 )
 
@@ -62,6 +66,7 @@ if TYPE_CHECKING:
     from wandb.sdk.artifacts.artifact import Artifact
 
 CheckRetryFnType = Callable[[Exception], Union[bool, timedelta]]
+T = TypeVar("T")
 
 
 logger = logging.getLogger(__name__)
@@ -629,6 +634,9 @@ def json_friendly(  # noqa: C901
         obj = None
     elif isinstance(obj, dict) and np:
         obj, converted = _sanitize_numpy_keys(obj)
+    elif isinstance(obj, set):
+        # set is not json serializable, so we convert it to tuple
+        obj = tuple(obj)
     else:
         converted = False
     if getsizeof(obj) > VALUE_BYTES_LIMIT:
@@ -875,7 +883,7 @@ def no_retry_auth(e: Any) -> bool:
     if not isinstance(e, requests.HTTPError):
         return True
     if e.response is None:
-        return True
+        return True  # type: ignore
     # Don't retry bad request errors; raise immediately
     if e.response.status_code in (400, 409):
         return False
@@ -1315,6 +1323,22 @@ def download_file_from_url(
             file.write(data)
 
 
+def download_file_into_memory(source_url: str, api_key: Optional[str] = None) -> bytes:
+    auth = None
+    if not _thread_local_api_settings.cookies:
+        auth = ("api", api_key or "")
+    response = requests.get(
+        source_url,
+        auth=auth,
+        headers=_thread_local_api_settings.headers,
+        cookies=_thread_local_api_settings.cookies,
+        stream=True,
+        timeout=5,
+    )
+    response.raise_for_status()
+    return response.content
+
+
 def isatty(ob: IO) -> bool:
     return hasattr(ob, "isatty") and ob.isatty()
 
@@ -1484,7 +1508,7 @@ def rand_alphanumeric(
 def fsync_open(
     path: StrPath, mode: str = "w", encoding: Optional[str] = None
 ) -> Generator[IO[Any], None, None]:
-    """Open a path for I/O and guarante that the file is flushed and synced."""
+    """Open a path for I/O and guarantee that the file is flushed and synced."""
     with open(path, mode, encoding=encoding) as f:
         yield f
 
@@ -1526,8 +1550,9 @@ def _is_databricks() -> bool:
     return False
 
 
-def _is_py_path(path: str) -> bool:
-    return path.endswith(".py")
+def _is_py_or_dockerfile(path: str) -> bool:
+    file = os.path.basename(path)
+    return file.endswith(".py") or file.startswith("Dockerfile")
 
 
 def check_windows_valid_filename(path: Union[int, str]) -> bool:
@@ -1753,3 +1778,61 @@ def cast_dictlike_to_dict(d: Dict[str, Any]) -> Dict[str, Any]:
             d[k] = dict(v)
             cast_dictlike_to_dict(d[k])
     return d
+
+
+def remove_keys_with_none_values(
+    d: Union[Dict[str, Any], Any]
+) -> Union[Dict[str, Any], Any]:
+    # otherwise iterrows will create a bunch of ugly charts
+    if not isinstance(d, dict):
+        return d
+
+    if isinstance(d, dict):
+        new_dict = {}
+        for k, v in d.items():
+            new_v = remove_keys_with_none_values(v)
+            if new_v is not None and not (isinstance(new_v, dict) and len(new_v) == 0):
+                new_dict[k] = new_v
+        return new_dict if new_dict else None
+
+
+def batched(n: int, iterable: Iterable[T]) -> Generator[List[T], None, None]:
+    i = iter(iterable)
+    batch = list(itertools.islice(i, n))
+    while batch:
+        yield batch
+        batch = list(itertools.islice(i, n))
+
+
+def random_string(length: int = 12) -> str:
+    """Generate a random string of a given length.
+
+    :param length: Length of the string to generate.
+    :return: Random string.
+    """
+    return "".join(
+        secrets.choice(string.ascii_lowercase + string.digits) for _ in range(length)
+    )
+
+
+def sample_with_exponential_decay_weights(
+    xs: Union[Iterable, Iterable[Iterable]],
+    ys: Iterable[Iterable],
+    keys: Optional[Iterable] = None,
+    sample_size: int = 1500,
+) -> Tuple[List, List, Optional[List]]:
+    """Sample from a list of lists with weights that decay exponentially.
+
+    May be used with the wandb.plot.line_series function.
+    """
+    xs_array = np.array(xs)
+    ys_array = np.array(ys)
+    keys_array = np.array(keys) if keys else None
+    weights = np.exp(-np.arange(len(xs_array)) / len(xs_array))
+    weights /= np.sum(weights)
+    sampled_indices = np.random.choice(len(xs_array), size=sample_size, p=weights)
+    sampled_xs = xs_array[sampled_indices].tolist()
+    sampled_ys = ys_array[sampled_indices].tolist()
+    sampled_keys = keys_array[sampled_indices].tolist() if keys else None
+
+    return sampled_xs, sampled_ys, sampled_keys

wandb/wandb_torch.py

@@ -147,69 +147,55 @@ class TorchHistory:
     def log_tensor_stats(self, tensor, name):
         """Add distribution statistics on a tensor's elements to the current History entry"""
         # TODO Handle the case of duplicate names.
-
-        if isinstance(tensor, tuple) or isinstance(tensor, list):
-            while (isinstance(tensor, tuple) or isinstance(tensor, list)) and (
-                isinstance(tensor[0], tuple) or isinstance(tensor[0], list)
+        if isinstance(tensor, (tuple, list)):
+            while isinstance(tensor, (tuple, list)) and isinstance(
+                tensor[0], (tuple, list)
             ):
                 tensor = [item for sublist in tensor for item in sublist]
-            tensor = torch.cat([t.reshape(-1) for t in tensor])
+            tensor = torch.cat([t.detach().clone().reshape(-1) for t in tensor])
 
+        tensor = tensor.detach().clone()
         # checking for inheritance from _TensorBase didn't work for some reason
         if not hasattr(tensor, "shape"):
             cls = type(tensor)
             raise TypeError(f"Expected Tensor, not {cls.__module__}.{cls.__name__}")
 
-        # HalfTensors on cpu do not support view(), upconvert to 32bit
-        if isinstance(tensor, torch.HalfTensor):
-            tensor = tensor.clone().type(torch.FloatTensor).detach()
-
         # Sparse tensors have a bunch of implicit zeros. In order to histo them correctly,
         # we have to count them up and add them to the histo ourselves.
         sparse_zeros = None
         if tensor.is_sparse:
             # Have to call this on a sparse tensor before most other ops.
-            tensor = tensor.cpu().coalesce().clone().detach()
+            tensor = tensor.cpu().coalesce()
 
             backing_values = tensor._values()
-            non_zero_values = backing_values.numel()
-            all_values = tensor.numel()
-            sparse_zeros = all_values - non_zero_values
+            sparse_zeros = tensor.numel() - backing_values.numel()
             tensor = backing_values
 
         flat = tensor.reshape(-1)
 
-        # For pytorch 0.3 we use unoptimized numpy histograms (detach is new in 0.4)
-        if not hasattr(flat, "detach"):
-            tensor = flat.cpu().clone().numpy()
-            wandb.run._log({name: wandb.Histogram(tensor)}, commit=False)
-            return
-
         if flat.is_cuda:
-            # TODO(jhr): see if pytorch will accept something upstream to check cuda support for ops
-            # until then, we are going to have to catch a specific exception to check for histc support.
             if self._is_cuda_histc_supported is None:
-                self._is_cuda_histc_supported = True
-                check = torch.cuda.FloatTensor(1).fill_(0)
                 try:
-                    check = flat.histc(bins=self._num_bins)
-                except RuntimeError as e:
-                    # Only work around missing support with specific exception
-                    # if str(e).startswith("_th_histc is not implemented"):
-                    #    self._is_cuda_histc_supported = False
-                    # On second thought, 0.4.1 doesnt have support and maybe there are other issues
-                    # lets disable more broadly for now
+                    flat.histc(bins=self._num_bins)
+                except RuntimeError:
                     self._is_cuda_histc_supported = False
-
-            if not self._is_cuda_histc_supported:
-                flat = flat.cpu().clone().detach()
+                else:
+                    self._is_cuda_histc_supported = True
 
             # As of torch 1.0.1.post2+nightly, float16 cuda summary ops are not supported (convert to float32)
-            if isinstance(flat, torch.cuda.HalfTensor):
-                flat = flat.clone().type(torch.cuda.FloatTensor).detach()
+            if not self._is_cuda_histc_supported:
+                flat = flat.cpu()
+            elif not isinstance(
+                flat, (torch.cuda.FloatTensor, torch.cuda.DoubleTensor)
+            ):
+                flat = flat.type(torch.cuda.FloatTensor)
 
-        if isinstance(flat, torch.HalfTensor):
-            flat = flat.clone().type(torch.FloatTensor).detach()
+        # Since we use histc, we need to make sure that torch supports the operation on CPU,
+        # otherwise we'll get a runtime error. Hence, we need to upcast to float32.
+        if not flat.is_cuda and not isinstance(
+            flat, (torch.FloatTensor, torch.DoubleTensor)
+        ):
+            flat = flat.type(torch.FloatTensor)
 
         # Skip logging if all values are nan or inf or the tensor is empty.
         if self._no_finite_values(flat):
@@ -229,7 +215,7 @@ class TorchHistory:
         if tmin > tmax:
             tmin, tmax = tmax, tmin
         tensor = flat.histc(bins=self._num_bins, min=tmin, max=tmax)
-        tensor = tensor.cpu().clone().detach()
+        tensor = tensor.cpu().detach().clone()
         bins = torch.linspace(tmin, tmax, steps=self._num_bins + 1)
 
         # Add back zeroes from a sparse tensor.

{wandb-0.15.9.dist-info → wandb-0.15.11.dist-info}/METADATA

@@ -1,21 +1,46 @@
 Metadata-Version: 2.1
 Name: wandb
-Version: 0.15.9
-Summary: A CLI and library for interacting with the Weights and Biases API.
-Home-page: https://github.com/wandb/wandb
-Author: Weights & Biases
-Author-email: support@wandb.com
-License: MIT license
+Version: 0.15.11
+Summary: A CLI and library for interacting with the Weights & Biases API.
+Author-email: Weights & Biases <support@wandb.com>
+License: MIT License
+        
+        Copyright (c) 2021 Weights and Biases, Inc.
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Source, https://github.com/wandb/wandb
+Project-URL: Bug Reports, https://github.com/wandb/wandb/issues
+Project-URL: Documentation, https://docs.wandb.ai/
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Natural Language :: English
-Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Logging
@@ -23,25 +48,24 @@ Classifier: Topic :: System :: Monitoring
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: Click (!=8.0.0,>=7.1)
-Requires-Dist: GitPython (!=3.1.29,>=1.0.0)
-Requires-Dist: requests (<3,>=2.0.0)
-Requires-Dist: psutil (>=5.0.0)
-Requires-Dist: sentry-sdk (>=1.0.0)
-Requires-Dist: docker-pycreds (>=0.4.0)
+Requires-Dist: Click !=8.0.0,>=7.1
+Requires-Dist: GitPython !=3.1.29,>=1.0.0
+Requires-Dist: requests <3,>=2.0.0
+Requires-Dist: psutil >=5.0.0
+Requires-Dist: sentry-sdk >=1.0.0
+Requires-Dist: docker-pycreds >=0.4.0
 Requires-Dist: PyYAML
 Requires-Dist: pathtools
 Requires-Dist: setproctitle
 Requires-Dist: setuptools
-Requires-Dist: appdirs (>=1.4.3)
+Requires-Dist: appdirs >=1.4.3
 Requires-Dist: typing-extensions ; python_version < "3.10"
-Requires-Dist: dataclasses ; python_version < "3.7"
-Requires-Dist: protobuf (!=4.21.0,<5,>=3.12.0) ; python_version < "3.9" and sys_platform == "linux"
-Requires-Dist: protobuf (!=4.21.0,<5,>=3.15.0) ; python_version == "3.9" and sys_platform == "linux"
-Requires-Dist: protobuf (!=4.21.0,<5,>=3.19.0) ; python_version > "3.9" and sys_platform == "linux"
-Requires-Dist: protobuf (!=4.21.0,<5,>=3.19.0) ; sys_platform != "linux"
+Requires-Dist: protobuf !=4.21.0,<5,>=3.12.0 ; python_version < "3.9" and sys_platform == "linux"
+Requires-Dist: protobuf !=4.21.0,<5,>=3.15.0 ; python_version == "3.9" and sys_platform == "linux"
+Requires-Dist: protobuf !=4.21.0,<5,>=3.19.0 ; python_version > "3.9" and sys_platform == "linux"
+Requires-Dist: protobuf !=4.21.0,<5,>=3.19.0 ; sys_platform != "linux"
 Provides-Extra: async
-Requires-Dist: httpx (>=0.22.0) ; extra == 'async'
+Requires-Dist: httpx >=0.22.0 ; extra == 'async'
 Provides-Extra: aws
 Requires-Dist: boto3 ; extra == 'aws'
 Provides-Extra: azure
@@ -49,8 +73,6 @@ Requires-Dist: azure-identity ; extra == 'azure'
 Requires-Dist: azure-storage-blob ; extra == 'azure'
 Provides-Extra: gcp
 Requires-Dist: google-cloud-storage ; extra == 'gcp'
-Provides-Extra: grpc
-Requires-Dist: grpcio (>=1.27.2) ; extra == 'grpc'
 Provides-Extra: kubeflow
 Requires-Dist: kubernetes ; extra == 'kubeflow'
 Requires-Dist: minio ; extra == 'kubeflow'
@@ -74,6 +96,7 @@ Requires-Dist: optuna ; extra == 'launch'
 Requires-Dist: nbconvert ; extra == 'launch'
 Requires-Dist: nbformat ; extra == 'launch'
 Requires-Dist: typing-extensions ; extra == 'launch'
+Requires-Dist: PyYAML >=6.0.0 ; extra == 'launch'
 Provides-Extra: media
 Requires-Dist: numpy ; extra == 'media'
 Requires-Dist: moviepy ; extra == 'media'
@@ -84,10 +107,12 @@ Requires-Dist: plotly ; extra == 'media'
 Requires-Dist: rdkit-pypi ; extra == 'media'
 Provides-Extra: models
 Requires-Dist: cloudpickle ; extra == 'models'
+Provides-Extra: nexus
+Requires-Dist: wandb-core >=0.16.0b1 ; extra == 'nexus'
 Provides-Extra: perf
 Requires-Dist: orjson ; extra == 'perf'
 Provides-Extra: sweeps
-Requires-Dist: sweeps (>=0.2.0) ; extra == 'sweeps'
+Requires-Dist: sweeps >=0.2.0 ; extra == 'sweeps'
 
 <div align="center">
   <img src="https://i.imgur.com/RUtiVzH.png" width="600" /><br><br>