wandb 0.18.2__py3-none-musllinux_1_2_x86_64.whl

wandb/sdk/lib/apikey.py

@@ -0,0 +1,273 @@
+"""apikey util."""
+
+import os
+import platform
+import stat
+import sys
+import textwrap
+from functools import partial
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Union
+from urllib.parse import urlparse
+
+# import Literal
+if sys.version_info >= (3, 8):
+    from typing import Literal
+else:
+    from typing_extensions import Literal
+
+import click
+from requests.utils import NETRC_FILES, get_netrc_auth
+
+import wandb
+from wandb.apis import InternalApi
+from wandb.errors import term
+from wandb.util import _is_databricks, isatty, prompt_choices
+
+from .wburls import wburls
+
+LOGIN_CHOICE_ANON = "Private W&B dashboard, no account required"
+LOGIN_CHOICE_NEW = "Create a W&B account"
+LOGIN_CHOICE_EXISTS = "Use an existing W&B account"
+LOGIN_CHOICE_DRYRUN = "Don't visualize my results"
+LOGIN_CHOICE_NOTTY = "Unconfigured"
+LOGIN_CHOICES = [
+    LOGIN_CHOICE_ANON,
+    LOGIN_CHOICE_NEW,
+    LOGIN_CHOICE_EXISTS,
+    LOGIN_CHOICE_DRYRUN,
+]
+
+Mode = Literal["allow", "must", "never", "false", "true"]
+
+if TYPE_CHECKING:
+    from wandb.sdk.wandb_settings import Settings
+
+
+getpass = partial(click.prompt, hide_input=True, err=True)
+
+
+def _fixup_anon_mode(default: Optional[Mode]) -> Optional[Mode]:
+    # Convert weird anonymode values from legacy settings files
+    # into one of our expected values.
+    anon_mode = default or "never"
+    mapping: Dict[Mode, Mode] = {"true": "allow", "false": "never"}
+    return mapping.get(anon_mode, anon_mode)
+
+
+def get_netrc_file_path() -> str:
+    """Return the path to the netrc file."""
+    # if the NETRC environment variable is set, use that
+    netrc_file = os.environ.get("NETRC")
+    if netrc_file:
+        return os.path.expanduser(netrc_file)
+
+    # if either .netrc or _netrc exists in the home directory, use that
+    for netrc_file in NETRC_FILES:
+        home_dir = os.path.expanduser("~")
+        if os.path.exists(os.path.join(home_dir, netrc_file)):
+            return os.path.join(home_dir, netrc_file)
+
+    # otherwise, use .netrc on non-Windows platforms and _netrc on Windows
+    netrc_file = ".netrc" if platform.system() != "Windows" else "_netrc"
+
+    return os.path.join(os.path.expanduser("~"), netrc_file)
+
+
+def prompt_api_key(  # noqa: C901
+    settings: "Settings",
+    api: Optional[InternalApi] = None,
+    input_callback: Optional[Callable] = None,
+    browser_callback: Optional[Callable] = None,
+    no_offline: bool = False,
+    no_create: bool = False,
+    local: bool = False,
+) -> Union[str, bool, None]:
+    """Prompt for api key.
+
+    Returns:
+        str - if key is configured
+        None - if dryrun is selected
+        False - if unconfigured (notty)
+    """
+    input_callback = input_callback or getpass
+    log_string = term.LOG_STRING
+    api = api or InternalApi(settings)
+    anon_mode = _fixup_anon_mode(settings.anonymous)  # type: ignore
+    jupyter = settings._jupyter or False
+    app_url = api.app_url
+
+    choices = [choice for choice in LOGIN_CHOICES]
+    if anon_mode == "never":
+        # Omit LOGIN_CHOICE_ANON as a choice if the env var is set to never
+        choices.remove(LOGIN_CHOICE_ANON)
+    if (jupyter and not settings.login_timeout) or no_offline:
+        choices.remove(LOGIN_CHOICE_DRYRUN)
+    if (jupyter and not settings.login_timeout) or no_create:
+        choices.remove(LOGIN_CHOICE_NEW)
+
+    if jupyter and "google.colab" in sys.modules:
+        log_string = term.LOG_STRING_NOCOLOR
+        key = wandb.jupyter.attempt_colab_login(app_url)  # type: ignore
+        if key is not None:
+            write_key(settings, key, api=api)
+            return key  # type: ignore
+
+    if anon_mode == "must":
+        result = LOGIN_CHOICE_ANON
+    # If we're not in an interactive environment, default to dry-run.
+    elif (
+        not jupyter and (not isatty(sys.stdout) or not isatty(sys.stdin))
+    ) or _is_databricks():
+        result = LOGIN_CHOICE_NOTTY
+    elif local:
+        result = LOGIN_CHOICE_EXISTS
+    elif len(choices) == 1:
+        result = choices[0]
+    else:
+        result = prompt_choices(
+            choices, input_timeout=settings.login_timeout, jupyter=jupyter
+        )
+
+    api_ask = (
+        f"{log_string}: Paste an API key from your profile and hit enter, "
+        "or press ctrl+c to quit"
+    )
+    if result == LOGIN_CHOICE_ANON:
+        key = api.create_anonymous_api_key()
+
+        write_key(settings, key, api=api, anonymous=True)
+        return key  # type: ignore
+    elif result == LOGIN_CHOICE_NEW:
+        key = browser_callback(signup=True) if browser_callback else None
+
+        if not key:
+            wandb.termlog(f"Create an account here: {app_url}/authorize?signup=true")
+            key = input_callback(api_ask).strip()
+
+        write_key(settings, key, api=api)
+        return key  # type: ignore
+    elif result == LOGIN_CHOICE_EXISTS:
+        key = browser_callback() if browser_callback else None
+
+        if not key:
+            if not (settings.is_local or local):
+                host = app_url
+                for prefix in ("http://", "https://"):
+                    if app_url.startswith(prefix):
+                        host = app_url[len(prefix) :]
+                wandb.termlog(
+                    f"Logging into {host}. (Learn how to deploy a W&B server locally: {wburls.get('wandb_server')})"
+                )
+            wandb.termlog(
+                f"You can find your API key in your browser here: {app_url}/authorize"
+            )
+            key = input_callback(api_ask).strip()
+        write_key(settings, key, api=api)
+        return key  # type: ignore
+    elif result == LOGIN_CHOICE_NOTTY:
+        # TODO: Needs refactor as this needs to be handled by caller
+        return False
+    elif result == LOGIN_CHOICE_DRYRUN:
+        return None
+    else:
+        # Jupyter environments don't have a tty, but we can still try logging in using
+        # the browser callback if one is supplied.
+        key, anonymous = (
+            browser_callback() if jupyter and browser_callback else (None, False)
+        )
+
+        write_key(settings, key, api=api)
+        return key  # type: ignore
+
+
+def write_netrc(host: str, entity: str, key: str) -> Optional[bool]:
+    """Add our host and key to .netrc."""
+    _, key_suffix = key.split("-", 1) if "-" in key else ("", key)
+    if len(key_suffix) != 40:
+        wandb.termerror(
+            "API-key must be exactly 40 characters long: {} ({} chars)".format(
+                key_suffix, len(key_suffix)
+            )
+        )
+        return None
+    try:
+        normalized_host = urlparse(host).netloc.split(":")[0]
+        netrc_path = get_netrc_file_path()
+        wandb.termlog(
+            f"Appending key for {normalized_host} to your netrc file: {netrc_path}"
+        )
+        machine_line = f"machine {normalized_host}"
+        orig_lines = None
+        try:
+            with open(netrc_path) as f:
+                orig_lines = f.read().strip().split("\n")
+        except OSError:
+            pass
+        with open(netrc_path, "w") as f:
+            if orig_lines:
+                # delete this machine from the file if it's already there.
+                skip = 0
+                for line in orig_lines:
+                    # we fix invalid netrc files with an empty host that we wrote before
+                    # verifying host...
+                    if line == "machine " or machine_line in line:
+                        skip = 2
+                    elif skip:
+                        skip -= 1
+                    else:
+                        f.write("{}\n".format(line))
+            f.write(
+                textwrap.dedent(
+                    """\
+            machine {host}
+              login {entity}
+              password {key}
+            """
+                ).format(host=normalized_host, entity=entity, key=key)
+            )
+        os.chmod(netrc_path, stat.S_IRUSR | stat.S_IWUSR)
+        return True
+    except OSError:
+        wandb.termerror(f"Unable to read {netrc_path}")
+        return None
+
+
+def write_key(
+    settings: "Settings",
+    key: Optional[str],
+    api: Optional["InternalApi"] = None,
+    anonymous: bool = False,
+) -> None:
+    if not key:
+        raise ValueError("No API key specified.")
+
+    # TODO(jhr): api shouldn't be optional or it shouldn't be passed, clean up callers
+    api = api or InternalApi()
+
+    # Normal API keys are 40-character hex strings. On-prem API keys have a
+    # variable-length prefix, a dash, then the 40-char string.
+    _, suffix = key.split("-", 1) if "-" in key else ("", key)
+
+    if len(suffix) != 40:
+        raise ValueError(
+            "API key must be 40 characters long, yours was {}".format(len(key))
+        )
+
+    if anonymous:
+        api.set_setting("anonymous", "true", globally=True, persist=True)
+    else:
+        api.clear_setting("anonymous", globally=True, persist=True)
+
+    write_netrc(settings.base_url, "user", key)
+
+
+def api_key(settings: Optional["Settings"] = None) -> Optional[str]:
+    if settings is None:
+        settings = wandb.setup().settings  # type: ignore
+        assert settings is not None
+    if settings.api_key:
+        return settings.api_key
+    auth = get_netrc_auth(settings.base_url)
+    if auth:
+        return auth[-1]
+    return None

wandb/sdk/lib/capped_dict.py

@@ -0,0 +1,26 @@
+import collections
+from typing import Any, Optional
+
+
+class CappedDict(collections.OrderedDict):
+    default_max_size = 50
+
+    def __init__(self, max_size: Optional[int] = None) -> None:
+        self.max_size = max_size or self.default_max_size
+        super().__init__()
+
+    def __setitem__(self, key: str, val: Any) -> None:
+        if key not in self:
+            max_size = self.max_size - 1
+            self._prune_dict(max_size)
+        super().__setitem__(key, val)
+
+    def update(self, **kwargs: Any) -> None:  # type: ignore[override]
+        super().update(**kwargs)
+        self._prune_dict(self.max_size)
+
+    def _prune_dict(self, max_size: int) -> None:
+        if len(self) >= max_size:
+            diff = len(self) - max_size
+            for k in list(self.keys())[:diff]:
+                del self[k]

wandb/sdk/lib/config_util.py

@@ -0,0 +1,101 @@
+import json
+import logging
+import os
+from typing import Any, Dict, Optional
+
+import yaml
+
+import wandb
+from wandb.errors import Error
+from wandb.util import load_yaml
+
+from . import filesystem
+
+logger = logging.getLogger("wandb")
+
+
+class ConfigError(Error):
+    pass
+
+
+def dict_from_proto_list(obj_list):
+    d = dict()
+    for item in obj_list:
+        d[item.key] = dict(desc=None, value=json.loads(item.value_json))
+    return d
+
+
+def dict_strip_value_dict(config_dict):
+    d = dict()
+    for k, v in config_dict.items():
+        d[k] = v["value"]
+    return d
+
+
+def dict_no_value_from_proto_list(obj_list):
+    d = dict()
+    for item in obj_list:
+        possible_dict = json.loads(item.value_json)
+        if not isinstance(possible_dict, dict) or "value" not in possible_dict:
+            continue
+        d[item.key] = possible_dict["value"]
+
+    return d
+
+
+# TODO(jhr): these functions should go away once we merge jobspec PR
+def save_config_file_from_dict(config_filename, config_dict):
+    s = b"wandb_version: 1"
+    if config_dict:  # adding an empty dictionary here causes a parse error
+        s += b"\n\n" + yaml.dump(
+            config_dict,
+            Dumper=yaml.SafeDumper,
+            default_flow_style=False,
+            allow_unicode=True,
+            encoding="utf-8",
+            sort_keys=False,
+        )
+    data = s.decode("utf-8")
+    filesystem.mkdir_exists_ok(os.path.dirname(config_filename))
+    with open(config_filename, "w") as conf_file:
+        conf_file.write(data)
+
+
+def dict_from_config_file(
+    filename: str, must_exist: bool = False
+) -> Optional[Dict[str, Any]]:
+    if not os.path.exists(filename):
+        if must_exist:
+            raise ConfigError("config file {} doesn't exist".format(filename))
+        logger.debug("no default config file found in {}".format(filename))
+        return None
+    try:
+        conf_file = open(filename)
+    except OSError:
+        raise ConfigError("Couldn't read config file: {}".format(filename))
+    try:
+        loaded = load_yaml(conf_file)
+    except yaml.parser.ParserError:
+        raise ConfigError("Invalid YAML in config yaml")
+    if loaded is None:
+        wandb.termwarn(
+            "Found an empty default config file (config-defaults.yaml). Proceeding with no defaults."
+        )
+        return None
+    config_version = loaded.pop("wandb_version", None)
+    if config_version is not None and config_version != 1:
+        raise ConfigError("Unknown config version")
+    data = dict()
+    for k, v in loaded.items():
+        data[k] = v["value"]
+    return data
+
+
+def merge_dicts(dest: dict, src: dict) -> dict:
+    """Recursively merge two dictionaries. Similar to Lodash's _.merge()."""
+    for key, value in src.items():
+        if isinstance(value, dict) and key in dest and isinstance(dest[key], dict):
+            merge_dicts(dest[key], value)
+        else:
+            dest[key] = value
+    return dest

wandb/sdk/lib/credentials.py

@@ -0,0 +1,141 @@
+import json
+import os
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import requests.utils
+
+from wandb.errors import AuthenticationError
+
+DEFAULT_WANDB_CREDENTIALS_FILE = Path(
+    os.path.expanduser("~/.config/wandb/credentials.json")
+)
+
+_expires_at_fmt = "%Y-%m-%d %H:%M:%S"
+
+
+def access_token(base_url: str, token_file: Path, credentials_file: Path) -> str:
+    """Retrieve an access token from the credentials file.
+
+    If no access token exists, create a new one by exchanging the identity
+    token from the token file, and save it to the credentials file.
+
+    Args:
+        base_url (str): The base URL of the server
+        token_file (pathlib.Path): The path to the file containing the
+        identity token
+        credentials_file (pathlib.Path): The path to file used to save
+        temporary access tokens
+
+    Returns:
+        str: The access token
+    """
+    if not credentials_file.exists():
+        _write_credentials_file(base_url, token_file, credentials_file)
+
+    data = _fetch_credentials(base_url, token_file, credentials_file)
+    return data["access_token"]
+
+
+def _write_credentials_file(base_url: str, token_file: Path, credentials_file: Path):
+    """Obtain an access token from the server and write it to the credentials file.
+
+    Args:
+        base_url (str): The base URL of the server
+        token_file (pathlib.Path): The path to the file containing the
+        identity token
+        credentials_file (pathlib.Path): The path to file used to save
+        temporary access tokens
+    """
+    credentials = _create_access_token(base_url, token_file)
+    data = {"credentials": {base_url: credentials}}
+    with open(credentials_file, "w") as file:
+        json.dump(data, file, indent=4)
+
+        # Set file permissions to be read/write by the owner only
+        os.chmod(credentials_file, 0o600)
+
+
+def _fetch_credentials(base_url: str, token_file: Path, credentials_file: Path) -> dict:
+    """Fetch the access token from the credentials file.
+
+    If the access token has expired, fetch a new one from the server and save it
+    to the credentials file.
+
+    Args:
+        base_url (str): The base URL of the server
+        token_file (pathlib.Path): The path to the file containing the
+        identity token
+        credentials_file (pathlib.Path): The path to file used to save
+        temporary access tokens
+
+    Returns:
+        dict: The credentials including the access token.
+    """
+    creds = {}
+    with open(credentials_file) as file:
+        data = json.load(file)
+        if "credentials" not in data:
+            data["credentials"] = {}
+        if base_url in data["credentials"]:
+            creds = data["credentials"][base_url]
+
+    expires_at = datetime.utcnow()
+    if "expires_at" in creds:
+        expires_at = datetime.strptime(creds["expires_at"], _expires_at_fmt)
+
+    if expires_at <= datetime.utcnow():
+        creds = _create_access_token(base_url, token_file)
+        with open(credentials_file, "w") as file:
+            data["credentials"][base_url] = creds
+            json.dump(data, file, indent=4)
+
+    return creds
+
+
+def _create_access_token(base_url: str, token_file: Path) -> dict:
+    """Exchange an identity token for an access token from the server.
+
+    Args:
+        base_url (str): The base URL of the server.
+        token_file (pathlib.Path): The path to the file containing the
+        identity token
+
+    Returns:
+        dict: The access token and its expiration.
+
+    Raises:
+        FileNotFoundError: If the token file is not found.
+        OSError: If there is an issue reading the token file.
+        AuthenticationError: If the server fails to provide an access token.
+    """
+    try:
+        with open(token_file) as file:
+            token = file.read().strip()
+    except FileNotFoundError as e:
+        raise FileNotFoundError(f"Identity token file not found: {token_file}") from e
+    except OSError as e:
+        raise OSError(
+            f"Failed to read the identity token from file: {token_file}"
+        ) from e
+
+    url = f"{base_url}/oidc/token"
+    data = {
+        "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
+        "assertion": token,
+    }
+    headers = {"Content-Type": "application/x-www-form-urlencoded"}
+
+    response = requests.post(url, data=data, headers=headers)
+
+    if response.status_code != 200:
+        raise AuthenticationError(
+            f"Failed to retrieve access token: {response.status_code}, {response.text}"
+        )
+
+    resp_json = response.json()
+    expires_at = datetime.utcnow() + timedelta(seconds=float(resp_json["expires_in"]))
+    resp_json["expires_at"] = expires_at.strftime(_expires_at_fmt)
+    del resp_json["expires_in"]
+
+    return resp_json

wandb/sdk/lib/deprecate.py

@@ -0,0 +1,42 @@
+__all__ = ["deprecate", "Deprecated"]
+
+from typing import TYPE_CHECKING, Optional, Tuple
+
+import wandb
+from wandb.proto.wandb_deprecated import DEPRECATED_FEATURES, Deprecated
+from wandb.proto.wandb_telemetry_pb2 import Deprecated as TelemetryDeprecated
+
+# avoid cycle, use string type reference
+if TYPE_CHECKING:
+    from .. import wandb_run
+
+
+deprecated_field_names: Tuple[str, ...] = tuple(
+    str(v) for k, v in Deprecated.__dict__.items() if not k.startswith("_")
+)
+
+
+def deprecate(
+    field_name: DEPRECATED_FEATURES,
+    warning_message: str,
+    run: Optional["wandb_run.Run"] = None,
+) -> None:
+    """Warn the user that a feature has been deprecated.
+
+    Also stores the information about the event in telemetry.
+
+    Args:
+        field_name: The name of the feature that has been deprecated.
+                    Defined in wandb/proto/wandb_telemetry.proto::Deprecated
+        warning_message: The message to display to the user.
+        run: The run to whose telemetry the event will be added.
+    """
+    known_fields = TelemetryDeprecated.DESCRIPTOR.fields_by_name.keys()
+    if field_name not in known_fields:
+        raise ValueError(
+            f"Unknown field name: {field_name}. Known fields: {known_fields}"
+        )
+    _run = run or wandb.run
+    with wandb.wandb_lib.telemetry.context(run=_run) as tel:  # type: ignore[attr-defined]
+        setattr(tel.deprecated, field_name, True)
+    wandb.termwarn(warning_message, repeat=False)

wandb/sdk/lib/disabled.py

@@ -0,0 +1,29 @@
+from typing import Any
+
+from wandb.sdk.lib import deprecate
+
+
+class SummaryDisabled(dict):
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+    def __getattr__(self, key):
+        return self[key]
+
+    def __getitem__(self, key):
+        val = dict.__getitem__(self, key)
+        if isinstance(val, dict) and not isinstance(val, SummaryDisabled):
+            val = SummaryDisabled(val)
+            self[key] = val
+        return val
+
+
+class RunDisabled:
+    """Compatibility class for integrations that explicitly check for wandb.RunDisabled."""
+
+    def __getattr__(self, name: str) -> Any:
+        deprecate.deprecate(
+            field_name=deprecate.Deprecated.run_disabled,
+            warning_message="RunDisabled is deprecated and is a no-op. "
+            '`wandb.init(mode="disabled")` now returns and instance of `wandb.sdk.wandb_run.Run`.',
+        )

wandb/sdk/lib/exit_hooks.py

@@ -0,0 +1,54 @@
+import sys
+import traceback
+from types import TracebackType
+from typing import TYPE_CHECKING, Optional, Type
+
+import wandb
+from wandb.errors import Error
+
+if TYPE_CHECKING:
+    from typing import NoReturn
+
+
+class ExitHooks:
+    exception: Optional[BaseException] = None
+
+    def __init__(self) -> None:
+        self.exit_code = 0
+        self.exception = None
+
+    def hook(self) -> None:
+        self._orig_exit = sys.exit
+        sys.exit = self.exit
+        self._orig_excepthook = (
+            sys.excepthook
+            if sys.excepthook
+            != sys.__excepthook__  # respect hooks by other libraries like pdb
+            else None
+        )
+        sys.excepthook = self.exc_handler  # type: ignore
+
+    def exit(self, code: object = 0) -> "NoReturn":
+        orig_code = code
+        code = code if code is not None else 0
+        code = code if isinstance(code, int) else 1
+        self.exit_code = code
+        self._orig_exit(orig_code)  # type: ignore
+
+    def was_ctrl_c(self) -> bool:
+        return isinstance(self.exception, KeyboardInterrupt)
+
+    def exc_handler(
+        self, exc_type: Type[BaseException], exc: BaseException, tb: TracebackType
+    ) -> None:
+        self.exit_code = 1
+        self.exception = exc
+        if issubclass(exc_type, Error):
+            wandb.termerror(str(exc), repeat=False)
+
+        if self.was_ctrl_c():
+            self.exit_code = 255
+
+        traceback.print_exception(exc_type, exc, tb)
+        if self._orig_excepthook:
+            self._orig_excepthook(exc_type, exc, tb)