wandb 0.19.1rc1__py3-none-musllinux_1_2_aarch64.whl

wandb/integration/sagemaker/auth.py

@@ -0,0 +1,28 @@
+import os
+
+import wandb
+from wandb import env
+
+
+def sagemaker_auth(overrides=None, path=".", api_key=None):
+    """Write a secrets.env file with the W&B ApiKey and any additional secrets passed.
+
+    Args:
+        overrides (dict, optional): Additional environment variables to write
+                                    to secrets.env
+        path (str, optional): The path to write the secrets file.
+    """
+    settings = wandb.setup().settings
+    current_api_key = wandb.wandb_lib.apikey.api_key(settings=settings)
+
+    overrides = overrides or dict()
+    api_key = overrides.get(env.API_KEY, api_key or current_api_key)
+    if api_key is None:
+        raise ValueError(
+            "Can't find W&B ApiKey, set the WANDB_API_KEY env variable "
+            "or run `wandb login`"
+        )
+    overrides[env.API_KEY] = api_key
+    with open(os.path.join(path, "secrets.env"), "w") as file:
+        for k, v in overrides.items():
+            file.write(f"{k}={v}\n")

wandb/integration/sagemaker/config.py

@@ -0,0 +1,49 @@
+import json
+import os
+import re
+import warnings
+from typing import Any, Dict
+
+from . import files as sm_files
+
+
+def parse_sm_config() -> Dict[str, Any]:
+    """Parses SageMaker configuration.
+
+    Returns:
+        A dictionary of SageMaker config keys/values
+        or an empty dict if not found.
+        SM_TRAINING_ENV is a json string of the
+        training environment variables set by SageMaker
+        and is only available when running in SageMaker,
+        but not in local mode.
+        SM_TRAINING_ENV is set by the SageMaker container and
+        contains arguments such as hyperparameters
+        and arguments passed to the training job.
+    """
+    conf = {}
+
+    if os.path.exists(sm_files.SM_PARAM_CONFIG) and os.path.exists(
+        sm_files.SM_RESOURCE_CONFIG
+    ):
+        conf["sagemaker_training_job_name"] = os.getenv("TRAINING_JOB_NAME")
+
+        # Hyperparameter searches quote configs...
+        with open(sm_files.SM_PARAM_CONFIG) as fid:
+            for key, val in json.load(fid).items():
+                cast = val.strip('"')
+                if re.match(r"^-?[\d]+$", cast):
+                    cast = int(cast)
+                elif re.match(r"^-?[.\d]+$", cast):
+                    cast = float(cast)
+                conf[key] = cast
+
+    if "SM_TRAINING_ENV" in os.environ:
+        try:
+            conf = {**conf, **json.loads(os.environ["SM_TRAINING_ENV"])}
+        except json.JSONDecodeError:
+            warnings.warn(
+                "Failed to parse SM_TRAINING_ENV not valid JSON string", stacklevel=2
+            )
+
+    return conf

wandb/integration/sagemaker/files.py

@@ -0,0 +1,3 @@
+SM_PARAM_CONFIG = "/opt/ml/input/config/hyperparameters.json"
+SM_RESOURCE_CONFIG = "/opt/ml/input/config/resourceconfig.json"
+SM_SECRETS = "secrets.env"

wandb/integration/sagemaker/resources.py

@@ -0,0 +1,34 @@
+import os
+import secrets
+import socket
+import string
+from typing import Dict, Tuple
+
+from . import files as sm_files
+
+
+def parse_sm_secrets() -> Dict[str, str]:
+    """We read our api_key from secrets.env in SageMaker."""
+    env_dict = dict()
+    # Set secret variables
+    if os.path.exists(sm_files.SM_SECRETS):
+        for line in open(sm_files.SM_SECRETS):
+            key, val = line.strip().split("=", 1)
+            env_dict[key] = val
+    return env_dict
+
+
+def parse_sm_resources() -> Tuple[Dict[str, str], Dict[str, str]]:
+    run_dict = dict()
+    run_id = os.getenv("TRAINING_JOB_NAME")
+
+    if run_id and os.getenv("WANDB_RUN_ID") is None:
+        suffix = "".join(
+            secrets.choice(string.ascii_lowercase + string.digits) for _ in range(6)
+        )
+        run_dict["run_id"] = "-".join(
+            [run_id, suffix, os.getenv("CURRENT_HOST", socket.gethostname())]
+        )
+    run_dict["run_group"] = os.getenv("TRAINING_JOB_NAME")
+    env_dict = parse_sm_secrets()
+    return run_dict, env_dict

wandb/integration/sb3/__init__.py

@@ -0,0 +1,3 @@
+from .sb3 import WandbCallback
+
+__all__ = ["WandbCallback"]

wandb/integration/sb3/sb3.py

@@ -0,0 +1,147 @@
+"""W&B callback for sb3.
+
+Really simple callback to get logging for each tree
+
+Example usage:
+
+```python
+import gym
+from stable_baselines3 import PPO
+from stable_baselines3.common.monitor import Monitor
+from stable_baselines3.common.vec_env import DummyVecEnv, VecVideoRecorder
+import wandb
+from wandb.integration.sb3 import WandbCallback
+
+
+config = {
+    "policy_type": "MlpPolicy",
+    "total_timesteps": 25000,
+    "env_name": "CartPole-v1",
+}
+run = wandb.init(
+    project="sb3",
+    config=config,
+    sync_tensorboard=True,  # auto-upload sb3's tensorboard metrics
+    monitor_gym=True,  # auto-upload the videos of agents playing the game
+    save_code=True,  # optional
+)
+
+
+def make_env():
+    env = gym.make(config["env_name"])
+    env = Monitor(env)  # record stats such as returns
+    return env
+
+
+env = DummyVecEnv([make_env])
+env = VecVideoRecorder(
+    env, "videos", record_video_trigger=lambda x: x % 2000 == 0, video_length=200
+)
+model = PPO(config["policy_type"], env, verbose=1, tensorboard_log=f"runs")
+model.learn(
+    total_timesteps=config["total_timesteps"],
+    callback=WandbCallback(
+        model_save_path=f"models/{run.id}",
+        gradient_save_freq=100,
+        log="all",
+    ),
+)
+```
+"""
+
+import logging
+import os
+from typing import Literal, Optional
+
+from stable_baselines3.common.callbacks import BaseCallback  # type: ignore
+
+import wandb
+from wandb.sdk.lib import telemetry as wb_telemetry
+
+logger = logging.getLogger(__name__)
+
+
+class WandbCallback(BaseCallback):
+    """Callback for logging experiments to Weights and Biases.
+
+    Log SB3 experiments to Weights and Biases
+        - Added model tracking and uploading
+        - Added complete hyperparameters recording
+        - Added gradient logging
+        - Note that `wandb.init(...)` must be called before the WandbCallback can be used.
+
+    Args:
+        verbose: The verbosity of sb3 output
+        model_save_path: Path to the folder where the model will be saved, The default value is `None` so the model is not logged
+        model_save_freq: Frequency to save the model
+        gradient_save_freq: Frequency to log gradient. The default value is 0 so the gradients are not logged
+        log: What to log. One of "gradients", "parameters", or "all".
+    """
+
+    def __init__(
+        self,
+        verbose: int = 0,
+        model_save_path: Optional[str] = None,
+        model_save_freq: int = 0,
+        gradient_save_freq: int = 0,
+        log: Optional[Literal["gradients", "parameters", "all"]] = "all",
+    ) -> None:
+        super().__init__(verbose)
+        if wandb.run is None:
+            raise wandb.Error("You must call wandb.init() before WandbCallback()")
+        with wb_telemetry.context() as tel:
+            tel.feature.sb3 = True
+        self.model_save_freq = model_save_freq
+        self.model_save_path = model_save_path
+        self.gradient_save_freq = gradient_save_freq
+        if log not in ["gradients", "parameters", "all", None]:
+            wandb.termwarn(
+                "`log` must be one of `None`, 'gradients', 'parameters', or 'all', "
+                "falling back to 'all'"
+            )
+            log = "all"
+        self.log = log
+        # Create folder if needed
+        if self.model_save_path is not None:
+            os.makedirs(self.model_save_path, exist_ok=True)
+            self.path = os.path.join(self.model_save_path, "model.zip")
+        else:
+            assert (
+                self.model_save_freq == 0
+            ), "to use the `model_save_freq` you have to set the `model_save_path` parameter"
+
+    def _init_callback(self) -> None:
+        d = {}
+        if "algo" not in d:
+            d["algo"] = type(self.model).__name__
+        for key in self.model.__dict__:
+            if key in wandb.config:
+                continue
+            if type(self.model.__dict__[key]) in [float, int, str]:
+                d[key] = self.model.__dict__[key]
+            else:
+                d[key] = str(self.model.__dict__[key])
+        if self.gradient_save_freq > 0:
+            wandb.watch(
+                self.model.policy,
+                log_freq=self.gradient_save_freq,
+                log=self.log,
+            )
+        wandb.config.setdefaults(d)
+
+    def _on_step(self) -> bool:
+        if self.model_save_freq > 0:
+            if self.model_save_path is not None:
+                if self.n_calls % self.model_save_freq == 0:
+                    self.save_model()
+        return True
+
+    def _on_training_end(self) -> None:
+        if self.model_save_path is not None:
+            self.save_model()
+
+    def save_model(self) -> None:
+        self.model.save(self.path)
+        wandb.save(self.path, base_path=self.model_save_path)
+        if self.verbose > 1:
+            logger.info(f"Saving model checkpoint to {self.path}")

wandb/integration/sklearn/__init__.py

@@ -0,0 +1,37 @@
+"""Create informative charts for scikit-learn models and log them to W&B."""
+
+from .plot import (
+    plot_calibration_curve,
+    plot_class_proportions,
+    plot_classifier,
+    plot_clusterer,
+    plot_confusion_matrix,
+    plot_elbow_curve,
+    plot_feature_importances,
+    plot_learning_curve,
+    plot_outlier_candidates,
+    plot_precision_recall,
+    plot_regressor,
+    plot_residuals,
+    plot_roc,
+    plot_silhouette,
+    plot_summary_metrics,
+)
+
+__all__ = [
+    "plot_classifier",
+    "plot_clusterer",
+    "plot_regressor",
+    "plot_summary_metrics",
+    "plot_learning_curve",
+    "plot_feature_importances",
+    "plot_class_proportions",
+    "plot_calibration_curve",
+    "plot_roc",
+    "plot_precision_recall",
+    "plot_confusion_matrix",
+    "plot_elbow_curve",
+    "plot_silhouette",
+    "plot_residuals",
+    "plot_outlier_candidates",
+]

wandb/integration/sklearn/calculate/__init__.py

@@ -0,0 +1,32 @@
+"""Calculates and formats metrics and charts for introspecting sklearn models.
+
+The functions in these modules are designed to be called by functions from the
+plot submodule that have been exported into the namespace of the wandb.sklearn
+submodule, rather than being called directly.
+"""
+
+from .calibration_curves import calibration_curves
+from .class_proportions import class_proportions
+from .confusion_matrix import confusion_matrix
+from .decision_boundaries import decision_boundaries
+from .elbow_curve import elbow_curve
+from .feature_importances import feature_importances
+from .learning_curve import learning_curve
+from .outlier_candidates import outlier_candidates
+from .residuals import residuals
+from .silhouette import silhouette
+from .summary_metrics import summary_metrics
+
+__all__ = [
+    "calibration_curves",
+    "class_proportions",
+    "confusion_matrix",
+    "decision_boundaries",
+    "elbow_curve",
+    "feature_importances",
+    "learning_curve",
+    "outlier_candidates",
+    "residuals",
+    "silhouette",
+    "summary_metrics",
+]

wandb/integration/sklearn/calculate/calibration_curves.py

@@ -0,0 +1,125 @@
+from warnings import simplefilter
+
+import numpy as np
+import sklearn
+from sklearn import model_selection, naive_bayes
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.linear_model import LogisticRegression
+
+import wandb
+from wandb.integration.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def calibration_curves(clf, X, y, clf_name):  # noqa: N803
+    # ComplementNB (introduced in 0.20.0) requires non-negative features
+    if int(sklearn.__version__.split(".")[1]) >= 20 and isinstance(
+        clf, naive_bayes.ComplementNB
+    ):
+        X = X - X.min()  # noqa:N806
+
+    # Calibrated with isotonic calibration
+    isotonic = CalibratedClassifierCV(clf, cv=2, method="isotonic")
+
+    # Calibrated with sigmoid calibration
+    sigmoid = CalibratedClassifierCV(clf, cv=2, method="sigmoid")
+
+    # Logistic regression with no calibration as baseline
+    lr = LogisticRegression(C=1.0)
+
+    model_column = []  # color
+    frac_positives_column = []  # y axis
+    mean_pred_value_column = []  # x axis
+    hist_column = []  # barchart y
+    edge_column = []  # barchart x
+
+    # Add curve for perfectly calibrated model
+    # format: model, fraction_of_positives, mean_predicted_value
+    model_column.append("Perfectly calibrated")
+    frac_positives_column.append(0)
+    mean_pred_value_column.append(0)
+    hist_column.append(0)
+    edge_column.append(0)
+    model_column.append("Perfectly calibrated")
+    hist_column.append(0)
+    edge_column.append(0)
+    frac_positives_column.append(1)
+    mean_pred_value_column.append(1)
+
+    x_train, x_test, y_train, y_test = model_selection.train_test_split(
+        X, y, test_size=0.9, random_state=42
+    )
+
+    # Add curve for LogisticRegression baseline and other models
+
+    models = [lr, isotonic, sigmoid]
+    names = ["Logistic", f"{clf_name} Isotonic", f"{clf_name} Sigmoid"]
+
+    for model, name in zip(models, names):
+        model.fit(x_train, y_train)
+        if hasattr(model, "predict_proba"):
+            prob_pos = model.predict_proba(x_test)[:, 1]
+        else:  # use decision function
+            prob_pos = model.decision_function(x_test)
+            prob_pos = (prob_pos - prob_pos.min()) / (prob_pos.max() - prob_pos.min())
+
+        hist, edges = np.histogram(prob_pos, bins=10, density=False)
+        frac_positives, mean_pred_value = sklearn.calibration.calibration_curve(
+            y_test, prob_pos, n_bins=10
+        )
+
+        # format: model, fraction_of_positives, mean_predicted_value
+        num_entries = len(frac_positives)
+        for i in range(num_entries):
+            hist_column.append(hist[i])
+            edge_column.append(edges[i])
+            model_column.append(name)
+            frac_positives_column.append(utils.round_3(frac_positives[i]))
+            mean_pred_value_column.append(utils.round_3(mean_pred_value[i]))
+            if utils.check_against_limit(
+                i,
+                "calibration_curve",
+                utils.chart_limit - 2,
+            ):
+                break
+
+    table = make_table(
+        model_column,
+        frac_positives_column,
+        mean_pred_value_column,
+        hist_column,
+        edge_column,
+    )
+    chart = wandb.visualize("wandb/calibration/v1", table)
+
+    return chart
+
+
+def make_table(
+    model_column,
+    frac_positives_column,
+    mean_pred_value_column,
+    hist_column,
+    edge_column,
+):
+    columns = [
+        "model",
+        "fraction_of_positives",
+        "mean_predicted_value",
+        "hist_dict",
+        "edge_dict",
+    ]
+
+    data = list(
+        zip(
+            model_column,
+            frac_positives_column,
+            mean_pred_value_column,
+            hist_column,
+            edge_column,
+        )
+    )
+
+    return wandb.Table(columns=columns, data=data)

wandb/integration/sklearn/calculate/class_proportions.py

@@ -0,0 +1,68 @@
+from warnings import simplefilter
+
+import numpy as np
+from sklearn.utils.multiclass import unique_labels
+
+import wandb
+from wandb.integration.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def class_proportions(y_train, y_test, labels):
+    # Get the unique values from the dataset
+    targets = (y_train,) if y_test is None else (y_train, y_test)
+    class_ids = np.array(unique_labels(*targets))
+
+    # Compute the class counts
+    counts_train = np.array([(y_train == c).sum() for c in class_ids])
+    counts_test = np.array([(y_test == c).sum() for c in class_ids])
+
+    class_column, dataset_column, count_column = make_columns(
+        class_ids, counts_train, counts_test
+    )
+
+    if labels is not None and (
+        isinstance(class_column[0], int) or isinstance(class_column[0], np.integer)
+    ):
+        class_column = get_named_labels(labels, class_column)
+
+    table = make_table(class_column, dataset_column, count_column)
+    chart = wandb.visualize("wandb/class_proportions/v1", table)
+
+    return chart
+
+
+def make_table(class_column, dataset_column, count_column):
+    columns = ["class", "dataset", "count"]
+    data = list(zip(class_column, dataset_column, count_column))
+
+    return wandb.Table(data=data, columns=columns)
+
+
+def make_columns(class_ids, counts_train, counts_test):
+    class_column, dataset_column, count_column = [], [], []
+
+    for i in range(len(class_ids)):
+        # add class counts from training set
+        class_column.append(class_ids[i])
+        dataset_column.append("train")
+        count_column.append(counts_train[i])
+        # add class counts from test set
+        class_column.append(class_ids[i])
+        dataset_column.append("test")
+        count_column.append(counts_test[i])
+
+        if utils.check_against_limit(
+            i,
+            "class_proportions",
+            utils.chart_limit,
+        ):
+            break
+
+    return class_column, dataset_column, count_column
+
+
+def get_named_labels(labels, numeric_labels):
+    return np.array([labels[num_label] for num_label in numeric_labels])

wandb/integration/sklearn/calculate/confusion_matrix.py

@@ -0,0 +1,93 @@
+import itertools
+from warnings import simplefilter
+
+import numpy as np
+from sklearn import metrics
+from sklearn.utils.multiclass import unique_labels
+
+import wandb
+
+from .. import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def validate_labels(*args, **kwargs):  # FIXME
+    raise AssertionError()
+
+
+def confusion_matrix(
+    y_true=None,
+    y_pred=None,
+    labels=None,
+    true_labels=None,
+    pred_labels=None,
+    normalize=False,
+):
+    """Compute the confusion matrix to evaluate the performance of a classification.
+
+    Called by plot_confusion_matrix to visualize roc curves. Please use the function
+    plot_confusion_matrix() if you wish to visualize your confusion matrix.
+    """
+    cm = metrics.confusion_matrix(y_true, y_pred)
+
+    if labels is None:
+        classes = unique_labels(y_true, y_pred)
+    else:
+        classes = np.asarray(labels)
+
+    if normalize:
+        cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]
+        cm = np.around(cm, decimals=2)
+        cm[np.isnan(cm)] = 0.0
+
+    if true_labels is None:
+        true_classes = classes
+    else:
+        validate_labels(classes, true_labels, "true_labels")
+
+        true_label_indexes = np.in1d(classes, true_labels)
+
+        true_classes = classes[true_label_indexes]
+        cm = cm[true_label_indexes]
+
+    if pred_labels is None:
+        pred_classes = classes
+    else:
+        validate_labels(classes, pred_labels, "pred_labels")
+
+        pred_label_indexes = np.in1d(classes, pred_labels)
+
+        pred_classes = classes[pred_label_indexes]
+        cm = cm[:, pred_label_indexes]
+
+    table = make_table(cm, pred_classes, true_classes, labels)
+    chart = wandb.visualize("wandb/confusion_matrix/v1", table)
+
+    return chart
+
+
+def make_table(cm, pred_classes, true_classes, labels):
+    data, count = [], 0
+    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
+        if labels is not None and (
+            isinstance(pred_classes[i], int) or isinstance(pred_classes[0], np.integer)
+        ):
+            pred = labels[pred_classes[i]]
+            true = labels[true_classes[j]]
+        else:
+            pred = pred_classes[i]
+            true = true_classes[j]
+        data.append([pred, true, cm[i, j]])
+        count += 1
+        if utils.check_against_limit(
+            count,
+            "confusion_matrix",
+            utils.chart_limit,
+        ):
+            break
+
+    table = wandb.Table(columns=["Predicted", "Actual", "Count"], data=data)
+
+    return table

wandb/integration/sklearn/calculate/decision_boundaries.py

@@ -0,0 +1,40 @@
+from warnings import simplefilter
+
+import wandb
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def decision_boundaries(
+    decision_boundary_x,
+    decision_boundary_y,
+    decision_boundary_color,
+    train_x,
+    train_y,
+    train_color,
+    test_x,
+    test_y,
+    test_color,
+):
+    x_dict, y_dict, color_dict = [], [], []
+    for i in range(min(len(decision_boundary_x), 100)):
+        x_dict.append(decision_boundary_x[i])
+        y_dict.append(decision_boundary_y[i])
+        color_dict.append(decision_boundary_color)
+    for i in range(300):
+        x_dict.append(test_x[i])
+        y_dict.append(test_y[i])
+        color_dict.append(test_color[i])
+    for i in range(min(len(train_x), 600)):
+        x_dict.append(train_x[i])
+        y_dict.append(train_y[i])
+        color_dict.append(train_color[i])
+
+    return wandb.visualize(
+        "wandb/decision_boundaries/v1",
+        wandb.Table(
+            columns=["x", "y", "color"],
+            data=[[x_dict[i], y_dict[i], color_dict[i]] for i in range(len(x_dict))],
+        ),
+    )