wandb 0.18.0__py3-none-macosx_10_13_x86_64.whl

wandb/sdk/wandb_sync.py

@@ -0,0 +1,75 @@
+import pathlib
+from typing import TYPE_CHECKING, Optional
+
+from ..errors.term import termerror, termlog
+from . import wandb_setup
+from .backend.backend import Backend
+from .lib.mailbox import Mailbox
+from .lib.runid import generate_id
+
+if TYPE_CHECKING:
+    from wandb.proto import wandb_internal_pb2
+
+
+def _sync(
+    path: str,
+    run_id: Optional[str] = None,
+    project: Optional[str] = None,
+    entity: Optional[str] = None,
+    mark_synced: Optional[bool] = None,
+    append: Optional[bool] = None,
+    skip_console: Optional[bool] = None,
+) -> "wandb_internal_pb2.SyncResponse":
+    p = pathlib.Path(path)
+
+    wl = wandb_setup.setup()
+    assert wl is not None
+
+    stream_id = generate_id()
+
+    settings = wl.settings.to_proto()
+    # update sync_file setting to point to the passed path
+    settings.sync_file.value = str(p.absolute())
+    settings.sync_dir.value = str(p.parent.absolute())
+    settings.files_dir.value = str(p.parent.absolute() / "files")
+    settings._sync.value = True
+    settings.run_id.value = stream_id  # TODO: remove this
+    if append:
+        settings.resume.value = "allow"
+
+    manager = wl._get_manager()
+    manager._inform_init(settings=settings, run_id=stream_id)
+
+    mailbox = Mailbox()
+    backend = Backend(settings=wl.settings, manager=manager, mailbox=mailbox)
+    backend.ensure_launched()
+
+    assert backend.interface
+    backend.interface._stream_id = stream_id  # type: ignore
+
+    mailbox.enable_keepalive()
+
+    # TODO: let's add extra sync messages here so we get the url in the beginning
+    handle = backend.interface.deliver_sync(
+        start_offset=0,
+        final_offset=-1,
+        entity=entity,
+        project=project,
+        run_id=run_id,
+        skip_output_raw=skip_console,
+    )
+    result = handle.wait(timeout=-1)
+    assert result and result.response
+    response = result.response.sync_response
+    if response.url:
+        termlog(f"Synced {p} to {response.url}")
+        # create a .synced file in the directory if mark_synced is true
+        if mark_synced:
+            with open(f"{p}.synced", "w"):
+                pass
+    else:
+        termerror(f"Failed to sync {p}")
+    if response.error and response.error.message:
+        termerror(response.error.message)
+
+    return response

wandb/sdk/wandb_watch.py

@@ -0,0 +1,128 @@
+"""watch."""
+
+import logging
+from typing import Optional
+
+try:
+    from typing import Literal
+except ImportError:
+    from typing_extensions import Literal  # type: ignore
+
+import wandb
+
+from .lib import telemetry
+
+logger = logging.getLogger("wandb")
+
+_global_watch_idx = 0
+
+
+def watch(
+    models,
+    criterion=None,
+    log: Optional[Literal["gradients", "parameters", "all"]] = "gradients",
+    log_freq: int = 1000,
+    idx: Optional[int] = None,
+    log_graph: bool = False,
+):
+    """Hook into the torch model to collect gradients and the topology.
+
+    Should be extended to accept arbitrary ML models.
+
+    Args:
+        models: (torch.Module) The model to hook, can be a tuple
+        criterion: (torch.F) An optional loss value being optimized
+        log: (str) One of "gradients", "parameters", "all", or None
+        log_freq: (int) log gradients and parameters every N batches
+        idx: (int) an index to be used when calling wandb.watch on multiple models
+        log_graph: (boolean) log graph topology
+
+    Returns:
+        `wandb.Graph`: The graph object that will populate after the first backward pass
+
+    Raises:
+        ValueError: If called before `wandb.init` or if any of models is not a torch.nn.Module.
+    """
+    global _global_watch_idx
+
+    with telemetry.context() as tel:
+        tel.feature.watch = True
+
+    logger.info("Watching")
+
+    if wandb.run is None:
+        raise ValueError("You must call `wandb.init` before calling watch")
+
+    if log not in {"gradients", "parameters", "all", None}:
+        raise ValueError("log must be one of 'gradients', 'parameters', 'all', or None")
+
+    log_parameters = log in {"parameters", "all"}
+    log_gradients = log in {"gradients", "all"}
+
+    if not isinstance(models, (tuple, list)):
+        models = (models,)
+
+    torch = wandb.util.get_module(
+        "torch", required="wandb.watch only works with pytorch, couldn't import torch."
+    )
+
+    for model in models:
+        if not isinstance(model, torch.nn.Module):
+            raise ValueError(
+                "Expected a pytorch model (torch.nn.Module). Received "
+                + str(type(model))
+            )
+
+    graphs = []
+    prefix = ""
+
+    if idx is None:
+        idx = _global_watch_idx
+    for local_idx, model in enumerate(models):
+        global_idx = idx + local_idx
+        _global_watch_idx += 1
+        if global_idx > 0:
+            # TODO: this makes ugly chart names like gradients/graph_1conv1d.bias
+            prefix = "graph_%i" % global_idx
+
+        if log_parameters:
+            wandb.run._torch.add_log_parameters_hook(
+                model,
+                prefix=prefix,
+                log_freq=log_freq,
+            )
+
+        if log_gradients:
+            wandb.run._torch.add_log_gradients_hook(
+                model,
+                prefix=prefix,
+                log_freq=log_freq,
+            )
+
+        if log_graph:
+            graph = wandb.run._torch.hook_torch(model, criterion, graph_idx=global_idx)
+            graphs.append(graph)
+            # NOTE: the graph is set in run.summary by hook_torch on the backward pass
+    return graphs
+
+
+def unwatch(models=None):
+    """Remove pytorch model topology, gradient and parameter hooks.
+
+    Args:
+        models: (list) Optional list of pytorch models that have had watch called on them
+    """
+    if models:
+        if not isinstance(models, (tuple, list)):
+            models = (models,)
+        for model in models:
+            if not hasattr(model, "_wandb_hook_names"):
+                wandb.termwarn("{} model has not been watched".format(model))
+            else:
+                for name in model._wandb_hook_names:
+                    wandb.run._torch.unhook(name)
+                delattr(model, "_wandb_hook_names")
+                # TODO: we should also remove recursively model._wandb_watch_called
+
+    else:
+        wandb.run._torch.unhook_all()

wandb/sklearn/__init__.py

@@ -0,0 +1,37 @@
+"""Create informative charts for scikit-learn models and log them to W&B."""
+
+from .plot import (
+    plot_calibration_curve,
+    plot_class_proportions,
+    plot_classifier,
+    plot_clusterer,
+    plot_confusion_matrix,
+    plot_elbow_curve,
+    plot_feature_importances,
+    plot_learning_curve,
+    plot_outlier_candidates,
+    plot_precision_recall,
+    plot_regressor,
+    plot_residuals,
+    plot_roc,
+    plot_silhouette,
+    plot_summary_metrics,
+)
+
+__all__ = [
+    "plot_classifier",
+    "plot_clusterer",
+    "plot_regressor",
+    "plot_summary_metrics",
+    "plot_learning_curve",
+    "plot_feature_importances",
+    "plot_class_proportions",
+    "plot_calibration_curve",
+    "plot_roc",
+    "plot_precision_recall",
+    "plot_confusion_matrix",
+    "plot_elbow_curve",
+    "plot_silhouette",
+    "plot_residuals",
+    "plot_outlier_candidates",
+]

wandb/sklearn/calculate/__init__.py

@@ -0,0 +1,32 @@
+"""Calculates and formats metrics and charts for introspecting sklearn models.
+
+The functions in these modules are designed to be called by functions from the
+plot submodule that have been exported into the namespace of the wandb.sklearn
+submodule, rather than being called directly.
+"""
+
+from .calibration_curves import calibration_curves
+from .class_proportions import class_proportions
+from .confusion_matrix import confusion_matrix
+from .decision_boundaries import decision_boundaries
+from .elbow_curve import elbow_curve
+from .feature_importances import feature_importances
+from .learning_curve import learning_curve
+from .outlier_candidates import outlier_candidates
+from .residuals import residuals
+from .silhouette import silhouette
+from .summary_metrics import summary_metrics
+
+__all__ = [
+    "calibration_curves",
+    "class_proportions",
+    "confusion_matrix",
+    "decision_boundaries",
+    "elbow_curve",
+    "feature_importances",
+    "learning_curve",
+    "outlier_candidates",
+    "residuals",
+    "silhouette",
+    "summary_metrics",
+]

wandb/sklearn/calculate/calibration_curves.py

@@ -0,0 +1,125 @@
+from warnings import simplefilter
+
+import numpy as np
+import sklearn
+from sklearn import model_selection, naive_bayes
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.linear_model import LogisticRegression
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def calibration_curves(clf, X, y, clf_name):
+    # ComplementNB (introduced in 0.20.0) requires non-negative features
+    if int(sklearn.__version__.split(".")[1]) >= 20 and isinstance(
+        clf, naive_bayes.ComplementNB
+    ):
+        X = X - X.min()
+
+    # Calibrated with isotonic calibration
+    isotonic = CalibratedClassifierCV(clf, cv=2, method="isotonic")
+
+    # Calibrated with sigmoid calibration
+    sigmoid = CalibratedClassifierCV(clf, cv=2, method="sigmoid")
+
+    # Logistic regression with no calibration as baseline
+    lr = LogisticRegression(C=1.0)
+
+    model_column = []  # color
+    frac_positives_column = []  # y axis
+    mean_pred_value_column = []  # x axis
+    hist_column = []  # barchart y
+    edge_column = []  # barchart x
+
+    # Add curve for perfectly calibrated model
+    # format: model, fraction_of_positives, mean_predicted_value
+    model_column.append("Perfectly calibrated")
+    frac_positives_column.append(0)
+    mean_pred_value_column.append(0)
+    hist_column.append(0)
+    edge_column.append(0)
+    model_column.append("Perfectly calibrated")
+    hist_column.append(0)
+    edge_column.append(0)
+    frac_positives_column.append(1)
+    mean_pred_value_column.append(1)
+
+    X_train, X_test, y_train, y_test = model_selection.train_test_split(
+        X, y, test_size=0.9, random_state=42
+    )
+
+    # Add curve for LogisticRegression baseline and other models
+
+    models = [lr, isotonic, sigmoid]
+    names = ["Logistic", f"{clf_name} Isotonic", f"{clf_name} Sigmoid"]
+
+    for model, name in zip(models, names):
+        model.fit(X_train, y_train)
+        if hasattr(model, "predict_proba"):
+            prob_pos = model.predict_proba(X_test)[:, 1]
+        else:  # use decision function
+            prob_pos = model.decision_function(X_test)
+            prob_pos = (prob_pos - prob_pos.min()) / (prob_pos.max() - prob_pos.min())
+
+        hist, edges = np.histogram(prob_pos, bins=10, density=False)
+        frac_positives, mean_pred_value = sklearn.calibration.calibration_curve(
+            y_test, prob_pos, n_bins=10
+        )
+
+        # format: model, fraction_of_positives, mean_predicted_value
+        num_entries = len(frac_positives)
+        for i in range(num_entries):
+            hist_column.append(hist[i])
+            edge_column.append(edges[i])
+            model_column.append(name)
+            frac_positives_column.append(utils.round_3(frac_positives[i]))
+            mean_pred_value_column.append(utils.round_3(mean_pred_value[i]))
+            if utils.check_against_limit(
+                i,
+                "calibration_curve",
+                utils.chart_limit - 2,
+            ):
+                break
+
+    table = make_table(
+        model_column,
+        frac_positives_column,
+        mean_pred_value_column,
+        hist_column,
+        edge_column,
+    )
+    chart = wandb.visualize("wandb/calibration/v1", table)
+
+    return chart
+
+
+def make_table(
+    model_column,
+    frac_positives_column,
+    mean_pred_value_column,
+    hist_column,
+    edge_column,
+):
+    columns = [
+        "model",
+        "fraction_of_positives",
+        "mean_predicted_value",
+        "hist_dict",
+        "edge_dict",
+    ]
+
+    data = list(
+        zip(
+            model_column,
+            frac_positives_column,
+            mean_pred_value_column,
+            hist_column,
+            edge_column,
+        )
+    )
+
+    return wandb.Table(columns=columns, data=data)

wandb/sklearn/calculate/class_proportions.py

@@ -0,0 +1,68 @@
+from warnings import simplefilter
+
+import numpy as np
+from sklearn.utils.multiclass import unique_labels
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def class_proportions(y_train, y_test, labels):
+    # Get the unique values from the dataset
+    targets = (y_train,) if y_test is None else (y_train, y_test)
+    class_ids = np.array(unique_labels(*targets))
+
+    # Compute the class counts
+    counts_train = np.array([(y_train == c).sum() for c in class_ids])
+    counts_test = np.array([(y_test == c).sum() for c in class_ids])
+
+    class_column, dataset_column, count_column = make_columns(
+        class_ids, counts_train, counts_test
+    )
+
+    if labels is not None and (
+        isinstance(class_column[0], int) or isinstance(class_column[0], np.integer)
+    ):
+        class_column = get_named_labels(labels, class_column)
+
+    table = make_table(class_column, dataset_column, count_column)
+    chart = wandb.visualize("wandb/class_proportions/v1", table)
+
+    return chart
+
+
+def make_table(class_column, dataset_column, count_column):
+    columns = ["class", "dataset", "count"]
+    data = list(zip(class_column, dataset_column, count_column))
+
+    return wandb.Table(data=data, columns=columns)
+
+
+def make_columns(class_ids, counts_train, counts_test):
+    class_column, dataset_column, count_column = [], [], []
+
+    for i in range(len(class_ids)):
+        # add class counts from training set
+        class_column.append(class_ids[i])
+        dataset_column.append("train")
+        count_column.append(counts_train[i])
+        # add class counts from test set
+        class_column.append(class_ids[i])
+        dataset_column.append("test")
+        count_column.append(counts_test[i])
+
+        if utils.check_against_limit(
+            i,
+            "class_proportions",
+            utils.chart_limit,
+        ):
+            break
+
+    return class_column, dataset_column, count_column
+
+
+def get_named_labels(labels, numeric_labels):
+    return np.array([labels[num_label] for num_label in numeric_labels])

wandb/sklearn/calculate/confusion_matrix.py

@@ -0,0 +1,92 @@
+import itertools
+from warnings import simplefilter
+
+import numpy as np
+from sklearn import metrics
+from sklearn.utils.multiclass import unique_labels
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def validate_labels(*args, **kwargs):  # FIXME
+    assert False
+
+
+def confusion_matrix(
+    y_true=None,
+    y_pred=None,
+    labels=None,
+    true_labels=None,
+    pred_labels=None,
+    normalize=False,
+):
+    """Compute the confusion matrix to evaluate the performance of a classification.
+
+    Called by plot_confusion_matrix to visualize roc curves. Please use the function
+    plot_confusion_matrix() if you wish to visualize your confusion matrix.
+    """
+    cm = metrics.confusion_matrix(y_true, y_pred)
+
+    if labels is None:
+        classes = unique_labels(y_true, y_pred)
+    else:
+        classes = np.asarray(labels)
+
+    if normalize:
+        cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]
+        cm = np.around(cm, decimals=2)
+        cm[np.isnan(cm)] = 0.0
+
+    if true_labels is None:
+        true_classes = classes
+    else:
+        validate_labels(classes, true_labels, "true_labels")
+
+        true_label_indexes = np.in1d(classes, true_labels)
+
+        true_classes = classes[true_label_indexes]
+        cm = cm[true_label_indexes]
+
+    if pred_labels is None:
+        pred_classes = classes
+    else:
+        validate_labels(classes, pred_labels, "pred_labels")
+
+        pred_label_indexes = np.in1d(classes, pred_labels)
+
+        pred_classes = classes[pred_label_indexes]
+        cm = cm[:, pred_label_indexes]
+
+    table = make_table(cm, pred_classes, true_classes, labels)
+    chart = wandb.visualize("wandb/confusion_matrix/v1", table)
+
+    return chart
+
+
+def make_table(cm, pred_classes, true_classes, labels):
+    data, count = [], 0
+    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
+        if labels is not None and (
+            isinstance(pred_classes[i], int) or isinstance(pred_classes[0], np.integer)
+        ):
+            pred = labels[pred_classes[i]]
+            true = labels[true_classes[j]]
+        else:
+            pred = pred_classes[i]
+            true = true_classes[j]
+        data.append([pred, true, cm[i, j]])
+        count += 1
+        if utils.check_against_limit(
+            count,
+            "confusion_matrix",
+            utils.chart_limit,
+        ):
+            break
+
+    table = wandb.Table(columns=["Predicted", "Actual", "Count"], data=data)
+
+    return table

wandb/sklearn/calculate/decision_boundaries.py

@@ -0,0 +1,40 @@
+from warnings import simplefilter
+
+import wandb
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def decision_boundaries(
+    decision_boundary_x,
+    decision_boundary_y,
+    decision_boundary_color,
+    train_x,
+    train_y,
+    train_color,
+    test_x,
+    test_y,
+    test_color,
+):
+    x_dict, y_dict, color_dict = [], [], []
+    for i in range(min(len(decision_boundary_x), 100)):
+        x_dict.append(decision_boundary_x[i])
+        y_dict.append(decision_boundary_y[i])
+        color_dict.append(decision_boundary_color)
+    for i in range(300):
+        x_dict.append(test_x[i])
+        y_dict.append(test_y[i])
+        color_dict.append(test_color[i])
+    for i in range(min(len(train_x), 600)):
+        x_dict.append(train_x[i])
+        y_dict.append(train_y[i])
+        color_dict.append(train_color[i])
+
+    return wandb.visualize(
+        "wandb/decision_boundaries/v1",
+        wandb.Table(
+            columns=["x", "y", "color"],
+            data=[[x_dict[i], y_dict[i], color_dict[i]] for i in range(len(x_dict))],
+        ),
+    )

wandb/sklearn/calculate/elbow_curve.py

@@ -0,0 +1,55 @@
+import time
+from warnings import simplefilter
+
+import numpy as np
+from joblib import Parallel, delayed
+from sklearn.base import clone
+
+import wandb
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def elbow_curve(clusterer, X, cluster_ranges, n_jobs, show_cluster_time):
+    if cluster_ranges is None:
+        cluster_ranges = range(1, 10, 2)
+    else:
+        cluster_ranges = sorted(cluster_ranges)
+
+    clfs, times = _compute_results_parallel(n_jobs, clusterer, X, cluster_ranges)
+
+    clfs = np.absolute(clfs)
+
+    table = make_table(cluster_ranges, clfs, times)
+    chart = wandb.visualize("wandb/elbow/v1", table)
+
+    return chart
+
+
+def make_table(cluster_ranges, clfs, times):
+    columns = ["cluster_ranges", "errors", "clustering_time"]
+
+    data = list(zip(cluster_ranges, clfs, times))
+
+    table = wandb.Table(columns=columns, data=data)
+
+    return table
+
+
+def _compute_results_parallel(n_jobs, clusterer, X, cluster_ranges):
+    parallel_runner = Parallel(n_jobs=n_jobs)
+    _cluster_scorer = delayed(_clone_and_score_clusterer)
+    results = parallel_runner(_cluster_scorer(clusterer, X, i) for i in cluster_ranges)
+
+    clfs, times = zip(*results)
+
+    return clfs, times
+
+
+def _clone_and_score_clusterer(clusterer, X, n_clusters):
+    start = time.time()
+    clusterer = clone(clusterer)
+    setattr(clusterer, "n_clusters", n_clusters)
+
+    return clusterer.fit(X).score(X), time.time() - start