wandb 0.18.1__py3-none-macosx_11_0_x86_64.whl

wandb/plot/confusion_matrix.py

@@ -0,0 +1,99 @@
+from typing import Optional, Sequence
+
+import wandb
+from wandb import util
+
+chart_limit = wandb.Table.MAX_ROWS
+
+
+def confusion_matrix(
+    probs: Optional[Sequence[Sequence]] = None,
+    y_true: Optional[Sequence] = None,
+    preds: Optional[Sequence] = None,
+    class_names: Optional[Sequence[str]] = None,
+    title: Optional[str] = None,
+    split_table: Optional[bool] = False,
+):
+    """Compute a multi-run confusion matrix.
+
+    Arguments:
+        probs (2-d arr): Shape [n_examples, n_classes]
+        y_true (arr): Array of label indices.
+        preds (arr): Array of predicted label indices.
+        class_names (arr): Array of class names.
+        split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+    Returns:
+        Nothing. To see plots, go to your W&B run page then expand the 'media' tab
+        under 'auto visualizations'.
+
+    Example:
+        ```
+        vals = np.random.uniform(size=(10, 5))
+        probs = np.exp(vals)/np.sum(np.exp(vals), keepdims=True, axis=1)
+        y_true = np.random.randint(0, 5, size=(10))
+        labels = ["Cat", "Dog", "Bird", "Fish", "Horse"]
+        wandb.log({'confusion_matrix': wandb.plot.confusion_matrix(probs, y_true=y_true, class_names=labels)})
+        ```
+    """
+    np = util.get_module(
+        "numpy",
+        required="confusion matrix requires the numpy library, install with `pip install numpy`",
+    )
+    # change warning
+    assert probs is None or len(probs.shape) == 2, (
+        "confusion_matrix has been updated to accept"
+        " probabilities as the default first argument. Use preds=..."
+    )
+
+    assert (probs is None or preds is None) and not (
+        probs is None and preds is None
+    ), "Must provide probabilities or predictions but not both to confusion matrix"
+
+    if probs is not None:
+        preds = np.argmax(probs, axis=1).tolist()
+
+    assert len(preds) == len(
+        y_true
+    ), "Number of predictions and label indices must match"
+
+    if class_names is not None:
+        n_classes = len(class_names)
+        class_inds = [i for i in range(n_classes)]
+        assert max(preds) <= len(
+            class_names
+        ), "Higher predicted index than number of classes"
+        assert max(y_true) <= len(
+            class_names
+        ), "Higher label class index than number of classes"
+    else:
+        class_inds = set(preds).union(set(y_true))
+        n_classes = len(class_inds)
+        class_names = [f"Class_{i}" for i in range(1, n_classes + 1)]
+
+    # get mapping of inds to class index in case user has weird prediction indices
+    class_mapping = {}
+    for i, val in enumerate(sorted(list(class_inds))):
+        class_mapping[val] = i
+    counts = np.zeros((n_classes, n_classes))
+    for i in range(len(preds)):
+        counts[class_mapping[y_true[i]], class_mapping[preds[i]]] += 1
+
+    data = []
+    for i in range(n_classes):
+        for j in range(n_classes):
+            data.append([class_names[i], class_names[j], counts[i, j]])
+
+    fields = {
+        "Actual": "Actual",
+        "Predicted": "Predicted",
+        "nPredictions": "nPredictions",
+    }
+    title = title or ""
+    return wandb.plot_table(
+        "wandb/confusion_matrix/v1",
+        wandb.Table(columns=["Actual", "Predicted", "nPredictions"], data=data),
+        fields,
+        {"title": title},
+        split_table=split_table,
+    )

wandb/plot/histogram.py

@@ -0,0 +1,36 @@
+from typing import Optional
+
+import wandb
+
+
+def histogram(
+    table: wandb.Table,
+    value: str,
+    title: Optional[str] = None,
+    split_table: Optional[bool] = False,
+):
+    """Construct a histogram plot.
+
+    Arguments:
+        table (wandb.Table): Table of data.
+        value (string): Name of column to use as data for bucketing.
+        title (string): Plot title.
+        split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+    Returns:
+        A plot object, to be passed to wandb.log()
+
+    Example:
+        ```
+        data = [[i, random.random() + math.sin(i / 10)] for i in range(100)]
+        table = wandb.Table(data=data, columns=["step", "height"])
+        wandb.log({'histogram-plot1': wandb.plot.histogram(table, "height")})
+        ```
+    """
+    return wandb.plot_table(
+        "wandb/histogram/v0",
+        table,
+        {"value": value},
+        {"title": title},
+        split_table=split_table,
+    )

wandb/plot/line.py

@@ -0,0 +1,40 @@
+from typing import Optional
+
+import wandb
+
+
+def line(
+    table: wandb.Table,
+    x: str,
+    y: str,
+    stroke: Optional[str] = None,
+    title: Optional[str] = None,
+    split_table: Optional[bool] = False,
+):
+    """Construct a line plot.
+
+    Arguments:
+        table (wandb.Table): Table of data.
+        x (string): Name of column to as for x-axis values.
+        y (string): Name of column to as for y-axis values.
+        stroke (string): Name of column to map to the line stroke scale.
+        title (string): Plot title.
+        split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+    Returns:
+        A plot object, to be passed to wandb.log()
+
+    Example:
+        ```
+        data = [[i, random.random() + math.sin(i / 10)] for i in range(100)]
+        table = wandb.Table(data=data, columns=["step", "height"])
+        wandb.log({'line-plot1': wandb.plot.line(table, "step", "height")})
+        ```
+    """
+    return wandb.plot_table(
+        "wandb/line/v0",
+        table,
+        {"x": x, "y": y, "stroke": stroke},
+        {"title": title},
+        split_table=split_table,
+    )

wandb/plot/line_series.py

@@ -0,0 +1,88 @@
+import typing as t
+from collections.abc import Iterable
+
+import wandb
+
+
+def line_series(
+    xs: t.Union[t.Iterable, t.Iterable[t.Iterable]],
+    ys: t.Iterable[t.Iterable],
+    keys: t.Optional[t.Iterable] = None,
+    title: t.Optional[str] = None,
+    xname: t.Optional[str] = None,
+    split_table: t.Optional[bool] = False,
+):
+    """Construct a line series plot.
+
+    Arguments:
+        xs (array of arrays, or array): Array of arrays of x values
+        ys (array of arrays): Array of y values
+        keys (array): Array of labels for the line plots
+        title (string): Plot title.
+        xname: Title of x-axis
+        split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+    Returns:
+        A plot object, to be passed to wandb.log()
+
+    Example:
+        When logging a singular array for xs, all ys are plotted against that xs
+        <!--yeadoc-test:plot-line-series-single-->
+        ```python
+        import wandb
+
+        run = wandb.init()
+        xs = [i for i in range(10)]
+        ys = [[i for i in range(10)], [i**2 for i in range(10)]]
+        run.log(
+            {"line-series-plot1": wandb.plot.line_series(xs, ys, title="title", xname="step")}
+        )
+        run.finish()
+        ```
+        xs can also contain an array of arrays for having different steps for each metric
+        <!--yeadoc-test:plot-line-series-double-->
+        ```python
+        import wandb
+
+        run = wandb.init()
+        xs = [[i for i in range(10)], [2 * i for i in range(10)]]
+        ys = [[i for i in range(10)], [i**2 for i in range(10)]]
+        run.log(
+            {"line-series-plot2": wandb.plot.line_series(xs, ys, title="title", xname="step")}
+        )
+        run.finish()
+        ```
+    """
+    if not isinstance(xs, Iterable):
+        raise TypeError(f"Expected xs to be an array instead got {type(xs)}")
+
+    if not isinstance(ys, Iterable):
+        raise TypeError(f"Expected ys to be an array instead got {type(xs)}")
+
+    for y in ys:
+        if not isinstance(y, Iterable):
+            raise TypeError(
+                f"Expected ys to be an array of arrays instead got {type(y)}"
+            )
+
+    if not isinstance(xs[0], Iterable) or isinstance(xs[0], (str, bytes)):
+        xs = [xs for _ in range(len(ys))]
+    assert len(xs) == len(ys), "Number of x-lines and y-lines must match"
+
+    if keys is not None:
+        assert len(keys) == len(ys), "Number of keys and y-lines must match"
+    data = [
+        [x, f"key_{i}" if keys is None else keys[i], y]
+        for i, (xx, yy) in enumerate(zip(xs, ys))
+        for x, y in zip(xx, yy)
+    ]
+
+    table = wandb.Table(data=data, columns=["step", "lineKey", "lineVal"])
+
+    return wandb.plot_table(
+        "wandb/lineseries/v0",
+        table,
+        {"step": "step", "lineKey": "lineKey", "lineVal": "lineVal"},
+        {"title": title, "xname": xname or "x"},
+        split_table=split_table,
+    )

wandb/plot/pr_curve.py

@@ -0,0 +1,136 @@
+from typing import Optional
+
+import wandb
+from wandb import util
+
+from .utils import test_missing, test_types
+
+
+def pr_curve(
+    y_true=None,
+    y_probas=None,
+    labels=None,
+    classes_to_plot=None,
+    interp_size=21,
+    title=None,
+    split_table: Optional[bool] = False,
+):
+    """Compute the tradeoff between precision and recall for different thresholds.
+
+    A high area under the curve represents both high recall and high precision, where
+    high precision relates to a low false positive rate, and high recall relates to a
+    low false negative rate. High scores for both show that the classifier is returning
+    accurate results (high precision), and returning a majority of all positive results
+    (high recall). PR curve is useful when the classes are very imbalanced.
+
+    Arguments:
+        y_true (arr): true sparse labels y_probas (arr): Target scores, can either be
+            probability estimates, confidence values, or non-thresholded measure of
+            decisions. shape: (*y_true.shape, num_classes)
+        labels (list): Named labels for target variable (y). Makes plots easier to read
+            by replacing target values with corresponding index. For example labels =
+            ['dog', 'cat', 'owl'] all 0s are replaced by 'dog', 1s by 'cat'.
+        classes_to_plot (list): unique values of y_true to include in the plot
+        interp_size (int): the recall values will be fixed to `interp_size` points
+            uniform on [0, 1] and the precision will be interpolated for these recall
+            values.
+        split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+    Returns:
+        Nothing. To see plots, go to your W&B run page then expand the 'media' tab under
+        'auto visualizations'.
+
+    Example:
+        ```
+        wandb.log({"pr-curve": wandb.plot.pr_curve(y_true, y_probas, labels)})
+        ```
+    """
+    np = util.get_module(
+        "numpy",
+        required="roc requires the numpy library, install with `pip install numpy`",
+    )
+    pd = util.get_module(
+        "pandas",
+        required="roc requires the pandas library, install with `pip install pandas`",
+    )
+    sklearn_metrics = util.get_module(
+        "sklearn.metrics",
+        "roc requires the scikit library, install with `pip install scikit-learn`",
+    )
+    sklearn_utils = util.get_module(
+        "sklearn.utils",
+        "roc requires the scikit library, install with `pip install scikit-learn`",
+    )
+
+    def _step(x):
+        y = np.array(x)
+        for i in range(1, len(y)):
+            y[i] = max(y[i], y[i - 1])
+        return y
+
+    y_true = np.array(y_true)
+    y_probas = np.array(y_probas)
+
+    if not test_missing(y_true=y_true, y_probas=y_probas):
+        return
+    if not test_types(y_true=y_true, y_probas=y_probas):
+        return
+
+    classes = np.unique(y_true)
+    if classes_to_plot is None:
+        classes_to_plot = classes
+
+    precision = dict()
+    interp_recall = np.linspace(0, 1, interp_size)[::-1]
+    indices_to_plot = np.where(np.isin(classes, classes_to_plot))[0]
+    for i in indices_to_plot:
+        if labels is not None and (
+            isinstance(classes[i], int) or isinstance(classes[0], np.integer)
+        ):
+            class_label = labels[classes[i]]
+        else:
+            class_label = classes[i]
+
+        cur_precision, cur_recall, _ = sklearn_metrics.precision_recall_curve(
+            y_true, y_probas[:, i], pos_label=classes[i]
+        )
+        # smooth the precision (monotonically increasing)
+        cur_precision = _step(cur_precision)
+
+        # reverse order so that recall in ascending
+        cur_precision = cur_precision[::-1]
+        cur_recall = cur_recall[::-1]
+        indices = np.searchsorted(cur_recall, interp_recall, side="left")
+        precision[class_label] = cur_precision[indices]
+
+    df = pd.DataFrame(
+        {
+            "class": np.hstack([[k] * len(v) for k, v in precision.items()]),
+            "precision": np.hstack(list(precision.values())),
+            "recall": np.tile(interp_recall, len(precision)),
+        }
+    )
+    df = df.round(3)
+
+    if len(df) > wandb.Table.MAX_ROWS:
+        wandb.termwarn(
+            "wandb uses only %d data points to create the plots." % wandb.Table.MAX_ROWS
+        )
+        # different sampling could be applied, possibly to ensure endpoints are kept
+        df = sklearn_utils.resample(
+            df,
+            replace=False,
+            n_samples=wandb.Table.MAX_ROWS,
+            random_state=42,
+            stratify=df["class"],
+        ).sort_values(["precision", "recall", "class"])
+
+    table = wandb.Table(dataframe=df)
+    title = title or "Precision v. Recall"
+    return wandb.plot_table(
+        "wandb/area-under-curve/v0",
+        table,
+        {"x": "recall", "y": "precision", "class": "class"},
+        {"title": title},
+        split_table=split_table,
+    )

wandb/plot/roc_curve.py

@@ -0,0 +1,118 @@
+from typing import Optional
+
+import wandb
+from wandb import util
+
+from .utils import test_missing, test_types
+
+
+def roc_curve(
+    y_true=None,
+    y_probas=None,
+    labels=None,
+    classes_to_plot=None,
+    title=None,
+    split_table: Optional[bool] = False,
+):
+    """Calculate and visualize receiver operating characteristic (ROC) scores.
+
+    Arguments:
+        y_true (arr): true sparse labels
+        y_probas (arr): Target scores, can either be probability estimates, confidence
+                         values, or non-thresholded measure of decisions.
+                         shape: (*y_true.shape, num_classes)
+        labels (list): Named labels for target variable (y). Makes plots easier to
+                        read by replacing target values with corresponding index.
+                        For example labels = ['dog', 'cat', 'owl'] all 0s are
+                        replaced by 'dog', 1s by 'cat'.
+        classes_to_plot (list): unique values of y_true to include in the plot
+        split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+    Returns:
+        Nothing. To see plots, go to your W&B run page then expand the 'media' tab
+            under 'auto visualizations'.
+
+    Example:
+        ```
+        wandb.log({'roc-curve': wandb.plot.roc_curve(y_true, y_probas, labels)})
+        ```
+    """
+    np = util.get_module(
+        "numpy",
+        required="roc requires the numpy library, install with `pip install numpy`",
+    )
+    pd = util.get_module(
+        "pandas",
+        required="roc requires the pandas library, install with `pip install pandas`",
+    )
+    sklearn_metrics = util.get_module(
+        "sklearn.metrics",
+        "roc requires the scikit library, install with `pip install scikit-learn`",
+    )
+    sklearn_utils = util.get_module(
+        "sklearn.utils",
+        "roc requires the scikit library, install with `pip install scikit-learn`",
+    )
+
+    y_true = np.array(y_true)
+    y_probas = np.array(y_probas)
+
+    if not test_missing(y_true=y_true, y_probas=y_probas):
+        return
+    if not test_types(y_true=y_true, y_probas=y_probas):
+        return
+
+    classes = np.unique(y_true)
+    if classes_to_plot is None:
+        classes_to_plot = classes
+
+    fpr = dict()
+    tpr = dict()
+    indices_to_plot = np.where(np.isin(classes, classes_to_plot))[0]
+    for i in indices_to_plot:
+        if labels is not None and (
+            isinstance(classes[i], int) or isinstance(classes[0], np.integer)
+        ):
+            class_label = labels[classes[i]]
+        else:
+            class_label = classes[i]
+
+        fpr[class_label], tpr[class_label], _ = sklearn_metrics.roc_curve(
+            y_true, y_probas[..., i], pos_label=classes[i]
+        )
+
+    df = pd.DataFrame(
+        {
+            "class": np.hstack([[k] * len(v) for k, v in fpr.items()]),
+            "fpr": np.hstack(list(fpr.values())),
+            "tpr": np.hstack(list(tpr.values())),
+        }
+    )
+    df = df.round(3)
+
+    if len(df) > wandb.Table.MAX_ROWS:
+        wandb.termwarn(
+            "wandb uses only %d data points to create the plots." % wandb.Table.MAX_ROWS
+        )
+        # different sampling could be applied, possibly to ensure endpoints are kept
+        df = sklearn_utils.resample(
+            df,
+            replace=False,
+            n_samples=wandb.Table.MAX_ROWS,
+            random_state=42,
+            stratify=df["class"],
+        ).sort_values(["fpr", "tpr", "class"])
+
+    table = wandb.Table(dataframe=df)
+    title = title or "ROC"
+    return wandb.plot_table(
+        "wandb/area-under-curve/v0",
+        table,
+        {"x": "fpr", "y": "tpr", "class": "class"},
+        {
+            "title": title,
+            "x-axis-title": "False positive rate",
+            "y-axis-title": "True positive rate",
+        },
+        split_table=split_table,
+    )

wandb/plot/scatter.py

@@ -0,0 +1,32 @@
+from typing import Optional
+
+import wandb
+
+
+def scatter(table, x, y, title=None, split_table: Optional[bool] = False):
+    """Construct a scatter plot.
+
+    Arguments:
+        table (wandb.Table): Table of data.
+        x (string): Name of column to as for x-axis values.
+        y (string): Name of column to as for y-axis values.
+        title (string): Plot title.
+        split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+    Returns:
+        A plot object, to be passed to wandb.log()
+
+    Example:
+        ```
+        data = [[i, random.random() + math.sin(i / 10)] for i in range(100)]
+        table = wandb.Table(data=data, columns=["step", "height"])
+        wandb.log({'scatter-plot1': wandb.plot.scatter(table, "step", "height")})
+        ```
+    """
+    return wandb.plot_table(
+        "wandb/scatter/v0",
+        table,
+        {"x": x, "y": y},
+        {"title": title},
+        split_table=split_table,
+    )