rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/utils/plotting.py

@@ -0,0 +1,362 @@
+import logging
+import itertools
+import os
+from functools import wraps
+
+import numpy as np
+from typing import Any, Callable, List, Optional, Text, TypeVar, Union, Tuple
+import matplotlib
+from matplotlib.ticker import FormatStrFormatter
+
+import rasa.shared.utils.io
+from rasa.constants import RESULTS_FILE
+
+logger = logging.getLogger(__name__)
+
+
+def _fix_matplotlib_backend() -> None:
+    """Tries to fix a broken matplotlib backend."""
+    try:
+        backend = matplotlib.get_backend()
+    except Exception:  # skipcq:PYL-W0703
+        logger.error(
+            "Cannot retrieve Matplotlib backend, likely due to a compatibility "
+            "issue with system dependencies. Please refer to the documentation: "
+            "https://matplotlib.org/stable/tutorials/introductory/usage.html#backends"
+        )
+        raise
+
+    # At first, matplotlib will be initialized with default OS-specific
+    # available backend
+    if backend == "TkAgg":
+        try:
+            # on OSX sometimes the tkinter package is broken and can't be imported.
+            # we'll try to import it and if it fails we will use a different backend
+            import tkinter
+        except (ImportError, ModuleNotFoundError):
+            logger.debug("Setting matplotlib backend to 'agg'")
+            matplotlib.use("agg")
+
+    # if no backend is set by default, we'll try to set it up manually
+    elif backend is None:  # pragma: no cover
+        try:
+            # If the `tkinter` package is available, we can use the `TkAgg` backend
+            import tkinter  # noqa: F401
+
+            logger.debug("Setting matplotlib backend to 'TkAgg'")
+            matplotlib.use("TkAgg")
+        except (ImportError, ModuleNotFoundError):
+            logger.debug("Setting matplotlib backend to 'agg'")
+            matplotlib.use("agg")
+
+
+ReturnType = TypeVar("ReturnType")
+FuncType = Callable[..., ReturnType]
+_MATPLOTLIB_BACKEND_FIXED = False
+
+
+def _needs_matplotlib_backend(func: FuncType) -> FuncType:
+    """Decorator to fix matplotlib backend before calling a function."""
+
+    @wraps(func)
+    def inner(*args: Any, **kwargs: Any) -> ReturnType:  # type: ignore
+        """Replacement function that fixes matplotlib backend."""
+        global _MATPLOTLIB_BACKEND_FIXED
+        if not _MATPLOTLIB_BACKEND_FIXED:
+            _fix_matplotlib_backend()
+            _MATPLOTLIB_BACKEND_FIXED = True
+        return func(*args, **kwargs)
+
+    return inner
+
+
+@_needs_matplotlib_backend
+def plot_confusion_matrix(
+    confusion_matrix: np.ndarray,
+    classes: Union[np.ndarray, List[Text]],
+    normalize: bool = False,
+    title: Text = "Confusion matrix",
+    color_map: Any = None,
+    zmin: int = 1,
+    output_file: Optional[Text] = None,
+) -> None:
+    """Print and plot the provided confusion matrix.
+    Normalization can be applied by setting `normalize=True`.
+
+    Args:
+        confusion_matrix: confusion matrix to plot
+        classes: class labels
+        normalize: If set to true, normalization will be applied.
+        title: title of the plot
+        color_map: color mapping
+        zmin:
+        output_file: output file to save plot to
+
+    """
+    import matplotlib.pyplot as plt
+    from matplotlib.colors import LogNorm
+
+    zmax = confusion_matrix.max() if len(confusion_matrix) > 0 else 1
+    plt.clf()
+    if not color_map:
+        color_map = plt.cm.Blues
+    plt.imshow(
+        confusion_matrix,
+        interpolation="nearest",
+        cmap=color_map,
+        aspect="auto",
+        norm=LogNorm(vmin=zmin, vmax=zmax),
+    )
+    plt.title(title)
+    plt.colorbar()
+    tick_marks = np.arange(len(classes))
+    plt.xticks(tick_marks, classes, rotation=90)
+    plt.yticks(tick_marks, classes)
+
+    if normalize:
+        confusion_matrix = (
+            confusion_matrix.astype("float")
+            / confusion_matrix.sum(axis=1)[:, np.newaxis]
+        )
+        logger.info(f"Normalized confusion matrix: \n{confusion_matrix}")
+    else:
+        logger.info(f"Confusion matrix, without normalization: \n{confusion_matrix}")
+
+    thresh = zmax / 2.0
+    for i, j in itertools.product(
+        range(confusion_matrix.shape[0]), range(confusion_matrix.shape[1])
+    ):
+        plt.text(
+            j,
+            i,
+            confusion_matrix[i, j],
+            horizontalalignment="center",
+            color="white" if confusion_matrix[i, j] > thresh else "black",
+        )
+
+    plt.ylabel("True label")
+    plt.xlabel("Predicted label")
+
+    # save confusion matrix to file before showing it
+    if output_file:
+        fig = plt.gcf()
+        fig.set_size_inches(20, 20)
+        fig.savefig(output_file, bbox_inches="tight")
+
+
+def _extract_paired_histogram_specification(
+    histogram_data: List[List[float]],
+    num_bins: int,
+    density: bool,
+    x_pad_fraction: float,
+    y_pad_fraction: float,
+) -> Tuple[List[float], List[List[float]], List[float], Tuple[float, float]]:
+    """Extracts all information from the data needed to plot a paired histogram.
+
+    Args:
+        histogram_data: Two data vectors
+        num_bins: Number of bins to be used for the histogram
+        density: If true, generate information for a probability density histogram
+        x_pad_fraction: Percentage of extra space in the horizontal direction
+        y_pad_fraction: Percentage of extra space in the vertical direction
+
+    Returns:
+        The bins, values, ranges of either x-axis, and the range of the y-axis
+
+    Raises:
+        ValueError: If histogram_data does not contain values.
+    """
+    if not histogram_data or not np.concatenate(histogram_data).size:
+        rasa.shared.utils.io.raise_warning("No data to plot paired histogram.")
+        raise ValueError("No data to plot paired histogram.")
+    min_data_value: float = np.min(np.concatenate(histogram_data))
+    max_data_value: float = np.max(np.concatenate(histogram_data))
+    bin_width = (max_data_value - min_data_value) / num_bins
+    bins = [
+        min_data_value + i * bin_width
+        # `bins` describes the _boundaries_ of the bins, so we need
+        # 2 extra - one at the beginning and one at the end
+        for i in range(num_bins + 2)
+    ]
+    histograms = [
+        # A list of counts - how often a value in `data` falls into a particular bin
+        list(np.histogram(data, bins=bins, density=density)[0])
+        for data in histogram_data
+    ]
+
+    y_padding = 0.5 * bin_width + y_pad_fraction * bin_width
+
+    if density:
+        # Get the maximum count across both histograms, and scale it
+        # with `x_pad_fraction`
+        v = max([(1.0 + x_pad_fraction) * max(histogram) for histogram in histograms])
+        # When we plot the PDF, let both x-axes run to the same value
+        # so it's easier to compare visually
+        x_ranges = [v, v]
+    else:
+        # For the left and right histograms, get the largest counts and scale them
+        # by `x_pad_fraction` to get the maximum x-values displayed
+        x_ranges = [(1.0 + x_pad_fraction) * max(histogram) for histogram in histograms]
+
+    try:
+        bin_of_first_non_zero_tally = min(
+            [[bool(v) for v in histogram].index(True) for histogram in histograms]
+        )
+    except ValueError:
+        bin_of_first_non_zero_tally = 0
+
+    y_range = (
+        # Start plotting where the data starts (ignore empty bins at the low end)
+        bins[bin_of_first_non_zero_tally] - y_padding,
+        # The y_padding adds half a bin width, as we want the bars to be
+        # _centered_ on the bins. We take the next-to-last element of `bins`,
+        # because that is the beginning of the last bin.
+        bins[-2] + y_padding,
+    )
+
+    return bins, histograms, x_ranges, y_range
+
+
+@_needs_matplotlib_backend
+def plot_paired_histogram(
+    histogram_data: List[List[float]],
+    title: Text,
+    output_file: Optional[Text] = None,
+    num_bins: int = 25,
+    colors: Tuple[Text, Text] = ("#009292", "#920000"),  # (dark cyan, dark red)
+    axes_label: Tuple[Text, Text] = ("Correct", "Wrong"),
+    frame_label: Tuple[Text, Text] = ("Number of Samples", "Confidence"),
+    density: bool = False,
+    x_pad_fraction: float = 0.05,
+    y_pad_fraction: float = 0.10,
+) -> None:
+    """Plots a side-by-side comparative histogram of the confidence distribution.
+
+    Args:
+        histogram_data: Two data vectors
+        title: Title to be displayed above the plot
+        output_file: File to save the plot to
+        num_bins: Number of bins to be used for the histogram
+        colors: Left and right bar colors as hex color strings
+        axes_label: Labels shown above the left and right histogram,
+            respectively
+        frame_label: Labels shown below and on the left of the
+            histogram, respectively
+        density: If true, generate a probability density histogram
+        x_pad_fraction: Percentage of extra space in the horizontal direction
+        y_pad_fraction: Percentage of extra space in the vertical direction
+    """
+    if num_bins <= 2:
+        rasa.shared.utils.io.raise_warning(
+            f"Number {num_bins} of paired histogram bins must be at least 3."
+        )
+        return
+
+    try:
+        bins, tallies, x_ranges, y_range = _extract_paired_histogram_specification(
+            histogram_data,
+            num_bins,
+            density=density,
+            x_pad_fraction=x_pad_fraction,
+            y_pad_fraction=y_pad_fraction,
+        )
+    except (ValueError, TypeError) as e:
+        rasa.shared.utils.io.raise_warning(
+            f"Unable to plot paired histogram '{title}': {e}"
+        )
+        return
+    yticks = [float(f"{x:.2f}") for x in bins]
+
+    import matplotlib.pyplot as plt
+
+    plt.gcf().clear()
+
+    fig, axes = plt.subplots(ncols=2, sharey=True)
+    for side in range(2):
+        axes[side].barh(
+            bins[:-1],
+            tallies[side],
+            height=np.diff(bins),
+            align="center",
+            color=colors[side],
+            linewidth=1,
+            edgecolor="white",
+        )
+        axes[side].set(title=axes_label[side])
+        axes[side].set(yticks=yticks, xlim=(0, x_ranges[side]), ylim=y_range)
+
+    axes[0].yaxis.set_major_formatter(FormatStrFormatter("%.2f"))
+    axes[0].yaxis.set_minor_formatter(FormatStrFormatter("%.2f"))
+
+    axes[0].invert_xaxis()
+    axes[0].yaxis.tick_right()
+
+    # Add the title
+    fig.suptitle(title, fontsize="x-large", fontweight="bold")
+
+    # Add hidden plot to correctly add x and y labels (frame_label)
+    fig.add_subplot(111, frameon=False)
+
+    # Hide tick and tick label of the unused axis
+    plt.tick_params(labelcolor="none", top=False, bottom=False, left=False, right=False)
+    plt.xlabel(frame_label[0])
+    plt.ylabel(frame_label[1])
+
+    if output_file:
+        fig = plt.gcf()
+        fig.set_size_inches(10, 10)
+        fig.tight_layout(w_pad=0)
+        fig.savefig(output_file, bbox_inches="tight")
+
+
+@_needs_matplotlib_backend
+def plot_curve(
+    output_directory: Text,
+    number_of_examples: List[int],
+    x_label_text: Text,
+    y_label_text: Text,
+    graph_path: Text,
+) -> None:
+    """Plot the results from a model comparison.
+
+    Args:
+        output_directory: Output directory to save resulting plots to
+        number_of_examples: Number of examples per run
+        x_label_text: text for the x axis
+        y_label_text: text for the y axis
+        graph_path: output path of the plot
+    """
+    import matplotlib.pyplot as plt
+
+    plt.gcf().clear()
+
+    ax = plt.gca()
+
+    # load results from file
+    data = rasa.shared.utils.io.read_json_file(
+        os.path.join(output_directory, RESULTS_FILE)
+    )
+    x = number_of_examples
+
+    # compute mean of all the runs for different configs
+    for label in data.keys():
+        if len(data[label]) == 0:
+            continue
+        mean = np.mean(data[label], axis=0)
+        std = np.std(data[label], axis=0)
+        ax.plot(x, mean, label=label, marker=".")
+        ax.fill_between(
+            x,
+            [m - s for m, s in zip(mean, std)],
+            [m + s for m, s in zip(mean, std)],
+            color="#6b2def",
+            alpha=0.2,
+        )
+    ax.legend(loc=4)
+
+    ax.set_xlabel(x_label_text)
+    ax.set_ylabel(y_label_text)
+
+    plt.savefig(graph_path, format="pdf")
+
+    logger.info(f"Comparison graph saved to '{graph_path}'.")

rasa/utils/sanic_error_handler.py

@@ -0,0 +1,32 @@
+from sanic import Sanic
+from sanic.handlers import ErrorHandler
+from sanic.request import Request
+from sanic.exceptions import ServerError
+
+
+# TODO: remove custom handler when upgrading to sanic >= 24
+#       the underlying issue https://github.com/sanic-org/sanic/issues/2572
+#       has been fixed in sanic 24
+class IgnoreWSServerErrorHandler(ErrorHandler):
+    @staticmethod
+    def log(request: Request, exception: Exception) -> None:
+        try:
+            if (
+                request.url.startswith("ws")
+                and isinstance(exception, ServerError)
+                and exception.args
+                and (
+                    exception.args[0]
+                    == "Invalid response type None (need HTTPResponse)"
+                )
+            ):
+                # in case we are in a websocket connection, we don't want to log the
+                # the error, as this is a bug in sanic
+                return
+        except Exception:
+            pass
+        ErrorHandler.log(request, exception)  # type: ignore
+
+
+def register_custom_sanic_error_handler(app: Sanic) -> None:
+    app.error_handler = IgnoreWSServerErrorHandler()

rasa/utils/singleton.py

@@ -0,0 +1,23 @@
+from typing import Any, Dict
+
+
+class Singleton(type):
+    """Singleton metaclass."""
+
+    _instances: Dict[Any, Any] = {}  # noqa: RUF012
+
+    def __call__(cls, *args: Any, **kwargs: Any) -> Any:
+        """Call the class.
+
+        Args:
+            *args: Arguments.
+            **kwargs: Keyword arguments.
+        """
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+
+        return cls._instances[cls]
+
+    def clear(cls) -> None:
+        """Clear the class."""
+        cls._instances = {}

rasa/utils/tensorflow/callback.py

@@ -0,0 +1,112 @@
+from pathlib import Path
+from typing import Dict, Text, Any, Optional
+
+import logging
+import tensorflow as tf
+from tqdm import tqdm
+
+import rasa.shared.utils.io
+
+logger = logging.getLogger(__name__)
+
+
+class RasaTrainingLogger(tf.keras.callbacks.Callback):
+    """Callback for logging the status of training."""
+
+    def __init__(self, epochs: int, silent: bool) -> None:
+        """Initializes the callback.
+
+        Args:
+            epochs: Total number of epochs.
+            silent: If 'True' the entire progressbar wrapper is disabled.
+        """
+        super().__init__()
+
+        disable = silent or rasa.shared.utils.io.is_logging_disabled()
+        self.progress_bar = tqdm(range(epochs), desc="Epochs", disable=disable)
+
+    def on_epoch_end(self, epoch: int, logs: Optional[Dict[Text, Any]] = None) -> None:
+        """Updates the logging output on every epoch end.
+
+        Args:
+            epoch: The current epoch.
+            logs: The training metrics.
+        """
+        self.progress_bar.update(1)
+        self.progress_bar.set_postfix(logs)
+
+    def on_train_end(self, logs: Optional[Dict[Text, Any]] = None) -> None:
+        """Closes the progress bar after training.
+
+        Args:
+            logs: The training metrics.
+        """
+        self.progress_bar.close()
+
+
+class RasaModelCheckpoint(tf.keras.callbacks.Callback):
+    """Callback for saving intermediate model checkpoints."""
+
+    def __init__(self, checkpoint_dir: Path) -> None:
+        """Initializes the callback.
+
+        Args:
+            checkpoint_dir: Directory to store checkpoints to.
+        """
+        super().__init__()
+
+        self.checkpoint_file = checkpoint_dir / "checkpoint.tf_model"
+        self.best_metrics_so_far: Dict[Text, Any] = {}
+
+    def on_epoch_end(self, epoch: int, logs: Optional[Dict[Text, Any]] = None) -> None:
+        """Save the model on epoch end if the model has improved.
+
+        Args:
+            epoch: The current epoch.
+            logs: The training metrics.
+        """
+        if self._does_model_improve(logs):
+            logger.debug(f"Creating model checkpoint at epoch={epoch + 1} ...")
+            self.model.save_weights(
+                self.checkpoint_file, overwrite=True, save_format="tf"
+            )
+
+    def _does_model_improve(self, curr_results: Dict[Text, Any]) -> bool:
+        """Checks whether the current results are better than the best so far.
+
+        Results are considered better if each metric is equal or better than the best so
+        far, and at least one is better.
+
+        Args:
+            curr_results: The training metrics for this epoch.
+        """
+        curr_metric_names = [
+            k
+            for k in curr_results.keys()
+            if k.startswith("val") and (k.endswith("_acc") or k.endswith("_f1"))
+        ]
+        # the "val" prefix is prepended to metrics in fit if _should_eval returns true
+        # for this particular epoch
+        if len(curr_metric_names) == 0:
+            # the metrics are not validation metrics
+            return False
+        # initialize best_metrics_so_far with the first results
+        if not self.best_metrics_so_far:
+            for metric_name in curr_metric_names:
+                self.best_metrics_so_far[metric_name] = float(curr_results[metric_name])
+            return True
+
+        at_least_one_improved = False
+        improved_metrics = {}
+        for metric_name in self.best_metrics_so_far.keys():
+            if float(curr_results[metric_name]) < self.best_metrics_so_far[metric_name]:
+                # at least one of the values is worse
+                return False
+            if float(curr_results[metric_name]) > self.best_metrics_so_far[metric_name]:
+                at_least_one_improved = True
+                improved_metrics[metric_name] = float(curr_results[metric_name])
+
+        # all current values >= previous best and at least one is better
+        if at_least_one_improved:
+            self.best_metrics_so_far.update(improved_metrics)
+        return at_least_one_improved

rasa/utils/tensorflow/constants.py

@@ -0,0 +1,116 @@
+# constants for configuration parameters of our tensorflow models
+
+LABEL = "label"
+IDS = "ids"
+# LABEL_PAD_ID is used to pad multi-label training examples.
+# It should be < 0 to avoid index out of bounds errors by tf.one_hot.
+LABEL_PAD_ID = -1
+HIDDEN_LAYERS_SIZES = "hidden_layers_sizes"
+SHARE_HIDDEN_LAYERS = "share_hidden_layers"
+
+TRANSFORMER_SIZE = "transformer_size"
+NUM_TRANSFORMER_LAYERS = "number_of_transformer_layers"
+NUM_HEADS = "number_of_attention_heads"
+UNIDIRECTIONAL_ENCODER = "unidirectional_encoder"
+KEY_RELATIVE_ATTENTION = "use_key_relative_attention"
+VALUE_RELATIVE_ATTENTION = "use_value_relative_attention"
+MAX_RELATIVE_POSITION = "max_relative_position"
+
+BATCH_SIZES = "batch_size"
+BATCH_STRATEGY = "batch_strategy"
+EPOCHS = "epochs"
+RANDOM_SEED = "random_seed"
+LEARNING_RATE = "learning_rate"
+
+DENSE_DIMENSION = "dense_dimension"
+CONCAT_DIMENSION = "concat_dimension"
+EMBEDDING_DIMENSION = "embedding_dimension"
+ENCODING_DIMENSION = "encoding_dimension"
+
+SIMILARITY_TYPE = "similarity_type"
+LOSS_TYPE = "loss_type"
+NUM_NEG = "number_of_negative_examples"
+MAX_POS_SIM = "maximum_positive_similarity"
+MAX_NEG_SIM = "maximum_negative_similarity"
+USE_MAX_NEG_SIM = "use_maximum_negative_similarity"
+
+SCALE_LOSS = "scale_loss"
+REGULARIZATION_CONSTANT = "regularization_constant"
+NEGATIVE_MARGIN_SCALE = "negative_margin_scale"
+DROP_RATE = "drop_rate"
+DROP_RATE_ATTENTION = "drop_rate_attention"
+DROP_RATE_DIALOGUE = "drop_rate_dialogue"
+DROP_RATE_LABEL = "drop_rate_label"
+CONSTRAIN_SIMILARITIES = "constrain_similarities"
+
+CONNECTION_DENSITY = "connection_density"
+
+EVAL_NUM_EPOCHS = "evaluate_every_number_of_epochs"
+EVAL_NUM_EXAMPLES = "evaluate_on_number_of_examples"
+
+INTENT_CLASSIFICATION = "intent_classification"
+ENTITY_RECOGNITION = "entity_recognition"
+MASKED_LM = "use_masked_language_model"
+
+SPARSE_INPUT_DROPOUT = "use_sparse_input_dropout"
+DENSE_INPUT_DROPOUT = "use_dense_input_dropout"
+
+RANKING_LENGTH = "ranking_length"
+RENORMALIZE_CONFIDENCES = "renormalize_confidences"
+MODEL_CONFIDENCE = "model_confidence"
+
+BILOU_FLAG = "BILOU_flag"
+
+RETRIEVAL_INTENT = "retrieval_intent"
+
+USE_TEXT_AS_LABEL = "use_text_as_label"
+
+SOFTMAX = "softmax"
+MARGIN = "margin"
+AUTO = "auto"
+INNER = "inner"
+COSINE = "cosine"
+CROSS_ENTROPY = "cross_entropy"
+
+BALANCED = "balanced"
+
+SEQUENCE = "sequence"
+SEQUENCE_LENGTH = f"{SEQUENCE}_lengths"
+SENTENCE = "sentence"
+
+POOLING = "pooling"
+MAX_POOLING = "max"
+MEAN_POOLING = "mean"
+
+TENSORBOARD_LOG_DIR = "tensorboard_log_directory"
+TENSORBOARD_LOG_LEVEL = "tensorboard_log_level"
+
+SEQUENCE_FEATURES = "sequence_features"
+SENTENCE_FEATURES = "sentence_features"
+
+FEATURIZERS = "featurizers"
+CHECKPOINT_MODEL = "checkpoint_model"
+
+MASK = "mask"
+
+IGNORE_INTENTS_LIST = "ignore_intents_list"
+
+TOLERANCE = "tolerance"
+
+POSITIVE_SCORES_KEY = "positive_scores"
+
+NEGATIVE_SCORES_KEY = "negative_scores"
+
+RANKING_KEY = "label_ranking"
+QUERY_INTENT_KEY = "query_intent"
+SCORE_KEY = "score"
+THRESHOLD_KEY = "threshold"
+SEVERITY_KEY = "severity"
+NAME = "name"
+
+TF_DETERMINISTIC_OPS = "TF_DETERMINISTIC_OPS"
+EPOCH_OVERRIDE = "epoch_override"
+
+USE_GPU = "use_gpu"
+RUN_EAGERLY = "run_eagerly"
+DROP_SMALL_LAST_BATCH = "drop_small_last_batch"