keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl

keras_hub/src/metrics/perplexity.py

@@ -0,0 +1,181 @@
+# Copyright 2024 The KerasHub Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import keras
+from keras import ops
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.utils.tensor_utils import is_float_dtype
+
+
+@keras_hub_export("keras_hub.metrics.Perplexity")
+class Perplexity(keras.metrics.Metric):
+    """Perplexity metric.
+
+    This class implements the perplexity metric. In short, this class calculates
+    the cross entropy loss and takes its exponent.
+    Note: This implementation is not suitable for fixed-size windows.
+
+    Args:
+        from_logits: bool. If True, `y_pred` (input to `update_state()`) should
+            be the logits as returned by the model. Otherwise, `y_pred` is a
+            tensor of probabilities.
+        mask_token_id: int. ID of the token to be masked. If provided, the mask
+            is computed for this class. Note that if this field is provided, and
+            if the `sample_weight` field in `update_state()` is also provided,
+            we will compute the final `sample_weight` as the element-wise
+            product of the mask and the `sample_weight`.
+        dtype: string or tf.dtypes.Dtype. Precision of metric computation. If
+               not specified, it defaults to `"float32"`.
+        name: string. Name of the metric instance.
+        **kwargs: Other keyword arguments.
+
+    Examples:
+
+    1. Calculate perplexity by calling update_state() and result().
+    1.1. `sample_weight`, and `mask_token_id` are not provided.
+    >>> np.random.seed(42)
+    >>> perplexity = keras_hub.metrics.Perplexity(name="perplexity")
+    >>> target = np.random.randint(10, size=[2, 5])
+    >>> logits = np.random.uniform(size=(2, 5, 10))
+    >>> perplexity.update_state(target, logits)
+    >>> perplexity.result()
+    <tf.Tensor: shape=(), dtype=float32, numpy=14.352535>
+
+    1.2. `sample_weight` specified (masking token with ID 0).
+    >>> np.random.seed(42)
+    >>> perplexity = keras_hub.metrics.Perplexity(name="perplexity")
+    >>> target = np.random.randint(10, size=[2, 5])
+    >>> logits = np.random.uniform(size=(2, 5, 10))
+    >>> sample_weight = (target != 0).astype("float32")
+    >>> perplexity.update_state(target, logits, sample_weight)
+    >>> perplexity.result()
+    <tf.Tensor: shape=(), dtype=float32, numpy=14.352535>
+
+    2. Call perplexity directly.
+    >>> np.random.seed(42)
+    >>> perplexity = keras_hub.metrics.Perplexity(name="perplexity")
+    >>> target = np.random.randint(10, size=[2, 5])
+    >>> logits = np.random.uniform(size=(2, 5, 10))
+    >>> perplexity(target, logits)
+    <tf.Tensor: shape=(), dtype=float32, numpy=14.352535>
+
+    3. Provide the padding token ID and let the class compute the mask on its
+       own.
+    >>> np.random.seed(42)
+    >>> perplexity = keras_hub.metrics.Perplexity(mask_token_id=0)
+    >>> target = np.random.randint(10, size=[2, 5])
+    >>> logits = np.random.uniform(size=(2, 5, 10))
+    >>> perplexity(target, logits)
+    <tf.Tensor: shape=(), dtype=float32, numpy=14.352535>
+    """
+
+    def __init__(
+        self,
+        from_logits=False,
+        mask_token_id=None,
+        dtype="float32",
+        name="perplexity",
+        **kwargs,
+    ):
+        if not is_float_dtype(dtype):
+            raise ValueError(
+                "`dtype` must be a floating point type. "
+                f"Received: dtype={dtype}"
+            )
+
+        super().__init__(name=name, dtype=dtype, **kwargs)
+
+        self._crossentropy = keras.losses.SparseCategoricalCrossentropy(
+            from_logits=from_logits, reduction="sum"
+        )
+
+        self.from_logits = from_logits
+        self.mask_token_id = mask_token_id
+
+        self._aggregate_crossentropy = self.add_weight(
+            shape=(),
+            initializer="zeros",
+            dtype=self.dtype,
+            name="aggregate_crossentropy",
+        )
+        self._number_of_samples = self.add_weight(
+            shape=(),
+            initializer="zeros",
+            dtype=self.dtype,
+            name="number_of_samples",
+        )
+
+    def update_state(self, y_true, y_pred, sample_weight=None):
+        # y_true shape: (batch_size, seq_len)
+        # y_pred shape: (batch_size, seq_len, vocab_size)
+        y_true = ops.cast(y_true, self.dtype)
+        y_pred = ops.cast(y_pred, self.dtype)
+
+        if sample_weight is not None:
+            sample_weight = ops.cast(sample_weight, self.dtype)
+
+        batch_size = ops.cast(ops.shape(y_true)[0], self.dtype)
+
+        if self.mask_token_id is not None:
+            mask = ops.cast(
+                ops.logical_not(ops.equal(y_true, self.mask_token_id)),
+                self.dtype,
+            )
+            if sample_weight is None:
+                sample_weight = mask
+            else:
+                sample_weight = ops.multiply(mask, sample_weight)
+
+        # Calculate the Cross Entropy Loss.
+        crossentropy_value = ops.cast(
+            self._crossentropy(y_true, y_pred, sample_weight=sample_weight),
+            self.dtype,
+        )  # scalar
+
+        # Divide the loss by the number of non-masked tokens
+        if sample_weight is not None:
+            crossentropy_value = crossentropy_value / ops.sum(
+                sample_weight
+            )  # scalar
+        else:
+            crossentropy_value = crossentropy_value / (
+                ops.cast(ops.shape(y_true)[0], self.dtype)
+                * ops.cast(ops.shape(y_true)[1], self.dtype)
+            )  # scalar
+
+        self._aggregate_crossentropy.assign_add(batch_size * crossentropy_value)
+        self._number_of_samples.assign_add(batch_size)
+
+    def result(self):
+        perplexity_score = ops.where(
+            ops.equal(ops.convert_to_tensor(self._number_of_samples), 0),
+            0,
+            ops.exp(self._aggregate_crossentropy / self._number_of_samples),
+        )
+        return perplexity_score
+
+    def reset_state(self):
+        self._aggregate_crossentropy.assign(0.0)
+        self._number_of_samples.assign(0.0)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "from_logits": self.from_logits,
+                "mask_token_id": self.mask_token_id,
+            }
+        )
+        return config

keras_hub/src/metrics/rouge_base.py

@@ -0,0 +1,204 @@
+# Copyright 2024 The KerasHub Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import keras
+from keras import ops
+
+from keras_hub.src.utils.tensor_utils import is_float_dtype
+from keras_hub.src.utils.tensor_utils import tensor_to_list
+
+try:
+    import tensorflow as tf
+except ImportError:
+    tf = None
+
+try:
+    from rouge_score import rouge_scorer
+except ImportError:
+    rouge_scorer = None
+
+
+class RougeBase(keras.metrics.Metric):
+    """ROUGE metric.
+
+    This class implements two variants of the ROUGE metric - ROUGE-N,
+    and ROUGE-L.
+
+    Note on input shapes:
+    For `y_true` and `y_pred`, this class supports scalar values and batch
+    inputs of shapes `()`, `(batch_size,)` and `(batch_size, 1)`.
+
+    Args:
+        variant: string. One of "rougeN", "rougeL". For "rougeN", N lies in
+            the range [1, 9]. Defaults to `"rouge2"`.
+        use_stemmer: bool. Whether Porter Stemmer should be used to strip word
+            suffixes to improve matching. Defaults to `False`.
+        dtype: string or tf.dtypes.Dtype. Precision of metric computation. If
+            not specified, it defaults to `"float32"`.
+        name: string. Name of the metric instance.
+        **kwargs: Other keyword arguments.
+
+    References:
+        - [Lin et al., 2004](https://aclanthology.org/W04-1013/)
+    """
+
+    def __init__(
+        self,
+        variant="rouge2",
+        use_stemmer=False,
+        dtype="float32",
+        name="rouge",
+        **kwargs,
+    ):
+        super().__init__(name=name, dtype=dtype, **kwargs)
+
+        if rouge_scorer is None:
+            raise ImportError(
+                f"{self.__class__.__name__} requires the `rouge_score` "
+                "package. Please install it with `pip install rouge-score`."
+            )
+
+        if not is_float_dtype(dtype):
+            raise ValueError(
+                "`dtype` must be a floating point type. "
+                f"Received: dtype={dtype}"
+            )
+
+        if variant not in tuple(
+            ("rouge" + str(order) for order in range(1, 10))
+        ) + ("rougeL",):
+            raise ValueError(
+                "Invalid variant of ROUGE. Should be one of: rougeN, rougeL, "
+                "with N ranging from 1 to 9. Received: "
+                f"variant={variant}"
+            )
+
+        self.variant = variant
+        self.use_stemmer = use_stemmer
+
+        # To-do: Add split_summaries and tokenizer options after the maintainers
+        # of rouge_scorer have released a new version.
+        self._rouge_scorer = rouge_scorer.RougeScorer(
+            rouge_types=[self.variant],
+            use_stemmer=use_stemmer,
+        )
+
+        self._rouge_precision = self.add_weight(
+            shape=(),
+            initializer="zeros",
+            dtype=self.dtype,
+            name="rouge_precision",
+        )
+        self._rouge_recall = self.add_weight(
+            shape=(),
+            initializer="zeros",
+            dtype=self.dtype,
+            name="rouge_recall",
+        )
+        self._rouge_f1_score = self.add_weight(
+            shape=(),
+            initializer="zeros",
+            dtype=self.dtype,
+            name="rouge_f1_score",
+        )
+
+        self._number_of_samples = self.add_weight(
+            shape=(),
+            initializer="zeros",
+            dtype=self.dtype,
+            name="number_of_samples",
+        )
+
+    def update_state(self, y_true, y_pred, sample_weight=None):
+        # Three possible shapes for y_true and y_pred: Python string,
+        # [batch_size] and [batch_size, 1]. In the latter two cases, we have
+        # strings in the tensor/list.
+
+        def validate_and_fix_rank(inputs, tensor_name):
+            if not isinstance(inputs, tf.Tensor):
+                inputs = tf.convert_to_tensor(inputs)
+
+            if inputs.shape.rank == 0:
+                return inputs[tf.newaxis]
+            elif inputs.shape.rank == 1:
+                return inputs
+            elif inputs.shape.rank == 2:
+                if inputs.shape[1] != 1:
+                    raise ValueError(
+                        f"{tensor_name} must be of shape `[batch_size, 1]`. "
+                        f"Found shape: {inputs.shape}"
+                    )
+                else:
+                    return tf.squeeze(inputs, axis=1)
+            else:
+                raise ValueError(
+                    f"{tensor_name} must be of rank 0 (scalar input), 1 or 2. "
+                    f"Found rank: {inputs.shape.rank}"
+                )
+
+        y_true = validate_and_fix_rank(y_true, "y_true")
+        y_pred = validate_and_fix_rank(y_pred, "y_pred")
+
+        batch_size = tf.shape(y_true)[0]
+
+        def calculate_rouge_score(reference, hypothesis):
+            reference = tensor_to_list(reference)
+            hypothesis = tensor_to_list(hypothesis)
+            score = self._rouge_scorer.score(reference, hypothesis)[
+                self.variant
+            ]
+            return score.precision, score.recall, score.fmeasure
+
+        for batch_idx in range(batch_size):
+            score = calculate_rouge_score(y_true[batch_idx], y_pred[batch_idx])
+            self._rouge_precision.assign_add(score[0])
+            self._rouge_recall.assign_add(score[1])
+            self._rouge_f1_score.assign_add(score[2])
+
+        self._number_of_samples.assign_add(
+            ops.cast(batch_size, dtype=self.dtype)
+        )
+
+    def result(self):
+        if self._number_of_samples == 0:
+            return {
+                "precision": 0.0,
+                "recall": 0.0,
+                "f1_score": 0.0,
+            }
+
+        rouge_precision = self._rouge_precision / self._number_of_samples
+        rouge_recall = self._rouge_recall / self._number_of_samples
+        rouge_f1_score = self._rouge_f1_score / self._number_of_samples
+        return {
+            "precision": rouge_precision,
+            "recall": rouge_recall,
+            "f1_score": rouge_f1_score,
+        }
+
+    def reset_state(self):
+        self._rouge_precision.assign(0.0)
+        self._rouge_recall.assign(0.0)
+        self._rouge_f1_score.assign(0.0)
+        self._number_of_samples.assign(0.0)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "variant": self.variant,
+                "use_stemmer": self.use_stemmer,
+            }
+        )
+        return config

keras_hub/src/metrics/rouge_l.py

@@ -0,0 +1,97 @@
+# Copyright 2024 The KerasHub Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.metrics.rouge_base import RougeBase
+
+
+@keras_hub_export("keras_hub.metrics.RougeL")
+class RougeL(RougeBase):
+    """ROUGE-L metric.
+
+    This class implements the ROUGE-L variant of the ROUGE metric. The ROUGE-L
+    metric is traditionally used for evaluating summarisation systems.
+    Succinctly put, ROUGE-L is a score based on the length of the longest
+    common subsequence present in the reference text and the hypothesis text.
+
+    Note on input shapes:
+    For `y_true` and `y_pred`, this class supports scalar values and batch
+    inputs of shapes `()`, `(batch_size,)` and `(batch_size, 1)`.
+
+    Args:
+        use_stemmer: bool. Whether Porter Stemmer should be used to strip word
+            suffixes to improve matching. Defaults to `False`.
+        dtype: string or tf.dtypes.Dtype. Precision of metric computation. If
+               not specified, it defaults to `"float32"`.
+        name: string. Name of the metric instance.
+        **kwargs: Other keyword arguments.
+
+    References:
+        - [Lin et al., 2004](https://aclanthology.org/W04-1013/)
+
+    Examples:
+
+    1. Python string.
+    >>> rouge_l = keras_hub.metrics.RougeL()
+    >>> y_true = "the tiny little cat was found under the big funny bed"
+    >>> y_pred = "the cat was under the bed"
+    >>> rouge_l(y_true, y_pred)["f1_score"]
+    <tf.Tensor: shape=(), dtype=float32, numpy=0.7058824>
+
+    2. List inputs.
+    a. Python list.
+    >>> rouge_l = keras_hub.metrics.RougeL()
+    >>> y_true = [
+    ...     "the tiny little cat was found under the big funny bed",
+    ...     "i really love contributing to KerasHub",
+    ... ]
+    >>> y_pred = [
+    ...     "the cat was under the bed",
+    ...     "i love contributing to KerasHub",
+    ... ]
+    >>> rouge_l(y_true, y_pred)["f1_score"]
+    <tf.Tensor: shape=(), dtype=float32, numpy=0.80748665>
+
+
+    3. 2D inputs.
+    >>> rouge_l = keras_hub.metrics.RougeL()
+    >>> y_true = [
+    ...     ["the tiny little cat was found under the big funny bed"],
+    ...     ["i really love contributing to KerasHub"],
+    ... ]
+    >>> y_pred = [
+    ...     ["the cat was under the bed"],
+    ...     ["i love contributing to KerasHub"],
+    ... ]
+    >>> rouge_l(y_true, y_pred)["f1_score"]
+    <tf.Tensor: shape=(), dtype=float32, numpy=0.80748665>
+    """
+
+    def __init__(
+        self,
+        use_stemmer=False,
+        name="rouge-l",
+        **kwargs,
+    ):
+        super().__init__(
+            variant="rougeL",
+            use_stemmer=use_stemmer,
+            name=name,
+            **kwargs,
+        )
+
+    def get_config(self):
+        config = super().get_config()
+        del config["variant"]
+        return config

keras_hub/src/metrics/rouge_n.py

@@ -0,0 +1,125 @@
+# Copyright 2024 The KerasHub Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.metrics.rouge_base import RougeBase
+
+
+@keras_hub_export("keras_hub.metrics.RougeN")
+class RougeN(RougeBase):
+    """ROUGE-N metric.
+
+    This class implements the ROUGE-N variant of the ROUGE metric. The ROUGE-N
+    metric is traditionally used for evaluating summarisation systems.
+    Succinctly put, ROUGE-N is a score based on the number of matching n-grams
+    between the reference text and the hypothesis text.
+
+    Note on input shapes:
+    For `y_true` and `y_pred`, this class supports scalar values and batch
+    inputs of shapes `()`, `(batch_size,)` and `(batch_size, 1)`.
+
+    Args:
+        order: The order of n-grams which are to be matched. It should lie in
+            range [1, 9]. Defaults to `2`.
+        use_stemmer: bool. Whether Porter Stemmer should be used to strip word
+            suffixes to improve matching. Defaults to `False`.
+        dtype: string or tf.dtypes.Dtype. Precision of metric computation. If
+               not specified, it defaults to `"float32"`.
+        name: string. Name of the metric instance.
+        **kwargs: Other keyword arguments.
+
+    References:
+        - [Lin et al., 2004](https://aclanthology.org/W04-1013/)
+
+    Examples:
+
+    1. Python string.
+    >>> rouge_n = keras_hub.metrics.RougeN(order=2)
+    >>> y_true = "the tiny little cat was found under the big funny bed"
+    >>> y_pred = "the cat was under the bed"
+    >>> rouge_n(y_true, y_pred)["f1_score"]
+    <tf.Tensor: shape=(), dtype=float32, numpy=0.26666668>
+
+    2. List inputs.
+    >>> rouge_n = keras_hub.metrics.RougeN(order=2)
+    >>> y_true = [
+    ...     "the tiny little cat was found under the big funny bed",
+    ...     "i really love contributing to KerasHub",
+    ... ]
+    >>> y_pred = [
+    ...     "the cat was under the bed",
+    ...     "i love contributing to KerasHub",
+    ... ]
+    >>> rouge_n(y_true, y_pred)["f1_score"]
+    <tf.Tensor: shape=(), dtype=float32, numpy=0.4666667>
+
+    3. 2D inputs.
+    >>> rouge_n = keras_hub.metrics.RougeN(order=2)
+    >>> y_true =[
+    ...     ["the tiny little cat was found under the big funny bed"],
+    ...     ["i really love contributing to KerasHub"],
+    ... ]
+    >>> y_pred =[
+    ...     ["the cat was under the bed"],
+    ...     ["i love contributing to KerasHub"],
+    ... ]
+    >>> rouge_n(y_true, y_pred)["f1_score"]
+    <tf.Tensor: shape=(), dtype=float32, numpy=0.4666667>
+
+    4. Trigrams.
+    >>> rouge_n = keras_hub.metrics.RougeN(order=3)
+    >>> y_true = [
+    ...     "the tiny little cat was found under the big funny bed",
+    ...     "i really love contributing to KerasHub",
+    ... ]
+    >>> y_pred = [
+    ...     "the cat was under the bed",
+    ...     "i love contributing to KerasHub",
+    ... ]
+    >>> rouge_n(y_true, y_pred)["f1_score"]
+    <tf.Tensor: shape=(), dtype=float32, numpy=0.2857143>
+    """
+
+    def __init__(
+        self,
+        order=2,
+        use_stemmer=False,
+        name="rouge-n",
+        **kwargs,
+    ):
+        if order not in range(1, 10):
+            raise ValueError(
+                "Invalid `order` value. Should lie in the range [1, 9]."
+                f"Received order={order}"
+            )
+
+        super().__init__(
+            variant=f"rouge{order}",
+            use_stemmer=use_stemmer,
+            name=name,
+            **kwargs,
+        )
+
+        self.order = order
+
+    def get_config(self):
+        config = super().get_config()
+        del config["variant"]
+
+        config.update(
+            {
+                "order": self.order,
+            }
+        )
+        return config

keras_hub/src/models/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 The KerasHub Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

keras_hub/src/models/albert/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2024 The KerasHub Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keras_hub.src.models.albert.albert_backbone import AlbertBackbone
+from keras_hub.src.models.albert.albert_presets import backbone_presets
+from keras_hub.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_hub.src.utils.preset_utils import register_presets
+
+register_presets(backbone_presets, (AlbertBackbone, AlbertTokenizer))