nltkor 1.2.14__cp311-cp311-macosx_13_0_x86_64.whl

nltkor/similarity/classical.py

@@ -0,0 +1,245 @@
+"""
+string2string similarity
+src = https://github.com/stanfordnlp/string2string
+
+
+MIT License
+
+Copyright (c) 2023 Mirac Suzgun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+"""
+
+"""
+This module contains the classes for the similarity metrics and functions.
+"""
+
+
+from typing import List, Union, Tuple, Optional
+import numpy as np
+
+# # Import the LongestCommonSubsequence class
+
+# for dev purposes
+import sys
+# sys.path.append("/Users/dowon/nltk_ko/nltk/metrics")
+from nltkor.alignment import LongestCommonSubsequence, LongestCommonSubstring
+# from alignment import LongestCommonSubsequence, LongestCommonSubstring
+
+# Longest Common Subsequence based similarity class
+class LCSubsequenceSimilarity(LongestCommonSubsequence):
+    """
+    This class contains the Longest Common Subsequence similarity metric.
+
+    This class inherits from the LongestCommonSubsequence class.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+
+    def compute(self,
+        str1: Union[str, List[str]],
+        str2: Union[str, List[str]],
+        denominator: str = 'max',
+    ) -> float:
+        """
+        Returns the LCS-similarity between two strings.
+
+        Arguments:
+            str1 (Union[str, List[str]]): The first string or list of strings.
+            str2 (Union[str, List[str]]): The second string or list of strings.
+            denominator (str): The denominator to use. Options are 'max' and 'sum'. Default is 'max'.
+
+        Returns:
+            float: The similarity between the two strings.
+
+        Raises:
+            ValueError: If the denominator is invalid.
+        """
+
+        # Get the numerator
+        numerator, _ = super().compute(str1, str2)
+
+        if denominator == 'max':
+            return (numerator / max(len(str1), len(str2)))
+        elif denominator == 'sum':
+            return (2. * numerator / (len(str1) + len(str2)))
+        else:
+            raise ValueError('Invalid denominator.')
+
+
+
+# Longest Common Substring based similarity class
+class LCSubstringSimilarity(LongestCommonSubstring):
+    """
+    This class contains the Longest Common Substring similarity metric.
+
+    This class inherits from the LongestCommonSubstring class.
+    """
+    def __init__(self):
+        super().__init__()
+
+
+    def compute(self,
+        str1: Union[str, List[str]],
+        str2: Union[str, List[str]],
+        denominator: str = 'max',
+    ) -> float:
+        """
+        Returns the LCS-similarity between two strings.
+
+        Arguments:
+            str1 (Union[str, List[str]]): The first string or list of strings.
+            str2 (Union[str, List[str]]): The second string or list of strings.
+            denominator (str): The denominator to use. Options are 'max' and 'sum'. Default is 'max'.
+
+        Returns:
+            float: The similarity between the two strings.
+
+        Raises:
+            ValueError: If the denominator is invalid.
+        """
+        # Get the numerator
+        numerator, _ = super().compute(str1, str2)
+
+        if denominator == 'max':
+            return (numerator / max(len(str1), len(str2)))
+        elif denominator == 'sum':
+            return (2. * numerator / (len(str1) + len(str2)))
+        else:
+            raise ValueError('Invalid denominator.')
+
+
+# Jaro similarity class
+class JaroSimilarity:
+    """
+    This class contains the Jaro similarity metric.
+    """
+
+    def __init__(self):
+        pass
+
+
+    def compute(self,
+        str1: Union[str, List[str]],
+        str2: Union[str, List[str]],
+    ) -> float:
+        """
+        This function returns the Jaro similarity between two strings.
+
+        Arguments:
+            str1 (Union[str, List[str]]): The first string or list of strings.
+            str2 (Union[str, List[str]]): The second string or list of strings.
+
+        Returns:
+            float: The Jaro similarity between the two strings.
+        """
+        # Get the length of the strings
+        len1 = len(str1)
+        len2 = len(str2)
+
+        # Get the maximum distance, which we denote by k
+        k = max(len1, len2) // 2 - 1
+
+        # Initialize the number of matching characters and the number of transpositions
+        num_matches = 0
+        num_transpositions = 0
+
+        # Initialize the list of matching flags for the strings
+        matches1 = [False] * len1
+        matches2 = [False] * len2
+
+        # Loop through the characters in the first string and find the matching characters
+        for i in range(len1):
+            # Get the lower and upper bounds for the search
+            lower_bound = max(0, i - k)
+            upper_bound = min(len2, i + k + 1)
+
+            # Loop through the characters in the second string
+            for j in range(lower_bound, upper_bound):
+                # Check if the characters match
+                if not matches2[j] and str1[i] == str2[j]:
+                    # Increment the number of matches
+                    num_matches += 1
+
+                    # Set the matching flags
+                    matches1[i] = True
+                    matches2[j] = True
+
+                    # Break out of the loop
+                    break
+
+        # Check if there are no matches
+        if num_matches == 0:
+            return 0.
+
+        # Loop through again but this time find the number of transpositions
+        # That is, the number of times where there are two matching characters but there is another "matched" character in between them
+        moving_index = 0
+        for i in range(len1):
+            # Check if the character is a match
+            if matches1[i]:
+                # Find the next match
+                for j in range(moving_index, len2):
+                    # Check if the character is a match
+                    if matches2[j]:
+                        # Set the moving index
+                        moving_index = j + 1
+
+                        # Check if the characters are not in the right order
+                        if str1[i] != str2[j]:
+                            # Increment the number of transpositions
+                            num_transpositions += 1
+
+                        # Break out of the loop
+                        break
+
+        num_transpositions = num_transpositions // 2
+
+        # Return the Jaro similarity
+        return (num_matches / len1 + num_matches / len2 + (num_matches - num_transpositions) / num_matches) / 3.0
+
+def demo():
+    """
+    This function demonstrates the similarity metrics.
+    """
+    # Initialize the similarity metrics
+    lcs_sim = LCSubsequenceSimilarity()
+    lcs_sub_sim = LCSubstringSimilarity()
+    jaro_sim = JaroSimilarity()
+
+    # Initialize the strings
+    str1 = '제가 나와 있는 곳은 경남 거제시 옥포동 덕포 해수욕장에 나와 있습니다.'
+    str2 = '강한 바람에 간판이나 지붕이 떨어지는 등 피해가 잇따르기도 했습니다.'
+
+    # Get the similarity metrics
+    lcs_sim_score = lcs_sim.compute(str1, str2)
+    lcs_sub_sim_score = lcs_sub_sim.compute(str1, str2)
+    jaro_sim_score = jaro_sim.compute(str1, str2)
+
+    # Print the results
+    print('Longest Common Subsequence Similarity: {}'.format(lcs_sim_score))
+    print('Longest Common Substring Similarity: {}'.format(lcs_sub_sim_score))
+    print('Jaro Similarity: {}'.format(jaro_sim_score))
+
+if __name__ == '__main__':
+    demo()

nltkor/similarity/cosine_similarity.py

@@ -0,0 +1,175 @@
+"""
+string2string similarity
+src = https://github.com/stanfordnlp/string2string
+
+
+MIT License
+
+Copyright (c) 2023 Mirac Suzgun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+"""
+
+from typing import List, Union, Tuple
+import torch
+from torch import Tensor
+from torch.nn import functional as F
+import numpy as np
+
+# for dev purposes
+import sys
+# sys.path.append("/Users/dowon/nltk_ko/nltk/misc")
+from nltkor.misc.string2string_word_embeddings import GloVeEmbeddings
+# from string2string_word_embeddings import GloVeEmbeddings
+
+
+# Cosine similarity class
+class CosineSimilarity:
+    def __init__(self) -> None:
+        r"""
+        This function initializes the CosineSimilarity class.
+        """
+        pass
+
+
+    # Compute (tensor)
+    def _compute_tensor(self,
+        x1: Tensor,
+        x2: Tensor,
+        dim: int = 1,
+        eps: float = 1e-8
+    ) -> Tensor:
+        r"""
+        Computes the cosine similarity between two tensors along a given dimension.
+
+        Arguments:
+            x1 (Tensor): First tensor.
+            x2 (Tensor): Second tensor.
+            dim (int): Dimension to compute cosine similarity.
+            eps (float): Epsilon value.
+
+        Returns:
+            Tensor: Cosine similarity between two tensors along a given dimension.
+        """
+        # Make sure that x1 and x2 are float tensors
+        if x1.dtype != torch.float:
+            x1 = x1.float()
+        if x2.dtype != torch.float:
+            x2 = x2.float()
+        # Compute cosine similarity between two tensors
+        return F.cosine_similarity(x1, x2, dim, eps)
+
+
+    # Compute (numpy)
+    def _compute_numpy(self,
+        x1: np.ndarray,
+        x2: np.ndarray,
+        dim: int = 1,
+        eps: float = 1e-8
+    ) -> np.ndarray:
+        r"""
+        Computes the cosine similarity between two numpy arrays along a given dimension.
+
+        Arguments:
+            x1 (np.ndarray): First numpy array.
+            x2 (np.ndarray): Second numpy array.
+            dim (int): Dimension (or axis in the numpy realm) to compute cosine similarity.
+            eps (float): Epsilon value (to prevent division by zero).
+
+        Returns:
+            np.ndarray: Cosine similarity between two numpy arrays along a given dimension.
+        """
+        # Compute cosine similarity between two numpy arrays along a given dimension "dim"
+        return np.sum(x1 * x2, axis=dim) / np.maximum(np.linalg.norm(x1, axis=dim) * np.linalg.norm(x2, axis=dim), eps)
+
+
+    # Compute
+    def compute(self,
+        x1: Union[Tensor, np.ndarray],
+        x2: Union[Tensor, np.ndarray],
+        dim: int = 0,
+        eps: float = 1e-8
+    ) -> Union[Tensor, np.ndarray]:
+        r"""
+        Computes the cosine similarity between two tensors (or numpy arrays) along a given dimension.
+
+        * For two (non-zero) vectors, :math:`x_1` and :math:`x_2`, the cosine similarity is defined as follows:
+
+            .. math::
+                :nowrap:
+
+                \begin{align}
+                    \texttt{cosine-similarity}(x_1, x_2) & = |x_1|| \ ||x_2|| \cos(\theta) \\
+                    & = \frac{x_1 \cdot x_2}{||x_1|| \ ||x_2||} \\
+                    & = \frac{\sum_{i=1}^n x_{1i} x_{2i}}{\sqrt{\sum_{i=1}^n x_{1i}^2} \sqrt{\sum_{i=1}^n x_{2i}^2}}
+                \end{align}
+
+            where :math:`\theta` denotes the angle between the vectors, :math:`\cdot` the dot product, and :math:`||\cdot||` the norm operator.
+
+        * In practice, the cosine similarity is computed as follows:
+
+            .. math::
+                :nowrap:
+
+                \begin{align}
+                    \texttt{cosine-similarity}(x_1, x_2) & = \frac{x_1 \cdot x_2}{\max(||x_1|| ||x_2||, \epsilon)}
+                \end{align}
+
+            where :math:`\epsilon` is a small value to avoid division by zero.
+
+
+        Arguments:
+            x1 (Union[Tensor, np.ndarray]): First tensor (or numpy array).
+            x2 (Union[Tensor, np.ndarray]): Second tensor (or numpy array).
+            dim (int): Dimension to compute cosine similarity (default: 0).
+            eps (float): Epsilon value (to avoid division by zero).
+
+        Returns:
+            Union[Tensor, np.ndarray]: Cosine similarity between two tensors (or numpy arrays) along a given dimension.
+
+        Raises:
+            TypeError: If x1 and x2 are not of the same type (either tensor or numpy array).
+            TypeError: If x1 and x2 are not tensors or numpy arrays.
+        """
+        # Check if x1 and x2 are of the same type (either tensor or numpy array)
+        if type(x1) != type(x2):
+            raise TypeError("x1 and x2 must be of the same type (either tensor or numpy array).")
+
+        # If x1 and x2 are tensors
+        if type(x1) == Tensor:
+            # Compute cosine similarity
+            return self._compute_tensor(x1, x2, dim, eps)
+        # If x1 and x2 are numpy arrays
+        elif type(x1) == np.ndarray:
+            # Compute cosine similarity
+            return self._compute_numpy(x1, x2, dim, eps)
+        # If x1 and x2 are not tensors or numpy arrays
+        else:
+            raise TypeError("x1 and x2 must be either tensors or numpy arrays.")
+
+def demo():
+    array1 = np.array([20, 65, 1])
+    array2 = np.array([98, 67, 548])
+
+    print("demo : ", CosineSimilarity().compute(array1, array2))
+
+if __name__ == "__main__":
+    demo()

nltkor/tag/__init__.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Taggers
+#
+# Copyright (C) 2001-2020 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+NLTK Taggers
+
+This package contains classes and interfaces for part-of-speech
+tagging, or simply "tagging".
+
+A "tag" is a case-sensitive string that specifies some property of a token,
+such as its part of speech.  Tagged tokens are encoded as tuples
+``(tag, token)``.  For example, the following tagged token combines
+the word ``'fly'`` with a noun part of speech tag (``'NN'``):
+
+    >>> tagged_tok = ('fly', 'NN')
+
+An off-the-shelf tagger is available for English. It uses the Penn Treebank tagset:
+
+    >>> from nltk import pos_tag, word_tokenize
+    >>> pos_tag(word_tokenize("John's big idea isn't all that bad."))
+    [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'), ('idea', 'NN'), ('is', 'VBZ'),
+    ("n't", 'RB'), ('all', 'PDT'), ('that', 'DT'), ('bad', 'JJ'), ('.', '.')]
+
+A Russian tagger is also available if you specify lang="rus". It uses
+the Russian National Corpus tagset:
+
+    >>> pos_tag(word_tokenize("Илья оторопел и дважды перечитал бумажку."), lang='rus')    # doctest: +SKIP
+    [('Илья', 'S'), ('оторопел', 'V'), ('и', 'CONJ'), ('дважды', 'ADV'), ('перечитал', 'V'),
+    ('бумажку', 'S'), ('.', 'NONLEX')]
+
+This package defines several taggers, which take a list of tokens,
+assign a tag to each one, and return the resulting list of tagged tokens.
+Most of the taggers are built automatically based on a training corpus.
+For example, the unigram tagger tags each word *w* by checking what
+the most frequent tag for *w* was in a training corpus:
+
+    >>> from nltk.corpus import brown
+    >>> from nltk.tag import UnigramTagger
+    >>> tagger = UnigramTagger(brown.tagged_sents(categories='news')[:500])
+    >>> sent = ['Mitchell', 'decried', 'the', 'high', 'rate', 'of', 'unemployment']
+    >>> for word, tag in tagger.tag(sent):
+    ...     print(word, '->', tag)
+    Mitchell -> NP
+    decried -> None
+    the -> AT
+    high -> JJ
+    rate -> NN
+    of -> IN
+    unemployment -> None
+
+Note that words that the tagger has not seen during training receive a tag
+of ``None``.
+
+We evaluate a tagger on data that was not seen during training:
+
+    >>> tagger.evaluate(brown.tagged_sents(categories='news')[500:600])
+    0.73...
+
+For more information, please consult chapter 5 of the NLTK Book.
+"""
+
+
+from nltkor.tag.espresso_tag import EspressoTagger
+#import nltkor.tag
+from nltkor.tag.libs import taggers
+from .libs import PickleConverter