nltk 3.2.1__zip → 3.3__zip

nltk-3.3/nltk/stem/arlstem.py

@@ -0,0 +1,369 @@
+# -*- coding: utf-8 -*-
+#
+# Natural Language Toolkit: ARLSTem Stemmer
+#
+# Copyright (C) 2001-2018 NLTK Project
+#
+# Author: Kheireddine Abainia (x-programer) <k.abainia@gmail.com>
+# Algorithms: Kheireddine Abainia <k.abainia@gmail.com>
+#                         Siham Ouamour
+#                         Halim Sayoud
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+
+"""
+ARLSTem Arabic Stemmer
+The details about the implementation of this algorithm are described in:
+K. Abainia, S. Ouamour and H. Sayoud, A Novel Robust Arabic Light Stemmer ,
+Journal of Experimental & Theoretical Artificial Intelligence (JETAI'17),
+Vol. 29, No. 3, 2017, pp. 557-573.
+The ARLSTem is a light Arabic stemmer that is based on removing the affixes
+from the word (i.e. prefixes, suffixes and infixes). It was evaluated and
+compared to several other stemmers using Paice's parameters (under-stemming
+index, over-stemming index and stemming weight), and the results showed that
+ARLSTem is promising and producing high performances. This stemmer is not
+based on any dictionary and can be used on-line effectively.
+"""
+from __future__ import unicode_literals
+import re
+
+from nltk.stem.api import StemmerI
+
+
+class ARLSTem(StemmerI):
+    '''
+    ARLSTem stemmer : a light Arabic Stemming algorithm without any dictionary.
+    Department of Telecommunication & Information Processing. USTHB University,
+    Algiers, Algeria.
+    ARLSTem.stem(token) returns the Arabic stem for the input token.
+    The ARLSTem Stemmer requires that all tokens are encoded using Unicode
+    encoding.
+    '''
+
+    def __init__(self):
+        # different Alif with hamza
+        self.re_hamzated_alif = re.compile(r'[\u0622\u0623\u0625]')
+        self.re_alifMaqsura = re.compile(r'[\u0649]')
+        self.re_diacritics = re.compile(r'[\u064B-\u065F]')
+
+        # Alif Laam, Laam Laam, Fa Laam, Fa Ba
+        self.pr2 = [
+            '\u0627\u0644', '\u0644\u0644',
+            '\u0641\u0644', '\u0641\u0628'
+            ]
+        # Ba Alif Laam, Kaaf Alif Laam, Waaw Alif Laam
+        self.pr3 = [
+            '\u0628\u0627\u0644',
+            '\u0643\u0627\u0644',
+            '\u0648\u0627\u0644'
+            ]
+        # Fa Laam Laam, Waaw Laam Laam
+        self.pr32 = ['\u0641\u0644\u0644', '\u0648\u0644\u0644']
+        # Fa Ba Alif Laam, Waaw Ba Alif Laam, Fa Kaaf Alif Laam
+        self.pr4 = [
+            '\u0641\u0628\u0627\u0644',
+            '\u0648\u0628\u0627\u0644',
+            '\u0641\u0643\u0627\u0644'
+            ]
+
+        # Kaf Yaa, Kaf Miim
+        self.su2 = [
+            '\u0643\u064A',
+            '\u0643\u0645'
+            ]
+        # Ha Alif, Ha Miim
+        self.su22 = ['\u0647\u0627', '\u0647\u0645']
+        # Kaf Miim Alif, Kaf Noon Shadda
+        self.su3 = ['\u0643\u0645\u0627', '\u0643\u0646\u0651']
+        # Ha Miim Alif, Ha Noon Shadda
+        self.su32 = ['\u0647\u0645\u0627', '\u0647\u0646\u0651']
+
+        # Alif Noon, Ya Noon, Waaw Noon
+        self.pl_si2 = ['\u0627\u0646', '\u064A\u0646', '\u0648\u0646']
+        # Taa Alif Noon, Taa Ya Noon
+        self.pl_si3 = ['\u062A\u0627\u0646', '\u062A\u064A\u0646']
+
+        # Alif Noon, Waaw Noon
+        self.verb_su2 = ['\u0627\u0646', '\u0648\u0646']
+        # Siin Taa, Siin Yaa
+        self.verb_pr2 = ['\u0633\u062A', '\u0633\u064A']
+        # Siin Alif, Siin Noon
+        self.verb_pr22 = ['\u0633\u0627', '\u0633\u0646']
+        # Lam Noon, Lam Taa, Lam Yaa, Lam Hamza
+        self.verb_pr33 = ['\u0644\u0646', '\u0644\u062A', '\u0644\u064A', '\u0644\u0623']
+        # Taa Miim Alif, Taa Noon Shadda
+        self.verb_suf3 = ['\u062A\u0645\u0627', '\u062A\u0646\u0651']
+        # Noon Alif, Taa Miim, Taa Alif, Waaw Alif
+        self.verb_suf2 = [
+            '\u0646\u0627', '\u062A\u0645',
+            '\u062A\u0627', '\u0648\u0627'
+            ]
+        # Taa, Alif, Noon
+        self.verb_suf1 = ['\u062A', '\u0627', '\u0646']
+
+    def stem(self, token):
+        """
+            call this function to get the word's stem based on ARLSTem .
+        """
+        try:
+            if token is None:
+                raise ValueError("The word could not be stemmed, because \
+                                 it is empty !")
+            # remove Arabic diacritics and replace some letters with others
+            token = self.norm(token)
+            # strip common prefixes of the nouns
+            pre = self.pref(token)
+            if pre is not None:
+                token = pre
+            # strip the suffixes which are common to nouns and verbs
+            token = self.suff(token)
+            # transform a plural noun to a singular noun
+            ps = self.plur2sing(token)
+            if ps is None:
+                # transform from the feminine form to the masculine form
+                fm = self.fem2masc(token)
+                if fm is not None:
+                    return fm
+                else:
+                    if pre is None:  # if the prefixes are not stripped
+                        # strip the verb prefixes and suffixes
+                        return self.verb(token)
+            else:
+                return ps
+            return token
+        except ValueError as e:
+            print(e)
+
+    def norm(self, token):
+        """
+            normalize the word by removing diacritics, replacing hamzated Alif
+            with Alif replacing AlifMaqsura with Yaa and removing Waaw at the
+            beginning.
+        """
+        # strip Arabic diacritics
+        token = self.re_diacritics.sub('', token)
+        # replace Hamzated Alif with Alif bare
+        token = self.re_hamzated_alif.sub('\u0627', token)
+        # replace alifMaqsura with Yaa
+        token = self.re_alifMaqsura.sub('\u064A', token)
+        # strip the Waaw from the word beginning if the remaining is 3 letters
+        # at least
+        if token.startswith('\u0648') and len(token) > 3:
+            token = token[1:]
+        return token
+
+    def pref(self, token):
+        """
+            remove prefixes from the words' beginning.
+        """
+        if len(token) > 5:
+            for p3 in self.pr3:
+                if token.startswith(p3):
+                    return token[3:]
+        if len(token) > 6:
+            for p4 in self.pr4:
+                if token.startswith(p4):
+                    return token[4:]
+        if len(token) > 5:
+            for p3 in self.pr32:
+                if token.startswith(p3):
+                    return token[3:]
+        if len(token) > 4:
+            for p2 in self.pr2:
+                if token.startswith(p2):
+                    return token[2:]
+
+    def suff(self, token):
+        """
+            remove suffixes from the word's end.
+        """
+        if token.endswith('\u0643') and len(token) > 3:
+            return token[:-1]
+        if len(token) > 4:
+            for s2 in self.su2:
+                if token.endswith(s2):
+                    return token[:-2]
+        if len(token) > 5:
+            for s3 in self.su3:
+                if token.endswith(s3):
+                    return token[:-3]
+        if token.endswith('\u0647') and len(token) > 3:
+            token = token[:-1]
+            return token
+        if len(token) > 4:
+            for s2 in self.su22:
+                if token.endswith(s2):
+                    return token[:-2]
+        if len(token) > 5:
+            for s3 in self.su32:
+                if token.endswith(s3):
+                    return token[:-3]
+        if token.endswith('\u0646\u0627') and len(token) > 4:
+            return token[:-2]
+        return token
+
+    def fem2masc(self, token):
+        """
+            transform the word from the feminine form to the masculine form.
+        """
+        if token.endswith('\u0629') and len(token) > 3:
+            return token[:-1]
+
+    def plur2sing(self, token):
+        """
+            transform the word from the plural form to the singular form.
+        """
+        if len(token) > 4:
+            for ps2 in self.pl_si2:
+                if token.endswith(ps2):
+                    return token[:-2]
+        if len(token) > 5:
+            for ps3 in self.pl_si3:
+                if token.endswith(ps3):
+                    return token[:-3]
+        if len(token) > 3 and token.endswith('\u0627\u062A'):
+            return token[:-2]
+        if (len(token) > 3 and token.startswith('\u0627')
+           and token[2] == '\u0627'):
+            return token[:2] + token[3:]
+        if (len(token) > 4 and token.startswith('\u0627')
+           and token[-2] == '\u0627'):
+            return token[1:-2] + token[-1]
+
+    def verb(self, token):
+        """
+            stem the verb prefixes and suffixes or both
+        """
+        vb = self.verb_t1(token)
+        if vb is not None:
+            return vb
+        vb = self.verb_t2(token)
+        if vb is not None:
+            return vb
+        vb = self.verb_t3(token)
+        if vb is not None:
+            return vb
+        vb = self.verb_t4(token)
+        if vb is not None:
+            return vb
+        vb = self.verb_t5(token)
+        if vb is not None:
+            return vb
+        return self.verb_t6(token)
+
+    def verb_t1(self, token):
+        """
+            stem the present prefixes and suffixes
+        """
+        if len(token) > 5 and token.startswith('\u062A'):  # Taa
+            for s2 in self.pl_si2:
+                if token.endswith(s2):
+                    return token[1:-2]
+        if len(token) > 5 and token.startswith('\u064A'):  # Yaa
+            for s2 in self.verb_su2:
+                if token.endswith(s2):
+                    return token[1:-2]
+        if len(token) > 4 and token.startswith('\u0627'):  # Alif
+            # Waaw Alif
+            if len(token) > 5 and token.endswith('\u0648\u0627'):
+                return token[1:-2]
+            # Yaa
+            if token.endswith('\u064A'):
+                return token[1:-1]
+            # Alif
+            if token.endswith('\u0627'):
+                return token[1:-1]
+            # Noon
+            if token.endswith('\u0646'):
+                return token[1:-1]
+        # ^Yaa, Noon$
+        if (len(token) > 4
+           and token.startswith('\u064A')
+           and token.endswith('\u0646')):
+            return token[1:-1]
+        # ^Taa, Noon$
+        if (len(token) > 4
+           and token.startswith('\u062A')
+           and token.endswith('\u0646')):
+            return token[1:-1]
+
+    def verb_t2(self, token):
+        """
+            stem the future prefixes and suffixes
+        """
+        if len(token) > 6:
+            for s2 in self.pl_si2:
+                # ^Siin Taa
+                if (token.startswith(self.verb_pr2[0])
+                   and token.endswith(s2)):
+                    return token[2:-2]
+            # ^Siin Yaa, Alif Noon$
+            if (token.startswith(self.verb_pr2[1])
+               and token.endswith(self.pl_si2[0])):
+                return token[2:-2]
+            # ^Siin Yaa, Waaw Noon$
+            if (token.startswith(self.verb_pr2[1])
+               and token.endswith(self.pl_si2[2])):
+                return token[2:-2]
+        # ^Siin Taa, Noon$
+        if (len(token) > 5
+           and token.startswith(self.verb_pr2[0])
+           and token.endswith('\u0646')):
+            return token[2:-1]
+        # ^Siin Yaa, Noon$
+        if (len(token) > 5
+           and token.startswith(self.verb_pr2[1])
+           and token.endswith('\u0646')):
+            return token[2:-1]
+
+    def verb_t3(self, token):
+        """
+            stem the present suffixes
+        """
+        if len(token) > 5:
+            for su3 in self.verb_suf3:
+                if(token.endswith(su3)):
+                    return token[:-3]
+        if len(token) > 4:
+            for su2 in self.verb_suf2:
+                if token.endswith(su2):
+                    return token[:-2]
+        if len(token) > 3:
+            for su1 in self.verb_suf1:
+                if token.endswith(su1):
+                    return token[:-1]
+
+    def verb_t4(self, token):
+        """
+            stem the present prefixes
+        """
+        if len(token) > 3:
+            for pr1 in self.verb_suf1:
+                if token.startswith(pr1):
+                    return token[1:]
+            if token.startswith('\u064A'):
+                return token[1:]
+
+    def verb_t5(self, token):
+        """
+            stem the future prefixes
+        """
+        if len(token) > 4:
+            for pr2 in self.verb_pr22:
+                if token.startswith(pr2):
+                    return token[2:]
+            for pr2 in self.verb_pr2:
+                if token.startswith(pr2):
+                    return token[2:]
+        return token
+
+    def verb_t6(self, token):
+        """
+            stem the order prefixes
+        """
+        if len(token) > 4:
+            for pr3 in self.verb_pr33:
+                if token.startswith(pr3):
+                    return token[2:]
+        return token

{nltk-3.2.1 → nltk-3.3}/nltk/stem/isri.py

@@ -2,7 +2,7 @@
 #
 # Natural Language Toolkit: The ISRI Arabic Stemmer
 #
-# Copyright (C) 2001-2016 NLTK Proejct
+# Copyright (C) 2001-2018 NLTK Proejct
 # Algorithm: Kazem Taghva, Rania Elkhoury, and Jeffrey Coombs (2005)
 # Author: Hosam Algasaier <hosam_hme@yahoo.com>
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/stem/lancaster.py

@@ -1,12 +1,12 @@
 # Natural Language Toolkit: Stemmers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Tomcavage <stomcava@law.upenn.edu>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
 """
-A word stemmer based on the Lancaster stemming algorithm.
+A word stemmer based on the Lancaster (Paice/Husk) stemming algorithm.
 Paice, Chris D. "Another Stemmer." ACM SIGIR Forum 24.3 (1990): 56-61.
 """
 from __future__ import unicode_literals
@@ -44,10 +44,16 @@ class LancasterStemmer(StemmerI):
         'meant'
         >>> st.stem('cement')      # ditto
         'cem'
+        >>> st_pre = LancasterStemmer(strip_prefix_flag=True)
+        >>> st_pre.stem('kilometer') # Test Prefix
+        'met'
+        >>> st_custom = LancasterStemmer(rule_tuple=("ssen4>", "s1t."))
+        >>> st_custom.stem("ness") # Change s to t
+        'nest'
     """
 
     # The rule list is static since it doesn't change between instances
-    rule_tuple = (
+    default_rule_tuple = (
         "ai*2.",     # -ia > -   if intact
         "a*1.",      # -a > -    if intact
         "bb1.",      # -bb > -b
@@ -165,23 +171,33 @@ class LancasterStemmer(StemmerI):
         "zy1s."      # -yz > -ys
     )
 
-
-    def __init__(self):
+    def __init__(self, rule_tuple=None, strip_prefix_flag=False):
         """Create an instance of the Lancaster stemmer.
         """
         # Setup an empty rule dictionary - this will be filled in later
         self.rule_dictionary = {}
+        # Check if a user wants to strip prefix
+        self._strip_prefix = strip_prefix_flag
+        # Check if a user wants to use his/her own rule tuples.
+        self._rule_tuple = rule_tuple if rule_tuple else self.default_rule_tuple
 
-    def parseRules(self, rule_tuple):
+    def parseRules(self, rule_tuple=None):
         """Validate the set of rules used in this stemmer.
+
+        If this function is called as an individual method, without using stem
+        method, rule_tuple argument will be compiled into self.rule_dictionary.
+        If this function is called within stem, self._rule_tuple will be used.
+
         """
+        # If there is no argument for the function, use class' own rule tuple.
+        rule_tuple = rule_tuple if rule_tuple else self._rule_tuple
         valid_rule = re.compile("^[a-z]+\*?\d[a-z]*[>\.]?$")
         # Empty any old rules from the rule set before adding new ones
         self.rule_dictionary = {}
 
         for rule in rule_tuple:
             if not valid_rule.match(rule):
-                raise ValueError("The rule %s is invalid" % rule)
+                raise ValueError("The rule {0} is invalid".format(rule))
             first_letter = rule[0:1]
             if first_letter in self.rule_dictionary:
                 self.rule_dictionary[first_letter].append(rule)
@@ -193,13 +209,14 @@ class LancasterStemmer(StemmerI):
         """
         # Lower-case the word, since all the rules are lower-cased
         word = word.lower()
+        word = self.__stripPrefix(word) if self._strip_prefix else word
 
         # Save a copy of the original word
         intact_word = word
 
-        # If the user hasn't supplied any rules, setup the default rules
-        if len(self.rule_dictionary) == 0:
-            self.parseRules(LancasterStemmer.rule_tuple)
+        # If rule dictionary is empty, parse rule tuple.
+        if not self.rule_dictionary:
+            self.parseRules()
 
         return self.__doStemming(word, intact_word)
 
@@ -304,7 +321,17 @@ class LancasterStemmer(StemmerI):
             word += append_string
         return word
 
-    def __repr__(self):
-        return '<LancasterStemmer>'
+    def __stripPrefix(self, word):
+        """Remove prefix from a word.
 
+        This function originally taken from Whoosh.
 
+        """
+        for prefix in ("kilo", "micro", "milli", "intra", "ultra", "mega",
+                       "nano", "pico", "pseudo"):
+            if word.startswith(prefix):
+                return word[len(prefix):]
+        return word
+
+    def __repr__(self):
+        return '<LancasterStemmer>'