nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/sem/logic.py

@@ -2,7 +2,7 @@
 #
 # Author: Dan Garrette <dhgarrette@gmail.com>
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # URL: <http://nltk.org>
 # For license information, see LICENSE.TXT
 
@@ -15,12 +15,13 @@ from __future__ import print_function, unicode_literals
 import re
 import operator
 from collections import defaultdict
-from functools import reduce
+from functools import reduce, total_ordering
+
+from six import string_types
 
 from nltk.util import Trie
 from nltk.internals import Counter
-from nltk.compat import (total_ordering, string_types,
-                         python_2_unicode_compatible)
+from nltk.compat import python_2_unicode_compatible
 
 APP = 'APP'
 
@@ -802,7 +803,7 @@ def read_type(type_string):
 
 class TypeException(Exception):
     def __init__(self, msg):
-        Exception.__init__(self, msg)
+        super(TypeException, self).__init__(msg)
 
 class InconsistentTypeHierarchyException(TypeException):
     def __init__(self, variable, expression=None):
@@ -812,21 +813,20 @@ class InconsistentTypeHierarchyException(TypeException):
         else:
             msg = "The variable '%s' was found in multiple places with different"\
                 " types." % (variable)
-        Exception.__init__(self, msg)
+        super(InconsistentTypeHierarchyException, self).__init__(msg)
 
 class TypeResolutionException(TypeException):
     def __init__(self, expression, other_type):
-        Exception.__init__(self, "The type of '%s', '%s', cannot be "
-                           "resolved with type '%s'" % \
-                           (expression, expression.type, other_type))
+        super(TypeResolutionException, self).__init__(
+            "The type of '%s', '%s', cannot be resolved with type '%s'" %
+            (expression, expression.type, other_type))
 
 class IllegalTypeException(TypeException):
     def __init__(self, expression, other_type, allowed_type):
-        Exception.__init__(self, "Cannot set type of %s '%s' to '%s'; "
-                           "must match type '%s'." %
-                           (expression.__class__.__name__, expression,
-                            other_type, allowed_type))
-
+        super(IllegalTypeException, self).__init__(
+            "Cannot set type of %s '%s' to '%s'; must match type '%s'." %
+            (expression.__class__.__name__, expression, other_type,
+            allowed_type))
 
 def typecheck(expressions, signature=None):
     """

{nltk-3.2.1 → nltk-3.3}/nltk/sem/relextract.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Relation Extraction
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -25,7 +25,8 @@ from __future__ import print_function
 
 from collections import defaultdict
 import re
-from nltk.compat import htmlentitydefs
+
+from six.moves import html_entities
 
 # Dictionary that associates corpora with NE classes
 NE_CLASSES = {
@@ -79,7 +80,7 @@ def _join(lst, sep=' ', untag=False):
         from nltk.tag import tuple2str
         return sep.join(tuple2str(tup) for tup in lst)
 
-def descape_entity(m, defs=htmlentitydefs.entitydefs):
+def descape_entity(m, defs=html_entities.entitydefs):
     """
     Translate one entity to its ISO Latin value.
     Inspired by example from effbot.org
@@ -114,7 +115,7 @@ def list2sym(lst):
 
 def tree2semi_rel(tree):
     """
-    Group a chunk structure into a list of 'semi-relations' of the form (list(str), ``Tree``). 
+    Group a chunk structure into a list of 'semi-relations' of the form (list(str), ``Tree``).
 
     In order to facilitate the construction of (``Tree``, string, ``Tree``) triples, this
     identifies pairs whose first member is a list (possibly empty) of terminal
@@ -209,7 +210,7 @@ def extract_rels(subjclass, objclass, doc, corpus='ace', pattern=None, window=10
             objclass = _expand(objclass)
         else:
             raise ValueError("your value for the object type has not been recognized: %s" % objclass)
-        
+
     if corpus == 'ace' or corpus == 'conll2002':
         pairs = tree2semi_rel(doc)
     elif corpus == 'ieer':
@@ -334,7 +335,7 @@ def roles_demo(trace=0):
     commissioner|
     counsel|
     director|
-    economist|       
+    economist|
     editor|
     executive|
     foreman|
@@ -379,10 +380,10 @@ def ieer_headlines():
 
     from nltk.corpus import ieer
     from nltk.tree import Tree
-    
+
     print("IEER: First 20 Headlines")
-    print("=" * 45)  
-    
+    print("=" * 45)
+
     trees = [(doc.docno, doc.headline) for file in ieer.fileids() for doc in ieer.parsed_docs(file)]
     for tree in trees[:20]:
         print()
@@ -424,7 +425,7 @@ def conllned(trace=1):
         if trace:
                 lcon = rcon = True
         for rel in extract_rels('PER', 'ORG', doc, corpus='conll2002', pattern=VAN, window=10):
-            print(rtuple(rel, lcon=True, rcon=True))
+            print(rtuple(rel, lcon=lcon, rcon=rcon))
 
 #############################################
 ## Spanish CONLL2002: (PER, ORG)
@@ -473,11 +474,3 @@ if __name__ == '__main__':
     conllesp()
     ieer_headlines()
     ne_chunked()
-
-
-
-
-
-
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/sem/skolemize.py

@@ -2,7 +2,7 @@
 #
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 

{nltk-3.2.1 → nltk-3.3}/nltk/sem/util.py

@@ -2,7 +2,7 @@
 #
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 

{nltk-3.2.1 → nltk-3.3}/nltk/sentiment/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Sentiment Analysis
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/sentiment/sentiment_analyzer.py

@@ -2,7 +2,7 @@
 #
 # Natural Language Toolkit: Sentiment Analyzer
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Pierpaolo Pantone <24alsecondo@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/sentiment/util.py

@@ -2,7 +2,7 @@
 #
 # Natural Language Toolkit: Sentiment Analyzer
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Pierpaolo Pantone <24alsecondo@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -10,8 +10,8 @@
 """
 Utility methods for Sentiment Analysis.
 """
+from __future__ import division
 
-from copy import deepcopy
 import codecs
 import csv
 import json
@@ -20,6 +20,8 @@ import random
 import re
 import sys
 import time
+from copy import deepcopy
+from itertools import tee
 
 import nltk
 from nltk.corpus import CategorizedPlaintextCorpusReader
@@ -63,6 +65,7 @@ SAD = set([
     ':c', ':{', '>:\\', ';('
     ])
 
+
 def timer(method):
     """
     A timer decorator to measure execution performance of methods.
@@ -72,8 +75,8 @@ def timer(method):
         result = method(*args, **kw)
         end = time.time()
         tot_time = end - start
-        hours = int(tot_time / 3600)
-        mins = int((tot_time / 60) % 60)
+        hours = tot_time // 3600
+        mins = tot_time // 60 % 60
         # in Python 2.x round() will return a float, so we convert it to int
         secs = int(round(tot_time % 60))
         if hours == 0 and mins == 0 and secs < 10:
@@ -83,6 +86,13 @@ def timer(method):
         return result
     return timed
 
+
+def pairwise(iterable):
+    """s -> (s0,s1), (s1,s2), (s2, s3), ..."""
+    a, b = tee(iterable)
+    next(b, None)
+    return zip(a, b)
+
 #////////////////////////////////////////////////////////////
 #{ Feature extractor functions
 #////////////////////////////////////////////////////////////

{nltk-3.2.1 → nltk-3.3}/nltk/sentiment/vader.py

@@ -1,10 +1,11 @@
 # coding: utf-8
 # Natural Language Toolkit: vader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: C.J. Hutto <Clayton.Hutto@gtri.gatech.edu>
 #         Ewan Klein <ewan@inf.ed.ac.uk> (modifications)
 #         Pierpaolo Pantone <24alsecondo@gmail.com> (modifications)
+#         George Berry <geb97@cornell.edu> (modifications)
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 #
@@ -24,7 +25,9 @@ import codecs
 import math
 import re
 import string
+from itertools import product
 import nltk.data
+from .util import pairwise
 
 ##Constants##
 
@@ -39,19 +42,18 @@ C_INCR = 0.733
 N_SCALAR = -0.74
 
 # for removing punctuation
-REGEX_REMOVE_PUNCTUATION = re.compile('[%s]' % re.escape(string.punctuation))
+REGEX_REMOVE_PUNCTUATION = re.compile('[{0}]'.format(re.escape(string.punctuation)))
 
 PUNC_LIST = [".", "!", "?", ",", ";", ":", "-", "'", "\"",
              "!!", "!!!", "??", "???", "?!?", "!?!", "?!?!", "!?!?"]
-NEGATE = \
-["aint", "arent", "cannot", "cant", "couldnt", "darent", "didnt", "doesnt",
+NEGATE = {"aint", "arent", "cannot", "cant", "couldnt", "darent", "didnt", "doesnt",
  "ain't", "aren't", "can't", "couldn't", "daren't", "didn't", "doesn't",
  "dont", "hadnt", "hasnt", "havent", "isnt", "mightnt", "mustnt", "neither",
  "don't", "hadn't", "hasn't", "haven't", "isn't", "mightn't", "mustn't",
  "neednt", "needn't", "never", "none", "nope", "nor", "not", "nothing", "nowhere",
  "oughtnt", "shant", "shouldnt", "uhuh", "wasnt", "werent",
  "oughtn't", "shan't", "shouldn't", "uh-uh", "wasn't", "weren't",
- "without", "wont", "wouldnt", "won't", "wouldn't", "rarely", "seldom", "despite"]
+ "without", "wont", "wouldnt", "won't", "wouldn't", "rarely", "seldom", "despite"}
 
 # booster/dampener 'intensifiers' or 'degree adverbs'
 # http://en.wiktionary.org/wiki/Category:English_degree_adverbs
@@ -86,18 +88,14 @@ def negated(input_words, include_nt=True):
     """
     Determine if input contains negation words
     """
-    neg_words = []
-    neg_words.extend(NEGATE)
-    for word in neg_words:
-        if word in input_words:
-            return True
+    neg_words = NEGATE
+    if any(word.lower() in neg_words for word in input_words):
+        return True
     if include_nt:
-        for word in input_words:
-            if "n't" in word:
-                return True
-    if "least" in input_words:
-        i = input_words.index("least")
-        if i > 0 and input_words[i-1] != "at":
+        if any("n't" in word.lower() for word in input_words):
+            return True
+    for first, second in pairwise(input_words):
+        if second.lower() == "least" and first.lower() != 'at':
             return True
     return False
 
@@ -160,37 +158,38 @@ class SentiText(object):
         # adjacent punctuation (keeps emoticons & contractions)
         self.is_cap_diff = allcap_differential(self.words_and_emoticons)
 
-    def _words_only(self):
-        text_mod = REGEX_REMOVE_PUNCTUATION.sub('', self.text)
+    def _words_plus_punc(self):
+        """
+        Returns mapping of form:
+        {
+            'cat,': 'cat',
+            ',cat': 'cat',
+        }
+        """
+        no_punc_text = REGEX_REMOVE_PUNCTUATION.sub('', self.text)
         # removes punctuation (but loses emoticons & contractions)
-        words_only = text_mod.split()
-        # get rid of empty items or single letter "words" like 'a' and 'I'
-        words_only = [word for word in words_only if len(word) > 1]
-        return words_only
+        words_only = no_punc_text.split()
+        # remove singletons
+        words_only = set( w for w in words_only if len(w) > 1 )
+        # the product gives ('cat', ',') and (',', 'cat')
+        punc_before = {''.join(p): p[1] for p in product(PUNC_LIST, words_only)}
+        punc_after = {''.join(p): p[0] for p in product(words_only, PUNC_LIST)}
+        words_punc_dict = punc_before
+        words_punc_dict.update(punc_after)
+        return words_punc_dict
 
     def _words_and_emoticons(self):
+        """
+        Removes leading and trailing puncutation
+        Leaves contractions and most emoticons
+            Does not preserve punc-plus-letter emoticons (e.g. :D)
+        """
         wes = self.text.split()
-
-        # get rid of residual empty items or single letter words
+        words_punc_dict = self._words_plus_punc()
         wes = [we for we in wes if len(we) > 1]
-
-        for word in self._words_only():
-            for punct in PUNC_LIST:
-                pword = punct + word
-                x1 = wes.count(pword)
-                while x1 > 0:
-                    i = wes.index(pword)
-                    wes.remove(pword)
-                    wes.insert(i, word)
-                    x1 = wes.count(pword)
-
-                wordp = word + punct
-                x2 = wes.count(wordp)
-                while x2 > 0:
-                    i = wes.index(wordp)
-                    wes.remove(wordp)
-                    wes.insert(i, word)
-                    x2 = wes.count(wordp)
+        for i, we in enumerate(wes):
+            if we in words_punc_dict:
+                wes[i] = words_punc_dict[we]
         return wes
 
 class SentimentIntensityAnalyzer(object):

{nltk-3.2.1 → nltk-3.3}/nltk/stem/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Stemmers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
 #         Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
@@ -29,5 +29,3 @@ from nltk.stem.porter import PorterStemmer
 from nltk.stem.snowball import SnowballStemmer
 from nltk.stem.wordnet import WordNetLemmatizer
 from nltk.stem.rslp import RSLPStemmer
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/stem/api.py

@@ -1,18 +1,24 @@
 # Natural Language Toolkit: Stemmer Interface
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
 #         Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
+
+
+@add_metaclass(ABCMeta)
 class StemmerI(object):
     """
     A processing interface for removing morphological affixes from
     words.  This process is known as stemming.
 
     """
+    @abstractmethod
     def stem(self, token):
         """
         Strip affixes from the token and return the stem.
@@ -20,6 +26,3 @@ class StemmerI(object):
         :param token: The token that should be stemmed.
         :type token: str
         """
-        raise NotImplementedError()
-
-