nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/ccg/chart.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Combinatory Categorial Grammar
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Graeme Gange <ggange@csse.unimelb.edu.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -33,6 +33,8 @@ from __future__ import print_function, division, unicode_literals
 
 import itertools
 
+from six import string_types
+
 from nltk.parse import ParserI
 from nltk.parse.chart import AbstractChartRule, EdgeI, Chart
 from nltk.tree import Tree
@@ -42,7 +44,7 @@ from nltk.ccg.combinator import (ForwardT, BackwardT, ForwardApplication,
                                  BackwardApplication, ForwardComposition,
                                  BackwardComposition, ForwardSubstitution,
                                  BackwardBx, BackwardSx)
-from nltk.compat import python_2_unicode_compatible, string_types
+from nltk.compat import python_2_unicode_compatible
 from nltk.ccg.combinator import *
 from nltk.ccg.logic import *
 from nltk.sem.logic import *
@@ -264,11 +266,11 @@ class CCGChart(Chart):
         memo[edge] = trees
         return trees
 
-           
+
 def compute_semantics(children, edge):
     if children[0].label()[0].semantics() is None:
         return None
-        
+
     if len(children) is 2:
         if isinstance(edge.rule(), BackwardCombinator):
             children = [children[1],children[0]]
@@ -335,7 +337,7 @@ def printCCGTree(lwidth,tree):
 
     (token, op) = tree.label()
 
-    if op == u'Leaf':
+    if op == 'Leaf':
         return rwidth
 
     # Pad to the left with spaces, followed by a sequence of '-'

{nltk-3.2.1 → nltk-3.3}/nltk/ccg/combinator.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Combinatory Categorial Grammar
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Graeme Gange <ggange@csse.unimelb.edu.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -9,10 +9,14 @@ CCG Combinators
 """
 
 from __future__ import unicode_literals
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
 
 from nltk.compat import python_2_unicode_compatible
 from nltk.ccg.api import FunctionalCategory
 
+
+@add_metaclass(ABCMeta)
 class UndirectedBinaryCombinator(object):
     """
     Abstract class for representing a binary combinator.
@@ -24,12 +28,16 @@ class UndirectedBinaryCombinator(object):
     of the combinators; these restrictions must be added in the rule
     class.
     """
+    @abstractmethod
     def can_combine(self, function, argument):
-        raise NotImplementedError()
+        pass
+
+    @abstractmethod
+    def combine(self, function, argument):
+        pass
 
-    def combine (self, function, argument):
-        raise NotImplementedError()
 
+@add_metaclass(ABCMeta)
 class DirectedBinaryCombinator(object):
     """
     Wrapper for the undirected binary combinator.
@@ -37,11 +45,14 @@ class DirectedBinaryCombinator(object):
     the function, and which the argument.
     It then decides whether or not they can be combined.
     """
+    @abstractmethod
     def can_combine(self, left, right):
-        raise NotImplementedError()
+        pass
 
+    @abstractmethod
     def combine(self, left, right):
-        raise NotImplementedError()
+        pass
+
 
 @python_2_unicode_compatible
 class ForwardCombinator(DirectedBinaryCombinator):
@@ -67,6 +78,7 @@ class ForwardCombinator(DirectedBinaryCombinator):
     def __str__(self):
         return ">%s%s" % (self._combinator, self._suffix)
 
+
 @python_2_unicode_compatible
 class BackwardCombinator(DirectedBinaryCombinator):
     """
@@ -80,6 +92,7 @@ class BackwardCombinator(DirectedBinaryCombinator):
     def can_combine(self, left, right):
         return (self._combinator.can_combine(right, left) and
                 self._predicate(left, right))
+
     def combine(self, left, right):
         for cat in self._combinator.combine(right, left):
             yield cat
@@ -87,6 +100,7 @@ class BackwardCombinator(DirectedBinaryCombinator):
     def __str__(self):
         return "<%s%s" % (self._combinator, self._suffix)
 
+
 @python_2_unicode_compatible
 class UndirectedFunctionApplication(UndirectedBinaryCombinator):
     """
@@ -122,10 +136,12 @@ class UndirectedFunctionApplication(UndirectedBinaryCombinator):
 def forwardOnly(left, right):
     return left.dir().is_forward()
 
+
 # Ensures the right functor takes an argument on the left
 def backwardOnly(left, right):
     return right.dir().is_backward()
 
+
 # Application combinator instances
 ForwardApplication = ForwardCombinator(UndirectedFunctionApplication(),
                                        forwardOnly)
@@ -155,25 +171,29 @@ class UndirectedComposition(UndirectedBinaryCombinator):
             return
         if function.dir().can_compose() and argument.dir().can_compose():
             subs = function.arg().can_unify(argument.res())
-            if not subs is None:
+            if subs is not None:
                 yield FunctionalCategory(function.res().substitute(subs),
-                                         argument.arg().substitute(subs), argument.dir())
+                                         argument.arg().substitute(subs),
+                                         argument.dir())
 
     def __str__(self):
         return 'B'
 
+
 # Predicates for restricting application of straight composition.
 def bothForward(left, right):
     return left.dir().is_forward() and right.dir().is_forward()
 
+
 def bothBackward(left, right):
     return left.dir().is_backward() and right.dir().is_backward()
 
-# Predicates for crossed composition
 
+# Predicates for crossed composition
 def crossedDirs(left, right):
     return left.dir().is_forward() and right.dir().is_backward()
 
+
 def backwardBxConstraint(left, right):
     # The functors must be crossed inwards
     if not crossedDirs(left, right):
@@ -184,6 +204,7 @@ def backwardBxConstraint(left, right):
     # The resulting argument category is restricted to be primitive
     return left.arg().is_primitive()
 
+
 # Straight composition combinators
 ForwardComposition = ForwardCombinator(UndirectedComposition(),
                                        forwardOnly)
@@ -194,6 +215,7 @@ BackwardComposition = BackwardCombinator(UndirectedComposition(),
 BackwardBx = BackwardCombinator(UndirectedComposition(), backwardBxConstraint,
                                 suffix='x')
 
+
 @python_2_unicode_compatible
 class UndirectedSubstitution(UndirectedBinaryCombinator):
     """
@@ -219,17 +241,20 @@ class UndirectedSubstitution(UndirectedBinaryCombinator):
 
     def combine(self, function, argument):
         if self.can_combine(function, argument):
-            yield FunctionalCategory(function.res().res(), argument.arg(), argument.dir())
+            yield FunctionalCategory(function.res().res(), argument.arg(),
+                                     argument.dir())
 
     def __str__(self):
         return 'S'
 
+
 # Predicate for forward substitution
 def forwardSConstraint(left, right):
     if not bothForward(left, right):
         return False
     return left.res().dir().is_forward() and left.arg().is_primitive()
 
+
 # Predicate for backward crossed substitution
 def backwardSxConstraint(left, right):
     if not left.dir().can_cross() and right.dir().can_cross():
@@ -238,6 +263,7 @@ def backwardSxConstraint(left, right):
         return False
     return right.res().dir().is_backward() and right.arg().is_primitive()
 
+
 # Instances of substitution combinators
 ForwardSubstitution = ForwardCombinator(UndirectedSubstitution(),
                                         forwardSConstraint)
@@ -252,6 +278,7 @@ def innermostFunction(categ):
         categ = categ.res()
     return categ
 
+
 @python_2_unicode_compatible
 class UndirectedTypeRaise(UndirectedBinaryCombinator):
     """
@@ -288,12 +315,14 @@ class UndirectedTypeRaise(UndirectedBinaryCombinator):
         if subs is not None:
             xcat = arg.res().substitute(subs)
             yield FunctionalCategory(xcat,
-                                     FunctionalCategory(xcat, function, arg.dir()),
+                                     FunctionalCategory(xcat, function,
+                                                        arg.dir()),
                                      -(arg.dir()))
 
     def __str__(self):
         return 'T'
 
+
 # Predicates for type-raising
 # The direction of the innermost category must be towards
 # the primary functor.
@@ -303,10 +332,12 @@ def forwardTConstraint(left, right):
     arg = innermostFunction(right)
     return arg.dir().is_backward() and arg.res().is_primitive()
 
+
 def backwardTConstraint(left, right):
     arg = innermostFunction(left)
     return arg.dir().is_forward() and arg.res().is_primitive()
 
+
 # Instances of type-raising combinators
 ForwardT = ForwardCombinator(UndirectedTypeRaise(), forwardTConstraint)
 BackwardT = BackwardCombinator(UndirectedTypeRaise(), backwardTConstraint)

{nltk-3.2.1 → nltk-3.3}/nltk/ccg/lexicon.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Combinatory Categorial Grammar
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Graeme Gange <ggange@csse.unimelb.edu.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/ccg/logic.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Combinatory Categorial Grammar
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Tanin Na Nakorn (@tanin)
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/chat/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Chatbots
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Authors: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/chat/eliza.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Eliza
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Authors: Steven Bird <stevenbird1@gmail.com>
 #          Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/chat/iesha.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Teen Chatbot
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Selina Dennis <sjmd@csse.unimelb.edu.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/chat/rude.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Rude Chatbot
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Peter Spiller <pspiller@csse.unimelb.edu.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/chat/suntsu.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Sun Tsu-Bot
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Sam Huston 2007
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/chat/util.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Chatbot Utilities
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Authors: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -11,7 +11,9 @@ from __future__ import print_function
 
 import re
 import random
-from nltk import compat
+
+from six.moves import input
+
 
 reflections = {
   "i am"       : "you are",
@@ -109,12 +111,12 @@ class Chat(object):
 
     # Hold a conversation with a chatbot
     def converse(self, quit="quit"):
-        input = ""
-        while input != quit:
-            input = quit
-            try: input = compat.raw_input(">")
+        user_input = ""
+        while user_input != quit:
+            user_input = quit
+            try: user_input = input(">")
             except EOFError:
-                print(input)
-            if input:
-                while input[-1] in "!.": input = input[:-1]
-                print(self.respond(input))
+                print(user_input)
+            if user_input:
+                while user_input[-1] in "!.": user_input = user_input[:-1]
+                print(self.respond(user_input))

{nltk-3.2.1 → nltk-3.3}/nltk/chat/zen.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Zen Chatbot
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Amy Holland <amyrh@csse.unimelb.edu.au>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/chunk/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Chunkers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/chunk/api.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Chunk parsing API
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/chunk/named_entity.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Chunk parsing API
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -9,6 +9,7 @@
 Named entity chunker
 """
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os, re, pickle
 from xml.etree import ElementTree as ET
@@ -101,9 +102,9 @@ class NEChunkParserTagger(ClassifierBasedTagger):
             'nextpos': nextpos,
             'prevword': prevword,
             'nextword': nextword,
-            'word+nextpos': '%s+%s' % (word.lower(), nextpos),
-            'pos+prevtag': '%s+%s' % (pos, prevtag),
-            'shape+prevtag': '%s+%s' % (prevshape, prevtag),
+            'word+nextpos': '{0}+{1}'.format(word.lower(), nextpos),
+            'pos+prevtag': '{0}+{1}'.format(pos, prevtag),
+            'shape+prevtag': '{0}+{1}'.format(prevshape, prevtag),
             }
 
         return features
@@ -159,9 +160,9 @@ class NEChunkParser(ChunkParserI):
                 if len(child) == 0:
                     print("Warning -- empty chunk in sentence")
                     continue
-                toks.append((child[0], 'B-%s' % child.label()))
+                toks.append((child[0], 'B-{0}'.format(child.label())))
                 for tok in child[1:]:
-                    toks.append((tok, 'I-%s' % child.label()))
+                    toks.append((tok, 'I-{0}'.format(child.label())))
             else:
                 toks.append((child, 'O'))
         return toks
@@ -210,7 +211,7 @@ def load_ace_data(roots, fmt='binary', skip_bnews=True):
                         yield sent
 
 def load_ace_file(textfile, fmt):
-    print('  - %s' % os.path.split(textfile)[1])
+    print('  - {0}'.format(os.path.split(textfile)[1]))
     annfile = textfile+'.tmx.rdc.xml'
 
     # Read the xml file, and get a list of entities
@@ -281,12 +282,12 @@ def cmp_chunks(correct, guessed):
     for (w, ct), (w, gt) in zip(correct, guessed):
         if ct == gt == 'O':
             if not ellipsis:
-                print("  %-15s %-15s %s" % (ct, gt, w))
-                print('  %-15s %-15s %s' % ('...', '...', '...'))
+                print("  {:15} {:15} {2}".format(ct, gt, w))
+                print('  {:15} {:15} {2}'.format('...', '...', '...'))
                 ellipsis = True
         else:
             ellipsis = False
-            print("  %-15s %-15s %s" % (ct, gt, w))
+            print("  {:15} {:15} {2}".format(ct, gt, w))
 
 def build_model(fmt='binary'):
     print('Loading training data...')
@@ -313,8 +314,8 @@ def build_model(fmt='binary'):
         if i < 3: cmp_chunks(correct, guess)
     print(chunkscore)
 
-    outfilename = '/tmp/ne_chunker_%s.pickle' % fmt
-    print('Saving chunker to %s...' % outfilename)
+    outfilename = '/tmp/ne_chunker_{0}.pickle'.format(fmt)
+    print('Saving chunker to {0}...'.format(outfilename))
 
     with open(outfilename, 'wb') as outfile:
         pickle.dump(cp, outfile, -1)
@@ -328,4 +329,3 @@ if __name__ == '__main__':
 
     build_model('binary')
     build_model('multiclass')
-

{nltk-3.2.1 → nltk-3.3}/nltk/chunk/regexp.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Regular Expression Chunkers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
 # URL: <http://nltk.org/>
@@ -10,9 +10,11 @@ from __future__ import division
 
 import re
 
+from six import string_types
+
 from nltk.tree import Tree
 from nltk.chunk.api import ChunkParserI
-from nltk.compat import python_2_unicode_compatible, string_types, unicode_repr
+from nltk.compat import python_2_unicode_compatible, unicode_repr
 
 ##//////////////////////////////////////////////////////
 ##  ChunkString
@@ -829,9 +831,13 @@ class ChunkRuleWithContext(RegexpChunkRule):
 # this should probably be made more strict than it is -- e.g., it
 # currently accepts 'foo'.
 CHUNK_TAG_PATTERN = re.compile(r'^((%s|<%s>)*)$' %
-                                ('[^\{\}<>]+',
+                                ('([^\{\}<>]|\{\d+,?\}|\{\d*,\d+\})+',
                                  '[^\{\}<>]+'))
 
+
+
+
+
 def tag_pattern2re_pattern(tag_pattern):
     """
     Convert a tag pattern to a regular expression pattern.  A "tag

{nltk-3.2.1 → nltk-3.3}/nltk/chunk/util.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Chunk format conversions
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
 # URL: <http://nltk.org/>
@@ -288,10 +288,10 @@ class ChunkScore(object):
         :rtype: str
         """
         return ("ChunkParse score:\n" +
-                ("    IOB Accuracy: %5.1f%%\n" % (self.accuracy()*100)) +
-                ("    Precision:    %5.1f%%\n" % (self.precision()*100)) +
-                ("    Recall:       %5.1f%%\n" % (self.recall()*100))+
-                ("    F-Measure:    %5.1f%%" % (self.f_measure()*100)))
+                ("    IOB Accuracy: {:5.1f}%%\n".format(self.accuracy()*100)) +
+                ("    Precision:    {:5.1f}%%\n".format(self.precision()*100)) +
+                ("    Recall:       {:5.1f}%%\n".format(self.recall()*100))+
+                ("    F-Measure:    {:5.1f}%%".format(self.f_measure()*100)))
 
 # extract chunks, and assign unique id, the absolute position of
 # the first word of the chunk
@@ -334,13 +334,13 @@ def tagstr2tree(s, chunk_label="NP", root_label="S", sep='/',
         text = match.group()
         if text[0] == '[':
             if len(stack) != 1:
-                raise ValueError('Unexpected [ at char %d' % match.start())
+                raise ValueError('Unexpected [ at char {:d}'.format(match.start()))
             chunk = Tree(chunk_label, [])
             stack[-1].append(chunk)
             stack.append(chunk)
         elif text[0] == ']':
             if len(stack) != 2:
-                raise ValueError('Unexpected ] at char %d' % match.start())
+                raise ValueError('Unexpected ] at char {:d}'.format(match.start()))
             stack.pop()
         else:
             if sep is None:
@@ -352,7 +352,7 @@ def tagstr2tree(s, chunk_label="NP", root_label="S", sep='/',
                 stack[-1].append((word, tag))
 
     if len(stack) != 1:
-        raise ValueError('Expected ] at char %d' % len(s))
+        raise ValueError('Expected ] at char {:d}'.format(len(s)))
     return stack[0]
 
 ### CONLL
@@ -384,7 +384,7 @@ def conllstr2tree(s, chunk_types=('NP', 'PP', 'VP'), root_label="S"):
         # Decode the line.
         match = _LINE_RE.match(line)
         if match is None:
-            raise ValueError('Error on line %d' % lineno)
+            raise ValueError('Error on line {:d}'.format(lineno))
         (word, tag, state, chunk_type) = match.groups()
 
         # If it's a chunk type we don't care about, treat it as O.
@@ -461,7 +461,7 @@ def conlltags2tree(sentence, chunk_types=('NP','PP','VP'),
         elif chunktag == 'O':
             tree.append((word,postag))
         else:
-            raise ValueError("Bad conll tag %r" % chunktag)
+            raise ValueError("Bad conll tag {0!r}".format(chunktag))
     return tree
 
 def tree2conllstr(t):
@@ -512,8 +512,8 @@ def _ieer_read_text(s, root_label):
             else:
                 stack[-1].append(piece)
         except (IndexError, ValueError):
-            raise ValueError('Bad IEER string (error at character %d)' %
-                             piece_m.start())
+            raise ValueError('Bad IEER string (error at character {:d})'.format \
+                             (piece_m.start()))
     if len(stack) != 1:
         raise ValueError('Bad IEER string')
     return stack[0]

{nltk-3.2.1 → nltk-3.3}/nltk/classify/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Classifiers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/classify/api.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Classifier Interface
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
 # URL: <http://nltk.org/>