nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/propbank.py

@@ -1,18 +1,19 @@
 # Natural Language Toolkit: PropBank Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
 from __future__ import unicode_literals
 import re
+from functools import total_ordering
 from xml.etree import ElementTree
 
-from nltk import compat
+from six import string_types
+
 from nltk.tree import Tree
 from nltk.internals import raise_unorderable_types
-from nltk.compat import total_ordering
 
 from nltk.corpus.reader.util import *
 from nltk.corpus.reader.api import *
@@ -48,7 +49,7 @@ class PropbankCorpusReader(CorpusReader):
             necessary to resolve the tree pointers used by propbank.
         """
         # If framefiles is specified as a regexp, expand it.
-        if isinstance(framefiles, compat.string_types):
+        if isinstance(framefiles, string_types):
             framefiles = find_corpus_fileids(root, framefiles)
         framefiles = list(framefiles)
         # Initialze the corpus reader.
@@ -67,7 +68,7 @@ class PropbankCorpusReader(CorpusReader):
         :return: the text contents of the given fileids, as a single string.
         """
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, ): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def instances(self, baseform=None):
@@ -107,9 +108,7 @@ class PropbankCorpusReader(CorpusReader):
         for roleset in etree.findall('predicate/roleset'):
             if roleset.attrib['id'] == roleset_id:
                 return roleset
-        else:
-            raise ValueError('Roleset %s not found in %s' %
-                             (roleset_id, framefile))
+        raise ValueError('Roleset %s not found in %s' % (roleset_id, framefile))
 
     def rolesets(self, baseform=None):
         """
@@ -472,10 +471,9 @@ class PropbankInflection(object):
 
     @staticmethod
     def parse(s):
-        if not isinstance(s, compat.string_types):
+        if not isinstance(s, string_types):
             raise TypeError('expected a string')
         if (len(s) != 5 or
             not PropbankInflection._VALIDATE.match(s)):
             raise ValueError('Bad propbank inflection string %r' % s)
         return PropbankInflection(*s)
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/pros_cons.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Pros and Cons Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Pierpaolo Pantone <24alsecondo@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -27,6 +27,8 @@ Related papers:
 """
 import re
 
+from six import string_types
+
 from nltk.corpus.reader.api import *
 from nltk.tokenize import *
 
@@ -75,7 +77,7 @@ class ProsConsCorpusReader(CategorizedCorpusReader, CorpusReader):
         fileids = self._resolve(fileids, categories)
         if fileids is None:
             fileids = self._fileids
-        elif isinstance(fileids, compat.string_types):
+        elif isinstance(fileids, string_types):
             fileids = [fileids]
         return concat([self.CorpusView(path, self._read_sent_block, encoding=enc)
             for (path, enc, fileid) in self.abspaths(fileids, True, True)])
@@ -95,7 +97,7 @@ class ProsConsCorpusReader(CategorizedCorpusReader, CorpusReader):
         fileids = self._resolve(fileids, categories)
         if fileids is None:
             fileids = self._fileids
-        elif isinstance(fileids, compat.string_types):
+        elif isinstance(fileids, string_types):
             fileids = [fileids]
         return concat([self.CorpusView(path, self._read_word_block, encoding=enc)
             for (path, enc, fileid) in self.abspaths(fileids, True, True)])

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/reviews.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Product Reviews Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Pierpaolo Pantone <24alsecondo@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -61,6 +61,8 @@ Note: Some of the files (e.g. "ipod.txt", "Canon PowerShot SD500.txt") do not
 
 from __future__ import division
 
+from six import string_types
+
 import re
 
 from nltk.corpus.reader.api import *

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/rte.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: RTE Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author:  Ewan Klein <ewan@inf.ed.ac.uk>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -33,6 +33,9 @@ file, taking values 1, 2 or 3. The GID is formatted 'm-n', where 'm' is the
 challenge number and 'n' is the pair ID.
 """
 from __future__ import unicode_literals
+
+from six import string_types
+
 from nltk import compat
 from nltk.corpus.reader.util import *
 from nltk.corpus.reader.api import *
@@ -137,10 +140,5 @@ class RTECorpusReader(XMLCorpusReader):
         :type: list
         :rtype: list(RTEPair)
         """
-        if isinstance(fileids, compat.string_types): fileids = [fileids]
+        if isinstance(fileids, string_types): fileids = [fileids]
         return concat([self._read_etree(self.xml(fileid)) for fileid in fileids])
-
-
-
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/semcor.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: SemCor Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Nathan Schneider <nschneid@cs.cmu.edu>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/senseval.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Senseval 2 Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@cs.mu.oz.au>
 #         Steven Bird <stevenbird1@gmail.com> (modifications)
 # URL: <http://nltk.org/>
@@ -23,6 +23,8 @@ is tagged with a sense identifier, and supplied with context.
 """
 from __future__ import print_function, unicode_literals
 
+from six import string_types
+
 import re
 from xml.etree import ElementTree
 
@@ -56,7 +58,7 @@ class SensevalCorpusReader(CorpusReader):
         :return: the text contents of the given fileids, as a single string.
         """
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def _entry(self, tree):

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/sentiwordnet.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: SentiWordNet
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Christopher Potts <cgpotts@stanford.edu>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -74,7 +74,7 @@ class SentiWordNetCorpusReader(CorpusReader):
             pos, offset = vals
             if pos == 's':
                 pos = 'a'
-            synset = wn._synset_from_pos_and_offset(pos, offset)
+            synset = wn.synset_from_pos_and_offset(pos, offset)
             return SentiSynset(pos_score, neg_score, synset)
         else:
             synset = wn.synset(vals[0])
@@ -102,7 +102,7 @@ class SentiWordNetCorpusReader(CorpusReader):
         for key, fields in self._db.items():
             pos, offset = key
             pos_score, neg_score = fields
-            synset = wn._synset_from_pos_and_offset(pos, offset)
+            synset = wn.synset_from_pos_and_offset(pos, offset)
             yield SentiSynset(pos_score, neg_score, synset)
 
 

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/sinica_treebank.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Sinica Treebank Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/string_category.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: String Category Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -18,6 +18,7 @@ NUM:date When did Hawaii become a state ?
 """
 
 # based on PPAttachmentCorpusReader
+from six import string_types
 
 from nltk import compat
 from nltk.corpus.reader.util import *
@@ -38,7 +39,7 @@ class StringCategoryCorpusReader(CorpusReader):
 
     def tuples(self, fileids=None):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([StreamBackedCorpusView(fileid, self._read_tuple_block,
                                               encoding=enc)
                        for (fileid, enc) in self.abspaths(fileids, True)])
@@ -48,7 +49,7 @@ class StringCategoryCorpusReader(CorpusReader):
         :return: the text contents of the given fileids, as a single string.
         """
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def _read_tuple_block(self, stream):

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/switchboard.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Switchboard Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/tagged.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Tagged Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 #         Jacob Perkins <japerk@gmail.com>
@@ -13,7 +13,8 @@ A reader for corpora whose documents contain part-of-speech-tagged words.
 
 import os
 
-from nltk import compat
+from six import string_types
+
 from nltk.tag import str2tuple, map_tag
 from nltk.tokenize import *
 
@@ -65,7 +66,7 @@ class TaggedCorpusReader(CorpusReader):
         :rtype: str
         """
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def words(self, fileids=None):

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/timit.py

@@ -126,6 +126,8 @@ import re
 import tempfile
 import time
 
+from six import string_types
+
 from nltk import compat
 from nltk.tree import Tree
 from nltk.internals import import_from_stdlib
@@ -162,7 +164,7 @@ class TimitCorpusReader(CorpusReader):
         :param root: The root directory for this corpus.
         """
         # Ensure that wave files don't get treated as unicode data:
-        if isinstance(encoding, compat.string_types):
+        if isinstance(encoding, string_types):
             encoding = [('.*\.wav', None), ('.*', encoding)]
 
         CorpusReader.__init__(self, root,
@@ -205,11 +207,11 @@ class TimitCorpusReader(CorpusReader):
         region, gender, sentence type, or sentence number, if
         specified.
         """
-        if isinstance(dialect, compat.string_types): dialect = [dialect]
-        if isinstance(sex, compat.string_types): sex = [sex]
-        if isinstance(spkrid, compat.string_types): spkrid = [spkrid]
-        if isinstance(sent_type, compat.string_types): sent_type = [sent_type]
-        if isinstance(sentid, compat.string_types): sentid = [sentid]
+        if isinstance(dialect, string_types): dialect = [dialect]
+        if isinstance(sex, string_types): sex = [sex]
+        if isinstance(spkrid, string_types): spkrid = [spkrid]
+        if isinstance(sent_type, string_types): sent_type = [sent_type]
+        if isinstance(sentid, string_types): sentid = [sentid]
 
         utterances = self._utterances[:]
         if dialect is not None:
@@ -307,7 +309,7 @@ class TimitCorpusReader(CorpusReader):
 
     def phone_trees(self, utterances=None):
         if utterances is None: utterances = self._utterances
-        if isinstance(utterances, compat.string_types): utterances = [utterances]
+        if isinstance(utterances, string_types): utterances = [utterances]
 
         trees = []
         for utterance in utterances:
@@ -372,7 +374,7 @@ class TimitCorpusReader(CorpusReader):
 
     def _utterance_fileids(self, utterances, extension):
         if utterances is None: utterances = self._utterances
-        if isinstance(utterances, compat.string_types): utterances = [utterances]
+        if isinstance(utterances, string_types): utterances = [utterances]
         return ['%s%s' % (u, extension) for u in utterances]
 
     def play(self, utterance, start=0, end=None):

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/toolbox.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Toolbox Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Greg Aumann <greg_aumann@sil.org>
 #         Stuart Robinson <Stuart.Robinson@mpi.nl>
 #         Steven Bird <stevenbird1@gmail.com>
@@ -16,9 +16,9 @@ import os
 import re
 import codecs
 
-from nltk import compat
-from nltk.toolbox import ToolboxData
+from six import string_types
 
+from nltk.toolbox import ToolboxData
 from nltk.corpus.reader.util import *
 from nltk.corpus.reader.api import *
 
@@ -57,7 +57,7 @@ class ToolboxCorpusReader(CorpusReader):
 
     def raw(self, fileids):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
 

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/twitter.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Twitter Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -13,7 +13,8 @@ have been serialised into line-delimited JSON.
 import json
 import os
 
-from nltk import compat
+from six import string_types
+
 from nltk.tokenize import TweetTokenizer
 
 from nltk.corpus.reader.util import StreamBackedCorpusView, concat, ZipFilePathPointer
@@ -135,7 +136,7 @@ class TwitterCorpusReader(CorpusReader):
         """
         if fileids is None:
             fileids = self._fileids
-        elif isinstance(fileids, compat.string_types):
+        elif isinstance(fileids, string_types):
             fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
@@ -152,6 +153,3 @@ class TwitterCorpusReader(CorpusReader):
             tweet = json.loads(line)
             tweets.append(tweet)
         return tweets
-
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/util.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Corpus Reader Utilities
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -10,6 +10,7 @@ import os
 import bisect
 import re
 import tempfile
+from six import string_types, text_type
 from functools import reduce
 try:
     import cPickle as pickle
@@ -20,7 +21,6 @@ except ImportError:
 try: from xml.etree import cElementTree as ElementTree
 except ImportError: from xml.etree import ElementTree
 
-from nltk.compat import string_types, text_type
 from nltk.tokenize import wordpunct_tokenize
 from nltk.internals import slice_bounds
 from nltk.data import PathPointer, FileSystemPathPointer, ZipFilePathPointer
@@ -281,6 +281,11 @@ class StreamBackedCorpusView(AbstractLazySequence):
         if self._stream is None:
             self._open()
 
+        # If the file is empty, the while loop will never run.
+        # This *seems* to be all the state we need to set:
+        if self._eofpos == 0:
+            self._len = 0
+
         # Each iteration through this loop, we read a single block
         # from the stream.
         while filepos < self._eofpos:
@@ -797,4 +802,3 @@ def tagged_treebank_para_block_reader(stream):
         # Content line:
         else:
             para += line
-