nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/chasen.py

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Masato Hagiwara <hagisan@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -9,9 +9,10 @@ from __future__ import print_function
 
 import sys
 
+from six import string_types
+
 from nltk.corpus.reader import util
 
-from nltk import compat
 from nltk.corpus.reader.util import *
 from nltk.corpus.reader.api import *
 
@@ -23,7 +24,7 @@ class ChasenCorpusReader(CorpusReader):
 
     def raw(self, fileids=None):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def words(self, fileids=None):
@@ -132,7 +133,7 @@ def test():
     jeita = LazyCorpusLoader(
         'jeita', ChasenCorpusReader, r'.*chasen', encoding='utf-8')
 
-    assert isinstance(jeita.tagged_words()[0][1], compat.string_types)
+    assert isinstance(jeita.tagged_words()[0][1], string_types)
 
 if __name__ == '__main__':
     demo()

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/childes.py

@@ -1,6 +1,6 @@
 # CHILDES XML Corpus Reader
 
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Tomonori Nagano <tnagano@gc.cuny.edu>
 #         Alexis Dimitriadis <A.Dimitriadis@uu.nl>
 # URL: <http://nltk.org/>
@@ -9,15 +9,15 @@
 """
 Corpus reader for the XML version of the CHILDES corpus.
 """
-from __future__ import print_function
+from __future__ import print_function, division
 
 __docformat__ = 'epytext en'
 
 import re
 from collections import defaultdict
+from six import string_types
 
 from nltk.util import flatten, LazyMap, LazyConcatenation
-from nltk.compat import string_types
 
 from nltk.corpus.reader.util import concat
 from nltk.corpus.reader.xmldocs import XMLCorpusReader, ElementTree
@@ -123,7 +123,7 @@ class CHILDESCorpusReader(XMLCorpusReader):
         if not self._lazy:
             return [self._get_words(fileid, speaker, sent, stem, relation,
                 pos, strip_space, replace) for fileid in self.abspaths(fileids)]
-        
+
         get_words = lambda fileid: self._get_words(fileid, speaker, sent, stem, relation,
             pos, strip_space, replace)
         return LazyConcatenation(LazyMap(get_words, self.abspaths(fileids)))
@@ -153,7 +153,7 @@ class CHILDESCorpusReader(XMLCorpusReader):
         if not self._lazy:
             return [self._get_words(fileid, speaker, sent, stem, relation,
                 pos, strip_space, replace) for fileid in self.abspaths(fileids)]
-        
+
         get_words = lambda fileid: self._get_words(fileid, speaker, sent, stem, relation,
             pos, strip_space, replace)
         return LazyConcatenation(LazyMap(get_words, self.abspaths(fileids)))
@@ -278,9 +278,9 @@ class CHILDESCorpusReader(XMLCorpusReader):
             thisWordList = flatten(results)
             # count number of morphemes
             # (e.g., 'read' = 1 morpheme but 'read-PAST' is 2 morphemes)
-            numWords = float(len(flatten([word.split('-')
-                                          for word in thisWordList]))) - numFillers
-            numSents = float(len(results)) - sentDiscount
+            numWords = len(flatten([word.split('-')
+                                          for word in thisWordList])) - numFillers
+            numSents = len(results) - sentDiscount
             mlu = numWords/numSents
         except ZeroDivisionError:
             mlu = 0
@@ -520,4 +520,3 @@ def demo(corpus_root=None):
 
 if __name__ == "__main__":
     demo()
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/chunked.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Chunked Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -13,9 +13,10 @@ documents.
 
 import os.path, codecs
 
+from six import string_types
+
 import nltk
 from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
-from nltk import compat
 from nltk.tree import Tree
 from nltk.tokenize import *
 from nltk.chunk import tagstr2tree
@@ -53,7 +54,7 @@ class ChunkedCorpusReader(CorpusReader):
         :rtype: str
         """
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def words(self, fileids=None):
@@ -209,4 +210,3 @@ class ChunkedCorpusView(StreamBackedCorpusView):
             else:
                 raise ValueError('expected child to be Tree or tuple')
         return tree
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/cmudict.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Carnegie Mellon Pronouncing Dictionary Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -47,6 +47,8 @@ ZH      seizure S IY ZH ER
 
 import codecs
 
+from six import string_types
+
 from nltk import compat
 from nltk.util import Index
 
@@ -68,7 +70,7 @@ class CMUDictCorpusReader(CorpusReader):
         :return: the cmudict lexicon as a raw string.
         """
         fileids = self._fileids
-        if isinstance(fileids, compat.string_types):
+        if isinstance(fileids, string_types):
             fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/comparative_sents.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Comparative Sentence Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Pierpaolo Pantone <24alsecondo@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -35,6 +35,8 @@ Related papers:
 """
 import re
 
+from six import string_types
+
 from nltk.corpus.reader.api import *
 from nltk.tokenize import *
 
@@ -120,7 +122,7 @@ class ComparativeSentencesCorpusReader(CorpusReader):
         """
         if fileids is None:
             fileids = self._fileids
-        elif isinstance(fileids, compat.string_types):
+        elif isinstance(fileids, string_types):
             fileids = [fileids]
         return concat([self.CorpusView(path, self._read_comparison_block, encoding=enc)
             for (path, enc, fileid) in self.abspaths(fileids, True, True)])

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/conll.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: CONLL Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -16,6 +16,8 @@ import os
 import codecs
 import textwrap
 
+from six import string_types
+
 from nltk import compat
 from nltk.tree import Tree
 from nltk.util import LazyMap, LazyConcatenation
@@ -33,7 +35,11 @@ class ConllCorpusReader(CorpusReader):
     annotation type.  The set of columns used by CoNLL-style files can
     vary from corpus to corpus; the ``ConllCorpusReader`` constructor
     therefore takes an argument, ``columntypes``, which is used to
-    specify the columns that are used by a given corpus.
+    specify the columns that are used by a given corpus. By default
+    columns are split by consecutive whitespaces, with the
+    ``separator`` argument you can set a string to split by (e.g.
+    ``\'\t\'``).
+
 
     @todo: Add support for reading from corpora where different
         parallel files contain different columns.
@@ -68,11 +74,11 @@ class ConllCorpusReader(CorpusReader):
     def __init__(self, root, fileids, columntypes,
                  chunk_types=None, root_label='S', pos_in_tree=False,
                  srl_includes_roleset=True, encoding='utf8',
-                 tree_class=Tree, tagset=None):
+                 tree_class=Tree, tagset=None, separator=None):
         for columntype in columntypes:
             if columntype not in self.COLUMN_TYPES:
                 raise ValueError('Bad column type %r' % columntype)
-        if isinstance(chunk_types, compat.string_types):
+        if isinstance(chunk_types, string_types):
             chunk_types = [chunk_types]
         self._chunk_types = chunk_types
         self._colmap = dict((c,i) for (i,c) in enumerate(columntypes))
@@ -82,6 +88,7 @@ class ConllCorpusReader(CorpusReader):
         self._tree_class = tree_class
         CorpusReader.__init__(self, root, fileids, encoding)
         self._tagset = tagset
+        self.sep = separator
 
     #/////////////////////////////////////////////////////////////////
     # Data Access Methods
@@ -89,7 +96,7 @@ class ConllCorpusReader(CorpusReader):
 
     def raw(self, fileids=None):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def words(self, fileids=None):
@@ -192,7 +199,7 @@ class ConllCorpusReader(CorpusReader):
             block = block.strip()
             if not block: continue
 
-            grid = [line.split() for line in block.split('\n')]
+            grid = [line.split(self.sep) for line in block.split('\n')]
 
             # If there's a docstart row, then discard. ([xx] eventually it
             # would be good to actually use it)
@@ -280,16 +287,16 @@ class ConllCorpusReader(CorpusReader):
             right = right.count(')')*')' # only keep ')'.
             treestr += '%s (%s %s) %s' % (left, pos_tag, word, right)
         try:
-            tree = self._tree_class.parse(treestr)
+            tree = self._tree_class.fromstring(treestr)
         except (ValueError, IndexError):
-            tree = self._tree_class.parse('(%s %s)' %
+            tree = self._tree_class.fromstring('(%s %s)' %
                                           (self._root_label, treestr))
 
         if not pos_in_tree:
             for subtree in tree.subtrees():
                 for i, child in enumerate(subtree):
                     if (isinstance(child, Tree) and len(child)==1 and
-                        isinstance(child[0], compat.string_types)):
+                        isinstance(child[0], string_types)):
                         subtree[i] = (child[0], child.label())
 
         return tree
@@ -492,7 +499,7 @@ class ConllSRLInstanceList(list):
 
     def _tree2conll(self, tree, wordnum, words, pos, synt):
         assert isinstance(tree, Tree)
-        if len(tree) == 1 and isinstance(tree[0], compat.string_types):
+        if len(tree) == 1 and isinstance(tree[0], string_types):
             pos[wordnum] = tree.label()
             assert words[wordnum] == tree[0]
             return wordnum+1
@@ -514,8 +521,8 @@ class ConllChunkCorpusReader(ConllCorpusReader):
     pos, and chunk.
     """
     def __init__(self, root, fileids, chunk_types, encoding='utf8',
-                 tagset=None):
+                 tagset=None, separator=None):
         ConllCorpusReader.__init__(
             self, root, fileids, ('words', 'pos', 'chunk'),
             chunk_types=chunk_types, encoding=encoding,
-            tagset=tagset)
+            tagset=tagset, separator=separator)

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/crubadan.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: An Crubadan N-grams Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Avital Pekker <avital.pekker@utoronto.ca>
 #
 # URL: <http://nltk.org/>
@@ -93,7 +93,7 @@ class CrubadanCorpusReader(CorpusReader):
         ngram_file = path.join(self.root, crubadan_code + '-3grams.txt')
 
         if not path.isfile(ngram_file):
-            raise Runtime("No N-gram file found for requested language.")
+            raise RuntimeError("No N-gram file found for requested language.")
 
         counts = FreqDist()
         if PY3:

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/dependency.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Dependency Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Kepa Sarasola <kepa.sarasola@ehu.es>
 #         Iker Manterola <returntothehangar@hotmail.com>
 #