nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/xmldocs.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: XML Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -18,7 +18,8 @@ import codecs
 try: from xml.etree import cElementTree as ElementTree
 except ImportError: from xml.etree import ElementTree
 
-from nltk import compat
+from six import string_types
+
 from nltk.data import SeekableUnicodeStreamReader
 from nltk.tokenize import WordPunctTokenizer
 from nltk.internals import ElementWrapper
@@ -42,7 +43,7 @@ class XMLCorpusReader(CorpusReader):
         # Make sure we have exactly one file -- no concatenating XML.
         if fileid is None and len(self._fileids) == 1:
             fileid = self._fileids[0]
-        if not isinstance(fileid, compat.string_types):
+        if not isinstance(fileid, string_types):
             raise TypeError('Expected a single file identifier string')
         # Read the XML in using ElementTree.
         elt = ElementTree.parse(self.abspath(fileid).open()).getroot()
@@ -79,7 +80,7 @@ class XMLCorpusReader(CorpusReader):
 
     def raw(self, fileids=None):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
 
@@ -156,7 +157,11 @@ class XMLCorpusView(StreamBackedCorpusView):
 
     def _detect_encoding(self, fileid):
         if isinstance(fileid, PathPointer):
-            s = fileid.open().readline()
+            try:
+                infile = fileid.open()
+                s = infile.readline()
+            finally:
+                infile.close()
         else:
             with open(fileid, 'rb') as infile:
                 s = infile.readline()

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/ycoe.py

@@ -22,7 +22,8 @@ to the YCOE standard: http://www-users.york.ac.uk/~lang22/YCOE/YcoeHome.htm
 import os
 import re
 
-from nltk import compat
+from six import string_types
+
 from nltk.tokenize import RegexpTokenizer
 from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
 from nltk.corpus.reader.tagged import TaggedCorpusReader
@@ -63,7 +64,7 @@ class YCOECorpusReader(CorpusReader):
         """
         if fileids is None:
             return self._documents
-        if isinstance(fileids, compat.string_types):
+        if isinstance(fileids, string_types):
             fileids = [fileids]
         for f in fileids:
             if f not in self._fileids:
@@ -78,7 +79,7 @@ class YCOECorpusReader(CorpusReader):
         """
         if documents is None:
             return self._fileids
-        elif isinstance(documents, compat.string_types):
+        elif isinstance(documents, string_types):
             documents = [documents]
         return sorted(set(['%s.pos' % doc for doc in documents] +
                           ['%s.psd' % doc for doc in documents]))
@@ -91,7 +92,7 @@ class YCOECorpusReader(CorpusReader):
         if documents is None:
             documents = self._documents
         else:
-            if isinstance(documents, compat.string_types):
+            if isinstance(documents, string_types):
                 documents = [documents]
             for document in documents:
                 if document not in self._documents:
@@ -140,8 +141,8 @@ class YCOETaggedCorpusReader(TaggedCorpusReader):
 #: A list of all documents and their titles in ycoe.
 documents = {
     'coadrian.o34': 'Adrian and Ritheus',
-    'coaelhom.o3': '�lfric, Supplemental Homilies',
-    'coaelive.o3': '�lfric\'s Lives of Saints',
+    'coaelhom.o3': 'Ælfric, Supplemental Homilies',
+    'coaelive.o3': 'Ælfric\'s Lives of Saints',
     'coalcuin': 'Alcuin De virtutibus et vitiis',
     'coalex.o23': 'Alexander\'s Letter to Aristotle',
     'coapollo.o3': 'Apollonius of Tyre',
@@ -153,8 +154,8 @@ documents = {
     'cobyrhtf.o3': 'Byrhtferth\'s Manual',
     'cocanedgD': 'Canons of Edgar (D)',
     'cocanedgX': 'Canons of Edgar (X)',
-    'cocathom1.o3': '�lfric\'s Catholic Homilies I',
-    'cocathom2.o3': '�lfric\'s Catholic Homilies II',
+    'cocathom1.o3': 'Ælfric\'s Catholic Homilies I',
+    'cocathom2.o3': 'Ælfric\'s Catholic Homilies II',
     'cochad.o24': 'Saint Chad',
     'cochdrul': 'Chrodegang of Metz, Rule',
     'cochristoph': 'Saint Christopher',
@@ -173,7 +174,7 @@ documents = {
     'codocu4.o24': 'Documents 4 (O2/O4)',
     'coeluc1': 'Honorius of Autun, Elucidarium 1',
     'coeluc2': 'Honorius of Autun, Elucidarium 1',
-    'coepigen.o3': '�lfric\'s Epilogue to Genesis',
+    'coepigen.o3': 'Ælfric\'s Epilogue to Genesis',
     'coeuphr': 'Saint Euphrosyne',
     'coeust': 'Saint Eustace and his companions',
     'coexodusP': 'Exodus (P)',
@@ -188,8 +189,8 @@ documents = {
     'colaece.o2': 'Leechdoms',
     'colaw1cn.o3': 'Laws, Cnut I',
     'colaw2cn.o3': 'Laws, Cnut II',
-    'colaw5atr.o3': 'Laws, �thelred V',
-    'colaw6atr.o3': 'Laws, �thelred VI',
+    'colaw5atr.o3': 'Laws, Æthelred V',
+    'colaw6atr.o3': 'Laws, Æthelred VI',
     'colawaf.o2': 'Laws, Alfred',
     'colawafint.o2': 'Alfred\'s Introduction to Laws',
     'colawger.o34': 'Laws, Gerefa',
@@ -197,14 +198,14 @@ documents = {
     'colawnorthu.o3': 'Northumbra Preosta Lagu',
     'colawwllad.o4': 'Laws, William I, Lad',
     'coleofri.o4': 'Leofric',
-    'colsigef.o3': '�lfric\'s Letter to Sigefyrth',
-    'colsigewB': '�lfric\'s Letter to Sigeweard (B)',
-    'colsigewZ.o34': '�lfric\'s Letter to Sigeweard (Z)',
-    'colwgeat': '�lfric\'s Letter to Wulfgeat',
-    'colwsigeT': '�lfric\'s Letter to Wulfsige (T)',
-    'colwsigeXa.o34': '�lfric\'s Letter to Wulfsige (Xa)',
-    'colwstan1.o3': '�lfric\'s Letter to Wulfstan I',
-    'colwstan2.o3': '�lfric\'s Letter to Wulfstan II',
+    'colsigef.o3': 'Ælfric\'s Letter to Sigefyrth',
+    'colsigewB': 'Ælfric\'s Letter to Sigeweard (B)',
+    'colsigewZ.o34': 'Ælfric\'s Letter to Sigeweard (Z)',
+    'colwgeat': 'Ælfric\'s Letter to Wulfgeat',
+    'colwsigeT': 'Ælfric\'s Letter to Wulfsige (T)',
+    'colwsigeXa.o34': 'Ælfric\'s Letter to Wulfsige (Xa)',
+    'colwstan1.o3': 'Ælfric\'s Letter to Wulfstan I',
+    'colwstan2.o3': 'Ælfric\'s Letter to Wulfstan II',
     'comargaC.o34': 'Saint Margaret (C)',
     'comargaT': 'Saint Margaret (T)',
     'comart1': 'Martyrology, I',
@@ -219,11 +220,11 @@ documents = {
     'conicodE': 'Gospel of Nicodemus (E)',
     'coorosiu.o2': 'Orosius',
     'cootest.o3': 'Heptateuch',
-    'coprefcath1.o3': '�lfric\'s Preface to Catholic Homilies I',
-    'coprefcath2.o3': '�lfric\'s Preface to Catholic Homilies II',
+    'coprefcath1.o3': 'Ælfric\'s Preface to Catholic Homilies I',
+    'coprefcath2.o3': 'Ælfric\'s Preface to Catholic Homilies II',
     'coprefcura.o2': 'Preface to the Cura Pastoralis',
-    'coprefgen.o3': '�lfric\'s Preface to Genesis',
-    'copreflives.o3': '�lfric\'s Preface to Lives of Saints',
+    'coprefgen.o3': 'Ælfric\'s Preface to Genesis',
+    'copreflives.o3': 'Ælfric\'s Preface to Lives of Saints',
     'coprefsolilo': 'Preface to Augustine\'s Soliloquies',
     'coquadru.o23': 'Pseudo-Apuleius, Medicina de quadrupedibus',
     'corood': 'History of the Holy Rood-Tree',
@@ -231,7 +232,7 @@ documents = {
     'cosolilo': 'St. Augustine\'s Soliloquies',
     'cosolsat1.o4': 'Solomon and Saturn I',
     'cosolsat2': 'Solomon and Saturn II',
-    'cotempo.o3': '�lfric\'s De Temporibus Anni',
+    'cotempo.o3': 'Ælfric\'s De Temporibus Anni',
     'coverhom': 'Vercelli Homilies',
     'coverhomE': 'Vercelli Homilies (E)',
     'coverhomL': 'Vercelli Homilies (L)',

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/util.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Corpus Reader Utility Functions
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -38,12 +38,29 @@ class LazyCorpusLoader(object):
     NLTK data package.  Once they've properly installed the data
     package (or modified ``nltk.data.path`` to point to its location),
     they can then use the corpus object without restarting python.
+    
+    :param name: The name of the corpus
+    :type name: str
+    :param reader_cls: The specific CorpusReader class, e.g. PlaintextCorpusReader, WordListCorpusReader
+    :type reader: nltk.corpus.reader.api.CorpusReader
+    :param nltk_data_subdir: The subdirectory where the corpus is stored.
+    :type nltk_data_subdir: str
+    :param *args: Any other non-keywords arguments that `reader_cls` might need.
+    :param *kargs: Any other keywords arguments that `reader_cls` might need.
     """
     def __init__(self, name, reader_cls, *args, **kwargs):
         from nltk.corpus.reader.api import CorpusReader
         assert issubclass(reader_cls, CorpusReader)
         self.__name = self.__name__ = name
         self.__reader_cls = reader_cls
+        # If nltk_data_subdir is set explicitly 
+        if 'nltk_data_subdir' in kwargs:
+            # Use the specified subdirectory path
+            self.subdir = kwargs['nltk_data_subdir']
+            # Pops the `nltk_data_subdir` argument, we don't need it anymore.
+            kwargs.pop('nltk_data_subdir', None)
+        else: # Otherwise use 'nltk_data/corpora'
+            self.subdir = 'corpora'
         self.__args = args
         self.__kwargs = kwargs
 
@@ -52,15 +69,15 @@ class LazyCorpusLoader(object):
         zip_name = re.sub(r'(([^/]*)(/.*)?)', r'\2.zip/\1/', self.__name)
         if TRY_ZIPFILE_FIRST:
             try:
-                root = nltk.data.find('corpora/%s' % zip_name)
+                root = nltk.data.find('{}/{}'.format(self.subdir, zip_name))
             except LookupError as e:
-                try: root = nltk.data.find('corpora/%s' % self.__name)
+                try: root = nltk.data.find('{}/{}'.format(self.subdir, self.__name))
                 except LookupError: raise e
         else:
             try:
-                root = nltk.data.find('corpora/%s' % self.__name)
+                root = nltk.data.find('{}/{}'.format(self.subdir, self.__name))
             except LookupError as e:
-                try: root = nltk.data.find('corpora/%s' % zip_name)
+                try: root = nltk.data.find('{}/{}'.format(self.subdir, zip_name))
                 except LookupError: raise e
 
         # Load the corpus.

{nltk-3.2.1 → nltk-3.3}/nltk/data.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Utility functions
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -32,17 +32,30 @@ to a local file.
 """
 from __future__ import print_function, unicode_literals
 from __future__ import division
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
 
-import sys
+import functools
+import textwrap
 import io
 import os
-import textwrap
 import re
+import sys
 import zipfile
 import codecs
 
 from gzip import GzipFile, READ as GZ_READ, WRITE as GZ_WRITE
 
+try: # Python 3.
+    textwrap_indent = functools.partial(textwrap.indent, prefix='  ')
+except AttributeError: # Python 2; indent() not available for Python2.
+    textwrap_fill = functools.partial(textwrap.fill,
+                                        initial_indent='  ',
+                                        subsequent_indent='  ',
+                                        replace_whitespace=False)
+    def textwrap_indent(text):
+        return '\n'.join(textwrap_fill(line) for line in text.splitlines())
+
 try:
     from zlib import Z_SYNC_FLUSH as FLUSH
 except ImportError:
@@ -53,12 +66,12 @@ try:
 except ImportError:
     import pickle
 
+from six import string_types, text_type
+from six.moves.urllib.request import urlopen, url2pathname
+
 # this import should be more specific:
 import nltk
-
-from nltk.compat import py3_data, add_py3_data
-from nltk.compat import text_type, string_types, BytesIO, urlopen, url2pathname
-
+from nltk.compat import py3_data, add_py3_data, BytesIO
 
 ######################################################################
 # Search Path
@@ -82,6 +95,7 @@ if sys.platform.startswith('win'):
     path += [
         str(r'C:\nltk_data'), str(r'D:\nltk_data'), str(r'E:\nltk_data'),
         os.path.join(sys.prefix, str('nltk_data')),
+        os.path.join(sys.prefix, str('share'), str('nltk_data')),
         os.path.join(sys.prefix, str('lib'), str('nltk_data')),
         os.path.join(
             os.environ.get(str('APPDATA'), str('C:\\')), str('nltk_data'))
@@ -92,7 +106,10 @@ else:
         str('/usr/share/nltk_data'),
         str('/usr/local/share/nltk_data'),
         str('/usr/lib/nltk_data'),
-        str('/usr/local/lib/nltk_data')
+        str('/usr/local/lib/nltk_data'),
+        os.path.join(sys.prefix, str('nltk_data')),
+        os.path.join(sys.prefix, str('share'), str('nltk_data')),
+        os.path.join(sys.prefix, str('lib'), str('nltk_data'))
     ]
 
 
@@ -100,8 +117,8 @@ else:
 # Util Functions
 ######################################################################
 
-def gzip_open_unicode(filename, mode="rb", compresslevel=9,
-                      encoding='utf-8', fileobj=None, errors=None, newline=None):
+def gzip_open_unicode(filename, mode="rb", compresslevel=9, encoding='utf-8',
+                      fileobj=None, errors=None, newline=None):
     if fileobj is None:
         fileobj = GzipFile(filename, mode, compresslevel, fileobj)
     return io.TextIOWrapper(fileobj, encoding, errors, newline)
@@ -241,6 +258,7 @@ def normalize_resource_name(resource_name, allow_relative=True, relative_path=No
 # Path Pointers
 ######################################################################
 
+@add_metaclass(ABCMeta)
 class PathPointer(object):
     """
     An abstract base class for 'path pointers,' used by NLTK's data
@@ -251,6 +269,7 @@ class PathPointer(object):
     by reading that zipfile.
     """
 
+    @abstractmethod
     def open(self, encoding=None):
         """
         Return a seekable read-only stream that can be used to read
@@ -259,8 +278,8 @@ class PathPointer(object):
         :raise IOError: If the path specified by this pointer does
             not contain a readable file.
         """
-        raise NotImplementedError('abstract base class')
 
+    @abstractmethod
     def file_size(self):
         """
         Return the size of the file pointed to by this path pointer,
@@ -269,8 +288,8 @@ class PathPointer(object):
         :raise IOError: If the path specified by this pointer does
             not contain a readable file.
         """
-        raise NotImplementedError('abstract base class')
 
+    @abstractmethod
     def join(self, fileid):
         """
         Return a new path pointer formed by starting at the path
@@ -279,7 +298,6 @@ class PathPointer(object):
         should be separated by forward slashes, regardless of
         the underlying file system's path seperator character.
         """
-        raise NotImplementedError('abstract base class')
 
 
 class FileSystemPathPointer(PathPointer, text_type):
@@ -369,11 +387,6 @@ class BufferedGzipFile(GzipFile):
         """
         GzipFile.__init__(self, filename, mode, compresslevel, fileobj)
         self._size = kwargs.get('size', self.SIZE)
-        # Note: In > Python3.5, GzipFile is already using a 
-        # buffered reader in the backend which has a variable self._buffer
-        # See https://github.com/nltk/nltk/issues/1308
-        if sys.version.startswith('3.5'):
-            sys.stderr.write("Use the native Python gzip.GzipFile instead.")
         self._nltk_buffer = BytesIO()
         # cStringIO does not support len.
         self._len = 0
@@ -445,7 +458,13 @@ class GzipFileSystemPathPointer(FileSystemPathPointer):
     """
 
     def open(self, encoding=None):
-        stream = BufferedGzipFile(self._path, 'rb')
+        # Note: In >= Python3.5, GzipFile is already using a
+        # buffered reader in the backend which has a variable self._buffer
+        # See https://github.com/nltk/nltk/issues/1308
+        if sys.version.startswith('2.7') or sys.version.startswith('3.4'):
+            stream = BufferedGzipFile(self._path, 'rb')
+        else:
+            stream = GzipFile(self._path, 'rb')
         if encoding:
             stream = SeekableUnicodeStreamReader(stream, encoding)
         return stream
@@ -472,7 +491,7 @@ class ZipFilePathPointer(PathPointer):
         entry = normalize_resource_name(entry, True, '/').lstrip('/')
 
         # Check that the entry exists:
-        if entry:
+        if entry != '.':
             try:
                 zipfile.getinfo(entry)
             except Exception:
@@ -510,7 +529,13 @@ class ZipFilePathPointer(PathPointer):
         data = self._zipfile.read(self._entry)
         stream = BytesIO(data)
         if self._entry.endswith('.gz'):
-            stream = BufferedGzipFile(self._entry, fileobj=stream)
+            # Note: In >= Python3.5, GzipFile is already using a
+            # buffered reader in the backend which has a variable self._buffer
+            # See https://github.com/nltk/nltk/issues/1308
+            if sys.version.startswith('2.7') or sys.version.startswith('3.4'):
+                stream = BufferedGzipFile(self._entry, fileobj=stream)
+            else:
+                stream = GzipFile(self._entry, fileobj=stream)
         elif encoding is not None:
             stream = SeekableUnicodeStreamReader(stream, encoding)
         return stream
@@ -527,7 +552,9 @@ class ZipFilePathPointer(PathPointer):
             self._zipfile.filename, self._entry)
 
     def __str__(self):
-        return os.path.normpath(os.path.join(self._zipfile.filename, self._entry))
+        return os.path.normpath(os.path.join(self._zipfile.filename,
+                                             self._entry))
+
 
 ######################################################################
 # Access Functions
@@ -629,15 +656,22 @@ def find(resource_name, paths=None):
             except LookupError:
                 pass
 
+    # Identify the package (i.e. the .zip file) to download.
+    resource_zipname = resource_name.split('/')[1]
+    if resource_zipname.endswith('.zip'):
+        resource_zipname = resource_zipname.rpartition('.')[0]
     # Display a friendly error message if the resource wasn't found:
-    msg = textwrap.fill(
-        'Resource %r not found.  Please use the NLTK Downloader to '
-        'obtain the resource:  >>> nltk.download()' %
-        (resource_name,), initial_indent='  ', subsequent_indent='  ',
-        width=66)
+    msg = str("Resource \33[93m{resource}\033[0m not found.\n"
+              "Please use the NLTK Downloader to obtain the resource:\n\n"
+              "\33[31m" # To display red text in terminal.
+              ">>> import nltk\n"
+              ">>> nltk.download(\'{resource}\')\n"
+              "\033[0m").format(resource=resource_zipname)
+    msg = textwrap_indent(msg)
+
     msg += '\n  Searched in:' + ''.join('\n    - %r' % d for d in paths)
     sep = '*' * 70
-    resource_not_found = '\n%s\n%s\n%s' % (sep, msg, sep)
+    resource_not_found = '\n%s\n%s\n%s\n' % (sep, msg, sep)
     raise LookupError(resource_not_found)
 
 
@@ -678,6 +712,7 @@ def retrieve(resource_url, filename=None, verbose=True):
 
     infile.close()
 
+
 #: A dictionary describing the formats that are supported by NLTK's
 #: load() method.  Keys are format names, and values are format
 #: descriptions.
@@ -979,7 +1014,7 @@ class OpenOnDemandZipFile(zipfile.ZipFile):
         zipfile.ZipFile.__init__(self, filename)
         assert self.filename == filename
         self.close()
-        # After closing a ZipFile object, the _fileRefCnt needs to be cleared 
+        # After closing a ZipFile object, the _fileRefCnt needs to be cleared
         # for Python2and3 compatible code.
         self._fileRefCnt = 0
 
@@ -1444,6 +1479,7 @@ class SeekableUnicodeStreamReader(object):
 
         return None
 
+
 __all__ = ['path', 'PathPointer', 'FileSystemPathPointer', 'BufferedGzipFile',
            'GzipFileSystemPathPointer', 'GzipFileSystemPathPointer',
            'find', 'retrieve', 'FORMATS', 'AUTO_FORMATS', 'load',

{nltk-3.2.1 → nltk-3.3}/nltk/decorators.py

@@ -56,7 +56,11 @@ def getinfo(func):
     'self, x, y, *args, **kw'
     """
     assert inspect.ismethod(func) or inspect.isfunction(func)
-    regargs, varargs, varkwargs, defaults = inspect.getargspec(func)
+    if sys.version_info[0] >= 3:
+        argspec = inspect.getfullargspec(func)
+    else:
+        argspec = inspect.getargspec(func)
+    regargs, varargs, varkwargs, defaults = argspec[:4]
     argnames = list(regargs)
     if varargs:
         argnames.append(varargs)