nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/toolbox.py

@@ -1,68 +1,68 @@
-# Natural Language Toolkit: Toolbox Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Greg Aumann <greg_aumann@sil.org>
-#         Stuart Robinson <Stuart.Robinson@mpi.nl>
-#         Steven Bird <stevenbird1@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Module for reading, writing and manipulating
-Toolbox databases and settings fileids.
-"""
-
-import os
-import re
-import codecs
-
-from nltk import compat
-from nltk.toolbox import ToolboxData
-
-from nltk.corpus.reader.util import *
-from nltk.corpus.reader.api import *
-
-class ToolboxCorpusReader(CorpusReader):
-    def xml(self, fileids, key=None):
-        return concat([ToolboxData(path, enc).parse(key=key)
-                       for (path, enc) in self.abspaths(fileids, True)])
-
-    def fields(self, fileids, strip=True, unwrap=True, encoding='utf8',
-               errors='strict', unicode_fields=None):
-        return concat([list(ToolboxData(fileid,enc).fields(
-                             strip, unwrap, encoding, errors, unicode_fields))
-                       for (fileid, enc)
-                       in self.abspaths(fileids, include_encoding=True)])
-
-    # should probably be done lazily:
-    def entries(self, fileids, **kwargs):
-        if 'key' in kwargs:
-            key = kwargs['key']
-            del kwargs['key']
-        else:
-            key = 'lx'  # the default key in MDF
-        entries = []
-        for marker, contents in self.fields(fileids, **kwargs):
-            if marker == key:
-                entries.append((contents, []))
-            else:
-                try:
-                    entries[-1][-1].append((marker, contents))
-                except IndexError:
-                    pass
-        return entries
-
-    def words(self, fileids, key='lx'):
-        return [contents for marker, contents in self.fields(fileids) if marker == key]
-
-    def raw(self, fileids):
-        if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
-        return concat([self.open(f).read() for f in fileids])
-
-
-def demo():
-    pass
-
-if __name__ == '__main__':
-    demo()
+# Natural Language Toolkit: Toolbox Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Greg Aumann <greg_aumann@sil.org>
+#         Stuart Robinson <Stuart.Robinson@mpi.nl>
+#         Steven Bird <stevenbird1@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Module for reading, writing and manipulating
+Toolbox databases and settings fileids.
+"""
+
+import os
+import re
+import codecs
+
+from six import string_types
+
+from nltk.toolbox import ToolboxData
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class ToolboxCorpusReader(CorpusReader):
+    def xml(self, fileids, key=None):
+        return concat([ToolboxData(path, enc).parse(key=key)
+                       for (path, enc) in self.abspaths(fileids, True)])
+
+    def fields(self, fileids, strip=True, unwrap=True, encoding='utf8',
+               errors='strict', unicode_fields=None):
+        return concat([list(ToolboxData(fileid,enc).fields(
+                             strip, unwrap, encoding, errors, unicode_fields))
+                       for (fileid, enc)
+                       in self.abspaths(fileids, include_encoding=True)])
+
+    # should probably be done lazily:
+    def entries(self, fileids, **kwargs):
+        if 'key' in kwargs:
+            key = kwargs['key']
+            del kwargs['key']
+        else:
+            key = 'lx'  # the default key in MDF
+        entries = []
+        for marker, contents in self.fields(fileids, **kwargs):
+            if marker == key:
+                entries.append((contents, []))
+            else:
+                try:
+                    entries[-1][-1].append((marker, contents))
+                except IndexError:
+                    pass
+        return entries
+
+    def words(self, fileids, key='lx'):
+        return [contents for marker, contents in self.fields(fileids) if marker == key]
+
+    def raw(self, fileids):
+        if fileids is None: fileids = self._fileids
+        elif isinstance(fileids, string_types): fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])
+
+
+def demo():
+    pass
+
+if __name__ == '__main__':
+    demo()

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/twitter.py

@@ -1,157 +1,155 @@
-# Natural Language Toolkit: Twitter Corpus Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Ewan Klein <ewan@inf.ed.ac.uk>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-A reader for corpora that consist of Tweets. It is assumed that the Tweets
-have been serialised into line-delimited JSON.
-"""
-
-import json
-import os
-
-from nltk import compat
-from nltk.tokenize import TweetTokenizer
-
-from nltk.corpus.reader.util import StreamBackedCorpusView, concat, ZipFilePathPointer
-from nltk.corpus.reader.api import CorpusReader
-
-
-class TwitterCorpusReader(CorpusReader):
-    """
-    Reader for corpora that consist of Tweets represented as a list of line-delimited JSON.
-
-    Individual Tweets can be tokenized using the default tokenizer, or by a
-    custom tokenizer specified as a parameter to the constructor.
-
-    Construct a new Tweet corpus reader for a set of documents
-    located at the given root directory.
-
-    If you made your own tweet collection in a directory called
-    `twitter-files`, then you can initialise the reader as::
-
-        from nltk.corpus import TwitterCorpusReader
-        reader = TwitterCorpusReader(root='/path/to/twitter-files', '.*\.json')
-
-    However, the recommended approach is to set the relevant directory as the
-    value of the environmental variable `TWITTER`, and then invoke the reader
-    as follows::
-
-       root = os.environ['TWITTER']
-       reader = TwitterCorpusReader(root, '.*\.json')
-
-    If you want to work directly with the raw Tweets, the `json` library can
-    be used::
-
-       import json
-       for tweet in reader.docs():
-           print(json.dumps(tweet, indent=1, sort_keys=True))
-
-    """
-
-    CorpusView = StreamBackedCorpusView
-    """
-    The corpus view class used by this reader.
-    """
-
-    def __init__(self, root, fileids=None,
-                 word_tokenizer=TweetTokenizer(),
-                 encoding='utf8'):
-        """
-
-        :param root: The root directory for this corpus.
-
-        :param fileids: A list or regexp specifying the fileids in this corpus.
-
-        :param word_tokenizer: Tokenizer for breaking the text of Tweets into
-        smaller units, including but not limited to words.
-
-        """
-        CorpusReader.__init__(self, root, fileids, encoding)
-
-        for path in self.abspaths(self._fileids):
-            if isinstance(path, ZipFilePathPointer):
-                pass
-            elif os.path.getsize(path) == 0:
-                raise ValueError("File {} is empty".format(path))
-        """Check that all user-created corpus files are non-empty."""
-
-        self._word_tokenizer = word_tokenizer
-
-
-
-    def docs(self, fileids=None):
-        """
-        Returns the full Tweet objects, as specified by `Twitter
-        documentation on Tweets
-        <https://dev.twitter.com/docs/platform-objects/tweets>`_
-
-        :return: the given file(s) as a list of dictionaries deserialised
-        from JSON.
-        :rtype: list(dict)
-        """
-        return concat([self.CorpusView(path, self._read_tweets, encoding=enc)
-                       for (path, enc, fileid) in self.abspaths(fileids, True, True)])
-
-
-    def strings(self, fileids=None):
-        """
-        Returns only the text content of Tweets in the file(s)
-
-        :return: the given file(s) as a list of Tweets.
-        :rtype: list(str)
-        """
-        fulltweets = self.docs(fileids)
-        tweets = []
-        for jsono in fulltweets:
-            try:
-                text = jsono['text']
-                if isinstance(text, bytes):
-                    text = text.decode(self.encoding)
-                tweets.append(text)
-            except KeyError:
-                pass
-        return tweets
-
-
-    def tokenized(self, fileids=None):
-        """
-        :return: the given file(s) as a list of the text content of Tweets as
-        as a list of words, screenanames, hashtags, URLs and punctuation symbols.
-
-        :rtype: list(list(str))
-        """
-        tweets = self.strings(fileids)
-        tokenizer = self._word_tokenizer
-        return [tokenizer.tokenize(t) for t in tweets]
-
-
-    def raw(self, fileids=None):
-        """
-        Return the corpora in their raw form.
-        """
-        if fileids is None:
-            fileids = self._fileids
-        elif isinstance(fileids, compat.string_types):
-            fileids = [fileids]
-        return concat([self.open(f).read() for f in fileids])
-
-
-    def _read_tweets(self, stream):
-        """
-        Assumes that each line in ``stream`` is a JSON-serialised object.
-        """
-        tweets = []
-        for i in range(10):
-            line = stream.readline()
-            if not line:
-                return tweets
-            tweet = json.loads(line)
-            tweets.append(tweet)
-        return tweets
-
-
-
+# Natural Language Toolkit: Twitter Corpus Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Ewan Klein <ewan@inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A reader for corpora that consist of Tweets. It is assumed that the Tweets
+have been serialised into line-delimited JSON.
+"""
+
+import json
+import os
+
+from six import string_types
+
+from nltk.tokenize import TweetTokenizer
+
+from nltk.corpus.reader.util import StreamBackedCorpusView, concat, ZipFilePathPointer
+from nltk.corpus.reader.api import CorpusReader
+
+
+class TwitterCorpusReader(CorpusReader):
+    """
+    Reader for corpora that consist of Tweets represented as a list of line-delimited JSON.
+
+    Individual Tweets can be tokenized using the default tokenizer, or by a
+    custom tokenizer specified as a parameter to the constructor.
+
+    Construct a new Tweet corpus reader for a set of documents
+    located at the given root directory.
+
+    If you made your own tweet collection in a directory called
+    `twitter-files`, then you can initialise the reader as::
+
+        from nltk.corpus import TwitterCorpusReader
+        reader = TwitterCorpusReader(root='/path/to/twitter-files', '.*\.json')
+
+    However, the recommended approach is to set the relevant directory as the
+    value of the environmental variable `TWITTER`, and then invoke the reader
+    as follows::
+
+       root = os.environ['TWITTER']
+       reader = TwitterCorpusReader(root, '.*\.json')
+
+    If you want to work directly with the raw Tweets, the `json` library can
+    be used::
+
+       import json
+       for tweet in reader.docs():
+           print(json.dumps(tweet, indent=1, sort_keys=True))
+
+    """
+
+    CorpusView = StreamBackedCorpusView
+    """
+    The corpus view class used by this reader.
+    """
+
+    def __init__(self, root, fileids=None,
+                 word_tokenizer=TweetTokenizer(),
+                 encoding='utf8'):
+        """
+
+        :param root: The root directory for this corpus.
+
+        :param fileids: A list or regexp specifying the fileids in this corpus.
+
+        :param word_tokenizer: Tokenizer for breaking the text of Tweets into
+        smaller units, including but not limited to words.
+
+        """
+        CorpusReader.__init__(self, root, fileids, encoding)
+
+        for path in self.abspaths(self._fileids):
+            if isinstance(path, ZipFilePathPointer):
+                pass
+            elif os.path.getsize(path) == 0:
+                raise ValueError("File {} is empty".format(path))
+        """Check that all user-created corpus files are non-empty."""
+
+        self._word_tokenizer = word_tokenizer
+
+
+
+    def docs(self, fileids=None):
+        """
+        Returns the full Tweet objects, as specified by `Twitter
+        documentation on Tweets
+        <https://dev.twitter.com/docs/platform-objects/tweets>`_
+
+        :return: the given file(s) as a list of dictionaries deserialised
+        from JSON.
+        :rtype: list(dict)
+        """
+        return concat([self.CorpusView(path, self._read_tweets, encoding=enc)
+                       for (path, enc, fileid) in self.abspaths(fileids, True, True)])
+
+
+    def strings(self, fileids=None):
+        """
+        Returns only the text content of Tweets in the file(s)
+
+        :return: the given file(s) as a list of Tweets.
+        :rtype: list(str)
+        """
+        fulltweets = self.docs(fileids)
+        tweets = []
+        for jsono in fulltweets:
+            try:
+                text = jsono['text']
+                if isinstance(text, bytes):
+                    text = text.decode(self.encoding)
+                tweets.append(text)
+            except KeyError:
+                pass
+        return tweets
+
+
+    def tokenized(self, fileids=None):
+        """
+        :return: the given file(s) as a list of the text content of Tweets as
+        as a list of words, screenanames, hashtags, URLs and punctuation symbols.
+
+        :rtype: list(list(str))
+        """
+        tweets = self.strings(fileids)
+        tokenizer = self._word_tokenizer
+        return [tokenizer.tokenize(t) for t in tweets]
+
+
+    def raw(self, fileids=None):
+        """
+        Return the corpora in their raw form.
+        """
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])
+
+
+    def _read_tweets(self, stream):
+        """
+        Assumes that each line in ``stream`` is a JSON-serialised object.
+        """
+        tweets = []
+        for i in range(10):
+            line = stream.readline()
+            if not line:
+                return tweets
+            tweet = json.loads(line)
+            tweets.append(tweet)
+        return tweets

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/udhr.py

@@ -1,80 +1,80 @@
-# -*- coding: utf-8 -*-
-"""
-UDHR corpus reader. It mostly deals with encodings.
-"""
-from __future__ import absolute_import, unicode_literals
-
-from nltk.corpus.reader.util import find_corpus_fileids
-from nltk.corpus.reader.plaintext import PlaintextCorpusReader
-
-class UdhrCorpusReader(PlaintextCorpusReader):
-
-    ENCODINGS = [
-        ('.*-Latin1$', 'latin-1'),
-        ('.*-Hebrew$', 'hebrew'),
-        ('.*-Arabic$', 'cp1256'),
-        ('Czech_Cesky-UTF8', 'cp1250'), # yeah
-        ('.*-Cyrillic$', 'cyrillic'),
-        ('.*-SJIS$', 'SJIS'),
-        ('.*-GB2312$', 'GB2312'),
-        ('.*-Latin2$', 'ISO-8859-2'),
-        ('.*-Greek$', 'greek'),
-        ('.*-UTF8$', 'utf-8'),
-
-        ('Hungarian_Magyar-Unicode', 'utf-16-le'),
-        ('Amahuaca', 'latin1'),
-        ('Turkish_Turkce-Turkish', 'latin5'),
-        ('Lithuanian_Lietuviskai-Baltic', 'latin4'),
-        ('Japanese_Nihongo-EUC', 'EUC-JP'),
-        ('Japanese_Nihongo-JIS', 'iso2022_jp'),
-        ('Chinese_Mandarin-HZ', 'hz'),
-        ('Abkhaz\-Cyrillic\+Abkh', 'cp1251'),
-    ]
-
-    SKIP = set([
-        # The following files are not fully decodable because they
-        # were truncated at wrong bytes:
-        'Burmese_Myanmar-UTF8',
-        'Japanese_Nihongo-JIS',
-        'Chinese_Mandarin-HZ',
-        'Chinese_Mandarin-UTF8',
-        'Gujarati-UTF8',
-        'Hungarian_Magyar-Unicode',
-        'Lao-UTF8',
-        'Magahi-UTF8',
-        'Marathi-UTF8',
-        'Tamil-UTF8',
-
-        # Unfortunately, encodings required for reading
-        # the following files are not supported by Python:
-        'Vietnamese-VPS',
-        'Vietnamese-VIQR',
-        'Vietnamese-TCVN',
-        'Magahi-Agra',
-        'Bhojpuri-Agra',
-        'Esperanto-T61', # latin3 raises an exception
-
-        # The following files are encoded for specific fonts:
-        'Burmese_Myanmar-WinResearcher',
-        'Armenian-DallakHelv',
-        'Tigrinya_Tigrigna-VG2Main',
-        'Amharic-Afenegus6..60375', # ?
-        'Navaho_Dine-Navajo-Navaho-font',
-
-        # What are these?
-        'Azeri_Azerbaijani_Cyrillic-Az.Times.Cyr.Normal0117',
-        'Azeri_Azerbaijani_Latin-Az.Times.Lat0117',
-
-        # The following files are unintended:
-        'Czech-Latin2-err',
-        'Russian_Russky-UTF8~',
-    ])
-
-
-    def __init__(self, root='udhr'):
-        fileids = find_corpus_fileids(root, r'(?!README|\.).*')
-        super(UdhrCorpusReader, self).__init__(
-            root,
-            [fileid for fileid in fileids if fileid not in self.SKIP],
-            encoding=self.ENCODINGS
-        )
+# -*- coding: utf-8 -*-
+"""
+UDHR corpus reader. It mostly deals with encodings.
+"""
+from __future__ import absolute_import, unicode_literals
+
+from nltk.corpus.reader.util import find_corpus_fileids
+from nltk.corpus.reader.plaintext import PlaintextCorpusReader
+
+class UdhrCorpusReader(PlaintextCorpusReader):
+
+    ENCODINGS = [
+        ('.*-Latin1$', 'latin-1'),
+        ('.*-Hebrew$', 'hebrew'),
+        ('.*-Arabic$', 'cp1256'),
+        ('Czech_Cesky-UTF8', 'cp1250'), # yeah
+        ('.*-Cyrillic$', 'cyrillic'),
+        ('.*-SJIS$', 'SJIS'),
+        ('.*-GB2312$', 'GB2312'),
+        ('.*-Latin2$', 'ISO-8859-2'),
+        ('.*-Greek$', 'greek'),
+        ('.*-UTF8$', 'utf-8'),
+
+        ('Hungarian_Magyar-Unicode', 'utf-16-le'),
+        ('Amahuaca', 'latin1'),
+        ('Turkish_Turkce-Turkish', 'latin5'),
+        ('Lithuanian_Lietuviskai-Baltic', 'latin4'),
+        ('Japanese_Nihongo-EUC', 'EUC-JP'),
+        ('Japanese_Nihongo-JIS', 'iso2022_jp'),
+        ('Chinese_Mandarin-HZ', 'hz'),
+        ('Abkhaz\-Cyrillic\+Abkh', 'cp1251'),
+    ]
+
+    SKIP = set([
+        # The following files are not fully decodable because they
+        # were truncated at wrong bytes:
+        'Burmese_Myanmar-UTF8',
+        'Japanese_Nihongo-JIS',
+        'Chinese_Mandarin-HZ',
+        'Chinese_Mandarin-UTF8',
+        'Gujarati-UTF8',
+        'Hungarian_Magyar-Unicode',
+        'Lao-UTF8',
+        'Magahi-UTF8',
+        'Marathi-UTF8',
+        'Tamil-UTF8',
+
+        # Unfortunately, encodings required for reading
+        # the following files are not supported by Python:
+        'Vietnamese-VPS',
+        'Vietnamese-VIQR',
+        'Vietnamese-TCVN',
+        'Magahi-Agra',
+        'Bhojpuri-Agra',
+        'Esperanto-T61', # latin3 raises an exception
+
+        # The following files are encoded for specific fonts:
+        'Burmese_Myanmar-WinResearcher',
+        'Armenian-DallakHelv',
+        'Tigrinya_Tigrigna-VG2Main',
+        'Amharic-Afenegus6..60375', # ?
+        'Navaho_Dine-Navajo-Navaho-font',
+
+        # What are these?
+        'Azeri_Azerbaijani_Cyrillic-Az.Times.Cyr.Normal0117',
+        'Azeri_Azerbaijani_Latin-Az.Times.Lat0117',
+
+        # The following files are unintended:
+        'Czech-Latin2-err',
+        'Russian_Russky-UTF8~',
+    ])
+
+
+    def __init__(self, root='udhr'):
+        fileids = find_corpus_fileids(root, r'(?!README|\.).*')
+        super(UdhrCorpusReader, self).__init__(
+            root,
+            [fileid for fileid in fileids if fileid not in self.SKIP],
+            encoding=self.ENCODINGS
+        )