nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/chunked.py

@@ -1,212 +1,212 @@
-# Natural Language Toolkit: Chunked Corpus Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Steven Bird <stevenbird1@gmail.com>
-#         Edward Loper <edloper@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-A reader for corpora that contain chunked (and optionally tagged)
-documents.
-"""
-
-import os.path, codecs
-
-import nltk
-from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
-from nltk import compat
-from nltk.tree import Tree
-from nltk.tokenize import *
-from nltk.chunk import tagstr2tree
-from nltk.corpus.reader.util import *
-from nltk.corpus.reader.api import *
-
-class ChunkedCorpusReader(CorpusReader):
-    """
-    Reader for chunked (and optionally tagged) corpora.  Paragraphs
-    are split using a block reader.  They are then tokenized into
-    sentences using a sentence tokenizer.  Finally, these sentences
-    are parsed into chunk trees using a string-to-chunktree conversion
-    function.  Each of these steps can be performed using a default
-    function or a custom function.  By default, paragraphs are split
-    on blank lines; sentences are listed one per line; and sentences
-    are parsed into chunk trees using ``nltk.chunk.tagstr2tree``.
-    """
-    def __init__(self, root, fileids, extension='',
-                 str2chunktree=tagstr2tree,
-                 sent_tokenizer=RegexpTokenizer('\n', gaps=True),
-                 para_block_reader=read_blankline_block,
-                 encoding='utf8', tagset=None):
-        """
-        :param root: The root directory for this corpus.
-        :param fileids: A list or regexp specifying the fileids in this corpus.
-        """
-        CorpusReader.__init__(self, root, fileids, encoding)
-        self._cv_args = (str2chunktree, sent_tokenizer, para_block_reader, tagset)
-        """Arguments for corpus views generated by this corpus: a tuple
-        (str2chunktree, sent_tokenizer, para_block_tokenizer)"""
-
-    def raw(self, fileids=None):
-        """
-        :return: the given file(s) as a single string.
-        :rtype: str
-        """
-        if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
-        return concat([self.open(f).read() for f in fileids])
-
-    def words(self, fileids=None):
-        """
-        :return: the given file(s) as a list of words
-            and punctuation symbols.
-        :rtype: list(str)
-        """
-        return concat([ChunkedCorpusView(f, enc, 0, 0, 0, 0, *self._cv_args)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def sents(self, fileids=None):
-        """
-        :return: the given file(s) as a list of
-            sentences or utterances, each encoded as a list of word
-            strings.
-        :rtype: list(list(str))
-        """
-        return concat([ChunkedCorpusView(f, enc, 0, 1, 0, 0, *self._cv_args)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def paras(self, fileids=None):
-        """
-        :return: the given file(s) as a list of
-            paragraphs, each encoded as a list of sentences, which are
-            in turn encoded as lists of word strings.
-        :rtype: list(list(list(str)))
-        """
-        return concat([ChunkedCorpusView(f, enc, 0, 1, 1, 0, *self._cv_args)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def tagged_words(self, fileids=None, tagset=None):
-        """
-        :return: the given file(s) as a list of tagged
-            words and punctuation symbols, encoded as tuples
-            ``(word,tag)``.
-        :rtype: list(tuple(str,str))
-        """
-        return concat([ChunkedCorpusView(f, enc, 1, 0, 0, 0, *self._cv_args, target_tagset=tagset)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def tagged_sents(self, fileids=None, tagset=None):
-        """
-        :return: the given file(s) as a list of
-            sentences, each encoded as a list of ``(word,tag)`` tuples.
-
-        :rtype: list(list(tuple(str,str)))
-        """
-        return concat([ChunkedCorpusView(f, enc, 1, 1, 0, 0, *self._cv_args, target_tagset=tagset)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def tagged_paras(self, fileids=None, tagset=None):
-        """
-        :return: the given file(s) as a list of
-            paragraphs, each encoded as a list of sentences, which are
-            in turn encoded as lists of ``(word,tag)`` tuples.
-        :rtype: list(list(list(tuple(str,str))))
-        """
-        return concat([ChunkedCorpusView(f, enc, 1, 1, 1, 0, *self._cv_args, target_tagset=tagset)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def chunked_words(self, fileids=None, tagset=None):
-        """
-        :return: the given file(s) as a list of tagged
-            words and chunks.  Words are encoded as ``(word, tag)``
-            tuples (if the corpus has tags) or word strings (if the
-            corpus has no tags).  Chunks are encoded as depth-one
-            trees over ``(word,tag)`` tuples or word strings.
-        :rtype: list(tuple(str,str) and Tree)
-        """
-        return concat([ChunkedCorpusView(f, enc, 1, 0, 0, 1, *self._cv_args, target_tagset=tagset)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def chunked_sents(self, fileids=None, tagset=None):
-        """
-        :return: the given file(s) as a list of
-            sentences, each encoded as a shallow Tree.  The leaves
-            of these trees are encoded as ``(word, tag)`` tuples (if
-            the corpus has tags) or word strings (if the corpus has no
-            tags).
-        :rtype: list(Tree)
-        """
-        return concat([ChunkedCorpusView(f, enc, 1, 1, 0, 1, *self._cv_args, target_tagset=tagset)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def chunked_paras(self, fileids=None, tagset=None):
-        """
-        :return: the given file(s) as a list of
-            paragraphs, each encoded as a list of sentences, which are
-            in turn encoded as a shallow Tree.  The leaves of these
-            trees are encoded as ``(word, tag)`` tuples (if the corpus
-            has tags) or word strings (if the corpus has no tags).
-        :rtype: list(list(Tree))
-        """
-        return concat([ChunkedCorpusView(f, enc, 1, 1, 1, 1, *self._cv_args, target_tagset=tagset)
-                       for (f, enc) in self.abspaths(fileids, True)])
-
-    def _read_block(self, stream):
-        return [tagstr2tree(t) for t in read_blankline_block(stream)]
-
-class ChunkedCorpusView(StreamBackedCorpusView):
-    def __init__(self, fileid, encoding, tagged, group_by_sent,
-                 group_by_para, chunked, str2chunktree, sent_tokenizer,
-                 para_block_reader, source_tagset=None, target_tagset=None):
-        StreamBackedCorpusView.__init__(self, fileid, encoding=encoding)
-        self._tagged = tagged
-        self._group_by_sent = group_by_sent
-        self._group_by_para = group_by_para
-        self._chunked = chunked
-        self._str2chunktree = str2chunktree
-        self._sent_tokenizer = sent_tokenizer
-        self._para_block_reader = para_block_reader
-        self._source_tagset = source_tagset
-        self._target_tagset = target_tagset
-
-    def read_block(self, stream):
-        block = []
-        for para_str in self._para_block_reader(stream):
-            para = []
-            for sent_str in self._sent_tokenizer.tokenize(para_str):
-                sent = self._str2chunktree(sent_str, source_tagset=self._source_tagset,
-                                           target_tagset=self._target_tagset)
-
-                # If requested, throw away the tags.
-                if not self._tagged:
-                    sent = self._untag(sent)
-
-                # If requested, throw away the chunks.
-                if not self._chunked:
-                    sent = sent.leaves()
-
-                # Add the sentence to `para`.
-                if self._group_by_sent:
-                    para.append(sent)
-                else:
-                    para.extend(sent)
-
-            # Add the paragraph to `block`.
-            if self._group_by_para:
-                block.append(para)
-            else:
-                block.extend(para)
-
-        # Return the block
-        return block
-
-    def _untag(self, tree):
-        for i, child in enumerate(tree):
-            if isinstance(child, Tree):
-                self._untag(child)
-            elif isinstance(child, tuple):
-                tree[i] = child[0]
-            else:
-                raise ValueError('expected child to be Tree or tuple')
-        return tree
-
+# Natural Language Toolkit: Chunked Corpus Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+#         Edward Loper <edloper@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A reader for corpora that contain chunked (and optionally tagged)
+documents.
+"""
+
+import os.path, codecs
+
+from six import string_types
+
+import nltk
+from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
+from nltk.tree import Tree
+from nltk.tokenize import *
+from nltk.chunk import tagstr2tree
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class ChunkedCorpusReader(CorpusReader):
+    """
+    Reader for chunked (and optionally tagged) corpora.  Paragraphs
+    are split using a block reader.  They are then tokenized into
+    sentences using a sentence tokenizer.  Finally, these sentences
+    are parsed into chunk trees using a string-to-chunktree conversion
+    function.  Each of these steps can be performed using a default
+    function or a custom function.  By default, paragraphs are split
+    on blank lines; sentences are listed one per line; and sentences
+    are parsed into chunk trees using ``nltk.chunk.tagstr2tree``.
+    """
+    def __init__(self, root, fileids, extension='',
+                 str2chunktree=tagstr2tree,
+                 sent_tokenizer=RegexpTokenizer('\n', gaps=True),
+                 para_block_reader=read_blankline_block,
+                 encoding='utf8', tagset=None):
+        """
+        :param root: The root directory for this corpus.
+        :param fileids: A list or regexp specifying the fileids in this corpus.
+        """
+        CorpusReader.__init__(self, root, fileids, encoding)
+        self._cv_args = (str2chunktree, sent_tokenizer, para_block_reader, tagset)
+        """Arguments for corpus views generated by this corpus: a tuple
+        (str2chunktree, sent_tokenizer, para_block_tokenizer)"""
+
+    def raw(self, fileids=None):
+        """
+        :return: the given file(s) as a single string.
+        :rtype: str
+        """
+        if fileids is None: fileids = self._fileids
+        elif isinstance(fileids, string_types): fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])
+
+    def words(self, fileids=None):
+        """
+        :return: the given file(s) as a list of words
+            and punctuation symbols.
+        :rtype: list(str)
+        """
+        return concat([ChunkedCorpusView(f, enc, 0, 0, 0, 0, *self._cv_args)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def sents(self, fileids=None):
+        """
+        :return: the given file(s) as a list of
+            sentences or utterances, each encoded as a list of word
+            strings.
+        :rtype: list(list(str))
+        """
+        return concat([ChunkedCorpusView(f, enc, 0, 1, 0, 0, *self._cv_args)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def paras(self, fileids=None):
+        """
+        :return: the given file(s) as a list of
+            paragraphs, each encoded as a list of sentences, which are
+            in turn encoded as lists of word strings.
+        :rtype: list(list(list(str)))
+        """
+        return concat([ChunkedCorpusView(f, enc, 0, 1, 1, 0, *self._cv_args)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def tagged_words(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of tagged
+            words and punctuation symbols, encoded as tuples
+            ``(word,tag)``.
+        :rtype: list(tuple(str,str))
+        """
+        return concat([ChunkedCorpusView(f, enc, 1, 0, 0, 0, *self._cv_args, target_tagset=tagset)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def tagged_sents(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of
+            sentences, each encoded as a list of ``(word,tag)`` tuples.
+
+        :rtype: list(list(tuple(str,str)))
+        """
+        return concat([ChunkedCorpusView(f, enc, 1, 1, 0, 0, *self._cv_args, target_tagset=tagset)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def tagged_paras(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of
+            paragraphs, each encoded as a list of sentences, which are
+            in turn encoded as lists of ``(word,tag)`` tuples.
+        :rtype: list(list(list(tuple(str,str))))
+        """
+        return concat([ChunkedCorpusView(f, enc, 1, 1, 1, 0, *self._cv_args, target_tagset=tagset)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def chunked_words(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of tagged
+            words and chunks.  Words are encoded as ``(word, tag)``
+            tuples (if the corpus has tags) or word strings (if the
+            corpus has no tags).  Chunks are encoded as depth-one
+            trees over ``(word,tag)`` tuples or word strings.
+        :rtype: list(tuple(str,str) and Tree)
+        """
+        return concat([ChunkedCorpusView(f, enc, 1, 0, 0, 1, *self._cv_args, target_tagset=tagset)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def chunked_sents(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of
+            sentences, each encoded as a shallow Tree.  The leaves
+            of these trees are encoded as ``(word, tag)`` tuples (if
+            the corpus has tags) or word strings (if the corpus has no
+            tags).
+        :rtype: list(Tree)
+        """
+        return concat([ChunkedCorpusView(f, enc, 1, 1, 0, 1, *self._cv_args, target_tagset=tagset)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def chunked_paras(self, fileids=None, tagset=None):
+        """
+        :return: the given file(s) as a list of
+            paragraphs, each encoded as a list of sentences, which are
+            in turn encoded as a shallow Tree.  The leaves of these
+            trees are encoded as ``(word, tag)`` tuples (if the corpus
+            has tags) or word strings (if the corpus has no tags).
+        :rtype: list(list(Tree))
+        """
+        return concat([ChunkedCorpusView(f, enc, 1, 1, 1, 1, *self._cv_args, target_tagset=tagset)
+                       for (f, enc) in self.abspaths(fileids, True)])
+
+    def _read_block(self, stream):
+        return [tagstr2tree(t) for t in read_blankline_block(stream)]
+
+class ChunkedCorpusView(StreamBackedCorpusView):
+    def __init__(self, fileid, encoding, tagged, group_by_sent,
+                 group_by_para, chunked, str2chunktree, sent_tokenizer,
+                 para_block_reader, source_tagset=None, target_tagset=None):
+        StreamBackedCorpusView.__init__(self, fileid, encoding=encoding)
+        self._tagged = tagged
+        self._group_by_sent = group_by_sent
+        self._group_by_para = group_by_para
+        self._chunked = chunked
+        self._str2chunktree = str2chunktree
+        self._sent_tokenizer = sent_tokenizer
+        self._para_block_reader = para_block_reader
+        self._source_tagset = source_tagset
+        self._target_tagset = target_tagset
+
+    def read_block(self, stream):
+        block = []
+        for para_str in self._para_block_reader(stream):
+            para = []
+            for sent_str in self._sent_tokenizer.tokenize(para_str):
+                sent = self._str2chunktree(sent_str, source_tagset=self._source_tagset,
+                                           target_tagset=self._target_tagset)
+
+                # If requested, throw away the tags.
+                if not self._tagged:
+                    sent = self._untag(sent)
+
+                # If requested, throw away the chunks.
+                if not self._chunked:
+                    sent = sent.leaves()
+
+                # Add the sentence to `para`.
+                if self._group_by_sent:
+                    para.append(sent)
+                else:
+                    para.extend(sent)
+
+            # Add the paragraph to `block`.
+            if self._group_by_para:
+                block.append(para)
+            else:
+                block.extend(para)
+
+        # Return the block
+        return block
+
+    def _untag(self, tree):
+        for i, child in enumerate(tree):
+            if isinstance(child, Tree):
+                self._untag(child)
+            elif isinstance(child, tuple):
+                tree[i] = child[0]
+            else:
+                raise ValueError('expected child to be Tree or tuple')
+        return tree

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/cmudict.py

@@ -1,95 +1,97 @@
-# Natural Language Toolkit: Carnegie Mellon Pronouncing Dictionary Corpus Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Steven Bird <stevenbird1@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-The Carnegie Mellon Pronouncing Dictionary [cmudict.0.6]
-ftp://ftp.cs.cmu.edu/project/speech/dict/
-Copyright 1998 Carnegie Mellon University
-
-File Format: Each line consists of an uppercased word, a counter
-(for alternative pronunciations), and a transcription.  Vowels are
-marked for stress (1=primary, 2=secondary, 0=no stress).  E.g.:
-NATURAL 1 N AE1 CH ER0 AH0 L
-
-The dictionary contains 127069 entries.  Of these, 119400 words are assigned
-a unique pronunciation, 6830 words have two pronunciations, and 839 words have
-three or more pronunciations.  Many of these are fast-speech variants.
-
-Phonemes: There are 39 phonemes, as shown below:
-
-Phoneme Example Translation    Phoneme Example Translation
-------- ------- -----------    ------- ------- -----------
-AA      odd     AA D           AE      at      AE T
-AH      hut     HH AH T        AO      ought   AO T
-AW      cow     K AW           AY      hide    HH AY D
-B       be      B IY           CH      cheese  CH IY Z
-D       dee     D IY           DH      thee    DH IY
-EH      Ed      EH D           ER      hurt    HH ER T
-EY      ate     EY T           F       fee     F IY
-G       green   G R IY N       HH      he      HH IY
-IH      it      IH T           IY      eat     IY T
-JH      gee     JH IY          K       key     K IY
-L       lee     L IY           M       me      M IY
-N       knee    N IY           NG      ping    P IH NG
-OW      oat     OW T           OY      toy     T OY
-P       pee     P IY           R       read    R IY D
-S       sea     S IY           SH      she     SH IY
-T       tea     T IY           TH      theta   TH EY T AH
-UH      hood    HH UH D        UW      two     T UW
-V       vee     V IY           W       we      W IY
-Y       yield   Y IY L D       Z       zee     Z IY
-ZH      seizure S IY ZH ER
-"""
-
-import codecs
-
-from nltk import compat
-from nltk.util import Index
-
-from nltk.corpus.reader.util import *
-from nltk.corpus.reader.api import *
-
-class CMUDictCorpusReader(CorpusReader):
-    def entries(self):
-        """
-        :return: the cmudict lexicon as a list of entries
-        containing (word, transcriptions) tuples.
-        """
-        return concat([StreamBackedCorpusView(fileid, read_cmudict_block,
-                                              encoding=enc)
-                       for fileid, enc in self.abspaths(None, True)])
-
-    def raw(self):
-        """
-        :return: the cmudict lexicon as a raw string.
-        """
-        fileids = self._fileids
-        if isinstance(fileids, compat.string_types):
-            fileids = [fileids]
-        return concat([self.open(f).read() for f in fileids])
-
-    def words(self):
-        """
-        :return: a list of all words defined in the cmudict lexicon.
-        """
-        return [word.lower() for (word, _) in self.entries()]
-
-    def dict(self):
-        """
-        :return: the cmudict lexicon as a dictionary, whose keys are
-        lowercase words and whose values are lists of pronunciations.
-        """
-        return dict(Index(self.entries()))
-
-def read_cmudict_block(stream):
-    entries = []
-    while len(entries) < 100: # Read 100 at a time.
-        line = stream.readline()
-        if line == '': return entries # end of file.
-        pieces = line.split()
-        entries.append( (pieces[0].lower(), pieces[2:]) )
-    return entries
+# Natural Language Toolkit: Carnegie Mellon Pronouncing Dictionary Corpus Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+The Carnegie Mellon Pronouncing Dictionary [cmudict.0.6]
+ftp://ftp.cs.cmu.edu/project/speech/dict/
+Copyright 1998 Carnegie Mellon University
+
+File Format: Each line consists of an uppercased word, a counter
+(for alternative pronunciations), and a transcription.  Vowels are
+marked for stress (1=primary, 2=secondary, 0=no stress).  E.g.:
+NATURAL 1 N AE1 CH ER0 AH0 L
+
+The dictionary contains 127069 entries.  Of these, 119400 words are assigned
+a unique pronunciation, 6830 words have two pronunciations, and 839 words have
+three or more pronunciations.  Many of these are fast-speech variants.
+
+Phonemes: There are 39 phonemes, as shown below:
+
+Phoneme Example Translation    Phoneme Example Translation
+------- ------- -----------    ------- ------- -----------
+AA      odd     AA D           AE      at      AE T
+AH      hut     HH AH T        AO      ought   AO T
+AW      cow     K AW           AY      hide    HH AY D
+B       be      B IY           CH      cheese  CH IY Z
+D       dee     D IY           DH      thee    DH IY
+EH      Ed      EH D           ER      hurt    HH ER T
+EY      ate     EY T           F       fee     F IY
+G       green   G R IY N       HH      he      HH IY
+IH      it      IH T           IY      eat     IY T
+JH      gee     JH IY          K       key     K IY
+L       lee     L IY           M       me      M IY
+N       knee    N IY           NG      ping    P IH NG
+OW      oat     OW T           OY      toy     T OY
+P       pee     P IY           R       read    R IY D
+S       sea     S IY           SH      she     SH IY
+T       tea     T IY           TH      theta   TH EY T AH
+UH      hood    HH UH D        UW      two     T UW
+V       vee     V IY           W       we      W IY
+Y       yield   Y IY L D       Z       zee     Z IY
+ZH      seizure S IY ZH ER
+"""
+
+import codecs
+
+from six import string_types
+
+from nltk import compat
+from nltk.util import Index
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class CMUDictCorpusReader(CorpusReader):
+    def entries(self):
+        """
+        :return: the cmudict lexicon as a list of entries
+        containing (word, transcriptions) tuples.
+        """
+        return concat([StreamBackedCorpusView(fileid, read_cmudict_block,
+                                              encoding=enc)
+                       for fileid, enc in self.abspaths(None, True)])
+
+    def raw(self):
+        """
+        :return: the cmudict lexicon as a raw string.
+        """
+        fileids = self._fileids
+        if isinstance(fileids, string_types):
+            fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])
+
+    def words(self):
+        """
+        :return: a list of all words defined in the cmudict lexicon.
+        """
+        return [word.lower() for (word, _) in self.entries()]
+
+    def dict(self):
+        """
+        :return: the cmudict lexicon as a dictionary, whose keys are
+        lowercase words and whose values are lists of pronunciations.
+        """
+        return dict(Index(self.entries()))
+
+def read_cmudict_block(stream):
+    entries = []
+    while len(entries) < 100: # Read 100 at a time.
+        line = stream.readline()
+        if line == '': return entries # end of file.
+        pieces = line.split()
+        entries.append( (pieces[0].lower(), pieces[2:]) )
+    return entries