nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/corpus/europarl_raw.py

@@ -1,44 +1,44 @@
-# Natural Language Toolkit: Europarl Corpus Readers
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author:  Nitin Madnani <nmadnani@umiacs.umd.edu>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-import re
-from nltk.corpus.util import LazyCorpusLoader
-from nltk.corpus.reader import *
-
-# Create a new corpus reader instance for each European language
-danish = LazyCorpusLoader(
-    'europarl_raw/danish', EuroparlCorpusReader, r'ep-.*\.da', encoding='utf-8')
-
-dutch = LazyCorpusLoader(
-    'europarl_raw/dutch', EuroparlCorpusReader, r'ep-.*\.nl', encoding='utf-8')
-
-english = LazyCorpusLoader(
-    'europarl_raw/english', EuroparlCorpusReader, r'ep-.*\.en', encoding='utf-8')
-
-finnish = LazyCorpusLoader(
-    'europarl_raw/finnish', EuroparlCorpusReader, r'ep-.*\.fi', encoding='utf-8')
-
-french = LazyCorpusLoader(
-    'europarl_raw/french', EuroparlCorpusReader, r'ep-.*\.fr', encoding='utf-8')
-
-german = LazyCorpusLoader(
-    'europarl_raw/german', EuroparlCorpusReader, r'ep-.*\.de', encoding='utf-8')
-
-greek = LazyCorpusLoader(
-    'europarl_raw/greek', EuroparlCorpusReader, r'ep-.*\.el', encoding='utf-8')
-
-italian = LazyCorpusLoader(
-    'europarl_raw/italian', EuroparlCorpusReader, r'ep-.*\.it', encoding='utf-8')
-
-portuguese = LazyCorpusLoader(
-    'europarl_raw/portuguese', EuroparlCorpusReader, r'ep-.*\.pt', encoding='utf-8')
-
-spanish = LazyCorpusLoader(
-    'europarl_raw/spanish', EuroparlCorpusReader, r'ep-.*\.es', encoding='utf-8')
-
-swedish = LazyCorpusLoader(
-    'europarl_raw/swedish', EuroparlCorpusReader, r'ep-.*\.sv', encoding='utf-8')
+# Natural Language Toolkit: Europarl Corpus Readers
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author:  Nitin Madnani <nmadnani@umiacs.umd.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import re
+from nltk.corpus.util import LazyCorpusLoader
+from nltk.corpus.reader import *
+
+# Create a new corpus reader instance for each European language
+danish = LazyCorpusLoader(
+    'europarl_raw/danish', EuroparlCorpusReader, r'ep-.*\.da', encoding='utf-8')
+
+dutch = LazyCorpusLoader(
+    'europarl_raw/dutch', EuroparlCorpusReader, r'ep-.*\.nl', encoding='utf-8')
+
+english = LazyCorpusLoader(
+    'europarl_raw/english', EuroparlCorpusReader, r'ep-.*\.en', encoding='utf-8')
+
+finnish = LazyCorpusLoader(
+    'europarl_raw/finnish', EuroparlCorpusReader, r'ep-.*\.fi', encoding='utf-8')
+
+french = LazyCorpusLoader(
+    'europarl_raw/french', EuroparlCorpusReader, r'ep-.*\.fr', encoding='utf-8')
+
+german = LazyCorpusLoader(
+    'europarl_raw/german', EuroparlCorpusReader, r'ep-.*\.de', encoding='utf-8')
+
+greek = LazyCorpusLoader(
+    'europarl_raw/greek', EuroparlCorpusReader, r'ep-.*\.el', encoding='utf-8')
+
+italian = LazyCorpusLoader(
+    'europarl_raw/italian', EuroparlCorpusReader, r'ep-.*\.it', encoding='utf-8')
+
+portuguese = LazyCorpusLoader(
+    'europarl_raw/portuguese', EuroparlCorpusReader, r'ep-.*\.pt', encoding='utf-8')
+
+spanish = LazyCorpusLoader(
+    'europarl_raw/spanish', EuroparlCorpusReader, r'ep-.*\.es', encoding='utf-8')
+
+swedish = LazyCorpusLoader(
+    'europarl_raw/swedish', EuroparlCorpusReader, r'ep-.*\.sv', encoding='utf-8')

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/__init__.py

@@ -1,145 +1,147 @@
-# Natural Language Toolkit: Corpus Readers
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Steven Bird <stevenbird1@gmail.com>
-#         Edward Loper <edloper@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-NLTK corpus readers.  The modules in this package provide functions
-that can be used to read corpus fileids in a variety of formats.  These
-functions can be used to read both the corpus fileids that are
-distributed in the NLTK corpus package, and corpus fileids that are part
-of external corpora.
-
-Corpus Reader Functions
-=======================
-Each corpus module defines one or more "corpus reader functions",
-which can be used to read documents from that corpus.  These functions
-take an argument, ``item``, which is used to indicate which document
-should be read from the corpus:
-
-- If ``item`` is one of the unique identifiers listed in the corpus
-  module's ``items`` variable, then the corresponding document will
-  be loaded from the NLTK corpus package.
-- If ``item`` is a fileid, then that file will be read.
-
-Additionally, corpus reader functions can be given lists of item
-names; in which case, they will return a concatenation of the
-corresponding documents.
-
-Corpus reader functions are named based on the type of information
-they return.  Some common examples, and their return types, are:
-
-- words(): list of str
-- sents(): list of (list of str)
-- paras(): list of (list of (list of str))
-- tagged_words(): list of (str,str) tuple
-- tagged_sents(): list of (list of (str,str))
-- tagged_paras(): list of (list of (list of (str,str)))
-- chunked_sents(): list of (Tree w/ (str,str) leaves)
-- parsed_sents(): list of (Tree with str leaves)
-- parsed_paras(): list of (list of (Tree with str leaves))
-- xml(): A single xml ElementTree
-- raw(): unprocessed corpus contents
-
-For example, to read a list of the words in the Brown Corpus, use
-``nltk.corpus.brown.words()``:
-
-    >>> from nltk.corpus import brown
-    >>> print(", ".join(brown.words()))
-    The, Fulton, County, Grand, Jury, said, ...
-
-"""
-
-from nltk.corpus.reader.plaintext import *
-from nltk.corpus.reader.util import *
-from nltk.corpus.reader.api import *
-from nltk.corpus.reader.tagged import *
-from nltk.corpus.reader.cmudict import *
-from nltk.corpus.reader.conll import *
-from nltk.corpus.reader.chunked import *
-from nltk.corpus.reader.wordlist import *
-from nltk.corpus.reader.xmldocs import *
-from nltk.corpus.reader.ppattach import *
-from nltk.corpus.reader.senseval import *
-from nltk.corpus.reader.ieer import *
-from nltk.corpus.reader.sinica_treebank import *
-from nltk.corpus.reader.bracket_parse import *
-from nltk.corpus.reader.indian import *
-from nltk.corpus.reader.toolbox import *
-from nltk.corpus.reader.timit import *
-from nltk.corpus.reader.ycoe import *
-from nltk.corpus.reader.rte import *
-from nltk.corpus.reader.string_category import *
-from nltk.corpus.reader.propbank import *
-from nltk.corpus.reader.verbnet import *
-from nltk.corpus.reader.bnc import *
-from nltk.corpus.reader.nps_chat import *
-from nltk.corpus.reader.wordnet import *
-from nltk.corpus.reader.switchboard import *
-from nltk.corpus.reader.dependency import *
-from nltk.corpus.reader.nombank import *
-from nltk.corpus.reader.ipipan import *
-from nltk.corpus.reader.pl196x import *
-from nltk.corpus.reader.knbc import *
-from nltk.corpus.reader.chasen import *
-from nltk.corpus.reader.childes import *
-from nltk.corpus.reader.aligned import *
-from nltk.corpus.reader.lin import *
-from nltk.corpus.reader.semcor import *
-from nltk.corpus.reader.framenet import *
-from nltk.corpus.reader.udhr import *
-from nltk.corpus.reader.bnc import *
-from nltk.corpus.reader.sentiwordnet import *
-from nltk.corpus.reader.twitter import *
-from nltk.corpus.reader.nkjp import *
-from nltk.corpus.reader.crubadan import *
-from nltk.corpus.reader.mte import *
-from nltk.corpus.reader.reviews import *
-from nltk.corpus.reader.opinion_lexicon import *
-from nltk.corpus.reader.pros_cons import *
-from nltk.corpus.reader.categorized_sents import *
-from nltk.corpus.reader.comparative_sents import *
-from nltk.corpus.reader.panlex_lite import *
-
-# Make sure that nltk.corpus.reader.bracket_parse gives the module, not
-# the function bracket_parse() defined in nltk.tree:
-from nltk.corpus.reader import bracket_parse
-
-__all__ = [
-    'CorpusReader', 'CategorizedCorpusReader',
-    'PlaintextCorpusReader', 'find_corpus_fileids',
-    'TaggedCorpusReader', 'CMUDictCorpusReader',
-    'ConllChunkCorpusReader', 'WordListCorpusReader',
-    'PPAttachmentCorpusReader', 'SensevalCorpusReader',
-    'IEERCorpusReader', 'ChunkedCorpusReader',
-    'SinicaTreebankCorpusReader', 'BracketParseCorpusReader',
-    'IndianCorpusReader', 'ToolboxCorpusReader',
-    'TimitCorpusReader', 'YCOECorpusReader',
-    'MacMorphoCorpusReader', 'SyntaxCorpusReader',
-    'AlpinoCorpusReader', 'RTECorpusReader',
-    'StringCategoryCorpusReader','EuroparlCorpusReader',
-    'CategorizedBracketParseCorpusReader',
-    'CategorizedTaggedCorpusReader',
-    'CategorizedPlaintextCorpusReader',
-    'PortugueseCategorizedPlaintextCorpusReader',
-    'tagged_treebank_para_block_reader',
-    'PropbankCorpusReader', 'VerbnetCorpusReader',
-    'BNCCorpusReader', 'ConllCorpusReader',
-    'XMLCorpusReader', 'NPSChatCorpusReader',
-    'SwadeshCorpusReader', 'WordNetCorpusReader',
-    'WordNetICCorpusReader', 'SwitchboardCorpusReader',
-    'DependencyCorpusReader', 'NombankCorpusReader',
-    'IPIPANCorpusReader', 'Pl196xCorpusReader',
-    'TEICorpusView', 'KNBCorpusReader', 'ChasenCorpusReader',
-    'CHILDESCorpusReader', 'AlignedCorpusReader',
-    'TimitTaggedCorpusReader', 'LinThesaurusCorpusReader',
-    'SemcorCorpusReader', 'FramenetCorpusReader', 'UdhrCorpusReader',
-    'BNCCorpusReader', 'SentiWordNetCorpusReader', 'SentiSynset',
-    'TwitterCorpusReader', 'NKJPCorpusReader', 'CrubadanCorpusReader',
-    'MTECorpusReader', 'ReviewsCorpusReader', 'OpinionLexiconCorpusReader',
-    'ProsConsCorpusReader', 'CategorizedSentencesCorpusReader',
-    'ComparativeSentencesCorpusReader', 'PanLexLiteCorpusReader'
-]
+# Natural Language Toolkit: Corpus Readers
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+#         Edward Loper <edloper@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+NLTK corpus readers.  The modules in this package provide functions
+that can be used to read corpus fileids in a variety of formats.  These
+functions can be used to read both the corpus fileids that are
+distributed in the NLTK corpus package, and corpus fileids that are part
+of external corpora.
+
+Corpus Reader Functions
+=======================
+Each corpus module defines one or more "corpus reader functions",
+which can be used to read documents from that corpus.  These functions
+take an argument, ``item``, which is used to indicate which document
+should be read from the corpus:
+
+- If ``item`` is one of the unique identifiers listed in the corpus
+  module's ``items`` variable, then the corresponding document will
+  be loaded from the NLTK corpus package.
+- If ``item`` is a fileid, then that file will be read.
+
+Additionally, corpus reader functions can be given lists of item
+names; in which case, they will return a concatenation of the
+corresponding documents.
+
+Corpus reader functions are named based on the type of information
+they return.  Some common examples, and their return types, are:
+
+- words(): list of str
+- sents(): list of (list of str)
+- paras(): list of (list of (list of str))
+- tagged_words(): list of (str,str) tuple
+- tagged_sents(): list of (list of (str,str))
+- tagged_paras(): list of (list of (list of (str,str)))
+- chunked_sents(): list of (Tree w/ (str,str) leaves)
+- parsed_sents(): list of (Tree with str leaves)
+- parsed_paras(): list of (list of (Tree with str leaves))
+- xml(): A single xml ElementTree
+- raw(): unprocessed corpus contents
+
+For example, to read a list of the words in the Brown Corpus, use
+``nltk.corpus.brown.words()``:
+
+    >>> from nltk.corpus import brown
+    >>> print(", ".join(brown.words()))
+    The, Fulton, County, Grand, Jury, said, ...
+
+"""
+
+from nltk.corpus.reader.plaintext import *
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.tagged import *
+from nltk.corpus.reader.cmudict import *
+from nltk.corpus.reader.conll import *
+from nltk.corpus.reader.chunked import *
+from nltk.corpus.reader.wordlist import *
+from nltk.corpus.reader.xmldocs import *
+from nltk.corpus.reader.ppattach import *
+from nltk.corpus.reader.senseval import *
+from nltk.corpus.reader.ieer import *
+from nltk.corpus.reader.sinica_treebank import *
+from nltk.corpus.reader.bracket_parse import *
+from nltk.corpus.reader.indian import *
+from nltk.corpus.reader.toolbox import *
+from nltk.corpus.reader.timit import *
+from nltk.corpus.reader.ycoe import *
+from nltk.corpus.reader.rte import *
+from nltk.corpus.reader.string_category import *
+from nltk.corpus.reader.propbank import *
+from nltk.corpus.reader.verbnet import *
+from nltk.corpus.reader.bnc import *
+from nltk.corpus.reader.nps_chat import *
+from nltk.corpus.reader.wordnet import *
+from nltk.corpus.reader.switchboard import *
+from nltk.corpus.reader.dependency import *
+from nltk.corpus.reader.nombank import *
+from nltk.corpus.reader.ipipan import *
+from nltk.corpus.reader.pl196x import *
+from nltk.corpus.reader.knbc import *
+from nltk.corpus.reader.chasen import *
+from nltk.corpus.reader.childes import *
+from nltk.corpus.reader.aligned import *
+from nltk.corpus.reader.lin import *
+from nltk.corpus.reader.semcor import *
+from nltk.corpus.reader.framenet import *
+from nltk.corpus.reader.udhr import *
+from nltk.corpus.reader.bnc import *
+from nltk.corpus.reader.sentiwordnet import *
+from nltk.corpus.reader.twitter import *
+from nltk.corpus.reader.nkjp import *
+from nltk.corpus.reader.crubadan import *
+from nltk.corpus.reader.mte import *
+from nltk.corpus.reader.reviews import *
+from nltk.corpus.reader.opinion_lexicon import *
+from nltk.corpus.reader.pros_cons import *
+from nltk.corpus.reader.categorized_sents import *
+from nltk.corpus.reader.comparative_sents import *
+from nltk.corpus.reader.panlex_lite import *
+
+# Make sure that nltk.corpus.reader.bracket_parse gives the module, not
+# the function bracket_parse() defined in nltk.tree:
+from nltk.corpus.reader import bracket_parse
+
+__all__ = [
+    'CorpusReader', 'CategorizedCorpusReader',
+    'PlaintextCorpusReader', 'find_corpus_fileids',
+    'TaggedCorpusReader', 'CMUDictCorpusReader',
+    'ConllChunkCorpusReader', 'WordListCorpusReader',
+    'PPAttachmentCorpusReader', 'SensevalCorpusReader',
+    'IEERCorpusReader', 'ChunkedCorpusReader',
+    'SinicaTreebankCorpusReader', 'BracketParseCorpusReader',
+    'IndianCorpusReader', 'ToolboxCorpusReader',
+    'TimitCorpusReader', 'YCOECorpusReader',
+    'MacMorphoCorpusReader', 'SyntaxCorpusReader',
+    'AlpinoCorpusReader', 'RTECorpusReader',
+    'StringCategoryCorpusReader','EuroparlCorpusReader',
+    'CategorizedBracketParseCorpusReader',
+    'CategorizedTaggedCorpusReader',
+    'CategorizedPlaintextCorpusReader',
+    'PortugueseCategorizedPlaintextCorpusReader',
+    'tagged_treebank_para_block_reader',
+    'PropbankCorpusReader', 'VerbnetCorpusReader',
+    'BNCCorpusReader', 'ConllCorpusReader',
+    'XMLCorpusReader', 'NPSChatCorpusReader',
+    'SwadeshCorpusReader', 'WordNetCorpusReader',
+    'WordNetICCorpusReader', 'SwitchboardCorpusReader',
+    'DependencyCorpusReader', 'NombankCorpusReader',
+    'IPIPANCorpusReader', 'Pl196xCorpusReader',
+    'TEICorpusView', 'KNBCorpusReader', 'ChasenCorpusReader',
+    'CHILDESCorpusReader', 'AlignedCorpusReader',
+    'TimitTaggedCorpusReader', 'LinThesaurusCorpusReader',
+    'SemcorCorpusReader', 'FramenetCorpusReader', 'UdhrCorpusReader',
+    'BNCCorpusReader', 'SentiWordNetCorpusReader', 'SentiSynset',
+    'TwitterCorpusReader', 'NKJPCorpusReader', 'CrubadanCorpusReader',
+    'MTECorpusReader', 'ReviewsCorpusReader', 'OpinionLexiconCorpusReader',
+    'ProsConsCorpusReader', 'CategorizedSentencesCorpusReader',
+    'ComparativeSentencesCorpusReader', 'PanLexLiteCorpusReader',
+    'NonbreakingPrefixesCorpusReader', 'UnicharsCorpusReader',
+    'MWAPPDBCorpusReader',
+]

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/aligned.py

@@ -1,114 +1,115 @@
-# Natural Language Toolkit: Aligned Corpus Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# URL: <http://nltk.org/>
-# Author: Steven Bird <stevenbird1@gmail.com>
-# For license information, see LICENSE.TXT
-
-from nltk import compat
-from nltk.tokenize import WhitespaceTokenizer, RegexpTokenizer
-from nltk.translate import AlignedSent, Alignment
-
-from nltk.corpus.reader.api import CorpusReader
-from nltk.corpus.reader.util import StreamBackedCorpusView, concat,\
-    read_alignedsent_block
-
-class AlignedCorpusReader(CorpusReader):
-    """
-    Reader for corpora of word-aligned sentences.  Tokens are assumed
-    to be separated by whitespace.  Sentences begin on separate lines.
-    """
-    def __init__(self, root, fileids,
-                 sep='/', word_tokenizer=WhitespaceTokenizer(),
-                 sent_tokenizer=RegexpTokenizer('\n', gaps=True),
-                 alignedsent_block_reader=read_alignedsent_block,
-                 encoding='latin1'):
-        """
-        Construct a new Aligned Corpus reader for a set of documents
-        located at the given root directory.  Example usage:
-
-            >>> root = '/...path to corpus.../'
-            >>> reader = AlignedCorpusReader(root, '.*', '.txt') # doctest: +SKIP
-
-        :param root: The root directory for this corpus.
-        :param fileids: A list or regexp specifying the fileids in this corpus.
-        """
-        CorpusReader.__init__(self, root, fileids, encoding)
-        self._sep = sep
-        self._word_tokenizer = word_tokenizer
-        self._sent_tokenizer = sent_tokenizer
-        self._alignedsent_block_reader = alignedsent_block_reader
-
-    def raw(self, fileids=None):
-        """
-        :return: the given file(s) as a single string.
-        :rtype: str
-        """
-        if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
-        return concat([self.open(f).read() for f in fileids])
-
-    def words(self, fileids=None):
-        """
-        :return: the given file(s) as a list of words
-            and punctuation symbols.
-        :rtype: list(str)
-        """
-        return concat([AlignedSentCorpusView(fileid, enc, False, False,
-                                             self._word_tokenizer,
-                                             self._sent_tokenizer,
-                                             self._alignedsent_block_reader)
-                       for (fileid, enc) in self.abspaths(fileids, True)])
-
-    def sents(self, fileids=None):
-        """
-        :return: the given file(s) as a list of
-            sentences or utterances, each encoded as a list of word
-            strings.
-        :rtype: list(list(str))
-        """
-        return concat([AlignedSentCorpusView(fileid, enc, False, True,
-                                             self._word_tokenizer,
-                                             self._sent_tokenizer,
-                                             self._alignedsent_block_reader)
-                       for (fileid, enc) in self.abspaths(fileids, True)])
-
-    def aligned_sents(self, fileids=None):
-        """
-        :return: the given file(s) as a list of AlignedSent objects.
-        :rtype: list(AlignedSent)
-        """
-        return concat([AlignedSentCorpusView(fileid, enc, True, True,
-                                             self._word_tokenizer,
-                                             self._sent_tokenizer,
-                                             self._alignedsent_block_reader)
-                       for (fileid, enc) in self.abspaths(fileids, True)])
-
-class AlignedSentCorpusView(StreamBackedCorpusView):
-    """
-    A specialized corpus view for aligned sentences.
-    ``AlignedSentCorpusView`` objects are typically created by
-    ``AlignedCorpusReader`` (not directly by nltk users).
-    """
-    def __init__(self, corpus_file, encoding, aligned, group_by_sent,
-                 word_tokenizer, sent_tokenizer, alignedsent_block_reader):
-        self._aligned = aligned
-        self._group_by_sent = group_by_sent
-        self._word_tokenizer = word_tokenizer
-        self._sent_tokenizer = sent_tokenizer
-        self._alignedsent_block_reader = alignedsent_block_reader
-        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
-
-    def read_block(self, stream):
-        block = [self._word_tokenizer.tokenize(sent_str)
-                 for alignedsent_str in self._alignedsent_block_reader(stream)
-                 for sent_str in self._sent_tokenizer.tokenize(alignedsent_str)]
-        if self._aligned:
-            block[2] = Alignment.fromstring(" ".join(block[2])) # kludge; we shouldn't have tokenized the alignment string
-            block = [AlignedSent(*block)]
-        elif self._group_by_sent:
-            block = [block[0]]
-        else:
-            block = block[0]
-
-        return block
+# Natural Language Toolkit: Aligned Corpus Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# URL: <http://nltk.org/>
+# Author: Steven Bird <stevenbird1@gmail.com>
+# For license information, see LICENSE.TXT
+
+from six import string_types
+
+from nltk.tokenize import WhitespaceTokenizer, RegexpTokenizer
+from nltk.translate import AlignedSent, Alignment
+
+from nltk.corpus.reader.api import CorpusReader
+from nltk.corpus.reader.util import StreamBackedCorpusView, concat,\
+    read_alignedsent_block
+
+class AlignedCorpusReader(CorpusReader):
+    """
+    Reader for corpora of word-aligned sentences.  Tokens are assumed
+    to be separated by whitespace.  Sentences begin on separate lines.
+    """
+    def __init__(self, root, fileids,
+                 sep='/', word_tokenizer=WhitespaceTokenizer(),
+                 sent_tokenizer=RegexpTokenizer('\n', gaps=True),
+                 alignedsent_block_reader=read_alignedsent_block,
+                 encoding='latin1'):
+        """
+        Construct a new Aligned Corpus reader for a set of documents
+        located at the given root directory.  Example usage:
+
+            >>> root = '/...path to corpus.../'
+            >>> reader = AlignedCorpusReader(root, '.*', '.txt') # doctest: +SKIP
+
+        :param root: The root directory for this corpus.
+        :param fileids: A list or regexp specifying the fileids in this corpus.
+        """
+        CorpusReader.__init__(self, root, fileids, encoding)
+        self._sep = sep
+        self._word_tokenizer = word_tokenizer
+        self._sent_tokenizer = sent_tokenizer
+        self._alignedsent_block_reader = alignedsent_block_reader
+
+    def raw(self, fileids=None):
+        """
+        :return: the given file(s) as a single string.
+        :rtype: str
+        """
+        if fileids is None: fileids = self._fileids
+        elif isinstance(fileids, string_types): fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])
+
+    def words(self, fileids=None):
+        """
+        :return: the given file(s) as a list of words
+            and punctuation symbols.
+        :rtype: list(str)
+        """
+        return concat([AlignedSentCorpusView(fileid, enc, False, False,
+                                             self._word_tokenizer,
+                                             self._sent_tokenizer,
+                                             self._alignedsent_block_reader)
+                       for (fileid, enc) in self.abspaths(fileids, True)])
+
+    def sents(self, fileids=None):
+        """
+        :return: the given file(s) as a list of
+            sentences or utterances, each encoded as a list of word
+            strings.
+        :rtype: list(list(str))
+        """
+        return concat([AlignedSentCorpusView(fileid, enc, False, True,
+                                             self._word_tokenizer,
+                                             self._sent_tokenizer,
+                                             self._alignedsent_block_reader)
+                       for (fileid, enc) in self.abspaths(fileids, True)])
+
+    def aligned_sents(self, fileids=None):
+        """
+        :return: the given file(s) as a list of AlignedSent objects.
+        :rtype: list(AlignedSent)
+        """
+        return concat([AlignedSentCorpusView(fileid, enc, True, True,
+                                             self._word_tokenizer,
+                                             self._sent_tokenizer,
+                                             self._alignedsent_block_reader)
+                       for (fileid, enc) in self.abspaths(fileids, True)])
+
+class AlignedSentCorpusView(StreamBackedCorpusView):
+    """
+    A specialized corpus view for aligned sentences.
+    ``AlignedSentCorpusView`` objects are typically created by
+    ``AlignedCorpusReader`` (not directly by nltk users).
+    """
+    def __init__(self, corpus_file, encoding, aligned, group_by_sent,
+                 word_tokenizer, sent_tokenizer, alignedsent_block_reader):
+        self._aligned = aligned
+        self._group_by_sent = group_by_sent
+        self._word_tokenizer = word_tokenizer
+        self._sent_tokenizer = sent_tokenizer
+        self._alignedsent_block_reader = alignedsent_block_reader
+        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
+
+    def read_block(self, stream):
+        block = [self._word_tokenizer.tokenize(sent_str)
+                 for alignedsent_str in self._alignedsent_block_reader(stream)
+                 for sent_str in self._sent_tokenizer.tokenize(alignedsent_str)]
+        if self._aligned:
+            block[2] = Alignment.fromstring(" ".join(block[2])) # kludge; we shouldn't have tokenized the alignment string
+            block = [AlignedSent(*block)]
+        elif self._group_by_sent:
+            block = [block[0]]
+        else:
+            block = block[0]
+
+        return block