nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/ieer.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: IEER Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -22,6 +22,8 @@ APW_19980429, NYT_19980315, NYT_19980403, and NYT_19980407.
 """
 from __future__ import unicode_literals
 
+from six import string_types
+
 import nltk
 from nltk import compat
 from nltk.corpus.reader.api import *
@@ -66,7 +68,7 @@ class IEERCorpusReader(CorpusReader):
     """
     def raw(self, fileids=None):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def docs(self, fileids=None):
@@ -108,4 +110,3 @@ class IEERCorpusReader(CorpusReader):
             if line.strip() == '</DOC>': break
         # Return the document
         return ['\n'.join(out)]
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/indian.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Indian Language POS-Tagged Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -18,7 +18,8 @@ Contents:
   - Telugu: IIIT Hyderabad
 """
 
-from nltk import compat
+from six import string_types
+
 from nltk.tag import str2tuple, map_tag
 
 from nltk.corpus.reader.util import *
@@ -58,7 +59,7 @@ class IndianCorpusReader(CorpusReader):
 
     def raw(self, fileids=None):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
 
@@ -82,5 +83,3 @@ class IndianCorpusView(StreamBackedCorpusView):
             return [sent]
         else:
             return sent
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/ipipan.py

@@ -1,13 +1,14 @@
 # Natural Language Toolkit: IPI PAN Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Konrad Goluchowski <kodie@mimuw.edu.pl>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
 import functools
 
-from nltk import compat
+from six import string_types
+
 from nltk.corpus.reader.util import StreamBackedCorpusView, concat
 from nltk.corpus.reader.api import CorpusReader
 
@@ -93,11 +94,11 @@ class IPIPANCorpusReader(CorpusReader):
         if channels is None and domains is None and \
                 categories is None:
             return CorpusReader.fileids(self)
-        if isinstance(channels, compat.string_types):
+        if isinstance(channels, string_types):
             channels = [channels]
-        if isinstance(domains, compat.string_types):
+        if isinstance(domains, string_types):
             domains = [domains]
-        if isinstance(categories, compat.string_types):
+        if isinstance(categories, string_types):
             categories = [categories]
         if channels:
             return self._list_morph_files_by('channel', channels)

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/knbc.py

@@ -1,6 +1,6 @@
 #! /usr/bin/env python
 # KNB Corpus reader
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Masato Hagiwara <hagisan@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -9,8 +9,8 @@
 from __future__ import print_function
 
 import re
+from six import string_types
 
-from nltk.compat import string_types
 from nltk.parse import DependencyGraph
 
 from nltk.corpus.reader.util import (

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/lin.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Lin's Thesaurus
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Dan Blanchard <dblanchard@ets.org>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.txt

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/mte.py

@@ -2,13 +2,14 @@
 A reader for corpora whose documents are in MTE format.
 """
 import os
+import re
 from functools import reduce
-from nltk import compat
+
+from six import string_types
+
 from nltk.corpus.reader import concat, TaggedCorpusReader
 from nltk.corpus.reader.xmldocs import XMLCorpusView
 
-import xml.etree.ElementTree as etree
-import re
 
 def xpath(root, path, ns):
     return root.findall(path, ns)
@@ -179,7 +180,7 @@ class MTECorpusReader(TaggedCorpusReader):
 
     def __fileids(self, fileids):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         # filter wrong userinput
         fileids = filter(lambda x : x in self._fileids, fileids)
         # filter multext-east sourcefiles that are not compatible to the teip5 specification

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/nkjp.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: NKJP Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Gabriela Kaczka
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -8,7 +8,8 @@
 import functools
 import os
 import tempfile
-from nltk import compat
+
+from six import string_types
 
 from nltk.corpus.reader.util import concat
 from nltk.corpus.reader.xmldocs import XMLCorpusReader, XMLCorpusView
@@ -53,7 +54,7 @@ class NKJPCorpusReader(XMLCorpusReader):
         x.header(fileids=['WilkDom', '/home/USER/nltk_data/corpora/nkjp/WilkWilczy'])
         x.tagged_words(fileids=['WilkDom', '/home/USER/nltk_data/corpora/nkjp/WilkWilczy'], tags=['subst', 'comp'])
         """
-        if isinstance(fileids, compat.string_types):
+        if isinstance(fileids, string_types):
             XMLCorpusReader.__init__(self, root, fileids + '.*/header.xml')
         else:
             XMLCorpusReader.__init__(self, root, [fileid + '/header.xml' for fileid in fileids])

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/nombank.py

@@ -1,17 +1,20 @@
 # Natural Language Toolkit: NomBank Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Authors: Paul Bedaride <paul.bedaride@gmail.com>
 #          Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
 from __future__ import unicode_literals
+from xml.etree import ElementTree
+from functools import total_ordering
+
+from six import string_types
 
 from nltk.tree import Tree
-from xml.etree import ElementTree
 from nltk.internals import raise_unorderable_types
-from nltk.compat import total_ordering, python_2_unicode_compatible, string_types
+from nltk.compat import python_2_unicode_compatible
 
 from nltk.corpus.reader.util import *
 from nltk.corpus.reader.api import *
@@ -66,7 +69,7 @@ class NombankCorpusReader(CorpusReader):
         :return: the text contents of the given fileids, as a single string.
         """
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def instances(self, baseform=None):
@@ -108,9 +111,7 @@ class NombankCorpusReader(CorpusReader):
         for roleset in etree.findall('predicate/roleset'):
             if roleset.attrib['id'] == roleset_id:
                 return roleset
-        else:
-            raise ValueError('Roleset %s not found in %s' %
-                             (roleset_id, framefile))
+        raise ValueError('Roleset %s not found in %s' % (roleset_id, framefile))
 
     def rolesets(self, baseform=None):
         """
@@ -418,4 +419,3 @@ class NombankTreePointer(NombankPointer):
                 else:
                     wordnum += 1
                     stack.pop()
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/nps_chat.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: NPS Chat Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/opinion_lexicon.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Opinion Lexicon Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Pierpaolo Pantone <24alsecondo@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -27,8 +27,8 @@ Related papers:
     Comparing Opinions on the Web". Proceedings of the 14th International World
     Wide Web conference (WWW-2005), May 10-14, 2005, Chiba, Japan.
 """
+from six import string_types
 
-from nltk.compat import string_types
 from nltk.corpus.reader import WordListCorpusReader
 from nltk.corpus.reader.api import *
 
@@ -83,7 +83,7 @@ class OpinionLexiconCorpusReader(WordListCorpusReader):
         :rtype: list(str)
         """
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.CorpusView(path, self._read_word_block, encoding=enc)
             for (path, enc, fileid) in self.abspaths(fileids, True, True)])
 

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/panlex_lite.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: PanLex Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: David Kamholz <kamholz@panlex.org>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

nltk-3.3/nltk/corpus/reader/pl196x.py

@@ -0,0 +1,292 @@
+# Natural Language Toolkit:
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Piotr Kasprzyk <p.j.kasprzyk@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from six import string_types
+
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.xmldocs import XMLCorpusReader
+
+
+PARA = re.compile(r'<p(?: [^>]*){0,1}>(.*?)</p>')
+SENT = re.compile(r'<s(?: [^>]*){0,1}>(.*?)</s>')
+
+TAGGEDWORD = re.compile(r'<([wc](?: [^>]*){0,1}>)(.*?)</[wc]>')
+WORD = re.compile(r'<[wc](?: [^>]*){0,1}>(.*?)</[wc]>')
+
+TYPE = re.compile(r'type="(.*?)"')
+ANA = re.compile(r'ana="(.*?)"')
+
+TEXTID = re.compile(r'text id="(.*?)"')
+
+
+class TEICorpusView(StreamBackedCorpusView):
+    def __init__(self, corpus_file,
+                 tagged, group_by_sent, group_by_para,
+                 tagset=None, head_len=0, textids=None):
+
+        self._tagged = tagged
+        self._textids = textids
+
+        self._group_by_sent = group_by_sent
+        self._group_by_para = group_by_para
+        # WARNING -- skip header
+        StreamBackedCorpusView.__init__(self, corpus_file, startpos=head_len)
+
+    _pagesize = 4096
+
+    def read_block(self, stream):
+        block = stream.readlines(self._pagesize)
+        block = concat(block)
+        while (block.count('<text id') > block.count('</text>')) \
+                or block.count('<text id') == 0:
+            tmp = stream.readline()
+            if len(tmp) <= 0:
+                break
+            block += tmp
+
+        block = block.replace('\n', '')
+
+        textids = TEXTID.findall(block)
+        if self._textids:
+            for tid in textids:
+                if tid not in self._textids:
+                    beg = block.find(tid) - 1
+                    end = block[beg:].find('</text>') + len('</text>')
+                    block = block[:beg] + block[beg + end:]
+
+        output = []
+        for para_str in PARA.findall(block):
+            para = []
+            for sent_str in SENT.findall(para_str):
+                if not self._tagged:
+                    sent = WORD.findall(sent_str)
+                else:
+                    sent = list(
+                        map(self._parse_tag, TAGGEDWORD.findall(sent_str)))
+                if self._group_by_sent:
+                    para.append(sent)
+                else:
+                    para.extend(sent)
+            if self._group_by_para:
+                output.append(para)
+            else:
+                output.extend(para)
+        return output
+
+    def _parse_tag(self, tag_word_tuple):
+        (tag, word) = tag_word_tuple
+        if tag.startswith('w'):
+            tag = ANA.search(tag).group(1)
+        else:  # tag.startswith('c')
+            tag = TYPE.search(tag).group(1)
+        return word, tag
+
+
+class Pl196xCorpusReader(CategorizedCorpusReader, XMLCorpusReader):
+    head_len = 2770
+
+    def __init__(self, *args, **kwargs):
+        if 'textid_file' in kwargs:
+            self._textids = kwargs['textid_file']
+        else:
+            self._textids = None
+
+        XMLCorpusReader.__init__(self, *args)
+        CategorizedCorpusReader.__init__(self, kwargs)
+
+        self._init_textids()
+
+    def _init_textids(self):
+        self._f2t = defaultdict(list)
+        self._t2f = defaultdict(list)
+        if self._textids is not None:
+            with open(self._textids) as fp:
+                for line in fp:
+                    line = line.strip()
+                    file_id, text_ids = line.split(' ', 1)
+                    if file_id not in self.fileids():
+                        raise ValueError(
+                            'In text_id mapping file %s: %s not found'
+                            % (self._textids, file_id)
+                        )
+                    for text_id in text_ids.split(self._delimiter):
+                        self._add_textids(file_id, text_id)
+
+    def _add_textids(self, file_id, text_id):
+        self._f2t[file_id].append(text_id)
+        self._t2f[text_id].append(file_id)
+
+    def _resolve(self, fileids, categories, textids=None):
+        tmp = None
+        if len(filter(lambda accessor: accessor is None,
+                      (fileids, categories, textids))) != 1:
+
+            raise ValueError('Specify exactly one of: fileids, '
+                             'categories or textids')
+
+        if fileids is not None:
+            return fileids, None
+
+        if categories is not None:
+            return self.fileids(categories), None
+
+        if textids is not None:
+            if isinstance(textids, string_types):
+                textids = [textids]
+            files = sum((self._t2f[t] for t in textids), [])
+            tdict = dict()
+            for f in files:
+                tdict[f] = (set(self._f2t[f]) & set(textids))
+            return files, tdict
+
+    def decode_tag(self, tag):
+        # to be implemented
+        return tag
+
+    def textids(self, fileids=None, categories=None):
+        """
+        In the pl196x corpus each category is stored in single
+        file and thus both methods provide identical functionality. In order
+        to accommodate finer granularity, a non-standard textids() method was
+        implemented. All the main functions can be supplied with a list
+        of required chunks---giving much more control to the user.
+        """
+        fileids, _ = self._resolve(fileids, categories)
+        if fileids is None: return sorted(self._t2f)
+
+        if isinstance(fileids, string_types):
+            fileids = [fileids]
+        return sorted(sum((self._f2t[d] for d in fileids), []))
+
+    def words(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, False, False,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, False, False,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def sents(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, True, False,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, True, False,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def paras(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, True, True,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, True, True,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def tagged_words(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, False, False,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, False, False,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def tagged_sents(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, True, False,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, True, False,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def tagged_paras(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, True, True,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, True, True,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def xml(self, fileids=None, categories=None):
+        fileids, _ = self._resolve(fileids, categories)
+        if len(fileids) == 1:
+            return XMLCorpusReader.xml(self, fileids[0])
+        else:
+            raise TypeError('Expected a single file')
+
+    def raw(self, fileids=None, categories=None):
+        fileids, _ = self._resolve(fileids, categories)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/plaintext.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Plaintext Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 #         Nitin Madnani <nmadnani@umiacs.umd.edu>
@@ -11,10 +11,10 @@
 A reader for corpora that consist of plaintext documents.
 """
 
+from six import string_types
 import codecs
 
 import nltk.data
-from nltk.compat import string_types
 from nltk.tokenize import *
 
 from nltk.corpus.reader.util import *
@@ -75,7 +75,7 @@ class PlaintextCorpusReader(CorpusReader):
         for f in fileids:
             _fin = self.open(f)
             raw_texts.append(_fin.read())
-            _fin.close() 
+            _fin.close()
         return concat(raw_texts)
 
     def words(self, fileids=None):
@@ -230,4 +230,3 @@ class EuroparlCorpusReader(PlaintextCorpusReader):
 
     def paras(self, fileids=None):
         raise NotImplementedError('The Europarl corpus reader does not support paragraphs. Please use chapters() instead.')
-

{nltk-3.2.1 → nltk-3.3}/nltk/corpus/reader/ppattach.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: PP Attachment Corpus Reader
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -39,6 +39,8 @@ of the author.
 """
 from __future__ import unicode_literals
 
+from six import string_types
+
 from nltk import compat
 from nltk.corpus.reader.util import *
 from nltk.corpus.reader.api import *
@@ -76,7 +78,7 @@ class PPAttachmentCorpusReader(CorpusReader):
 
     def raw(self, fileids=None):
         if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
+        elif isinstance(fileids, string_types): fileids = [fileids]
         return concat([self.open(f).read() for f in fileids])
 
     def _read_tuple_block(self, stream):
@@ -92,4 +94,3 @@ class PPAttachmentCorpusReader(CorpusReader):
             return [PPAttachment(*line.split())]
         else:
             return []
-