nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/panlex_lite.py

@@ -1,165 +1,165 @@
-# Natural Language Toolkit: PanLex Corpus Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: David Kamholz <kamholz@panlex.org>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-CorpusReader for PanLex Lite, a stripped down version of PanLex distributed
-as an SQLite database. See the README.txt in the panlex_lite corpus directory
-for more information on PanLex Lite.
-"""
-
-import os
-import sqlite3
-
-from nltk.corpus.reader.api import CorpusReader
-
-class PanLexLiteCorpusReader(CorpusReader):
-    MEANING_Q = """
-        SELECT dnx2.mn, dnx2.uq, dnx2.ap, dnx2.ui, ex2.tt, ex2.lv
-        FROM dnx
-        JOIN ex ON (ex.ex = dnx.ex)
-        JOIN dnx dnx2 ON (dnx2.mn = dnx.mn)
-        JOIN ex ex2 ON (ex2.ex = dnx2.ex)
-        WHERE dnx.ex != dnx2.ex AND ex.tt = ? AND ex.lv = ?
-        ORDER BY dnx2.uq DESC
-    """
-
-    TRANSLATION_Q = """
-        SELECT s.tt, sum(s.uq) AS trq FROM (
-            SELECT ex2.tt, max(dnx.uq) AS uq
-            FROM dnx
-            JOIN ex ON (ex.ex = dnx.ex)
-            JOIN dnx dnx2 ON (dnx2.mn = dnx.mn)
-            JOIN ex ex2 ON (ex2.ex = dnx2.ex)
-            WHERE dnx.ex != dnx2.ex AND ex.lv = ? AND ex.tt = ? AND ex2.lv = ?
-            GROUP BY ex2.tt, dnx.ui
-        ) s
-        GROUP BY s.tt
-        ORDER BY trq DESC, s.tt
-    """
-
-    def __init__(self, root):
-        self._c = sqlite3.connect(os.path.join(root, 'db.sqlite')).cursor()
-
-        self._uid_lv = {}
-        self._lv_uid = {}
-
-        for row in self._c.execute('SELECT uid, lv FROM lv'):
-            self._uid_lv[row[0]] = row[1]
-            self._lv_uid[row[1]] = row[0]
-
-    def language_varieties(self, lc=None):
-        """
-        Return a list of PanLex language varieties.
-
-        :param lc: ISO 639 alpha-3 code. If specified, filters returned varieties
-            by this code. If unspecified, all varieties are returned.
-        :return: the specified language varieties as a list of tuples. The first
-            element is the language variety's seven-character uniform identifier,
-            and the second element is its default name.
-        :rtype: list(tuple)
-        """
-
-        if lc == None:
-            return self._c.execute('SELECT uid, tt FROM lv ORDER BY uid').fetchall()
-        else:
-            return self._c.execute('SELECT uid, tt FROM lv WHERE lc = ? ORDER BY uid', (lc,)).fetchall()
-
-    def meanings(self, expr_uid, expr_tt):
-        """
-        Return a list of meanings for an expression.
-
-        :param expr_uid: the expression's language variety, as a seven-character
-            uniform identifier.
-        :param expr_tt: the expression's text.
-        :return: a list of Meaning objects.
-        :rtype: list(Meaning)
-        """
-
-        expr_lv = self._uid_lv[expr_uid]
-
-        mn_info = {}
-
-        for i in self._c.execute(self.MEANING_Q, (expr_tt, expr_lv)):
-            mn = i[0]
-            uid = self._lv_uid[i[5]]
-
-            if not mn in mn_info:
-                mn_info[mn] = { 'uq': i[1], 'ap': i[2], 'ui': i[3], 'ex': { expr_uid: [expr_tt] } }
-
-            if not uid in mn_info[mn]['ex']:
-                mn_info[mn]['ex'][uid] = []
-
-            mn_info[mn]['ex'][uid].append(i[4])
-
-        return [ Meaning(mn, mn_info[mn]) for mn in mn_info ]
-
-    def translations(self, from_uid, from_tt, to_uid):
-        """
-        Return a list of translations for an expression into a single language
-            variety.
-
-        :param from_uid: the source expression's language variety, as a
-            seven-character uniform identifier.
-        :param from_tt: the source expression's text.
-        :param to_uid: the target language variety, as a seven-character
-            uniform identifier.
-        :return a list of translation tuples. The first element is the expression 
-            text and the second element is the translation quality.
-        :rtype: list(tuple)
-        """
-
-        from_lv = self._uid_lv[from_uid]
-        to_lv = self._uid_lv[to_uid]
-
-        return self._c.execute(self.TRANSLATION_Q, (from_lv, from_tt, to_lv)).fetchall()
-
-class Meaning(dict):
-    """
-    Represents a single PanLex meaning. A meaning is a translation set derived
-    from a single source.
-    """
-
-    def __init__(self, mn, attr):
-        super(Meaning, self).__init__(**attr)
-        self['mn'] = mn
-
-    def id(self):
-        """
-        :return: the meaning's id.
-        :rtype: int
-        """
-        return self['mn']
-
-    def quality(self):
-        """
-        :return: the meaning's source's quality (0=worst, 9=best).
-        :rtype: int
-        """
-        return self['uq']
-
-    def source(self):
-        """
-        :return: the meaning's source id.
-        :rtype: int
-        """
-        return self['ap']
-
-    def source_group(self):
-        """
-        :return: the meaning's source group id.
-        :rtype: int
-        """
-        return self['ui']
-
-    def expressions(self):
-        """
-        :return: the meaning's expressions as a dictionary whose keys are language
-            variety uniform identifiers and whose values are lists of expression
-            texts.
-        :rtype: dict
-        """
-        return self['ex']
+# Natural Language Toolkit: PanLex Corpus Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: David Kamholz <kamholz@panlex.org>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+CorpusReader for PanLex Lite, a stripped down version of PanLex distributed
+as an SQLite database. See the README.txt in the panlex_lite corpus directory
+for more information on PanLex Lite.
+"""
+
+import os
+import sqlite3
+
+from nltk.corpus.reader.api import CorpusReader
+
+class PanLexLiteCorpusReader(CorpusReader):
+    MEANING_Q = """
+        SELECT dnx2.mn, dnx2.uq, dnx2.ap, dnx2.ui, ex2.tt, ex2.lv
+        FROM dnx
+        JOIN ex ON (ex.ex = dnx.ex)
+        JOIN dnx dnx2 ON (dnx2.mn = dnx.mn)
+        JOIN ex ex2 ON (ex2.ex = dnx2.ex)
+        WHERE dnx.ex != dnx2.ex AND ex.tt = ? AND ex.lv = ?
+        ORDER BY dnx2.uq DESC
+    """
+
+    TRANSLATION_Q = """
+        SELECT s.tt, sum(s.uq) AS trq FROM (
+            SELECT ex2.tt, max(dnx.uq) AS uq
+            FROM dnx
+            JOIN ex ON (ex.ex = dnx.ex)
+            JOIN dnx dnx2 ON (dnx2.mn = dnx.mn)
+            JOIN ex ex2 ON (ex2.ex = dnx2.ex)
+            WHERE dnx.ex != dnx2.ex AND ex.lv = ? AND ex.tt = ? AND ex2.lv = ?
+            GROUP BY ex2.tt, dnx.ui
+        ) s
+        GROUP BY s.tt
+        ORDER BY trq DESC, s.tt
+    """
+
+    def __init__(self, root):
+        self._c = sqlite3.connect(os.path.join(root, 'db.sqlite')).cursor()
+
+        self._uid_lv = {}
+        self._lv_uid = {}
+
+        for row in self._c.execute('SELECT uid, lv FROM lv'):
+            self._uid_lv[row[0]] = row[1]
+            self._lv_uid[row[1]] = row[0]
+
+    def language_varieties(self, lc=None):
+        """
+        Return a list of PanLex language varieties.
+
+        :param lc: ISO 639 alpha-3 code. If specified, filters returned varieties
+            by this code. If unspecified, all varieties are returned.
+        :return: the specified language varieties as a list of tuples. The first
+            element is the language variety's seven-character uniform identifier,
+            and the second element is its default name.
+        :rtype: list(tuple)
+        """
+
+        if lc == None:
+            return self._c.execute('SELECT uid, tt FROM lv ORDER BY uid').fetchall()
+        else:
+            return self._c.execute('SELECT uid, tt FROM lv WHERE lc = ? ORDER BY uid', (lc,)).fetchall()
+
+    def meanings(self, expr_uid, expr_tt):
+        """
+        Return a list of meanings for an expression.
+
+        :param expr_uid: the expression's language variety, as a seven-character
+            uniform identifier.
+        :param expr_tt: the expression's text.
+        :return: a list of Meaning objects.
+        :rtype: list(Meaning)
+        """
+
+        expr_lv = self._uid_lv[expr_uid]
+
+        mn_info = {}
+
+        for i in self._c.execute(self.MEANING_Q, (expr_tt, expr_lv)):
+            mn = i[0]
+            uid = self._lv_uid[i[5]]
+
+            if not mn in mn_info:
+                mn_info[mn] = { 'uq': i[1], 'ap': i[2], 'ui': i[3], 'ex': { expr_uid: [expr_tt] } }
+
+            if not uid in mn_info[mn]['ex']:
+                mn_info[mn]['ex'][uid] = []
+
+            mn_info[mn]['ex'][uid].append(i[4])
+
+        return [ Meaning(mn, mn_info[mn]) for mn in mn_info ]
+
+    def translations(self, from_uid, from_tt, to_uid):
+        """
+        Return a list of translations for an expression into a single language
+            variety.
+
+        :param from_uid: the source expression's language variety, as a
+            seven-character uniform identifier.
+        :param from_tt: the source expression's text.
+        :param to_uid: the target language variety, as a seven-character
+            uniform identifier.
+        :return a list of translation tuples. The first element is the expression 
+            text and the second element is the translation quality.
+        :rtype: list(tuple)
+        """
+
+        from_lv = self._uid_lv[from_uid]
+        to_lv = self._uid_lv[to_uid]
+
+        return self._c.execute(self.TRANSLATION_Q, (from_lv, from_tt, to_lv)).fetchall()
+
+class Meaning(dict):
+    """
+    Represents a single PanLex meaning. A meaning is a translation set derived
+    from a single source.
+    """
+
+    def __init__(self, mn, attr):
+        super(Meaning, self).__init__(**attr)
+        self['mn'] = mn
+
+    def id(self):
+        """
+        :return: the meaning's id.
+        :rtype: int
+        """
+        return self['mn']
+
+    def quality(self):
+        """
+        :return: the meaning's source's quality (0=worst, 9=best).
+        :rtype: int
+        """
+        return self['uq']
+
+    def source(self):
+        """
+        :return: the meaning's source id.
+        :rtype: int
+        """
+        return self['ap']
+
+    def source_group(self):
+        """
+        :return: the meaning's source group id.
+        :rtype: int
+        """
+        return self['ui']
+
+    def expressions(self):
+        """
+        :return: the meaning's expressions as a dictionary whose keys are language
+            variety uniform identifiers and whose values are lists of expression
+            texts.
+        :rtype: dict
+        """
+        return self['ex']

nltk-3.3/nltk/corpus/reader/pl196x.py

@@ -0,0 +1,292 @@
+# Natural Language Toolkit:
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Piotr Kasprzyk <p.j.kasprzyk@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from six import string_types
+
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.xmldocs import XMLCorpusReader
+
+
+PARA = re.compile(r'<p(?: [^>]*){0,1}>(.*?)</p>')
+SENT = re.compile(r'<s(?: [^>]*){0,1}>(.*?)</s>')
+
+TAGGEDWORD = re.compile(r'<([wc](?: [^>]*){0,1}>)(.*?)</[wc]>')
+WORD = re.compile(r'<[wc](?: [^>]*){0,1}>(.*?)</[wc]>')
+
+TYPE = re.compile(r'type="(.*?)"')
+ANA = re.compile(r'ana="(.*?)"')
+
+TEXTID = re.compile(r'text id="(.*?)"')
+
+
+class TEICorpusView(StreamBackedCorpusView):
+    def __init__(self, corpus_file,
+                 tagged, group_by_sent, group_by_para,
+                 tagset=None, head_len=0, textids=None):
+
+        self._tagged = tagged
+        self._textids = textids
+
+        self._group_by_sent = group_by_sent
+        self._group_by_para = group_by_para
+        # WARNING -- skip header
+        StreamBackedCorpusView.__init__(self, corpus_file, startpos=head_len)
+
+    _pagesize = 4096
+
+    def read_block(self, stream):
+        block = stream.readlines(self._pagesize)
+        block = concat(block)
+        while (block.count('<text id') > block.count('</text>')) \
+                or block.count('<text id') == 0:
+            tmp = stream.readline()
+            if len(tmp) <= 0:
+                break
+            block += tmp
+
+        block = block.replace('\n', '')
+
+        textids = TEXTID.findall(block)
+        if self._textids:
+            for tid in textids:
+                if tid not in self._textids:
+                    beg = block.find(tid) - 1
+                    end = block[beg:].find('</text>') + len('</text>')
+                    block = block[:beg] + block[beg + end:]
+
+        output = []
+        for para_str in PARA.findall(block):
+            para = []
+            for sent_str in SENT.findall(para_str):
+                if not self._tagged:
+                    sent = WORD.findall(sent_str)
+                else:
+                    sent = list(
+                        map(self._parse_tag, TAGGEDWORD.findall(sent_str)))
+                if self._group_by_sent:
+                    para.append(sent)
+                else:
+                    para.extend(sent)
+            if self._group_by_para:
+                output.append(para)
+            else:
+                output.extend(para)
+        return output
+
+    def _parse_tag(self, tag_word_tuple):
+        (tag, word) = tag_word_tuple
+        if tag.startswith('w'):
+            tag = ANA.search(tag).group(1)
+        else:  # tag.startswith('c')
+            tag = TYPE.search(tag).group(1)
+        return word, tag
+
+
+class Pl196xCorpusReader(CategorizedCorpusReader, XMLCorpusReader):
+    head_len = 2770
+
+    def __init__(self, *args, **kwargs):
+        if 'textid_file' in kwargs:
+            self._textids = kwargs['textid_file']
+        else:
+            self._textids = None
+
+        XMLCorpusReader.__init__(self, *args)
+        CategorizedCorpusReader.__init__(self, kwargs)
+
+        self._init_textids()
+
+    def _init_textids(self):
+        self._f2t = defaultdict(list)
+        self._t2f = defaultdict(list)
+        if self._textids is not None:
+            with open(self._textids) as fp:
+                for line in fp:
+                    line = line.strip()
+                    file_id, text_ids = line.split(' ', 1)
+                    if file_id not in self.fileids():
+                        raise ValueError(
+                            'In text_id mapping file %s: %s not found'
+                            % (self._textids, file_id)
+                        )
+                    for text_id in text_ids.split(self._delimiter):
+                        self._add_textids(file_id, text_id)
+
+    def _add_textids(self, file_id, text_id):
+        self._f2t[file_id].append(text_id)
+        self._t2f[text_id].append(file_id)
+
+    def _resolve(self, fileids, categories, textids=None):
+        tmp = None
+        if len(filter(lambda accessor: accessor is None,
+                      (fileids, categories, textids))) != 1:
+
+            raise ValueError('Specify exactly one of: fileids, '
+                             'categories or textids')
+
+        if fileids is not None:
+            return fileids, None
+
+        if categories is not None:
+            return self.fileids(categories), None
+
+        if textids is not None:
+            if isinstance(textids, string_types):
+                textids = [textids]
+            files = sum((self._t2f[t] for t in textids), [])
+            tdict = dict()
+            for f in files:
+                tdict[f] = (set(self._f2t[f]) & set(textids))
+            return files, tdict
+
+    def decode_tag(self, tag):
+        # to be implemented
+        return tag
+
+    def textids(self, fileids=None, categories=None):
+        """
+        In the pl196x corpus each category is stored in single
+        file and thus both methods provide identical functionality. In order
+        to accommodate finer granularity, a non-standard textids() method was
+        implemented. All the main functions can be supplied with a list
+        of required chunks---giving much more control to the user.
+        """
+        fileids, _ = self._resolve(fileids, categories)
+        if fileids is None: return sorted(self._t2f)
+
+        if isinstance(fileids, string_types):
+            fileids = [fileids]
+        return sorted(sum((self._f2t[d] for d in fileids), []))
+
+    def words(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, False, False,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, False, False,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def sents(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, True, False,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, True, False,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def paras(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, True, True,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         False, True, True,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def tagged_words(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, False, False,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, False, False,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def tagged_sents(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, True, False,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, True, False,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def tagged_paras(self, fileids=None, categories=None, textids=None):
+        fileids, textids = self._resolve(fileids, categories, textids)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+
+        if textids:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, True, True,
+                                         head_len=self.head_len,
+                                         textids=textids[fileid])
+                           for fileid in fileids])
+        else:
+            return concat([TEICorpusView(self.abspath(fileid),
+                                         True, True, True,
+                                         head_len=self.head_len)
+                           for fileid in fileids])
+
+    def xml(self, fileids=None, categories=None):
+        fileids, _ = self._resolve(fileids, categories)
+        if len(fileids) == 1:
+            return XMLCorpusReader.xml(self, fileids[0])
+        else:
+            raise TypeError('Expected a single file')
+
+    def raw(self, fileids=None, categories=None):
+        fileids, _ = self._resolve(fileids, categories)
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])