nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/crubadan.py

@@ -1,116 +1,116 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: An Crubadan N-grams Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Avital Pekker <avital.pekker@utoronto.ca>
-#
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-An NLTK interface for the n-gram statistics gathered from
-the corpora for each language using An Crubadan.
-
-There are multiple potential applications for the data but
-this reader was created with the goal of using it in the
-context of language identification.
-
-For details about An Crubadan, this data, and its potential uses, see:
-http://borel.slu.edu/crubadan/index.html
-"""
-
-from __future__ import print_function, unicode_literals
-
-import re
-from nltk.compat import PY3
-from os import path
-from nltk.corpus.reader import CorpusReader
-from nltk.probability import FreqDist
-from nltk.data import ZipFilePathPointer
-
-class CrubadanCorpusReader(CorpusReader):
-    """
-    A corpus reader used to access language An Crubadan n-gram files.
-    """
-    
-    _LANG_MAPPER_FILE = 'table.txt'
-    _all_lang_freq = {}
-    
-    def __init__(self, root, fileids, encoding='utf8', tagset=None):
-        super(CrubadanCorpusReader, self).__init__(root, fileids, encoding='utf8')
-        self._lang_mapping_data = []
-        self._load_lang_mapping_data()
-        
-    def lang_freq(self, lang):
-        ''' Return n-gram FreqDist for a specific language
-            given ISO 639-3 language code '''
-        
-        if lang not in self._all_lang_freq:
-            self._all_lang_freq[lang] = self._load_lang_ngrams(lang)
-
-        return self._all_lang_freq[lang]
-    
-    def langs(self):
-        ''' Return a list of supported languages as ISO 639-3 codes '''
-        return [row[1] for row in self._lang_mapping_data]
-            
-    def iso_to_crubadan(self, lang):
-        ''' Return internal Crubadan code based on ISO 639-3 code '''
-        for i in self._lang_mapping_data:
-            if i[1].lower() == lang.lower():
-                return i[0]
-    
-    def crubadan_to_iso(self, lang):
-        ''' Return ISO 639-3 code given internal Crubadan code '''
-        for i in self._lang_mapping_data:
-            if i[0].lower() == lang.lower():
-                return i[1]
-    
-    def _load_lang_mapping_data(self):
-        ''' Load language mappings between codes and description from table.txt '''
-        if isinstance(self.root, ZipFilePathPointer):
-            raise RuntimeError("Please install the 'crubadan' corpus first, use nltk.download()")
-        
-        mapper_file = path.join(self.root, self._LANG_MAPPER_FILE)
-        if self._LANG_MAPPER_FILE not in self.fileids():
-            raise RuntimeError("Could not find language mapper file: " + mapper_file)
-
-        if PY3:
-            raw = open(mapper_file, 'r', encoding='utf-8').read().strip()
-        else:
-            raw = open(mapper_file, 'rU').read().decode('utf-8').strip()
-
-        self._lang_mapping_data = [row.split('\t') for row in raw.split('\n')]
-        
-    def _load_lang_ngrams(self, lang):
-        ''' Load single n-gram language file given the ISO 639-3 language code
-            and return its FreqDist '''
-
-        if lang not in self.langs():
-            raise RuntimeError("Unsupported language.")
-
-        crubadan_code = self.iso_to_crubadan(lang)
-        ngram_file = path.join(self.root, crubadan_code + '-3grams.txt')
-
-        if not path.isfile(ngram_file):
-            raise Runtime("No N-gram file found for requested language.")
-
-        counts = FreqDist()
-        if PY3:
-            f = open(ngram_file, 'r', encoding='utf-8')
-        else:
-            f = open(ngram_file, 'rU')
-
-        for line in f:
-            if PY3:
-                data = line.split(' ')
-            else:
-                data = line.decode('utf8').split(' ')
-
-            ngram = data[1].strip('\n')
-            freq = int(data[0])
-            
-            counts[ngram] = freq
-            
-        return counts
-        
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: An Crubadan N-grams Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Avital Pekker <avital.pekker@utoronto.ca>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+An NLTK interface for the n-gram statistics gathered from
+the corpora for each language using An Crubadan.
+
+There are multiple potential applications for the data but
+this reader was created with the goal of using it in the
+context of language identification.
+
+For details about An Crubadan, this data, and its potential uses, see:
+http://borel.slu.edu/crubadan/index.html
+"""
+
+from __future__ import print_function, unicode_literals
+
+import re
+from nltk.compat import PY3
+from os import path
+from nltk.corpus.reader import CorpusReader
+from nltk.probability import FreqDist
+from nltk.data import ZipFilePathPointer
+
+class CrubadanCorpusReader(CorpusReader):
+    """
+    A corpus reader used to access language An Crubadan n-gram files.
+    """
+    
+    _LANG_MAPPER_FILE = 'table.txt'
+    _all_lang_freq = {}
+    
+    def __init__(self, root, fileids, encoding='utf8', tagset=None):
+        super(CrubadanCorpusReader, self).__init__(root, fileids, encoding='utf8')
+        self._lang_mapping_data = []
+        self._load_lang_mapping_data()
+        
+    def lang_freq(self, lang):
+        ''' Return n-gram FreqDist for a specific language
+            given ISO 639-3 language code '''
+        
+        if lang not in self._all_lang_freq:
+            self._all_lang_freq[lang] = self._load_lang_ngrams(lang)
+
+        return self._all_lang_freq[lang]
+    
+    def langs(self):
+        ''' Return a list of supported languages as ISO 639-3 codes '''
+        return [row[1] for row in self._lang_mapping_data]
+            
+    def iso_to_crubadan(self, lang):
+        ''' Return internal Crubadan code based on ISO 639-3 code '''
+        for i in self._lang_mapping_data:
+            if i[1].lower() == lang.lower():
+                return i[0]
+    
+    def crubadan_to_iso(self, lang):
+        ''' Return ISO 639-3 code given internal Crubadan code '''
+        for i in self._lang_mapping_data:
+            if i[0].lower() == lang.lower():
+                return i[1]
+    
+    def _load_lang_mapping_data(self):
+        ''' Load language mappings between codes and description from table.txt '''
+        if isinstance(self.root, ZipFilePathPointer):
+            raise RuntimeError("Please install the 'crubadan' corpus first, use nltk.download()")
+        
+        mapper_file = path.join(self.root, self._LANG_MAPPER_FILE)
+        if self._LANG_MAPPER_FILE not in self.fileids():
+            raise RuntimeError("Could not find language mapper file: " + mapper_file)
+
+        if PY3:
+            raw = open(mapper_file, 'r', encoding='utf-8').read().strip()
+        else:
+            raw = open(mapper_file, 'rU').read().decode('utf-8').strip()
+
+        self._lang_mapping_data = [row.split('\t') for row in raw.split('\n')]
+        
+    def _load_lang_ngrams(self, lang):
+        ''' Load single n-gram language file given the ISO 639-3 language code
+            and return its FreqDist '''
+
+        if lang not in self.langs():
+            raise RuntimeError("Unsupported language.")
+
+        crubadan_code = self.iso_to_crubadan(lang)
+        ngram_file = path.join(self.root, crubadan_code + '-3grams.txt')
+
+        if not path.isfile(ngram_file):
+            raise RuntimeError("No N-gram file found for requested language.")
+
+        counts = FreqDist()
+        if PY3:
+            f = open(ngram_file, 'r', encoding='utf-8')
+        else:
+            f = open(ngram_file, 'rU')
+
+        for line in f:
+            if PY3:
+                data = line.split(' ')
+            else:
+                data = line.decode('utf8').split(' ')
+
+            ngram = data[1].strip('\n')
+            freq = int(data[0])
+            
+            counts[ngram] = freq
+            
+        return counts
+        

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/dependency.py

@@ -1,101 +1,101 @@
-# Natural Language Toolkit: Dependency Corpus Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Kepa Sarasola <kepa.sarasola@ehu.es>
-#         Iker Manterola <returntothehangar@hotmail.com>
-#
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-import codecs
-
-from nltk.parse import DependencyGraph
-from nltk.tokenize import *
-
-from nltk.corpus.reader.util import *
-from nltk.corpus.reader.api import *
-
-class DependencyCorpusReader(SyntaxCorpusReader):
-
-    def __init__(self, root, fileids, encoding='utf8',
-                 word_tokenizer=TabTokenizer(),
-                 sent_tokenizer=RegexpTokenizer('\n', gaps=True),
-                 para_block_reader=read_blankline_block):
-
-        CorpusReader.__init__(self, root, fileids, encoding)
-
-    #########################################################
-
-    def raw(self, fileids=None):
-        """
-        :return: the given file(s) as a single string.
-        :rtype: str
-        """
-        result = []
-        for fileid, encoding in self.abspaths(fileids, include_encoding=True):
-            if isinstance(fileid, PathPointer):
-                result.append(fileid.open(encoding=encoding).read())
-            else:
-                with codecs.open(fileid, "r", encoding) as fp:
-                    result.append(fp.read())
-        return concat(result)
-
-    def words(self, fileids=None):
-        return concat([DependencyCorpusView(fileid, False, False, False, encoding=enc)
-                       for fileid, enc in self.abspaths(fileids, include_encoding=True)])
-
-    def tagged_words(self, fileids=None):
-        return concat([DependencyCorpusView(fileid, True, False, False, encoding=enc)
-                       for fileid, enc in self.abspaths(fileids, include_encoding=True)])
-
-    def sents(self, fileids=None):
-        return concat([DependencyCorpusView(fileid, False, True, False, encoding=enc)
-                       for fileid, enc in self.abspaths(fileids, include_encoding=True)])
-
-    def tagged_sents(self, fileids=None):
-            return concat([DependencyCorpusView(fileid, True, True, False, encoding=enc)
-                           for fileid, enc in self.abspaths(fileids, include_encoding=True)])
-
-    def parsed_sents(self, fileids=None):
-        sents=concat([DependencyCorpusView(fileid, False, True, True, encoding=enc)
-                      for fileid, enc in self.abspaths(fileids, include_encoding=True)])
-        return [DependencyGraph(sent) for sent in sents]
-
-
-class DependencyCorpusView(StreamBackedCorpusView):
-    _DOCSTART = '-DOCSTART- -DOCSTART- O\n' #dokumentu hasiera definitzen da
-
-    def __init__(self, corpus_file, tagged, group_by_sent, dependencies,
-                 chunk_types=None, encoding='utf8'):
-        self._tagged = tagged
-        self._dependencies = dependencies
-        self._group_by_sent = group_by_sent
-        self._chunk_types = chunk_types
-        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
-
-    def read_block(self, stream):
-        # Read the next sentence.
-        sent = read_blankline_block(stream)[0].strip()
-        # Strip off the docstart marker, if present.
-        if sent.startswith(self._DOCSTART):
-            sent = sent[len(self._DOCSTART):].lstrip()
-
-        # extract word and tag from any of the formats
-        if not self._dependencies:
-            lines = [line.split('\t') for line in sent.split('\n')]
-            if len(lines[0]) == 3 or len(lines[0]) == 4:
-                sent = [(line[0], line[1]) for line in lines]
-            elif len(lines[0]) == 10:
-                sent = [(line[1], line[4]) for line in lines]
-            else:
-                raise ValueError('Unexpected number of fields in dependency tree file')
-
-            # discard tags if they weren't requested
-            if not self._tagged:
-                sent = [word for (word, tag) in sent]
-
-        # Return the result.
-        if self._group_by_sent:
-            return [sent]
-        else:
-            return list(sent)
+# Natural Language Toolkit: Dependency Corpus Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Kepa Sarasola <kepa.sarasola@ehu.es>
+#         Iker Manterola <returntothehangar@hotmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import codecs
+
+from nltk.parse import DependencyGraph
+from nltk.tokenize import *
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class DependencyCorpusReader(SyntaxCorpusReader):
+
+    def __init__(self, root, fileids, encoding='utf8',
+                 word_tokenizer=TabTokenizer(),
+                 sent_tokenizer=RegexpTokenizer('\n', gaps=True),
+                 para_block_reader=read_blankline_block):
+
+        CorpusReader.__init__(self, root, fileids, encoding)
+
+    #########################################################
+
+    def raw(self, fileids=None):
+        """
+        :return: the given file(s) as a single string.
+        :rtype: str
+        """
+        result = []
+        for fileid, encoding in self.abspaths(fileids, include_encoding=True):
+            if isinstance(fileid, PathPointer):
+                result.append(fileid.open(encoding=encoding).read())
+            else:
+                with codecs.open(fileid, "r", encoding) as fp:
+                    result.append(fp.read())
+        return concat(result)
+
+    def words(self, fileids=None):
+        return concat([DependencyCorpusView(fileid, False, False, False, encoding=enc)
+                       for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+
+    def tagged_words(self, fileids=None):
+        return concat([DependencyCorpusView(fileid, True, False, False, encoding=enc)
+                       for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+
+    def sents(self, fileids=None):
+        return concat([DependencyCorpusView(fileid, False, True, False, encoding=enc)
+                       for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+
+    def tagged_sents(self, fileids=None):
+            return concat([DependencyCorpusView(fileid, True, True, False, encoding=enc)
+                           for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+
+    def parsed_sents(self, fileids=None):
+        sents=concat([DependencyCorpusView(fileid, False, True, True, encoding=enc)
+                      for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+        return [DependencyGraph(sent) for sent in sents]
+
+
+class DependencyCorpusView(StreamBackedCorpusView):
+    _DOCSTART = '-DOCSTART- -DOCSTART- O\n' #dokumentu hasiera definitzen da
+
+    def __init__(self, corpus_file, tagged, group_by_sent, dependencies,
+                 chunk_types=None, encoding='utf8'):
+        self._tagged = tagged
+        self._dependencies = dependencies
+        self._group_by_sent = group_by_sent
+        self._chunk_types = chunk_types
+        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
+
+    def read_block(self, stream):
+        # Read the next sentence.
+        sent = read_blankline_block(stream)[0].strip()
+        # Strip off the docstart marker, if present.
+        if sent.startswith(self._DOCSTART):
+            sent = sent[len(self._DOCSTART):].lstrip()
+
+        # extract word and tag from any of the formats
+        if not self._dependencies:
+            lines = [line.split('\t') for line in sent.split('\n')]
+            if len(lines[0]) == 3 or len(lines[0]) == 4:
+                sent = [(line[0], line[1]) for line in lines]
+            elif len(lines[0]) == 10:
+                sent = [(line[1], line[4]) for line in lines]
+            else:
+                raise ValueError('Unexpected number of fields in dependency tree file')
+
+            # discard tags if they weren't requested
+            if not self._tagged:
+                sent = [word for (word, tag) in sent]
+
+        # Return the result.
+        if self._group_by_sent:
+            return [sent]
+        else:
+            return list(sent)