nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sentiwordnet.py

@@ -1,137 +1,137 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: SentiWordNet
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Christopher Potts <cgpotts@stanford.edu>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-An NLTK interface for SentiWordNet
-
-SentiWordNet is a lexical resource for opinion mining.
-SentiWordNet assigns to each synset of WordNet three
-sentiment scores: positivity, negativity, and objectivity.
-
-For details about SentiWordNet see:
-http://sentiwordnet.isti.cnr.it/
-
-    >>> from nltk.corpus import sentiwordnet as swn
-    >>> print(swn.senti_synset('breakdown.n.03'))
-    <breakdown.n.03: PosScore=0.0 NegScore=0.25>
-    >>> list(swn.senti_synsets('slow'))
-    [SentiSynset('decelerate.v.01'), SentiSynset('slow.v.02'),
-    SentiSynset('slow.v.03'), SentiSynset('slow.a.01'),
-    SentiSynset('slow.a.02'), SentiSynset('dense.s.04'),
-    SentiSynset('slow.a.04'), SentiSynset('boring.s.01'),
-    SentiSynset('dull.s.08'), SentiSynset('slowly.r.01'),
-    SentiSynset('behind.r.03')]
-    >>> happy = swn.senti_synsets('happy', 'a')
-    >>> happy0 = list(happy)[0]
-    >>> happy0.pos_score()
-    0.875
-    >>> happy0.neg_score()
-    0.0
-    >>> happy0.obj_score()
-    0.125
-"""
-
-import re
-from nltk.compat import python_2_unicode_compatible
-from nltk.corpus.reader import CorpusReader
-
-@python_2_unicode_compatible
-class SentiWordNetCorpusReader(CorpusReader):
-    def __init__(self, root, fileids, encoding='utf-8'):
-        """
-        Construct a new SentiWordNet Corpus Reader, using data from
-   	the specified file.
-        """        
-        super(SentiWordNetCorpusReader, self).__init__(root, fileids,
-                                                  encoding=encoding)
-        if len(self._fileids) != 1:
-            raise ValueError('Exactly one file must be specified')
-        self._db = {}
-        self._parse_src_file()
-
-    def _parse_src_file(self):
-        lines = self.open(self._fileids[0]).read().splitlines()
-        lines = filter((lambda x : not re.search(r"^\s*#", x)), lines)
-        for i, line in enumerate(lines):
-            fields = [field.strip() for field in re.split(r"\t+", line)]
-            try:            
-                pos, offset, pos_score, neg_score, synset_terms, gloss = fields
-            except:
-                raise ValueError('Line %s formatted incorrectly: %s\n' % (i, line))
-            if pos and offset:
-                offset = int(offset)
-                self._db[(pos, offset)] = (float(pos_score), float(neg_score))
-
-    def senti_synset(self, *vals):        
-        from nltk.corpus import wordnet as wn
-        if tuple(vals) in self._db:
-            pos_score, neg_score = self._db[tuple(vals)]
-            pos, offset = vals
-            if pos == 's':
-                pos = 'a'
-            synset = wn._synset_from_pos_and_offset(pos, offset)
-            return SentiSynset(pos_score, neg_score, synset)
-        else:
-            synset = wn.synset(vals[0])
-            pos = synset.pos()
-            if pos == 's':
-                pos = 'a'
-            offset = synset.offset()
-            if (pos, offset) in self._db:
-                pos_score, neg_score = self._db[(pos, offset)]
-                return SentiSynset(pos_score, neg_score, synset)
-            else:
-                return None
-
-    def senti_synsets(self, string, pos=None):
-        from nltk.corpus import wordnet as wn
-        sentis = []
-        synset_list = wn.synsets(string, pos)
-        for synset in synset_list:
-            sentis.append(self.senti_synset(synset.name()))
-        sentis = filter(lambda x : x, sentis)
-        return sentis
-
-    def all_senti_synsets(self):
-        from nltk.corpus import wordnet as wn
-        for key, fields in self._db.items():
-            pos, offset = key
-            pos_score, neg_score = fields
-            synset = wn._synset_from_pos_and_offset(pos, offset)
-            yield SentiSynset(pos_score, neg_score, synset)
-
-
-@python_2_unicode_compatible
-class SentiSynset(object):
-    def __init__(self, pos_score, neg_score, synset):
-        self._pos_score = pos_score
-        self._neg_score = neg_score
-        self._obj_score = 1.0 - (self._pos_score + self._neg_score)
-        self.synset = synset
-
-    def pos_score(self):
-        return self._pos_score
-
-    def neg_score(self):
-        return self._neg_score
-
-    def obj_score(self):
-        return self._obj_score
-
-    def __str__(self):
-        """Prints just the Pos/Neg scores for now."""
-        s = "<"
-        s += self.synset.name() + ": "
-        s += "PosScore=%s " % self._pos_score
-        s += "NegScore=%s" % self._neg_score
-        s += ">"
-        return s
-
-    def __repr__(self):
-        return "Senti" + repr(self.synset)
-                    
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: SentiWordNet
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Christopher Potts <cgpotts@stanford.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+An NLTK interface for SentiWordNet
+
+SentiWordNet is a lexical resource for opinion mining.
+SentiWordNet assigns to each synset of WordNet three
+sentiment scores: positivity, negativity, and objectivity.
+
+For details about SentiWordNet see:
+http://sentiwordnet.isti.cnr.it/
+
+    >>> from nltk.corpus import sentiwordnet as swn
+    >>> print(swn.senti_synset('breakdown.n.03'))
+    <breakdown.n.03: PosScore=0.0 NegScore=0.25>
+    >>> list(swn.senti_synsets('slow'))
+    [SentiSynset('decelerate.v.01'), SentiSynset('slow.v.02'),
+    SentiSynset('slow.v.03'), SentiSynset('slow.a.01'),
+    SentiSynset('slow.a.02'), SentiSynset('dense.s.04'),
+    SentiSynset('slow.a.04'), SentiSynset('boring.s.01'),
+    SentiSynset('dull.s.08'), SentiSynset('slowly.r.01'),
+    SentiSynset('behind.r.03')]
+    >>> happy = swn.senti_synsets('happy', 'a')
+    >>> happy0 = list(happy)[0]
+    >>> happy0.pos_score()
+    0.875
+    >>> happy0.neg_score()
+    0.0
+    >>> happy0.obj_score()
+    0.125
+"""
+
+import re
+from nltk.compat import python_2_unicode_compatible
+from nltk.corpus.reader import CorpusReader
+
+@python_2_unicode_compatible
+class SentiWordNetCorpusReader(CorpusReader):
+    def __init__(self, root, fileids, encoding='utf-8'):
+        """
+        Construct a new SentiWordNet Corpus Reader, using data from
+   	the specified file.
+        """        
+        super(SentiWordNetCorpusReader, self).__init__(root, fileids,
+                                                  encoding=encoding)
+        if len(self._fileids) != 1:
+            raise ValueError('Exactly one file must be specified')
+        self._db = {}
+        self._parse_src_file()
+
+    def _parse_src_file(self):
+        lines = self.open(self._fileids[0]).read().splitlines()
+        lines = filter((lambda x : not re.search(r"^\s*#", x)), lines)
+        for i, line in enumerate(lines):
+            fields = [field.strip() for field in re.split(r"\t+", line)]
+            try:            
+                pos, offset, pos_score, neg_score, synset_terms, gloss = fields
+            except:
+                raise ValueError('Line %s formatted incorrectly: %s\n' % (i, line))
+            if pos and offset:
+                offset = int(offset)
+                self._db[(pos, offset)] = (float(pos_score), float(neg_score))
+
+    def senti_synset(self, *vals):        
+        from nltk.corpus import wordnet as wn
+        if tuple(vals) in self._db:
+            pos_score, neg_score = self._db[tuple(vals)]
+            pos, offset = vals
+            if pos == 's':
+                pos = 'a'
+            synset = wn.synset_from_pos_and_offset(pos, offset)
+            return SentiSynset(pos_score, neg_score, synset)
+        else:
+            synset = wn.synset(vals[0])
+            pos = synset.pos()
+            if pos == 's':
+                pos = 'a'
+            offset = synset.offset()
+            if (pos, offset) in self._db:
+                pos_score, neg_score = self._db[(pos, offset)]
+                return SentiSynset(pos_score, neg_score, synset)
+            else:
+                return None
+
+    def senti_synsets(self, string, pos=None):
+        from nltk.corpus import wordnet as wn
+        sentis = []
+        synset_list = wn.synsets(string, pos)
+        for synset in synset_list:
+            sentis.append(self.senti_synset(synset.name()))
+        sentis = filter(lambda x : x, sentis)
+        return sentis
+
+    def all_senti_synsets(self):
+        from nltk.corpus import wordnet as wn
+        for key, fields in self._db.items():
+            pos, offset = key
+            pos_score, neg_score = fields
+            synset = wn.synset_from_pos_and_offset(pos, offset)
+            yield SentiSynset(pos_score, neg_score, synset)
+
+
+@python_2_unicode_compatible
+class SentiSynset(object):
+    def __init__(self, pos_score, neg_score, synset):
+        self._pos_score = pos_score
+        self._neg_score = neg_score
+        self._obj_score = 1.0 - (self._pos_score + self._neg_score)
+        self.synset = synset
+
+    def pos_score(self):
+        return self._pos_score
+
+    def neg_score(self):
+        return self._neg_score
+
+    def obj_score(self):
+        return self._obj_score
+
+    def __str__(self):
+        """Prints just the Pos/Neg scores for now."""
+        s = "<"
+        s += self.synset.name() + ": "
+        s += "PosScore=%s " % self._pos_score
+        s += "NegScore=%s" % self._neg_score
+        s += ">"
+        return s
+
+    def __repr__(self):
+        return "Senti" + repr(self.synset)
+                    

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/sinica_treebank.py

@@ -1,75 +1,75 @@
-# Natural Language Toolkit: Sinica Treebank Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Steven Bird <stevenbird1@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Sinica Treebank Corpus Sample
-
-http://rocling.iis.sinica.edu.tw/CKIP/engversion/treebank.htm
-
-10,000 parsed sentences, drawn from the Academia Sinica Balanced
-Corpus of Modern Chinese.  Parse tree notation is based on
-Information-based Case Grammar.  Tagset documentation is available
-at http://www.sinica.edu.tw/SinicaCorpus/modern_e_wordtype.html
-
-Language and Knowledge Processing Group, Institute of Information
-Science, Academia Sinica
-
-It is distributed with the Natural Language Toolkit under the terms of
-the Creative Commons Attribution-NonCommercial-ShareAlike License
-[http://creativecommons.org/licenses/by-nc-sa/2.5/].
-
-References:
-
-Feng-Yi Chen, Pi-Fang Tsai, Keh-Jiann Chen, and Chu-Ren Huang (1999)
-The Construction of Sinica Treebank. Computational Linguistics and
-Chinese Language Processing, 4, pp 87-104.
-
-Huang Chu-Ren, Keh-Jiann Chen, Feng-Yi Chen, Keh-Jiann Chen, Zhao-Ming
-Gao, and Kuang-Yu Chen. 2000. Sinica Treebank: Design Criteria,
-Annotation Guidelines, and On-line Interface. Proceedings of 2nd
-Chinese Language Processing Workshop, Association for Computational
-Linguistics.
-
-Chen Keh-Jiann and Yu-Ming Hsieh (2004) Chinese Treebanks and Grammar
-Extraction, Proceedings of IJCNLP-04, pp560-565.
-"""
-
-import os
-import re
-
-from nltk.tree import sinica_parse
-from nltk.tag import map_tag
-
-from nltk.corpus.reader.util import *
-from nltk.corpus.reader.api import *
-
-IDENTIFIER = re.compile(r'^#\S+\s')
-APPENDIX = re.compile(r'(?<=\))#.*$')
-TAGWORD = re.compile(r':([^:()|]+):([^:()|]+)')
-WORD = re.compile(r':[^:()|]+:([^:()|]+)')
-
-class SinicaTreebankCorpusReader(SyntaxCorpusReader):
-    """
-    Reader for the sinica treebank.
-    """
-    def _read_block(self, stream):
-        sent = stream.readline()
-        sent = IDENTIFIER.sub('', sent)
-        sent = APPENDIX.sub('', sent)
-        return [sent]
-
-    def _parse(self, sent):
-        return sinica_parse(sent)
-
-    def _tag(self, sent, tagset=None):
-        tagged_sent = [(w,t) for (t,w) in TAGWORD.findall(sent)]
-        if tagset and tagset != self._tagset:
-            tagged_sent = [(w, map_tag(self._tagset, tagset, t)) for (w,t) in tagged_sent]
-        return tagged_sent
-
-    def _word(self, sent):
-        return WORD.findall(sent)
+# Natural Language Toolkit: Sinica Treebank Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Sinica Treebank Corpus Sample
+
+http://rocling.iis.sinica.edu.tw/CKIP/engversion/treebank.htm
+
+10,000 parsed sentences, drawn from the Academia Sinica Balanced
+Corpus of Modern Chinese.  Parse tree notation is based on
+Information-based Case Grammar.  Tagset documentation is available
+at http://www.sinica.edu.tw/SinicaCorpus/modern_e_wordtype.html
+
+Language and Knowledge Processing Group, Institute of Information
+Science, Academia Sinica
+
+It is distributed with the Natural Language Toolkit under the terms of
+the Creative Commons Attribution-NonCommercial-ShareAlike License
+[http://creativecommons.org/licenses/by-nc-sa/2.5/].
+
+References:
+
+Feng-Yi Chen, Pi-Fang Tsai, Keh-Jiann Chen, and Chu-Ren Huang (1999)
+The Construction of Sinica Treebank. Computational Linguistics and
+Chinese Language Processing, 4, pp 87-104.
+
+Huang Chu-Ren, Keh-Jiann Chen, Feng-Yi Chen, Keh-Jiann Chen, Zhao-Ming
+Gao, and Kuang-Yu Chen. 2000. Sinica Treebank: Design Criteria,
+Annotation Guidelines, and On-line Interface. Proceedings of 2nd
+Chinese Language Processing Workshop, Association for Computational
+Linguistics.
+
+Chen Keh-Jiann and Yu-Ming Hsieh (2004) Chinese Treebanks and Grammar
+Extraction, Proceedings of IJCNLP-04, pp560-565.
+"""
+
+import os
+import re
+
+from nltk.tree import sinica_parse
+from nltk.tag import map_tag
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+IDENTIFIER = re.compile(r'^#\S+\s')
+APPENDIX = re.compile(r'(?<=\))#.*$')
+TAGWORD = re.compile(r':([^:()|]+):([^:()|]+)')
+WORD = re.compile(r':[^:()|]+:([^:()|]+)')
+
+class SinicaTreebankCorpusReader(SyntaxCorpusReader):
+    """
+    Reader for the sinica treebank.
+    """
+    def _read_block(self, stream):
+        sent = stream.readline()
+        sent = IDENTIFIER.sub('', sent)
+        sent = APPENDIX.sub('', sent)
+        return [sent]
+
+    def _parse(self, sent):
+        return sinica_parse(sent)
+
+    def _tag(self, sent, tagset=None):
+        tagged_sent = [(w,t) for (t,w) in TAGWORD.findall(sent)]
+        if tagset and tagset != self._tagset:
+            tagged_sent = [(w, map_tag(self._tagset, tagset, t)) for (w,t) in tagged_sent]
+        return tagged_sent
+
+    def _word(self, sent):
+        return WORD.findall(sent)

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/string_category.py

@@ -1,59 +1,60 @@
-# Natural Language Toolkit: String Category Corpus Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Steven Bird <stevenbird1@gmail.com>
-#         Edward Loper <edloper@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Read tuples from a corpus consisting of categorized strings.
-For example, from the question classification corpus:
-
-NUM:dist How far is it from Denver to Aspen ?
-LOC:city What county is Modesto , California in ?
-HUM:desc Who was Galileo ?
-DESC:def What is an atom ?
-NUM:date When did Hawaii become a state ?
-"""
-
-# based on PPAttachmentCorpusReader
-
-from nltk import compat
-from nltk.corpus.reader.util import *
-from nltk.corpus.reader.api import *
-
-# [xx] Should the order of the tuple be reversed -- in most other places
-# in nltk, we use the form (data, tag) -- e.g., tagged words and
-# labeled texts for classifiers.
-class StringCategoryCorpusReader(CorpusReader):
-    def __init__(self, root, fileids, delimiter=' ', encoding='utf8'):
-        """
-        :param root: The root directory for this corpus.
-        :param fileids: A list or regexp specifying the fileids in this corpus.
-        :param delimiter: Field delimiter
-        """
-        CorpusReader.__init__(self, root, fileids, encoding)
-        self._delimiter = delimiter
-
-    def tuples(self, fileids=None):
-        if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
-        return concat([StreamBackedCorpusView(fileid, self._read_tuple_block,
-                                              encoding=enc)
-                       for (fileid, enc) in self.abspaths(fileids, True)])
-
-    def raw(self, fileids=None):
-        """
-        :return: the text contents of the given fileids, as a single string.
-        """
-        if fileids is None: fileids = self._fileids
-        elif isinstance(fileids, compat.string_types): fileids = [fileids]
-        return concat([self.open(f).read() for f in fileids])
-
-    def _read_tuple_block(self, stream):
-        line = stream.readline().strip()
-        if line:
-            return [tuple(line.split(self._delimiter, 1))]
-        else:
-            return []
+# Natural Language Toolkit: String Category Corpus Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+#         Edward Loper <edloper@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Read tuples from a corpus consisting of categorized strings.
+For example, from the question classification corpus:
+
+NUM:dist How far is it from Denver to Aspen ?
+LOC:city What county is Modesto , California in ?
+HUM:desc Who was Galileo ?
+DESC:def What is an atom ?
+NUM:date When did Hawaii become a state ?
+"""
+
+# based on PPAttachmentCorpusReader
+from six import string_types
+
+from nltk import compat
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+# [xx] Should the order of the tuple be reversed -- in most other places
+# in nltk, we use the form (data, tag) -- e.g., tagged words and
+# labeled texts for classifiers.
+class StringCategoryCorpusReader(CorpusReader):
+    def __init__(self, root, fileids, delimiter=' ', encoding='utf8'):
+        """
+        :param root: The root directory for this corpus.
+        :param fileids: A list or regexp specifying the fileids in this corpus.
+        :param delimiter: Field delimiter
+        """
+        CorpusReader.__init__(self, root, fileids, encoding)
+        self._delimiter = delimiter
+
+    def tuples(self, fileids=None):
+        if fileids is None: fileids = self._fileids
+        elif isinstance(fileids, string_types): fileids = [fileids]
+        return concat([StreamBackedCorpusView(fileid, self._read_tuple_block,
+                                              encoding=enc)
+                       for (fileid, enc) in self.abspaths(fileids, True)])
+
+    def raw(self, fileids=None):
+        """
+        :return: the text contents of the given fileids, as a single string.
+        """
+        if fileids is None: fileids = self._fileids
+        elif isinstance(fileids, string_types): fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])
+
+    def _read_tuple_block(self, stream):
+        line = stream.readline().strip()
+        if line:
+            return [tuple(line.split(self._delimiter, 1))]
+        else:
+            return []