nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/corpus/reader/reviews.py

@@ -1,328 +1,330 @@
-# Natural Language Toolkit: Product Reviews Corpus Reader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Pierpaolo Pantone <24alsecondo@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-CorpusReader for reviews corpora (syntax based on Customer Review Corpus).
-
-- Customer Review Corpus information -
-Annotated by: Minqing Hu and Bing Liu, 2004.
-    Department of Computer Sicence
-    University of Illinois at Chicago
-
-Contact: Bing Liu, liub@cs.uic.edu
-        http://www.cs.uic.edu/~liub
-
-Distributed with permission.
-
-The "product_reviews_1" and "product_reviews_2" datasets respectively contain
-annotated customer reviews of 5 and 9 products from amazon.com.
-
-Related papers:
-
-- Minqing Hu and Bing Liu. "Mining and summarizing customer reviews".
-    Proceedings of the ACM SIGKDD International Conference on Knowledge
-    Discovery & Data Mining (KDD-04), 2004.
-
-- Minqing Hu and Bing Liu. "Mining Opinion Features in Customer Reviews".
-    Proceedings of Nineteeth National Conference on Artificial Intelligence
-    (AAAI-2004), 2004.
-
-- Xiaowen Ding, Bing Liu and Philip S. Yu. "A Holistic Lexicon-Based Appraoch to
-    Opinion Mining." Proceedings of First ACM International Conference on Web
-    Search and Data Mining (WSDM-2008), Feb 11-12, 2008, Stanford University,
-    Stanford, California, USA.
-
-Symbols used in the annotated reviews:
-
-    [t] : the title of the review: Each [t] tag starts a review.
-    xxxx[+|-n]: xxxx is a product feature.
-    [+n]: Positive opinion, n is the opinion strength: 3 strongest, and 1 weakest.
-          Note that the strength is quite subjective.
-          You may want ignore it, but only considering + and -
-    [-n]: Negative opinion
-    ##  : start of each sentence. Each line is a sentence.
-    [u] : feature not appeared in the sentence.
-    [p] : feature not appeared in the sentence. Pronoun resolution is needed.
-    [s] : suggestion or recommendation.
-    [cc]: comparison with a competing product from a different brand.
-    [cs]: comparison with a competing product from the same brand.
-
-Note: Some of the files (e.g. "ipod.txt", "Canon PowerShot SD500.txt") do not
-    provide separation between different reviews. This is due to the fact that
-    the dataset was specifically designed for aspect/feature-based sentiment
-    analysis, for which sentence-level annotation is sufficient. For document-
-    level classification and analysis, this peculiarity should be taken into
-    consideration.
-"""
-
-from __future__ import division
-
-import re
-
-from nltk.corpus.reader.api import *
-from nltk.tokenize import *
-
-TITLE = re.compile(r'^\[t\](.*)$') # [t] Title
-FEATURES = re.compile(r'((?:(?:\w+\s)+)?\w+)\[((?:\+|\-)\d)\]') # find 'feature' in feature[+3]
-NOTES = re.compile(r'\[(?!t)(p|u|s|cc|cs)\]') # find 'p' in camera[+2][p]
-SENT = re.compile(r'##(.*)$') # find tokenized sentence
-
-
-@compat.python_2_unicode_compatible
-class Review(object):
-    """
-    A Review is the main block of a ReviewsCorpusReader.
-    """
-    def __init__(self, title=None, review_lines=None):
-        """
-        :param title: the title of the review.
-        :param review_lines: the list of the ReviewLines that belong to the Review.
-        """
-        self.title = title
-        if review_lines is None:
-            self.review_lines = []
-        else:
-            self.review_lines = review_lines
-
-    def add_line(self, review_line):
-        """
-        Add a line (ReviewLine) to the review.
-
-        :param review_line: a ReviewLine instance that belongs to the Review.
-        """
-        assert isinstance(review_line, ReviewLine)
-        self.review_lines.append(review_line)
-
-    def features(self):
-        """
-        Return a list of features in the review. Each feature is a tuple made of
-        the specific item feature and the opinion strength about that feature.
-
-        :return: all features of the review as a list of tuples (feat, score).
-        :rtype: list(tuple)
-        """
-        features = []
-        for review_line in self.review_lines:
-            features.extend(review_line.features)
-        return features
-
-    def sents(self):
-        """
-        Return all tokenized sentences in the review.
-
-        :return: all sentences of the review as lists of tokens.
-        :rtype: list(list(str))
-        """
-        return [review_line.sent for review_line in self.review_lines]
-
-    def __repr__(self):
-        return 'Review(title=\"{}\", review_lines={})'.format(self.title, self.review_lines)
-
-
-@compat.python_2_unicode_compatible
-class ReviewLine(object):
-    """
-    A ReviewLine represents a sentence of the review, together with (optional)
-    annotations of its features and notes about the reviewed item.
-    """
-    def __init__(self, sent, features=None, notes=None):
-        self.sent = sent
-        if features is None:
-            self.features = []
-        else:
-            self.features = features
-
-        if notes is None:
-            self.notes = []
-        else:
-            self.notes = notes
-
-    def __repr__(self):
-        return ('ReviewLine(features={}, notes={}, sent={})'.format(
-            self.features, self.notes, self.sent))
-
-
-class ReviewsCorpusReader(CorpusReader):
-    """
-    Reader for the Customer Review Data dataset by Hu, Liu (2004).
-    Note: we are not applying any sentence tokenization at the moment, just word
-    tokenization.
-
-        >>> from nltk.corpus import product_reviews_1
-        >>> camera_reviews = product_reviews_1.reviews('Canon_G3.txt')
-        >>> review = camera_reviews[0]
-        >>> review.sents()[0]
-        ['i', 'recently', 'purchased', 'the', 'canon', 'powershot', 'g3', 'and', 'am',
-        'extremely', 'satisfied', 'with', 'the', 'purchase', '.']
-        >>> review.features()
-        [('canon powershot g3', '+3'), ('use', '+2'), ('picture', '+2'),
-        ('picture quality', '+1'), ('picture quality', '+1'), ('camera', '+2'),
-        ('use', '+2'), ('feature', '+1'), ('picture quality', '+3'), ('use', '+1'),
-        ('option', '+1')]
-
-    We can also reach the same information directly from the stream:
-
-        >>> product_reviews_1.features('Canon_G3.txt')
-        [('canon powershot g3', '+3'), ('use', '+2'), ...]
-
-    We can compute stats for specific product features:
-
-        >>> from __future__ import division
-        >>> n_reviews = len([(feat,score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture'])
-        >>> tot = sum([int(score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture'])
-        >>> # We use float for backward compatibility with division in Python2.7
-        >>> mean = tot / n_reviews
-        >>> print(n_reviews, tot, mean)
-        15 24 1.6
-    """
-    CorpusView = StreamBackedCorpusView
-
-    def __init__(self, root, fileids, word_tokenizer=WordPunctTokenizer(),
-                 encoding='utf8'):
-        """
-        :param root: The root directory for the corpus.
-        :param fileids: a list or regexp specifying the fileids in the corpus.
-        :param word_tokenizer: a tokenizer for breaking sentences or paragraphs
-            into words. Default: `WordPunctTokenizer`
-        :param encoding: the encoding that should be used to read the corpus.
-        """
-
-        CorpusReader.__init__(self, root, fileids, encoding)
-        self._word_tokenizer = word_tokenizer
-
-    def features(self, fileids=None):
-        """
-        Return a list of features. Each feature is a tuple made of the specific
-        item feature and the opinion strength about that feature.
-
-        :param fileids: a list or regexp specifying the ids of the files whose
-            features have to be returned.
-        :return: all features for the item(s) in the given file(s).
-        :rtype: list(tuple)
-        """
-        if fileids is None:
-            fileids = self._fileids
-        elif isinstance(fileids, string_types):
-            fileids = [fileids]
-        return concat([self.CorpusView(fileid, self._read_features, encoding=enc)
-                       for (fileid, enc) in self.abspaths(fileids, True)])
-
-    def raw(self, fileids=None):
-        """
-        :param fileids: a list or regexp specifying the fileids of the files that
-            have to be returned as a raw string.
-        :return: the given file(s) as a single string.
-        :rtype: str
-        """
-        if fileids is None:
-            fileids = self._fileids
-        elif isinstance(fileids, string_types):
-            fileids = [fileids]
-        return concat([self.open(f).read() for f in fileids])
-
-    def readme(self):
-        """
-        Return the contents of the corpus README.txt file.
-        """
-        return self.open("README.txt").read()
-
-    def reviews(self, fileids=None):
-        """
-        Return all the reviews as a list of Review objects. If `fileids` is
-        specified, return all the reviews from each of the specified files.
-
-        :param fileids: a list or regexp specifying the ids of the files whose
-            reviews have to be returned.
-        :return: the given file(s) as a list of reviews.
-        """
-        if fileids is None:
-            fileids = self._fileids
-        return concat([self.CorpusView(fileid, self._read_review_block, encoding=enc)
-                       for (fileid, enc) in self.abspaths(fileids, True)])
-
-    def sents(self, fileids=None):
-        """
-        Return all sentences in the corpus or in the specified files.
-
-        :param fileids: a list or regexp specifying the ids of the files whose
-            sentences have to be returned.
-        :return: the given file(s) as a list of sentences, each encoded as a
-            list of word strings.
-        :rtype: list(list(str))
-        """
-        return concat([self.CorpusView(path, self._read_sent_block, encoding=enc)
-                       for (path, enc, fileid)
-                       in self.abspaths(fileids, True, True)])
-
-    def words(self, fileids=None):
-        """
-        Return all words and punctuation symbols in the corpus or in the specified
-        files.
-
-        :param fileids: a list or regexp specifying the ids of the files whose
-            words have to be returned.
-        :return: the given file(s) as a list of words and punctuation symbols.
-        :rtype: list(str)
-        """
-        return concat([self.CorpusView(path, self._read_word_block, encoding=enc)
-                       for (path, enc, fileid)
-                       in self.abspaths(fileids, True, True)])
-
-    def _read_features(self, stream):
-        features = []
-        for i in range(20):
-            line = stream.readline()
-            if not line:
-                return features
-            features.extend(re.findall(FEATURES, line))
-        return features
-
-    def _read_review_block(self, stream):
-        while True:
-            line = stream.readline()
-            if not line:
-                return [] # end of file.
-            title_match = re.match(TITLE, line)
-            if title_match:
-                review = Review(title=title_match.group(1).strip()) # We create a new review
-                break
-
-        # Scan until we find another line matching the regexp, or EOF.
-        while True:
-            oldpos = stream.tell()
-            line = stream.readline()
-            # End of file:
-            if not line:
-                return [review]
-            # Start of a new review: backup to just before it starts, and
-            # return the review we've already collected.
-            if re.match(TITLE, line):
-                stream.seek(oldpos)
-                return [review]
-            # Anything else is part of the review line.
-            feats = re.findall(FEATURES, line)
-            notes = re.findall(NOTES, line)
-            sent = re.findall(SENT, line)
-            if sent:
-                sent = self._word_tokenizer.tokenize(sent[0])
-            review_line = ReviewLine(sent=sent, features=feats, notes=notes)
-            review.add_line(review_line)
-
-    def _read_sent_block(self, stream):
-        sents = []
-        for review in self._read_review_block(stream):
-            sents.extend([sent for sent in review.sents()])
-        return sents
-
-    def _read_word_block(self, stream):
-        words = []
-        for i in range(20): # Read 20 lines at a time.
-            line = stream.readline()
-            sent = re.findall(SENT, line)
-            if sent:
-                words.extend(self._word_tokenizer.tokenize(sent[0]))
-        return words
+# Natural Language Toolkit: Product Reviews Corpus Reader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Pierpaolo Pantone <24alsecondo@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+CorpusReader for reviews corpora (syntax based on Customer Review Corpus).
+
+- Customer Review Corpus information -
+Annotated by: Minqing Hu and Bing Liu, 2004.
+    Department of Computer Sicence
+    University of Illinois at Chicago
+
+Contact: Bing Liu, liub@cs.uic.edu
+        http://www.cs.uic.edu/~liub
+
+Distributed with permission.
+
+The "product_reviews_1" and "product_reviews_2" datasets respectively contain
+annotated customer reviews of 5 and 9 products from amazon.com.
+
+Related papers:
+
+- Minqing Hu and Bing Liu. "Mining and summarizing customer reviews".
+    Proceedings of the ACM SIGKDD International Conference on Knowledge
+    Discovery & Data Mining (KDD-04), 2004.
+
+- Minqing Hu and Bing Liu. "Mining Opinion Features in Customer Reviews".
+    Proceedings of Nineteeth National Conference on Artificial Intelligence
+    (AAAI-2004), 2004.
+
+- Xiaowen Ding, Bing Liu and Philip S. Yu. "A Holistic Lexicon-Based Appraoch to
+    Opinion Mining." Proceedings of First ACM International Conference on Web
+    Search and Data Mining (WSDM-2008), Feb 11-12, 2008, Stanford University,
+    Stanford, California, USA.
+
+Symbols used in the annotated reviews:
+
+    [t] : the title of the review: Each [t] tag starts a review.
+    xxxx[+|-n]: xxxx is a product feature.
+    [+n]: Positive opinion, n is the opinion strength: 3 strongest, and 1 weakest.
+          Note that the strength is quite subjective.
+          You may want ignore it, but only considering + and -
+    [-n]: Negative opinion
+    ##  : start of each sentence. Each line is a sentence.
+    [u] : feature not appeared in the sentence.
+    [p] : feature not appeared in the sentence. Pronoun resolution is needed.
+    [s] : suggestion or recommendation.
+    [cc]: comparison with a competing product from a different brand.
+    [cs]: comparison with a competing product from the same brand.
+
+Note: Some of the files (e.g. "ipod.txt", "Canon PowerShot SD500.txt") do not
+    provide separation between different reviews. This is due to the fact that
+    the dataset was specifically designed for aspect/feature-based sentiment
+    analysis, for which sentence-level annotation is sufficient. For document-
+    level classification and analysis, this peculiarity should be taken into
+    consideration.
+"""
+
+from __future__ import division
+
+from six import string_types
+
+import re
+
+from nltk.corpus.reader.api import *
+from nltk.tokenize import *
+
+TITLE = re.compile(r'^\[t\](.*)$') # [t] Title
+FEATURES = re.compile(r'((?:(?:\w+\s)+)?\w+)\[((?:\+|\-)\d)\]') # find 'feature' in feature[+3]
+NOTES = re.compile(r'\[(?!t)(p|u|s|cc|cs)\]') # find 'p' in camera[+2][p]
+SENT = re.compile(r'##(.*)$') # find tokenized sentence
+
+
+@compat.python_2_unicode_compatible
+class Review(object):
+    """
+    A Review is the main block of a ReviewsCorpusReader.
+    """
+    def __init__(self, title=None, review_lines=None):
+        """
+        :param title: the title of the review.
+        :param review_lines: the list of the ReviewLines that belong to the Review.
+        """
+        self.title = title
+        if review_lines is None:
+            self.review_lines = []
+        else:
+            self.review_lines = review_lines
+
+    def add_line(self, review_line):
+        """
+        Add a line (ReviewLine) to the review.
+
+        :param review_line: a ReviewLine instance that belongs to the Review.
+        """
+        assert isinstance(review_line, ReviewLine)
+        self.review_lines.append(review_line)
+
+    def features(self):
+        """
+        Return a list of features in the review. Each feature is a tuple made of
+        the specific item feature and the opinion strength about that feature.
+
+        :return: all features of the review as a list of tuples (feat, score).
+        :rtype: list(tuple)
+        """
+        features = []
+        for review_line in self.review_lines:
+            features.extend(review_line.features)
+        return features
+
+    def sents(self):
+        """
+        Return all tokenized sentences in the review.
+
+        :return: all sentences of the review as lists of tokens.
+        :rtype: list(list(str))
+        """
+        return [review_line.sent for review_line in self.review_lines]
+
+    def __repr__(self):
+        return 'Review(title=\"{}\", review_lines={})'.format(self.title, self.review_lines)
+
+
+@compat.python_2_unicode_compatible
+class ReviewLine(object):
+    """
+    A ReviewLine represents a sentence of the review, together with (optional)
+    annotations of its features and notes about the reviewed item.
+    """
+    def __init__(self, sent, features=None, notes=None):
+        self.sent = sent
+        if features is None:
+            self.features = []
+        else:
+            self.features = features
+
+        if notes is None:
+            self.notes = []
+        else:
+            self.notes = notes
+
+    def __repr__(self):
+        return ('ReviewLine(features={}, notes={}, sent={})'.format(
+            self.features, self.notes, self.sent))
+
+
+class ReviewsCorpusReader(CorpusReader):
+    """
+    Reader for the Customer Review Data dataset by Hu, Liu (2004).
+    Note: we are not applying any sentence tokenization at the moment, just word
+    tokenization.
+
+        >>> from nltk.corpus import product_reviews_1
+        >>> camera_reviews = product_reviews_1.reviews('Canon_G3.txt')
+        >>> review = camera_reviews[0]
+        >>> review.sents()[0]
+        ['i', 'recently', 'purchased', 'the', 'canon', 'powershot', 'g3', 'and', 'am',
+        'extremely', 'satisfied', 'with', 'the', 'purchase', '.']
+        >>> review.features()
+        [('canon powershot g3', '+3'), ('use', '+2'), ('picture', '+2'),
+        ('picture quality', '+1'), ('picture quality', '+1'), ('camera', '+2'),
+        ('use', '+2'), ('feature', '+1'), ('picture quality', '+3'), ('use', '+1'),
+        ('option', '+1')]
+
+    We can also reach the same information directly from the stream:
+
+        >>> product_reviews_1.features('Canon_G3.txt')
+        [('canon powershot g3', '+3'), ('use', '+2'), ...]
+
+    We can compute stats for specific product features:
+
+        >>> from __future__ import division
+        >>> n_reviews = len([(feat,score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture'])
+        >>> tot = sum([int(score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture'])
+        >>> # We use float for backward compatibility with division in Python2.7
+        >>> mean = tot / n_reviews
+        >>> print(n_reviews, tot, mean)
+        15 24 1.6
+    """
+    CorpusView = StreamBackedCorpusView
+
+    def __init__(self, root, fileids, word_tokenizer=WordPunctTokenizer(),
+                 encoding='utf8'):
+        """
+        :param root: The root directory for the corpus.
+        :param fileids: a list or regexp specifying the fileids in the corpus.
+        :param word_tokenizer: a tokenizer for breaking sentences or paragraphs
+            into words. Default: `WordPunctTokenizer`
+        :param encoding: the encoding that should be used to read the corpus.
+        """
+
+        CorpusReader.__init__(self, root, fileids, encoding)
+        self._word_tokenizer = word_tokenizer
+
+    def features(self, fileids=None):
+        """
+        Return a list of features. Each feature is a tuple made of the specific
+        item feature and the opinion strength about that feature.
+
+        :param fileids: a list or regexp specifying the ids of the files whose
+            features have to be returned.
+        :return: all features for the item(s) in the given file(s).
+        :rtype: list(tuple)
+        """
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+        return concat([self.CorpusView(fileid, self._read_features, encoding=enc)
+                       for (fileid, enc) in self.abspaths(fileids, True)])
+
+    def raw(self, fileids=None):
+        """
+        :param fileids: a list or regexp specifying the fileids of the files that
+            have to be returned as a raw string.
+        :return: the given file(s) as a single string.
+        :rtype: str
+        """
+        if fileids is None:
+            fileids = self._fileids
+        elif isinstance(fileids, string_types):
+            fileids = [fileids]
+        return concat([self.open(f).read() for f in fileids])
+
+    def readme(self):
+        """
+        Return the contents of the corpus README.txt file.
+        """
+        return self.open("README.txt").read()
+
+    def reviews(self, fileids=None):
+        """
+        Return all the reviews as a list of Review objects. If `fileids` is
+        specified, return all the reviews from each of the specified files.
+
+        :param fileids: a list or regexp specifying the ids of the files whose
+            reviews have to be returned.
+        :return: the given file(s) as a list of reviews.
+        """
+        if fileids is None:
+            fileids = self._fileids
+        return concat([self.CorpusView(fileid, self._read_review_block, encoding=enc)
+                       for (fileid, enc) in self.abspaths(fileids, True)])
+
+    def sents(self, fileids=None):
+        """
+        Return all sentences in the corpus or in the specified files.
+
+        :param fileids: a list or regexp specifying the ids of the files whose
+            sentences have to be returned.
+        :return: the given file(s) as a list of sentences, each encoded as a
+            list of word strings.
+        :rtype: list(list(str))
+        """
+        return concat([self.CorpusView(path, self._read_sent_block, encoding=enc)
+                       for (path, enc, fileid)
+                       in self.abspaths(fileids, True, True)])
+
+    def words(self, fileids=None):
+        """
+        Return all words and punctuation symbols in the corpus or in the specified
+        files.
+
+        :param fileids: a list or regexp specifying the ids of the files whose
+            words have to be returned.
+        :return: the given file(s) as a list of words and punctuation symbols.
+        :rtype: list(str)
+        """
+        return concat([self.CorpusView(path, self._read_word_block, encoding=enc)
+                       for (path, enc, fileid)
+                       in self.abspaths(fileids, True, True)])
+
+    def _read_features(self, stream):
+        features = []
+        for i in range(20):
+            line = stream.readline()
+            if not line:
+                return features
+            features.extend(re.findall(FEATURES, line))
+        return features
+
+    def _read_review_block(self, stream):
+        while True:
+            line = stream.readline()
+            if not line:
+                return [] # end of file.
+            title_match = re.match(TITLE, line)
+            if title_match:
+                review = Review(title=title_match.group(1).strip()) # We create a new review
+                break
+
+        # Scan until we find another line matching the regexp, or EOF.
+        while True:
+            oldpos = stream.tell()
+            line = stream.readline()
+            # End of file:
+            if not line:
+                return [review]
+            # Start of a new review: backup to just before it starts, and
+            # return the review we've already collected.
+            if re.match(TITLE, line):
+                stream.seek(oldpos)
+                return [review]
+            # Anything else is part of the review line.
+            feats = re.findall(FEATURES, line)
+            notes = re.findall(NOTES, line)
+            sent = re.findall(SENT, line)
+            if sent:
+                sent = self._word_tokenizer.tokenize(sent[0])
+            review_line = ReviewLine(sent=sent, features=feats, notes=notes)
+            review.add_line(review_line)
+
+    def _read_sent_block(self, stream):
+        sents = []
+        for review in self._read_review_block(stream):
+            sents.extend([sent for sent in review.sents()])
+        return sents
+
+    def _read_word_block(self, stream):
+        words = []
+        for i in range(20): # Read 20 lines at a time.
+            line = stream.readline()
+            sent = re.findall(SENT, line)
+            if sent:
+                words.extend(self._word_tokenizer.tokenize(sent[0]))
+        return words