nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/sentiment/vader.py

@@ -1,445 +1,443 @@
-# coding: utf-8
-# Natural Language Toolkit: vader
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: C.J. Hutto <Clayton.Hutto@gtri.gatech.edu>
-#         Ewan Klein <ewan@inf.ed.ac.uk> (modifications)
-#         Pierpaolo Pantone <24alsecondo@gmail.com> (modifications)
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-#
-# Modifications to the original VADER code have been made in order to
-# integrate it into NLTK. These have involved changes to
-# ensure Python 3 compatibility, and refactoring to achieve greater modularity.
-
-"""
-If you use the VADER sentiment analysis tools, please cite:
-
-Hutto, C.J. & Gilbert, E.E. (2014). VADER: A Parsimonious Rule-based Model for
-Sentiment Analysis of Social Media Text. Eighth International Conference on
-Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014.
-"""
-
-import codecs
-import math
-import os
-import re
-import string
-
-##Constants##
-
-# (empirically derived mean sentiment intensity rating increase for booster words)
-B_INCR = 0.293
-B_DECR = -0.293
-
-# (empirically derived mean sentiment intensity rating increase for using
-# ALLCAPs to emphasize a word)
-C_INCR = 0.733
-
-N_SCALAR = -0.74
-
-# for removing punctuation
-REGEX_REMOVE_PUNCTUATION = re.compile('[%s]' % re.escape(string.punctuation))
-
-PUNC_LIST = [".", "!", "?", ",", ";", ":", "-", "'", "\"",
-             "!!", "!!!", "??", "???", "?!?", "!?!", "?!?!", "!?!?"]
-NEGATE = \
-["aint", "arent", "cannot", "cant", "couldnt", "darent", "didnt", "doesnt",
- "ain't", "aren't", "can't", "couldn't", "daren't", "didn't", "doesn't",
- "dont", "hadnt", "hasnt", "havent", "isnt", "mightnt", "mustnt", "neither",
- "don't", "hadn't", "hasn't", "haven't", "isn't", "mightn't", "mustn't",
- "neednt", "needn't", "never", "none", "nope", "nor", "not", "nothing", "nowhere",
- "oughtnt", "shant", "shouldnt", "uhuh", "wasnt", "werent",
- "oughtn't", "shan't", "shouldn't", "uh-uh", "wasn't", "weren't",
- "without", "wont", "wouldnt", "won't", "wouldn't", "rarely", "seldom", "despite"]
-
-# booster/dampener 'intensifiers' or 'degree adverbs'
-# http://en.wiktionary.org/wiki/Category:English_degree_adverbs
-
-BOOSTER_DICT = \
-{"absolutely": B_INCR, "amazingly": B_INCR, "awfully": B_INCR, "completely": B_INCR, "considerably": B_INCR,
- "decidedly": B_INCR, "deeply": B_INCR, "effing": B_INCR, "enormously": B_INCR,
- "entirely": B_INCR, "especially": B_INCR, "exceptionally": B_INCR, "extremely": B_INCR,
- "fabulously": B_INCR, "flipping": B_INCR, "flippin": B_INCR,
- "fricking": B_INCR, "frickin": B_INCR, "frigging": B_INCR, "friggin": B_INCR, "fully": B_INCR, "fucking": B_INCR,
- "greatly": B_INCR, "hella": B_INCR, "highly": B_INCR, "hugely": B_INCR, "incredibly": B_INCR,
- "intensely": B_INCR, "majorly": B_INCR, "more": B_INCR, "most": B_INCR, "particularly": B_INCR,
- "purely": B_INCR, "quite": B_INCR, "really": B_INCR, "remarkably": B_INCR,
- "so": B_INCR, "substantially": B_INCR,
- "thoroughly": B_INCR, "totally": B_INCR, "tremendously": B_INCR,
- "uber": B_INCR, "unbelievably": B_INCR, "unusually": B_INCR, "utterly": B_INCR,
- "very": B_INCR,
- "almost": B_DECR, "barely": B_DECR, "hardly": B_DECR, "just enough": B_DECR,
- "kind of": B_DECR, "kinda": B_DECR, "kindof": B_DECR, "kind-of": B_DECR,
- "less": B_DECR, "little": B_DECR, "marginally": B_DECR, "occasionally": B_DECR, "partly": B_DECR,
- "scarcely": B_DECR, "slightly": B_DECR, "somewhat": B_DECR,
- "sort of": B_DECR, "sorta": B_DECR, "sortof": B_DECR, "sort-of": B_DECR}
-
-# check for special case idioms using a sentiment-laden keyword known to SAGE
-SPECIAL_CASE_IDIOMS = {"the shit": 3, "the bomb": 3, "bad ass": 1.5, "yeah right": -2,
-                       "cut the mustard": 2, "kiss of death": -1.5, "hand to mouth": -2}
-
-
-##Static methods##
-
-def negated(input_words, include_nt=True):
-    """
-    Determine if input contains negation words
-    """
-    neg_words = []
-    neg_words.extend(NEGATE)
-    for word in neg_words:
-        if word in input_words:
-            return True
-    if include_nt:
-        for word in input_words:
-            if "n't" in word:
-                return True
-    if "least" in input_words:
-        i = input_words.index("least")
-        if i > 0 and input_words[i-1] != "at":
-            return True
-    return False
-
-
-def normalize(score, alpha=15):
-    """
-    Normalize the score to be between -1 and 1 using an alpha that
-    approximates the max expected value
-    """
-    norm_score = score/math.sqrt((score*score) + alpha)
-    return norm_score
-
-
-def allcap_differential(words):
-    """
-    Check whether just some words in the input are ALL CAPS
-
-    :param list words: The words to inspect
-    :returns: `True` if some but not all items in `words` are ALL CAPS
-    """
-    is_different = False
-    allcap_words = 0
-    for word in words:
-        if word.isupper():
-            allcap_words += 1
-    cap_differential = len(words) - allcap_words
-    if cap_differential > 0 and cap_differential < len(words):
-        is_different = True
-    return is_different
-
-
-def scalar_inc_dec(word, valence, is_cap_diff):
-    """
-    Check if the preceding words increase, decrease, or negate/nullify the
-    valence
-    """
-    scalar = 0.0
-    word_lower = word.lower()
-    if word_lower in BOOSTER_DICT:
-        scalar = BOOSTER_DICT[word_lower]
-        if valence < 0:
-            scalar *= -1
-        #check if booster/dampener word is in ALLCAPS (while others aren't)
-        if word.isupper() and is_cap_diff:
-            if valence > 0:
-                scalar += C_INCR
-            else: scalar -= C_INCR
-    return scalar
-
-class SentiText(object):
-    """
-    Identify sentiment-relevant string-level properties of input text.
-    """
-    def __init__(self, text):
-        if not isinstance(text, str):
-            text = str(text.encode('utf-8'))
-        self.text = text
-        self.words_and_emoticons = self._words_and_emoticons()
-        # doesn't separate words from\
-        # adjacent punctuation (keeps emoticons & contractions)
-        self.is_cap_diff = allcap_differential(self.words_and_emoticons)
-
-    def _words_only(self):
-        text_mod = REGEX_REMOVE_PUNCTUATION.sub('', self.text)
-        # removes punctuation (but loses emoticons & contractions)
-        words_only = text_mod.split()
-        # get rid of empty items or single letter "words" like 'a' and 'I'
-        words_only = [word for word in words_only if len(word) > 1]
-        return words_only
-
-    def _words_and_emoticons(self):
-        wes = self.text.split()
-
-        # get rid of residual empty items or single letter words
-        wes = [we for we in wes if len(we) > 1]
-
-        for word in self._words_only():
-            for punct in PUNC_LIST:
-                pword = punct + word
-                x1 = wes.count(pword)
-                while x1 > 0:
-                    i = wes.index(pword)
-                    wes.remove(pword)
-                    wes.insert(i, word)
-                    x1 = wes.count(pword)
-
-                wordp = word + punct
-                x2 = wes.count(wordp)
-                while x2 > 0:
-                    i = wes.index(wordp)
-                    wes.remove(wordp)
-                    wes.insert(i, word)
-                    x2 = wes.count(wordp)
-        return wes
-
-class SentimentIntensityAnalyzer(object):
-    """
-    Give a sentiment intensity score to sentences.
-    """
-    def __init__(self, lexicon_file="vader_lexicon.txt"):
-        self.lexicon_file = os.path.join(os.path.dirname(__file__), lexicon_file)
-        self.lexicon = self.make_lex_dict()
-
-    def make_lex_dict(self):
-        """
-        Convert lexicon file to a dictionary
-        """
-        lex_dict = {}
-        with codecs.open(self.lexicon_file, encoding='utf8') as infile:
-            for line in infile:
-                (word, measure) = line.strip().split('\t')[0:2]
-                lex_dict[word] = float(measure)
-        return lex_dict
-
-    def polarity_scores(self, text):
-        """
-        Return a float for sentiment strength based on the input text.
-        Positive values are positive valence, negative value are negative
-        valence.
-        """
-        sentitext = SentiText(text)
-        #text, words_and_emoticons, is_cap_diff = self.preprocess(text)
-
-        sentiments = []
-        words_and_emoticons = sentitext.words_and_emoticons
-        for item in words_and_emoticons:
-            valence = 0
-            i = words_and_emoticons.index(item)
-            if (i < len(words_and_emoticons) - 1 and item.lower() == "kind" and \
-                words_and_emoticons[i+1].lower() == "of") or \
-                item.lower() in BOOSTER_DICT:
-                sentiments.append(valence)
-                continue
-
-            sentiments = self.sentiment_valence(valence, sentitext, item, i, sentiments)
-
-        sentiments = self._but_check(words_and_emoticons, sentiments)
-
-        return self.score_valence(sentiments, text)
-
-    def sentiment_valence(self, valence, sentitext, item, i, sentiments):
-        is_cap_diff = sentitext.is_cap_diff
-        words_and_emoticons = sentitext.words_and_emoticons
-        item_lowercase = item.lower()
-        if item_lowercase in self.lexicon:
-            #get the sentiment valence
-            valence = self.lexicon[item_lowercase]
-
-            #check if sentiment laden word is in ALL CAPS (while others aren't)
-            if item.isupper() and is_cap_diff:
-                if valence > 0:
-                    valence += C_INCR
-                else:
-                    valence -= C_INCR
-
-            for start_i in range(0,3):
-                if i > start_i and words_and_emoticons[i-(start_i+1)].lower() not in self.lexicon:
-                    # dampen the scalar modifier of preceding words and emoticons
-                    # (excluding the ones that immediately preceed the item) based
-                    # on their distance from the current item.
-                    s = scalar_inc_dec(words_and_emoticons[i-(start_i+1)], valence, is_cap_diff)
-                    if start_i == 1 and s != 0:
-                        s = s*0.95
-                    if start_i == 2 and s != 0:
-                        s = s*0.9
-                    valence = valence+s
-                    valence = self._never_check(valence, words_and_emoticons, start_i, i)
-                    if start_i == 2:
-                        valence = self._idioms_check(valence, words_and_emoticons, i)
-
-                        # future work: consider other sentiment-laden idioms
-                        # other_idioms =
-                        # {"back handed": -2, "blow smoke": -2, "blowing smoke": -2,
-                        #  "upper hand": 1, "break a leg": 2,
-                        #  "cooking with gas": 2, "in the black": 2, "in the red": -2,
-                        #  "on the ball": 2,"under the weather": -2}
-
-            valence = self._least_check(valence, words_and_emoticons, i)
-
-        sentiments.append(valence)
-        return sentiments
-
-    def _least_check(self, valence, words_and_emoticons, i):
-        # check for negation case using "least"
-        if i > 1 and words_and_emoticons[i-1].lower() not in self.lexicon \
-           and words_and_emoticons[i-1].lower() == "least":
-            if words_and_emoticons[i-2].lower() != "at" and words_and_emoticons[i-2].lower() != "very":
-                valence = valence*N_SCALAR
-        elif i > 0 and words_and_emoticons[i-1].lower() not in self.lexicon \
-             and words_and_emoticons[i-1].lower() == "least":
-            valence = valence*N_SCALAR
-        return valence
-
-    def _but_check(self, words_and_emoticons, sentiments):
-        # check for modification in sentiment due to contrastive conjunction 'but'
-        if 'but' in words_and_emoticons or 'BUT' in words_and_emoticons:
-            try:
-                bi = words_and_emoticons.index('but')
-            except ValueError:
-                bi = words_and_emoticons.index('BUT')
-            for sentiment in sentiments:
-                si = sentiments.index(sentiment)
-                if si < bi:
-                    sentiments.pop(si)
-                    sentiments.insert(si, sentiment*0.5)
-                elif si > bi:
-                    sentiments.pop(si)
-                    sentiments.insert(si, sentiment*1.5)
-        return sentiments
-
-    def _idioms_check(self, valence, words_and_emoticons, i):
-        onezero = "{0} {1}".format(words_and_emoticons[i-1], words_and_emoticons[i])
-
-        twoonezero = "{0} {1} {2}".format(words_and_emoticons[i-2],
-                                       words_and_emoticons[i-1], words_and_emoticons[i])
-
-        twoone = "{0} {1}".format(words_and_emoticons[i-2], words_and_emoticons[i-1])
-
-        threetwoone = "{0} {1} {2}".format(words_and_emoticons[i-3],
-                                        words_and_emoticons[i-2], words_and_emoticons[i-1])
-
-        threetwo = "{0} {1}".format(words_and_emoticons[i-3], words_and_emoticons[i-2])
-
-        sequences = [onezero, twoonezero, twoone, threetwoone, threetwo]
-
-        for seq in sequences:
-            if seq in SPECIAL_CASE_IDIOMS:
-                valence = SPECIAL_CASE_IDIOMS[seq]
-                break
-
-        if len(words_and_emoticons)-1 > i:
-            zeroone = "{0} {1}".format(words_and_emoticons[i], words_and_emoticons[i+1])
-            if zeroone in SPECIAL_CASE_IDIOMS:
-                valence = SPECIAL_CASE_IDIOMS[zeroone]
-        if len(words_and_emoticons)-1 > i+1:
-            zeroonetwo = "{0} {1} {2}".format(words_and_emoticons[i], words_and_emoticons[i+1], words_and_emoticons[i+2])
-            if zeroonetwo in SPECIAL_CASE_IDIOMS:
-                valence = SPECIAL_CASE_IDIOMS[zeroonetwo]
-
-        # check for booster/dampener bi-grams such as 'sort of' or 'kind of'
-        if threetwo in BOOSTER_DICT or twoone in BOOSTER_DICT:
-            valence = valence+B_DECR
-        return valence
-
-    def _never_check(self, valence, words_and_emoticons, start_i, i):
-        if start_i == 0:
-            if negated([words_and_emoticons[i-1]]):
-                    valence = valence*N_SCALAR
-        if start_i == 1:
-            if words_and_emoticons[i-2] == "never" and\
-               (words_and_emoticons[i-1] == "so" or
-                words_and_emoticons[i-1] == "this"):
-                valence = valence*1.5
-            elif negated([words_and_emoticons[i-(start_i+1)]]):
-                valence = valence*N_SCALAR
-        if start_i == 2:
-            if words_and_emoticons[i-3] == "never" and \
-               (words_and_emoticons[i-2] == "so" or words_and_emoticons[i-2] == "this") or \
-               (words_and_emoticons[i-1] == "so" or words_and_emoticons[i-1] == "this"):
-                valence = valence*1.25
-            elif negated([words_and_emoticons[i-(start_i+1)]]):
-                valence = valence*N_SCALAR
-        return valence
-
-    def _punctuation_emphasis(self, sum_s, text):
-        # add emphasis from exclamation points and question marks
-        ep_amplifier = self._amplify_ep(text)
-        qm_amplifier = self._amplify_qm(text)
-        punct_emph_amplifier = ep_amplifier+qm_amplifier
-        return punct_emph_amplifier
-
-    def _amplify_ep(self, text):
-        # check for added emphasis resulting from exclamation points (up to 4 of them)
-        ep_count = text.count("!")
-        if ep_count > 4:
-            ep_count = 4
-        # (empirically derived mean sentiment intensity rating increase for
-        # exclamation points)
-        ep_amplifier = ep_count*0.292
-        return ep_amplifier
-
-    def _amplify_qm(self, text):
-        # check for added emphasis resulting from question marks (2 or 3+)
-        qm_count = text.count("?")
-        qm_amplifier = 0
-        if qm_count > 1:
-            if qm_count <= 3:
-                # (empirically derived mean sentiment intensity rating increase for
-                # question marks)
-                qm_amplifier = qm_count*0.18
-            else:
-                qm_amplifier = 0.96
-        return qm_amplifier
-
-    def _sift_sentiment_scores(self, sentiments):
-        # want separate positive versus negative sentiment scores
-        pos_sum = 0.0
-        neg_sum = 0.0
-        neu_count = 0
-        for sentiment_score in sentiments:
-            if sentiment_score > 0:
-                pos_sum += (float(sentiment_score) +1) # compensates for neutral words that are counted as 1
-            if sentiment_score < 0:
-                neg_sum += (float(sentiment_score) -1) # when used with math.fabs(), compensates for neutrals
-            if sentiment_score == 0:
-                neu_count += 1
-        return pos_sum, neg_sum, neu_count
-
-    def score_valence(self, sentiments, text):
-        if sentiments:
-            sum_s = float(sum(sentiments))
-            # compute and add emphasis from punctuation in text
-            punct_emph_amplifier = self._punctuation_emphasis(sum_s, text)
-            if sum_s > 0:
-                sum_s += punct_emph_amplifier
-            elif  sum_s < 0:
-                sum_s -= punct_emph_amplifier
-
-            compound = normalize(sum_s)
-            # discriminate between positive, negative and neutral sentiment scores
-            pos_sum, neg_sum, neu_count = self._sift_sentiment_scores(sentiments)
-
-            if pos_sum > math.fabs(neg_sum):
-                pos_sum += (punct_emph_amplifier)
-            elif pos_sum < math.fabs(neg_sum):
-                neg_sum -= (punct_emph_amplifier)
-
-            total = pos_sum + math.fabs(neg_sum) + neu_count
-            pos = math.fabs(pos_sum / total)
-            neg = math.fabs(neg_sum / total)
-            neu = math.fabs(neu_count / total)
-
-        else:
-            compound = 0.0
-            pos = 0.0
-            neg = 0.0
-            neu = 0.0
-
-        sentiment_dict = \
-            {"neg" : round(neg, 3),
-             "neu" : round(neu, 3),
-             "pos" : round(pos, 3),
-             "compound" : round(compound, 4)}
-
-        return sentiment_dict
+# coding: utf-8
+# Natural Language Toolkit: vader
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: C.J. Hutto <Clayton.Hutto@gtri.gatech.edu>
+#         Ewan Klein <ewan@inf.ed.ac.uk> (modifications)
+#         Pierpaolo Pantone <24alsecondo@gmail.com> (modifications)
+#         George Berry <geb97@cornell.edu> (modifications)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+# Modifications to the original VADER code have been made in order to
+# integrate it into NLTK. These have involved changes to
+# ensure Python 3 compatibility, and refactoring to achieve greater modularity.
+
+"""
+If you use the VADER sentiment analysis tools, please cite:
+
+Hutto, C.J. & Gilbert, E.E. (2014). VADER: A Parsimonious Rule-based Model for
+Sentiment Analysis of Social Media Text. Eighth International Conference on
+Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014.
+"""
+
+import codecs
+import math
+import re
+import string
+from itertools import product
+import nltk.data
+from .util import pairwise
+
+##Constants##
+
+# (empirically derived mean sentiment intensity rating increase for booster words)
+B_INCR = 0.293
+B_DECR = -0.293
+
+# (empirically derived mean sentiment intensity rating increase for using
+# ALLCAPs to emphasize a word)
+C_INCR = 0.733
+
+N_SCALAR = -0.74
+
+# for removing punctuation
+REGEX_REMOVE_PUNCTUATION = re.compile('[{0}]'.format(re.escape(string.punctuation)))
+
+PUNC_LIST = [".", "!", "?", ",", ";", ":", "-", "'", "\"",
+             "!!", "!!!", "??", "???", "?!?", "!?!", "?!?!", "!?!?"]
+NEGATE = {"aint", "arent", "cannot", "cant", "couldnt", "darent", "didnt", "doesnt",
+ "ain't", "aren't", "can't", "couldn't", "daren't", "didn't", "doesn't",
+ "dont", "hadnt", "hasnt", "havent", "isnt", "mightnt", "mustnt", "neither",
+ "don't", "hadn't", "hasn't", "haven't", "isn't", "mightn't", "mustn't",
+ "neednt", "needn't", "never", "none", "nope", "nor", "not", "nothing", "nowhere",
+ "oughtnt", "shant", "shouldnt", "uhuh", "wasnt", "werent",
+ "oughtn't", "shan't", "shouldn't", "uh-uh", "wasn't", "weren't",
+ "without", "wont", "wouldnt", "won't", "wouldn't", "rarely", "seldom", "despite"}
+
+# booster/dampener 'intensifiers' or 'degree adverbs'
+# http://en.wiktionary.org/wiki/Category:English_degree_adverbs
+
+BOOSTER_DICT = \
+{"absolutely": B_INCR, "amazingly": B_INCR, "awfully": B_INCR, "completely": B_INCR, "considerably": B_INCR,
+ "decidedly": B_INCR, "deeply": B_INCR, "effing": B_INCR, "enormously": B_INCR,
+ "entirely": B_INCR, "especially": B_INCR, "exceptionally": B_INCR, "extremely": B_INCR,
+ "fabulously": B_INCR, "flipping": B_INCR, "flippin": B_INCR,
+ "fricking": B_INCR, "frickin": B_INCR, "frigging": B_INCR, "friggin": B_INCR, "fully": B_INCR, "fucking": B_INCR,
+ "greatly": B_INCR, "hella": B_INCR, "highly": B_INCR, "hugely": B_INCR, "incredibly": B_INCR,
+ "intensely": B_INCR, "majorly": B_INCR, "more": B_INCR, "most": B_INCR, "particularly": B_INCR,
+ "purely": B_INCR, "quite": B_INCR, "really": B_INCR, "remarkably": B_INCR,
+ "so": B_INCR, "substantially": B_INCR,
+ "thoroughly": B_INCR, "totally": B_INCR, "tremendously": B_INCR,
+ "uber": B_INCR, "unbelievably": B_INCR, "unusually": B_INCR, "utterly": B_INCR,
+ "very": B_INCR,
+ "almost": B_DECR, "barely": B_DECR, "hardly": B_DECR, "just enough": B_DECR,
+ "kind of": B_DECR, "kinda": B_DECR, "kindof": B_DECR, "kind-of": B_DECR,
+ "less": B_DECR, "little": B_DECR, "marginally": B_DECR, "occasionally": B_DECR, "partly": B_DECR,
+ "scarcely": B_DECR, "slightly": B_DECR, "somewhat": B_DECR,
+ "sort of": B_DECR, "sorta": B_DECR, "sortof": B_DECR, "sort-of": B_DECR}
+
+# check for special case idioms using a sentiment-laden keyword known to SAGE
+SPECIAL_CASE_IDIOMS = {"the shit": 3, "the bomb": 3, "bad ass": 1.5, "yeah right": -2,
+                       "cut the mustard": 2, "kiss of death": -1.5, "hand to mouth": -2}
+
+
+##Static methods##
+
+def negated(input_words, include_nt=True):
+    """
+    Determine if input contains negation words
+    """
+    neg_words = NEGATE
+    if any(word.lower() in neg_words for word in input_words):
+        return True
+    if include_nt:
+        if any("n't" in word.lower() for word in input_words):
+            return True
+    for first, second in pairwise(input_words):
+        if second.lower() == "least" and first.lower() != 'at':
+            return True
+    return False
+
+
+def normalize(score, alpha=15):
+    """
+    Normalize the score to be between -1 and 1 using an alpha that
+    approximates the max expected value
+    """
+    norm_score = score/math.sqrt((score*score) + alpha)
+    return norm_score
+
+
+def allcap_differential(words):
+    """
+    Check whether just some words in the input are ALL CAPS
+
+    :param list words: The words to inspect
+    :returns: `True` if some but not all items in `words` are ALL CAPS
+    """
+    is_different = False
+    allcap_words = 0
+    for word in words:
+        if word.isupper():
+            allcap_words += 1
+    cap_differential = len(words) - allcap_words
+    if cap_differential > 0 and cap_differential < len(words):
+        is_different = True
+    return is_different
+
+
+def scalar_inc_dec(word, valence, is_cap_diff):
+    """
+    Check if the preceding words increase, decrease, or negate/nullify the
+    valence
+    """
+    scalar = 0.0
+    word_lower = word.lower()
+    if word_lower in BOOSTER_DICT:
+        scalar = BOOSTER_DICT[word_lower]
+        if valence < 0:
+            scalar *= -1
+        #check if booster/dampener word is in ALLCAPS (while others aren't)
+        if word.isupper() and is_cap_diff:
+            if valence > 0:
+                scalar += C_INCR
+            else: scalar -= C_INCR
+    return scalar
+
+class SentiText(object):
+    """
+    Identify sentiment-relevant string-level properties of input text.
+    """
+    def __init__(self, text):
+        if not isinstance(text, str):
+            text = str(text.encode('utf-8'))
+        self.text = text
+        self.words_and_emoticons = self._words_and_emoticons()
+        # doesn't separate words from\
+        # adjacent punctuation (keeps emoticons & contractions)
+        self.is_cap_diff = allcap_differential(self.words_and_emoticons)
+
+    def _words_plus_punc(self):
+        """
+        Returns mapping of form:
+        {
+            'cat,': 'cat',
+            ',cat': 'cat',
+        }
+        """
+        no_punc_text = REGEX_REMOVE_PUNCTUATION.sub('', self.text)
+        # removes punctuation (but loses emoticons & contractions)
+        words_only = no_punc_text.split()
+        # remove singletons
+        words_only = set( w for w in words_only if len(w) > 1 )
+        # the product gives ('cat', ',') and (',', 'cat')
+        punc_before = {''.join(p): p[1] for p in product(PUNC_LIST, words_only)}
+        punc_after = {''.join(p): p[0] for p in product(words_only, PUNC_LIST)}
+        words_punc_dict = punc_before
+        words_punc_dict.update(punc_after)
+        return words_punc_dict
+
+    def _words_and_emoticons(self):
+        """
+        Removes leading and trailing puncutation
+        Leaves contractions and most emoticons
+            Does not preserve punc-plus-letter emoticons (e.g. :D)
+        """
+        wes = self.text.split()
+        words_punc_dict = self._words_plus_punc()
+        wes = [we for we in wes if len(we) > 1]
+        for i, we in enumerate(wes):
+            if we in words_punc_dict:
+                wes[i] = words_punc_dict[we]
+        return wes
+
+class SentimentIntensityAnalyzer(object):
+    """
+    Give a sentiment intensity score to sentences.
+    """
+    def __init__(self, lexicon_file="sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt"):
+        self.lexicon_file = nltk.data.load(lexicon_file)
+        self.lexicon = self.make_lex_dict()
+
+    def make_lex_dict(self):
+        """
+        Convert lexicon file to a dictionary
+        """
+        lex_dict = {}
+        for line in self.lexicon_file.split('\n'):
+            (word, measure) = line.strip().split('\t')[0:2]
+            lex_dict[word] = float(measure)
+        return lex_dict
+
+    def polarity_scores(self, text):
+        """
+        Return a float for sentiment strength based on the input text.
+        Positive values are positive valence, negative value are negative
+        valence.
+        """
+        sentitext = SentiText(text)
+        #text, words_and_emoticons, is_cap_diff = self.preprocess(text)
+
+        sentiments = []
+        words_and_emoticons = sentitext.words_and_emoticons
+        for item in words_and_emoticons:
+            valence = 0
+            i = words_and_emoticons.index(item)
+            if (i < len(words_and_emoticons) - 1 and item.lower() == "kind" and \
+                words_and_emoticons[i+1].lower() == "of") or \
+                item.lower() in BOOSTER_DICT:
+                sentiments.append(valence)
+                continue
+
+            sentiments = self.sentiment_valence(valence, sentitext, item, i, sentiments)
+
+        sentiments = self._but_check(words_and_emoticons, sentiments)
+
+        return self.score_valence(sentiments, text)
+
+    def sentiment_valence(self, valence, sentitext, item, i, sentiments):
+        is_cap_diff = sentitext.is_cap_diff
+        words_and_emoticons = sentitext.words_and_emoticons
+        item_lowercase = item.lower()
+        if item_lowercase in self.lexicon:
+            #get the sentiment valence
+            valence = self.lexicon[item_lowercase]
+
+            #check if sentiment laden word is in ALL CAPS (while others aren't)
+            if item.isupper() and is_cap_diff:
+                if valence > 0:
+                    valence += C_INCR
+                else:
+                    valence -= C_INCR
+
+            for start_i in range(0,3):
+                if i > start_i and words_and_emoticons[i-(start_i+1)].lower() not in self.lexicon:
+                    # dampen the scalar modifier of preceding words and emoticons
+                    # (excluding the ones that immediately preceed the item) based
+                    # on their distance from the current item.
+                    s = scalar_inc_dec(words_and_emoticons[i-(start_i+1)], valence, is_cap_diff)
+                    if start_i == 1 and s != 0:
+                        s = s*0.95
+                    if start_i == 2 and s != 0:
+                        s = s*0.9
+                    valence = valence+s
+                    valence = self._never_check(valence, words_and_emoticons, start_i, i)
+                    if start_i == 2:
+                        valence = self._idioms_check(valence, words_and_emoticons, i)
+
+                        # future work: consider other sentiment-laden idioms
+                        # other_idioms =
+                        # {"back handed": -2, "blow smoke": -2, "blowing smoke": -2,
+                        #  "upper hand": 1, "break a leg": 2,
+                        #  "cooking with gas": 2, "in the black": 2, "in the red": -2,
+                        #  "on the ball": 2,"under the weather": -2}
+
+            valence = self._least_check(valence, words_and_emoticons, i)
+
+        sentiments.append(valence)
+        return sentiments
+
+    def _least_check(self, valence, words_and_emoticons, i):
+        # check for negation case using "least"
+        if i > 1 and words_and_emoticons[i-1].lower() not in self.lexicon \
+           and words_and_emoticons[i-1].lower() == "least":
+            if words_and_emoticons[i-2].lower() != "at" and words_and_emoticons[i-2].lower() != "very":
+                valence = valence*N_SCALAR
+        elif i > 0 and words_and_emoticons[i-1].lower() not in self.lexicon \
+             and words_and_emoticons[i-1].lower() == "least":
+            valence = valence*N_SCALAR
+        return valence
+
+    def _but_check(self, words_and_emoticons, sentiments):
+        # check for modification in sentiment due to contrastive conjunction 'but'
+        if 'but' in words_and_emoticons or 'BUT' in words_and_emoticons:
+            try:
+                bi = words_and_emoticons.index('but')
+            except ValueError:
+                bi = words_and_emoticons.index('BUT')
+            for sentiment in sentiments:
+                si = sentiments.index(sentiment)
+                if si < bi:
+                    sentiments.pop(si)
+                    sentiments.insert(si, sentiment*0.5)
+                elif si > bi:
+                    sentiments.pop(si)
+                    sentiments.insert(si, sentiment*1.5)
+        return sentiments
+
+    def _idioms_check(self, valence, words_and_emoticons, i):
+        onezero = "{0} {1}".format(words_and_emoticons[i-1], words_and_emoticons[i])
+
+        twoonezero = "{0} {1} {2}".format(words_and_emoticons[i-2],
+                                       words_and_emoticons[i-1], words_and_emoticons[i])
+
+        twoone = "{0} {1}".format(words_and_emoticons[i-2], words_and_emoticons[i-1])
+
+        threetwoone = "{0} {1} {2}".format(words_and_emoticons[i-3],
+                                        words_and_emoticons[i-2], words_and_emoticons[i-1])
+
+        threetwo = "{0} {1}".format(words_and_emoticons[i-3], words_and_emoticons[i-2])
+
+        sequences = [onezero, twoonezero, twoone, threetwoone, threetwo]
+
+        for seq in sequences:
+            if seq in SPECIAL_CASE_IDIOMS:
+                valence = SPECIAL_CASE_IDIOMS[seq]
+                break
+
+        if len(words_and_emoticons)-1 > i:
+            zeroone = "{0} {1}".format(words_and_emoticons[i], words_and_emoticons[i+1])
+            if zeroone in SPECIAL_CASE_IDIOMS:
+                valence = SPECIAL_CASE_IDIOMS[zeroone]
+        if len(words_and_emoticons)-1 > i+1:
+            zeroonetwo = "{0} {1} {2}".format(words_and_emoticons[i], words_and_emoticons[i+1], words_and_emoticons[i+2])
+            if zeroonetwo in SPECIAL_CASE_IDIOMS:
+                valence = SPECIAL_CASE_IDIOMS[zeroonetwo]
+
+        # check for booster/dampener bi-grams such as 'sort of' or 'kind of'
+        if threetwo in BOOSTER_DICT or twoone in BOOSTER_DICT:
+            valence = valence+B_DECR
+        return valence
+
+    def _never_check(self, valence, words_and_emoticons, start_i, i):
+        if start_i == 0:
+            if negated([words_and_emoticons[i-1]]):
+                    valence = valence*N_SCALAR
+        if start_i == 1:
+            if words_and_emoticons[i-2] == "never" and\
+               (words_and_emoticons[i-1] == "so" or
+                words_and_emoticons[i-1] == "this"):
+                valence = valence*1.5
+            elif negated([words_and_emoticons[i-(start_i+1)]]):
+                valence = valence*N_SCALAR
+        if start_i == 2:
+            if words_and_emoticons[i-3] == "never" and \
+               (words_and_emoticons[i-2] == "so" or words_and_emoticons[i-2] == "this") or \
+               (words_and_emoticons[i-1] == "so" or words_and_emoticons[i-1] == "this"):
+                valence = valence*1.25
+            elif negated([words_and_emoticons[i-(start_i+1)]]):
+                valence = valence*N_SCALAR
+        return valence
+
+    def _punctuation_emphasis(self, sum_s, text):
+        # add emphasis from exclamation points and question marks
+        ep_amplifier = self._amplify_ep(text)
+        qm_amplifier = self._amplify_qm(text)
+        punct_emph_amplifier = ep_amplifier+qm_amplifier
+        return punct_emph_amplifier
+
+    def _amplify_ep(self, text):
+        # check for added emphasis resulting from exclamation points (up to 4 of them)
+        ep_count = text.count("!")
+        if ep_count > 4:
+            ep_count = 4
+        # (empirically derived mean sentiment intensity rating increase for
+        # exclamation points)
+        ep_amplifier = ep_count*0.292
+        return ep_amplifier
+
+    def _amplify_qm(self, text):
+        # check for added emphasis resulting from question marks (2 or 3+)
+        qm_count = text.count("?")
+        qm_amplifier = 0
+        if qm_count > 1:
+            if qm_count <= 3:
+                # (empirically derived mean sentiment intensity rating increase for
+                # question marks)
+                qm_amplifier = qm_count*0.18
+            else:
+                qm_amplifier = 0.96
+        return qm_amplifier
+
+    def _sift_sentiment_scores(self, sentiments):
+        # want separate positive versus negative sentiment scores
+        pos_sum = 0.0
+        neg_sum = 0.0
+        neu_count = 0
+        for sentiment_score in sentiments:
+            if sentiment_score > 0:
+                pos_sum += (float(sentiment_score) +1) # compensates for neutral words that are counted as 1
+            if sentiment_score < 0:
+                neg_sum += (float(sentiment_score) -1) # when used with math.fabs(), compensates for neutrals
+            if sentiment_score == 0:
+                neu_count += 1
+        return pos_sum, neg_sum, neu_count
+
+    def score_valence(self, sentiments, text):
+        if sentiments:
+            sum_s = float(sum(sentiments))
+            # compute and add emphasis from punctuation in text
+            punct_emph_amplifier = self._punctuation_emphasis(sum_s, text)
+            if sum_s > 0:
+                sum_s += punct_emph_amplifier
+            elif  sum_s < 0:
+                sum_s -= punct_emph_amplifier
+
+            compound = normalize(sum_s)
+            # discriminate between positive, negative and neutral sentiment scores
+            pos_sum, neg_sum, neu_count = self._sift_sentiment_scores(sentiments)
+
+            if pos_sum > math.fabs(neg_sum):
+                pos_sum += (punct_emph_amplifier)
+            elif pos_sum < math.fabs(neg_sum):
+                neg_sum -= (punct_emph_amplifier)
+
+            total = pos_sum + math.fabs(neg_sum) + neu_count
+            pos = math.fabs(pos_sum / total)
+            neg = math.fabs(neg_sum / total)
+            neu = math.fabs(neu_count / total)
+
+        else:
+            compound = 0.0
+            pos = 0.0
+            neg = 0.0
+            neu = 0.0
+
+        sentiment_dict = \
+            {"neg" : round(neg, 3),
+             "neu" : round(neu, 3),
+             "pos" : round(pos, 3),
+             "compound" : round(compound, 4)}
+
+        return sentiment_dict