nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/sem/util.py

@@ -1,249 +1,249 @@
-# Natural Language Toolkit: Semantic Interpretation
-#
-# Author: Ewan Klein <ewan@inf.ed.ac.uk>
-#
-# Copyright (C) 2001-2015 NLTK Project
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Utility functions for batch-processing sentences: parsing and
-extraction of the semantic representation of the root node of the the
-syntax tree, followed by evaluation of the semantic representation in
-a first-order model.
-"""
-from __future__ import print_function, unicode_literals
-
-import codecs
-from nltk.sem import evaluate
-
-
-##############################################################
-## Utility functions for connecting parse output to semantics
-##############################################################
-
-def parse_sents(inputs, grammar, trace=0):
-    """
-    Convert input sentences into syntactic trees.
-
-    :param inputs: sentences to be parsed
-    :type inputs: list(str)
-    :param grammar: ``FeatureGrammar`` or name of feature-based grammar
-    :type grammar: nltk.grammar.FeatureGrammar
-    :rtype: list(nltk.tree.Tree) or dict(list(str)): list(Tree)
-    :return: a mapping from input sentences to a list of ``Tree``s
-    """
-    # put imports here to avoid circult dependencies
-    from nltk.grammar import FeatureGrammar
-    from nltk.parse import FeatureChartParser, load_parser
-
-    if isinstance(grammar, FeatureGrammar):
-        cp = FeatureChartParser(grammar)
-    else:
-        cp = load_parser(grammar, trace=trace)
-    parses = []
-    for sent in inputs:
-        tokens = sent.split() # use a tokenizer?
-        syntrees = list(cp.parse(tokens))
-        parses.append(syntrees)
-    return parses
-
-def root_semrep(syntree, semkey='SEM'):
-    """
-    Find the semantic representation at the root of a tree.
-
-    :param syntree: a parse ``Tree``
-    :param semkey: the feature label to use for the root semantics in the tree
-    :return: the semantic representation at the root of a ``Tree``
-    :rtype: sem.Expression
-    """
-    from nltk.grammar import FeatStructNonterminal
-
-    node = syntree.label()
-    assert isinstance(node, FeatStructNonterminal)
-    try:
-        return node[semkey]
-    except KeyError:
-        print(node, end=' ')
-        print("has no specification for the feature %s" % semkey)
-    raise
-
-def interpret_sents(inputs, grammar, semkey='SEM', trace=0):
-    """
-    Add the semantic representation to each syntactic parse tree
-    of each input sentence.
-
-    :param inputs: a list of sentences
-    :type inputs: list(str)
-    :param grammar: ``FeatureGrammar`` or name of feature-based grammar
-    :type grammar: nltk.grammar.FeatureGrammar
-    :return: a mapping from sentences to lists of pairs (parse-tree, semantic-representations)
-    :rtype: list(list(tuple(nltk.tree.Tree, nltk.sem.logic.ConstantExpression)))
-    """
-    return [[(syn, root_semrep(syn, semkey)) for syn in syntrees]
-            for syntrees in parse_sents(inputs, grammar, trace=trace)]
-
-def evaluate_sents(inputs, grammar, model, assignment, trace=0):
-    """
-    Add the truth-in-a-model value to each semantic representation
-    for each syntactic parse of each input sentences.
-
-    :param inputs: a list of sentences
-    :type inputs: list(str)
-    :param grammar: ``FeatureGrammar`` or name of feature-based grammar
-    :type grammar: nltk.grammar.FeatureGrammar
-    :return: a mapping from sentences to lists of triples (parse-tree, semantic-representations, evaluation-in-model)
-    :rtype: list(list(tuple(nltk.tree.Tree, nltk.sem.logic.ConstantExpression, bool or dict(str): bool)))
-    """
-    return [[(syn, sem, model.evaluate("%s" % sem, assignment, trace=trace))
-            for (syn, sem) in interpretations]
-            for interpretations in interpret_sents(inputs, grammar)]
-
-
-def demo_model0():
-    global m0, g0
-    #Initialize a valuation of non-logical constants."""
-    v = [('john', 'b1'),
-        ('mary', 'g1'),
-        ('suzie', 'g2'),
-        ('fido', 'd1'),
-        ('tess', 'd2'),
-        ('noosa', 'n'),
-        ('girl', set(['g1', 'g2'])),
-        ('boy', set(['b1', 'b2'])),
-        ('dog', set(['d1', 'd2'])),
-        ('bark', set(['d1', 'd2'])),
-        ('walk', set(['b1', 'g2', 'd1'])),
-        ('chase', set([('b1', 'g1'), ('b2', 'g1'), ('g1', 'd1'), ('g2', 'd2')])),
-        ('see', set([('b1', 'g1'), ('b2', 'd2'), ('g1', 'b1'),('d2', 'b1'), ('g2', 'n')])),
-        ('in', set([('b1', 'n'), ('b2', 'n'), ('d2', 'n')])),
-        ('with', set([('b1', 'g1'), ('g1', 'b1'), ('d1', 'b1'), ('b1', 'd1')]))
-     ]
-    #Read in the data from ``v``
-    val = evaluate.Valuation(v)
-    #Bind ``dom`` to the ``domain`` property of ``val``
-    dom = val.domain
-    #Initialize a model with parameters ``dom`` and ``val``.
-    m0 = evaluate.Model(dom, val)
-    #Initialize a variable assignment with parameter ``dom``
-    g0 = evaluate.Assignment(dom)
-
-
-def read_sents(filename, encoding='utf8'):
-    with codecs.open(filename, 'r', encoding) as fp:
-        sents = [l.rstrip() for l in fp]
-
-    # get rid of blank lines
-    sents = [l for l in sents if len(l) > 0]
-    sents = [l for l in sents if not l[0] == '#']
-    return sents
-
-def demo_legacy_grammar():
-    """
-    Check that interpret_sents() is compatible with legacy grammars that use
-    a lowercase 'sem' feature.
-
-    Define 'test.fcfg' to be the following
-
-    """
-    from nltk.grammar import FeatureGrammar
-
-    g = FeatureGrammar.fromstring("""
-    % start S
-    S[sem=<hello>] -> 'hello'
-    """)
-    print("Reading grammar: %s" % g)
-    print("*" * 20)
-    for reading in interpret_sents(['hello'], g, semkey='sem'):
-        syn, sem = reading[0]
-        print()
-        print("output: ", sem)
-
-def demo():
-    import sys
-    from optparse import OptionParser
-    description = \
-    """
-    Parse and evaluate some sentences.
-    """
-
-    opts = OptionParser(description=description)
-
-    opts.set_defaults(evaluate=True, beta=True, syntrace=0,
-                      semtrace=0, demo='default', grammar='', sentences='')
-
-    opts.add_option("-d", "--demo", dest="demo",
-                    help="choose demo D; omit this for the default demo, or specify 'chat80'", metavar="D")
-    opts.add_option("-g", "--gram", dest="grammar",
-                    help="read in grammar G", metavar="G")
-    opts.add_option("-m", "--model", dest="model",
-                        help="import model M (omit '.py' suffix)", metavar="M")
-    opts.add_option("-s", "--sentences", dest="sentences",
-                        help="read in a file of test sentences S", metavar="S")
-    opts.add_option("-e", "--no-eval", action="store_false", dest="evaluate",
-                    help="just do a syntactic analysis")
-    opts.add_option("-b", "--no-beta-reduction", action="store_false",
-                    dest="beta", help="don't carry out beta-reduction")
-    opts.add_option("-t", "--syntrace", action="count", dest="syntrace",
-                    help="set syntactic tracing on; requires '-e' option")
-    opts.add_option("-T", "--semtrace", action="count", dest="semtrace",
-                    help="set semantic tracing on")
-
-    (options, args) = opts.parse_args()
-
-    SPACER = '-' * 30
-
-    demo_model0()
-
-    sents = [
-    'Fido sees a boy with Mary',
-    'John sees Mary',
-    'every girl chases a dog',
-    'every boy chases a girl',
-    'John walks with a girl in Noosa',
-    'who walks']
-
-    gramfile = 'grammars/sample_grammars/sem2.fcfg'
-
-    if options.sentences:
-        sentsfile = options.sentences
-    if options.grammar:
-        gramfile = options.grammar
-    if options.model:
-        exec("import %s as model" % options.model)
-
-    if sents is None:
-        sents = read_sents(sentsfile)
-
-    # Set model and assignment
-    model = m0
-    g = g0
-
-    if options.evaluate:
-        evaluations = \
-            evaluate_sents(sents, gramfile, model, g, trace=options.semtrace)
-    else:
-        semreps = \
-            interpret_sents(sents, gramfile, trace=options.syntrace)
-
-    for i, sent in enumerate(sents):
-        n = 1
-        print('\nSentence: %s' % sent)
-        print(SPACER)
-        if options.evaluate:
-
-            for (syntree, semrep, value) in evaluations[i]:
-                if isinstance(value, dict):
-                    value = set(value.keys())
-                print('%d:  %s' % (n, semrep))
-                print(value)
-                n += 1
-        else:
-
-            for (syntree, semrep) in semreps[i]:
-                print('%d:  %s' % (n, semrep))
-                n += 1
-
-if __name__ == "__main__":
-    demo()
-    demo_legacy_grammar()
+# Natural Language Toolkit: Semantic Interpretation
+#
+# Author: Ewan Klein <ewan@inf.ed.ac.uk>
+#
+# Copyright (C) 2001-2018 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Utility functions for batch-processing sentences: parsing and
+extraction of the semantic representation of the root node of the the
+syntax tree, followed by evaluation of the semantic representation in
+a first-order model.
+"""
+from __future__ import print_function, unicode_literals
+
+import codecs
+from nltk.sem import evaluate
+
+
+##############################################################
+## Utility functions for connecting parse output to semantics
+##############################################################
+
+def parse_sents(inputs, grammar, trace=0):
+    """
+    Convert input sentences into syntactic trees.
+
+    :param inputs: sentences to be parsed
+    :type inputs: list(str)
+    :param grammar: ``FeatureGrammar`` or name of feature-based grammar
+    :type grammar: nltk.grammar.FeatureGrammar
+    :rtype: list(nltk.tree.Tree) or dict(list(str)): list(Tree)
+    :return: a mapping from input sentences to a list of ``Tree``s
+    """
+    # put imports here to avoid circult dependencies
+    from nltk.grammar import FeatureGrammar
+    from nltk.parse import FeatureChartParser, load_parser
+
+    if isinstance(grammar, FeatureGrammar):
+        cp = FeatureChartParser(grammar)
+    else:
+        cp = load_parser(grammar, trace=trace)
+    parses = []
+    for sent in inputs:
+        tokens = sent.split() # use a tokenizer?
+        syntrees = list(cp.parse(tokens))
+        parses.append(syntrees)
+    return parses
+
+def root_semrep(syntree, semkey='SEM'):
+    """
+    Find the semantic representation at the root of a tree.
+
+    :param syntree: a parse ``Tree``
+    :param semkey: the feature label to use for the root semantics in the tree
+    :return: the semantic representation at the root of a ``Tree``
+    :rtype: sem.Expression
+    """
+    from nltk.grammar import FeatStructNonterminal
+
+    node = syntree.label()
+    assert isinstance(node, FeatStructNonterminal)
+    try:
+        return node[semkey]
+    except KeyError:
+        print(node, end=' ')
+        print("has no specification for the feature %s" % semkey)
+    raise
+
+def interpret_sents(inputs, grammar, semkey='SEM', trace=0):
+    """
+    Add the semantic representation to each syntactic parse tree
+    of each input sentence.
+
+    :param inputs: a list of sentences
+    :type inputs: list(str)
+    :param grammar: ``FeatureGrammar`` or name of feature-based grammar
+    :type grammar: nltk.grammar.FeatureGrammar
+    :return: a mapping from sentences to lists of pairs (parse-tree, semantic-representations)
+    :rtype: list(list(tuple(nltk.tree.Tree, nltk.sem.logic.ConstantExpression)))
+    """
+    return [[(syn, root_semrep(syn, semkey)) for syn in syntrees]
+            for syntrees in parse_sents(inputs, grammar, trace=trace)]
+
+def evaluate_sents(inputs, grammar, model, assignment, trace=0):
+    """
+    Add the truth-in-a-model value to each semantic representation
+    for each syntactic parse of each input sentences.
+
+    :param inputs: a list of sentences
+    :type inputs: list(str)
+    :param grammar: ``FeatureGrammar`` or name of feature-based grammar
+    :type grammar: nltk.grammar.FeatureGrammar
+    :return: a mapping from sentences to lists of triples (parse-tree, semantic-representations, evaluation-in-model)
+    :rtype: list(list(tuple(nltk.tree.Tree, nltk.sem.logic.ConstantExpression, bool or dict(str): bool)))
+    """
+    return [[(syn, sem, model.evaluate("%s" % sem, assignment, trace=trace))
+            for (syn, sem) in interpretations]
+            for interpretations in interpret_sents(inputs, grammar)]
+
+
+def demo_model0():
+    global m0, g0
+    #Initialize a valuation of non-logical constants."""
+    v = [('john', 'b1'),
+        ('mary', 'g1'),
+        ('suzie', 'g2'),
+        ('fido', 'd1'),
+        ('tess', 'd2'),
+        ('noosa', 'n'),
+        ('girl', set(['g1', 'g2'])),
+        ('boy', set(['b1', 'b2'])),
+        ('dog', set(['d1', 'd2'])),
+        ('bark', set(['d1', 'd2'])),
+        ('walk', set(['b1', 'g2', 'd1'])),
+        ('chase', set([('b1', 'g1'), ('b2', 'g1'), ('g1', 'd1'), ('g2', 'd2')])),
+        ('see', set([('b1', 'g1'), ('b2', 'd2'), ('g1', 'b1'),('d2', 'b1'), ('g2', 'n')])),
+        ('in', set([('b1', 'n'), ('b2', 'n'), ('d2', 'n')])),
+        ('with', set([('b1', 'g1'), ('g1', 'b1'), ('d1', 'b1'), ('b1', 'd1')]))
+     ]
+    #Read in the data from ``v``
+    val = evaluate.Valuation(v)
+    #Bind ``dom`` to the ``domain`` property of ``val``
+    dom = val.domain
+    #Initialize a model with parameters ``dom`` and ``val``.
+    m0 = evaluate.Model(dom, val)
+    #Initialize a variable assignment with parameter ``dom``
+    g0 = evaluate.Assignment(dom)
+
+
+def read_sents(filename, encoding='utf8'):
+    with codecs.open(filename, 'r', encoding) as fp:
+        sents = [l.rstrip() for l in fp]
+
+    # get rid of blank lines
+    sents = [l for l in sents if len(l) > 0]
+    sents = [l for l in sents if not l[0] == '#']
+    return sents
+
+def demo_legacy_grammar():
+    """
+    Check that interpret_sents() is compatible with legacy grammars that use
+    a lowercase 'sem' feature.
+
+    Define 'test.fcfg' to be the following
+
+    """
+    from nltk.grammar import FeatureGrammar
+
+    g = FeatureGrammar.fromstring("""
+    % start S
+    S[sem=<hello>] -> 'hello'
+    """)
+    print("Reading grammar: %s" % g)
+    print("*" * 20)
+    for reading in interpret_sents(['hello'], g, semkey='sem'):
+        syn, sem = reading[0]
+        print()
+        print("output: ", sem)
+
+def demo():
+    import sys
+    from optparse import OptionParser
+    description = \
+    """
+    Parse and evaluate some sentences.
+    """
+
+    opts = OptionParser(description=description)
+
+    opts.set_defaults(evaluate=True, beta=True, syntrace=0,
+                      semtrace=0, demo='default', grammar='', sentences='')
+
+    opts.add_option("-d", "--demo", dest="demo",
+                    help="choose demo D; omit this for the default demo, or specify 'chat80'", metavar="D")
+    opts.add_option("-g", "--gram", dest="grammar",
+                    help="read in grammar G", metavar="G")
+    opts.add_option("-m", "--model", dest="model",
+                        help="import model M (omit '.py' suffix)", metavar="M")
+    opts.add_option("-s", "--sentences", dest="sentences",
+                        help="read in a file of test sentences S", metavar="S")
+    opts.add_option("-e", "--no-eval", action="store_false", dest="evaluate",
+                    help="just do a syntactic analysis")
+    opts.add_option("-b", "--no-beta-reduction", action="store_false",
+                    dest="beta", help="don't carry out beta-reduction")
+    opts.add_option("-t", "--syntrace", action="count", dest="syntrace",
+                    help="set syntactic tracing on; requires '-e' option")
+    opts.add_option("-T", "--semtrace", action="count", dest="semtrace",
+                    help="set semantic tracing on")
+
+    (options, args) = opts.parse_args()
+
+    SPACER = '-' * 30
+
+    demo_model0()
+
+    sents = [
+    'Fido sees a boy with Mary',
+    'John sees Mary',
+    'every girl chases a dog',
+    'every boy chases a girl',
+    'John walks with a girl in Noosa',
+    'who walks']
+
+    gramfile = 'grammars/sample_grammars/sem2.fcfg'
+
+    if options.sentences:
+        sentsfile = options.sentences
+    if options.grammar:
+        gramfile = options.grammar
+    if options.model:
+        exec("import %s as model" % options.model)
+
+    if sents is None:
+        sents = read_sents(sentsfile)
+
+    # Set model and assignment
+    model = m0
+    g = g0
+
+    if options.evaluate:
+        evaluations = \
+            evaluate_sents(sents, gramfile, model, g, trace=options.semtrace)
+    else:
+        semreps = \
+            interpret_sents(sents, gramfile, trace=options.syntrace)
+
+    for i, sent in enumerate(sents):
+        n = 1
+        print('\nSentence: %s' % sent)
+        print(SPACER)
+        if options.evaluate:
+
+            for (syntree, semrep, value) in evaluations[i]:
+                if isinstance(value, dict):
+                    value = set(value.keys())
+                print('%d:  %s' % (n, semrep))
+                print(value)
+                n += 1
+        else:
+
+            for (syntree, semrep) in semreps[i]:
+                print('%d:  %s' % (n, semrep))
+                n += 1
+
+if __name__ == "__main__":
+    demo()
+    demo_legacy_grammar()

{nltk-3.2 → nltk-3.3}/nltk/sentiment/__init__.py

@@ -1,14 +1,14 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: Sentiment Analysis
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Ewan Klein <ewan@inf.ed.ac.uk>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-NLTK Sentiment Analysis Package
-
-"""
-from nltk.sentiment.sentiment_analyzer import SentimentAnalyzer
-from nltk.sentiment.vader import SentimentIntensityAnalyzer
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Sentiment Analysis
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Ewan Klein <ewan@inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+NLTK Sentiment Analysis Package
+
+"""
+from nltk.sentiment.sentiment_analyzer import SentimentAnalyzer
+from nltk.sentiment.vader import SentimentIntensityAnalyzer