nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/grammar.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Context Free Grammars
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 #         Jason Narad <jason.narad@gmail.com>
@@ -68,18 +68,21 @@ The operation of replacing the left hand side (*lhs*) of a production
 with the right hand side (*rhs*) in a tree (*tree*) is known as
 "expanding" *lhs* to *rhs* in *tree*.
 """
-from __future__ import print_function, unicode_literals, division 
+from __future__ import print_function, unicode_literals, division
 
 import re
+from functools import total_ordering
+
+from six import string_types
 
 from nltk.util import transitive_closure, invert_graph
-from nltk.compat import (string_types, total_ordering, text_type,
-                         python_2_unicode_compatible, unicode_repr)
+from nltk.compat import python_2_unicode_compatible, unicode_repr
 from nltk.internals import raise_unorderable_types
 
 from nltk.probability import ImmutableProbabilisticMixIn
 from nltk.featstruct import FeatStruct, FeatDict, FeatStructReader, SLASH, TYPE
 
+
 #################################################################
 # Nonterminal
 #################################################################
@@ -104,6 +107,7 @@ class Nonterminal(object):
     :ivar _symbol: The node value corresponding to this
         ``Nonterminal``.  This value must be immutable and hashable.
     """
+
     def __init__(self, symbol):
         """
         Construct a new non-terminal from the given symbol.
@@ -179,6 +183,21 @@ class Nonterminal(object):
         """
         return Nonterminal('%s/%s' % (self._symbol, rhs._symbol))
 
+    def __truediv__(self, rhs):
+        """
+        Return a new nonterminal whose symbol is ``A/B``, where ``A`` is
+        the symbol for this nonterminal, and ``B`` is the symbol for rhs.
+        This function allows use of the slash ``/`` operator with
+        the future import of division.
+
+        :param rhs: The nonterminal used to form the right hand side
+            of the new nonterminal.
+        :type rhs: Nonterminal
+        :rtype: Nonterminal
+        """
+        return self.__div__(rhs)
+
+
 def nonterminals(symbols):
     """
     Given a string containing a list of symbol names, return a list of
@@ -192,19 +211,25 @@ def nonterminals(symbols):
         in the same order as the symbols names.
     :rtype: list(Nonterminal)
     """
-    if ',' in symbols: symbol_list = symbols.split(',')
-    else: symbol_list = symbols.split()
+    if ',' in symbols:
+        symbol_list = symbols.split(',')
+    else:
+        symbol_list = symbols.split()
     return [Nonterminal(s.strip()) for s in symbol_list]
 
+
 class FeatStructNonterminal(FeatDict, Nonterminal):
     """A feature structure that's also a nonterminal.  It acts as its
     own symbol, and automatically freezes itself when hashed."""
+
     def __hash__(self):
         self.freeze()
         return FeatStruct.__hash__(self)
+
     def symbol(self):
         return self
 
+
 def is_nonterminal(item):
     """
     :return: True if the item is a ``Nonterminal``.
@@ -360,6 +385,7 @@ class DependencyProduction(Production):
     A dependency grammar production.  Each production maps a single
     head word to an unordered list of one or more modifier words.
     """
+
     def __str__(self):
         """
         Return a verbose string representation of the ``DependencyProduction``.
@@ -384,6 +410,7 @@ class ProbabilisticProduction(Production, ImmutableProbabilisticMixIn):
 
     :see: ``Production``
     """
+
     def __init__(self, lhs, rhs, **prob):
         """
         Construct a new ``ProbabilisticProduction``.
@@ -399,7 +426,7 @@ class ProbabilisticProduction(Production, ImmutableProbabilisticMixIn):
 
     def __str__(self):
         return Production.__unicode__(self) + \
-            (' [1.0]' if (self.prob() == 1.0) else ' [%g]' % self.prob())
+               (' [1.0]' if (self.prob() == 1.0) else ' [%g]' % self.prob())
 
     def __eq__(self, other):
         return (type(self) == type(other) and
@@ -413,6 +440,7 @@ class ProbabilisticProduction(Production, ImmutableProbabilisticMixIn):
     def __hash__(self):
         return hash((self._lhs, self._rhs, self.prob()))
 
+
 #################################################################
 # Grammars
 #################################################################
@@ -427,6 +455,7 @@ class CFG(object):
     If you need efficient key-based access to productions, you
     can use a subclass to implement it.
     """
+
     def __init__(self, start, productions, calculate_leftcorners=True):
         """
         Create a new context-free grammar, from the given start state
@@ -511,13 +540,13 @@ class CFG(object):
     @classmethod
     def fromstring(cls, input, encoding=None):
         """
-        Return the ``CFG`` corresponding to the input string(s).
+        Return the grammar instance corresponding to the input string(s).
 
         :param input: a grammar, either in the form of a string or as a list of strings.
         """
         start, productions = read_grammar(input, standard_nonterm_parser,
                                           encoding=encoding)
-        return CFG(start, productions)
+        return cls(start, productions)
 
     def start(self):
         """
@@ -723,6 +752,7 @@ class FeatureGrammar(CFG):
     productions.  The set of terminals and nonterminals
     is implicitly specified by the productions.
     """
+
     def __init__(self, start, productions):
         """
         Create a new feature-based grammar, from the given start
@@ -770,9 +800,9 @@ class FeatureGrammar(CFG):
 
     @classmethod
     def fromstring(cls, input, features=None, logic_parser=None, fstruct_reader=None,
-               encoding=None):
+                   encoding=None):
         """
-        Return a feature structure based ``FeatureGrammar``.
+        Return a feature structure based grammar.
 
         :param input: a grammar, either in the form of a string or else
         as a list of strings.
@@ -794,8 +824,7 @@ class FeatureGrammar(CFG):
 
         start, productions = read_grammar(input, fstruct_reader.read_partial,
                                           encoding=encoding)
-        return FeatureGrammar(start, productions)
-
+        return cls(start, productions)
 
     def productions(self, lhs=None, rhs=None, empty=False):
         """
@@ -859,6 +888,7 @@ class FeatureGrammar(CFG):
         else:
             return item
 
+
 @total_ordering
 @python_2_unicode_compatible
 class FeatureValueType(object):
@@ -867,6 +897,7 @@ class FeatureValueType(object):
     from ordinary strings.  This is to stop the ``FeatStruct``
     ``FOO[]`` from being compare equal to the terminal "FOO".
     """
+
     def __init__(self, value):
         self._value = value
         self._hash = hash(value)
@@ -896,6 +927,7 @@ class DependencyGrammar(object):
     productions.  Each production specifies a head/modifier relationship
     between a pair of words.
     """
+
     def __init__(self, productions):
         """
         Create a new dependency grammar, from the set of ``Productions``.
@@ -910,13 +942,14 @@ class DependencyGrammar(object):
         productions = []
         for linenum, line in enumerate(input.split('\n')):
             line = line.strip()
-            if line.startswith('#') or line=='': continue
-            try: productions += _read_dependency_production(line)
+            if line.startswith('#') or line == '': continue
+            try:
+                productions += _read_dependency_production(line)
             except ValueError:
                 raise ValueError('Unable to parse line %s: %s' % (linenum, line))
         if len(productions) == 0:
             raise ValueError('No productions found!')
-        return DependencyGrammar(productions)
+        return cls(productions)
 
     def contains(self, head, mod):
         """
@@ -931,7 +964,7 @@ class DependencyGrammar(object):
         """
         for production in self._productions:
             for possibleMod in production._rhs:
-                if(production._lhs == head and possibleMod == mod):
+                if (production._lhs == head and possibleMod == mod):
                     return True
         return False
 
@@ -948,7 +981,7 @@ class DependencyGrammar(object):
         """
         for production in self._productions:
             for possibleMod in production._rhs:
-                if(production._lhs == head and possibleMod == mod):
+                if (production._lhs == head and possibleMod == mod):
                     return True
         return False
 
@@ -963,7 +996,6 @@ class DependencyGrammar(object):
     #                   return True
     #   return False
 
-
     def __str__(self):
         """
         Return a verbose string representation of the ``DependencyGrammar``
@@ -1006,7 +1038,7 @@ class ProbabilisticDependencyGrammar(object):
         """
         for production in self._productions:
             for possibleMod in production._rhs:
-                if(production._lhs == head and possibleMod == mod):
+                if (production._lhs == head and possibleMod == mod):
                     return True
         return False
 
@@ -1079,15 +1111,14 @@ class PCFG(CFG):
             probs[production.lhs()] = (probs.get(production.lhs(), 0) +
                                        production.prob())
         for (lhs, p) in probs.items():
-            if not ((1-PCFG.EPSILON) < p <
-                    (1+PCFG.EPSILON)):
+            if not ((1 - PCFG.EPSILON) < p <
+                    (1 + PCFG.EPSILON)):
                 raise ValueError("Productions for %r do not sum to 1" % lhs)
 
-
     @classmethod
     def fromstring(cls, input, encoding=None):
         """
-        Return a probabilistic ``PCFG`` corresponding to the
+        Return a probabilistic context-free grammar corresponding to the
         input string(s).
 
         :param input: a grammar, either in the form of a string or else
@@ -1095,7 +1126,7 @@ class PCFG(CFG):
         """
         start, productions = read_grammar(input, standard_nonterm_parser,
                                           probabilistic=True, encoding=encoding)
-        return PCFG(start, productions)
+        return cls(start, productions)
 
 
 #################################################################
@@ -1127,10 +1158,10 @@ def induce_pcfg(start, productions):
 
     for prod in productions:
         lcount[prod.lhs()] = lcount.get(prod.lhs(), 0) + 1
-        pcount[prod]       = pcount.get(prod,       0) + 1
+        pcount[prod] = pcount.get(prod, 0) + 1
 
     prods = [ProbabilisticProduction(p.lhs(), p.rhs(),
-                                prob=pcount[p] / lcount[p.lhs()])
+                                     prob=pcount[p] / lcount[p.lhs()])
              for p in pcount]
     return PCFG(start, prods)
 
@@ -1145,12 +1176,14 @@ def _read_cfg_production(input):
     """
     return _read_production(input, standard_nonterm_parser)
 
+
 def _read_pcfg_production(input):
     """
     Return a list of PCFG ``ProbabilisticProductions``.
     """
     return _read_production(input, standard_nonterm_parser, probabilistic=True)
 
+
 def _read_fcfg_production(input, fstruct_reader):
     """
     Return a list of feature-based ``Productions``.
@@ -1165,6 +1198,7 @@ _PROBABILITY_RE = re.compile(r'( \[ [\d\.]+ \] ) \s*', re.VERBOSE)
 _TERMINAL_RE = re.compile(r'( "[^"]+" | \'[^\']+\' ) \s*', re.VERBOSE)
 _DISJUNCTION_RE = re.compile(r'\| \s*', re.VERBOSE)
 
+
 def _read_production(line, nonterm_parser, probabilistic=False):
     """
     Parse a grammar rule, given as a string, and return
@@ -1251,9 +1285,9 @@ def read_grammar(input, nonterm_parser, probabilistic=False, encoding=None):
     continue_line = ''
     for linenum, line in enumerate(lines):
         line = continue_line + line.strip()
-        if line.startswith('#') or line=='': continue
+        if line.startswith('#') or line == '': continue
         if line.endswith('\\'):
-            continue_line = line[:-1].rstrip()+' '
+            continue_line = line[:-1].rstrip() + ' '
             continue
         continue_line = ''
         try:
@@ -1270,7 +1304,7 @@ def read_grammar(input, nonterm_parser, probabilistic=False, encoding=None):
                 productions += _read_production(line, nonterm_parser, probabilistic)
         except ValueError as e:
             raise ValueError('Unable to parse line %s: %s\n%s' %
-                             (linenum+1, line, e))
+                             (linenum + 1, line, e))
 
     if not productions:
         raise ValueError('No productions found!')
@@ -1278,8 +1312,10 @@ def read_grammar(input, nonterm_parser, probabilistic=False, encoding=None):
         start = productions[0].lhs()
     return (start, productions)
 
+
 _STANDARD_NONTERM_RE = re.compile('( [\w/][\w/^<>-]* ) \s*', re.VERBOSE)
 
+
 def standard_nonterm_parser(string, pos):
     m = _STANDARD_NONTERM_RE.match(string, pos)
     if not m: raise ValueError('Expected a nonterminal, found: '
@@ -1300,15 +1336,16 @@ _READ_DG_RE = re.compile(r'''^\s*                # leading whitespace
                                  | \|              # disjunction
                                  )
                                  \s*)              # trailing space
-                                 *$''',            # zero or more copies
-                             re.VERBOSE)
+                                 *$''',  # zero or more copies
+                         re.VERBOSE)
 _SPLIT_DG_RE = re.compile(r'''('[^']'|[-=]+>|"[^"]+"|'[^']+'|\|)''')
 
+
 def _read_dependency_production(s):
     if not _READ_DG_RE.match(s):
         raise ValueError('Bad production string')
     pieces = _SPLIT_DG_RE.split(s)
-    pieces = [p for i,p in enumerate(pieces) if i%2==1]
+    pieces = [p for i, p in enumerate(pieces) if i % 2 == 1]
     lhside = pieces[0].strip('\'\"')
     rhsides = [[]]
     for piece in pieces[2:]:
@@ -1333,9 +1370,9 @@ def cfg_demo():
     # Create some nonterminals
     S, NP, VP, PP = nonterminals('S, NP, VP, PP')
     N, V, P, Det = nonterminals('N, V, P, Det')
-    VP_slash_NP = VP/NP
+    VP_slash_NP = VP / NP
 
-    print('Some nonterminals:', [S, NP, VP, PP, N, V, P, Det, VP/NP])
+    print('Some nonterminals:', [S, NP, VP, PP, N, V, P, Det, VP / NP])
     print('    S.symbol() =>', repr(S.symbol()))
     print()
 
@@ -1357,9 +1394,10 @@ def cfg_demo():
     print('    grammar.start()       =>', repr(grammar.start()))
     print('    grammar.productions() =>', end=' ')
     # Use string.replace(...) is to line-wrap the output.
-    print(repr(grammar.productions()).replace(',', ',\n'+' '*25))
+    print(repr(grammar.productions()).replace(',', ',\n' + ' ' * 25))
     print()
 
+
 toy_pcfg1 = PCFG.fromstring("""
     S -> NP VP [1.0]
     NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
@@ -1397,6 +1435,7 @@ toy_pcfg2 = PCFG.fromstring("""
     Det  -> 'my'          [.28]
     """)
 
+
 def pcfg_demo():
     """
     A demonstration showing how a ``PCFG`` can be created and used.
@@ -1421,7 +1460,7 @@ def pcfg_demo():
     print('    grammar.start()       =>', repr(grammar.start()))
     print('    grammar.productions() =>', end=' ')
     # Use .replace(...) is to line-wrap the output.
-    print(repr(grammar.productions()).replace(',', ',\n'+' '*26))
+    print(repr(grammar.productions()).replace(',', ',\n' + ' ' * 26))
     print()
 
     # extract productions from three trees and induce the PCFG
@@ -1431,8 +1470,8 @@ def pcfg_demo():
     item = treebank._fileids[0]
     for tree in treebank.parsed_sents(item)[:3]:
         # perform optional tree transformations, e.g.:
-        tree.collapse_unary(collapsePOS = False)
-        tree.chomsky_normal_form(horzMarkov = 2)
+        tree.collapse_unary(collapsePOS=False)
+        tree.chomsky_normal_form(horzMarkov=2)
 
         productions += tree.productions()
 
@@ -1447,19 +1486,21 @@ def pcfg_demo():
     parser.trace(3)
 
     # doesn't work as tokens are different:
-    #sent = treebank.tokenized('wsj_0001.mrg')[0]
+    # sent = treebank.tokenized('wsj_0001.mrg')[0]
 
     sent = treebank.parsed_sents(item)[0].leaves()
     print(sent)
     for parse in parser.parse(sent):
         print(parse)
 
+
 def fcfg_demo():
     import nltk.data
     g = nltk.data.load('grammars/book_grammars/feat0.fcfg')
     print(g)
     print()
 
+
 def dg_demo():
     """
     A demonstration showing the creation and inspection of a
@@ -1472,6 +1513,7 @@ def dg_demo():
     """)
     print(grammar)
 
+
 def sdg_demo():
     """
     A demonstration of how to read a string representation of
@@ -1497,6 +1539,7 @@ def sdg_demo():
     tree = dg.tree()
     print(tree.pprint())
 
+
 def demo():
     cfg_demo()
     pcfg_demo()
@@ -1504,13 +1547,13 @@ def demo():
     dg_demo()
     sdg_demo()
 
+
 if __name__ == '__main__':
     demo()
 
 __all__ = ['Nonterminal', 'nonterminals',
            'CFG', 'Production',
-	   'PCFG', 'ProbabilisticProduction',
-	   'DependencyGrammar', 'DependencyProduction',
+           'PCFG', 'ProbabilisticProduction',
+           'DependencyGrammar', 'DependencyProduction',
            'ProbabilisticDependencyGrammar',
-	   'induce_pcfg', 'read_grammar']
-
+           'induce_pcfg', 'read_grammar']

{nltk-3.2.1 → nltk-3.3}/nltk/help.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit (NLTK) Help
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Authors: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/inference/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Inference
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Dan Garrette <dhgarrette@gmail.com>
 #         Ewan Klein <ewan@inf.ed.ac.uk>
 #

{nltk-3.2.1 → nltk-3.3}/nltk/inference/api.py

@@ -18,10 +18,14 @@ goal *G*, the model builder tries to find a counter-model, in the sense of a mod
 the assumptions plus the negation of *G*.
 """
 from __future__ import print_function
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
 
 import threading
 import time
 
+
+@add_metaclass(ABCMeta)
 class Prover(object):
     """
     Interface for trying to prove a goal from assumptions.  Both the goal and
@@ -34,13 +38,15 @@ class Prover(object):
         """
         return self._prove(goal, assumptions, verbose)[0]
 
+    @abstractmethod
     def _prove(self, goal=None, assumptions=None, verbose=False):
         """
         :return: Whether the proof was successful or not, along with the proof
         :rtype: tuple: (bool, str)
         """
-        raise NotImplementedError()
 
+
+@add_metaclass(ABCMeta)
 class ModelBuilder(object):
     """
     Interface for trying to build a model of set of formulas.
@@ -56,20 +62,22 @@ class ModelBuilder(object):
         """
         return self._build_model(goal, assumptions, verbose)[0]
 
+    @abstractmethod
     def _build_model(self, goal=None, assumptions=None, verbose=False):
         """
         Perform the actual model building.
         :return: Whether a model was generated, and the model itself
         :rtype: tuple(bool, sem.Valuation)
         """
-        raise NotImplementedError()
 
 
+@add_metaclass(ABCMeta)
 class TheoremToolCommand(object):
     """
     This class holds a goal and a list of assumptions to be used in proving
     or model building.
     """
+    @abstractmethod
     def add_assumptions(self, new_assumptions):
         """
         Add new assumptions to the assumption list.
@@ -77,8 +85,8 @@ class TheoremToolCommand(object):
         :param new_assumptions: new assumptions
         :type new_assumptions: list(sem.Expression)
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def retract_assumptions(self, retracted, debug=False):
         """
         Retract assumptions from the assumption list.
@@ -89,29 +97,28 @@ class TheoremToolCommand(object):
         :param retracted: assumptions to be retracted
         :type retracted: list(sem.Expression)
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def assumptions(self):
         """
         List the current assumptions.
 
         :return: list of ``Expression``
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def goal(self):
         """
         Return the goal
 
         :return: ``Expression``
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def print_assumptions(self):
         """
         Print the list of the current assumptions.
         """
-        raise NotImplementedError()
 
 
 class ProverCommand(TheoremToolCommand):
@@ -119,26 +126,26 @@ class ProverCommand(TheoremToolCommand):
     This class holds a ``Prover``, a goal, and a list of assumptions.  When
     prove() is called, the ``Prover`` is executed with the goal and assumptions.
     """
+    @abstractmethod
     def prove(self, verbose=False):
         """
         Perform the actual proof.
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def proof(self, simplify=True):
         """
         Return the proof string
         :param simplify: bool simplify the proof?
         :return: str
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def get_prover(self):
         """
         Return the prover object
         :return: ``Prover``
         """
-        raise NotImplementedError()
 
 
 class ModelBuilderCommand(TheoremToolCommand):
@@ -147,14 +154,15 @@ class ModelBuilderCommand(TheoremToolCommand):
     When build_model() is called, the ``ModelBuilder`` is executed with the goal
     and assumptions.
     """
+    @abstractmethod
     def build_model(self, verbose=False):
         """
         Perform the actual model building.
         :return: A model if one is generated; None otherwise.
         :rtype: sem.Valuation
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def model(self, format=None):
         """
         Return a string representation of the model
@@ -162,14 +170,13 @@ class ModelBuilderCommand(TheoremToolCommand):
         :param simplify: bool simplify the proof?
         :return: str
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def get_model_builder(self):
         """
         Return the model builder object
         :return: ``ModelBuilder``
         """
-        raise NotImplementedError()
 
 
 class BaseTheoremToolCommand(TheoremToolCommand):
@@ -369,8 +376,8 @@ class TheoremToolCommandDecorator(TheoremToolCommand):
         """
         self._command = command
 
-        #The decorator has its own versions of 'result' different from the
-        #underlying command
+        # The decorator has its own versions of 'result' different from the
+        # underlying command
         self._result = None
 
     def assumptions(self):
@@ -402,8 +409,8 @@ class ProverCommandDecorator(TheoremToolCommandDecorator, ProverCommand):
         """
         TheoremToolCommandDecorator.__init__(self, proverCommand)
 
-        #The decorator has its own versions of 'result' and 'proof'
-        #because they may be different from the underlying command
+        # The decorator has its own versions of 'result' and 'proof'
+        # because they may be different from the underlying command
         self._proof = None
 
     def prove(self, verbose=False):
@@ -449,8 +456,8 @@ class ModelBuilderCommandDecorator(TheoremToolCommandDecorator, ModelBuilderComm
         """
         TheoremToolCommandDecorator.__init__(self, modelBuilderCommand)
 
-        #The decorator has its own versions of 'result' and 'valuation'
-        #because they may be different from the underlying command
+        # The decorator has its own versions of 'result' and 'valuation'
+        # because they may be different from the underlying command
         self._model = None
 
     def build_model(self, verbose=False):
@@ -528,6 +535,7 @@ class ParallelProverBuilder(Prover, ModelBuilder):
         else:
             return None
 
+
 class ParallelProverBuilderCommand(BaseProverCommand, BaseModelBuilderCommand):
     """
     This command stores both a prover and a model builder and when either