nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/inference/discourse.py

@@ -43,6 +43,8 @@ The set of all threads for a discourse is the Cartesian product of all the readi
 those threads which are consistent (taking into account any background assumptions).
 """
 from __future__ import print_function
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
 import os
 
 from operator import and_, add
@@ -60,13 +62,14 @@ from nltk.inference.mace import MaceCommand
 from nltk.inference.prover9 import Prover9Command
 
 
+@add_metaclass(ABCMeta)
 class ReadingCommand(object):
+    @abstractmethod
     def parse_to_readings(self, sentence):
         """
         :param sentence: the sentence to read
         :type sentence: str
         """
-        raise NotImplementedError()
 
     def process_thread(self, sentence_readings):
         """
@@ -80,6 +83,7 @@ class ReadingCommand(object):
         """
         return sentence_readings
 
+    @abstractmethod
     def combine_readings(self, readings):
         """
         :param readings: readings to combine
@@ -87,8 +91,8 @@ class ReadingCommand(object):
         :return: one combined reading
         :rtype: Expression
         """
-        raise NotImplementedError()
 
+    @abstractmethod
     def to_fol(self, expression):
         """
         Convert this expression into a First-Order Logic expression.
@@ -98,7 +102,6 @@ class ReadingCommand(object):
         :return: a FOL version of the input expression
         :rtype: Expression
         """
-        raise NotImplementedError()
 
 
 class CfgReadingCommand(ReadingCommand):
@@ -299,7 +302,6 @@ class DiscourseTester(object):
             if (tid, True) in consistency_checked:
                 self._filtered_threads[tid] = thread
 
-
     def _show_readings(self, sentence=None):
         """
         Print out the readings for  the discourse (or a single sentence).
@@ -492,6 +494,7 @@ class DiscourseTester(object):
 #L2 = ['a', 'b', 'c']
 #print multiply(L1,L2)
 
+
 def load_fol(s):
     """
     Temporarily duplicated from ``nltk.sem.util``.
@@ -505,17 +508,18 @@ def load_fol(s):
     statements = []
     for linenum, line in enumerate(s.splitlines()):
         line = line.strip()
-        if line.startswith('#') or line=='': continue
+        if line.startswith('#') or line == '':
+            continue
         try:
             statements.append(Expression.fromstring(line))
         except Exception:
             raise ValueError('Unable to parse line %s: %s' % (linenum, line))
     return statements
 
+
 ###############################
 # Demo
 ###############################
-
 def discourse_demo(reading_command=None):
     """
     Illustrate the various methods of ``DiscourseTester``
@@ -524,7 +528,7 @@ def discourse_demo(reading_command=None):
                          reading_command)
     dt.models()
     print()
-    #dt.grammar()
+    # dt.grammar()
     print()
     dt.sentences()
     print()
@@ -555,12 +559,12 @@ def discourse_demo(reading_command=None):
     dt.add_sentence('A person dances', informchk=True)
     dt = DiscourseTester(['Vincent is a boxer', 'Fido is a boxer',
                           'Vincent is married', 'Fido barks'],
-                          reading_command)
+                         reading_command)
     dt.readings(filter=True)
     import nltk.data
     background_file = os.path.join('grammars', 'book_grammars', 'background.fol')
     background = nltk.data.load(background_file)
-    
+
     print()
     dt.add_background(background, verbose=False)
     dt.background()
@@ -590,19 +594,19 @@ def drt_discourse_demo(reading_command=None):
 def spacer(num=30):
     print('-' * num)
 
+
 def demo():
     discourse_demo()
 
-    tagger = RegexpTagger(
-        [('^(chases|runs)$', 'VB'),
-         ('^(a)$', 'ex_quant'),
-         ('^(every)$', 'univ_quant'),
-         ('^(dog|boy)$', 'NN'),
-         ('^(he)$', 'PRP')
-    ])
+    tagger = RegexpTagger([('^(chases|runs)$', 'VB'),
+                           ('^(a)$', 'ex_quant'),
+                           ('^(every)$', 'univ_quant'),
+                           ('^(dog|boy)$', 'NN'),
+                           ('^(he)$', 'PRP')])
     depparser = MaltParser(tagger=tagger)
     drt_discourse_demo(DrtGlueReadingCommand(remove_duplicates=False,
                                              depparser=depparser))
 
+
 if __name__ == '__main__':
     demo()

{nltk-3.2.1 → nltk-3.3}/nltk/inference/nonmonotonic.py

@@ -2,7 +2,7 @@
 #
 # Author: Daniel H. Garrette <dhgarrette@gmail.com>
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # URL: <http://nltk.org>
 # For license information, see LICENSE.TXT
 

{nltk-3.2.1 → nltk-3.3}/nltk/inference/prover9.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Interface to the Prover9 Theorem Prover
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Dan Garrette <dhgarrette@gmail.com>
 #         Ewan Klein <ewan@inf.ed.ac.uk>
 #

{nltk-3.2.1 → nltk-3.3}/nltk/inference/resolution.py

@@ -2,7 +2,7 @@
 #
 # Author: Dan Garrette <dhgarrette@gmail.com>
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # URL: <http://nltk.org>
 # For license information, see LICENSE.TXT
 

{nltk-3.2.1 → nltk-3.3}/nltk/inference/tableau.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: First-Order Tableau Theorem Prover
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Dan Garrette <dhgarrette@gmail.com>
 #
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/internals.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Internal utility functions
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 #         Nitin Madnani <nmadnani@ets.org>
@@ -25,6 +25,8 @@ try:
 except ImportError:
     from xml.etree import ElementTree
 
+from six import string_types
+
 from nltk import __file__
 from nltk import compat
 
@@ -35,7 +37,7 @@ from nltk import compat
 _java_bin = None
 _java_options = []
 # [xx] add classpath option to config_java?
-def config_java(bin=None, options=None, verbose=True):
+def config_java(bin=None, options=None, verbose=False):
     """
     Configure nltk's java interface, by letting nltk know where it can
     find the Java binary, and what extra options (if any) should be
@@ -56,7 +58,7 @@ def config_java(bin=None, options=None, verbose=True):
     _java_bin = find_binary('java', bin, env_vars=['JAVAHOME', 'JAVA_HOME'], verbose=verbose, binary_names=['java.exe'])
 
     if options is not None:
-        if isinstance(options, compat.string_types):
+        if isinstance(options, string_types):
             options = options.split()
         _java_options = list(options)
 
@@ -104,7 +106,7 @@ def java(cmd, classpath=None, stdin=None, stdout=None, stderr=None,
     if stdin == 'pipe': stdin = subprocess.PIPE
     if stdout == 'pipe': stdout = subprocess.PIPE
     if stderr == 'pipe': stderr = subprocess.PIPE
-    if isinstance(cmd, compat.string_types):
+    if isinstance(cmd, string_types):
         raise TypeError('cmd should be a list of strings')
 
     # Make sure we know where a java binary is.
@@ -112,7 +114,7 @@ def java(cmd, classpath=None, stdin=None, stdout=None, stderr=None,
         config_java()
 
     # Set up the classpath.
-    if isinstance(classpath, compat.string_types):
+    if isinstance(classpath, string_types):
         classpaths=[classpath]
     else:
         classpaths=list(classpath)
@@ -173,21 +175,21 @@ def read_str(s, start_position):
     containing the value of the string literal and the position where
     it ends.  Otherwise, raise a ``ReadError``.
 
-    :param s: A string that will be checked to see if within which a 
+    :param s: A string that will be checked to see if within which a
         Python string literal exists.
     :type s: str
-    
+
     :param start_position: The specified beginning position of the string ``s``
         to begin regex matching.
     :type start_position: int
-    
-    :return: A tuple containing the matched string literal evaluated as a 
+
+    :return: A tuple containing the matched string literal evaluated as a
         string and the end position of the string literal.
     :rtype: tuple(str, int)
 
     :raise ReadError: If the ``_STRING_START_RE`` regex doesn't return a
-        match in ``s`` at ``start_position``, i.e., open quote. If the 
-        ``_STRING_END_RE`` regex doesn't return a match in ``s`` at the 
+        match in ``s`` at ``start_position``, i.e., open quote. If the
+        ``_STRING_END_RE`` regex doesn't return a match in ``s`` at the
         end of the first match, i.e., close quote.
     :raise ValueError: If an invalid string (i.e., contains an invalid
         escape sequence) is passed into the ``eval``.
@@ -227,14 +229,14 @@ def read_int(s, start_position):
     value of the integer and the position where it ends.  Otherwise,
     raise a ``ReadError``.
 
-    :param s: A string that will be checked to see if within which a 
+    :param s: A string that will be checked to see if within which a
         Python integer exists.
     :type s: str
-    
+
     :param start_position: The specified beginning position of the string ``s``
         to begin regex matching.
     :type start_position: int
-    
+
     :return: A tuple containing the matched integer casted to an int,
         and the end position of the int in ``s``.
     :rtype: tuple(int, int)
@@ -246,7 +248,7 @@ def read_int(s, start_position):
     >>> from nltk.internals import read_int
     >>> read_int('42 is the answer', 0)
     (42, 2)
-    
+
     """
     m = _READ_INT_RE.match(s, start_position)
     if not m: raise ReadError('integer', start_position)
@@ -260,14 +262,14 @@ def read_number(s, start_position):
     containing the value of the number and the position where it ends.
     Otherwise, raise a ``ReadError``.
 
-    :param s: A string that will be checked to see if within which a 
+    :param s: A string that will be checked to see if within which a
         Python number exists.
     :type s: str
-    
+
     :param start_position: The specified beginning position of the string ``s``
         to begin regex matching.
     :type start_position: int
-    
+
     :return: A tuple containing the matched number casted to a ``float``,
         and the end position of the number in ``s``.
     :rtype: tuple(float, int)
@@ -279,7 +281,7 @@ def read_number(s, start_position):
     >>> from nltk.internals import read_number
     >>> read_number('Pi is 3.14159', 6)
     (3.14159, 13)
-    
+
     """
     m = _READ_NUMBER_VALUE.match(s, start_position)
     if not m or not (m.group(1) or m.group(2)):
@@ -451,7 +453,7 @@ class Counter:
 ##########################################################################
 
 def find_file_iter(filename, env_vars=(), searchpath=(),
-    file_names=None, url=None, verbose=True, finding_dir=False):
+    file_names=None, url=None, verbose=False, finding_dir=False):
     """
     Search for a file to be used by nltk.
 
@@ -463,10 +465,10 @@ def find_file_iter(filename, env_vars=(), searchpath=(),
     :param verbose: Whether or not to print path when a file is found.
     """
     file_names = [filename] + (file_names or [])
-    assert isinstance(filename, compat.string_types)
-    assert not isinstance(file_names, compat.string_types)
-    assert not isinstance(searchpath, compat.string_types)
-    if isinstance(env_vars, compat.string_types):
+    assert isinstance(filename, string_types)
+    assert not isinstance(file_names, string_types)
+    assert not isinstance(searchpath, string_types)
+    if isinstance(env_vars, string_types):
         env_vars = env_vars.split()
     yielded = False
 
@@ -498,7 +500,7 @@ def find_file_iter(filename, env_vars=(), searchpath=(),
             if finding_dir: # This is to file a directory instead of file
                 yielded = True
                 yield os.environ[env_var]
-        		
+
             for env_dir in os.environ[env_var].split(os.pathsep):
                 # Check if the environment variable contains a direct path to the bin
                 if os.path.isfile(env_dir):
@@ -568,19 +570,19 @@ def find_file_iter(filename, env_vars=(), searchpath=(),
 
 
 def find_file(filename, env_vars=(), searchpath=(),
-        file_names=None, url=None, verbose=True):
+        file_names=None, url=None, verbose=False):
     return next(find_file_iter(filename, env_vars, searchpath,
                                file_names, url, verbose))
 
 
 def find_dir(filename, env_vars=(), searchpath=(),
-        file_names=None, url=None, verbose=True):
+        file_names=None, url=None, verbose=False):
     return next(find_file_iter(filename, env_vars, searchpath,
                                file_names, url, verbose, finding_dir=True))
 
 
 def find_binary_iter(name, path_to_bin=None, env_vars=(), searchpath=(),
-                binary_names=None, url=None, verbose=True):
+                binary_names=None, url=None, verbose=False):
     """
     Search for a file to be used by nltk.
 
@@ -597,12 +599,12 @@ def find_binary_iter(name, path_to_bin=None, env_vars=(), searchpath=(),
         yield file
 
 def find_binary(name, path_to_bin=None, env_vars=(), searchpath=(),
-                binary_names=None, url=None, verbose=True):
+                binary_names=None, url=None, verbose=False):
     return next(find_binary_iter(name, path_to_bin, env_vars, searchpath,
                                  binary_names, url, verbose))
 
 def find_jar_iter(name_pattern, path_to_jar=None, env_vars=(),
-        searchpath=(), url=None, verbose=True, is_regex=False):
+        searchpath=(), url=None, verbose=False, is_regex=False):
     """
     Search for a jar that is used by nltk.
 
@@ -615,9 +617,9 @@ def find_jar_iter(name_pattern, path_to_jar=None, env_vars=(),
     :param is_regex: Whether name is a regular expression.
     """
 
-    assert isinstance(name_pattern, compat.string_types)
-    assert not isinstance(searchpath, compat.string_types)
-    if isinstance(env_vars, compat.string_types):
+    assert isinstance(name_pattern, string_types)
+    assert not isinstance(searchpath, string_types)
+    if isinstance(env_vars, string_types):
         env_vars = env_vars.split()
     yielded = False
 
@@ -648,7 +650,7 @@ def find_jar_iter(name_pattern, path_to_jar=None, env_vars=(),
                                 print('[Found %s: %s]' % (name_pattern, cp))
                             yielded = True
                             yield cp
-                    # The case where user put directory containing the jar file in the classpath 
+                    # The case where user put directory containing the jar file in the classpath
                     if os.path.isdir(cp):
                         if not is_regex:
                             if os.path.isfile(os.path.join(cp,name_pattern)):
@@ -657,14 +659,14 @@ def find_jar_iter(name_pattern, path_to_jar=None, env_vars=(),
                                 yielded = True
                                 yield os.path.join(cp,name_pattern)
                         else:
-                            # Look for file using regular expression 
+                            # Look for file using regular expression
                             for file_name in os.listdir(cp):
                                 if re.match(name_pattern,file_name):
                                     if verbose:
                                         print('[Found %s: %s]' % (name_pattern, os.path.join(cp,file_name)))
                                     yielded = True
                                     yield os.path.join(cp,file_name)
-                                
+
             else:
                 jar_env = os.environ[env_var]
                 jar_iter = ((os.path.join(jar_env, path_to_jar) for path_to_jar in os.listdir(jar_env))
@@ -714,21 +716,21 @@ def find_jar_iter(name_pattern, path_to_jar=None, env_vars=(),
         raise LookupError('\n\n%s\n%s\n%s' % (div, msg, div))
 
 def find_jar(name_pattern, path_to_jar=None, env_vars=(),
-        searchpath=(), url=None, verbose=True, is_regex=False):
+        searchpath=(), url=None, verbose=False, is_regex=False):
     return next(find_jar_iter(name_pattern, path_to_jar, env_vars,
                          searchpath, url, verbose, is_regex))
 
-                
+
 def find_jars_within_path(path_to_jars):
-	return [os.path.join(root, filename) 
-			for root, dirnames, filenames in os.walk(path_to_jars) 
+	return [os.path.join(root, filename)
+			for root, dirnames, filenames in os.walk(path_to_jars)
 			for filename in fnmatch.filter(filenames, '*.jar')]
 
 def _decode_stdoutdata(stdoutdata):
     """ Convert data read from stdout/stderr to unicode """
     if not isinstance(stdoutdata, bytes):
         return stdoutdata
-    
+
     encoding = getattr(sys.__stdout__, "encoding", locale.getpreferredencoding())
     if encoding is None:
         return stdoutdata.decode()
@@ -795,7 +797,7 @@ class ElementWrapper(object):
             <Element "<?xml version='1.0' encoding='utf8'?>\n<test />">
 
         """
-        if isinstance(etree, compat.string_types):
+        if isinstance(etree, string_types):
             etree = ElementTree.fromstring(etree)
         self.__dict__['_etree'] = etree
 
@@ -978,5 +980,3 @@ def is_writable(path):
 
 def raise_unorderable_types(ordering, a, b):
     raise TypeError("unorderable types: %s() %s %s()" % (type(a).__name__, ordering, type(b).__name__))
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/jsontags.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: JSON Encoder/Decoder Helpers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Xu <xxu@student.unimelb.edu.au>
 #
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/metrics/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Metrics
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -27,3 +27,4 @@ from nltk.metrics.association     import (NgramAssocMeasures, BigramAssocMeasure
                                           TrigramAssocMeasures, ContingencyMeasures)
 from nltk.metrics.spearman        import (spearman_correlation, ranks_from_sequence,
                                           ranks_from_scores)
+from nltk.metrics.aline           import align

{nltk-3.2.1 → nltk-3.3}/nltk/metrics/agreement.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Agreement Metrics
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Tom Lippincott <tom@cs.columbia.edu>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -74,9 +74,11 @@ import logging
 from itertools import groupby
 from operator import itemgetter
 
+from six import iteritems
+
 from nltk.probability import FreqDist, ConditionalFreqDist
 from nltk.internals import deprecated
-from nltk.compat import python_2_unicode_compatible, iteritems
+from nltk.compat import python_2_unicode_compatible
 
 from nltk.metrics.distance import binary_distance
 
@@ -97,8 +99,15 @@ class AnnotationTask(object):
     """
 
     def __init__(self, data=None, distance=binary_distance):
-        """Initialize an empty annotation task.
+        """Initialize an annotation task.
+
+        The data argument can be None (to create an empty annotation task) or a sequence of 3-tuples,
+        each representing a coder's labeling of an item:
+            (coder,item,label)
 
+        The distance argument is a function taking two arguments (labels) and producing a numerical distance.
+        The distance from a label to itself should be zero:
+            distance(l,l) = 0
         """
         self.distance = distance
         self.I = set()
@@ -114,9 +123,9 @@ class AnnotationTask(object):
                                 ",".join(x['labels'])), self.data))
 
     def load_array(self, array):
-        """Load the results of annotation.
+        """Load an sequence of annotation results, appending to any data already loaded.
 
-        The argument is a list of 3-tuples, each representing a coder's labeling of an item:
+        The argument is a sequence of 3-tuples, each representing a coder's labeling of an item:
             (coder,item,label)
         """
         for coder, item, labels in array:
@@ -304,6 +313,15 @@ class AnnotationTask(object):
         """Krippendorff 1980
 
         """
+        # check for degenerate cases
+        if len(self.K)==0:
+            raise ValueError("Cannot calculate alpha, no data present!")
+        if len(self.K) == 1:
+            log.debug("Only one annotation value, allpha returning 1.")
+            return 1
+        if len(self.C)==1 and len(self.I) == 1:
+            raise ValueError("Cannot calculate alpha, only one coder and item present!")
+
         De = 0.0
 
         label_freqs = FreqDist(x['labels'] for x in self.data)
@@ -311,9 +329,12 @@ class AnnotationTask(object):
             nj = label_freqs[j]
             for l in self.K:
                 De += float(nj * label_freqs[l]) * self.distance(j, l)
-        De = (1.0 / (len(self.I) * len(self.C) * (len(self.I) * len(self.C) - 1))) * De
-        log.debug("Expected disagreement: %f", De)
-        ret = 1.0 - (self.Do_alpha() / De)
+        try:
+            De = (1.0 / (len(self.I) * len(self.C) * (len(self.I) * len(self.C) - 1))) * De
+            log.debug("Expected disagreement: %f", De)
+            ret = 1.0 - (self.Do_alpha() / De)
+        except ZeroDivisionError:
+            raise ValueError("Cannot calculate alpha, expected disagreement zero, check the distance function!")
         return ret
 
     def weighted_kappa_pairwise(self, cA, cB, max_distance=1.0):
@@ -398,4 +419,3 @@ if __name__ == '__main__':
         print(getattr(task, options.agreement)())
 
     logging.shutdown()
-