nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/metrics/confusionmatrix.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Confusion Matrices
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/metrics/distance.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Distance Metrics
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 #         Tom Lippincott <tom@cs.columbia.edu>
@@ -34,7 +34,7 @@ def _edit_dist_init(len1, len2):
     return lev
 
 
-def _edit_dist_step(lev, i, j, s1, s2, transpositions=False):
+def _edit_dist_step(lev, i, j, s1, s2, substitution_cost=1, transpositions=False):
     c1 = s1[i - 1]
     c2 = s2[j - 1]
 
@@ -43,7 +43,7 @@ def _edit_dist_step(lev, i, j, s1, s2, transpositions=False):
     # skipping a character in s2
     b = lev[i][j - 1] + 1
     # substitution
-    c = lev[i - 1][j - 1] + (c1 != c2)
+    c = lev[i - 1][j - 1] + (substitution_cost if c1 != c2 else 0)
 
     # transposition
     d = c + 1  # never picked by default
@@ -55,7 +55,7 @@ def _edit_dist_step(lev, i, j, s1, s2, transpositions=False):
     lev[i][j] = min(a, b, c, d)
 
 
-def edit_distance(s1, s2, transpositions=False):
+def edit_distance(s1, s2, substitution_cost=1, transpositions=False):
     """
     Calculate the Levenshtein edit-distance between two strings.
     The edit distance is the number of characters that need to be
@@ -65,6 +65,9 @@ def edit_distance(s1, s2, transpositions=False):
     "rain" -> "sain" -> "shin" -> "shine".  These operations could have
     been done in other orders, but at least three steps are needed.
 
+    Allows specifying the cost of substitution edits (e.g., "a" -> "b"),
+    because sometimes it makes sense to assign greater penalties to substitutions.
+
     This also optionally allows transposition edits (e.g., "ab" -> "ba"),
     though this is disabled by default.
 
@@ -72,6 +75,7 @@ def edit_distance(s1, s2, transpositions=False):
     :param transpositions: Whether to allow transposition edits
     :type s1: str
     :type s2: str
+    :type substitution_cost: int
     :type transpositions: bool
     :rtype int
     """
@@ -83,7 +87,8 @@ def edit_distance(s1, s2, transpositions=False):
     # iterate over the array
     for i in range(len1):
         for j in range(len2):
-            _edit_dist_step(lev, i + 1, j + 1, s1, s2, transpositions=transpositions)
+            _edit_dist_step(lev, i + 1, j + 1, s1, s2,
+                            substitution_cost=substitution_cost, transpositions=transpositions)
     return lev[len1][len2]
 
 
@@ -116,7 +121,7 @@ def masi_distance(label1, label2):
 
     >>> from nltk.metrics import masi_distance
     >>> masi_distance(set([1, 2]), set([1, 2, 3, 4]))
-    0.665...
+    0.665
 
     Passonneau 2006, Measuring Agreement on Set-Valued Items (MASI)
     for Semantic and Pragmatic Annotation.
@@ -135,7 +140,7 @@ def masi_distance(label1, label2):
     else:
         m = 0
 
-    return 1 - (len_intersection / len_union) * m
+    return 1 - len_intersection / len_union * m
 
 
 def interval_distance(label1,label2):

{nltk-3.2.1 → nltk-3.3}/nltk/metrics/paice.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Agreement Metrics
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Lauri Hallila <laurihallila@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/metrics/scores.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Evaluation
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
@@ -12,12 +12,13 @@ import operator
 from random import shuffle
 from functools import reduce
 
+from six.moves import range, zip
+
 try:
     from scipy.stats.stats import betai
 except ImportError:
     betai = None
 
-from nltk.compat import xrange, izip
 from nltk.util import LazyConcatenation, LazyMap
 
 def accuracy(reference, test):
@@ -37,7 +38,7 @@ def accuracy(reference, test):
     """
     if len(reference) != len(test):
         raise ValueError("Lists must have the same length.")
-    return sum(x == y for x, y in izip(reference, test)) / len(test)
+    return sum(x == y for x, y in zip(reference, test)) / len(test)
 
 def precision(reference, test):
     """
@@ -132,7 +133,7 @@ def log_likelihood(reference, test):
 
     # Return the average value of dist.logprob(val).
     total_likelihood = sum(dist.logprob(val)
-                            for (val, dist) in izip(reference, test))
+                            for (val, dist) in zip(reference, test))
     return total_likelihood / len(reference)
 
 def approxrand(a, b, **kwargs):
@@ -159,7 +160,7 @@ def approxrand(a, b, **kwargs):
     shuffles = kwargs.get('shuffles', 999)
     # there's no point in trying to shuffle beyond all possible permutations
     shuffles = \
-        min(shuffles, reduce(operator.mul, xrange(1, len(a) + len(b) + 1)))
+        min(shuffles, reduce(operator.mul, range(1, len(a) + len(b) + 1)))
     stat = kwargs.get('statistic', lambda lst: sum(lst) / len(lst))
     verbose = kwargs.get('verbose', False)
 
@@ -176,7 +177,7 @@ def approxrand(a, b, **kwargs):
     lst = LazyConcatenation([a, b])
     indices = list(range(len(a) + len(b)))
 
-    for i in xrange(shuffles):
+    for i in range(shuffles):
         if verbose and i % 10 == 0:
             print('shuffle: %d' % i)
 
@@ -225,4 +226,3 @@ def demo():
 
 if __name__ == '__main__':
     demo()
-

{nltk-3.2.1 → nltk-3.3}/nltk/metrics/segmentation.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Text Segmentation Metrics
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 #         David Doukhan <david.doukhan@gmail.com>
@@ -46,7 +46,8 @@ try:
 except ImportError:
     pass
 
-from nltk.compat import xrange
+from six.moves import range
+
 
 def windowdiff(seg1, seg2, k, boundary="1", weighted=False):
     """
@@ -213,7 +214,7 @@ def pk(ref, hyp, k=None, boundary='1'):
         k = int(round(len(ref) / (ref.count(boundary) * 2.)))
 
     err = 0
-    for i in xrange(len(ref)-k +1):
+    for i in range(len(ref)-k +1):
         r = ref[i:i+k].count(boundary) > 0
         h = hyp[i:i+k].count(boundary) > 0
         if r != h:
@@ -228,5 +229,3 @@ def setup_module(module):
         import numpy
     except ImportError:
         raise SkipTest("numpy is required for nltk.metrics.segmentation")
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/metrics/spearman.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Spearman Rank Correlation
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Joel Nothman <jnothman@student.usyd.edu.au>
 # URL: <http://nltk.org>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/misc/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Miscellaneous modules
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/misc/chomsky.py

@@ -118,7 +118,9 @@ scope of a complex symbol.
 
 import textwrap, random
 from itertools import chain, islice
-from nltk.compat import izip
+
+from six.moves import zip
+
 
 def generate_chomsky(times=5, line_length=72):
     parts = []
@@ -126,7 +128,7 @@ def generate_chomsky(times=5, line_length=72):
         phraselist = list(map(str.strip, part.splitlines()))
         random.shuffle(phraselist)
         parts.append(phraselist)
-    output = chain(*islice(izip(*parts), 0, times))
+    output = chain(*islice(zip(*parts), 0, times))
     print(textwrap.fill(" ".join(output), line_length))
 
 if __name__ == '__main__':

{nltk-3.2.1 → nltk-3.3}/nltk/misc/minimalset.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Minimal Sets
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/misc/sort.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: List Sorting
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/misc/wordfinder.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Word Finder
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/parse/__init__.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Parsers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>
@@ -78,3 +78,4 @@ from nltk.parse.malt import MaltParser
 from nltk.parse.evaluate import DependencyEvaluator
 from nltk.parse.transitionparser import TransitionParser
 from nltk.parse.bllip import BllipParser
+from nltk.parse.corenlp import CoreNLPParser, CoreNLPDependencyParser

{nltk-3.2.1 → nltk-3.3}/nltk/parse/api.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Parser API
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 #         Edward Loper <edloper@gmail.com>
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/parse/bllip.py

@@ -2,7 +2,7 @@
 #
 # Author: David McClosky <dmcc@bigasterisk.com>
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 

{nltk-3.2.1 → nltk-3.3}/nltk/parse/chart.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: A Chart Parser
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com>
 #         Jean Mark Gawron <gawron@mail.sdsu.edu>
@@ -40,14 +40,15 @@ from __future__ import print_function, division, unicode_literals
 import itertools
 import re
 import warnings
+from functools import total_ordering
+
+from six.moves import range
 
-from nltk import compat
 from nltk.tree import Tree
 from nltk.grammar import PCFG, is_nonterminal, is_terminal
 from nltk.util import OrderedDict
 from nltk.internals import raise_unorderable_types
-from nltk.compat import (total_ordering, python_2_unicode_compatible,
-                         unicode_repr)
+from nltk.compat import python_2_unicode_compatible, unicode_repr
 
 from nltk.parse.api import ParserI
 
@@ -1147,7 +1148,7 @@ class EmptyPredictRule(AbstractChartRule):
     NUM_EDGES = 0
     def apply(self, chart, grammar):
         for prod in grammar.productions(empty=True):
-            for index in compat.xrange(chart.num_leaves() + 1):
+            for index in range(chart.num_leaves() + 1):
                 new_edge = TreeEdge.from_production(prod, index)
                 if chart.insert(new_edge, ()):
                     yield new_edge
@@ -1313,7 +1314,7 @@ class ChartParser(ParserI):
         # Width, for printing trace edges.
         trace_edge_width = self._trace_chart_width // (chart.num_leaves() + 1)
         if trace: print(chart.pretty_format_leaves(trace_edge_width))
-        
+
         if self._use_agenda:
             # Use an agenda-based algorithm.
             for axiom in self._axioms:
@@ -1630,7 +1631,7 @@ def demo(choice=None,
         t = time.time()
         chart = cp.chart_parse(tokens)
         parses = list(chart.parses(grammar.start()))
-        
+
         times[strategies[strategy][0]] = time.time()-t
         print("Nr edges in chart:", len(chart.edges()))
         if numparses: