nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/util.py

@@ -1,11 +1,13 @@
 # Natural Language Toolkit: Utility functions
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 from __future__ import print_function
 
+import sys
+import inspect
 import locale
 import re
 import types
@@ -19,19 +21,23 @@ from pprint import pprint
 from collections import defaultdict, deque
 from sys import version_info
 
+from six import class_types, string_types, text_type
+from six.moves.urllib.request import (build_opener, install_opener, getproxies,
+                                      ProxyHandler, ProxyBasicAuthHandler,
+                                      ProxyDigestAuthHandler,
+                                      HTTPPasswordMgrWithDefaultRealm)
+
 from nltk.internals import slice_bounds, raise_unorderable_types
-from nltk.compat import (class_types, text_type, string_types, total_ordering,
-                         python_2_unicode_compatible, getproxies,
-			 ProxyHandler, build_opener, install_opener,
-			 HTTPPasswordMgrWithDefaultRealm,
-			 ProxyBasicAuthHandler, ProxyDigestAuthHandler)
+from nltk.collections import *
+from nltk.compat import python_2_unicode_compatible
+
+
 
 ######################################################################
 # Short usage message
 ######################################################################
 
 def usage(obj, selfname='self'):
-    import inspect
     str(obj) # In case it's lazy, this will load it.
 
     if not isinstance(obj, class_types):
@@ -42,7 +48,11 @@ def usage(obj, selfname='self'):
         if name.startswith('_'): continue
         if getattr(method, '__deprecated__', False): continue
 
-        args, varargs, varkw, defaults = inspect.getargspec(method)
+        if sys.version_info[0] >= 3:
+            getargspec = inspect.getfullargspec
+        else:
+            getargspec = inspect.getargspec
+        args, varargs, varkw, defaults = getargspec(method)[:4]
         if (args and args[0]=='self' and
             (defaults is None or len(args)>len(defaults))):
             args = args[1:]
@@ -378,18 +388,18 @@ def flatten(*args):
 # Ngram iteration
 ##########################################################################
 
-def pad_sequence(sequence, n, pad_left=False, pad_right=False, 
+def pad_sequence(sequence, n, pad_left=False, pad_right=False,
                  left_pad_symbol=None, right_pad_symbol=None):
     """
     Returns a padded sequence of items before ngram extraction.
-    
+
         >>> list(pad_sequence([1,2,3,4,5], 2, pad_left=True, pad_right=True, left_pad_symbol='<s>', right_pad_symbol='</s>'))
         ['<s>', 1, 2, 3, 4, 5, '</s>']
         >>> list(pad_sequence([1,2,3,4,5], 2, pad_left=True, left_pad_symbol='<s>'))
         ['<s>', 1, 2, 3, 4, 5]
         >>> list(pad_sequence([1,2,3,4,5], 2, pad_right=True, right_pad_symbol='</s>'))
         [1, 2, 3, 4, 5, '</s>']
-    
+
     :param sequence: the source data to be padded
     :type sequence: sequence or iter
     :param n: the degree of the ngrams
@@ -413,7 +423,7 @@ def pad_sequence(sequence, n, pad_left=False, pad_right=False,
 
 # add a flag to pad the sequence so we get peripheral ngrams?
 
-def ngrams(sequence, n, pad_left=False, pad_right=False, 
+def ngrams(sequence, n, pad_left=False, pad_right=False,
            left_pad_symbol=None, right_pad_symbol=None):
     """
     Return the ngrams generated from a sequence of items, as an iterator.
@@ -423,7 +433,7 @@ def ngrams(sequence, n, pad_left=False, pad_right=False,
         >>> list(ngrams([1,2,3,4,5], 3))
         [(1, 2, 3), (2, 3, 4), (3, 4, 5)]
 
-    Use ngrams for a list version of this function.  Set pad_left
+    Wrap with list for a list version of this function.  Set pad_left
     or pad_right to true in order to get additional ngrams:
 
         >>> list(ngrams([1,2,3,4,5], 2, pad_right=True))
@@ -452,7 +462,7 @@ def ngrams(sequence, n, pad_left=False, pad_right=False,
     """
     sequence = pad_sequence(sequence, n, pad_left, pad_right,
                             left_pad_symbol, right_pad_symbol)
-        
+
     history = []
     while n > 1:
         history.append(next(sequence))
@@ -503,13 +513,13 @@ def trigrams(sequence, **kwargs):
 def everygrams(sequence, min_len=1, max_len=-1, **kwargs):
     """
     Returns all possible ngrams generated from a sequence of items, as an iterator.
-    
+
         >>> sent = 'a b c'.split()
         >>> list(everygrams(sent))
         [('a',), ('b',), ('c',), ('a', 'b'), ('b', 'c'), ('a', 'b', 'c')]
         >>> list(everygrams(sent, max_len=2))
         [('a',), ('b',), ('c',), ('a', 'b'), ('b', 'c')]
-        
+
     :param sequence: the source data to be converted into trigrams
     :type sequence: sequence or iter
     :param min_len: minimum length of the ngrams, aka. n-gram order/degree of ngram
@@ -518,7 +528,7 @@ def everygrams(sequence, min_len=1, max_len=-1, **kwargs):
     :type  max_len: int
     :rtype: iter(tuple)
     """
-    
+
     if max_len == -1:
         max_len = len(sequence)
     for n in range(min_len, max_len+1):
@@ -530,13 +540,13 @@ def skipgrams(sequence, n, k, **kwargs):
     Returns all possible skipgrams generated from a sequence of items, as an iterator.
     Skipgrams are ngrams that allows tokens to be skipped.
     Refer to http://homepages.inf.ed.ac.uk/ballison/pdf/lrec_skipgrams.pdf
-    
+
         >>> sent = "Insurgents killed in ongoing fighting".split()
         >>> list(skipgrams(sent, 2, 2))
         [('Insurgents', 'killed'), ('Insurgents', 'in'), ('Insurgents', 'ongoing'), ('killed', 'in'), ('killed', 'ongoing'), ('killed', 'fighting'), ('in', 'ongoing'), ('in', 'fighting'), ('ongoing', 'fighting')]
         >>> list(skipgrams(sent, 3, 2))
         [('Insurgents', 'killed', 'in'), ('Insurgents', 'killed', 'ongoing'), ('Insurgents', 'killed', 'fighting'), ('Insurgents', 'in', 'ongoing'), ('Insurgents', 'in', 'fighting'), ('Insurgents', 'ongoing', 'fighting'), ('killed', 'in', 'ongoing'), ('killed', 'in', 'fighting'), ('killed', 'ongoing', 'fighting'), ('in', 'ongoing', 'fighting')]
-    
+
     :param sequence: the source data to be converted into trigrams
     :type sequence: sequence or iter
     :param n: the degree of the ngrams
@@ -545,13 +555,13 @@ def skipgrams(sequence, n, k, **kwargs):
     :type  k: int
     :rtype: iter(tuple)
     """
-    
+
     # Pads the sequence as desired by **kwargs.
     if 'pad_left' in kwargs or 'pad_right' in kwargs:
         sequence = pad_sequence(sequence, n, **kwargs)
-    
+
     # Note when iterating through the ngrams, the pad_right here is not
-    # the **kwargs padding, it's for the algorithm to detect the SENTINEL 
+    # the **kwargs padding, it's for the algorithm to detect the SENTINEL
     # object on the right pad to stop inner loop.
     SENTINEL = object()
     for ngram in ngrams(sequence, n + k, pad_right=True, right_pad_symbol=SENTINEL):
@@ -562,531 +572,6 @@ def skipgrams(sequence, n, k, **kwargs):
                 continue
             yield head + skip_tail
 
-##########################################################################
-# Ordered Dictionary
-##########################################################################
-
-class OrderedDict(dict):
-    def __init__(self, data=None, **kwargs):
-        self._keys = self.keys(data, kwargs.get('keys'))
-        self._default_factory = kwargs.get('default_factory')
-        if data is None:
-            dict.__init__(self)
-        else:
-            dict.__init__(self, data)
-
-    def __delitem__(self, key):
-        dict.__delitem__(self, key)
-        self._keys.remove(key)
-
-    def __getitem__(self, key):
-        try:
-            return dict.__getitem__(self, key)
-        except KeyError:
-            return self.__missing__(key)
-
-    def __iter__(self):
-        return (key for key in self.keys())
-
-    def __missing__(self, key):
-        if not self._default_factory and key not in self._keys:
-            raise KeyError()
-        return self._default_factory()
-
-    def __setitem__(self, key, item):
-        dict.__setitem__(self, key, item)
-        if key not in self._keys:
-            self._keys.append(key)
-
-    def clear(self):
-        dict.clear(self)
-        self._keys.clear()
-
-    def copy(self):
-        d = dict.copy(self)
-        d._keys = self._keys
-        return d
-
-    def items(self):
-        # returns iterator under python 3 and list under python 2
-        return zip(self.keys(), self.values())
-
-    def keys(self, data=None, keys=None):
-        if data:
-            if keys:
-                assert isinstance(keys, list)
-                assert len(data) == len(keys)
-                return keys
-            else:
-                assert isinstance(data, dict) or \
-                       isinstance(data, OrderedDict) or \
-                       isinstance(data, list)
-                if isinstance(data, dict) or isinstance(data, OrderedDict):
-                    return data.keys()
-                elif isinstance(data, list):
-                    return [key for (key, value) in data]
-        elif '_keys' in self.__dict__:
-            return self._keys
-        else:
-            return []
-
-    def popitem(self):
-        if not self._keys:
-            raise KeyError()
-
-        key = self._keys.pop()
-        value = self[key]
-        del self[key]
-        return (key, value)
-
-    def setdefault(self, key, failobj=None):
-        dict.setdefault(self, key, failobj)
-        if key not in self._keys:
-            self._keys.append(key)
-
-    def update(self, data):
-        dict.update(self, data)
-        for key in self.keys(data):
-            if key not in self._keys:
-                self._keys.append(key)
-
-    def values(self):
-        # returns iterator under python 3
-        return map(self.get, self._keys)
-
-######################################################################
-# Lazy Sequences
-######################################################################
-
-@total_ordering
-@python_2_unicode_compatible
-class AbstractLazySequence(object):
-    """
-    An abstract base class for read-only sequences whose values are
-    computed as needed.  Lazy sequences act like tuples -- they can be
-    indexed, sliced, and iterated over; but they may not be modified.
-
-    The most common application of lazy sequences in NLTK is for
-    corpus view objects, which provide access to the contents of a
-    corpus without loading the entire corpus into memory, by loading
-    pieces of the corpus from disk as needed.
-
-    The result of modifying a mutable element of a lazy sequence is
-    undefined.  In particular, the modifications made to the element
-    may or may not persist, depending on whether and when the lazy
-    sequence caches that element's value or reconstructs it from
-    scratch.
-
-    Subclasses are required to define two methods: ``__len__()``
-    and ``iterate_from()``.
-    """
-    def __len__(self):
-        """
-        Return the number of tokens in the corpus file underlying this
-        corpus view.
-        """
-        raise NotImplementedError('should be implemented by subclass')
-
-    def iterate_from(self, start):
-        """
-        Return an iterator that generates the tokens in the corpus
-        file underlying this corpus view, starting at the token number
-        ``start``.  If ``start>=len(self)``, then this iterator will
-        generate no tokens.
-        """
-        raise NotImplementedError('should be implemented by subclass')
-
-    def __getitem__(self, i):
-        """
-        Return the *i* th token in the corpus file underlying this
-        corpus view.  Negative indices and spans are both supported.
-        """
-        if isinstance(i, slice):
-            start, stop = slice_bounds(self, i)
-            return LazySubsequence(self, start, stop)
-        else:
-            # Handle negative indices
-            if i < 0: i += len(self)
-            if i < 0: raise IndexError('index out of range')
-            # Use iterate_from to extract it.
-            try:
-                return next(self.iterate_from(i))
-            except StopIteration:
-                raise IndexError('index out of range')
-
-    def __iter__(self):
-        """Return an iterator that generates the tokens in the corpus
-        file underlying this corpus view."""
-        return self.iterate_from(0)
-
-    def count(self, value):
-        """Return the number of times this list contains ``value``."""
-        return sum(1 for elt in self if elt==value)
-
-    def index(self, value, start=None, stop=None):
-        """Return the index of the first occurrence of ``value`` in this
-        list that is greater than or equal to ``start`` and less than
-        ``stop``.  Negative start and stop values are treated like negative
-        slice bounds -- i.e., they count from the end of the list."""
-        start, stop = slice_bounds(self, slice(start, stop))
-        for i, elt in enumerate(islice(self, start, stop)):
-            if elt == value: return i+start
-        raise ValueError('index(x): x not in list')
-
-    def __contains__(self, value):
-        """Return true if this list contains ``value``."""
-        return bool(self.count(value))
-
-    def __add__(self, other):
-        """Return a list concatenating self with other."""
-        return LazyConcatenation([self, other])
-
-    def __radd__(self, other):
-        """Return a list concatenating other with self."""
-        return LazyConcatenation([other, self])
-
-    def __mul__(self, count):
-        """Return a list concatenating self with itself ``count`` times."""
-        return LazyConcatenation([self] * count)
-
-    def __rmul__(self, count):
-        """Return a list concatenating self with itself ``count`` times."""
-        return LazyConcatenation([self] * count)
-
-    _MAX_REPR_SIZE = 60
-    def __repr__(self):
-        """
-        Return a string representation for this corpus view that is
-        similar to a list's representation; but if it would be more
-        than 60 characters long, it is truncated.
-        """
-        pieces = []
-        length = 5
-        for elt in self:
-            pieces.append(repr(elt))
-            length += len(pieces[-1]) + 2
-            if length > self._MAX_REPR_SIZE and len(pieces) > 2:
-                return '[%s, ...]' % text_type(', ').join(pieces[:-1])
-        else:
-            return '[%s]' % text_type(', ').join(pieces)
-
-    def __eq__(self, other):
-        return (type(self) == type(other) and list(self) == list(other))
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __lt__(self, other):
-        if type(other) != type(self):
-            raise_unorderable_types("<", self, other)
-        return list(self) < list(other)
-
-    def __hash__(self):
-        """
-        :raise ValueError: Corpus view objects are unhashable.
-        """
-        raise ValueError('%s objects are unhashable' %
-                         self.__class__.__name__)
-
-
-class LazySubsequence(AbstractLazySequence):
-    """
-    A subsequence produced by slicing a lazy sequence.  This slice
-    keeps a reference to its source sequence, and generates its values
-    by looking them up in the source sequence.
-    """
-
-    MIN_SIZE = 100
-    """
-    The minimum size for which lazy slices should be created.  If
-    ``LazySubsequence()`` is called with a subsequence that is
-    shorter than ``MIN_SIZE``, then a tuple will be returned instead.
-    """
-
-    def __new__(cls, source, start, stop):
-        """
-        Construct a new slice from a given underlying sequence.  The
-        ``start`` and ``stop`` indices should be absolute indices --
-        i.e., they should not be negative (for indexing from the back
-        of a list) or greater than the length of ``source``.
-        """
-        # If the slice is small enough, just use a tuple.
-        if stop-start < cls.MIN_SIZE:
-            return list(islice(source.iterate_from(start), stop-start))
-        else:
-            return object.__new__(cls)
-
-    def __init__(self, source, start, stop):
-        self._source = source
-        self._start = start
-        self._stop = stop
-
-    def __len__(self):
-        return self._stop - self._start
-
-    def iterate_from(self, start):
-        return islice(self._source.iterate_from(start+self._start),
-                      max(0, len(self)-start))
-
-
-class LazyConcatenation(AbstractLazySequence):
-    """
-    A lazy sequence formed by concatenating a list of lists.  This
-    underlying list of lists may itself be lazy.  ``LazyConcatenation``
-    maintains an index that it uses to keep track of the relationship
-    between offsets in the concatenated lists and offsets in the
-    sublists.
-    """
-    def __init__(self, list_of_lists):
-        self._list = list_of_lists
-        self._offsets = [0]
-
-    def __len__(self):
-        if len(self._offsets) <= len(self._list):
-            for tok in self.iterate_from(self._offsets[-1]): pass
-        return self._offsets[-1]
-
-    def iterate_from(self, start_index):
-        if start_index < self._offsets[-1]:
-            sublist_index = bisect.bisect_right(self._offsets, start_index)-1
-        else:
-            sublist_index = len(self._offsets)-1
-
-        index = self._offsets[sublist_index]
-
-        # Construct an iterator over the sublists.
-        if isinstance(self._list, AbstractLazySequence):
-            sublist_iter = self._list.iterate_from(sublist_index)
-        else:
-            sublist_iter = islice(self._list, sublist_index, None)
-
-        for sublist in sublist_iter:
-            if sublist_index == (len(self._offsets)-1):
-                assert index+len(sublist) >= self._offsets[-1], (
-                        'offests not monotonic increasing!')
-                self._offsets.append(index+len(sublist))
-            else:
-                assert self._offsets[sublist_index+1] == index+len(sublist), (
-                        'inconsistent list value (num elts)')
-
-            for value in sublist[max(0, start_index-index):]:
-                yield value
-
-            index += len(sublist)
-            sublist_index += 1
-
-
-class LazyMap(AbstractLazySequence):
-    """
-    A lazy sequence whose elements are formed by applying a given
-    function to each element in one or more underlying lists.  The
-    function is applied lazily -- i.e., when you read a value from the
-    list, ``LazyMap`` will calculate that value by applying its
-    function to the underlying lists' value(s).  ``LazyMap`` is
-    essentially a lazy version of the Python primitive function
-    ``map``.  In particular, the following two expressions are
-    equivalent:
-
-        >>> from nltk.util import LazyMap
-        >>> function = str
-        >>> sequence = [1,2,3]
-        >>> map(function, sequence) # doctest: +SKIP
-        ['1', '2', '3']
-        >>> list(LazyMap(function, sequence))
-        ['1', '2', '3']
-
-    Like the Python ``map`` primitive, if the source lists do not have
-    equal size, then the value None will be supplied for the
-    'missing' elements.
-
-    Lazy maps can be useful for conserving memory, in cases where
-    individual values take up a lot of space.  This is especially true
-    if the underlying list's values are constructed lazily, as is the
-    case with many corpus readers.
-
-    A typical example of a use case for this class is performing
-    feature detection on the tokens in a corpus.  Since featuresets
-    are encoded as dictionaries, which can take up a lot of memory,
-    using a ``LazyMap`` can significantly reduce memory usage when
-    training and running classifiers.
-    """
-    def __init__(self, function, *lists, **config):
-        """
-        :param function: The function that should be applied to
-            elements of ``lists``.  It should take as many arguments
-            as there are ``lists``.
-        :param lists: The underlying lists.
-        :param cache_size: Determines the size of the cache used
-            by this lazy map.  (default=5)
-        """
-        if not lists:
-            raise TypeError('LazyMap requires at least two args')
-
-        self._lists = lists
-        self._func = function
-        self._cache_size = config.get('cache_size', 5)
-        self._cache = ({} if self._cache_size > 0 else None)
-
-        # If you just take bool() of sum() here _all_lazy will be true just
-        # in case n >= 1 list is an AbstractLazySequence.  Presumably this
-        # isn't what's intended.
-        self._all_lazy = sum(isinstance(lst, AbstractLazySequence)
-                             for lst in lists) == len(lists)
-
-    def iterate_from(self, index):
-        # Special case: one lazy sublist
-        if len(self._lists) == 1 and self._all_lazy:
-            for value in self._lists[0].iterate_from(index):
-                yield self._func(value)
-            return
-
-        # Special case: one non-lazy sublist
-        elif len(self._lists) == 1:
-            while True:
-                try: yield self._func(self._lists[0][index])
-                except IndexError: return
-                index += 1
-
-        # Special case: n lazy sublists
-        elif self._all_lazy:
-            iterators = [lst.iterate_from(index) for lst in self._lists]
-            while True:
-                elements = []
-                for iterator in iterators:
-                    try: elements.append(next(iterator))
-                    except: elements.append(None)
-                if elements == [None] * len(self._lists):
-                    return
-                yield self._func(*elements)
-                index += 1
-
-        # general case
-        else:
-            while True:
-                try: elements = [lst[index] for lst in self._lists]
-                except IndexError:
-                    elements = [None] * len(self._lists)
-                    for i, lst in enumerate(self._lists):
-                        try: elements[i] = lst[index]
-                        except IndexError: pass
-                    if elements == [None] * len(self._lists):
-                        return
-                yield self._func(*elements)
-                index += 1
-
-    def __getitem__(self, index):
-        if isinstance(index, slice):
-            sliced_lists = [lst[index] for lst in self._lists]
-            return LazyMap(self._func, *sliced_lists)
-        else:
-            # Handle negative indices
-            if index < 0: index += len(self)
-            if index < 0: raise IndexError('index out of range')
-            # Check the cache
-            if self._cache is not None and index in self._cache:
-                return self._cache[index]
-            # Calculate the value
-            try: val = next(self.iterate_from(index))
-            except StopIteration:
-                raise IndexError('index out of range')
-            # Update the cache
-            if self._cache is not None:
-                if len(self._cache) > self._cache_size:
-                    self._cache.popitem() # discard random entry
-                self._cache[index] = val
-            # Return the value
-            return val
-
-    def __len__(self):
-        return max(len(lst) for lst in self._lists)
-
-
-class LazyZip(LazyMap):
-    """
-    A lazy sequence whose elements are tuples, each containing the i-th
-    element from each of the argument sequences.  The returned list is
-    truncated in length to the length of the shortest argument sequence. The
-    tuples are constructed lazily -- i.e., when you read a value from the
-    list, ``LazyZip`` will calculate that value by forming a tuple from
-    the i-th element of each of the argument sequences.
-
-    ``LazyZip`` is essentially a lazy version of the Python primitive function
-    ``zip``.  In particular, an evaluated LazyZip is equivalent to a zip:
-
-        >>> from nltk.util import LazyZip
-        >>> sequence1, sequence2 = [1, 2, 3], ['a', 'b', 'c']
-        >>> zip(sequence1, sequence2) # doctest: +SKIP
-        [(1, 'a'), (2, 'b'), (3, 'c')]
-        >>> list(LazyZip(sequence1, sequence2))
-        [(1, 'a'), (2, 'b'), (3, 'c')]
-        >>> sequences = [sequence1, sequence2, [6,7,8,9]]
-        >>> list(zip(*sequences)) == list(LazyZip(*sequences))
-        True
-
-    Lazy zips can be useful for conserving memory in cases where the argument
-    sequences are particularly long.
-
-    A typical example of a use case for this class is combining long sequences
-    of gold standard and predicted values in a classification or tagging task
-    in order to calculate accuracy.  By constructing tuples lazily and
-    avoiding the creation of an additional long sequence, memory usage can be
-    significantly reduced.
-    """
-    def __init__(self, *lists):
-        """
-        :param lists: the underlying lists
-        :type lists: list(list)
-        """
-        LazyMap.__init__(self, lambda *elts: elts, *lists)
-
-    def iterate_from(self, index):
-        iterator = LazyMap.iterate_from(self, index)
-        while index < len(self):
-            yield next(iterator)
-            index += 1
-        return
-
-    def __len__(self):
-        return min(len(lst) for lst in self._lists)
-
-
-class LazyEnumerate(LazyZip):
-    """
-    A lazy sequence whose elements are tuples, each ontaining a count (from
-    zero) and a value yielded by underlying sequence.  ``LazyEnumerate`` is
-    useful for obtaining an indexed list. The tuples are constructed lazily
-    -- i.e., when you read a value from the list, ``LazyEnumerate`` will
-    calculate that value by forming a tuple from the count of the i-th
-    element and the i-th element of the underlying sequence.
-
-    ``LazyEnumerate`` is essentially a lazy version of the Python primitive
-    function ``enumerate``.  In particular, the following two expressions are
-    equivalent:
-
-        >>> from nltk.util import LazyEnumerate
-        >>> sequence = ['first', 'second', 'third']
-        >>> list(enumerate(sequence))
-        [(0, 'first'), (1, 'second'), (2, 'third')]
-        >>> list(LazyEnumerate(sequence))
-        [(0, 'first'), (1, 'second'), (2, 'third')]
-
-    Lazy enumerations can be useful for conserving memory in cases where the
-    argument sequences are particularly long.
-
-    A typical example of a use case for this class is obtaining an indexed
-    list for a long sequence of values.  By constructing tuples lazily and
-    avoiding the creation of an additional long sequence, memory usage can be
-    significantly reduced.
-    """
-
-    def __init__(self, lst):
-        """
-        :param lst: the underlying list
-        :type lst: list
-        """
-        LazyZip.__init__(self, range(len(lst)), lst)
-
-
 ######################################################################
 # Binary Search in a File
 ######################################################################
@@ -1186,7 +671,7 @@ def set_proxy(proxy, user=None, password=''):
             raise ValueError('Could not detect default proxy settings')
 
     # Set up the proxy handler
-    proxy_handler = ProxyHandler({'http': proxy})
+    proxy_handler = ProxyHandler({'https': proxy, 'http': proxy})
     opener = build_opener(proxy_handler)
 
     if user is not None:
@@ -1210,9 +695,9 @@ def elementtree_indent(elem, level=0):
     """
     Recursive function to indent an ElementTree._ElementInterface
     used for pretty printing. Run indent on elem and then output
-    in the normal way. 
-    
-    :param elem: element to be indented. will be modified. 
+    in the normal way.
+
+    :param elem: element to be indented. will be modified.
     :type elem: ElementTree._ElementInterface
     :param level: level of indentation for this element
     :type level: nonnegative integer
@@ -1239,17 +724,17 @@ def elementtree_indent(elem, level=0):
 def choose(n, k):
     """
     This function is a fast way to calculate binomial coefficients, commonly
-    known as nCk, i.e. the number of combinations of n things taken k at a time. 
+    known as nCk, i.e. the number of combinations of n things taken k at a time.
     (https://en.wikipedia.org/wiki/Binomial_coefficient).
-    
-    This is the *scipy.special.comb()* with long integer computation but this 
+
+    This is the *scipy.special.comb()* with long integer computation but this
     approximation is faster, see https://github.com/nltk/nltk/issues/1181
-    
+
         >>> choose(4, 2)
         6
         >>> choose(6, 2)
         15
-    
+
     :param n: The number of things.
     :type n: int
     :param r: The number of times a thing is taken.
@@ -1264,101 +749,3 @@ def choose(n, k):
         return ntok // ktok
     else:
         return 0
-
-######################################################################
-# Trie Implementation
-######################################################################
-class Trie(defaultdict):
-    """A Trie implementation for strings"""
-    LEAF = True
-
-    def __init__(self, strings=None):
-        """Builds a Trie object, which is built around a ``defaultdict``
-        
-        If ``strings`` is provided, it will add the ``strings``, which
-        consist of a ``list`` of ``strings``, to the Trie. 
-        Otherwise, it'll construct an empty Trie.
-
-        :param strings: List of strings to insert into the trie 
-            (Default is ``None``)
-        :type strings: list(str)
-
-        """
-        defaultdict.__init__(self, Trie)
-        if strings:
-            for string in strings:
-                self.insert(string)
-
-    def insert(self, string):
-        """Inserts ``string`` into the Trie
-
-        :param string: String to insert into the trie
-        :type string: str
-
-        :Example:
-
-        >>> from nltk.util import Trie
-        >>> trie = Trie(["ab"])
-        >>> trie
-        defaultdict(<class 'nltk.util.Trie'>, {'a': defaultdict(<class 'nltk.util.Trie'>, {'b': defaultdict(<class 'nltk.util.Trie'>, {True: None})})})
-
-        """
-        if len(string):
-            self[string[0]].insert(string[1:])
-        else:
-            # mark the string is complete
-            self[Trie.LEAF] = None
-
-    def __str__(self):
-        return str(self.as_dict())
-
-    def as_dict(self, d=None):
-        """Convert ``defaultdict`` to common ``dict`` representation.
-
-        :param: A defaultdict containing strings mapped to nested defaultdicts.
-            This is the structure of the trie. (Default is None)
-        :type: defaultdict(str -> defaultdict)
-        :return: Even though ``defaultdict`` is a subclass of ``dict`` and thus
-            can be converted to a simple ``dict`` using ``dict()``, in our case
-            it's a nested ``defaultdict``, so here's a quick trick to provide to
-            us the ``dict`` representation of the ``Trie`` without 
-            ``defaultdict(<class 'nltk.util.Trie'>, ...``
-        :rtype: dict(str -> dict(bool -> None))
-            Note: there can be an arbitrarily deeply nested 
-            ``dict(str -> dict(str -> dict(..))``, but the last
-            level will have ``dict(str -> dict(bool -> None))``
-
-        :Example:
-
-        >>> from nltk.util import Trie
-        >>> trie = Trie(["abc", "def"])
-        >>> expected = {'a': {'b': {'c': {True: None}}}, 'd': {'e': {'f': {True: None}}}}
-        >>> trie.as_dict() == expected
-        True
-
-        """
-        def _default_to_regular(d):
-            """
-            Source: http://stackoverflow.com/a/26496899/4760801
-
-            :param d: Nested ``defaultdict`` to convert to regular ``dict``
-            :type d: defaultdict(str -> defaultdict(...))
-            :return: A dict representation of the defaultdict
-            :rtype: dict(str -> dict(str -> ...))
-
-            :Example:
-
-            >>> from collections import defaultdict
-            >>> d = defaultdict(defaultdict)
-            >>> d["one"]["two"] = "three"
-            >>> d
-            defaultdict(<type 'collections.defaultdict'>, {'one': defaultdict(None, {'two': 'three'})})
-            >>> _default_to_regular(d)
-            {'one': {'two': 'three'}}
-
-            """
-            if isinstance(d, defaultdict):
-                d = {k: _default_to_regular(v) for k, v in d.items()}
-            return d
-        
-        return _default_to_regular(self)