nltk 3.2.1__zip → 3.3__zip

nltk-3.2.1/nltk/test/unit/test_stem.py

@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import print_function, unicode_literals
-import unittest
-from nltk.stem.snowball import SnowballStemmer
-
-
-class SnowballTest(unittest.TestCase):
-
-    def test_russian(self):
-        # Russian words both consisting of Cyrillic
-        # and Roman letters can be stemmed.
-        stemmer_russian = SnowballStemmer("russian")
-        assert stemmer_russian.stem("авантненькая") == "авантненьк"
-        assert stemmer_russian.stem("avenantnen'kai^a") == "avenantnen'k"
-
-    def test_german(self):
-        stemmer_german = SnowballStemmer("german")
-        stemmer_german2 = SnowballStemmer("german", ignore_stopwords=True)
-
-        assert stemmer_german.stem("Schr\xe4nke") == 'schrank'
-        assert stemmer_german2.stem("Schr\xe4nke") == 'schrank'
-
-        assert stemmer_german.stem("keinen") == 'kein'
-        assert stemmer_german2.stem("keinen") == 'keinen'
-
-    def test_spanish(self):
-        stemmer = SnowballStemmer('spanish')
-
-        assert stemmer.stem("Visionado") == 'vision'
-
-        # The word 'algue' was raising an IndexError
-        assert stemmer.stem("algue") == 'algu'
-
-    def test_short_strings_bug(self):
-        stemmer = SnowballStemmer('english')
-        assert stemmer.stem("y's") == 'y'

nltk-3.2.1/nltk/test/unit/test_tokenize.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Unit tests for nltk.tokenize.
-See also nltk/test/tokenize.doctest
-"""
-
-from __future__ import unicode_literals
-from nltk.tokenize import TweetTokenizer
-import unittest
-
-class TestTokenize(unittest.TestCase):
-
-    def test_tweet_tokenizer(self):
-        """
-        Test TweetTokenizer using words with special and accented characters.
-        """
-
-        tokenizer = TweetTokenizer(strip_handles=True, reduce_len=True)
-        s9 = "@myke: Let's test these words: resumé España München français"
-        tokens = tokenizer.tokenize(s9)
-        expected = [':', "Let's", 'test', 'these', 'words', ':', 'resumé',
-                    'España', 'München', 'français']
-        self.assertEqual(tokens, expected)

nltk-3.2.1/nltk/test/unit/translate/test_bleu.py

@@ -1,138 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Tests for BLEU translation evaluation metric
-"""
-
-import unittest
-from nltk.translate.bleu_score import modified_precision, brevity_penalty
-from nltk.translate.bleu_score import sentence_bleu, corpus_bleu
-
-
-class TestBLEU(unittest.TestCase):
-    def test_modified_precision(self):
-        """
-        Examples from the original BLEU paper 
-        http://www.aclweb.org/anthology/P02-1040.pdf
-        """
-        # Example 1: the "the*" example.
-        # Reference sentences.
-        ref1 = 'the cat is on the mat'.split()
-        ref2 = 'there is a cat on the mat'.split()
-        # Hypothesis sentence(s).
-        hyp1 = 'the the the the the the the'.split()
-        
-        references = [ref1, ref2] 
-        
-        # Testing modified unigram precision.
-        hyp1_unigram_precision =  float(modified_precision(references, hyp1, n=1))
-        assert (round(hyp1_unigram_precision, 4) == 0.2857)
-        # With assertAlmostEqual at 4 place precision.
-        self.assertAlmostEqual(hyp1_unigram_precision, 0.28571428, places=4)
-        
-        # Testing modified bigram precision.
-        assert(float(modified_precision(references, hyp1, n=2)) == 0.0)
-        
-        
-        # Example 2: the "of the" example.
-        # Reference sentences
-        ref1 = str('It is a guide to action that ensures that the military '
-                   'will forever heed Party commands').split()
-        ref2 = str('It is the guiding principle which guarantees the military '
-                   'forces always being under the command of the Party').split()
-        ref3 = str('It is the practical guide for the army always to heed '
-                   'the directions of the party').split()
-        # Hypothesis sentence(s).
-        hyp1 = 'of the'.split()
-        
-        references = [ref1, ref2, ref3] 
-        # Testing modified unigram precision.
-        assert (float(modified_precision(references, hyp1, n=1)) == 1.0)
-        
-        # Testing modified bigram precision.
-        assert(float(modified_precision(references, hyp1, n=2)) == 1.0)
-        
-
-        # Example 3: Proper MT outputs.
-        hyp1 = str('It is a guide to action which ensures that the military '
-                   'always obeys the commands of the party').split()
-        hyp2 = str('It is to insure the troops forever hearing the activity '
-                   'guidebook that party direct').split()
-        
-        references = [ref1, ref2, ref3]
-        
-        # Unigram precision.
-        hyp1_unigram_precision = float(modified_precision(references, hyp1, n=1))
-        hyp2_unigram_precision = float(modified_precision(references, hyp2, n=1))
-        # Test unigram precision with assertAlmostEqual at 4 place precision.
-        self.assertAlmostEqual(hyp1_unigram_precision, 0.94444444, places=4)
-        self.assertAlmostEqual(hyp2_unigram_precision, 0.57142857, places=4)
-        # Test unigram precision with rounding.
-        assert (round(hyp1_unigram_precision, 4) == 0.9444)
-        assert (round(hyp2_unigram_precision, 4) == 0.5714)
-        
-        # Bigram precision
-        hyp1_bigram_precision = float(modified_precision(references, hyp1, n=2))
-        hyp2_bigram_precision = float(modified_precision(references, hyp2, n=2))
-        # Test bigram precision with assertAlmostEqual at 4 place precision.
-        self.assertAlmostEqual(hyp1_bigram_precision, 0.58823529, places=4)
-        self.assertAlmostEqual(hyp2_bigram_precision, 0.07692307, places=4)
-        # Test bigram precision with rounding.
-        assert (round(hyp1_bigram_precision, 4) == 0.5882)
-        assert (round(hyp2_bigram_precision, 4) == 0.0769)
-        
-    def test_zero_matches(self):
-        # Test case where there's 0 matches
-        references = ['The candidate has no alignment to any of the references'.split()]
-        hypothesis = 'John loves Mary'.split()
-        
-        # Test BLEU to nth order of n-grams, where n is len(hypothesis). 
-        for n in range(1,len(hypothesis)):
-            weights = [1.0/n] * n # Uniform weights.
-            assert(sentence_bleu(references, hypothesis, weights) == 0)
-    
-    def test_full_matches(self):    
-        # Test case where there's 100% matches
-        references = ['John loves Mary'.split()]
-        hypothesis = 'John loves Mary'.split()
-    
-        # Test BLEU to nth order of n-grams, where n is len(hypothesis). 
-        for n in range(1,len(hypothesis)):
-            weights = [1.0/n] * n # Uniform weights.
-            assert(sentence_bleu(references, hypothesis, weights) == 1.0)
-    
-    def test_partial_matches_hypothesis_longer_than_reference(self):
-        references = ['John loves Mary'.split()]
-        hypothesis = 'John loves Mary who loves Mike'.split()
-        self.assertAlmostEqual(sentence_bleu(references, hypothesis), 0.4729, places=4)
-            
-
-@unittest.skip("Skipping fringe cases for BLEU.")
-class TestBLEUFringeCases(unittest.TestCase):
-
-    def test_case_where_n_is_bigger_than_hypothesis_length(self):
-        # Test BLEU to nth order of n-grams, where n > len(hypothesis).
-        # TODO: Currently this test breaks the BLEU implementation (13.03.2016)
-        references = ['John loves Mary'.split()]
-        hypothesis = 'John loves Mary'.split()
-        n = len(hypothesis) + 1 # 
-        weights = [1.0/n] * n # Uniform weights.
-        assert(sentence_bleu(references, hypothesis, weights) == 1.0)
-    
-    def test_empty_hypothesis(self):
-        # Test case where there's hypothesis is empty.
-        # TODO: Currently this test breaks the BLEU implementation (13.03.2016)
-        references = ['The candidate has no alignment to any of the references'.split()]
-        hypothesis = []
-        assert(sentence_bleu(references, hypothesis) == 0)
-        
-    def test_empty_references(self):
-        # Test case where there's reference is empty.
-        # TODO: Currently this test breaks the BLEU implementation (13.03.2016)
-        references = [[]]
-        hypothesis = 'John loves Mary'.split()
-        assert(sentence_bleu(references, hypothesis) == 0)
-        
-        
-    def test_brevity_penalty(self):
-        pass
-    

nltk-3.2.1/nltk/tokenize/stanford_segmenter.py

@@ -1,157 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: Interface to the Stanford Chinese Segmenter
-#
-# Copyright (C) 2001-2016 NLTK Project
-# Author: 52nlp <52nlpcn@gmail.com>
-#         Casper Lehmann-Strøm <casperlehmann@gmail.com>
-#
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-from __future__ import unicode_literals, print_function
-
-import tempfile
-import os
-import json
-from subprocess import PIPE
-
-from nltk import compat
-from nltk.internals import find_jar, config_java, java, _java_options
-
-from nltk.tokenize.api import TokenizerI
-
-_stanford_url = 'http://nlp.stanford.edu/software'
-
-class StanfordSegmenter(TokenizerI):
-    r"""
-    Interface to the Stanford Segmenter
-    >>> from nltk.tokenize.stanford_segmenter import StanfordSegmenter
-    >>> segmenter = StanfordSegmenter(
-    ...     path_to_jar="stanford-segmenter-3.6.0.jar",
-    ...     path_to_slf4j = "slf4j-api.jar"
-    ...     path_to_sihan_corpora_dict="./data",
-    ...     path_to_model="./data/pku.gz",
-    ...     path_to_dict="./data/dict-chris6.ser.gz")
-    >>> sentence = u"这是斯坦福中文分词器测试"
-    >>> segmenter.segment(sentence)
-    >>> u'\u8fd9 \u662f \u65af\u5766\u798f \u4e2d\u6587 \u5206\u8bcd\u5668 \u6d4b\u8bd5\n'
-    >>> segmenter.segment_file("test.simp.utf8")
-    >>> u'\u9762\u5bf9 \u65b0 \u4e16\u7eaa \uff0c \u4e16\u754c \u5404\u56fd ...
-    """
-
-    _JAR = 'stanford-segmenter.jar'
-    _SLF4J = 'slf4j-api.jar'
-
-    def __init__(self, path_to_jar=None, path_to_slf4j=None,
-            path_to_sihan_corpora_dict=None,
-            path_to_model=None, path_to_dict=None,
-            encoding='UTF-8', options=None,
-            verbose=False, java_options='-mx2g'):
-        stanford_segmenter = find_jar(
-                self._JAR, path_to_jar,
-                env_vars=('STANFORD_SEGMENTER',),
-                searchpath=(), url=_stanford_url,
-                verbose=verbose)
-        slf4j = find_jar(
-                self._SLF4J, path_to_slf4j,
-                env_vars=('SLF4J',),
-                searchpath=(), url=_stanford_url,
-                verbose=verbose)
-
-        # This is passed to java as the -cp option, the segmenter needs slf4j.
-        self._stanford_jar = ':'.join(
-            [_ for _ in [stanford_segmenter, slf4j] if not _ is None])
-
-        self._sihan_corpora_dict = path_to_sihan_corpora_dict
-        self._model = path_to_model
-        self._dict = path_to_dict
-
-        self._encoding = encoding
-        self.java_options = java_options
-        options = {} if options is None else options
-        self._options_cmd = ','.join('{0}={1}'.format(key, json.dumps(val)) for key, val in options.items())
-
-    def tokenize(self, s):
-        super().tokenize(s)
-
-    def segment_file(self, input_file_path):
-        """
-        """
-        cmd = [
-            'edu.stanford.nlp.ie.crf.CRFClassifier',
-            '-sighanCorporaDict', self._sihan_corpora_dict,
-            '-textFile', input_file_path,
-            '-sighanPostProcessing', 'true',
-            '-keepAllWhitespaces', 'false',
-            '-loadClassifier', self._model,
-            '-serDictionary', self._dict
-        ]
-
-        stdout = self._execute(cmd)
-
-        return stdout
-
-    def segment(self, tokens):
-        return self.segment_sents([tokens])
-
-    def segment_sents(self, sentences):
-        """
-        """
-        encoding = self._encoding
-        # Create a temporary input file
-        _input_fh, self._input_file_path = tempfile.mkstemp(text=True)
-
-        # Write the actural sentences to the temporary input file
-        _input_fh = os.fdopen(_input_fh, 'wb')
-        _input = '\n'.join((' '.join(x) for x in sentences))
-        if isinstance(_input, compat.text_type) and encoding:
-            _input = _input.encode(encoding)
-        _input_fh.write(_input)
-        _input_fh.close()
-
-        cmd = [
-            'edu.stanford.nlp.ie.crf.CRFClassifier',
-            '-sighanCorporaDict', self._sihan_corpora_dict,
-            '-textFile', self._input_file_path,
-            '-sighanPostProcessing', 'true',
-            '-keepAllWhitespaces', 'false',
-            '-loadClassifier', self._model,
-            '-serDictionary', self._dict
-        ]
-
-        stdout = self._execute(cmd)
-
-        # Delete the temporary file
-        os.unlink(self._input_file_path)
-
-        return stdout
-
-    def _execute(self, cmd, verbose=False):
-        encoding = self._encoding
-        cmd.extend(['-inputEncoding', encoding])
-        _options_cmd = self._options_cmd
-        if _options_cmd:
-            cmd.extend(['-options', self._options_cmd])
-
-        default_options = ' '.join(_java_options)
-
-        # Configure java.
-        config_java(options=self.java_options, verbose=verbose)
-
-        stdout, _stderr = java(
-            cmd,classpath=self._stanford_jar, stdout=PIPE, stderr=PIPE)
-        stdout = stdout.decode(encoding)
-
-        # Return java configurations to their default values.
-        config_java(options=default_options, verbose=False)
-
-        return stdout
-
-def setup_module(module):
-    from nose import SkipTest
-
-    try:
-        StanfordSegmenter()
-    except LookupError:
-        raise SkipTest('doctests from nltk.tokenize.stanford_segmenter are skipped because the stanford segmenter jar doesn\'t exist')

nltk-3.2.1/nltk/tokenize/treebank.py

@@ -1,124 +0,0 @@
-# Natural Language Toolkit: Tokenizers
-#
-# Copyright (C) 2001-2016 NLTK Project
-# Author: Edward Loper <edloper@gmail.com>
-#         Michael Heilman <mheilman@cmu.edu> (re-port from http://www.cis.upenn.edu/~treebank/tokenizer.sed)
-#
-# URL: <http://nltk.sourceforge.net>
-# For license information, see LICENSE.TXT
-
-r"""
-
-Penn Treebank Tokenizer
-
-The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
-This implementation is a port of the tokenizer sed script written by Robert McIntyre
-and available at http://www.cis.upenn.edu/~treebank/tokenizer.sed.
-"""
-
-import re
-from nltk.tokenize.api import TokenizerI
-
-
-class TreebankWordTokenizer(TokenizerI):
-    """
-    The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
-    This is the method that is invoked by ``word_tokenize()``.  It assumes that the
-    text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
-
-    This tokenizer performs the following steps:
-
-    - split standard contractions, e.g. ``don't`` -> ``do n't`` and ``they'll`` -> ``they 'll``
-    - treat most punctuation characters as separate tokens
-    - split off commas and single quotes, when followed by whitespace
-    - separate periods that appear at the end of line
-
-        >>> from nltk.tokenize import TreebankWordTokenizer
-        >>> s = '''Good muffins cost $3.88\\nin New York.  Please buy me\\ntwo of them.\\nThanks.'''
-        >>> TreebankWordTokenizer().tokenize(s)
-        ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks', '.']
-        >>> s = "They'll save and invest more."
-        >>> TreebankWordTokenizer().tokenize(s)
-        ['They', "'ll", 'save', 'and', 'invest', 'more', '.']
-        >>> s = "hi, my name can't hello,"
-        >>> TreebankWordTokenizer().tokenize(s)
-        ['hi', ',', 'my', 'name', 'ca', "n't", 'hello', ',']
-    """
-
-    #starting quotes
-    STARTING_QUOTES = [
-        (re.compile(r'^\"'), r'``'),
-        (re.compile(r'(``)'), r' \1 '),
-        (re.compile(r'([ (\[{<])"'), r'\1 `` '),
-    ]
-
-    #punctuation
-    PUNCTUATION = [
-        (re.compile(r'([:,])([^\d])'), r' \1 \2'),
-        (re.compile(r'([:,])$'), r' \1 '),
-        (re.compile(r'\.\.\.'), r' ... '),
-        (re.compile(r'[;@#$%&]'), r' \g<0> '),
-        (re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'), r'\1 \2\3 '),
-        (re.compile(r'[?!]'), r' \g<0> '),
-
-        (re.compile(r"([^'])' "), r"\1 ' "),
-    ]
-
-    #parens, brackets, etc.
-    PARENS_BRACKETS = [
-        (re.compile(r'[\]\[\(\)\{\}\<\>]'), r' \g<0> '),
-        (re.compile(r'--'), r' -- '),
-    ]
-
-    #ending quotes
-    ENDING_QUOTES = [
-        (re.compile(r'"'), " '' "),
-        (re.compile(r'(\S)(\'\')'), r'\1 \2 '),
-
-        (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
-        (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
-    ]
-
-    # List of contractions adapted from Robert MacIntyre's tokenizer.
-    CONTRACTIONS2 = [re.compile(r"(?i)\b(can)(not)\b"),
-                     re.compile(r"(?i)\b(d)('ye)\b"),
-                     re.compile(r"(?i)\b(gim)(me)\b"),
-                     re.compile(r"(?i)\b(gon)(na)\b"),
-                     re.compile(r"(?i)\b(got)(ta)\b"),
-                     re.compile(r"(?i)\b(lem)(me)\b"),
-                     re.compile(r"(?i)\b(mor)('n)\b"),
-                     re.compile(r"(?i)\b(wan)(na) ")]
-    CONTRACTIONS3 = [re.compile(r"(?i) ('t)(is)\b"),
-                     re.compile(r"(?i) ('t)(was)\b")]
-    CONTRACTIONS4 = [re.compile(r"(?i)\b(whad)(dd)(ya)\b"),
-                     re.compile(r"(?i)\b(wha)(t)(cha)\b")]
-
-    def tokenize(self, text):
-        for regexp, substitution in self.STARTING_QUOTES:
-            text = regexp.sub(substitution, text)
-
-        for regexp, substitution in self.PUNCTUATION:
-            text = regexp.sub(substitution, text)
-
-        for regexp, substitution in self.PARENS_BRACKETS:
-            text = regexp.sub(substitution, text)
-
-        #add extra space to make things easier
-        text = " " + text + " "
-
-        for regexp, substitution in self.ENDING_QUOTES:
-            text = regexp.sub(substitution, text)
-
-        for regexp in self.CONTRACTIONS2:
-            text = regexp.sub(r' \1 \2 ', text)
-        for regexp in self.CONTRACTIONS3:
-            text = regexp.sub(r' \1 \2 ', text)
-
-        # We are not using CONTRACTIONS4 since
-        # they are also commented out in the SED scripts
-        # for regexp in self.CONTRACTIONS4:
-        #     text = regexp.sub(r' \1 \2 \3 ', text)
-
-        return text.split()
-
-

nltk-3.2.1/nltk/tokenize/util.py

@@ -1,90 +0,0 @@
-# Natural Language Toolkit: Tokenizer Utilities
-#
-# Copyright (C) 2001-2016 NLTK Project
-# Author: Steven Bird <stevenbird1@gmail.com>
-# URL: <http://nltk.sourceforge.net>
-# For license information, see LICENSE.TXT
-
-from re import finditer
-
-def string_span_tokenize(s, sep):
-    r"""
-    Return the offsets of the tokens in *s*, as a sequence of ``(start, end)``
-    tuples, by splitting the string at each occurrence of *sep*.
-
-        >>> from nltk.tokenize.util import string_span_tokenize
-        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
-        ... two of them.\n\nThanks.'''
-        >>> list(string_span_tokenize(s, " "))
-        [(0, 4), (5, 12), (13, 17), (18, 26), (27, 30), (31, 36), (37, 37),
-        (38, 44), (45, 48), (49, 55), (56, 58), (59, 73)]
-
-    :param s: the string to be tokenized
-    :type s: str
-    :param sep: the token separator
-    :type sep: str
-    :rtype: iter(tuple(int, int))
-    """
-    if len(sep) == 0:
-        raise ValueError("Token delimiter must not be empty")
-    left = 0
-    while True:
-        try:
-            right = s.index(sep, left)
-            if right != 0:
-                yield left, right
-        except ValueError:
-            if left != len(s):
-                yield left, len(s)
-            break
-
-        left = right + len(sep)
-
-def regexp_span_tokenize(s, regexp):
-    r"""
-    Return the offsets of the tokens in *s*, as a sequence of ``(start, end)``
-    tuples, by splitting the string at each successive match of *regexp*.
-
-        >>> from nltk.tokenize.util import regexp_span_tokenize
-        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
-        ... two of them.\n\nThanks.'''
-        >>> list(regexp_span_tokenize(s, r'\s'))
-        [(0, 4), (5, 12), (13, 17), (18, 23), (24, 26), (27, 30), (31, 36),
-        (38, 44), (45, 48), (49, 51), (52, 55), (56, 58), (59, 64), (66, 73)]
-
-    :param s: the string to be tokenized
-    :type s: str
-    :param regexp: regular expression that matches token separators (must not be empty)
-    :type regexp: str
-    :rtype: iter(tuple(int, int))
-    """
-    left = 0
-    for m in finditer(regexp, s):
-        right, next = m.span()
-        if right != left:
-            yield left, right
-        left = next
-    yield left, len(s)
-
-def spans_to_relative(spans):
-    r"""
-    Return a sequence of relative spans, given a sequence of spans.
-
-        >>> from nltk.tokenize import WhitespaceTokenizer
-        >>> from nltk.tokenize.util import spans_to_relative
-        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
-        ... two of them.\n\nThanks.'''
-        >>> list(spans_to_relative(WhitespaceTokenizer().span_tokenize(s)))
-        [(0, 4), (1, 7), (1, 4), (1, 5), (1, 2), (1, 3), (1, 5), (2, 6),
-        (1, 3), (1, 2), (1, 3), (1, 2), (1, 5), (2, 7)]
-
-    :param spans: a sequence of (start, end) offsets of the tokens
-    :type spans: iter(tuple(int, int))
-    :rtype: iter(tuple(int, int))
-    """
-    prev = 0
-    for left, right in spans:
-        yield left - prev, right - left
-        prev = right
-
-