nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/stem/util.py

@@ -1,12 +1,22 @@
-# Natural Language Toolkit: Stemmer Utilities
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Helder <he7d3r@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-def suffix_replace(original, old, new):
-    """
-    Replaces the old suffix of the original string by a new suffix
-    """
-    return original[:-len(old)] + new
+# Natural Language Toolkit: Stemmer Utilities
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Helder <he7d3r@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+def suffix_replace(original, old, new):
+    """
+    Replaces the old suffix of the original string by a new suffix
+    """
+    return original[:-len(old)] + new
+
+def prefix_replace(original, old, new):
+    """
+     Replaces the old prefix of the original string by a new suffix
+    :param original: string
+    :param old: string
+    :param new: string
+    :return: string
+    """
+    return new + original[len(old):]

{nltk-3.2 → nltk-3.3}/nltk/stem/wordnet.py

@@ -1,51 +1,51 @@
-# Natural Language Toolkit: WordNet stemmer interface
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Steven Bird <stevenbird1@gmail.com>
-#         Edward Loper <edloper@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-from __future__ import unicode_literals
-
-from nltk.corpus.reader.wordnet import NOUN
-from nltk.corpus import wordnet
-from nltk.compat import python_2_unicode_compatible
-
-@python_2_unicode_compatible
-class WordNetLemmatizer(object):
-    """
-    WordNet Lemmatizer
-
-    Lemmatize using WordNet's built-in morphy function.
-    Returns the input word unchanged if it cannot be found in WordNet.
-
-        >>> from nltk.stem import WordNetLemmatizer
-        >>> wnl = WordNetLemmatizer()
-        >>> print(wnl.lemmatize('dogs'))
-        dog
-        >>> print(wnl.lemmatize('churches'))
-        church
-        >>> print(wnl.lemmatize('aardwolves'))
-        aardwolf
-        >>> print(wnl.lemmatize('abaci'))
-        abacus
-        >>> print(wnl.lemmatize('hardrock'))
-        hardrock
-    """
-
-    def __init__(self):
-        pass
-
-    def lemmatize(self, word, pos=NOUN):
-        lemmas = wordnet._morphy(word, pos)
-        return min(lemmas, key=len) if lemmas else word
-
-    def __repr__(self):
-        return '<WordNetLemmatizer>'
-
-
-# unload wordnet
-def teardown_module(module=None):
-    from nltk.corpus import wordnet
-    wordnet._unload()
-
+# Natural Language Toolkit: WordNet stemmer interface
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Steven Bird <stevenbird1@gmail.com>
+#         Edward Loper <edloper@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import unicode_literals
+
+from nltk.corpus.reader.wordnet import NOUN
+from nltk.corpus import wordnet
+from nltk.compat import python_2_unicode_compatible
+
+@python_2_unicode_compatible
+class WordNetLemmatizer(object):
+    """
+    WordNet Lemmatizer
+
+    Lemmatize using WordNet's built-in morphy function.
+    Returns the input word unchanged if it cannot be found in WordNet.
+
+        >>> from nltk.stem import WordNetLemmatizer
+        >>> wnl = WordNetLemmatizer()
+        >>> print(wnl.lemmatize('dogs'))
+        dog
+        >>> print(wnl.lemmatize('churches'))
+        church
+        >>> print(wnl.lemmatize('aardwolves'))
+        aardwolf
+        >>> print(wnl.lemmatize('abaci'))
+        abacus
+        >>> print(wnl.lemmatize('hardrock'))
+        hardrock
+    """
+
+    def __init__(self):
+        pass
+
+    def lemmatize(self, word, pos=NOUN):
+        lemmas = wordnet._morphy(word, pos)
+        return min(lemmas, key=len) if lemmas else word
+
+    def __repr__(self):
+        return '<WordNetLemmatizer>'
+
+
+# unload wordnet
+def teardown_module(module=None):
+    from nltk.corpus import wordnet
+    wordnet._unload()
+

{nltk-3.2 → nltk-3.3}/nltk/tag/__init__.py

@@ -1,127 +1,152 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: Taggers
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Edward Loper <edloper@gmail.com>
-#         Steven Bird <stevenbird1@gmail.com> (minor additions)
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-"""
-NLTK Taggers
-
-This package contains classes and interfaces for part-of-speech
-tagging, or simply "tagging".
-
-A "tag" is a case-sensitive string that specifies some property of a token,
-such as its part of speech.  Tagged tokens are encoded as tuples
-``(tag, token)``.  For example, the following tagged token combines
-the word ``'fly'`` with a noun part of speech tag (``'NN'``):
-
-    >>> tagged_tok = ('fly', 'NN')
-
-An off-the-shelf tagger is available.  It uses the Penn Treebank tagset:
-
-    >>> from nltk import pos_tag, word_tokenize
-    >>> pos_tag(word_tokenize("John's big idea isn't all that bad."))
-    [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'), ('idea', 'NN'), ('is', 'VBZ'),
-    ("n't", 'RB'), ('all', 'PDT'), ('that', 'DT'), ('bad', 'JJ'), ('.', '.')]
-
-This package defines several taggers, which take a list of tokens,
-assign a tag to each one, and return the resulting list of tagged tokens.
-Most of the taggers are built automatically based on a training corpus.
-For example, the unigram tagger tags each word *w* by checking what
-the most frequent tag for *w* was in a training corpus:
-
-    >>> from nltk.corpus import brown
-    >>> from nltk.tag import UnigramTagger
-    >>> tagger = UnigramTagger(brown.tagged_sents(categories='news')[:500])
-    >>> sent = ['Mitchell', 'decried', 'the', 'high', 'rate', 'of', 'unemployment']
-    >>> for word, tag in tagger.tag(sent):
-    ...     print(word, '->', tag)
-    Mitchell -> NP
-    decried -> None
-    the -> AT
-    high -> JJ
-    rate -> NN
-    of -> IN
-    unemployment -> None
-
-Note that words that the tagger has not seen during training receive a tag
-of ``None``.
-
-We evaluate a tagger on data that was not seen during training:
-
-    >>> tagger.evaluate(brown.tagged_sents(categories='news')[500:600])
-    0.73...
-
-For more information, please consult chapter 5 of the NLTK Book.
-"""
-from __future__ import print_function
-
-from nltk.tag.api           import TaggerI
-from nltk.tag.util          import str2tuple, tuple2str, untag
-from nltk.tag.sequential    import (SequentialBackoffTagger, ContextTagger,
-                                    DefaultTagger, NgramTagger, UnigramTagger,
-                                    BigramTagger, TrigramTagger, AffixTagger,
-                                    RegexpTagger, ClassifierBasedTagger,
-                                    ClassifierBasedPOSTagger)
-from nltk.tag.brill         import BrillTagger
-from nltk.tag.brill_trainer import BrillTaggerTrainer
-from nltk.tag.tnt           import TnT
-from nltk.tag.hunpos        import HunposTagger
-from nltk.tag.stanford      import StanfordTagger, StanfordPOSTagger, StanfordNERTagger
-from nltk.tag.hmm           import HiddenMarkovModelTagger, HiddenMarkovModelTrainer
-from nltk.tag.senna         import SennaTagger, SennaChunkTagger, SennaNERTagger
-from nltk.tag.mapping       import tagset_mapping, map_tag
-from nltk.tag.crf           import CRFTagger
-from nltk.tag.perceptron    import PerceptronTagger
-
-from nltk.data import load
-
-def _pos_tag(tokens, tagset, tagger):
-    tagged_tokens = tagger.tag(tokens)
-    if tagset:
-        tagged_tokens = [(token, map_tag('en-ptb', tagset, tag)) for (token, tag) in tagged_tokens]
-    return tagged_tokens
-
-def pos_tag(tokens, tagset=None):
-    """
-    Use NLTK's currently recommended part of speech tagger to
-    tag the given list of tokens.
-
-        >>> from nltk.tag import pos_tag
-        >>> from nltk.tokenize import word_tokenize
-        >>> pos_tag(word_tokenize("John's big idea isn't all that bad."))
-        [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'), ('idea', 'NN'), ('is', 'VBZ'),
-        ("n't", 'RB'), ('all', 'PDT'), ('that', 'DT'), ('bad', 'JJ'), ('.', '.')]
-        >>> pos_tag(word_tokenize("John's big idea isn't all that bad."), tagset='universal')
-        [('John', 'NOUN'), ("'s", 'PRT'), ('big', 'ADJ'), ('idea', 'NOUN'), ('is', 'VERB'),
-        ("n't", 'ADV'), ('all', 'DET'), ('that', 'DET'), ('bad', 'ADJ'), ('.', '.')]
-
-    NB. Use `pos_tag_sents()` for efficient tagging of more than one sentence.
-
-    :param tokens: Sequence of tokens to be tagged
-    :type tokens: list(str)
-    :param tagset: the tagset to be used, e.g. universal, wsj, brown
-    :type tagset: str
-    :return: The tagged tokens
-    :rtype: list(tuple(str, str))
-    """
-    tagger = PerceptronTagger()
-    return _pos_tag(tokens, tagset, tagger)    
-
-
-def pos_tag_sents(sentences, tagset=None):
-    """
-    Use NLTK's currently recommended part of speech tagger to tag the
-    given list of sentences, each consisting of a list of tokens.
-
-    :param tokens: List of sentences to be tagged
-    :type tokens: list(list(str))
-    :param tagset: the tagset to be used, e.g. universal, wsj, brown
-    :type tagset: str
-    :return: The list of tagged sentences
-    :rtype: list(list(tuple(str, str)))
-    """
-    tagger = PerceptronTagger()
-    return [_pos_tag(sent, tagset, tagger) for sent in sentences]
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Taggers
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+NLTK Taggers
+
+This package contains classes and interfaces for part-of-speech
+tagging, or simply "tagging".
+
+A "tag" is a case-sensitive string that specifies some property of a token,
+such as its part of speech.  Tagged tokens are encoded as tuples
+``(tag, token)``.  For example, the following tagged token combines
+the word ``'fly'`` with a noun part of speech tag (``'NN'``):
+
+    >>> tagged_tok = ('fly', 'NN')
+
+An off-the-shelf tagger is available for English. It uses the Penn Treebank tagset:
+
+    >>> from nltk import pos_tag, word_tokenize
+    >>> pos_tag(word_tokenize("John's big idea isn't all that bad."))
+    [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'), ('idea', 'NN'), ('is', 'VBZ'),
+    ("n't", 'RB'), ('all', 'PDT'), ('that', 'DT'), ('bad', 'JJ'), ('.', '.')]
+
+A Russian tagger is also available if you specify lang="rus". It uses 
+the Russian National Corpus tagset:
+
+    >>> pos_tag(word_tokenize("Илья оторопел и дважды перечитал бумажку."), lang='rus')    # doctest: +SKIP
+    [('Илья', 'S'), ('оторопел', 'V'), ('и', 'CONJ'), ('дважды', 'ADV'), ('перечитал', 'V'),
+    ('бумажку', 'S'), ('.', 'NONLEX')]
+
+This package defines several taggers, which take a list of tokens,
+assign a tag to each one, and return the resulting list of tagged tokens.
+Most of the taggers are built automatically based on a training corpus.
+For example, the unigram tagger tags each word *w* by checking what
+the most frequent tag for *w* was in a training corpus:
+
+    >>> from nltk.corpus import brown
+    >>> from nltk.tag import UnigramTagger
+    >>> tagger = UnigramTagger(brown.tagged_sents(categories='news')[:500])
+    >>> sent = ['Mitchell', 'decried', 'the', 'high', 'rate', 'of', 'unemployment']
+    >>> for word, tag in tagger.tag(sent):
+    ...     print(word, '->', tag)
+    Mitchell -> NP
+    decried -> None
+    the -> AT
+    high -> JJ
+    rate -> NN
+    of -> IN
+    unemployment -> None
+
+Note that words that the tagger has not seen during training receive a tag
+of ``None``.
+
+We evaluate a tagger on data that was not seen during training:
+
+    >>> tagger.evaluate(brown.tagged_sents(categories='news')[500:600])
+    0.73...
+
+For more information, please consult chapter 5 of the NLTK Book.
+"""
+from __future__ import print_function
+
+from nltk.tag.api           import TaggerI
+from nltk.tag.util          import str2tuple, tuple2str, untag
+from nltk.tag.sequential    import (SequentialBackoffTagger, ContextTagger,
+                                    DefaultTagger, NgramTagger, UnigramTagger,
+                                    BigramTagger, TrigramTagger, AffixTagger,
+                                    RegexpTagger, ClassifierBasedTagger,
+                                    ClassifierBasedPOSTagger)
+from nltk.tag.brill         import BrillTagger
+from nltk.tag.brill_trainer import BrillTaggerTrainer
+from nltk.tag.tnt           import TnT
+from nltk.tag.hunpos        import HunposTagger
+from nltk.tag.stanford      import StanfordTagger, StanfordPOSTagger, StanfordNERTagger
+from nltk.tag.hmm           import HiddenMarkovModelTagger, HiddenMarkovModelTrainer
+from nltk.tag.senna         import SennaTagger, SennaChunkTagger, SennaNERTagger
+from nltk.tag.mapping       import tagset_mapping, map_tag
+from nltk.tag.crf           import CRFTagger
+from nltk.tag.perceptron    import PerceptronTagger
+
+from nltk.data import load, find
+
+RUS_PICKLE = 'taggers/averaged_perceptron_tagger_ru/averaged_perceptron_tagger_ru.pickle'
+
+
+def _get_tagger(lang=None):
+    if lang == 'rus':
+        tagger = PerceptronTagger(False)
+        ap_russian_model_loc = 'file:' + str(find(RUS_PICKLE))
+        tagger.load(ap_russian_model_loc)
+    else:
+        tagger = PerceptronTagger()
+    return tagger
+
+
+def _pos_tag(tokens, tagset, tagger):
+    tagged_tokens = tagger.tag(tokens)
+    if tagset:
+        tagged_tokens = [(token, map_tag('en-ptb', tagset, tag)) for (token, tag) in tagged_tokens]
+    return tagged_tokens
+
+
+def pos_tag(tokens, tagset=None, lang='eng'):
+    """
+    Use NLTK's currently recommended part of speech tagger to
+    tag the given list of tokens.
+
+        >>> from nltk.tag import pos_tag
+        >>> from nltk.tokenize import word_tokenize
+        >>> pos_tag(word_tokenize("John's big idea isn't all that bad."))
+        [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'), ('idea', 'NN'), ('is', 'VBZ'),
+        ("n't", 'RB'), ('all', 'PDT'), ('that', 'DT'), ('bad', 'JJ'), ('.', '.')]
+        >>> pos_tag(word_tokenize("John's big idea isn't all that bad."), tagset='universal')
+        [('John', 'NOUN'), ("'s", 'PRT'), ('big', 'ADJ'), ('idea', 'NOUN'), ('is', 'VERB'),
+        ("n't", 'ADV'), ('all', 'DET'), ('that', 'DET'), ('bad', 'ADJ'), ('.', '.')]
+
+    NB. Use `pos_tag_sents()` for efficient tagging of more than one sentence.
+
+    :param tokens: Sequence of tokens to be tagged
+    :type tokens: list(str)
+    :param tagset: the tagset to be used, e.g. universal, wsj, brown
+    :type tagset: str
+    :param lang: the ISO 639 code of the language, e.g. 'eng' for English, 'rus' for Russian
+    :type lang: str
+    :return: The tagged tokens
+    :rtype: list(tuple(str, str))
+    """
+    tagger = _get_tagger(lang)
+    return _pos_tag(tokens, tagset, tagger)    
+
+
+def pos_tag_sents(sentences, tagset=None, lang='eng'):
+    """
+    Use NLTK's currently recommended part of speech tagger to tag the
+    given list of sentences, each consisting of a list of tokens.
+
+    :param tokens: List of sentences to be tagged
+    :type tokens: list(list(str))
+    :param tagset: the tagset to be used, e.g. universal, wsj, brown
+    :type tagset: str
+    :param lang: the ISO 639 code of the language, e.g. 'eng' for English, 'rus' for Russian
+    :type lang: str
+    :return: The list of tagged sentences
+    :rtype: list(list(tuple(str, str)))
+    """
+    tagger = _get_tagger(lang)
+    return [_pos_tag(sent, tagset, tagger) for sent in sentences]

{nltk-3.2 → nltk-3.3}/nltk/tag/api.py

@@ -1,81 +1,86 @@
-# Natural Language Toolkit: Tagger Interface
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Edward Loper <edloper@gmail.com>
-#         Steven Bird <stevenbird1@gmail.com> (minor additions)
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Interface for tagging each token in a sentence with supplementary
-information, such as its part of speech.
-"""
-from nltk.internals import overridden
-from nltk.metrics import accuracy
-
-from nltk.tag.util import untag
-
-class TaggerI(object):
-    """
-    A processing interface for assigning a tag to each token in a list.
-    Tags are case sensitive strings that identify some property of each
-    token, such as its part of speech or its sense.
-
-    Some taggers require specific types for their tokens.  This is
-    generally indicated by the use of a sub-interface to ``TaggerI``.
-    For example, featureset taggers, which are subclassed from
-    ``FeaturesetTagger``, require that each token be a ``featureset``.
-
-    Subclasses must define:
-      - either ``tag()`` or ``tag_sents()`` (or both)
-    """
-    def tag(self, tokens):
-        """
-        Determine the most appropriate tag sequence for the given
-        token sequence, and return a corresponding list of tagged
-        tokens.  A tagged token is encoded as a tuple ``(token, tag)``.
-
-        :rtype: list(tuple(str, str))
-        """
-        if overridden(self.tag_sents):
-            return self.tag_sents([tokens])[0]
-        else:
-            raise NotImplementedError()
-
-    def tag_sents(self, sentences):
-        """
-        Apply ``self.tag()`` to each element of *sentences*.  I.e.:
-
-            return [self.tag(sent) for sent in sentences]
-        """
-        return [self.tag(sent) for sent in sentences]
-
-    def evaluate(self, gold):
-        """
-        Score the accuracy of the tagger against the gold standard.
-        Strip the tags from the gold standard text, retag it using
-        the tagger, then compute the accuracy score.
-
-        :type gold: list(list(tuple(str, str)))
-        :param gold: The list of tagged sentences to score the tagger on.
-        :rtype: float
-        """
-
-        tagged_sents = self.tag_sents(untag(sent) for sent in gold)
-        gold_tokens = sum(gold, [])
-        test_tokens = sum(tagged_sents, [])
-        return accuracy(gold_tokens, test_tokens)
-
-    def _check_params(self, train, model):
-        if (train and model) or (not train and not model):
-            raise ValueError('Must specify either training data or trained model.')
-
-class FeaturesetTaggerI(TaggerI):
-    """
-    A tagger that requires tokens to be ``featuresets``.  A featureset
-    is a dictionary that maps from feature names to feature
-    values.  See ``nltk.classify`` for more information about features
-    and featuresets.
-    """
-
-
+# Natural Language Toolkit: Tagger Interface
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interface for tagging each token in a sentence with supplementary
+information, such as its part of speech.
+"""
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
+from itertools import chain
+
+from nltk.internals import overridden
+from nltk.metrics import accuracy
+
+from nltk.tag.util import untag
+
+
+@add_metaclass(ABCMeta)
+class TaggerI(object):
+    """
+    A processing interface for assigning a tag to each token in a list.
+    Tags are case sensitive strings that identify some property of each
+    token, such as its part of speech or its sense.
+
+    Some taggers require specific types for their tokens.  This is
+    generally indicated by the use of a sub-interface to ``TaggerI``.
+    For example, featureset taggers, which are subclassed from
+    ``FeaturesetTagger``, require that each token be a ``featureset``.
+
+    Subclasses must define:
+      - either ``tag()`` or ``tag_sents()`` (or both)
+    """
+    @abstractmethod
+    def tag(self, tokens):
+        """
+        Determine the most appropriate tag sequence for the given
+        token sequence, and return a corresponding list of tagged
+        tokens.  A tagged token is encoded as a tuple ``(token, tag)``.
+
+        :rtype: list(tuple(str, str))
+        """
+        if overridden(self.tag_sents):
+            return self.tag_sents([tokens])[0]
+
+    def tag_sents(self, sentences):
+        """
+        Apply ``self.tag()`` to each element of *sentences*.  I.e.:
+
+            return [self.tag(sent) for sent in sentences]
+        """
+        return [self.tag(sent) for sent in sentences]
+
+    def evaluate(self, gold):
+        """
+        Score the accuracy of the tagger against the gold standard.
+        Strip the tags from the gold standard text, retag it using
+        the tagger, then compute the accuracy score.
+
+        :type gold: list(list(tuple(str, str)))
+        :param gold: The list of tagged sentences to score the tagger on.
+        :rtype: float
+        """
+
+        tagged_sents = self.tag_sents(untag(sent) for sent in gold)
+        gold_tokens = list(chain(*gold))
+        test_tokens = list(chain(*tagged_sents))
+        return accuracy(gold_tokens, test_tokens)
+
+    def _check_params(self, train, model):
+        if (train and model) or (not train and not model):
+            raise ValueError(
+                    'Must specify either training data or trained model.')
+
+
+class FeaturesetTaggerI(TaggerI):
+    """
+    A tagger that requires tokens to be ``featuresets``.  A featureset
+    is a dictionary that maps from feature names to feature
+    values.  See ``nltk.classify`` for more information about features
+    and featuresets.
+    """