nltk 3.2.1__zip → 3.3__zip

{nltk-3.2.1 → nltk-3.3}/nltk/tag/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Taggers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
 # URL: <http://nltk.org/>
@@ -19,13 +19,20 @@ the word ``'fly'`` with a noun part of speech tag (``'NN'``):
 
     >>> tagged_tok = ('fly', 'NN')
 
-An off-the-shelf tagger is available.  It uses the Penn Treebank tagset:
+An off-the-shelf tagger is available for English. It uses the Penn Treebank tagset:
 
     >>> from nltk import pos_tag, word_tokenize
     >>> pos_tag(word_tokenize("John's big idea isn't all that bad."))
     [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'), ('idea', 'NN'), ('is', 'VBZ'),
     ("n't", 'RB'), ('all', 'PDT'), ('that', 'DT'), ('bad', 'JJ'), ('.', '.')]
 
+A Russian tagger is also available if you specify lang="rus". It uses 
+the Russian National Corpus tagset:
+
+    >>> pos_tag(word_tokenize("Илья оторопел и дважды перечитал бумажку."), lang='rus')    # doctest: +SKIP
+    [('Илья', 'S'), ('оторопел', 'V'), ('и', 'CONJ'), ('дважды', 'ADV'), ('перечитал', 'V'),
+    ('бумажку', 'S'), ('.', 'NONLEX')]
+
 This package defines several taggers, which take a list of tokens,
 assign a tag to each one, and return the resulting list of tagged tokens.
 Most of the taggers are built automatically based on a training corpus.
@@ -76,7 +83,20 @@ from nltk.tag.mapping       import tagset_mapping, map_tag
 from nltk.tag.crf           import CRFTagger
 from nltk.tag.perceptron    import PerceptronTagger
 
-from nltk.data import load
+from nltk.data import load, find
+
+RUS_PICKLE = 'taggers/averaged_perceptron_tagger_ru/averaged_perceptron_tagger_ru.pickle'
+
+
+def _get_tagger(lang=None):
+    if lang == 'rus':
+        tagger = PerceptronTagger(False)
+        ap_russian_model_loc = 'file:' + str(find(RUS_PICKLE))
+        tagger.load(ap_russian_model_loc)
+    else:
+        tagger = PerceptronTagger()
+    return tagger
+
 
 def _pos_tag(tokens, tagset, tagger):
     tagged_tokens = tagger.tag(tokens)
@@ -84,7 +104,8 @@ def _pos_tag(tokens, tagset, tagger):
         tagged_tokens = [(token, map_tag('en-ptb', tagset, tag)) for (token, tag) in tagged_tokens]
     return tagged_tokens
 
-def pos_tag(tokens, tagset=None):
+
+def pos_tag(tokens, tagset=None, lang='eng'):
     """
     Use NLTK's currently recommended part of speech tagger to
     tag the given list of tokens.
@@ -104,14 +125,16 @@ def pos_tag(tokens, tagset=None):
     :type tokens: list(str)
     :param tagset: the tagset to be used, e.g. universal, wsj, brown
     :type tagset: str
+    :param lang: the ISO 639 code of the language, e.g. 'eng' for English, 'rus' for Russian
+    :type lang: str
     :return: The tagged tokens
     :rtype: list(tuple(str, str))
     """
-    tagger = PerceptronTagger()
+    tagger = _get_tagger(lang)
     return _pos_tag(tokens, tagset, tagger)    
 
 
-def pos_tag_sents(sentences, tagset=None):
+def pos_tag_sents(sentences, tagset=None, lang='eng'):
     """
     Use NLTK's currently recommended part of speech tagger to tag the
     given list of sentences, each consisting of a list of tokens.
@@ -120,8 +143,10 @@ def pos_tag_sents(sentences, tagset=None):
     :type tokens: list(list(str))
     :param tagset: the tagset to be used, e.g. universal, wsj, brown
     :type tagset: str
+    :param lang: the ISO 639 code of the language, e.g. 'eng' for English, 'rus' for Russian
+    :type lang: str
     :return: The list of tagged sentences
     :rtype: list(list(tuple(str, str)))
     """
-    tagger = PerceptronTagger()
+    tagger = _get_tagger(lang)
     return [_pos_tag(sent, tagset, tagger) for sent in sentences]

{nltk-3.2.1 → nltk-3.3}/nltk/tag/api.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Tagger Interface
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
 # URL: <http://nltk.org/>
@@ -10,11 +10,17 @@
 Interface for tagging each token in a sentence with supplementary
 information, such as its part of speech.
 """
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
+from itertools import chain
+
 from nltk.internals import overridden
 from nltk.metrics import accuracy
 
 from nltk.tag.util import untag
 
+
+@add_metaclass(ABCMeta)
 class TaggerI(object):
     """
     A processing interface for assigning a tag to each token in a list.
@@ -29,6 +35,7 @@ class TaggerI(object):
     Subclasses must define:
       - either ``tag()`` or ``tag_sents()`` (or both)
     """
+    @abstractmethod
     def tag(self, tokens):
         """
         Determine the most appropriate tag sequence for the given
@@ -39,8 +46,6 @@ class TaggerI(object):
         """
         if overridden(self.tag_sents):
             return self.tag_sents([tokens])[0]
-        else:
-            raise NotImplementedError()
 
     def tag_sents(self, sentences):
         """
@@ -62,13 +67,15 @@ class TaggerI(object):
         """
 
         tagged_sents = self.tag_sents(untag(sent) for sent in gold)
-        gold_tokens = sum(gold, [])
-        test_tokens = sum(tagged_sents, [])
+        gold_tokens = list(chain(*gold))
+        test_tokens = list(chain(*tagged_sents))
         return accuracy(gold_tokens, test_tokens)
 
     def _check_params(self, train, model):
         if (train and model) or (not train and not model):
-            raise ValueError('Must specify either training data or trained model.')
+            raise ValueError(
+                    'Must specify either training data or trained model.')
+
 
 class FeaturesetTaggerI(TaggerI):
     """
@@ -77,5 +84,3 @@ class FeaturesetTaggerI(TaggerI):
     values.  See ``nltk.classify`` for more information about features
     and featuresets.
     """
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/tag/brill.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Transformation-based learning
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Marcus Uneson <marcus.uneson@gmail.com>
 #   based on previous (nltk2) version by
 #   Christopher Maloof, Edward Loper, Steven Bird
@@ -10,9 +10,8 @@
 
 from __future__ import print_function, division
 
-from collections import defaultdict
+from collections import defaultdict, Counter
 
-from nltk.compat import Counter
 from nltk.tag import TaggerI
 from nltk.tbl import Feature, Template
 from nltk import jsontags
@@ -376,8 +375,9 @@ class BrillTagger(TaggerI):
             usedtpls = set([int(tid) for tid in tids])
             unused = [(tid, tpl) for (tid, tpl) in enumerate(Template.ALLTEMPLATES) if tid not in usedtpls]
             print("UNUSED TEMPLATES ({0})".format(len(unused)))
+
             for (tid, tpl) in unused:
-                print("{0:03d} {1:s}".format(tid, tpl))
+                print("{0:03d} {1:s}".format(tid, str(tpl)))
 
         if test_stats is None:
             print_train_stats()
@@ -422,5 +422,3 @@ class BrillTagger(TaggerI):
         testing_stats['finalerrors'] = errors[-1]
         testing_stats['finalacc'] = 1 - testing_stats['finalerrors']/testing_stats['tokencount']
         return (tagged_tokenses, testing_stats)
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/tag/crf.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Interface to the CRFSuite Tagger
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Long Duong <longdt219@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT

{nltk-3.2.1 → nltk-3.3}/nltk/tag/hmm.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Hidden Markov Model
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Trevor Cohn <tacohn@csse.unimelb.edu.au>
 #         Philip Blunsom <pcbl@csse.unimelb.edu.au>
 #         Tiago Tresoldi <tiago@tresoldi.pro.br> (fixes)
@@ -73,6 +73,8 @@ from __future__ import print_function, unicode_literals, division
 import re
 import itertools
 
+from six.moves import map, zip
+
 try:
     import numpy as np
 except ImportError:
@@ -85,7 +87,7 @@ from nltk.probability import (FreqDist, ConditionalFreqDist,
                               MLEProbDist, RandomProbDist)
 from nltk.metrics import accuracy
 from nltk.util import LazyMap, unique_list
-from nltk.compat import python_2_unicode_compatible, izip, imap
+from nltk.compat import python_2_unicode_compatible
 from nltk.tag.api import TaggerI
 
 
@@ -269,7 +271,7 @@ class HiddenMarkovModelTagger(TaggerI):
 
     def _tag(self, unlabeled_sequence):
         path = self._best_path(unlabeled_sequence)
-        return list(izip(unlabeled_sequence, path))
+        return list(zip(unlabeled_sequence, path))
 
     def _output_logprob(self, state, symbol):
         """
@@ -778,10 +780,10 @@ class HiddenMarkovModelTagger(TaggerI):
             return list(itertools.chain(*seq))
 
         test_sequence = self._transform(test_sequence)
-        predicted_sequence = list(imap(self._tag, imap(words, test_sequence)))
+        predicted_sequence = list(map(self._tag, map(words, test_sequence)))
 
         if verbose:
-            for test_sent, predicted_sent in izip(test_sequence, predicted_sequence):
+            for test_sent, predicted_sent in zip(test_sequence, predicted_sequence):
                 print('Test:',
                     ' '.join('%s/%s' % (token, tag)
                              for (token, tag) in test_sent))
@@ -799,8 +801,8 @@ class HiddenMarkovModelTagger(TaggerI):
                 print()
                 print('-' * 60)
 
-        test_tags = flatten(imap(tags, test_sequence))
-        predicted_tags = flatten(imap(tags, predicted_sequence))
+        test_tags = flatten(map(tags, test_sequence))
+        predicted_tags = flatten(map(tags, predicted_sequence))
 
         acc = accuracy(test_tags, predicted_tags)
         count = sum(len(sent) for sent in test_sequence)
@@ -840,9 +842,11 @@ class HiddenMarkovModelTrainer(object):
         :rtype: HiddenMarkovModelTagger
         :param labelled_sequences: the supervised training data, a set of
             labelled sequences of observations
+            ex: [ (word_1, tag_1),...,(word_n,tag_n) ] 
         :type labelled_sequences: list
         :param unlabeled_sequences: the unsupervised training data, a set of
             sequences of observations
+            ex: [ word_1, ..., word_n ]
         :type unlabeled_sequences: list
         :param kwargs: additional arguments to pass to the training methods
         """
@@ -1271,7 +1275,3 @@ def demo_bw():
     trainer = HiddenMarkovModelTrainer(states, symbols)
     hmm = trainer.train_unsupervised(training, model=model,
                                      max_iterations=1000)
-
-
-
-

{nltk-3.2.1 → nltk-3.3}/nltk/tag/hunpos.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Natural Language Toolkit: Interface to the HunPos POS-tagger
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Peter Ljunglöf <peter.ljunglof@heatherleaf.se>
 #         Dávid Márk Nemeskey <nemeskeyd@gmail.com> (modifications)
 #         Attila Zséder <zseder@gmail.com> (modifications)
@@ -15,9 +15,10 @@ A module for interfacing with the HunPos open-source POS-tagger.
 import os
 from subprocess import Popen, PIPE
 
+from six import text_type
+
 from nltk.internals import find_binary, find_file
 from nltk.tag.api import TaggerI
-from nltk import compat
 
 _hunpos_url = 'http://code.google.com/p/hunpos/'
 
@@ -105,7 +106,7 @@ class HunposTagger(TaggerI):
         """
         for token in tokens:
             assert "\n" not in token, "Tokens should not contain newlines"
-            if isinstance(token, compat.text_type):
+            if isinstance(token, text_type):
                 token = token.encode(self._encoding)
             self._hunpos.stdin.write(token + b"\n")
         # We write a final empty line to tell hunpos that the sentence is finished:
@@ -129,4 +130,3 @@ def setup_module(module):
         HunposTagger('en_wsj.model')
     except LookupError:
         raise SkipTest("HunposTagger is not available")
-

{nltk-3.2.1 → nltk-3.3}/nltk/tag/mapping.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Tagset Mapping
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Nathan Schneider <nathan@cmu.edu>
 #         Steven Bird <stevenbird1@gmail.com>
 # URL: <http://nltk.org/>

{nltk-3.2.1 → nltk-3.3}/nltk/tag/perceptron.py

@@ -28,7 +28,7 @@ class AveragedPerceptron(object):
     '''An averaged perceptron, as implemented by Matthew Honnibal.
 
     See more implementation details here:
-        http://spacy.io/blog/part-of-speech-POS-tagger-in-python/
+        https://explosion.ai/blog/part-of-speech-pos-tagger-in-python
     '''
 
     def __init__(self):
@@ -101,7 +101,7 @@ class PerceptronTagger(TaggerI):
     '''
     Greedy Averaged Perceptron tagger, as implemented by Matthew Honnibal.
     See more implementation details here:
-        http://spacy.io/blog/part-of-speech-POS-tagger-in-python/
+        https://explosion.ai/blog/part-of-speech-pos-tagger-in-python
     
     >>> from nltk.tag.perceptron import PerceptronTagger
 
@@ -165,18 +165,26 @@ class PerceptronTagger(TaggerI):
         '''Train a model from sentences, and save it at ``save_loc``. ``nr_iter``
         controls the number of Perceptron training iterations.
 
-        :param sentences: A list of (words, tags) tuples.
+        :param sentences: A list or iterator of sentences, where each sentence
+            is a list of (words, tags) tuples.
         :param save_loc: If not ``None``, saves a pickled model in this location.
         :param nr_iter: Number of training iterations.
         '''
+        # We'd like to allow ``sentences`` to be either a list or an iterator,
+        # the latter being especially important for a large training dataset.
+        # Because ``self._make_tagdict(sentences)`` runs regardless, we make
+        # it populate ``self._sentences`` (a list) with all the sentences.
+        # This saves the overheard of just iterating through ``sentences`` to
+        # get the list by ``sentences = list(sentences)``.
+
+        self._sentences = list()  # to be populated by self._make_tagdict...
         self._make_tagdict(sentences)
         self.model.classes = self.classes
         for iter_ in range(nr_iter):
             c = 0
             n = 0
-            for sentence  in sentences:
-                words = [word for word,tag in sentence]
-                tags  = [tag for word,tag in sentence]
+            for sentence in self._sentences:
+                words, tags = zip(*sentence)
                 
                 prev, prev2 = self.START
                 context = self.START + [self.normalize(w) for w in words] \
@@ -191,13 +199,19 @@ class PerceptronTagger(TaggerI):
                     prev = guess
                     c += guess == tags[i]
                     n += 1
-            random.shuffle(sentences)
+            random.shuffle(self._sentences)
             logging.info("Iter {0}: {1}/{2}={3}".format(iter_, c, n, _pc(c, n)))
+
+        # We don't need the training sentences anymore, and we don't want to
+        # waste space on them when we pickle the trained tagger.
+        self._sentences = None
+
         self.model.average_weights()
         # Pickle as a binary file
         if save_loc is not None:
             with open(save_loc, 'wb') as fout:
-                pickle.dump((self.model.weights, self.tagdict, self.classes), fout, -1)
+                # changed protocol from -1 to 2 to make pickling Python 2 compatible
+                pickle.dump((self.model.weights, self.tagdict, self.classes), fout, 2)
         
 
     def load(self, loc):
@@ -214,7 +228,7 @@ class PerceptronTagger(TaggerI):
         '''
         Normalization used in pre-processing.
         - All words are lower cased
-        - Digits in the range 1800-2100 are represented as !YEAR;
+        - Groups of digits of length 4 are represented as !YEAR;
         - Other digits are represented as !DIGITS
 
         :rtype: str
@@ -230,7 +244,7 @@ class PerceptronTagger(TaggerI):
 
     def _get_features(self, i, word, context, prev, prev2):
         '''Map tokens into a feature representation, implemented as a
-        {hashable: float} dict. If the features change, a new model must be
+        {hashable: int} dict. If the features change, a new model must be
         trained.
         '''
         def add(name, *args):
@@ -262,6 +276,7 @@ class PerceptronTagger(TaggerI):
         '''
         counts = defaultdict(lambda: defaultdict(int))
         for sentence in sentences:
+            self._sentences.append(sentence)
             for word, tag in sentence:
                 counts[word][tag] += 1
                 self.classes.add(tag)

{nltk-3.2.1 → nltk-3.3}/nltk/tag/senna.py

@@ -1,7 +1,7 @@
 # encoding: utf-8
 # Natural Language Toolkit: Senna POS Tagger
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Rami Al-Rfou' <ralrfou@cs.stonybrook.edu>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
@@ -10,29 +10,31 @@
 Senna POS tagger, NER Tagger, Chunk Tagger
 
 The input is:
-- path to the directory that contains SENNA executables. If the path is incorrect, 
-   SennaTagger will automatically search for executable file specified in SENNA environment variable 
+- path to the directory that contains SENNA executables. If the path is incorrect,
+   SennaTagger will automatically search for executable file specified in SENNA environment variable
 - (optionally) the encoding of the input data (default:utf-8)
 
+Note: Unit tests for this module can be found in test/unit/test_senna.py
+
     >>> from nltk.tag import SennaTagger
-    >>> tagger = SennaTagger('/usr/share/senna-v2.0')
-    >>> tagger.tag('What is the airspeed of an unladen swallow ?'.split())
+    >>> tagger = SennaTagger('/usr/share/senna-v3.0')
+    >>> tagger.tag('What is the airspeed of an unladen swallow ?'.split()) # doctest: +SKIP
     [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'),
     ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'NN'), ('?', '.')]
 
     >>> from nltk.tag import SennaChunkTagger
-    >>> chktagger = SennaChunkTagger('/usr/share/senna-v2.0')
-    >>> chktagger.tag('What is the airspeed of an unladen swallow ?'.split())
+    >>> chktagger = SennaChunkTagger('/usr/share/senna-v3.0')
+    >>> chktagger.tag('What is the airspeed of an unladen swallow ?'.split()) # doctest: +SKIP
     [('What', 'B-NP'), ('is', 'B-VP'), ('the', 'B-NP'), ('airspeed', 'I-NP'),
     ('of', 'B-PP'), ('an', 'B-NP'), ('unladen', 'I-NP'), ('swallow', 'I-NP'),
     ('?', 'O')]
 
     >>> from nltk.tag import SennaNERTagger
-    >>> nertagger = SennaNERTagger('/usr/share/senna-v2.0')
-    >>> nertagger.tag('Shakespeare theatre was in London .'.split())
+    >>> nertagger = SennaNERTagger('/usr/share/senna-v3.0')
+    >>> nertagger.tag('Shakespeare theatre was in London .'.split()) # doctest: +SKIP
     [('Shakespeare', 'B-PER'), ('theatre', 'O'), ('was', 'O'), ('in', 'O'),
     ('London', 'B-LOC'), ('.', 'O')]
-    >>> nertagger.tag('UN headquarters are in NY , USA .'.split())
+    >>> nertagger.tag('UN headquarters are in NY , USA .'.split()) # doctest: +SKIP
     [('UN', 'B-ORG'), ('headquarters', 'O'), ('are', 'O'), ('in', 'O'),
     ('NY', 'B-LOC'), (',', 'O'), ('USA', 'B-LOC'), ('.', 'O')]
 """
@@ -73,29 +75,29 @@ class SennaChunkTagger(Senna):
                 annotations = tagged_sents[i][j]
                 tagged_sents[i][j] = (annotations['word'], annotations['chk'])
         return tagged_sents
-        
+
     def bio_to_chunks(self, tagged_sent, chunk_type):
         """
         Extracts the chunks in a BIO chunk-tagged sentence.
-        
+
         >>> from nltk.tag import SennaChunkTagger
-        >>> chktagger = SennaChunkTagger('/usr/share/senna-v2.0')
+        >>> chktagger = SennaChunkTagger('/usr/share/senna-v3.0')
         >>> sent = 'What is the airspeed of an unladen swallow ?'.split()
-        >>> tagged_sent = chktagger.tag(sent)
-        >>> tagged_sent
+        >>> tagged_sent = chktagger.tag(sent) # doctest: +SKIP
+        >>> tagged_sent # doctest: +SKIP
         [('What', 'B-NP'), ('is', 'B-VP'), ('the', 'B-NP'), ('airspeed', 'I-NP'),
         ('of', 'B-PP'), ('an', 'B-NP'), ('unladen', 'I-NP'), ('swallow', 'I-NP'),
         ('?', 'O')]
-        >>> list(chktagger.bio_to_chunks(tagged_sent, chunk_type='NP'))
+        >>> list(chktagger.bio_to_chunks(tagged_sent, chunk_type='NP')) # doctest: +SKIP
         [('What', '0'), ('the airspeed', '2-3'), ('an unladen swallow', '5-6-7')]
-        
+
         :param tagged_sent: A list of tuples of word and BIO chunk tag.
         :type tagged_sent: list(tuple)
         :param tagged_sent: The chunk tag that users want to extract, e.g. 'NP' or 'VP'
         :type tagged_sent: str
-        
+
         :return: An iterable of tuples of chunks that users want to extract
-          and their corresponding indices. 
+          and their corresponding indices.
         :rtype: iter(tuple(str))
         """
         current_chunk = []
@@ -107,14 +109,14 @@ class SennaChunkTagger(Senna):
                 current_chunk_position.append((idx))
             else:
                 if current_chunk: # Flush the full chunk when out of an NP.
-                    _chunk_str = ' '.join(current_chunk) 
+                    _chunk_str = ' '.join(current_chunk)
                     _chunk_pos_str = '-'.join(map(str, current_chunk_position))
-                    yield _chunk_str, _chunk_pos_str 
+                    yield _chunk_str, _chunk_pos_str
                     current_chunk = []
                     current_chunk_position = []
         if current_chunk: # Flush the last chunk.
             yield ' '.join(current_chunk), '-'.join(map(str, current_chunk_position))
-    
+
 
 @python_2_unicode_compatible
 class SennaNERTagger(Senna):
@@ -139,7 +141,7 @@ class SennaNERTagger(Senna):
 def setup_module(module):
     from nose import SkipTest
     try:
-        tagger = Senna('/usr/share/senna-v2.0', ['pos', 'chk', 'ner'])
+        tagger = Senna('/usr/share/senna-v3.0', ['pos', 'chk', 'ner'])
     except OSError:
         raise SkipTest("Senna executable not found")
 

{nltk-3.2.1 → nltk-3.3}/nltk/tag/sequential.py

@@ -1,6 +1,6 @@
 # Natural Language Toolkit: Sequential Backoff Taggers
 #
-# Copyright (C) 2001-2016 NLTK Project
+# Copyright (C) 2001-2018 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Steven Bird <stevenbird1@gmail.com> (minor additions)
 #         Tiago Tresoldi <tresoldi@users.sf.net> (original affix tagger)
@@ -18,6 +18,7 @@ consulted instead.  Any SequentialBackoffTagger may serve as a
 backoff tagger for any other SequentialBackoffTagger.
 """
 from __future__ import print_function, unicode_literals
+from abc import abstractmethod
 
 import re
 
@@ -29,8 +30,9 @@ from nltk.tag.api import TaggerI, FeaturesetTaggerI
 
 from nltk import jsontags
 
+
 ######################################################################
-#{ Abstract Base Classes
+# Abstract Base Classes
 ######################################################################
 class SequentialBackoffTagger(TaggerI):
     """
@@ -79,9 +81,11 @@ class SequentialBackoffTagger(TaggerI):
         tag = None
         for tagger in self._taggers:
             tag = tagger.choose_tag(tokens, index, history)
-            if tag is not None:  break
+            if tag is not None:
+                break
         return tag
 
+    @abstractmethod
     def choose_tag(self, tokens, index, history):
         """
         Decide which tag should be used for the specified token, and
@@ -99,7 +103,6 @@ class SequentialBackoffTagger(TaggerI):
         :type history: list(str)
         :param history: A list of the tags for all words before *index*.
         """
-        raise NotImplementedError()
 
 
 @python_2_unicode_compatible
@@ -125,6 +128,7 @@ class ContextTagger(SequentialBackoffTagger):
         SequentialBackoffTagger.__init__(self, backoff)
         self._context_to_tag = (context_to_tag if context_to_tag else {})
 
+    @abstractmethod
     def context(self, tokens, index, history):
         """
         :return: the context that should be used to look up the tag
@@ -132,7 +136,6 @@ class ContextTagger(SequentialBackoffTagger):
             should not be handled by this tagger.
         :rtype: (hashable)
         """
-        raise NotImplementedError()
 
     def choose_tag(self, tokens, index, history):
         context = self.context(tokens, index, history)
@@ -180,11 +183,13 @@ class ContextTagger(SequentialBackoffTagger):
                 # Record the event.
                 token_count += 1
                 context = self.context(tokens, index, tags[:index])
-                if context is None: continue
+                if context is None:
+                    continue
                 fd[context][tag] += 1
                 # If the backoff got it wrong, this context is useful:
                 if (self.backoff is None or
-                    tag != self.backoff.tag_one(tokens, index, tags[:index])):
+                        tag != self.backoff.tag_one(
+                        tokens, index, tags[:index])):
                     useful_contexts.add(context)
 
         # Build the context_to_tag table -- for each context, figure
@@ -200,16 +205,16 @@ class ContextTagger(SequentialBackoffTagger):
         # Display some stats, if requested.
         if verbose:
             size = len(self._context_to_tag)
-            backoff = 100 - (hit_count * 100.0)/ token_count
+            backoff = 100 - (hit_count * 100.0) / token_count
             pruning = 100 - (size * 100.0) / len(fd.conditions())
             print("[Trained Unigram tagger:", end=' ')
             print("size=%d, backoff=%.2f%%, pruning=%.2f%%]" % (
                 size, backoff, pruning))
 
+
 ######################################################################
-#{ Tagger Classes
+# Tagger Classes
 ######################################################################
-
 @python_2_unicode_compatible
 @jsontags.register_tag
 class DefaultTagger(SequentialBackoffTagger):
@@ -295,7 +300,7 @@ class NgramTagger(ContextTagger):
         return cls(_n, model=_context_to_tag, backoff=backoff)
 
     def context(self, tokens, index, history):
-        tag_context = tuple(history[max(0,index-self._n+1):index])
+        tag_context = tuple(history[max(0, index-self._n+1):index])
         return tag_context, tokens[index]
 
 
@@ -651,8 +656,8 @@ class ClassifierBasedTagger(SequentialBackoffTagger, FeaturesetTaggerI):
             untagged_sentence, tags = zip(*sentence)
             for index in range(len(sentence)):
                 featureset = self.feature_detector(untagged_sentence,
-                                                    index, history)
-                classifier_corpus.append( (featureset, tags[index]) )
+                                                   index, history)
+                classifier_corpus.append((featureset, tags[index]))
                 history.append(tags[index])
 
         if verbose:
@@ -683,6 +688,7 @@ class ClassifierBasedTagger(SequentialBackoffTagger, FeaturesetTaggerI):
         """
         return self._classifier
 
+
 class ClassifierBasedPOSTagger(ClassifierBasedTagger):
     """
     A classifier based part of speech tagger.
@@ -732,5 +738,3 @@ class ClassifierBasedPOSTagger(ClassifierBasedTagger):
             'shape': shape,
             }
         return features
-
-