nltk 3.2__zip → 3.3__zip

{nltk-3.2 → nltk-3.3}/nltk/tag/hunpos.py

@@ -1,132 +1,132 @@
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: Interface to the HunPos POS-tagger
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Peter Ljunglöf <peter.ljunglof@heatherleaf.se>
-#         Dávid Márk Nemeskey <nemeskeyd@gmail.com> (modifications)
-#         Attila Zséder <zseder@gmail.com> (modifications)
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-A module for interfacing with the HunPos open-source POS-tagger.
-"""
-
-import os
-from subprocess import Popen, PIPE
-
-from nltk.internals import find_binary, find_file
-from nltk.tag.api import TaggerI
-from nltk import compat
-
-_hunpos_url = 'http://code.google.com/p/hunpos/'
-
-_hunpos_charset = 'ISO-8859-1'
-"""The default encoding used by hunpos: ISO-8859-1."""
-
-class HunposTagger(TaggerI):
-    """
-    A class for pos tagging with HunPos. The input is the paths to:
-     - a model trained on training data
-     - (optionally) the path to the hunpos-tag binary
-     - (optionally) the encoding of the training data (default: ISO-8859-1)
-
-    Example:
-
-        >>> from nltk.tag import HunposTagger
-        >>> ht = HunposTagger('en_wsj.model')
-        >>> ht.tag('What is the airspeed of an unladen swallow ?'.split())
-        [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]
-        >>> ht.close()
-
-    This class communicates with the hunpos-tag binary via pipes. When the
-    tagger object is no longer needed, the close() method should be called to
-    free system resources. The class supports the context manager interface; if
-    used in a with statement, the close() method is invoked automatically:
-
-        >>> with HunposTagger('en_wsj.model') as ht:
-        ...     ht.tag('What is the airspeed of an unladen swallow ?'.split())
-        ...
-        [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]
-    """
-
-    def __init__(self, path_to_model, path_to_bin=None,
-                 encoding=_hunpos_charset, verbose=False):
-        """
-        Starts the hunpos-tag executable and establishes a connection with it.
-
-        :param path_to_model: The model file.
-        :param path_to_bin: The hunpos-tag binary.
-        :param encoding: The encoding used by the model. Unicode tokens
-            passed to the tag() and tag_sents() methods are converted to
-            this charset when they are sent to hunpos-tag.
-            The default is ISO-8859-1 (Latin-1).
-
-            This parameter is ignored for str tokens, which are sent as-is.
-            The caller must ensure that tokens are encoded in the right charset.
-        """
-        self._closed = True
-        hunpos_paths = ['.', '/usr/bin', '/usr/local/bin', '/opt/local/bin',
-                        '/Applications/bin', '~/bin', '~/Applications/bin']
-        hunpos_paths = list(map(os.path.expanduser, hunpos_paths))
-
-        self._hunpos_bin = find_binary(
-            'hunpos-tag', path_to_bin,
-            env_vars=('HUNPOS_TAGGER',),
-            searchpath=hunpos_paths,
-            url=_hunpos_url,
-            verbose=verbose
-        )
-
-        self._hunpos_model = find_file(
-            path_to_model, env_vars=('HUNPOS_TAGGER',), verbose=verbose)
-        self._encoding = encoding
-        self._hunpos = Popen([self._hunpos_bin, self._hunpos_model],
-                             shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
-        self._closed = False
-
-    def __del__(self):
-        self.close()
-
-    def close(self):
-        """Closes the pipe to the hunpos executable."""
-        if not self._closed:
-            self._hunpos.communicate()
-            self._closed = True
-
-    def __enter__(self):
-        return self
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.close()
-
-    def tag(self, tokens):
-        """Tags a single sentence: a list of words.
-        The tokens should not contain any newline characters.
-        """
-        for token in tokens:
-            assert "\n" not in token, "Tokens should not contain newlines"
-            if isinstance(token, compat.text_type):
-                token = token.encode(self._encoding)
-            self._hunpos.stdin.write(token + b"\n")
-        # We write a final empty line to tell hunpos that the sentence is finished:
-        self._hunpos.stdin.write(b"\n")
-        self._hunpos.stdin.flush()
-
-        tagged_tokens = []
-        for token in tokens:
-            tagged = self._hunpos.stdout.readline().strip().split(b"\t")
-            tag = (tagged[1] if len(tagged) > 1 else None)
-            tagged_tokens.append((token, tag))
-        # We have to read (and dismiss) the final empty line:
-        self._hunpos.stdout.readline()
-
-        return tagged_tokens
-
-# skip doctests if Hunpos tagger is not installed
-def setup_module(module):
-    from nose import SkipTest
-    try:
-        HunposTagger('en_wsj.model')
-    except LookupError:
-        raise SkipTest("HunposTagger is not available")
-
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the HunPos POS-tagger
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Peter Ljunglöf <peter.ljunglof@heatherleaf.se>
+#         Dávid Márk Nemeskey <nemeskeyd@gmail.com> (modifications)
+#         Attila Zséder <zseder@gmail.com> (modifications)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A module for interfacing with the HunPos open-source POS-tagger.
+"""
+
+import os
+from subprocess import Popen, PIPE
+
+from six import text_type
+
+from nltk.internals import find_binary, find_file
+from nltk.tag.api import TaggerI
+
+_hunpos_url = 'http://code.google.com/p/hunpos/'
+
+_hunpos_charset = 'ISO-8859-1'
+"""The default encoding used by hunpos: ISO-8859-1."""
+
+class HunposTagger(TaggerI):
+    """
+    A class for pos tagging with HunPos. The input is the paths to:
+     - a model trained on training data
+     - (optionally) the path to the hunpos-tag binary
+     - (optionally) the encoding of the training data (default: ISO-8859-1)
+
+    Example:
+
+        >>> from nltk.tag import HunposTagger
+        >>> ht = HunposTagger('en_wsj.model')
+        >>> ht.tag('What is the airspeed of an unladen swallow ?'.split())
+        [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]
+        >>> ht.close()
+
+    This class communicates with the hunpos-tag binary via pipes. When the
+    tagger object is no longer needed, the close() method should be called to
+    free system resources. The class supports the context manager interface; if
+    used in a with statement, the close() method is invoked automatically:
+
+        >>> with HunposTagger('en_wsj.model') as ht:
+        ...     ht.tag('What is the airspeed of an unladen swallow ?'.split())
+        ...
+        [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]
+    """
+
+    def __init__(self, path_to_model, path_to_bin=None,
+                 encoding=_hunpos_charset, verbose=False):
+        """
+        Starts the hunpos-tag executable and establishes a connection with it.
+
+        :param path_to_model: The model file.
+        :param path_to_bin: The hunpos-tag binary.
+        :param encoding: The encoding used by the model. Unicode tokens
+            passed to the tag() and tag_sents() methods are converted to
+            this charset when they are sent to hunpos-tag.
+            The default is ISO-8859-1 (Latin-1).
+
+            This parameter is ignored for str tokens, which are sent as-is.
+            The caller must ensure that tokens are encoded in the right charset.
+        """
+        self._closed = True
+        hunpos_paths = ['.', '/usr/bin', '/usr/local/bin', '/opt/local/bin',
+                        '/Applications/bin', '~/bin', '~/Applications/bin']
+        hunpos_paths = list(map(os.path.expanduser, hunpos_paths))
+
+        self._hunpos_bin = find_binary(
+            'hunpos-tag', path_to_bin,
+            env_vars=('HUNPOS_TAGGER',),
+            searchpath=hunpos_paths,
+            url=_hunpos_url,
+            verbose=verbose
+        )
+
+        self._hunpos_model = find_file(
+            path_to_model, env_vars=('HUNPOS_TAGGER',), verbose=verbose)
+        self._encoding = encoding
+        self._hunpos = Popen([self._hunpos_bin, self._hunpos_model],
+                             shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        self._closed = False
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        """Closes the pipe to the hunpos executable."""
+        if not self._closed:
+            self._hunpos.communicate()
+            self._closed = True
+
+    def __enter__(self):
+        return self
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+    def tag(self, tokens):
+        """Tags a single sentence: a list of words.
+        The tokens should not contain any newline characters.
+        """
+        for token in tokens:
+            assert "\n" not in token, "Tokens should not contain newlines"
+            if isinstance(token, text_type):
+                token = token.encode(self._encoding)
+            self._hunpos.stdin.write(token + b"\n")
+        # We write a final empty line to tell hunpos that the sentence is finished:
+        self._hunpos.stdin.write(b"\n")
+        self._hunpos.stdin.flush()
+
+        tagged_tokens = []
+        for token in tokens:
+            tagged = self._hunpos.stdout.readline().strip().split(b"\t")
+            tag = (tagged[1] if len(tagged) > 1 else None)
+            tagged_tokens.append((token, tag))
+        # We have to read (and dismiss) the final empty line:
+        self._hunpos.stdout.readline()
+
+        return tagged_tokens
+
+# skip doctests if Hunpos tagger is not installed
+def setup_module(module):
+    from nose import SkipTest
+    try:
+        HunposTagger('en_wsj.model')
+    except LookupError:
+        raise SkipTest("HunposTagger is not available")

{nltk-3.2 → nltk-3.3}/nltk/tag/mapping.py

@@ -1,101 +1,101 @@
-# Natural Language Toolkit: Tagset Mapping
-#
-# Copyright (C) 2001-2015 NLTK Project
-# Author: Nathan Schneider <nathan@cmu.edu>
-#         Steven Bird <stevenbird1@gmail.com>
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Interface for converting POS tags from various treebanks
-to the universal tagset of Petrov, Das, & McDonald.
-
-The tagset consists of the following 12 coarse tags:
-
-VERB - verbs (all tenses and modes)
-NOUN - nouns (common and proper)
-PRON - pronouns
-ADJ - adjectives
-ADV - adverbs
-ADP - adpositions (prepositions and postpositions)
-CONJ - conjunctions
-DET - determiners
-NUM - cardinal numbers
-PRT - particles or other function words
-X - other: foreign words, typos, abbreviations
-. - punctuation
-
-@see: http://arxiv.org/abs/1104.2086 and http://code.google.com/p/universal-pos-tags/
-
-"""
-
-from __future__ import print_function, unicode_literals, division
-from collections import defaultdict
-from os.path import join
-
-from nltk.data import load
-
-_UNIVERSAL_DATA = "taggers/universal_tagset"
-_UNIVERSAL_TAGS = ('VERB','NOUN','PRON','ADJ','ADV','ADP','CONJ','DET','NUM','PRT','X','.')
-
-# _MAPPINGS = defaultdict(lambda: defaultdict(dict))
-# the mapping between tagset T1 and T2 returns UNK if appied to an unrecognized tag
-_MAPPINGS = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 'UNK')))
-
-
-def _load_universal_map(fileid):
-    contents = load(join(_UNIVERSAL_DATA, fileid+'.map'), format="text")
-
-    # When mapping to the Universal Tagset,
-    # map unknown inputs to 'X' not 'UNK'
-    _MAPPINGS[fileid]['universal'].default_factory = lambda: 'X'
-
-    for line in contents.splitlines():
-        line = line.strip()
-        if line == '':
-            continue
-        fine, coarse = line.split('\t')
-
-        assert coarse in _UNIVERSAL_TAGS, 'Unexpected coarse tag: {}'.format(coarse)
-        assert fine not in _MAPPINGS[fileid]['universal'], 'Multiple entries for original tag: {}'.format(fine)
-
-        _MAPPINGS[fileid]['universal'][fine] = coarse
-
-
-def tagset_mapping(source, target):
-    """
-    Retrieve the mapping dictionary between tagsets.
-
-    >>> tagset_mapping('ru-rnc', 'universal') == {'!': '.', 'A': 'ADJ', 'C': 'CONJ', 'AD': 'ADV',\
-            'NN': 'NOUN', 'VG': 'VERB', 'COMP': 'CONJ', 'NC': 'NUM', 'VP': 'VERB', 'P': 'ADP',\
-            'IJ': 'X', 'V': 'VERB', 'Z': 'X', 'VI': 'VERB', 'YES_NO_SENT': 'X', 'PTCL': 'PRT'}
-    True
-    """
-
-    if source not in _MAPPINGS or target not in _MAPPINGS[source]:
-        if target == 'universal':
-            _load_universal_map(source)
-    return _MAPPINGS[source][target]
-
-def map_tag(source, target, source_tag):
-    """
-    Maps the tag from the source tagset to the target tagset.
-
-    >>> map_tag('en-ptb', 'universal', 'VBZ')
-    'VERB'
-    >>> map_tag('en-ptb', 'universal', 'VBP')
-    'VERB'
-    >>> map_tag('en-ptb', 'universal', '``')
-    '.'
-    """
-
-    # we need a systematic approach to naming
-    if target == 'universal':
-        if source == 'wsj':
-            source = 'en-ptb'
-        if source == 'brown':
-            source = 'en-brown'
-
-    return tagset_mapping(source, target)[source_tag]
-
-
+# Natural Language Toolkit: Tagset Mapping
+#
+# Copyright (C) 2001-2018 NLTK Project
+# Author: Nathan Schneider <nathan@cmu.edu>
+#         Steven Bird <stevenbird1@gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interface for converting POS tags from various treebanks
+to the universal tagset of Petrov, Das, & McDonald.
+
+The tagset consists of the following 12 coarse tags:
+
+VERB - verbs (all tenses and modes)
+NOUN - nouns (common and proper)
+PRON - pronouns
+ADJ - adjectives
+ADV - adverbs
+ADP - adpositions (prepositions and postpositions)
+CONJ - conjunctions
+DET - determiners
+NUM - cardinal numbers
+PRT - particles or other function words
+X - other: foreign words, typos, abbreviations
+. - punctuation
+
+@see: http://arxiv.org/abs/1104.2086 and http://code.google.com/p/universal-pos-tags/
+
+"""
+
+from __future__ import print_function, unicode_literals, division
+from collections import defaultdict
+from os.path import join
+
+from nltk.data import load
+
+_UNIVERSAL_DATA = "taggers/universal_tagset"
+_UNIVERSAL_TAGS = ('VERB','NOUN','PRON','ADJ','ADV','ADP','CONJ','DET','NUM','PRT','X','.')
+
+# _MAPPINGS = defaultdict(lambda: defaultdict(dict))
+# the mapping between tagset T1 and T2 returns UNK if appied to an unrecognized tag
+_MAPPINGS = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 'UNK')))
+
+
+def _load_universal_map(fileid):
+    contents = load(join(_UNIVERSAL_DATA, fileid+'.map'), format="text")
+
+    # When mapping to the Universal Tagset,
+    # map unknown inputs to 'X' not 'UNK'
+    _MAPPINGS[fileid]['universal'].default_factory = lambda: 'X'
+
+    for line in contents.splitlines():
+        line = line.strip()
+        if line == '':
+            continue
+        fine, coarse = line.split('\t')
+
+        assert coarse in _UNIVERSAL_TAGS, 'Unexpected coarse tag: {}'.format(coarse)
+        assert fine not in _MAPPINGS[fileid]['universal'], 'Multiple entries for original tag: {}'.format(fine)
+
+        _MAPPINGS[fileid]['universal'][fine] = coarse
+
+
+def tagset_mapping(source, target):
+    """
+    Retrieve the mapping dictionary between tagsets.
+
+    >>> tagset_mapping('ru-rnc', 'universal') == {'!': '.', 'A': 'ADJ', 'C': 'CONJ', 'AD': 'ADV',\
+            'NN': 'NOUN', 'VG': 'VERB', 'COMP': 'CONJ', 'NC': 'NUM', 'VP': 'VERB', 'P': 'ADP',\
+            'IJ': 'X', 'V': 'VERB', 'Z': 'X', 'VI': 'VERB', 'YES_NO_SENT': 'X', 'PTCL': 'PRT'}
+    True
+    """
+
+    if source not in _MAPPINGS or target not in _MAPPINGS[source]:
+        if target == 'universal':
+            _load_universal_map(source)
+    return _MAPPINGS[source][target]
+
+def map_tag(source, target, source_tag):
+    """
+    Maps the tag from the source tagset to the target tagset.
+
+    >>> map_tag('en-ptb', 'universal', 'VBZ')
+    'VERB'
+    >>> map_tag('en-ptb', 'universal', 'VBP')
+    'VERB'
+    >>> map_tag('en-ptb', 'universal', '``')
+    '.'
+    """
+
+    # we need a systematic approach to naming
+    if target == 'universal':
+        if source == 'wsj':
+            source = 'en-ptb'
+        if source == 'brown':
+            source = 'en-brown'
+
+    return tagset_mapping(source, target)[source_tag]
+
+