nltkor 1.2.14__cp311-cp311-macosx_13_0_x86_64.whl

nltkor/tag/libs/attributes.py

@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+
+import logging
+import numpy as np
+
+from .word_dictionary import WordDictionary as WD
+from collections import defaultdict
+
+# dummy value to be used when POS is an additional attribute
+PADDING_POS = 'PADDING'
+
+class Caps(object):
+    """Dummy class for storing numeric values for capitalization."""
+    num_values = 5
+    lower = 0
+    title = 1
+    non_alpha = 2
+    other = 3
+    padding = 4
+
+
+class Token(object):
+    def __init__(self, word, morph_h='NA', pos_h='NA',  morph_t='NA', pos_t='NA', chunk='NA'):
+        """
+        A token representation that stores discrete attributes to be given as 
+        input to the neural network. 
+        """
+        self.word = word
+        self.morph_h = morph_h
+        self.pos_h = pos_h
+        self.pos_t = pos_t
+        self.morph_t = morph_t
+        self.chunk = chunk
+    
+    def __str__(self):
+        return str(self.word)
+    
+    def __repr__(self):
+        return self.word.__repr__()
+
+
+class Affix(object):
+    """Dummy class for manipulating suffixes and their related codes."""
+    # codes maps integers (affix sizes) to dicts. each dict maps a suffix of the given 
+    # size to its code
+    suffix_codes = {}
+    prefix_codes = {}
+    other = 0
+    padding = 1
+    num_suffixes_per_size = {}
+    num_prefixes_per_size = {}
+    
+    @classmethod
+    def load_suffixes(cls, md):
+        """
+        Loads suffixes from the suffix file.
+        """
+        cls.load_affixes(cls.suffix_codes, md.paths['suffixes'])
+        
+        # +2 because of the unkown suffix code and padding
+        cls.num_suffixes_per_size = {size: len(cls.suffix_codes[size]) + 2
+                                     for size in cls.suffix_codes}
+    
+    @classmethod
+    def load_prefixes(cls, md):
+        """
+        Loads prefixes from the prefix file.
+        """
+        cls.load_affixes(cls.prefix_codes, md.paths['prefixes'])
+        
+        # +2 because of the unkown prefix code and padding
+        cls.num_prefixes_per_size = {size: len(cls.prefix_codes[size]) + 2
+                                     for size in cls.prefix_codes}
+        
+    
+    @classmethod
+    def load_affixes(cls, codes, filename):
+        """
+        Parent function for loading prefixes and suffixes.
+        """
+        logger = logging.getLogger("Logger")
+        
+        # intermediate storage
+        affixes_by_size = defaultdict(list)
+        
+        try:
+            with open(filename, 'rb') as f:
+                for line in f:
+                    affix = unicode(line.strip(), 'utf-8')
+                    size = len(affix)
+                    affixes_by_size[size].append(affix)
+        except IOError:
+            logger.error("File %s doesn't exist." % filename)
+            raise
+        
+        for size in affixes_by_size:
+            # for each size, each affix has a code starting from 2
+            # 0 is reserved for unknown affixes
+            # 1 is reserved for padding pseudo-affixes
+            codes[size] = {affix: code 
+                           for code, affix in enumerate(affixes_by_size[size], 2)}
+    
+    @classmethod
+    def get_suffix(cls, word, size):
+        """
+        Return the suffix code for the given word. Consider a suffix
+        of the given size.
+        """
+        if word == WD.padding_left or word == WD.padding_right:
+            return cls.padding
+        
+        if len(word) <= size:
+            return cls.other
+        
+        suffix = word[-size:].lower()
+        code = cls.suffix_codes[size].get(suffix, cls.other)
+        return code
+    
+    @classmethod
+    def get_prefix(cls, word, size):
+        """
+        Return the suffix code for the given word. Consider a suffix
+        of the given size.
+        """
+        if word == WD.padding_left or word == WD.padding_right:
+            return cls.padding
+        
+        if len(word) <= size:
+            return cls.other
+        
+        prefix = word[:size].lower()
+        code = cls.prefix_codes[size].get(prefix, cls.other)
+        return code
+
+
+class TokenConverter(object):
+    
+    def __init__(self):
+        """
+        Class to convert tokens into indices to their feature vectos in
+        feature matrices.
+        """
+        self.extractors = []
+    
+    def add_extractor(self, extractor):
+        """
+        Adds an extractor function to the TokenConverter. In order to get a token's 
+        feature indices, the Converter will call each of its extraction functions passing
+        the token as a parameter. The result will be a list containing each result. 
+        """
+        self.extractors.append(extractor)
+    
+    def get_padding_left(self, tokens_as_string=True):
+        """
+        Returns an object to be used as the left padding in the sentence.
+        
+        :param tokens_as_string: if True, treat tokens as strings. 
+        If False, treat them as Token objects.
+        """
+        if tokens_as_string:
+            pad = WD.padding_left
+        else:
+            pad = Token(WD.padding_left, morph_h=WD.padding_left, \
+                morph_t=WD.padding_left, pos_h=PADDING_POS,  pos_t=PADDING_POS)
+        return self.convert(pad)
+    
+    def get_padding_right(self, tokens_as_string=True):
+        """
+        Returns an object to be used as the right padding in the sentence.
+        
+        :param tokens_as_string: if True, treat tokens as strings. 
+            If False, treat them as Token objects.
+        """
+        if tokens_as_string:
+            pad = WD.padding_right
+        else:
+            pad = Token(WD.padding_right, morph_h=WD.padding_right, \
+                morph_t=WD.padding_right, pos_h=PADDING_POS, pos_t=PADDING_POS)
+            #pad = Token(WD.padding_right, pos_t=PADDING_POS)
+        return self.convert(pad)
+    
+    def convert(self, token):
+        """
+        Converts a token into its feature indices.
+        """
+        indices = np.array([function(token) for function in self.extractors])
+        return indices
+
+
+def get_capitalization(word):
+    """
+    Returns a code describing the capitalization of the word:
+    lower, title, other or non-alpha (numbers and other tokens that can't be
+    capitalized).
+    """
+    if word == WD.padding_left or word == WD.padding_right:
+        return Caps.padding
+    
+    if not any(c.isalpha() for c in word):
+        # check if there is at least one letter
+        # (this is faster than using a regex)
+        return Caps.non_alpha
+    
+    if word.islower():
+        return Caps.lower
+    
+    # word.istitle() returns false for compunds like Low-cost
+    if len(word) == 1:
+        # if we reached here, there's a single upper case letter
+        return Caps.title
+    elif word[0].isupper() and word[1:].islower():
+        return Caps.title    
+    
+    return Caps.other
+
+def capitalize(word, capitalization):
+    """
+    Capitalizes the word in the desired format. If the capitalization is 
+    Caps.other, it is set all uppercase.
+    """
+    if capitalization == Caps.non_alpha or capitalization == Caps.padding:
+        return word
+    elif capitalization == Caps.lower:
+        return word.lower()
+    elif capitalization == Caps.title:
+        return word[0].upper() + word[1:].lower()
+    elif capitalization == Caps.other:
+        return word.upper()
+    else:
+        raise ValueError("Unknown capitalization type.")
+

nltkor/tag/libs/config.py

@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*- 
+
+"""
+Configuration data for the system.
+"""
+
+import os
+
+data_dir = None
+FILES = {}
+
+def get_config_paths(directory):
+    """Sets the data directory containing the data for the models."""
+    assert os.path.isdir(directory), 'Invalid data directory'
+
+    return { key: os.path.join(directory, value) for key, value in [ 
+				# cross-task data
+				('.', '.'), #for data_dir access
+				
+				# vocabulary file used as a fallback if a reader doesn't have a specific one
+				('vocabulary'                  , 'vocabulary.txt'),
+				('type_features'               , 'types-features.npy'),
+				('termvectors'                 , 'termvectors.txt'),
+
+				# POS
+				('network_pos'                 , 'pos-network.npz'),
+				('network_text_pos'            , 'pos-network.txt'),
+				('pos_tags'                    , 'pos-tags.txt'),
+				('pos_tag_dict'                , 'pos-tags.txt'),
+				('pos_co_lexicon'              , 'pos-co-lexicon.pickle'),
+				('pos_morph_lexicon'           , 'pos-morph-lexicon.pickle'),
+				('pos_prob_dict'               , 'pos-prob-dict.pickle'),
+				('pos_morph_lexicon_txt'       , 'pos-morph-lexicon.txt'),
+				('suffix'                      , 'suffixes.txt'),
+				('suffixes'                    , 'suffixes.txt'),
+				('prefix'                      , 'prefixes.txt'),
+				('prefixes'                    , 'prefixes.txt'),
+				('metadata_pos'                , 'metadata-pos.pickle'),
+				('metadata_text_pos'           , 'metadata-pos.txt'),
+				('type_features_pos'           , 'types-features-pos.npy'),
+				('caps_features_pos'           , 'caps-features-pos.npy'),
+				('suffix_features_pos'         , 'suffix-features-pos.npy'),
+				('prefix_features_pos'         , 'prefix-features-pos.npy'),
+				('vocabulary_pos'              , 'vocabulary-pos.txt'),
+				
+				# NER
+				('network_ner'                 , 'ner-network.npz'),
+				('network_text_ner'            , 'ner-network.txt'),
+				('ner_tags'                    , 'ner-tags.txt'),
+				('ner_tag_dict'                , 'ner-tags.txt'),
+				('ner_morph_lexicon'           , 'ner-morph-lexicon.txt'),
+				('suffix'                      , 'suffixes.txt'),
+				('suffixes'                    , 'suffixes.txt'),
+				('prefix'                      , 'prefixes.txt'),
+				('prefixes'                    , 'prefixes.txt'),
+				('metadata_ner'                , 'metadata-ner.pickle'),
+				('metadata_text_ner'           , 'metadata-ner.txt'),
+				('type_features_ner'           , 'types-features-ner.npy'),
+				('caps_features_ner'           , 'caps-features-ner.npy'),
+				('suffix_features_ner'         , 'suffix-features-ner.npy'),
+				('prefix_features_ner'         , 'prefix-features-ner.npy'),
+				('vocabulary_ner'              , 'vocabulary-ner.txt'),
+
+				# WSD
+				('network_wsd'                 , 'wsd-network.npz'),
+				('network_text_wsd'            , 'wsd-network.txt'),
+				('wsd_tags'                    , 'wsd-tags.txt'),
+				('wsd_tag_dict'                , 'wsd-tags.txt'),
+				('wsd_morph_lexicon'           , 'wsd-morph-lexicon.txt'),
+#				('suffix'                      , 'suffixes.txt'),
+#				('suffixes'                    , 'suffixes.txt'),
+#				('prefix'                      , 'prefixes.txt'),
+#				('prefixes'                    , 'prefixes.txt'),
+				('metadata_wsd'                , 'metadata-wsd.pickle'),
+				('metadata_text_wsd'           , 'metadata-wsd.txt'),
+				('type_features_wsd'           , 'types-features-wsd.npy'),
+				('caps_features_wsd'           , 'caps-features-wsd.npy'),
+				('suffix_features_wsd'         , 'suffix-features-wsd.npy'),
+				('prefix_features_wsd'         , 'prefix-features-wsd.npy'),
+				('vocabulary_wsd'              , 'vocabulary-wsd.txt'),
+
+				# dependency
+				('network_labeled_dependency', 'ldep-network.npz'),
+				('network_text_labeled_dependency', 'ldep-network.txt'),
+				('type_features_labeled_dependency', 'types-features-ldep.npy'),
+				('caps_features_labeled_dependency', 'caps-features-ldep.npy'),
+				('pos_features_labeled_dependency', 'pos-features-ldep.npy'),
+				('metadata_labeled_dependency', 'metadata-ldep.pickle'),
+				('metadata_text_labeled_dependency', 'metadata-ldep.txt'),
+				('dependency_tag_dict', 'dependency-tags.txt'),
+				('labeled_dependency_tag_dict', 'dependency-tags.txt'),
+				('vocabulary_labeled_dependency', 'vocabulary-ldep.txt'),
+				
+				('dependency_pos_tags', 'dep-pos-tags.txt'),
+				
+				('network_unlabeled_dependency', 'udep-network.npz'),
+				('network_text_unlabeled_dependency', 'udep-network.txt'),
+				('type_features_unlabeled_dependency', 'types-features-udep.npy'),
+				('caps_features_unlabeled_dependency', 'caps-features-udep.npy'),
+				('pos_features_unlabeled_dependency', 'pos-features-udep.npy'),
+				('metadata_unlabeled_dependency', 'metadata-udep.pickle'),
+				('metadata_text_unlabeled_dependency', 'metadata-udep.txt'),
+				('vocabulary_unlabeled_dependency', 'vocabulary-udep.txt'),
+				
+				# chunk
+				#('chunk_tag_dict'              , 'chunk-tag-dict.pickle'),
+				#('chunk_tags'                  , 'chunk-tags.txt'),
+				
+				# SRL
+				('network_srl'                  , 'srl-network.npz'),
+				('network_text_srl'             , 'srl-network.txt'),
+				#('network_srl_boundary'        , 'srl-id-network.npz'),
+				#('network_srl_classify'        , 'srl-class-network.npz'),
+				#('network_srl_predicates'      , 'srl-class-predicates.npz'),
+				#('srl_iob_tag_dict'            , 'srl-tags.txt'),
+				#('srl_iob_tags'                , 'srl-tags.txt'),
+				('srl_tags'                    , 'srl-tags.txt'),
+				#('srl_classify_tag_dict'       , 'srl-tags.txt'),
+				#('srl_classify_tags'           , 'srl-tags.txt'),
+				#('srl_predicates_tag_dict'     , 'srl-predicates-tags.txt'),
+				#('srl_predicates_tags'         , 'srl-predicates-tags.txt'),
+				('type_features_srl'            , 'types-features-srl.npy'),
+				('caps_features_srl'            , 'caps-features-srl.npy'),
+				('pos_features_srl'             , 'pos-features-srl.npy'),
+				#('chunk_features_classify'     , 'chunk-features-class.npy'),
+				#('type_features_boundary'      , 'types-features-id.npy'),
+				#('caps_features_boundary'      , 'caps-features-id.npy'),
+				#('pos_features_boundary'       , 'pos-features-id.npy'),
+				#('chunk_features_boundary'     , 'chunk-features-id.npy'),
+				#('type_features_classify'      , 'types-features-class.npy'),
+				#('caps_features_classify'      , 'caps-features-class.npy'),
+				#('pos_features_classify'       , 'pos-features-class.npy'),
+				#('chunk_features_classify'     , 'chunk-features-class.npy'),
+				#('type_features_1step'         , 'types-features-1step.npy'),
+				#('caps_features_1step'         , 'caps-features-1step.npy'),
+				#('pos_features_1step'          , 'pos-features-1step.npy'),
+				#('chunk_features_1step'        , 'chunk-features-1step.npy'),
+				#('type_features_srl_predicates', 'types-features-preds.npy'),
+				#('caps_features_srl_predicates', 'caps-features-preds.npy'),
+				#('pos_features_srl_predicates' , 'pos-features-preds.npy'),
+				('metadata_srl'                , 'metadata-srl.pickle'),
+				('metadata_text_srl'            , 'metadata-srl.txt'),
+				#('metadata_srl_boundary'       , 'metadata-srl-boundary.pickle'),
+				#('metadata_srl_classify'       , 'metadata-srl-classify.pickle'),
+				#('metadata_srl_predicates'     , 'metadata-srl-predicates.pickle'),
+				('vocabulary_srl', 'vocabulary-srl.txt'),
+				#('vocabulary_srl_boundary', 'vocabulary-srl-boundary.txt'),
+				#('vocabulary_srl_classify', 'vocabulary-srl-classify.txt'),
+				#('vocabulary_srl_predicates', 'vocabulary-srl-predicates.txt')
+				]
+    }
+
+
+def set_data_dir(directory):
+    """Sets the global data directory containing the data for the models."""
+    global data_dir, FILES
+    data_dir = directory
+    FILES = get_config_paths(directory)
+

nltkor/tag/libs/metadata.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+
+"""
+This script contains the definition of the Metadata class.
+It can also be invoked in order to create a Metada object
+and save it to a file in the data directory.
+"""
+
+import _pickle
+
+from . import config
+
+class Metadata(object):
+    """
+    Class for storing metadata about a neural network and its 
+    parameter files.
+    """
+    
+    def __init__(self, task, paths=None, use_caps=True, use_suffix=False, use_prefix=False,
+                 use_pos=False, use_chunk=False, use_lemma=False):
+        self.task = task
+        self.paths = paths if paths else config.FILES
+        self.use_caps = use_caps
+        self.use_suffix = use_suffix
+        self.use_prefix = use_prefix
+        self.use_pos = use_pos
+        self.use_chunk = use_chunk
+        self.use_lemma = use_lemma
+        self.metadata = 'metadata_%s' % task
+        self.network = 'network_%s' % task
+        self.network_text = 'network_text_%s' % task
+        self.tag_dict = '%s_tag_dict' % task
+        
+        # dependency edge filter doesn't use an actual neural network, so 
+        # we call it "model" to be more consistent
+        self.model = self.network
+        '''
+        if task == 'srl_boundary':
+            self.pred_dist_table = 'pred_dist_table_boundary'
+            self.target_dist_table = 'target_dist_table_boundary'
+            self.transitions = 'srl_transitions_boundary'
+            self.type_features = 'type_features_boundary'
+            self.caps_features = 'caps_features_boundary'
+            self.pos_features = 'pos_features_boundary'
+            self.chunk_features = 'chunk_features_boundary'
+            self.suffix_features = None
+            
+        elif task == 'srl_classify':
+            self.pred_dist_table = 'pred_dist_table_classify'
+            self.target_dist_table = 'target_dist_table_classify'
+            self.transitions = None
+            self.type_features = 'type_features_classify'
+            self.caps_features = 'caps_features_classify'
+            self.pos_features = 'pos_features_classify'
+            self.chunk_features = 'chunk_features_classify'
+            self.suffix_features = None
+        
+        elif task == 'srl':
+            # one step srl
+            self.pred_dist_table = 'pred_dist_table_1step'
+            self.target_dist_table = 'target_dist_table_1step'
+            self.transitions = 'srl_transitions_1step'
+            self.type_features = 'type_features_1step'
+            self.caps_features = 'caps_features_1step'
+            self.pos_features = 'pos_features_1step'
+            self.chunk_features = 'chunk_features_1step'
+            self.suffix_features = None
+
+        else:
+            self.type_features = 'type_features_%s' % task
+            self.caps_features = 'caps_features_%s' % task
+            self.pos_features = 'pos_features_%s' % task
+            self.chunk_features = 'chunk_features_%s' % task
+            self.suffix_features = 'suffix_features_%s' % task
+            self.prefix_features = 'prefix_features_%s' % task
+        '''
+        self.type_features = 'type_features_%s' % task
+        self.caps_features = 'caps_features_%s' % task
+        self.pos_features = 'pos_features_%s' % task
+        #self.chunk_features = 'chunk_features_%s' % task
+        self.suffix_features = 'suffix_features_%s' % task
+        self.prefix_features = 'prefix_features_%s' % task
+
+    def __str__(self):
+        """Shows the task at hand and which attributes are used."""
+        lines = []
+        lines.append("Metadata for task %s" % self.task)
+        for k in self.__dict__:
+            if isinstance(k, str) and k.startswith('use_'):
+                lines.append('%s: %s' % (k, self.__dict__[k]))
+        
+        return '\n'.join(lines)
+    
+    def save_to_file(self): 
+        """
+        Save the contents of the metadata to a file. The filename is determined according
+        to the task.
+        """
+        save_data = self.__dict__.copy()
+        filename = self.paths['metadata_%s' % self.task]
+        del(save_data['paths'])
+        
+        with open(filename, 'wb') as f:
+            _pickle.dump(save_data, f, 2)
+
+        filename = self.paths['metadata_text_%s' % self.task]
+        
+        with open(filename, 'wt') as f:
+            for k,v in save_data.items():
+                f.write("%s: %s\n" %(k,v))
+    
+    @classmethod
+    def load_from_file(cls, task, paths=None):
+        """
+        Reads the file containing the metadata for the given task and returns a 
+        Metadata object.
+        """
+        if paths is None:
+            paths = config.FILES
+        md = Metadata(None, paths)
+        
+        # the actual content of the file is the __dict__ member variable, which contain all
+        # the instance's data
+        with open(paths['metadata_%s' % task], 'rb') as f:
+            data = _pickle.load(f)
+        md.__dict__.update(data)
+        
+        return md
+

nltkor/tag/libs/ner/__init__.py

@@ -0,0 +1,2 @@
+
+from .ner_reader import NERReader

nltkor/tag/libs/ner/macmorphoreader.py

@@ -0,0 +1,7 @@
+
+import warnings
+
+# backwards compatibility
+from .ner_reader import *
+
+warnings.warn('Module macmorphoreader is deprecated. Use module pos_reader instead.')

nltkor/tag/libs/ner/ner_reader.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+
+"""
+Class for dealing with POS data.
+"""
+
+from ..reader import TaggerReader
+
+class ConllNER(object):
+	"""
+	Dummy class for storing column positions in a conll file.
+	"""
+	id = 0
+	word = 1
+	pos = 2
+	ner = 3
+	link = 4
+	SEP = '\t'
+
+class NERReader(TaggerReader):
+	"""
+	This class reads data from a POS corpus and turns it into a format
+	readable by the neural network for the POS tagging task.
+	"""
+	
+	def __init__(self, md=None, filename=None, load_dictionaries=True):
+		"""
+		Constructor
+		"""
+		self.rare_tag = None
+		self.sentences = []
+		if filename is not None:
+			try:
+				self._read_plain(filename)
+			except:
+				self._read_conll(filename)
+		
+		super(NERReader, self).__init__(md, load_dictionaries=load_dictionaries)
+		
+	@property
+	def task(self):
+		"""
+		Abstract Base Class (ABC) attribute.
+		"""
+		return 'ner'
+	
+	def _read_plain(self, filename):
+		"""
+		Read data from a "plain" file, with one sentence per line, each token
+		as token_tag.
+		"""
+		self.sentences = []
+		with open(filename, 'rt') as f:
+			for line in f:
+				#line = unicode(line, 'utf-8')
+				items = line.strip().split()
+				sentence = []
+				for item in items:
+					token, tag = item.rsplit('_', 1)
+					sentence.append((token, tag))
+					
+				self.sentences.append(sentence)
+	
+	def _read_conll(self, filename):
+		"""
+		Read data from a CoNLL formatted file. It expects at least 4 columns:
+		id, surface word, lemma (ignored, may be anything) 
+		and the POS tag.
+		"""
+		self.sentences = []
+		sentence = []
+		with open(filename, 'rt') as f:
+			for line in f:
+				line = line.strip()
+				if line == '':
+					if len(sentence) > 0:
+						self.sentences.append(sentence)
+					sentence = [] # 문장 끝나고 빈 라인이 여러개 있는 것에 대비 
+					continue
+				
+				fields = line.split(ConllNER.SEP)
+				word = fields[ConllNER.word]
+				pos = fields[ConllNER.pos]
+				ner = fields[ConllNER.ner]
+				link = fields[ConllNER.link]
+				sentence.append((word, ner))
+		
+		if len(sentence) > 0:
+			self.sentences.append(sentence)
+
+# backwards compatibility
+MacMorphoReader = NERReader