nltkor 1.2.0__cp39-cp39-macosx_10_9_x86_64.whl

nltkor/tag/libs/utils.py

@@ -0,0 +1,344 @@
+# -*- coding: utf-8 -*-
+
+"""
+Utility functions
+"""
+
+import re
+import os, sys
+import logging
+import nltk
+import nltkor
+#from nltkor import Kor_char
+from nltkor.tokenize import Ko_tokenize
+import numpy as np
+
+#from nltk.tokenize.regexp import RegexpTokenizer
+#from nltk.tokenize import TreebankWordTokenizer
+from . import attributes
+
+
+def get_word_from_morph_lexicon(root, word, tags, space_flag):
+	'''
+	space_flag: if True then including space, otherwise do not including space
+	'''
+
+	values = list()
+	value_data = list()
+	if not word: return root.keys()
+
+	current_dict = root
+	_end = '$$'
+	s = 0
+	for i, letter in enumerate(word):
+		#print(i, '>', letter, current_dict)
+		if letter in current_dict:
+			#print(letter, current_dict[letter])
+			current_dict = current_dict[letter]
+			if _end in current_dict :
+				for idx in range(i-s): values.pop()
+				values.append(word[s:i+1])
+				for idx in range(i-s): value_data.pop()
+				value_data.append(current_dict[_end])
+			else: values.append(letter); value_data.append(tags[i])
+		else:
+			#print('==', letter, values)
+			if space_flag or letter != ' ':
+				values.append(letter)				# 최장일치 : -1
+				value_data.append(tags[i])
+			s = i+1
+			current_dict = root
+	else:
+		if values: return values, value_data
+		else:	return list(word), tags
+
+def intersperse(lst, item):
+	result = [item] * (len(lst) * 2 - 1)
+	result[0::2] = lst
+	return result
+
+def get_word(root, word, tags, space_flag=False) :
+	'''
+	space_flag : True : 공백이 있어도 매칭됨
+							 False: 공백이 있으면 매칭 안됨
+	'''
+	word_list = get_word_from_morph_lexicon(root, word, tags, space_flag)
+	return word_list
+
+
+
+def tokenize(text, use_sent_tokenizer=True):
+		"""
+		Call the tokenizer function for the given language.
+		The returned tokens are in a list of lists, one for each sentence.
+
+		:param use_sent_tokenizer: True : use sentence tokenizer
+									False : sentence per line
+		"""
+		return tokenize_ko(text, use_sent_tokenizer)
+
+def tokenize_ko(text, use_sent_tokenizer=True, clean=True):
+		"""
+		text: string
+		Return a list of lists of the tokens in text, separated by sentences.
+		"""
+		if clean:
+			text = clean_kotext(text)
+
+		if use_sent_tokenizer:
+			## False: 띄어쓰기 무시, True: 띄어쓰기 고려
+			sentences = [Ko_tokenize.syllable(sentence, True) for sentence in Ko_tokenize.sentence(text)]
+		else:
+			sentences = [Ko_tokenize.syllable(text, True)]
+
+		return sentences
+
+def clean_kotext(text, correct=False):
+		"""
+		1. 특수 공백문자를 공백으로 처리
+		Apply some transformations to the text, such as
+		replacing digits for 9 and simplifying quotation marks.
+
+		:param correct: If True, tries to correct punctuation misspellings.
+		"""
+		# replaces different kinds of quotation marks with "
+		# take care not to remove apostrophes
+		'''
+		text = re.sub(r"(?u)(^|\W)[‘’′`']", r'\1"', text)
+		text = re.sub(r"(?u)[‘’`′'](\W|$)", r'"\1', text)
+		text = re.sub(r'(?u)[«»“”]', '"', text)
+
+		if correct:
+				# tries to fix mistyped tokens (common in Wikipedia-pt) as ,, '' ..
+				text = re.sub(r'(?<!\.)\.\.(?!\.)', '.', text) # take care with ellipses
+				text = re.sub(r'([,";:])\1,', r'\1', text)
+
+				# inserts space after leading hyphen. It happens sometimes in cases like:
+				# blablabla -that is, bloblobloblo
+				text = re.sub(' -(?=[^\W\d_])', ' - ', text)
+		'''
+
+		# replaces numbers with the 9's
+		text = re.sub(r'\xa0', ' ', text)
+		text = re.sub(u'　', ' ', text)
+		text = re.sub(u' ', ' ', text)
+		text = re.sub(u' +', ' ', text) # 연속된 공백 처리
+		# replaces numbers with the 9's, 사전쪽과 같이 판단할 것
+		#text = re.sub(r'\d', '9', text)
+		# replaces english character with the a's
+		#text = re.sub(r'[a-zA-Z]', 'a', text)
+		# replaces chinses character with the 家's
+		#for x in re.findall(r'[\u4e00-\u9fff]', text):
+		#	text = re.sub(x, '家', text)
+
+		# replaces special ellipsis character
+		#text = text.replace(u'…', '...')
+
+		return text
+
+
+
+def generate_feature_vectors(num_vectors, num_features, min_value=-0.1, max_value=0.1):
+		"""
+		Generates vectors of real numbers, to be used as word features.
+		Vectors are initialized randomly. Returns a 2-dim numpy array.
+		"""
+		logger = logging.getLogger("Logger")
+		#table = (max_value * 2) * np.random.random((num_vectors, num_vectors, num_features, num_features)) + min_value
+		table = (max_value * 2) * np.random.random((num_vectors, num_features)) + min_value
+		logger.debug("Generated %d feature vectors with %d features each." % (num_vectors, num_features))
+		print("Generated %d feature vectors with %d features each." % (num_vectors, num_features))
+
+		return table
+
+
+def count_lines(filename):
+		"""Counts and returns how many non empty lines in a file there are."""
+		with open(filename, 'r') as f:
+				lines = [x for x in list(f) if x.strip()]
+		return len(lines)
+
+def _create_affix_tables(affix, table_list, num_features):
+		"""
+		Internal helper function for loading suffix or prefix feature tables
+		into the given list.
+		affix should be either 'suffix' or 'prefix'.
+		"""
+		logger = logging.getLogger('Logger')
+		logger.info('Generating %s features...' % affix)
+		tensor = []
+		codes = getattr(attributes.Affix, '%s_codes' % affix)
+		num_affixes_per_size = getattr(attributes.Affix, 'num_%ses_per_size' % affix)
+		for size in codes:
+
+				# use num_*_per_size because it accounts for special suffix codes
+				num_affixes = num_affixes_per_size[size]
+				table = generate_feature_vectors(num_affixes, num_features)
+				tensor.append(table)
+
+		# affix attribute actually has a 3-dim tensor
+		# (concatenation of 2d tables, one for each suffix size)
+		for table in tensor:
+				table_list.append(table)
+
+def create_feature_tables(args, md, text_reader):
+		"""
+		Create the feature tables to be used by the network. If the args object
+		contains the load_features option as true, the feature table for word types
+		is loaded instead of being created. The actual number of
+		feature tables will depend on the argument options.
+
+		:param arguments: Parameters supplied to the program
+		:param md: metadata about the network
+		:param text_reader: The TextReader being used.
+		:returns: all the feature tables to be used
+		"""
+
+		logger = logging.getLogger("Logger")
+		feature_tables = []
+
+		if not args.load_types:
+				logger.info("Generating word type features...")
+				table_size = len(text_reader.word_dict)
+				types_table = generate_feature_vectors(table_size, args.num_features)
+		else:
+				logger.info("Loading word type features...")
+				# check if there is a word feature file specific for the task
+				# if not, load a generic one
+				filename = md.paths[md.type_features]
+				if os.path.exists(filename):
+						types_table = load_features_from_file(filename)
+				else:
+						filename = md.paths['type_features']
+						types_table = load_features_from_file(filename)
+
+				if len(types_table) < len(text_reader.word_dict):
+						# the type dictionary provided has more types than
+						# the number of feature vectors. So, let's generate
+						# feature vectors for the new types by replicating the vector
+						# associated with the RARE word
+						diff = len(text_reader.word_dict) - len(types_table)
+						logger.warning("Number of types in feature table and dictionary differ.")
+						logger.warning("Generating features for %d new types." % diff)
+						num_features = len(types_table[0])
+						new_vecs =	generate_feature_vectors(diff, num_features)
+						types_table = np.append(types_table, new_vecs, axis=0)
+
+				elif len(types_table) < len(text_reader.word_dict):
+						logger.warning("Number of features provided is greater than the number of tokens\
+						in the dictionary. The extra features will be ignored.")
+
+		feature_tables.append(types_table)	# head
+		#print(md.task)
+		#if md.task in ['labeled_dependency', 'unlabeled_dependency']:
+		#	feature_tables.append(types_table)	# tail
+
+		# Capitalization
+		if md.use_caps:
+				logger.info("Generating capitalization features...")
+				caps_table = generate_feature_vectors(attributes.Caps.num_values, args.caps)
+				feature_tables.append(caps_table)
+
+		# Prefixes
+		if md.use_prefix:
+				_create_affix_tables('prefix', feature_tables, args.prefix)
+
+		# Suffixes
+		if md.use_suffix:
+				_create_affix_tables('suffix', feature_tables, args.suffix)
+
+		# POS tags
+		if md.use_pos:
+				logger.info("Generating POS features...")
+				num_pos_tags = text_reader.get_num_pos_tags()
+				pos_table = generate_feature_vectors(num_pos_tags, args.pos)
+				#feature_tables.append(pos_table) # head    # 여기와 *_reader의 converter와 일치해야 한다.
+				feature_tables.append(pos_table) # tail
+
+		# chunk tags
+		if md.use_chunk:
+				logger.info("Generating chunk features...")
+				num_chunk_tags = count_lines(md.paths['chunk_tags'])
+				chunk_table = generate_feature_vectors(num_chunk_tags, args.chunk)
+				feature_tables.append(chunk_table)
+
+		#print(len(feature_tables))
+		return feature_tables
+
+
+
+def set_distance_features(max_dist=None,
+													num_target_features=None, num_pred_features=None):
+		"""
+		Returns the distance feature tables to be used by a convolutional network.
+		One table is for relative distance to the target predicate, the other
+		to the predicate.
+
+		:param max_dist: maximum distance to be used in new vectors.
+		"""
+		logger = logging.getLogger("Logger")
+
+		# max_dist before/after, 0 distance, and distances above the max
+		max_dist = 2 * (max_dist + 1) + 1
+		logger.info("Generating target word distance features...")
+		target_dist = generate_feature_vectors(max_dist, num_target_features)
+		logger.info("Generating predicate distance features...")
+		pred_dist = generate_feature_vectors(max_dist, num_pred_features)
+
+		return [target_dist, pred_dist]
+
+
+def set_logger(level):
+		"""Sets the logger to be used throughout the system."""
+		log_format = '%(message)s'
+		logging.basicConfig(format=log_format)
+		logger = logging.getLogger("Logger")
+		logger.setLevel(level)
+
+def load_features_from_file(features_file):
+		"""Reads a file with features written as binary data."""
+		return np.load(features_file)
+
+def save_features_to_file(table, features_file):
+		"""Saves a feature table to a given file, writing binary data."""
+		np.save(features_file, table)
+
+def convert_iobes_to_bracket(tag):
+		"""
+		Convert tags from the IOBES scheme to the CoNLL bracketing.
+
+		Example:
+		B-A0 -> (A0*
+		I-A0 -> *
+		E-A0 -> *)
+		S-A1 -> (A1*)
+		O		-> *
+		"""
+		if tag.startswith('I') or tag.startswith('O'):
+				return '*'
+		if tag.startswith('B'):
+				return '(%s*' % tag[2:]
+		if tag.startswith('E'):
+				return '*)'
+		if tag.startswith('S'):
+				return '(%s*)' % tag[2:]
+		else:
+				raise ValueError("Unknown tag: %s" % tag)
+
+def boundaries_to_arg_limits(boundaries):
+		"""
+		Converts a sequence of IOBES tags delimiting arguments to an array
+		of argument boundaries, used by the network.
+		"""
+		limits = []
+		start = None
+
+		for i, tag in enumerate(boundaries):
+				if tag == 'S':
+						limits.append([i, i])
+				elif tag == 'B':
+						start = i
+				elif tag == 'E':
+						limits.append([start, i])
+
+		return np.array(limits, np.int)

nltkor/tag/libs/word_dictionary.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+import itertools
+from collections import Counter, OrderedDict as OD
+from nltkor import Kor_char
+
+class WordDictionary(dict):
+	"""
+	Class to store words and their corresponding indices in
+	the network lookup table. Also deals with padding and
+	maps rare words to a special index.
+	"""
+
+	padding_left = '*LEFT*'
+	padding_right = '*RIGHT*'
+	rare = '*RARE*'
+
+	number_transformation = {ord(c): '9' for c in '012345678'}
+	#english_transformation = {ord(c): 'a' for c in 'abcdefghijklmnopqrstuvwxyz'}
+	# 한자 처리 필요 korChar.hanja_syllable() 사용 , utils.py 참조
+
+	def __init__(self, tokens, size=None, minimum_occurrences=None, wordlist=None):
+		"""
+		Fills a dictionary (to be used for indexing) with the most
+		common words in the given text.
+
+		:param tokens: Either a list of tokens or a list of lists of tokens
+			(each token represented as a string).
+		:param size: Maximum number of token indices
+			(not including paddings, rare, etc.).
+		:param minimum_occurrences: The minimum number of occurrences a token must
+			have in order to be included.
+		:param wordlist: Use this list of words to build the dictionary. Overrides tokens
+			if not None and ignores maximum size.
+		"""
+		if wordlist is None:
+			# work with the supplied tokens. extract frequencies.
+
+			# gets frequency count
+			c = self._get_frequency_count(tokens)
+
+			if minimum_occurrences is None:
+				minimum_occurrences = 1
+
+			words = [key for key, number in c.most_common()
+					 if number >= minimum_occurrences and key]
+
+			if size is not None and size < len(words):
+				words = words[:size]
+		else:
+			# using ordered dict as an ordered set
+			# (we need to keep the order and eliminate duplicates)
+			words = [word.lower().translate(WordDictionary.number_transformation) for word in wordlist]
+			values = [None] * len(words)
+			words = OD(zip(words, values)).keys()
+			#for c in words:
+			#	print c.encode('utf-8')
+
+		# verifies the maximum size
+		if size is None:
+			size = len(words)
+
+		# set all words in the dictionary
+		for word, num in zip(words, range(size)):
+			self[word] = num
+
+		# if the given words include one of the the rare or padding symbols, don't replace it
+		# 사전 생성 시에 가장 뒤에 padding, rare를 생성
+		special_symbols = [WordDictionary.rare.lower(),
+						   WordDictionary.padding_left.lower(),
+						   WordDictionary.padding_right.lower()]
+
+		for symbol in special_symbols:
+			if symbol not in words:
+				self[symbol] = size
+				size += 1
+
+		self.check()
+
+	@classmethod
+	def init_from_wordlist(cls, wordlist):
+		"""
+		Initializes the WordDictionary instance with a list of words, independently from their
+		frequencies. Every word in the list gets an entry.
+		"""
+		return cls(None, wordlist=wordlist)
+
+	@classmethod
+	def init_empty(cls):
+		"""
+		Initializes an empty Word Dictionary.
+		"""
+		return cls([[]])
+
+	def save(self, filename):
+		"""
+		Saves the word dictionary to the given file as a list of word types.
+
+		Special words (paddings and rare) are also included.
+		"""
+		sorted_words = sorted(self, key=self.get)
+		text = '\n'.join(sorted_words)
+		with open(filename, 'w') as f:
+			f.write(text)
+
+	@classmethod
+	def load(cls, filename):
+		"""
+		Loads a WordDictionary object from a vocabulary file.
+		"""
+		words = []
+		with open(filename, 'r') as f:
+			for word in f:
+				word = word.strip()
+				if word:
+					words.append(word)
+
+		return cls.init_from_wordlist(words)
+
+	def _get_frequency_count(self, token_list):
+		"""
+		Returns a token counter for tokens in token_list.
+
+		:param token_list: Either a list of tokens (as strings) or a list
+			of lists of tokens.
+		"""
+		if type(token_list[0]) == list:
+			c = Counter(t.lower().translate(WordDictionary.number_transformation)
+						for sent in token_list for t in sent)
+		else:
+			c = Counter(t.lower().translate(WordDictionary.number_transformation)
+						for t in token_list)
+		return c
+
+
+	def update_tokens(self, tokens, size=None, minimum_occurrences=1, freqs=None):
+		"""
+		Updates the dictionary, adding more types until size is reached.
+
+		:param freqs: a dictionary providing a token count.
+		"""
+		if freqs is None:
+			freqs = self._get_frequency_count(tokens)
+
+		if size is None or size == 0:
+			# size None or 0 means no size limit
+			size = len(freqs)
+
+		if self.num_tokens >= size:
+			return
+		else:
+			size_diff = size - self.num_tokens
+
+		# a new version of freqs with only tokens not present in the dictionary
+		# and above minimum frequency
+		candidate_tokens = dict((token, freqs[token])
+								for token in freqs
+								if token not in self and freqs[token] >= minimum_occurrences)
+
+		# order the types from the most frequent to the least
+		new_tokens = sorted(candidate_tokens, key=lambda x: candidate_tokens[x], reverse=True)
+
+		next_value = len(self)
+		for token in new_tokens:
+			self[token] = next_value
+			next_value += 1
+			size_diff -= 1
+			if size_diff == 0:
+				break
+
+		self.check()
+
+	def __contains__(self, key):
+		"""
+		Overrides the "in" operator. Case insensitive.
+		"""
+		transformed = key.lower().translate(WordDictionary.number_transformation)
+		return super(WordDictionary, self).__contains__(transformed)
+
+	def __setitem__(self, key, value):
+		"""
+		Overrides the [] write operator. It converts every key to lower case
+		before assignment.
+		"""
+		transformed = key.lower().translate(WordDictionary.number_transformation)
+		super(WordDictionary, self).__setitem__(transformed, value)
+
+	def __getitem__(self, key):
+		"""
+		Overrides the [] read operator.
+
+		Three differences from the original:
+		1) when given a word without an entry, it returns the value for the *UNKNOWN* key.
+		2) all entries are converted to lower case before verification.
+		3) digits are mapped to 9
+		"""
+		# faster than regexp
+		transformed = key.lower().translate(WordDictionary.number_transformation)
+		return super(WordDictionary, self).get(transformed, self.index_rare)
+
+	def get(self, key):
+		"""
+		Overrides the dictionary get method, so when given a word without an entry, it returns
+		the value for the *rare* key. Note that it is not possible to supply a default value as
+		in the dict class.
+		"""
+		# faster than regexp
+		#print (key, type(key))
+		transformed = key.lower().translate(WordDictionary.number_transformation)
+		#print key, transformed
+		return super(WordDictionary, self).get(transformed, self.index_rare)
+
+	def check(self):
+		"""
+		Checks the internal structure of the dictionary and makes necessary adjustments,
+		such as updating num_tokens.
+		"""
+		# since WordDictionary overrides __get__, we use the super call
+		# (the WordDictionary __get__ fails when self.index_rare is not set)
+		key = WordDictionary.rare.lower()
+		self.index_rare = super(WordDictionary, self).get(key)
+
+		self.index_padding_left = self[WordDictionary.padding_left]
+		self.index_padding_right = self[WordDictionary.padding_right]
+		self.num_tokens = len(self)
+
+	def get_words(self, indices):
+		"""
+		Returns the words represented by a sequence of indices.
+		"""
+		words = [w for w in self if self[w] in indices]
+		return words
+
+	def get_indices(self, words):
+		"""
+		Returns the indices corresponding to a sequence of tokens.
+		"""
+		indices = [self[w] for w in words]
+		return indices

nltkor/tag/libs/wsd/__init__.py

@@ -0,0 +1,2 @@
+
+from .wsd_reader import WSDReader

nltkor/tag/libs/wsd/macmorphoreader.py

@@ -0,0 +1,7 @@
+
+import warnings
+
+# backwards compatibility
+from .wsd_reader import *
+
+warnings.warn('Module macmorphoreader is deprecated. Use module pos_reader instead.')

nltkor/tag/libs/wsd/wsd_reader.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+"""
+Class for dealing with WSD data.
+"""
+
+from ..reader import TaggerReader
+
+class ConllWSD(object):
+	"""
+	Dummy class for storing column positions in a conll file.
+	"""
+	id = 0
+	word = 1
+	pos = 2
+	wsd = 3
+	SEP = '\t'
+
+class WSDReader(TaggerReader):
+	"""
+	This class reads data from a POS corpus and turns it into a format
+	readable by the neural network for the POS tagging task.
+	"""
+	
+	def __init__(self, md=None, filename=None, load_dictionaries=True):
+		"""
+		Constructor
+		"""
+		self.rare_tag = None
+		self.sentences = []
+		if filename is not None:
+			try:
+				self._read_plain(filename)
+			except:
+				self._read_conll(filename)
+		
+		super(WSDReader, self).__init__(md, load_dictionaries=load_dictionaries)
+		
+	@property
+	def task(self):
+		"""
+		Abstract Base Class (ABC) attribute.
+		"""
+		return 'wsd'
+	
+	def _read_plain(self, filename):
+		"""
+		Read data from a "plain" file, with one sentence per line, each token
+		as token_tag.
+		"""
+		self.sentences = []
+		with open(filename, 'rt') as f:
+			for line in f:
+				#line = unicode(line, 'utf-8')
+				items = line.strip().split()
+				sentence = []
+				for item in items:
+					token, tag = item.rsplit('_', 1)
+					sentence.append((token, tag))
+					
+				self.sentences.append(sentence)
+	
+	def _read_conll(self, filename):
+		"""
+		Read data from a CoNLL formatted file. It expects at least 4 columns:
+		id, surface word, lemma (ignored, may be anything) 
+		and the POS tag.
+		"""
+		self.sentences = []
+		sentence = []
+		with open(filename, 'rt') as f:
+			for line in f:
+				line = line.strip()
+				if line == '':
+					if len(sentence) > 0:
+						self.sentences.append(sentence)
+						sentence = []
+						continue
+				
+				fields = line.split(ConllWSD.SEP)
+				try:
+					word = fields[ConllWSD.word]
+					pos = fields[ConllWSD.pos]
+					wsd = fields[ConllWSD.wsd]
+				except: continue
+				sentence.append((word, wsd))
+				#sentence.append((word, pos, ner))
+		
+		if len(sentence) > 0:
+			self.sentences.append(sentence)
+
+# backwards compatibility
+MacMorphoReader = WSDReader