nltkor 1.2.14__cp311-cp311-macosx_13_0_x86_64.whl

nltkor/tag/espresso_tag.py

@@ -0,0 +1,220 @@
+import os
+
+# import sys
+# sys.path.append("/Users/dowon/nltk_ko/nltk/tag")
+# from libs import *
+
+"""
+This script will run a POS or SRL tagger on the input data and print the results
+to stdout.
+"""
+
+import argparse
+import logging
+
+if __package__ is None: # 스크립트로 실행할 때
+		import sys
+		from os import path
+		#print(path.dirname( path.dirname( path.abspath(__file__) ) ))
+		sys.path.append(path.dirname( path.dirname( path.abspath(__file__) ) ))
+		from libs import *
+else:
+	from .libs import *
+#from .libs import *
+import requests
+import zipfile
+
+logging.disable(logging.INFO) # disable INFO and DEBUG logging everywhere
+
+
+class EspressoTagger:
+		def __init__(self, data_dir=None, task="pos"):
+				self.data_dir = data_dir
+				if data_dir == None:
+						path=os.path.dirname(__file__)
+						path= path + '/data'
+						self.data_dir = path
+						#print(path)
+				self.path = ""
+				self.tagger = None
+
+				self.task = task.lower()
+				if not self._check_model():
+						self._download_model()
+
+				set_data_dir(self.data_dir)
+
+				if self.task == 'pos':
+								self.tagger = taggers.POSTagger(data_dir=self.data_dir)
+				elif self.task == 'ner':
+								self.tagger = taggers.NERTagger(data_dir=self.data_dir)
+				elif self.task == 'wsd':
+								self.tagger = taggers.WSDTagger(data_dir=self.data_dir)
+				elif self.task == 'srl':
+								self.tagger = taggers.SRLTagger(data_dir=self.data_dir)
+				elif self.task == 'dependency':
+								self.tagger = taggers.DependencyParser(data_dir=self.data_dir)
+				else:
+								raise ValueError('Unknown task: %s' % self.task)
+
+
+		def tag(self, text, use_sent_tokenizer=True, lemma=True):
+				"""
+				This function provides an interactive environment for running the system.
+				It receives text from the standard input, tokenizes it, and calls the function
+				given as a parameter to produce an answer.
+
+				:param task: 'pos', ner', 'wsd', 'srl' or 'dependency'
+				:param use_tokenizer: whether to use built-in tokenizer
+				"""
+
+				#use_sent_tokenizer = not use_sent_tokenizer
+				mode = 'standard' if lemma else 'eumjeol'
+				result = self.tagger.tag(text, use_sent_tokenizer, mode)
+				'''
+				else:
+								tokens = text.split()
+								if self.task != 'dependency':
+												result = [self.tagger.tag_tokens(tokens, True)]
+								else:
+												result = [self.tagger.tag_tokens(tokens)]
+				'''
+
+				return self._result_tagged(result, self.task)
+
+		def _result_tagged(self, tagged_sents, task):
+				"""
+				Prints the tagged text to stdout.
+
+				:param tagged_sents: sentences tagged according to any of espresso taggers.
+				:param task: the tagging task (either 'pos', 'ner', 'wsd', 'srl' or 'dependency')
+				"""
+
+				##TODO: print부분 return으로 변경
+				if task == 'pos':
+								return self._return_tagged_pos(tagged_sents)
+				elif task == 'ner':
+								return self._return_tagged_ner(tagged_sents)
+				elif task == 'wsd':
+								return self._return_tagged_wsd(tagged_sents)
+				elif task == 'srl':
+								return self._return_tagged_srl(tagged_sents)
+				elif task == 'dependency':
+								return self._return_parsed_dependency(tagged_sents)
+				else:
+								raise ValueError('Unknown task: %s' % task)
+
+
+
+		def _return_parsed_dependency(self, parsed_sents):
+				"""Prints one token per line and its head"""
+				result = []
+				temp_list = []
+				temp_list2 = []
+				for sent in parsed_sents:
+						temp_list = sent.to_conll().split('\t')
+						temp_list = temp_list[1:]
+						for ele in temp_list:
+								if '\n' in ele:
+										ele = ele[:ele.find('\n')]
+								temp_list2.append(ele)
+						result.append(self._dependency_after(temp_list2)[:])
+						temp_list2 = []
+
+				return result
+
+		def _return_tagged_pos(self, tagged_sents):
+			"""Prints one sentence per line as token_tag"""
+			result = []
+			for sent in tagged_sents:
+				result = result + list(sent)
+			return result
+
+		def _return_tagged_srl(self, tagged_sents):
+			result = []
+			for sent in tagged_sents:
+				# print (' '.join(sent.tokens))
+				temp_dict1 = {}
+				for predicate, arg_structure in sent.arg_structures:
+					# print ("test 1 :", predicate)
+					# print("te22 :", arg_structure)
+
+					temp_dict2 = {}
+					for label in arg_structure:
+						argument = ' '.join(arg_structure[label])
+						# line = '\t%s: %s' % (label, argument)
+						# print (line)
+						temp_dict2[label] = argument
+
+						# result.append((label, argument))
+						# print ('\n')
+					temp_dict1[predicate] = temp_dict2
+
+				result.append(temp_dict1)
+
+			return result
+
+		def _return_tagged_ner(self, tagged_sents):
+				"""Prints one sentence per line as token_tag"""
+				result = []
+				for sent in tagged_sents:
+						for item in sent:
+								#s = '_'.join(item)
+								result.append(item)
+
+				return result
+
+		def _return_tagged_wsd(self, tagged_sents):
+				"""Prints one sentence per line as token_tag"""
+				result = []
+				for sent in tagged_sents:
+						for item in sent:
+								s = '_'.join(item)
+								result.append(s)
+
+				return result
+
+		def _download_model(self):
+				"""Downloads the model from the server"""
+				temp_path = os.path.dirname(__file__) + '/data.zip'
+				url = "https://air.changwon.ac.kr/~airdemo/storage/espresso_data_1/data.zip"
+				print("Downloading Espresso5 model...")
+				with requests.get(url, stream=True) as r:
+						r.raise_for_status()
+						with open(temp_path, 'wb') as f:
+								for chunk in r.iter_content(chunk_size=8192):
+								# If you have chunk encoded response uncomment if
+								# and set chunk_size parameter to None.
+								#if chunk:
+										f.write(chunk)
+
+				if os.path.exists(self.data_dir):
+						os.rmdir(self.data_dir)
+
+				with zipfile.ZipFile(temp_path, "r") as zip_ref:
+						zip_ref.extractall(os.path.dirname(__file__))
+
+		def _check_model(self):
+				"""Checks if the model is available and downloads it if necessary"""
+				if not os.path.exists(self.data_dir):
+						return False
+				else:
+						return True
+
+		def _dependency_after(self, list):
+				len_list = len(list)
+				temp_list = []
+				repeat = len_list//3
+				for i in range(repeat):
+						index = i*3
+						tup1 = (i+1, )
+						tup2 = tuple(list[index:index+3])
+						tup = tup1 + tup2
+						temp_list.append(tup[:])
+
+
+				return temp_list
+
+if __name__ == '__main__':
+		tagger = EspressoTagger(task='pos')
+		print(tagger.tag("나는 아름다운 강산에 살고있다."))

nltkor/tag/libs/__init__.py

@@ -0,0 +1,10 @@
+# import to provide easier access for nlpnet users
+from .config import set_data_dir
+from . import taggers
+from . import utils
+
+from .taggers import POSTagger, NERTagger, WSDTagger, SRLTagger, DependencyParser
+from .utils import tokenize
+from .utils import PickleConverter
+
+__version__ = '1.2.0'

nltkor/tag/libs/arguments.py

@@ -0,0 +1,280 @@
+# -*- coding: utf-8 -*-
+
+"""
+Code for argument parsing and a few verifications. 
+These arguments are used by the training script.
+"""
+
+import argparse
+
+
+def fill_defaults(args, defaults_per_task):
+		"""
+		This function fills arguments not explicitly set (left as None)
+		with default values according to the chosen task.
+		
+		We can't rely on argparse to it because using subparsers with
+		set_defaults and a parent parser overwrites the defaults. 
+		"""
+		task = args.task
+		defaults = defaults_per_task[task]
+		for arg in args.__dict__:
+				if getattr(args, arg) is None and arg in defaults:
+						setattr(args, arg, defaults[arg])
+		
+
+def get_args():
+		parser = argparse.ArgumentParser(description="Train the Espresso "\
+																		 "for a given NLP task.")
+		subparsers = parser.add_subparsers(title='Tasks',
+																			 dest='task',
+																			 description='Task to train the Espresso for. '\
+																			 'Type %(prog)s [TASK] -h to get task-specific help.')
+		
+		defaults = {}
+		
+		# base parser with arguments not related to any model
+		base_parser = argparse.ArgumentParser(add_help=False)
+		base_parser.add_argument('-f', '--num_features', type=int,
+														 help='Number of features per word '\
+														 '(used to generate random vectors)',
+														 default=50, dest='num_features')
+		base_parser.add_argument('--load_network', action='store_true',
+														 help='Load previously saved network')
+		base_parser.add_argument('--load_features', action='store_true',
+														 help="Load previously saved word type features "\
+														 "(overrides -f and must also load a vocabulary file)", 
+														 dest='load_types')
+		base_parser.add_argument('-v', '--verbose', help='Verbose mode',
+														 action="store_true")
+		base_parser.add_argument('--gold', help='File with annotated data for training.', 
+														 type=str, required=True)
+		base_parser.add_argument('--data', help='Directory to save new models and load '\
+														 'partially trained ones (default: current directory)', type=str, 
+														 default='./data/')
+		base_parser.add_argument('--dev', help='Development (validation) data. If not given, '\
+														 'training data will be used to evaluate performance.',
+														 default=None)
+		
+		# parser with network arguments shared among most tasks
+		# each task-specific parser may define defaults
+		network_parser = argparse.ArgumentParser(add_help=False, parents=[base_parser])
+		
+		network_parser.add_argument('-w', '--window', type=int,
+																help='Size of the word window',
+																dest='window')	# 반드시 홀수로 할 것 	
+		network_parser.add_argument('-e', '--epochs', type=int, dest='iterations',
+																help='Number of training epochs')
+		network_parser.add_argument('-l', '--learning_rate', type=float,
+																help='Learning rate for network connections',
+																dest='learning_rate')
+		network_parser.add_argument('--lf', type=float,
+																help='Learning rate for features',
+																dest='learning_rate_features')
+		network_parser.add_argument('--lt', type=float,
+																help='Learning rate for tag transitions',
+																dest='learning_rate_transitions')
+		network_parser.add_argument('--decay', type=float, const=1, nargs='?', default=None,
+																help='Use learning rate decay. Optionally, '\
+																'supply decay factor (default 1)')
+		network_parser.add_argument('-a', '--accuracy', type=float,
+																help='Maximum desired accuracy per token.',
+																default=0, dest='accuracy')
+		network_parser.add_argument('-n', '--hidden', type=int,
+																help='Number of hidden neurons',
+																dest='hidden')
+		network_parser.add_argument('--caps', const=5, nargs='?', type=int, default=None,
+																help='Include capitalization features. '\
+																'Optionally, supply the number of features (default 5)')
+		
+				
+		# parser with arguments shared among convolutional-based tasks
+		conv_parser = argparse.ArgumentParser(add_help=False)
+		conv_parser.add_argument('-c', '--convolution', type=int, 
+														 help='Number of convolution neurons',
+														 dest='convolution')
+		conv_parser.add_argument('--pos', const=5, nargs='?', type=int, default=None,
+														 help='Include part-of-speech features. '\
+														 'Optionally, supply the number of features (default 5)')
+		conv_parser.add_argument('--max_dist', type=int, default=10,
+														 help='Maximum distance to have its own feature vector')
+		conv_parser.add_argument('--target_features', type=int, default=5,
+														 help='Number of features for distance to target word')
+		conv_parser.add_argument('--pred_features', type=int, default=5,
+														 help='Number of features for distance to predicate')
+				
+		
+		# POS argument parser
+		parser_pos = subparsers.add_parser('pos', help='POS tagging', 
+																			 parents=[network_parser])
+		parser_pos.add_argument('--suffix', const=2, nargs='?', type=int, default=None,
+														help='Include suffix features. Optionally, '\
+														'supply the number of features (default 2)')
+		parser_pos.add_argument('--suffix_size', type=int, default=5,
+														help='Use suffixes up to this size (in characters, default 5). '\
+														'Only used if --suffix is supplied')
+		parser_pos.add_argument('--prefix', const=2, nargs='?', type=int, default=None,
+														help='Include prefix features. Optionally, '\
+														'supply the number of features (default 2)')
+		parser_pos.add_argument('--prefix_size', type=int, default=5,
+														help='Use prefixes up to this size (in characters, default 5). '\
+														'Only used if --suffix is supplied')
+		defaults['pos'] = dict(window=9, hidden=300, iterations=20, 
+													 learning_rate=0.001, learning_rate_features=0.001,
+													 learning_rate_transitions=0.001 )
+
+		# NER argument parser
+		parser_ner = subparsers.add_parser('ner', help='Named Entity Tagging', 
+																			 parents=[network_parser])
+		parser_ner.add_argument('--suffix', const=2, nargs='?', type=int, default=None,
+														help='Include suffix features. Optionally, '\
+														'supply the number of features (default 2)')
+		parser_ner.add_argument('--suffix_size', type=int, default=5,
+														help='Use suffixes up to this size (in characters, default 5). '\
+														'Only used if --suffix is supplied')
+		parser_ner.add_argument('--prefix', const=2, nargs='?', type=int, default=None,
+														help='Include prefix features. Optionally, '\
+														'supply the number of features (default 2)')
+		parser_ner.add_argument('--prefix_size', type=int, default=5,
+														help='Use prefixes up to this size (in characters, default 5). '\
+														'Only used if --suffix is supplied')
+		defaults['ner'] = dict(window=5, hidden=300, iterations=20, 
+													 learning_rate=0.001, learning_rate_features=0.001,
+													 learning_rate_transitions=0.001)
+
+		# WSD argument parser
+		parser_wsd = subparsers.add_parser('wsd', help='Word Sense Disambiguation', 
+																			 parents=[network_parser])
+		parser_wsd.add_argument('--suffix', const=2, nargs='?', type=int, default=None,
+														help='Include suffix features. Optionally, '\
+														'supply the number of features (default 2)')
+		parser_wsd.add_argument('--suffix_size', type=int, default=5,
+														help='Use suffixes up to this size (in characters, default 5). '\
+														'Only used if --suffix is supplied')
+		parser_wsd.add_argument('--prefix', const=2, nargs='?', type=int, default=None,
+														help='Include prefix features. Optionally, '\
+														'supply the number of features (default 2)')
+		parser_wsd.add_argument('--prefix_size', type=int, default=5,
+														help='Use prefixes up to this size (in characters, default 5). '\
+														'Only used if --suffix is supplied')
+		defaults['wsd'] = dict(window=5, hidden=300, iterations=20, 
+													 learning_rate=0.001, learning_rate_features=0.001,
+													 learning_rate_transitions=0.001)
+
+	# Chunk argument parser
+		#parser_ner = subparsers.add_parser('ner', help='Named Entity Tagging', 
+		#																	 parents=[network_parser])
+		#parser_ner.add_argument('--suffix', const=2, nargs='?', type=int, default=None,
+		#												help='Include suffix features. Optionally, '\
+		#												'supply the number of features (default 2)')
+		#parser_ner.add_argument('--suffix_size', type=int, default=5,
+		#												help='Use suffixes up to this size (in characters, default 5). '\
+		#												'Only used if --suffix is supplied')
+		#parser_ner.add_argument('--prefix', const=2, nargs='?', type=int, default=None,
+		#												help='Include prefix features. Optionally, '\
+		#												'supply the number of features (default 2)')
+		#parser_ner.add_argument('--prefix_size', type=int, default=5,
+		#												help='Use prefixes up to this size (in characters, default 5). '\
+		#												'Only used if --suffix is supplied')
+		#defaults['ner'] = dict(window=7, hidden=300, iterations=20, 
+		#											 learning_rate=0.001, learning_rate_features=0.001,
+		#											 learning_rate_transitions=0.001)
+
+		
+		# dependency
+		parser_dep = subparsers.add_parser('dependency', help='Dependency parsing')
+		dep_subparsers = parser_dep.add_subparsers(title='Dependency parsing training steps',
+																							 dest='subtask',
+																							 description='Which step of the dependency training '\
+																							 '(detecting edges or labeling them)')
+		
+		dep_subparsers.add_parser('labeled', help='Labeling dependency edges',
+															parents=[network_parser, conv_parser])
+		dep_subparsers.add_parser('unlabeled', help='Dependency edge detection',
+															parents=[network_parser, conv_parser])
+		
+		defaults['dependency_filter'] = dict()
+		defaults['labeled_dependency'] = dict(window=9, hidden=200, iterations=20, 
+													 learning_rate=0.001, learning_rate_features=0.001,
+													 convolution=300, pos=50,
+													 learning_rate_transitions=0.001)
+		defaults['unlabeled_dependency'] = dict(window=9, hidden=200, iterations=20, 
+													 learning_rate=0.001, learning_rate_features=0.001,
+													 convolution=300, pos=50,
+													 learning_rate_transitions=0.001)
+		
+
+		# SRL argument parser
+		# There is another level of subparsers for predicate detection / 
+		# argument boundary identification / argument classification / 
+		# (id + class) in one step
+		parser_srl = subparsers.add_parser('srl', help='Semantic Role Labeling',
+																			parents=[network_parser, conv_parser],
+																			 formatter_class=argparse.RawDescriptionHelpFormatter)
+		#parser_srl.set_defaults(identify=False, predicates=False, classify=False)
+
+		desc = '''SRL has 3 steps: predicate	detection, argument identification and 
+argument classification. Each one depends on the one before.
+
+You need one model trained for each subtask (or one for predicate
+detection and another with the other 2 steps) in order to perform
+full SRL.
+
+Type %(prog)s [SUBTASK] -h to get subtask-specific help.'''
+		
+		'''
+		srl_subparsers = parser_srl.add_subparsers(title='SRL subtasks',
+																							 dest='subtask',
+																							 description=desc)
+		srl_subparsers.add_parser('pred', help='Predicate identification',
+															parents=[network_parser])
+		defaults['srl_predicates'] = dict(window=5, hidden=50, iterations=1, 
+																			learning_rate=0.01, learning_rate_features=0.01,
+																			learning_rate_transitions=0.01,
+																			predicates=True)
+		
+		srl_subparsers.add_parser('id', help='Argument identification',
+															parents=[network_parser, conv_parser])
+		defaults['srl_boundary'] = dict(window=3, hidden=150, convolution=150, 
+																		identify=True, iterations=15,
+																		learning_rate=0.001, learning_rate_features=0.001,
+																		learning_rate_transitions=0.001)
+		
+		srl_subparsers.add_parser('class', help='Argument classification',
+															parents=[network_parser, conv_parser])
+		defaults['srl_classify'] = dict(window=3, hidden=0, convolution=100, 
+																		classify=True, iterations=3,
+																		learning_rate=0.01, learning_rate_features=0.01,
+																		learning_rate_transitions=0.01)
+		srl_subparsers.add_parser('1step', parents=[network_parser, conv_parser],
+															help='Argument identification and '\
+															'classification together')
+		'''
+		defaults['srl'] = dict(window=3, hidden=150, convolution=200, iterations=8,
+														pos = 5,
+													 learning_rate=0.001, learning_rate_features=0.001,
+													 learning_rate_transitions=0.001)
+		
+		
+		args = parser.parse_args()
+		
+		#if args.task == 'srl':
+			#	if args.subtask == 'class':
+			#			args.task = 'srl_classify'
+			#			args.classify = True
+			#	elif args.subtask == 'id':
+			#			args.task = 'srl_boundary'
+			#			args.identify = True
+			#	elif args.subtask == 'pred':
+			#			args.task = 'srl_predicates'
+			#			args.predicates = True
+		if args.task == 'dependency':
+			if args.subtask == 'labeled':
+				args.task = 'labeled_dependency'
+				args.labeled = True
+			elif args.subtask == 'unlabeled':
+				args.task = 'unlabeled_dependency'
+				args.labeled = False
+				
+		fill_defaults(args, defaults)
+		return args