nltkor 1.2.18__tar.gz → 1.2.19__tar.gz

{nltkor-1.2.18 → nltkor-1.2.19}/PKG-INFO

@@ -1,11 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: nltkor
-Version: 1.2.18
-Summary: UNKNOWN
+Version: 1.2.19
 Home-page: https://modi.changwon.ac.kr/air_cwnu/nlp_tool/nltk_ko.git
-License: UNKNOWN
 Keywords: string matching,pattern matching,edit distance,string to string correction,string to string matching,Levenshtein edit distance,Hamming distance,Damerau-Levenshtein distance,Jaro-Winkler distance,longest common subsequence,longest common substring,dynamic programming,approximate string matching,semantic similarity,natural language processing,NLP,information retrieval,rouge,sacrebleu,bertscore,bartscore,fasttext,glove,cosine similarity,Smith-Waterman,Needleman-Wunsch,Hirschberg,Karp-Rabin,Knuth-Morris-Pratt,Boyer-Moore
-Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
@@ -15,6 +12,31 @@ Classifier: Operating System :: OS Independent
 Classifier: Typing :: Typed
 Requires-Python: >=3.7
 License-File: LICENSE.txt
-
-UNKNOWN
-
+Requires-Dist: Cython
+Requires-Dist: numpy<=1.26.4,>=1.23.5
+Requires-Dist: regex
+Requires-Dist: tqdm>=4.40.0
+Requires-Dist: joblib
+Requires-Dist: requests
+Requires-Dist: nltk>3.0
+Requires-Dist: pyarrow
+Requires-Dist: beautifulSoup4
+Requires-Dist: faiss-cpu==1.7.3
+Requires-Dist: datasets
+Requires-Dist: torch
+Requires-Dist: dill<0.3.9
+Requires-Dist: scikit-learn>=0.22.1
+Requires-Dist: transformers==4.42.2
+Requires-Dist: protobuf
+Requires-Dist: sentencepiece
+Requires-Dist: pandas
+Requires-Dist: bert_score
+Requires-Dist: chardet
+Requires-Dist: GPUtil
+Requires-Dist: fasttext
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python

{nltkor-1.2.18 → nltkor-1.2.19}/README.md

@@ -110,18 +110,13 @@ NLTKor는 한국어를 위한 NLTK이며 기존의 영어에서 사용하는 Wor
 
 ## 2. 사용 환경
 
-- 운영체제 : ubuntu 18.04, ubuntu 22.04, MacOS
+- 운영체제 : ubuntu 18.04, ubuntu 22.04, MacOS, Windows
 - 언어 : `python3.8`, `python3.9`, `python3.10`, `python3.11`
 - 라이브러리 : nltk>=1.1.3, numpy==1.23, faiss-cpu=1.7.3 **※ 해당 NLTKor는 영어 NLTK를 별도로 인스톨해야 함.**
 
 **주의사항**
 
-- Espresso5의 EspressoTagger의 사용가능 환경은 다음과 같다.
-
-| OS     | python                                    | 아키텍처      |
-| ------ | ----------------------------------------- | ------------- |
-| Mac    | python3.8                                 | arm64         |
-| ubuntu | python3.8 python3.9 python3.10 python3.11 | arm64, x86_64 |
+- Windows 환경에서 python 3.9~3.11을 사용할 경우 fasttext 라이브러리가 지원되지 않습니다. 대신 fasttext-wheel  라이브러리를 사용해야 합니다(pip install fasttext-wheel).
 
 ### 2.1 라이브러리 설치
 
@@ -1674,27 +1669,27 @@ Adding FAISS index...
 
 ```python
 root = {}
-dict_file = '텍스트 파일 경로'
-sc = SearchDic(root)
+dict_file = '텍스트파일 경로'
+sc = TRIESearch(root)
 with open(dict_file, 'r') as f:
     for line in f:
         if ';;' in line[:2]: continue
         k, v = line.strip().split('\t')
-        sc.build_search_dict(k, v)
+        sc.build_trie_search(k, v)
 # print(root)
 word = '고용 노동부'
-values, value_data = sc.search_dict(word, True)
+values, value_data = sc.trie_search(word, True)
 print(values, value_data)
 
 word = '2시뉴스외전'
-values, value_data = sc.search_dict(word, True)
+values, value_data = sc.trie_search( word, True)
 print(values, value_data)
 word = '2시 뉴스외전'
-values, value_data = sc.search_dict(word, True)
+values, value_data = sc.trie_search( word, True)
 print(values, value_data)
 
 word = 'gbc'
-values, value_data = sc.search_dict(word, True)
+values, value_data = sc.trie_search( word, True)
 print(values, value_data)
 ```
 **결과**

{nltkor-1.2.18 → nltkor-1.2.19}/nltkor/__init__.py

@@ -13,4 +13,4 @@ from nltkor import trans
 from nltkor import Kor_char
 from nltkor import etc
 
-__version__ = '1.2.18'
+__version__ = '1.2.19'

{nltkor-1.2.18 → nltkor-1.2.19}/nltkor/search/__init__.py

@@ -8,4 +8,4 @@ from .classical import (
 )
 from .faiss_search import FaissSearch
 from .kobert_tokenizer import KoBERTTokenizer
-from .search_dict import SearchDic
+from .trie_search import TRIESearch

nltkor-1.2.19/nltkor/search/test.py

@@ -0,0 +1,25 @@
+from trie_search import TRIESearch
+
+root = {}
+dict_file = '/Users/chanhyeok/Downloads/lexicon.txt'
+sc = TRIESearch(root)
+with open(dict_file, 'r') as f:
+    for line in f:
+        if ';;' in line[:2]: continue
+        k, v = line.strip().split('\t')
+        sc.build_trie_search(k, v)
+    # print(root)
+word = '고용 노동부'
+values, value_data = sc.trie_search(word, True)
+print(values, value_data)
+
+word = '2시뉴스외전'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)
+word = '2시 뉴스외전'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)
+
+word = 'gbc'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)

nltkor-1.2.18/nltkor/search/search_dict.py → nltkor-1.2.19/nltkor/search/trie_search.py

@@ -4,11 +4,11 @@ import numpy as np
 import json
 import argparse
 
-class SearchDic :
+class TRIESearch :
     def __init__ (self,root) :
         self.root = root
 
-    def build_search_dict(self, word, data) -> dict:
+    def build_trie_search(self, word, data) -> dict:
         current_dict = self.root
         _end_word_ = '$$'
         for letter in word:
@@ -19,7 +19,7 @@ class SearchDic :
 
 
 
-    def search_dict(self, word, space_flag=False):
+    def trie_search(self, word, space_flag=False):
         '''
         TRIE 탐색
         space_flag: if True then including space, otherwise do not including space
@@ -69,27 +69,27 @@ class SearchDic :
             return pickle.load(f)
 if __name__ == "__main__":
     root = {}
-    dict_file = '텍스트파일경로'
-    sc = SearchDic(root)
+    dict_file = '텍스트파일 경로'
+    sc = TRIESearch(root)
     with open(dict_file, 'r') as f:
         for line in f:
             if ';;' in line[:2]: continue
             k, v = line.strip().split('\t')
-            sc.build_search_dict(k, v)
+            sc.build_trie_search(k, v)
     # print(root)
     word = '고용 노동부'
-    values, value_data = sc.search_dict(word, True)
+    values, value_data = sc.trie_search(word, True)
     print(values, value_data)
 
     word = '2시뉴스외전'
-    values, value_data = sc.search_dict( word, True)
+    values, value_data = sc.trie_search( word, True)
     print(values, value_data)
     word = '2시 뉴스외전'
-    values, value_data = sc.search_dict( word, True)
+    values, value_data = sc.trie_search( word, True)
     print(values, value_data)
 
     word = 'gbc'
-    values, value_data = sc.search_dict( word, True)
+    values, value_data = sc.trie_search( word, True)
     print(values, value_data)