nltkor 1.2.17__tar.gz → 1.2.19__tar.gz

{nltkor-1.2.17 → nltkor-1.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: nltkor
-Version: 1.2.17
+Version: 1.2.19
 Home-page: https://modi.changwon.ac.kr/air_cwnu/nlp_tool/nltk_ko.git
 Keywords: string matching,pattern matching,edit distance,string to string correction,string to string matching,Levenshtein edit distance,Hamming distance,Damerau-Levenshtein distance,Jaro-Winkler distance,longest common subsequence,longest common substring,dynamic programming,approximate string matching,semantic similarity,natural language processing,NLP,information retrieval,rouge,sacrebleu,bertscore,bartscore,fasttext,glove,cosine similarity,Smith-Waterman,Needleman-Wunsch,Hirschberg,Karp-Rabin,Knuth-Morris-Pratt,Boyer-Moore
 Classifier: Programming Language :: Python :: 3.7
@@ -34,3 +34,9 @@ Requires-Dist: bert_score
 Requires-Dist: chardet
 Requires-Dist: GPUtil
 Requires-Dist: fasttext
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python

{nltkor-1.2.17 → nltkor-1.2.19}/README.md

@@ -8,6 +8,8 @@
 | 2    | 2024.5.22 | 차정원 | NLTKo 1.1.0 공개 |
 | 3    | 2025.2.5 | 이예나 | NLTKor 1.2.0 공개<br> bleu tensor 추가, entment 추가, accurancy norm 추가 |
 | 4    | 2025.4.3 | 이예나 | NLTKor 1.2.10 업데이트<br> espresso 오류 수정 |
+| 5    | 2025.5.21 | 정찬혁 | NLTKor 1.2.18 업데이트<br> TRIE 검색 추가|
+
 
 
 
@@ -89,6 +91,7 @@
   - [12.3 KMP 검색 알고리즘](#123-kmp-검색)
   - [12.4 Boyer-Moore 검색 알고리즘](#124-boyer-moore-검색)
   - [12.5 Faiss-Semantic 검색](#125-faiss-semantic-검색)
+  - [12.6 TRIE 검색](#126-trie-검색)
 - [13. 세종전자사전 (ssem)](#13-세종전자사전-ssem)
   - [13.1 객체 확인 방법](#131-객체-확인-방법)
   - [13.2 entry 접근법](#132-entry-접근법)
@@ -107,18 +110,13 @@ NLTKor는 한국어를 위한 NLTK이며 기존의 영어에서 사용하는 Wor
 
 ## 2. 사용 환경
 
-- 운영체제 : ubuntu 18.04, ubuntu 22.04, MacOS
+- 운영체제 : ubuntu 18.04, ubuntu 22.04, MacOS, Windows
 - 언어 : `python3.8`, `python3.9`, `python3.10`, `python3.11`
 - 라이브러리 : nltk>=1.1.3, numpy==1.23, faiss-cpu=1.7.3 **※ 해당 NLTKor는 영어 NLTK를 별도로 인스톨해야 함.**
 
 **주의사항**
 
-- Espresso5의 EspressoTagger의 사용가능 환경은 다음과 같다.
-
-| OS     | python                                    | 아키텍처      |
-| ------ | ----------------------------------------- | ------------- |
-| Mac    | python3.8                                 | arm64         |
-| ubuntu | python3.8 python3.9 python3.10 python3.11 | arm64, x86_64 |
+- Windows 환경에서 python 3.9~3.11을 사용할 경우 fasttext 라이브러리가 지원되지 않습니다. 대신 fasttext-wheel  라이브러리를 사용해야 합니다(pip install fasttext-wheel).
 
 ### 2.1 라이브러리 설치
 
@@ -609,6 +607,7 @@ accuracy = correct / len(examples)
 print(f"Accuracy: {accuracy * 100:.2f}%")
 print(f"Time: {sum(inference_times)/len(inference_times)}, memory: {sum(memory_usages)/len(memory_usages)}")
 ```
+**결과**
 ```
 Accuracy: 20.00
 Time: 0.05374705195426941, memory: 1409.9
@@ -1665,6 +1664,42 @@ Adding FAISS index...
 4     피아노 연주는 나를 편안하게 해줍니다.  [-0.242319867, 0.6492734551, -1.4172941446, 0....  34.069862
 ```
 
+#### 12.6 TRIE 검색
+- 텍스트 파일에 word가 포함되어 있는지 판단한다.
+
+```python
+root = {}
+dict_file = '텍스트파일 경로'
+sc = TRIESearch(root)
+with open(dict_file, 'r') as f:
+    for line in f:
+        if ';;' in line[:2]: continue
+        k, v = line.strip().split('\t')
+        sc.build_trie_search(k, v)
+# print(root)
+word = '고용 노동부'
+values, value_data = sc.trie_search(word, True)
+print(values, value_data)
+
+word = '2시뉴스외전'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)
+word = '2시 뉴스외전'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)
+
+word = 'gbc'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)
+```
+**결과**
+```
+['고용 노동부'] ['NN']
+['2시뉴스외전'] ['NN']
+['2시 뉴스외전'] ['NN']
+['bc'] ['ND']
+```
+
 ### 13. 세종전자사전 (ssem)
 
 우선 해당 기능을 사용하기 전에 인자 포맷에 대해 설명한다. 인자는 **entrys, entry, sense** 함수에서 사용한다. 인자 포맷을 설명하기 위해 예제는 체언의 '눈'과 용언의 '감다'를 이용하였다. 

{nltkor-1.2.17 → nltkor-1.2.19}/nltkor/__init__.py

@@ -13,4 +13,4 @@ from nltkor import trans
 from nltkor import Kor_char
 from nltkor import etc
 
-__version__ = '1.2.17'
+__version__ = '1.2.19'

{nltkor-1.2.17 → nltkor-1.2.19}/nltkor/metrics/__init__.py

@@ -53,7 +53,6 @@ from nltkor.metrics.eval import StringMetric
 """
 from nltkor.metrics.classical import DefaultMetric
 from nltkor.metrics.entment import EMR
-from nltkor.metrics.bleu_tensor import *
 #DefaultMetric = lazy_import.lazy_callable("nltkor.metrics.classical.DefaultMetric")
 #Mauve = lazy_import.lazy_callable("nltkor.metrics.mauve.Mauve")
 from nltkor.metrics.mauve import Mauve

{nltkor-1.2.17 → nltkor-1.2.19}/nltkor/search/__init__.py

@@ -7,4 +7,5 @@ from .classical import (
     BoyerMooreSearch,
 )
 from .faiss_search import FaissSearch
-from .kobert_tokenizer import KoBERTTokenizer
+from .kobert_tokenizer import KoBERTTokenizer
+from .trie_search import TRIESearch

nltkor-1.2.19/nltkor/search/test.py

@@ -0,0 +1,25 @@
+from trie_search import TRIESearch
+
+root = {}
+dict_file = '/Users/chanhyeok/Downloads/lexicon.txt'
+sc = TRIESearch(root)
+with open(dict_file, 'r') as f:
+    for line in f:
+        if ';;' in line[:2]: continue
+        k, v = line.strip().split('\t')
+        sc.build_trie_search(k, v)
+    # print(root)
+word = '고용 노동부'
+values, value_data = sc.trie_search(word, True)
+print(values, value_data)
+
+word = '2시뉴스외전'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)
+word = '2시 뉴스외전'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)
+
+word = 'gbc'
+values, value_data = sc.trie_search( word, True)
+print(values, value_data)

nltkor-1.2.19/nltkor/search/trie_search.py

@@ -0,0 +1,95 @@
+import re, os, sys
+import pandas as pd
+import numpy as np
+import json
+import argparse
+
+class TRIESearch :
+    def __init__ (self,root) :
+        self.root = root
+
+    def build_trie_search(self, word, data) -> dict:
+        current_dict = self.root
+        _end_word_ = '$$'
+        for letter in word:
+
+            current_dict = current_dict.setdefault(letter, {})
+        current_dict = current_dict.setdefault(_end_word_, data)
+         
+
+
+
+    def trie_search(self, word, space_flag=False):
+        '''
+        TRIE 탐색
+        space_flag: if True then including space, otherwise do not including space
+        '''
+
+        values = list()
+        value_data = list()
+        if not word: return self.root.keys()
+
+        current_dict = self.root
+        _end_word_ = '$$'
+        SPACE = ' '
+        s = 0
+        for i, letter in enumerate(word):
+            #print(i, s, '>', letter, values, value_data, current_dict)
+            if letter in current_dict:
+                #print('\t', letter, values, value_data, current_dict)
+                current_dict = current_dict[letter]
+                if _end_word_ in current_dict :
+                    values.append(word[s:i+1])
+                    value_data.append(current_dict[_end_word_])
+            elif space_flag and letter != SPACE and SPACE in current_dict:
+                look_ahead_dict = current_dict[SPACE]
+                # print('\t==', i, letter, values, look_ahead_dict)
+                if letter in look_ahead_dict:
+                    current_dict = look_ahead_dict[letter]
+            elif space_flag and letter == SPACE:
+                # print('\t##', i, letter, word[i+1], values)
+                continue
+            else:
+                # print('\t@@', i, letter, values)
+                s = i+1
+                current_dict = self.root
+        else:
+            if values: return values, value_data
+            else:	return list(word), value_data
+
+
+    def save_dict(self, file_path):
+        # root dictionary를 pickle 파일로 저장
+        with open(file_path, 'wb') as f:
+            pickle.dump(self.root, f)
+    
+    def load_dict(self,file_path) -> dict:
+        # pickle 퍄일을 읽어들인다.
+        with open(file_path, 'rb') as f:
+            return pickle.load(f)
+if __name__ == "__main__":
+    root = {}
+    dict_file = '텍스트파일 경로'
+    sc = TRIESearch(root)
+    with open(dict_file, 'r') as f:
+        for line in f:
+            if ';;' in line[:2]: continue
+            k, v = line.strip().split('\t')
+            sc.build_trie_search(k, v)
+    # print(root)
+    word = '고용 노동부'
+    values, value_data = sc.trie_search(word, True)
+    print(values, value_data)
+
+    word = '2시뉴스외전'
+    values, value_data = sc.trie_search( word, True)
+    print(values, value_data)
+    word = '2시 뉴스외전'
+    values, value_data = sc.trie_search( word, True)
+    print(values, value_data)
+
+    word = 'gbc'
+    values, value_data = sc.trie_search( word, True)
+    print(values, value_data)
+    
+