nltkor 1.2.17__tar.gz → 1.2.18__tar.gz

{nltkor-1.2.17 → nltkor-1.2.18}/PKG-INFO

@@ -1,8 +1,11 @@
 Metadata-Version: 2.1
 Name: nltkor
-Version: 1.2.17
+Version: 1.2.18
+Summary: UNKNOWN
 Home-page: https://modi.changwon.ac.kr/air_cwnu/nlp_tool/nltk_ko.git
+License: UNKNOWN
 Keywords: string matching,pattern matching,edit distance,string to string correction,string to string matching,Levenshtein edit distance,Hamming distance,Damerau-Levenshtein distance,Jaro-Winkler distance,longest common subsequence,longest common substring,dynamic programming,approximate string matching,semantic similarity,natural language processing,NLP,information retrieval,rouge,sacrebleu,bertscore,bartscore,fasttext,glove,cosine similarity,Smith-Waterman,Needleman-Wunsch,Hirschberg,Karp-Rabin,Knuth-Morris-Pratt,Boyer-Moore
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
@@ -12,25 +15,6 @@ Classifier: Operating System :: OS Independent
 Classifier: Typing :: Typed
 Requires-Python: >=3.7
 License-File: LICENSE.txt
-Requires-Dist: Cython
-Requires-Dist: numpy<=1.26.4,>=1.23.5
-Requires-Dist: regex
-Requires-Dist: tqdm>=4.40.0
-Requires-Dist: joblib
-Requires-Dist: requests
-Requires-Dist: nltk>3.0
-Requires-Dist: pyarrow
-Requires-Dist: beautifulSoup4
-Requires-Dist: faiss-cpu==1.7.3
-Requires-Dist: datasets
-Requires-Dist: torch
-Requires-Dist: dill<0.3.9
-Requires-Dist: scikit-learn>=0.22.1
-Requires-Dist: transformers==4.42.2
-Requires-Dist: protobuf
-Requires-Dist: sentencepiece
-Requires-Dist: pandas
-Requires-Dist: bert_score
-Requires-Dist: chardet
-Requires-Dist: GPUtil
-Requires-Dist: fasttext
+
+UNKNOWN
+

{nltkor-1.2.17 → nltkor-1.2.18}/README.md

@@ -8,6 +8,8 @@
 | 2    | 2024.5.22 | 차정원 | NLTKo 1.1.0 공개 |
 | 3    | 2025.2.5 | 이예나 | NLTKor 1.2.0 공개<br> bleu tensor 추가, entment 추가, accurancy norm 추가 |
 | 4    | 2025.4.3 | 이예나 | NLTKor 1.2.10 업데이트<br> espresso 오류 수정 |
+| 5    | 2025.5.21 | 정찬혁 | NLTKor 1.2.18 업데이트<br> TRIE 검색 추가|
+
 
 
 
@@ -89,6 +91,7 @@
   - [12.3 KMP 검색 알고리즘](#123-kmp-검색)
   - [12.4 Boyer-Moore 검색 알고리즘](#124-boyer-moore-검색)
   - [12.5 Faiss-Semantic 검색](#125-faiss-semantic-검색)
+  - [12.6 TRIE 검색](#126-trie-검색)
 - [13. 세종전자사전 (ssem)](#13-세종전자사전-ssem)
   - [13.1 객체 확인 방법](#131-객체-확인-방법)
   - [13.2 entry 접근법](#132-entry-접근법)
@@ -609,6 +612,7 @@ accuracy = correct / len(examples)
 print(f"Accuracy: {accuracy * 100:.2f}%")
 print(f"Time: {sum(inference_times)/len(inference_times)}, memory: {sum(memory_usages)/len(memory_usages)}")
 ```
+**결과**
 ```
 Accuracy: 20.00
 Time: 0.05374705195426941, memory: 1409.9
@@ -1665,6 +1669,42 @@ Adding FAISS index...
 4     피아노 연주는 나를 편안하게 해줍니다.  [-0.242319867, 0.6492734551, -1.4172941446, 0....  34.069862
 ```
 
+#### 12.6 TRIE 검색
+- 텍스트 파일에 word가 포함되어 있는지 판단한다.
+
+```python
+root = {}
+dict_file = '텍스트 파일 경로'
+sc = SearchDic(root)
+with open(dict_file, 'r') as f:
+    for line in f:
+        if ';;' in line[:2]: continue
+        k, v = line.strip().split('\t')
+        sc.build_search_dict(k, v)
+# print(root)
+word = '고용 노동부'
+values, value_data = sc.search_dict(word, True)
+print(values, value_data)
+
+word = '2시뉴스외전'
+values, value_data = sc.search_dict(word, True)
+print(values, value_data)
+word = '2시 뉴스외전'
+values, value_data = sc.search_dict(word, True)
+print(values, value_data)
+
+word = 'gbc'
+values, value_data = sc.search_dict(word, True)
+print(values, value_data)
+```
+**결과**
+```
+['고용 노동부'] ['NN']
+['2시뉴스외전'] ['NN']
+['2시 뉴스외전'] ['NN']
+['bc'] ['ND']
+```
+
 ### 13. 세종전자사전 (ssem)
 
 우선 해당 기능을 사용하기 전에 인자 포맷에 대해 설명한다. 인자는 **entrys, entry, sense** 함수에서 사용한다. 인자 포맷을 설명하기 위해 예제는 체언의 '눈'과 용언의 '감다'를 이용하였다. 

{nltkor-1.2.17 → nltkor-1.2.18}/nltkor/__init__.py

@@ -13,4 +13,4 @@ from nltkor import trans
 from nltkor import Kor_char
 from nltkor import etc
 
-__version__ = '1.2.17'
+__version__ = '1.2.18'

{nltkor-1.2.17 → nltkor-1.2.18}/nltkor/metrics/__init__.py

@@ -53,7 +53,6 @@ from nltkor.metrics.eval import StringMetric
 """
 from nltkor.metrics.classical import DefaultMetric
 from nltkor.metrics.entment import EMR
-from nltkor.metrics.bleu_tensor import *
 #DefaultMetric = lazy_import.lazy_callable("nltkor.metrics.classical.DefaultMetric")
 #Mauve = lazy_import.lazy_callable("nltkor.metrics.mauve.Mauve")
 from nltkor.metrics.mauve import Mauve

{nltkor-1.2.17 → nltkor-1.2.18}/nltkor/search/__init__.py

@@ -7,4 +7,5 @@ from .classical import (
     BoyerMooreSearch,
 )
 from .faiss_search import FaissSearch
-from .kobert_tokenizer import KoBERTTokenizer
+from .kobert_tokenizer import KoBERTTokenizer
+from .search_dict import SearchDic

nltkor-1.2.18/nltkor/search/search_dict.py

@@ -0,0 +1,95 @@
+import re, os, sys
+import pandas as pd
+import numpy as np
+import json
+import argparse
+
+class SearchDic :
+    def __init__ (self,root) :
+        self.root = root
+
+    def build_search_dict(self, word, data) -> dict:
+        current_dict = self.root
+        _end_word_ = '$$'
+        for letter in word:
+
+            current_dict = current_dict.setdefault(letter, {})
+        current_dict = current_dict.setdefault(_end_word_, data)
+         
+
+
+
+    def search_dict(self, word, space_flag=False):
+        '''
+        TRIE 탐색
+        space_flag: if True then including space, otherwise do not including space
+        '''
+
+        values = list()
+        value_data = list()
+        if not word: return self.root.keys()
+
+        current_dict = self.root
+        _end_word_ = '$$'
+        SPACE = ' '
+        s = 0
+        for i, letter in enumerate(word):
+            #print(i, s, '>', letter, values, value_data, current_dict)
+            if letter in current_dict:
+                #print('\t', letter, values, value_data, current_dict)
+                current_dict = current_dict[letter]
+                if _end_word_ in current_dict :
+                    values.append(word[s:i+1])
+                    value_data.append(current_dict[_end_word_])
+            elif space_flag and letter != SPACE and SPACE in current_dict:
+                look_ahead_dict = current_dict[SPACE]
+                # print('\t==', i, letter, values, look_ahead_dict)
+                if letter in look_ahead_dict:
+                    current_dict = look_ahead_dict[letter]
+            elif space_flag and letter == SPACE:
+                # print('\t##', i, letter, word[i+1], values)
+                continue
+            else:
+                # print('\t@@', i, letter, values)
+                s = i+1
+                current_dict = self.root
+        else:
+            if values: return values, value_data
+            else:	return list(word), value_data
+
+
+    def save_dict(self, file_path):
+        # root dictionary를 pickle 파일로 저장
+        with open(file_path, 'wb') as f:
+            pickle.dump(self.root, f)
+    
+    def load_dict(self,file_path) -> dict:
+        # pickle 퍄일을 읽어들인다.
+        with open(file_path, 'rb') as f:
+            return pickle.load(f)
+if __name__ == "__main__":
+    root = {}
+    dict_file = '텍스트파일경로'
+    sc = SearchDic(root)
+    with open(dict_file, 'r') as f:
+        for line in f:
+            if ';;' in line[:2]: continue
+            k, v = line.strip().split('\t')
+            sc.build_search_dict(k, v)
+    # print(root)
+    word = '고용 노동부'
+    values, value_data = sc.search_dict(word, True)
+    print(values, value_data)
+
+    word = '2시뉴스외전'
+    values, value_data = sc.search_dict( word, True)
+    print(values, value_data)
+    word = '2시 뉴스외전'
+    values, value_data = sc.search_dict( word, True)
+    print(values, value_data)
+
+    word = 'gbc'
+    values, value_data = sc.search_dict( word, True)
+    print(values, value_data)
+    
+