nahiarhdNLP 1.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mydatasets/__init__.py +0 -0
- mydatasets/loaders.py +262 -0
- nahiarhdnlp-1.0.3.dist-info/METADATA +346 -0
- nahiarhdnlp-1.0.3.dist-info/RECORD +28 -0
- nahiarhdnlp-1.0.3.dist-info/WHEEL +5 -0
- nahiarhdnlp-1.0.3.dist-info/licenses/LICENSE +21 -0
- nahiarhdnlp-1.0.3.dist-info/top_level.txt +2 -0
- preprocessing/__init__.py +74 -0
- preprocessing/cleaning/__init__.py +0 -0
- preprocessing/cleaning/indonesia_spellchecker/README.md +8 -0
- preprocessing/cleaning/indonesia_spellchecker/__init__.py +0 -0
- preprocessing/cleaning/indonesia_spellchecker/kamus.txt +30871 -0
- preprocessing/cleaning/indonesia_spellchecker/kata_dasar_kbbi.csv +28526 -0
- preprocessing/cleaning/indonesia_spellchecker/main.py +44 -0
- preprocessing/cleaning/indonesia_spellchecker/stemmer.py +280 -0
- preprocessing/cleaning/indonesia_spellchecker/teks.txt +1 -0
- preprocessing/cleaning/indonesia_spellchecker/wordlist.json +1 -0
- preprocessing/cleaning/spell_corrector.py +44 -0
- preprocessing/cleaning/text_cleaner.py +243 -0
- preprocessing/linguistic/__init__.py +0 -0
- preprocessing/linguistic/stemmer.py +23 -0
- preprocessing/linguistic/stopwords.py +324 -0
- preprocessing/normalization/__init__.py +0 -0
- preprocessing/normalization/emoji.py +217 -0
- preprocessing/normalization/slang.py +175 -0
- preprocessing/tokenization/__init__.py +0 -0
- preprocessing/tokenization/tokenizer.py +10 -0
- preprocessing/utils.py +624 -0
mydatasets/__init__.py
ADDED
|
File without changes
|
mydatasets/loaders.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import pickle
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from datasets import load_dataset
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DatasetLoader:
|
|
8
|
+
"""Loader for Indonesian NLP datasets from HuggingFace with caching."""
|
|
9
|
+
|
|
10
|
+
def __init__(self):
|
|
11
|
+
# Setup cache directory
|
|
12
|
+
self.cache_dir = Path.home() / ".nahiarhdNLP" / "cache"
|
|
13
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
14
|
+
|
|
15
|
+
def _get_cached_data(self, dataset_name: str):
|
|
16
|
+
"""Get cached dataset if exists."""
|
|
17
|
+
cache_file = self.cache_dir / f"{dataset_name}.pkl"
|
|
18
|
+
if cache_file.exists():
|
|
19
|
+
try:
|
|
20
|
+
with open(cache_file, "rb") as f:
|
|
21
|
+
return pickle.load(f)
|
|
22
|
+
except Exception:
|
|
23
|
+
# If cache corrupted, remove it
|
|
24
|
+
cache_file.unlink()
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
def _save_cached_data(self, dataset_name: str, data):
|
|
28
|
+
"""Save dataset to cache."""
|
|
29
|
+
cache_file = self.cache_dir / f"{dataset_name}.pkl"
|
|
30
|
+
try:
|
|
31
|
+
with open(cache_file, "wb") as f:
|
|
32
|
+
pickle.dump(data, f)
|
|
33
|
+
except Exception as e:
|
|
34
|
+
print(f"Warning: Could not cache {dataset_name}: {e}")
|
|
35
|
+
|
|
36
|
+
def load_stopwords_dataset(self, language="indonesian"):
|
|
37
|
+
"""Load stopwords with caching."""
|
|
38
|
+
cache_key = f"stopwords_{language}"
|
|
39
|
+
|
|
40
|
+
# Try cache first
|
|
41
|
+
cached_data = self._get_cached_data(cache_key)
|
|
42
|
+
if cached_data:
|
|
43
|
+
print(f"Using cached stopwords ({len(cached_data)} words)")
|
|
44
|
+
return cached_data
|
|
45
|
+
|
|
46
|
+
# Load from HuggingFace
|
|
47
|
+
print("Loading stopwords from HuggingFace...")
|
|
48
|
+
try:
|
|
49
|
+
ds = load_dataset("nahiar/indo-stopwords")
|
|
50
|
+
data = [item["stopword"] for item in list(ds["train"])]
|
|
51
|
+
|
|
52
|
+
# Cache the data
|
|
53
|
+
self._save_cached_data(cache_key, data)
|
|
54
|
+
return data
|
|
55
|
+
|
|
56
|
+
except Exception as e:
|
|
57
|
+
print(f"Error loading stopwords: {e}")
|
|
58
|
+
return self._get_fallback_stopwords()
|
|
59
|
+
|
|
60
|
+
def load_slang_dataset(self, language="indonesian"):
|
|
61
|
+
"""Load slang dataset with caching."""
|
|
62
|
+
cache_key = f"slang_{language}"
|
|
63
|
+
|
|
64
|
+
# Try cache first
|
|
65
|
+
cached_data = self._get_cached_data(cache_key)
|
|
66
|
+
if cached_data:
|
|
67
|
+
print(f"Using cached slang ({len(cached_data)} entries)")
|
|
68
|
+
return cached_data
|
|
69
|
+
|
|
70
|
+
# Load from HuggingFace
|
|
71
|
+
print("Loading slang from HuggingFace...")
|
|
72
|
+
try:
|
|
73
|
+
ds = load_dataset("nahiar/indonesia-slang")
|
|
74
|
+
data = [
|
|
75
|
+
{"slang": item["slang"], "formal": item["formal"]}
|
|
76
|
+
for item in list(ds["train"])
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
# Cache the data
|
|
80
|
+
self._save_cached_data(cache_key, data)
|
|
81
|
+
return data
|
|
82
|
+
|
|
83
|
+
except Exception as e:
|
|
84
|
+
print(f"Error loading slang: {e}")
|
|
85
|
+
return self._get_fallback_slang()
|
|
86
|
+
|
|
87
|
+
def load_emoji_dataset(self, language="indonesian"):
|
|
88
|
+
"""Load emoji dataset with caching."""
|
|
89
|
+
cache_key = f"emoji_{language}"
|
|
90
|
+
|
|
91
|
+
# Try cache first
|
|
92
|
+
cached_data = self._get_cached_data(cache_key)
|
|
93
|
+
if cached_data:
|
|
94
|
+
print(f"Using cached emoji ({len(cached_data)} entries)")
|
|
95
|
+
return cached_data
|
|
96
|
+
|
|
97
|
+
# Load from HuggingFace
|
|
98
|
+
print("Loading emoji from HuggingFace...")
|
|
99
|
+
try:
|
|
100
|
+
ds = load_dataset("nahiar/indo-emoji-dictionary")
|
|
101
|
+
data = [dict(item) for item in list(ds["train"])]
|
|
102
|
+
|
|
103
|
+
# Cache the data
|
|
104
|
+
self._save_cached_data(cache_key, data)
|
|
105
|
+
return data
|
|
106
|
+
|
|
107
|
+
except Exception as e:
|
|
108
|
+
print(f"Error loading emoji: {e}")
|
|
109
|
+
return self._get_fallback_emoji()
|
|
110
|
+
|
|
111
|
+
def _get_fallback_stopwords(self):
|
|
112
|
+
"""Fallback stopwords if HuggingFace fails."""
|
|
113
|
+
return [
|
|
114
|
+
"yang",
|
|
115
|
+
"dan",
|
|
116
|
+
"di",
|
|
117
|
+
"ke",
|
|
118
|
+
"dari",
|
|
119
|
+
"untuk",
|
|
120
|
+
"dengan",
|
|
121
|
+
"ini",
|
|
122
|
+
"itu",
|
|
123
|
+
"atau",
|
|
124
|
+
"juga",
|
|
125
|
+
"bisa",
|
|
126
|
+
"akan",
|
|
127
|
+
"sudah",
|
|
128
|
+
"belum",
|
|
129
|
+
"tidak",
|
|
130
|
+
"bukan",
|
|
131
|
+
"saya",
|
|
132
|
+
"kamu",
|
|
133
|
+
"dia",
|
|
134
|
+
"mereka",
|
|
135
|
+
"kami",
|
|
136
|
+
"kita",
|
|
137
|
+
"anda",
|
|
138
|
+
"beliau",
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
def _get_fallback_slang(self):
|
|
142
|
+
"""Fallback slang if HuggingFace fails."""
|
|
143
|
+
return [
|
|
144
|
+
{"slang": "gw", "formal": "saya"},
|
|
145
|
+
{"slang": "gue", "formal": "saya"},
|
|
146
|
+
{"slang": "lo", "formal": "kamu"},
|
|
147
|
+
{"slang": "lu", "formal": "kamu"},
|
|
148
|
+
{"slang": "yg", "formal": "yang"},
|
|
149
|
+
{"slang": "dgn", "formal": "dengan"},
|
|
150
|
+
{"slang": "utk", "formal": "untuk"},
|
|
151
|
+
{"slang": "dr", "formal": "dari"},
|
|
152
|
+
{"slang": "tp", "formal": "tapi"},
|
|
153
|
+
{"slang": "krn", "formal": "karena"},
|
|
154
|
+
{"slang": "jd", "formal": "jadi"},
|
|
155
|
+
{"slang": "sdh", "formal": "sudah"},
|
|
156
|
+
{"slang": "udh", "formal": "sudah"},
|
|
157
|
+
{"slang": "blm", "formal": "belum"},
|
|
158
|
+
{"slang": "ga", "formal": "tidak"},
|
|
159
|
+
{"slang": "gak", "formal": "tidak"},
|
|
160
|
+
{"slang": "nggak", "formal": "tidak"},
|
|
161
|
+
{"slang": "bgt", "formal": "banget"},
|
|
162
|
+
{"slang": "klo", "formal": "kalau"},
|
|
163
|
+
{"slang": "gimana", "formal": "bagaimana"},
|
|
164
|
+
{"slang": "gmn", "formal": "bagaimana"},
|
|
165
|
+
{"slang": "kenapa", "formal": "mengapa"},
|
|
166
|
+
{"slang": "knp", "formal": "mengapa"},
|
|
167
|
+
{"slang": "makasih", "formal": "terima kasih"},
|
|
168
|
+
{"slang": "mksh", "formal": "terima kasih"},
|
|
169
|
+
{"slang": "org", "formal": "orang"},
|
|
170
|
+
{"slang": "skrg", "formal": "sekarang"},
|
|
171
|
+
{"slang": "trs", "formal": "terus"},
|
|
172
|
+
{"slang": "emg", "formal": "memang"},
|
|
173
|
+
{"slang": "kyk", "formal": "seperti"},
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
def _get_fallback_emoji(self):
|
|
177
|
+
"""Fallback emoji if HuggingFace fails."""
|
|
178
|
+
return [
|
|
179
|
+
{
|
|
180
|
+
"emoji": "😀",
|
|
181
|
+
"name_id": "wajah_gembira",
|
|
182
|
+
"alias": "wajah_gembira",
|
|
183
|
+
"aliases": ["gembira", "senang", "happy"],
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
"emoji": "😂",
|
|
187
|
+
"name_id": "wajah_tertawa",
|
|
188
|
+
"alias": "wajah_tertawa",
|
|
189
|
+
"aliases": ["tertawa", "laugh", "ketawa"],
|
|
190
|
+
},
|
|
191
|
+
{
|
|
192
|
+
"emoji": "😍",
|
|
193
|
+
"name_id": "wajah_bercinta",
|
|
194
|
+
"alias": "wajah_bercinta",
|
|
195
|
+
"aliases": ["bercinta", "love", "cinta"],
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
"emoji": "😭",
|
|
199
|
+
"name_id": "wajah_menangis",
|
|
200
|
+
"alias": "wajah_menangis",
|
|
201
|
+
"aliases": ["menangis", "cry", "sedih"],
|
|
202
|
+
},
|
|
203
|
+
{
|
|
204
|
+
"emoji": "😡",
|
|
205
|
+
"name_id": "wajah_marah",
|
|
206
|
+
"alias": "wajah_marah",
|
|
207
|
+
"aliases": ["marah", "angry", "kesal"],
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
"emoji": "👍",
|
|
211
|
+
"name_id": "jempol_keatas",
|
|
212
|
+
"alias": "jempol_keatas",
|
|
213
|
+
"aliases": ["jempol", "thumbs_up", "bagus"],
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
"emoji": "👎",
|
|
217
|
+
"name_id": "jempol_kebawah",
|
|
218
|
+
"alias": "jempol_kebawah",
|
|
219
|
+
"aliases": ["jempol_bawah", "thumbs_down", "jelek"],
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
"emoji": "❤️",
|
|
223
|
+
"name_id": "hati_merah",
|
|
224
|
+
"alias": "hati_merah",
|
|
225
|
+
"aliases": ["hati", "love", "cinta"],
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
"emoji": "🔥",
|
|
229
|
+
"name_id": "api",
|
|
230
|
+
"alias": "api",
|
|
231
|
+
"aliases": ["panas", "fire", "keren"],
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
"emoji": "💯",
|
|
235
|
+
"name_id": "seratus",
|
|
236
|
+
"alias": "seratus",
|
|
237
|
+
"aliases": ["100", "perfect", "sempurna"],
|
|
238
|
+
},
|
|
239
|
+
]
|
|
240
|
+
|
|
241
|
+
def clear_cache(self):
|
|
242
|
+
"""Clear all cached data."""
|
|
243
|
+
import shutil
|
|
244
|
+
|
|
245
|
+
if self.cache_dir.exists():
|
|
246
|
+
shutil.rmtree(self.cache_dir)
|
|
247
|
+
print("Cache cleared!")
|
|
248
|
+
|
|
249
|
+
def get_cache_info(self):
|
|
250
|
+
"""Get cache information."""
|
|
251
|
+
if not self.cache_dir.exists():
|
|
252
|
+
return "No cache directory"
|
|
253
|
+
|
|
254
|
+
cache_files = list(self.cache_dir.glob("*.pkl"))
|
|
255
|
+
total_size = sum(f.stat().st_size for f in cache_files)
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
"cache_dir": str(self.cache_dir),
|
|
259
|
+
"files": len(cache_files),
|
|
260
|
+
"total_size_mb": total_size / 1024 / 1024,
|
|
261
|
+
"files_list": [f.name for f in cache_files],
|
|
262
|
+
}
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nahiarhdNLP
|
|
3
|
+
Version: 1.0.3
|
|
4
|
+
Summary: Advanced Indonesian Natural Language Processing Library
|
|
5
|
+
Author-email: Raihan Hidayatullah Djunaedi <raihanhd.dev@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://example.com
|
|
8
|
+
Project-URL: Documentation, https://example.com
|
|
9
|
+
Project-URL: Repository, https://github.com/raihanhd12/nahiarhdNLP
|
|
10
|
+
Project-URL: Issues, https://github.com/raihanhd12/nahiarhdNLP/issues
|
|
11
|
+
Keywords: nlp,indonesian,natural-language-processing,text-processing,bahasa-indonesia
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
24
|
+
Classifier: Operating System :: OS Independent
|
|
25
|
+
Requires-Python: >=3.8
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: pandas>=1.3.0
|
|
29
|
+
Requires-Dist: fsspec>=2021.10.1
|
|
30
|
+
Requires-Dist: huggingface_hub>=0.10.0
|
|
31
|
+
Requires-Dist: sastrawi>=1.0.1
|
|
32
|
+
Requires-Dist: datasets>=2.0.0
|
|
33
|
+
Requires-Dist: rich>=12.0.0
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
37
|
+
Requires-Dist: black>=22.0.0; extra == "dev"
|
|
38
|
+
Requires-Dist: isort>=5.10.0; extra == "dev"
|
|
39
|
+
Requires-Dist: flake8>=4.0.0; extra == "dev"
|
|
40
|
+
Requires-Dist: mypy>=0.991; extra == "dev"
|
|
41
|
+
Requires-Dist: pre-commit>=2.17.0; extra == "dev"
|
|
42
|
+
Provides-Extra: docs
|
|
43
|
+
Requires-Dist: sphinx>=4.0.0; extra == "docs"
|
|
44
|
+
Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
|
|
45
|
+
Requires-Dist: myst-parser>=0.17.0; extra == "docs"
|
|
46
|
+
Dynamic: license-file
|
|
47
|
+
|
|
48
|
+
# nahiarhdNLP - Advanced Indonesian Natural Language Processing Library
|
|
49
|
+
|
|
50
|
+
Advanced Indonesian Natural Language Processing Library dengan fitur preprocessing teks, normalisasi slang, konversi emoji, koreksi ejaan, dan banyak lagi.
|
|
51
|
+
|
|
52
|
+
## 🚀 Instalasi
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install nahiarhdNLP
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## 📦 Import Library
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
# Import package utama
|
|
62
|
+
import nahiarhdNLP
|
|
63
|
+
|
|
64
|
+
# Import module preprocessing
|
|
65
|
+
from nahiarhdNLP import preprocessing
|
|
66
|
+
|
|
67
|
+
# Import module datasets
|
|
68
|
+
from nahiarhdNLP import mydatasets
|
|
69
|
+
|
|
70
|
+
# Atau import fungsi spesifik
|
|
71
|
+
from nahiarhdNLP.preprocessing import preprocess, remove_html, replace_slang
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Contoh Penggunaan
|
|
75
|
+
|
|
76
|
+
### 1. 🎯 Fungsi Preprocess All-in-One
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from nahiarhdNLP import preprocessing
|
|
80
|
+
|
|
81
|
+
# Preprocessing lengkap dengan satu fungsi
|
|
82
|
+
teks = "Halooo emg siapa yg nanya? 😀"
|
|
83
|
+
hasil = preprocessing.preprocess(teks)
|
|
84
|
+
print(hasil)
|
|
85
|
+
# Output: "halo wajah_gembira"
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### 2. 🧹 TextCleaner - Membersihkan Teks
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from nahiarhdNLP.preprocessing import TextCleaner
|
|
92
|
+
|
|
93
|
+
cleaner = TextCleaner()
|
|
94
|
+
|
|
95
|
+
# Membersihkan URL
|
|
96
|
+
url_text = "kunjungi https://google.com sekarang!"
|
|
97
|
+
clean_result = cleaner.clean_urls(url_text)
|
|
98
|
+
print(clean_result)
|
|
99
|
+
# Output: "kunjungi sekarang!"
|
|
100
|
+
|
|
101
|
+
# Membersihkan mentions
|
|
102
|
+
mention_text = "Halo @user123 apa kabar?"
|
|
103
|
+
clean_result = cleaner.clean_mentions(mention_text)
|
|
104
|
+
print(clean_result)
|
|
105
|
+
# Output: "Halo apa kabar?"
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### 3. ✏️ SpellCorrector - Koreksi Ejaan
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from nahiarhdNLP.preprocessing import SpellCorrector
|
|
112
|
+
|
|
113
|
+
spell = SpellCorrector()
|
|
114
|
+
|
|
115
|
+
# Koreksi kata
|
|
116
|
+
word = "mencri"
|
|
117
|
+
corrected = spell.correct(word)
|
|
118
|
+
print(corrected)
|
|
119
|
+
# Output: "mencuri"
|
|
120
|
+
|
|
121
|
+
# Koreksi kalimat
|
|
122
|
+
sentence = "saya mencri informsi"
|
|
123
|
+
corrected = spell.correct_sentence(sentence)
|
|
124
|
+
print(corrected)
|
|
125
|
+
# Output: "saya mencuri informasi"
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### 4. 🚫 StopwordRemover - Menghapus Stopwords
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from nahiarhdNLP.preprocessing import StopwordRemover
|
|
132
|
+
|
|
133
|
+
stopword = StopwordRemover()
|
|
134
|
+
|
|
135
|
+
# Menghapus stopwords
|
|
136
|
+
text = "saya suka makan nasi goreng"
|
|
137
|
+
result = stopword.remove_stopwords(text)
|
|
138
|
+
print(result)
|
|
139
|
+
# Output: "suka makan nasi goreng"
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### 5. 🔄 SlangNormalizer - Normalisasi Slang
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from nahiarhdNLP.preprocessing import SlangNormalizer
|
|
146
|
+
|
|
147
|
+
slang = SlangNormalizer()
|
|
148
|
+
|
|
149
|
+
# Normalisasi kata slang
|
|
150
|
+
text = "gw lg di rmh"
|
|
151
|
+
result = slang.normalize(text)
|
|
152
|
+
print(result)
|
|
153
|
+
# Output: "saya lagi di rumah"
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### 6. 😀 EmojiConverter - Konversi Emoji
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
from nahiarhdNLP.preprocessing import EmojiConverter
|
|
160
|
+
|
|
161
|
+
emoji = EmojiConverter()
|
|
162
|
+
|
|
163
|
+
# Emoji ke teks
|
|
164
|
+
emoji_text = "😀 😂 😍"
|
|
165
|
+
text_result = emoji.emoji_to_text_convert(emoji_text)
|
|
166
|
+
print(text_result)
|
|
167
|
+
# Output: "wajah_gembira wajah_tertawa wajah_bercinta"
|
|
168
|
+
|
|
169
|
+
# Teks ke emoji
|
|
170
|
+
text = "wajah_gembira"
|
|
171
|
+
emoji_result = emoji.text_to_emoji_convert(text)
|
|
172
|
+
print(emoji_result)
|
|
173
|
+
# Output: "😀"
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
### 7. 🔪 Tokenizer - Tokenisasi
|
|
177
|
+
|
|
178
|
+
```python
|
|
179
|
+
from nahiarhdNLP.preprocessing import Tokenizer
|
|
180
|
+
|
|
181
|
+
tokenizer = Tokenizer()
|
|
182
|
+
|
|
183
|
+
# Tokenisasi teks
|
|
184
|
+
text = "ini contoh tokenisasi"
|
|
185
|
+
tokens = tokenizer.tokenize(text)
|
|
186
|
+
print(tokens)
|
|
187
|
+
# Output: ['ini', 'contoh', 'tokenisasi']
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### 8. 🛠️ Fungsi Individual
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
from nahiarhdNLP.preprocessing import (
|
|
194
|
+
remove_html, remove_url, remove_mentions,
|
|
195
|
+
replace_slang, emoji_to_words, correct_spelling
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
# Menghapus HTML
|
|
199
|
+
html_text = "website <a href='https://google.com'>google</a>"
|
|
200
|
+
clean_text = remove_html(html_text)
|
|
201
|
+
print(clean_text)
|
|
202
|
+
# Output: "website google"
|
|
203
|
+
|
|
204
|
+
# Menghapus URL
|
|
205
|
+
url_text = "kunjungi https://google.com sekarang!"
|
|
206
|
+
clean_text = remove_url(url_text)
|
|
207
|
+
print(clean_text)
|
|
208
|
+
# Output: "kunjungi sekarang!"
|
|
209
|
+
|
|
210
|
+
# Menghapus mentions
|
|
211
|
+
mention_text = "Halo @user123 apa kabar?"
|
|
212
|
+
clean_text = remove_mentions(mention_text)
|
|
213
|
+
print(clean_text)
|
|
214
|
+
# Output: "Halo apa kabar?"
|
|
215
|
+
|
|
216
|
+
# Normalisasi slang
|
|
217
|
+
slang_text = "emg siapa yg nanya?"
|
|
218
|
+
normal_text = replace_slang(slang_text)
|
|
219
|
+
print(normal_text)
|
|
220
|
+
# Output: "memang siapa yang bertanya?"
|
|
221
|
+
|
|
222
|
+
# Konversi emoji
|
|
223
|
+
emoji_text = "😀 😂 😍"
|
|
224
|
+
text_result = emoji_to_words(emoji_text)
|
|
225
|
+
print(text_result)
|
|
226
|
+
# Output: "wajah_gembira wajah_tertawa wajah_bercinta"
|
|
227
|
+
|
|
228
|
+
# Koreksi ejaan
|
|
229
|
+
spell_text = "saya mencri informsi"
|
|
230
|
+
corrected = correct_spelling(spell_text)
|
|
231
|
+
print(corrected)
|
|
232
|
+
# Output: "saya mencuri informasi"
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
### 9. 📊 Dataset Loader
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
from nahiarhdNLP.mydatasets import DatasetLoader
|
|
239
|
+
|
|
240
|
+
loader = DatasetLoader()
|
|
241
|
+
|
|
242
|
+
# Load stopwords
|
|
243
|
+
stopwords = loader.load_stopwords_dataset()
|
|
244
|
+
print(f"Jumlah stopwords: {len(stopwords)}")
|
|
245
|
+
|
|
246
|
+
# Load slang dictionary
|
|
247
|
+
slang_dict = loader.load_slang_dataset()
|
|
248
|
+
print(f"Jumlah slang: {len(slang_dict)}")
|
|
249
|
+
|
|
250
|
+
# Load emoji dictionary
|
|
251
|
+
emoji_dict = loader.load_emoji_dataset()
|
|
252
|
+
print(f"Jumlah emoji: {len(emoji_dict)}")
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
### 10. 🔄 Pipeline Custom
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
from nahiarhdNLP.preprocessing import pipeline, replace_word_elongation, replace_slang
|
|
259
|
+
|
|
260
|
+
# Buat pipeline custom
|
|
261
|
+
custom_pipeline = pipeline([
|
|
262
|
+
replace_word_elongation,
|
|
263
|
+
replace_slang
|
|
264
|
+
])
|
|
265
|
+
|
|
266
|
+
# Jalankan pipeline
|
|
267
|
+
text = "Knp emg gk mw makan kenapaaa???"
|
|
268
|
+
result = custom_pipeline(text)
|
|
269
|
+
print(result)
|
|
270
|
+
# Output: "mengapa memang tidak mau makan mengapa???"
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
## ⚙️ Parameter Preprocess
|
|
274
|
+
|
|
275
|
+
Fungsi `preprocess()` memiliki parameter opsional:
|
|
276
|
+
|
|
277
|
+
```python
|
|
278
|
+
result = nahiarhdNLP.preprocessing.preprocess(
|
|
279
|
+
text="Halooo emg siapa yg nanya? 😀",
|
|
280
|
+
remove_html_tags=True, # Hapus HTML tags
|
|
281
|
+
remove_urls=True, # Hapus URL
|
|
282
|
+
remove_stopwords_flag=True, # Hapus stopwords
|
|
283
|
+
replace_slang_flag=True, # Normalisasi slang
|
|
284
|
+
replace_elongation=True, # Atasi word elongation
|
|
285
|
+
convert_emoji=True, # Konversi emoji
|
|
286
|
+
correct_spelling_flag=False,# Koreksi ejaan (lambat)
|
|
287
|
+
stem_text_flag=False, # Stemming
|
|
288
|
+
to_lowercase=True # Lowercase
|
|
289
|
+
)
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
## 🚨 Error Handling
|
|
293
|
+
|
|
294
|
+
```python
|
|
295
|
+
try:
|
|
296
|
+
from nahiarhdNLP import preprocessing
|
|
297
|
+
result = preprocessing.preprocess("test")
|
|
298
|
+
except ImportError:
|
|
299
|
+
print("Package nahiarhdNLP belum terinstall")
|
|
300
|
+
print("Install dengan: pip install nahiarhdNLP")
|
|
301
|
+
except Exception as e:
|
|
302
|
+
print(f"Error: {e}")
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
## 💡 Tips Penggunaan
|
|
306
|
+
|
|
307
|
+
1. **Untuk preprocessing cepat**: Gunakan `preprocess()` dengan parameter default
|
|
308
|
+
2. **Untuk kontrol penuh**: Gunakan kelas individual (`TextCleaner`, `SpellCorrector`, dll)
|
|
309
|
+
3. **Untuk kustomisasi**: Gunakan `pipeline()` dengan fungsi yang diinginkan
|
|
310
|
+
4. **Untuk koreksi ejaan**: Aktifkan `correct_spelling_flag=True` (tapi lebih lambat)
|
|
311
|
+
5. **Untuk stemming**: Aktifkan `stem_text_flag=True` (perlu install Sastrawi)
|
|
312
|
+
6. **Untuk performa optimal**: Dataset akan di-cache otomatis setelah download pertama
|
|
313
|
+
7. **Untuk development**: Gunakan fallback data jika HuggingFace down
|
|
314
|
+
|
|
315
|
+
## ⚡ Performance & Caching
|
|
316
|
+
|
|
317
|
+
nahiarhdNLP menggunakan sistem caching untuk mempercepat loading dataset:
|
|
318
|
+
|
|
319
|
+
- **Pertama kali**: Download dataset dari HuggingFace (3-4 detik)
|
|
320
|
+
- **Kedua kali**: Load dari cache lokal (0.003 detik)
|
|
321
|
+
- **Cache tersimpan** di `~/.nahiarhdNLP/cache/`
|
|
322
|
+
- **Fallback data** jika HuggingFace down
|
|
323
|
+
|
|
324
|
+
```python
|
|
325
|
+
from nahiarhdNLP.mydatasets.loaders import DatasetLoader
|
|
326
|
+
|
|
327
|
+
loader = DatasetLoader()
|
|
328
|
+
|
|
329
|
+
# Clear cache jika perlu
|
|
330
|
+
loader.clear_cache()
|
|
331
|
+
|
|
332
|
+
# Cek info cache
|
|
333
|
+
cache_info = loader.get_cache_info()
|
|
334
|
+
print(cache_info)
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
## 📦 Dependencies
|
|
338
|
+
|
|
339
|
+
Package ini membutuhkan:
|
|
340
|
+
|
|
341
|
+
- `datasets` - untuk load dataset dari HuggingFace
|
|
342
|
+
- `sastrawi` - untuk stemming (opsional)
|
|
343
|
+
- `pandas` - untuk data processing
|
|
344
|
+
- `rich` - untuk output formatting
|
|
345
|
+
- `fsspec` - untuk file system operations
|
|
346
|
+
- `huggingface_hub` - untuk akses HuggingFace
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
mydatasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
mydatasets/loaders.py,sha256=A1indIdVZQgNZHBjkp_P9T8GVIGlWS_y1_Ur-37vOSQ,8728
|
|
3
|
+
nahiarhdnlp-1.0.3.dist-info/licenses/LICENSE,sha256=d_tuMzN2gqUK77K5r3lBPxxZzRuKg6Ku2RHqhHg83F8,1085
|
|
4
|
+
preprocessing/__init__.py,sha256=K6VyeZIfDrqTpGRh78ie1CNU0Xlz-4hFTaxW0bPL92I,1798
|
|
5
|
+
preprocessing/utils.py,sha256=nHQbSF4hyKGn11-sTQ0yeZjfdYZK_EPgl8JH_WMBqQQ,14561
|
|
6
|
+
preprocessing/cleaning/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
preprocessing/cleaning/spell_corrector.py,sha256=dK1L8Zm3bFuvhKlExXWCiZBuY_y5SmHjsLN9upMufD8,1501
|
|
8
|
+
preprocessing/cleaning/text_cleaner.py,sha256=NGjo8I5H30VZPxxnFRfnwlPKJ0echfl_gCEgCu6xLHI,6623
|
|
9
|
+
preprocessing/cleaning/indonesia_spellchecker/README.md,sha256=JCAcskkcf6jFy6nUOEqKATE3jYT8hl-EORCswD0Y_xw,154
|
|
10
|
+
preprocessing/cleaning/indonesia_spellchecker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
preprocessing/cleaning/indonesia_spellchecker/kamus.txt,sha256=7XCedVWD44AWPWzsKD1a4k2KgzujH8O7wEE9O5Nj9AU,303773
|
|
12
|
+
preprocessing/cleaning/indonesia_spellchecker/kata_dasar_kbbi.csv,sha256=9IGJIz6LgzN8YQGeuZAHhTjs9nUsojnNp7wavFXrauw,436281
|
|
13
|
+
preprocessing/cleaning/indonesia_spellchecker/main.py,sha256=YTYsrDfnTwbLShWqRLxqz4TZ2rzUnY8O1ION1ddseUw,1122
|
|
14
|
+
preprocessing/cleaning/indonesia_spellchecker/stemmer.py,sha256=yBby7oz1m9SPdWuCG7hvOFMxAFz5wCSCBuno_dnD-8k,10417
|
|
15
|
+
preprocessing/cleaning/indonesia_spellchecker/teks.txt,sha256=NBHzB_MxVEgvuJDrQTx-EK85aRINhuPM4_DBkm_xuiY,464
|
|
16
|
+
preprocessing/cleaning/indonesia_spellchecker/wordlist.json,sha256=mTKawBjE9dCVUEIeoJ6jk90JdW55gY63zNljsVss6Xc,940731
|
|
17
|
+
preprocessing/linguistic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
+
preprocessing/linguistic/stemmer.py,sha256=feqDW03EFbUtJvV2ryqO596qUD1kNqYO2_u57MhmxcU,630
|
|
19
|
+
preprocessing/linguistic/stopwords.py,sha256=8WCG5GAFAkbOc-nLSyCs0PoptmADWqOsc-TcH1ePAVw,8518
|
|
20
|
+
preprocessing/normalization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
preprocessing/normalization/emoji.py,sha256=yYvosdsYCnbmCm_b1XUbW9VgTz5pk0muMdbXvaE1BWo,6392
|
|
22
|
+
preprocessing/normalization/slang.py,sha256=Buqg9RXUYZC4Q6aMWC8fJ5ArTNAi_Nd1nMJHeOnyiA4,5265
|
|
23
|
+
preprocessing/tokenization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
+
preprocessing/tokenization/tokenizer.py,sha256=bCs_Ro5dKI3Gst6GhkYYyM-oRD9W6fN9jSq1_dk8hW8,207
|
|
25
|
+
nahiarhdnlp-1.0.3.dist-info/METADATA,sha256=rISDRvAy_Kp1RGYs4drcaSnpdeIpQ73Zg96OQpth83o,9342
|
|
26
|
+
nahiarhdnlp-1.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
27
|
+
nahiarhdnlp-1.0.3.dist-info/top_level.txt,sha256=YJvurcBBflLa8dIqcemq1BGB2nl3IuUx_8a9nZH2gPo,25
|
|
28
|
+
nahiarhdnlp-1.0.3.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Raihan Hidayatullah Djunaedi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|