nahiarhdNLP 1.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mydatasets/__init__.py ADDED
File without changes
mydatasets/loaders.py ADDED
@@ -0,0 +1,262 @@
1
+ import pickle
2
+ from pathlib import Path
3
+
4
+ from datasets import load_dataset
5
+
6
+
7
+ class DatasetLoader:
8
+ """Loader for Indonesian NLP datasets from HuggingFace with caching."""
9
+
10
+ def __init__(self):
11
+ # Setup cache directory
12
+ self.cache_dir = Path.home() / ".nahiarhdNLP" / "cache"
13
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
14
+
15
+ def _get_cached_data(self, dataset_name: str):
16
+ """Get cached dataset if exists."""
17
+ cache_file = self.cache_dir / f"{dataset_name}.pkl"
18
+ if cache_file.exists():
19
+ try:
20
+ with open(cache_file, "rb") as f:
21
+ return pickle.load(f)
22
+ except Exception:
23
+ # If cache corrupted, remove it
24
+ cache_file.unlink()
25
+ return None
26
+
27
+ def _save_cached_data(self, dataset_name: str, data):
28
+ """Save dataset to cache."""
29
+ cache_file = self.cache_dir / f"{dataset_name}.pkl"
30
+ try:
31
+ with open(cache_file, "wb") as f:
32
+ pickle.dump(data, f)
33
+ except Exception as e:
34
+ print(f"Warning: Could not cache {dataset_name}: {e}")
35
+
36
+ def load_stopwords_dataset(self, language="indonesian"):
37
+ """Load stopwords with caching."""
38
+ cache_key = f"stopwords_{language}"
39
+
40
+ # Try cache first
41
+ cached_data = self._get_cached_data(cache_key)
42
+ if cached_data:
43
+ print(f"Using cached stopwords ({len(cached_data)} words)")
44
+ return cached_data
45
+
46
+ # Load from HuggingFace
47
+ print("Loading stopwords from HuggingFace...")
48
+ try:
49
+ ds = load_dataset("nahiar/indo-stopwords")
50
+ data = [item["stopword"] for item in list(ds["train"])]
51
+
52
+ # Cache the data
53
+ self._save_cached_data(cache_key, data)
54
+ return data
55
+
56
+ except Exception as e:
57
+ print(f"Error loading stopwords: {e}")
58
+ return self._get_fallback_stopwords()
59
+
60
+ def load_slang_dataset(self, language="indonesian"):
61
+ """Load slang dataset with caching."""
62
+ cache_key = f"slang_{language}"
63
+
64
+ # Try cache first
65
+ cached_data = self._get_cached_data(cache_key)
66
+ if cached_data:
67
+ print(f"Using cached slang ({len(cached_data)} entries)")
68
+ return cached_data
69
+
70
+ # Load from HuggingFace
71
+ print("Loading slang from HuggingFace...")
72
+ try:
73
+ ds = load_dataset("nahiar/indonesia-slang")
74
+ data = [
75
+ {"slang": item["slang"], "formal": item["formal"]}
76
+ for item in list(ds["train"])
77
+ ]
78
+
79
+ # Cache the data
80
+ self._save_cached_data(cache_key, data)
81
+ return data
82
+
83
+ except Exception as e:
84
+ print(f"Error loading slang: {e}")
85
+ return self._get_fallback_slang()
86
+
87
+ def load_emoji_dataset(self, language="indonesian"):
88
+ """Load emoji dataset with caching."""
89
+ cache_key = f"emoji_{language}"
90
+
91
+ # Try cache first
92
+ cached_data = self._get_cached_data(cache_key)
93
+ if cached_data:
94
+ print(f"Using cached emoji ({len(cached_data)} entries)")
95
+ return cached_data
96
+
97
+ # Load from HuggingFace
98
+ print("Loading emoji from HuggingFace...")
99
+ try:
100
+ ds = load_dataset("nahiar/indo-emoji-dictionary")
101
+ data = [dict(item) for item in list(ds["train"])]
102
+
103
+ # Cache the data
104
+ self._save_cached_data(cache_key, data)
105
+ return data
106
+
107
+ except Exception as e:
108
+ print(f"Error loading emoji: {e}")
109
+ return self._get_fallback_emoji()
110
+
111
+ def _get_fallback_stopwords(self):
112
+ """Fallback stopwords if HuggingFace fails."""
113
+ return [
114
+ "yang",
115
+ "dan",
116
+ "di",
117
+ "ke",
118
+ "dari",
119
+ "untuk",
120
+ "dengan",
121
+ "ini",
122
+ "itu",
123
+ "atau",
124
+ "juga",
125
+ "bisa",
126
+ "akan",
127
+ "sudah",
128
+ "belum",
129
+ "tidak",
130
+ "bukan",
131
+ "saya",
132
+ "kamu",
133
+ "dia",
134
+ "mereka",
135
+ "kami",
136
+ "kita",
137
+ "anda",
138
+ "beliau",
139
+ ]
140
+
141
+ def _get_fallback_slang(self):
142
+ """Fallback slang if HuggingFace fails."""
143
+ return [
144
+ {"slang": "gw", "formal": "saya"},
145
+ {"slang": "gue", "formal": "saya"},
146
+ {"slang": "lo", "formal": "kamu"},
147
+ {"slang": "lu", "formal": "kamu"},
148
+ {"slang": "yg", "formal": "yang"},
149
+ {"slang": "dgn", "formal": "dengan"},
150
+ {"slang": "utk", "formal": "untuk"},
151
+ {"slang": "dr", "formal": "dari"},
152
+ {"slang": "tp", "formal": "tapi"},
153
+ {"slang": "krn", "formal": "karena"},
154
+ {"slang": "jd", "formal": "jadi"},
155
+ {"slang": "sdh", "formal": "sudah"},
156
+ {"slang": "udh", "formal": "sudah"},
157
+ {"slang": "blm", "formal": "belum"},
158
+ {"slang": "ga", "formal": "tidak"},
159
+ {"slang": "gak", "formal": "tidak"},
160
+ {"slang": "nggak", "formal": "tidak"},
161
+ {"slang": "bgt", "formal": "banget"},
162
+ {"slang": "klo", "formal": "kalau"},
163
+ {"slang": "gimana", "formal": "bagaimana"},
164
+ {"slang": "gmn", "formal": "bagaimana"},
165
+ {"slang": "kenapa", "formal": "mengapa"},
166
+ {"slang": "knp", "formal": "mengapa"},
167
+ {"slang": "makasih", "formal": "terima kasih"},
168
+ {"slang": "mksh", "formal": "terima kasih"},
169
+ {"slang": "org", "formal": "orang"},
170
+ {"slang": "skrg", "formal": "sekarang"},
171
+ {"slang": "trs", "formal": "terus"},
172
+ {"slang": "emg", "formal": "memang"},
173
+ {"slang": "kyk", "formal": "seperti"},
174
+ ]
175
+
176
+ def _get_fallback_emoji(self):
177
+ """Fallback emoji if HuggingFace fails."""
178
+ return [
179
+ {
180
+ "emoji": "😀",
181
+ "name_id": "wajah_gembira",
182
+ "alias": "wajah_gembira",
183
+ "aliases": ["gembira", "senang", "happy"],
184
+ },
185
+ {
186
+ "emoji": "😂",
187
+ "name_id": "wajah_tertawa",
188
+ "alias": "wajah_tertawa",
189
+ "aliases": ["tertawa", "laugh", "ketawa"],
190
+ },
191
+ {
192
+ "emoji": "😍",
193
+ "name_id": "wajah_bercinta",
194
+ "alias": "wajah_bercinta",
195
+ "aliases": ["bercinta", "love", "cinta"],
196
+ },
197
+ {
198
+ "emoji": "😭",
199
+ "name_id": "wajah_menangis",
200
+ "alias": "wajah_menangis",
201
+ "aliases": ["menangis", "cry", "sedih"],
202
+ },
203
+ {
204
+ "emoji": "😡",
205
+ "name_id": "wajah_marah",
206
+ "alias": "wajah_marah",
207
+ "aliases": ["marah", "angry", "kesal"],
208
+ },
209
+ {
210
+ "emoji": "👍",
211
+ "name_id": "jempol_keatas",
212
+ "alias": "jempol_keatas",
213
+ "aliases": ["jempol", "thumbs_up", "bagus"],
214
+ },
215
+ {
216
+ "emoji": "👎",
217
+ "name_id": "jempol_kebawah",
218
+ "alias": "jempol_kebawah",
219
+ "aliases": ["jempol_bawah", "thumbs_down", "jelek"],
220
+ },
221
+ {
222
+ "emoji": "❤️",
223
+ "name_id": "hati_merah",
224
+ "alias": "hati_merah",
225
+ "aliases": ["hati", "love", "cinta"],
226
+ },
227
+ {
228
+ "emoji": "🔥",
229
+ "name_id": "api",
230
+ "alias": "api",
231
+ "aliases": ["panas", "fire", "keren"],
232
+ },
233
+ {
234
+ "emoji": "💯",
235
+ "name_id": "seratus",
236
+ "alias": "seratus",
237
+ "aliases": ["100", "perfect", "sempurna"],
238
+ },
239
+ ]
240
+
241
+ def clear_cache(self):
242
+ """Clear all cached data."""
243
+ import shutil
244
+
245
+ if self.cache_dir.exists():
246
+ shutil.rmtree(self.cache_dir)
247
+ print("Cache cleared!")
248
+
249
+ def get_cache_info(self):
250
+ """Get cache information."""
251
+ if not self.cache_dir.exists():
252
+ return "No cache directory"
253
+
254
+ cache_files = list(self.cache_dir.glob("*.pkl"))
255
+ total_size = sum(f.stat().st_size for f in cache_files)
256
+
257
+ return {
258
+ "cache_dir": str(self.cache_dir),
259
+ "files": len(cache_files),
260
+ "total_size_mb": total_size / 1024 / 1024,
261
+ "files_list": [f.name for f in cache_files],
262
+ }
@@ -0,0 +1,346 @@
1
+ Metadata-Version: 2.4
2
+ Name: nahiarhdNLP
3
+ Version: 1.0.3
4
+ Summary: Advanced Indonesian Natural Language Processing Library
5
+ Author-email: Raihan Hidayatullah Djunaedi <raihanhd.dev@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://example.com
8
+ Project-URL: Documentation, https://example.com
9
+ Project-URL: Repository, https://github.com/raihanhd12/nahiarhdNLP
10
+ Project-URL: Issues, https://github.com/raihanhd12/nahiarhdNLP/issues
11
+ Keywords: nlp,indonesian,natural-language-processing,text-processing,bahasa-indonesia
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.8
18
+ Classifier: Programming Language :: Python :: 3.9
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
+ Classifier: Topic :: Text Processing :: Linguistic
24
+ Classifier: Operating System :: OS Independent
25
+ Requires-Python: >=3.8
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Requires-Dist: pandas>=1.3.0
29
+ Requires-Dist: fsspec>=2021.10.1
30
+ Requires-Dist: huggingface_hub>=0.10.0
31
+ Requires-Dist: sastrawi>=1.0.1
32
+ Requires-Dist: datasets>=2.0.0
33
+ Requires-Dist: rich>=12.0.0
34
+ Provides-Extra: dev
35
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
36
+ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
37
+ Requires-Dist: black>=22.0.0; extra == "dev"
38
+ Requires-Dist: isort>=5.10.0; extra == "dev"
39
+ Requires-Dist: flake8>=4.0.0; extra == "dev"
40
+ Requires-Dist: mypy>=0.991; extra == "dev"
41
+ Requires-Dist: pre-commit>=2.17.0; extra == "dev"
42
+ Provides-Extra: docs
43
+ Requires-Dist: sphinx>=4.0.0; extra == "docs"
44
+ Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
45
+ Requires-Dist: myst-parser>=0.17.0; extra == "docs"
46
+ Dynamic: license-file
47
+
48
+ # nahiarhdNLP - Advanced Indonesian Natural Language Processing Library
49
+
50
+ Advanced Indonesian Natural Language Processing Library dengan fitur preprocessing teks, normalisasi slang, konversi emoji, koreksi ejaan, dan banyak lagi.
51
+
52
+ ## 🚀 Instalasi
53
+
54
+ ```bash
55
+ pip install nahiarhdNLP
56
+ ```
57
+
58
+ ## 📦 Import Library
59
+
60
+ ```python
61
+ # Import package utama
62
+ import nahiarhdNLP
63
+
64
+ # Import module preprocessing
65
+ from nahiarhdNLP import preprocessing
66
+
67
+ # Import module datasets
68
+ from nahiarhdNLP import mydatasets
69
+
70
+ # Atau import fungsi spesifik
71
+ from nahiarhdNLP.preprocessing import preprocess, remove_html, replace_slang
72
+ ```
73
+
74
+ ## Contoh Penggunaan
75
+
76
+ ### 1. 🎯 Fungsi Preprocess All-in-One
77
+
78
+ ```python
79
+ from nahiarhdNLP import preprocessing
80
+
81
+ # Preprocessing lengkap dengan satu fungsi
82
+ teks = "Halooo emg siapa yg nanya? 😀"
83
+ hasil = preprocessing.preprocess(teks)
84
+ print(hasil)
85
+ # Output: "halo wajah_gembira"
86
+ ```
87
+
88
+ ### 2. 🧹 TextCleaner - Membersihkan Teks
89
+
90
+ ```python
91
+ from nahiarhdNLP.preprocessing import TextCleaner
92
+
93
+ cleaner = TextCleaner()
94
+
95
+ # Membersihkan URL
96
+ url_text = "kunjungi https://google.com sekarang!"
97
+ clean_result = cleaner.clean_urls(url_text)
98
+ print(clean_result)
99
+ # Output: "kunjungi sekarang!"
100
+
101
+ # Membersihkan mentions
102
+ mention_text = "Halo @user123 apa kabar?"
103
+ clean_result = cleaner.clean_mentions(mention_text)
104
+ print(clean_result)
105
+ # Output: "Halo apa kabar?"
106
+ ```
107
+
108
+ ### 3. ✏️ SpellCorrector - Koreksi Ejaan
109
+
110
+ ```python
111
+ from nahiarhdNLP.preprocessing import SpellCorrector
112
+
113
+ spell = SpellCorrector()
114
+
115
+ # Koreksi kata
116
+ word = "mencri"
117
+ corrected = spell.correct(word)
118
+ print(corrected)
119
+ # Output: "mencuri"
120
+
121
+ # Koreksi kalimat
122
+ sentence = "saya mencri informsi"
123
+ corrected = spell.correct_sentence(sentence)
124
+ print(corrected)
125
+ # Output: "saya mencuri informasi"
126
+ ```
127
+
128
+ ### 4. 🚫 StopwordRemover - Menghapus Stopwords
129
+
130
+ ```python
131
+ from nahiarhdNLP.preprocessing import StopwordRemover
132
+
133
+ stopword = StopwordRemover()
134
+
135
+ # Menghapus stopwords
136
+ text = "saya suka makan nasi goreng"
137
+ result = stopword.remove_stopwords(text)
138
+ print(result)
139
+ # Output: "suka makan nasi goreng"
140
+ ```
141
+
142
+ ### 5. 🔄 SlangNormalizer - Normalisasi Slang
143
+
144
+ ```python
145
+ from nahiarhdNLP.preprocessing import SlangNormalizer
146
+
147
+ slang = SlangNormalizer()
148
+
149
+ # Normalisasi kata slang
150
+ text = "gw lg di rmh"
151
+ result = slang.normalize(text)
152
+ print(result)
153
+ # Output: "saya lagi di rumah"
154
+ ```
155
+
156
+ ### 6. 😀 EmojiConverter - Konversi Emoji
157
+
158
+ ```python
159
+ from nahiarhdNLP.preprocessing import EmojiConverter
160
+
161
+ emoji = EmojiConverter()
162
+
163
+ # Emoji ke teks
164
+ emoji_text = "😀 😂 😍"
165
+ text_result = emoji.emoji_to_text_convert(emoji_text)
166
+ print(text_result)
167
+ # Output: "wajah_gembira wajah_tertawa wajah_bercinta"
168
+
169
+ # Teks ke emoji
170
+ text = "wajah_gembira"
171
+ emoji_result = emoji.text_to_emoji_convert(text)
172
+ print(emoji_result)
173
+ # Output: "😀"
174
+ ```
175
+
176
+ ### 7. 🔪 Tokenizer - Tokenisasi
177
+
178
+ ```python
179
+ from nahiarhdNLP.preprocessing import Tokenizer
180
+
181
+ tokenizer = Tokenizer()
182
+
183
+ # Tokenisasi teks
184
+ text = "ini contoh tokenisasi"
185
+ tokens = tokenizer.tokenize(text)
186
+ print(tokens)
187
+ # Output: ['ini', 'contoh', 'tokenisasi']
188
+ ```
189
+
190
+ ### 8. 🛠️ Fungsi Individual
191
+
192
+ ```python
193
+ from nahiarhdNLP.preprocessing import (
194
+ remove_html, remove_url, remove_mentions,
195
+ replace_slang, emoji_to_words, correct_spelling
196
+ )
197
+
198
+ # Menghapus HTML
199
+ html_text = "website <a href='https://google.com'>google</a>"
200
+ clean_text = remove_html(html_text)
201
+ print(clean_text)
202
+ # Output: "website google"
203
+
204
+ # Menghapus URL
205
+ url_text = "kunjungi https://google.com sekarang!"
206
+ clean_text = remove_url(url_text)
207
+ print(clean_text)
208
+ # Output: "kunjungi sekarang!"
209
+
210
+ # Menghapus mentions
211
+ mention_text = "Halo @user123 apa kabar?"
212
+ clean_text = remove_mentions(mention_text)
213
+ print(clean_text)
214
+ # Output: "Halo apa kabar?"
215
+
216
+ # Normalisasi slang
217
+ slang_text = "emg siapa yg nanya?"
218
+ normal_text = replace_slang(slang_text)
219
+ print(normal_text)
220
+ # Output: "memang siapa yang bertanya?"
221
+
222
+ # Konversi emoji
223
+ emoji_text = "😀 😂 😍"
224
+ text_result = emoji_to_words(emoji_text)
225
+ print(text_result)
226
+ # Output: "wajah_gembira wajah_tertawa wajah_bercinta"
227
+
228
+ # Koreksi ejaan
229
+ spell_text = "saya mencri informsi"
230
+ corrected = correct_spelling(spell_text)
231
+ print(corrected)
232
+ # Output: "saya mencuri informasi"
233
+ ```
234
+
235
+ ### 9. 📊 Dataset Loader
236
+
237
+ ```python
238
+ from nahiarhdNLP.mydatasets import DatasetLoader
239
+
240
+ loader = DatasetLoader()
241
+
242
+ # Load stopwords
243
+ stopwords = loader.load_stopwords_dataset()
244
+ print(f"Jumlah stopwords: {len(stopwords)}")
245
+
246
+ # Load slang dictionary
247
+ slang_dict = loader.load_slang_dataset()
248
+ print(f"Jumlah slang: {len(slang_dict)}")
249
+
250
+ # Load emoji dictionary
251
+ emoji_dict = loader.load_emoji_dataset()
252
+ print(f"Jumlah emoji: {len(emoji_dict)}")
253
+ ```
254
+
255
+ ### 10. 🔄 Pipeline Custom
256
+
257
+ ```python
258
+ from nahiarhdNLP.preprocessing import pipeline, replace_word_elongation, replace_slang
259
+
260
+ # Buat pipeline custom
261
+ custom_pipeline = pipeline([
262
+ replace_word_elongation,
263
+ replace_slang
264
+ ])
265
+
266
+ # Jalankan pipeline
267
+ text = "Knp emg gk mw makan kenapaaa???"
268
+ result = custom_pipeline(text)
269
+ print(result)
270
+ # Output: "mengapa memang tidak mau makan mengapa???"
271
+ ```
272
+
273
+ ## ⚙️ Parameter Preprocess
274
+
275
+ Fungsi `preprocess()` memiliki parameter opsional:
276
+
277
+ ```python
278
+ result = nahiarhdNLP.preprocessing.preprocess(
279
+ text="Halooo emg siapa yg nanya? 😀",
280
+ remove_html_tags=True, # Hapus HTML tags
281
+ remove_urls=True, # Hapus URL
282
+ remove_stopwords_flag=True, # Hapus stopwords
283
+ replace_slang_flag=True, # Normalisasi slang
284
+ replace_elongation=True, # Atasi word elongation
285
+ convert_emoji=True, # Konversi emoji
286
+ correct_spelling_flag=False,# Koreksi ejaan (lambat)
287
+ stem_text_flag=False, # Stemming
288
+ to_lowercase=True # Lowercase
289
+ )
290
+ ```
291
+
292
+ ## 🚨 Error Handling
293
+
294
+ ```python
295
+ try:
296
+ from nahiarhdNLP import preprocessing
297
+ result = preprocessing.preprocess("test")
298
+ except ImportError:
299
+ print("Package nahiarhdNLP belum terinstall")
300
+ print("Install dengan: pip install nahiarhdNLP")
301
+ except Exception as e:
302
+ print(f"Error: {e}")
303
+ ```
304
+
305
+ ## 💡 Tips Penggunaan
306
+
307
+ 1. **Untuk preprocessing cepat**: Gunakan `preprocess()` dengan parameter default
308
+ 2. **Untuk kontrol penuh**: Gunakan kelas individual (`TextCleaner`, `SpellCorrector`, dll)
309
+ 3. **Untuk kustomisasi**: Gunakan `pipeline()` dengan fungsi yang diinginkan
310
+ 4. **Untuk koreksi ejaan**: Aktifkan `correct_spelling_flag=True` (tapi lebih lambat)
311
+ 5. **Untuk stemming**: Aktifkan `stem_text_flag=True` (perlu install Sastrawi)
312
+ 6. **Untuk performa optimal**: Dataset akan di-cache otomatis setelah download pertama
313
+ 7. **Untuk development**: Gunakan fallback data jika HuggingFace down
314
+
315
+ ## ⚡ Performance & Caching
316
+
317
+ nahiarhdNLP menggunakan sistem caching untuk mempercepat loading dataset:
318
+
319
+ - **Pertama kali**: Download dataset dari HuggingFace (3-4 detik)
320
+ - **Kedua kali**: Load dari cache lokal (0.003 detik)
321
+ - **Cache tersimpan** di `~/.nahiarhdNLP/cache/`
322
+ - **Fallback data** jika HuggingFace down
323
+
324
+ ```python
325
+ from nahiarhdNLP.mydatasets.loaders import DatasetLoader
326
+
327
+ loader = DatasetLoader()
328
+
329
+ # Clear cache jika perlu
330
+ loader.clear_cache()
331
+
332
+ # Cek info cache
333
+ cache_info = loader.get_cache_info()
334
+ print(cache_info)
335
+ ```
336
+
337
+ ## 📦 Dependencies
338
+
339
+ Package ini membutuhkan:
340
+
341
+ - `datasets` - untuk load dataset dari HuggingFace
342
+ - `sastrawi` - untuk stemming (opsional)
343
+ - `pandas` - untuk data processing
344
+ - `rich` - untuk output formatting
345
+ - `fsspec` - untuk file system operations
346
+ - `huggingface_hub` - untuk akses HuggingFace
@@ -0,0 +1,28 @@
1
+ mydatasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ mydatasets/loaders.py,sha256=A1indIdVZQgNZHBjkp_P9T8GVIGlWS_y1_Ur-37vOSQ,8728
3
+ nahiarhdnlp-1.0.3.dist-info/licenses/LICENSE,sha256=d_tuMzN2gqUK77K5r3lBPxxZzRuKg6Ku2RHqhHg83F8,1085
4
+ preprocessing/__init__.py,sha256=K6VyeZIfDrqTpGRh78ie1CNU0Xlz-4hFTaxW0bPL92I,1798
5
+ preprocessing/utils.py,sha256=nHQbSF4hyKGn11-sTQ0yeZjfdYZK_EPgl8JH_WMBqQQ,14561
6
+ preprocessing/cleaning/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
+ preprocessing/cleaning/spell_corrector.py,sha256=dK1L8Zm3bFuvhKlExXWCiZBuY_y5SmHjsLN9upMufD8,1501
8
+ preprocessing/cleaning/text_cleaner.py,sha256=NGjo8I5H30VZPxxnFRfnwlPKJ0echfl_gCEgCu6xLHI,6623
9
+ preprocessing/cleaning/indonesia_spellchecker/README.md,sha256=JCAcskkcf6jFy6nUOEqKATE3jYT8hl-EORCswD0Y_xw,154
10
+ preprocessing/cleaning/indonesia_spellchecker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
+ preprocessing/cleaning/indonesia_spellchecker/kamus.txt,sha256=7XCedVWD44AWPWzsKD1a4k2KgzujH8O7wEE9O5Nj9AU,303773
12
+ preprocessing/cleaning/indonesia_spellchecker/kata_dasar_kbbi.csv,sha256=9IGJIz6LgzN8YQGeuZAHhTjs9nUsojnNp7wavFXrauw,436281
13
+ preprocessing/cleaning/indonesia_spellchecker/main.py,sha256=YTYsrDfnTwbLShWqRLxqz4TZ2rzUnY8O1ION1ddseUw,1122
14
+ preprocessing/cleaning/indonesia_spellchecker/stemmer.py,sha256=yBby7oz1m9SPdWuCG7hvOFMxAFz5wCSCBuno_dnD-8k,10417
15
+ preprocessing/cleaning/indonesia_spellchecker/teks.txt,sha256=NBHzB_MxVEgvuJDrQTx-EK85aRINhuPM4_DBkm_xuiY,464
16
+ preprocessing/cleaning/indonesia_spellchecker/wordlist.json,sha256=mTKawBjE9dCVUEIeoJ6jk90JdW55gY63zNljsVss6Xc,940731
17
+ preprocessing/linguistic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
18
+ preprocessing/linguistic/stemmer.py,sha256=feqDW03EFbUtJvV2ryqO596qUD1kNqYO2_u57MhmxcU,630
19
+ preprocessing/linguistic/stopwords.py,sha256=8WCG5GAFAkbOc-nLSyCs0PoptmADWqOsc-TcH1ePAVw,8518
20
+ preprocessing/normalization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
21
+ preprocessing/normalization/emoji.py,sha256=yYvosdsYCnbmCm_b1XUbW9VgTz5pk0muMdbXvaE1BWo,6392
22
+ preprocessing/normalization/slang.py,sha256=Buqg9RXUYZC4Q6aMWC8fJ5ArTNAi_Nd1nMJHeOnyiA4,5265
23
+ preprocessing/tokenization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
24
+ preprocessing/tokenization/tokenizer.py,sha256=bCs_Ro5dKI3Gst6GhkYYyM-oRD9W6fN9jSq1_dk8hW8,207
25
+ nahiarhdnlp-1.0.3.dist-info/METADATA,sha256=rISDRvAy_Kp1RGYs4drcaSnpdeIpQ73Zg96OQpth83o,9342
26
+ nahiarhdnlp-1.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
27
+ nahiarhdnlp-1.0.3.dist-info/top_level.txt,sha256=YJvurcBBflLa8dIqcemq1BGB2nl3IuUx_8a9nZH2gPo,25
28
+ nahiarhdnlp-1.0.3.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (80.9.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Raihan Hidayatullah Djunaedi
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ mydatasets
2
+ preprocessing