nahiarhdNLP 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. nahiarhdnlp-1.0.0/LICENSE +21 -0
  2. nahiarhdnlp-1.0.0/MANIFEST.in +11 -0
  3. nahiarhdnlp-1.0.0/PKG-INFO +466 -0
  4. nahiarhdnlp-1.0.0/README.md +419 -0
  5. nahiarhdnlp-1.0.0/pyproject.toml +97 -0
  6. nahiarhdnlp-1.0.0/requirements.txt +7 -0
  7. nahiarhdnlp-1.0.0/setup.cfg +4 -0
  8. nahiarhdnlp-1.0.0/setup.py +4 -0
  9. nahiarhdnlp-1.0.0/src/__init__.py +17 -0
  10. nahiarhdnlp-1.0.0/src/mydatasets/__init__.py +0 -0
  11. nahiarhdnlp-1.0.0/src/mydatasets/loaders.py +22 -0
  12. nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/PKG-INFO +466 -0
  13. nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/SOURCES.txt +35 -0
  14. nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/dependency_links.txt +1 -0
  15. nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/requires.txt +20 -0
  16. nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/top_level.txt +2 -0
  17. nahiarhdnlp-1.0.0/src/preprocessing/__init__.py +74 -0
  18. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/__init__.py +0 -0
  19. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/README.md +8 -0
  20. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/__init__.py +0 -0
  21. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/kamus.txt +30871 -0
  22. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/kata_dasar_kbbi.csv +28526 -0
  23. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/main.py +44 -0
  24. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/stemmer.py +280 -0
  25. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/teks.txt +1 -0
  26. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/wordlist.json +1 -0
  27. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/spell_corrector.py +44 -0
  28. nahiarhdnlp-1.0.0/src/preprocessing/cleaning/text_cleaner.py +243 -0
  29. nahiarhdnlp-1.0.0/src/preprocessing/linguistic/__init__.py +0 -0
  30. nahiarhdnlp-1.0.0/src/preprocessing/linguistic/stemmer.py +23 -0
  31. nahiarhdnlp-1.0.0/src/preprocessing/linguistic/stopwords.py +325 -0
  32. nahiarhdnlp-1.0.0/src/preprocessing/normalization/__init__.py +0 -0
  33. nahiarhdnlp-1.0.0/src/preprocessing/normalization/emoji.py +217 -0
  34. nahiarhdnlp-1.0.0/src/preprocessing/normalization/slang.py +175 -0
  35. nahiarhdnlp-1.0.0/src/preprocessing/tokenization/__init__.py +0 -0
  36. nahiarhdnlp-1.0.0/src/preprocessing/tokenization/tokenizer.py +10 -0
  37. nahiarhdnlp-1.0.0/src/preprocessing/utils.py +577 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Raihan Hidayatullah Djunaedi
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,11 @@
1
+ include README.md
2
+ include LICENSE
3
+ include requirements.txt
4
+ include pyproject.toml
5
+ recursive-include src *.py
6
+ recursive-include src *.txt
7
+ recursive-include src *.csv
8
+ recursive-include src *.json
9
+ recursive-include src *.md
10
+ recursive-exclude * __pycache__
11
+ recursive-exclude * *.py[co]
@@ -0,0 +1,466 @@
1
+ Metadata-Version: 2.4
2
+ Name: nahiarhdNLP
3
+ Version: 1.0.0
4
+ Summary: Advanced Indonesian Natural Language Processing Library
5
+ Author-email: Raihan Hidayatullah Djunaedi <raihanhd.dev@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/nahiarhdNLP/nahiarhdNLP
8
+ Project-URL: Documentation, https://nahiarhdNLP.readthedocs.io
9
+ Project-URL: Repository, https://github.com/nahiarhdNLP/nahiarhdNLP
10
+ Project-URL: Issues, https://github.com/nahiarhdNLP/nahiarhdNLP/issues
11
+ Keywords: nlp,indonesian,natural-language-processing,text-processing,bahasa-indonesia
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.8
18
+ Classifier: Programming Language :: Python :: 3.9
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
+ Classifier: Topic :: Text Processing :: Linguistic
24
+ Classifier: Operating System :: OS Independent
25
+ Requires-Python: >=3.8
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Requires-Dist: pandas>=1.3.0
29
+ Requires-Dist: fsspec>=2021.10.1
30
+ Requires-Dist: huggingface_hub>=0.10.0
31
+ Requires-Dist: sastrawi>=1.0.1
32
+ Requires-Dist: datasets>=2.0.0
33
+ Requires-Dist: rich>=12.0.0
34
+ Provides-Extra: dev
35
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
36
+ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
37
+ Requires-Dist: black>=22.0.0; extra == "dev"
38
+ Requires-Dist: isort>=5.10.0; extra == "dev"
39
+ Requires-Dist: flake8>=4.0.0; extra == "dev"
40
+ Requires-Dist: mypy>=0.991; extra == "dev"
41
+ Requires-Dist: pre-commit>=2.17.0; extra == "dev"
42
+ Provides-Extra: docs
43
+ Requires-Dist: sphinx>=4.0.0; extra == "docs"
44
+ Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
45
+ Requires-Dist: myst-parser>=0.17.0; extra == "docs"
46
+ Dynamic: license-file
47
+
48
+ # nahiarhdNLP
49
+
50
+ [![PyPI version](https://badge.fury.io/py/nahiarhdNLP.svg)](https://badge.fury.io/py/nahiarhdNLP)
51
+ [![Python Version](https://img.shields.io/pypi/pyversions/nahiarhdNLP.svg)](https://pypi.org/project/nahiarhdNLP/)
52
+ [![Test](https://github.com/nahiarhdNLP/nahiarhdNLP/workflows/Test/badge.svg)](https://github.com/nahiarhdNLP/nahiarhdNLP/actions)
53
+ [![Lint](https://github.com/nahiarhdNLP/nahiarhdNLP/workflows/Lint/badge.svg)](https://github.com/nahiarhdNLP/nahiarhdNLP/actions)
54
+ [![codecov](https://codecov.io/gh/nahiarhdNLP/nahiarhdNLP/branch/main/graph/badge.svg)](https://codecov.io/gh/nahiarhdNLP/nahiarhdNLP)
55
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
56
+
57
+ **Bahasa** | [English](README.md)
58
+
59
+ nahiarhdNLP adalah library python advanced yang bertujuan untuk memudahkan proyek NLP anda dengan fitur-fitur yang selalu up-to-date.
60
+
61
+ ## Installation
62
+
63
+ nahiarhdNLP dapat diinstall dengan mudah dengan menggunakan pip:
64
+
65
+ ```bash
66
+ $ pip install nahiarhdNLP
67
+ ```
68
+
69
+ Atau clone repository ini:
70
+
71
+ ```bash
72
+ git clone https://github.com/nahiarhdNLP/nahiarhdNLP.git
73
+ cd nahiarhdNLP
74
+ pip install -r requirements.txt
75
+ ```
76
+
77
+ ## Preprocessing
78
+
79
+ Modul `nahiarhdNLP.preprocessing` menyediakan beberapa fungsi umum untuk menyiapkan dan melakukan transformasi terhadap data teks mentah untuk digunakan pada konteks tertentu.
80
+
81
+ ### Generics
82
+
83
+ #### remove_html
84
+
85
+ Menghapus html tag yang terdapat di dalam teks
86
+
87
+ ```python
88
+ >>> from src.preprocessing import remove_html
89
+ >>> remove_html("website <a href='https://google.com'>google</a>")
90
+ "website google"
91
+ ```
92
+
93
+ #### remove_url
94
+
95
+ Menghapus url yang terdapat di dalam teks
96
+
97
+ ```python
98
+ >>> from src.preprocessing import remove_url
99
+ >>> remove_url("retrieved from https://gist.github.com/gruber/8891611")
100
+ "retrieved from "
101
+ ```
102
+
103
+ #### remove_stopwords
104
+
105
+ Stopwords merupakan kata yang diabaikan dalam pemrosesan dan biasanya disimpan di dalam stop lists. Stop list ini berisi daftar kata umum yang mempunyai fungsi tapi tidak mempunyai arti.
106
+
107
+ Menghapus stopwords yang terdapat di dalam teks. List stopwords bahasa Indonesia didapatkan dari dataset HuggingFace.
108
+
109
+ ```python
110
+ >>> from src.preprocessing import remove_stopwords
111
+ >>> remove_stopwords("siapa yang suruh makan?!!")
112
+ "suruh makan?!!"
113
+ ```
114
+
115
+ #### replace_slang
116
+
117
+ Mengganti kata gaul (slang) menjadi kata formal tanpa mengubah makna dari kata tersebut. List kata gaul (slang words) bahasa Indonesian didapatkan dari dataset HuggingFace.
118
+
119
+ ```python
120
+ >>> from src.preprocessing import replace_slang
121
+ >>> replace_slang("emg siapa yg nanya?")
122
+ "memang siapa yang bertanya?"
123
+ ```
124
+
125
+ #### replace_word_elongation
126
+
127
+ Word elongation adalah tindakan untuk menambahkan huruf ke kata, biasanya di akhir kata.
128
+
129
+ Menghandle word elongation:
130
+
131
+ ```python
132
+ >>> from src.preprocessing import replace_word_elongation
133
+ >>> replace_word_elongation("kenapaaa?")
134
+ "kenapaa?"
135
+ ```
136
+
137
+ ### Fungsi Pembersihan Individual
138
+
139
+ #### remove_mentions
140
+
141
+ Menghapus mentions (@username) dari teks.
142
+
143
+ ```python
144
+ >>> from src.preprocessing import remove_mentions
145
+ >>> remove_mentions("Halo @user123 dan @admin, apa kabar?")
146
+ "Halo dan , apa kabar?"
147
+ ```
148
+
149
+ #### remove_hashtags
150
+
151
+ Menghapus hashtags (#tag) dari teks.
152
+
153
+ ```python
154
+ >>> from src.preprocessing import remove_hashtags
155
+ >>> remove_hashtags("Hari ini #senin #libur #weekend")
156
+ "Hari ini"
157
+ ```
158
+
159
+ #### remove_numbers
160
+
161
+ Menghapus angka dari teks.
162
+
163
+ ```python
164
+ >>> from src.preprocessing import remove_numbers
165
+ >>> remove_numbers("Saya berumur 25 tahun dan punya 3 anak")
166
+ "Saya berumur tahun dan punya anak"
167
+ ```
168
+
169
+ #### remove_punctuation
170
+
171
+ Menghapus tanda baca dari teks.
172
+
173
+ ```python
174
+ >>> from src.preprocessing import remove_punctuation
175
+ >>> remove_punctuation("Halo, apa kabar?! Semoga sehat selalu...")
176
+ "Halo apa kabar Semoga sehat selalu"
177
+ ```
178
+
179
+ #### remove_extra_spaces
180
+
181
+ Menghapus spasi berlebih dari teks.
182
+
183
+ ```python
184
+ >>> from src.preprocessing import remove_extra_spaces
185
+ >>> remove_extra_spaces("Halo dunia yang indah")
186
+ "Halo dunia yang indah"
187
+ ```
188
+
189
+ #### remove_special_chars
190
+
191
+ Menghapus karakter khusus yang bukan alfanumerik atau spasi.
192
+
193
+ ```python
194
+ >>> from src.preprocessing import remove_special_chars
195
+ >>> remove_special_chars("Halo @#$%^&*() dunia!!!")
196
+ "Halo () dunia!!!"
197
+ ```
198
+
199
+ #### remove_whitespace
200
+
201
+ Membersihkan karakter whitespace (tab, newline, dll).
202
+
203
+ ```python
204
+ >>> from src.preprocessing import remove_whitespace
205
+ >>> remove_whitespace("Halo\n\tdunia\r\nyang indah")
206
+ "Halo dunia yang indah"
207
+ ```
208
+
209
+ #### to_lowercase
210
+
211
+ Mengubah teks menjadi huruf kecil.
212
+
213
+ ```python
214
+ >>> from src.preprocessing import to_lowercase
215
+ >>> to_lowercase("HALO Dunia Yang INDAH")
216
+ "halo dunia yang indah"
217
+ ```
218
+
219
+ ### Emoji
220
+
221
+ Preproses teks yang mengandung emoji.
222
+
223
+ #### emoji_to_words
224
+
225
+ Mengubah emoji yang berada dalam sebuah teks menjadi kata-kata yang sesuai dengan emoji tersebut.
226
+
227
+ ```python
228
+ >>> from src.preprocessing import emoji_to_words
229
+ >>> emoji_to_words("emoji 😀😁")
230
+ "emoji wajah_gembira wajah_gembira_dengan_mata_bahagia"
231
+ ```
232
+
233
+ #### words_to_emoji
234
+
235
+ Mengubah kata-kata dengan kode emoji menjadi emoji.
236
+
237
+ ```python
238
+ >>> from src.preprocessing import words_to_emoji
239
+ >>> words_to_emoji("emoji wajah_gembira")
240
+ "emoji 😀"
241
+ ```
242
+
243
+ ### Pipelining
244
+
245
+ Membuat pipeline dari sequence fungsi preprocessing:
246
+
247
+ ```python
248
+ >>> from src.preprocessing import pipeline, replace_word_elongation, replace_slang
249
+ >>> pipe = pipeline([replace_word_elongation, replace_slang])
250
+ >>> pipe("Knp emg gk mw makan kenapaaa???")
251
+ "Kenapa memang tidak mau makan kenapa??"
252
+ ```
253
+
254
+ ### Preprocessing All-in-One
255
+
256
+ Fungsi `preprocess` menyediakan preprocessing lengkap dengan berbagai opsi:
257
+
258
+ ```python
259
+ >>> from src.preprocessing import preprocess
260
+ >>> preprocess("Halooo emg siapa yg nanya? 😀")
261
+ "halo wajah_gembira"
262
+ ```
263
+
264
+ Dengan opsi kustomisasi:
265
+
266
+ ```python
267
+ >>> from src.preprocessing import preprocess
268
+ >>> preprocess(
269
+ ... "Halooo emg siapa yg nanya? 😀",
270
+ ... remove_html_tags=True,
271
+ ... remove_urls=True,
272
+ ... remove_stopwords_flag=True,
273
+ ... replace_slang_flag=True,
274
+ ... replace_elongation=True,
275
+ ... convert_emoji=True,
276
+ ... to_lowercase=True
277
+ ... )
278
+ "halo wajah_gembira"
279
+ ```
280
+
281
+ ## Fungsi Tambahan
282
+
283
+ ### Tokenization
284
+
285
+ ```python
286
+ >>> from src.preprocessing import tokenize
287
+ >>> tokenize("Saya suka makan nasi")
288
+ ['Saya', 'suka', 'makan', 'nasi']
289
+ ```
290
+
291
+ ### Text Cleaning
292
+
293
+ ```python
294
+ >>> from src.preprocessing import clean_text
295
+ >>> clean_text("Halooo!!! @user #trending https://example.com 😀")
296
+ "haloo!!"
297
+ ```
298
+
299
+ ### Spell Correction
300
+
301
+ ```python
302
+ >>> from src.preprocessing import correct_spelling
303
+ >>> correct_spelling("sya suka mkn nasi")
304
+ "saya suka makan nasi"
305
+ ```
306
+
307
+ ### Stemming
308
+
309
+ ```python
310
+ >>> from src.preprocessing import stem_text
311
+ >>> stem_text("bermain-main dengan senang")
312
+ "main dengan senang"
313
+ ```
314
+
315
+ ## Advanced Usage
316
+
317
+ ### Menggunakan Kelas Langsung
318
+
319
+ Jika Anda memerlukan kontrol lebih lanjut, Anda dapat menggunakan kelas-kelas secara langsung:
320
+
321
+ ```python
322
+ from src.preprocessing import TextCleaner, StopwordRemover, SlangNormalizer
323
+
324
+ # Inisialisasi dengan opsi kustom
325
+ cleaner = TextCleaner(
326
+ remove_urls=True,
327
+ remove_mentions=True,
328
+ remove_hashtags=True,
329
+ lowercase=True
330
+ )
331
+
332
+ stopword_remover = StopwordRemover(language="indonesian")
333
+ slang_normalizer = SlangNormalizer(language="indonesian")
334
+
335
+ # Gunakan
336
+ text = "Halooo @user ini contoh teks!!! https://example.com"
337
+ cleaned = cleaner.clean(text)
338
+ no_stopwords = stopword_remover.remove_stopwords(cleaned)
339
+ formal = slang_normalizer.normalize(no_stopwords)
340
+ ```
341
+
342
+ ### Pipeline Kustom
343
+
344
+ ```python
345
+ from src.preprocessing import pipeline
346
+ from src.preprocessing import (
347
+ remove_html,
348
+ remove_url,
349
+ replace_word_elongation,
350
+ emoji_to_words,
351
+ replace_slang,
352
+ remove_stopwords,
353
+ # Fungsi-fungsi individual
354
+ remove_mentions,
355
+ remove_hashtags,
356
+ remove_numbers,
357
+ remove_punctuation,
358
+ remove_extra_spaces,
359
+ to_lowercase
360
+ )
361
+
362
+ # Buat pipeline kustom
363
+ custom_pipe = pipeline([
364
+ remove_html,
365
+ remove_url,
366
+ remove_mentions,
367
+ remove_hashtags,
368
+ remove_numbers,
369
+ replace_word_elongation,
370
+ emoji_to_words,
371
+ replace_slang,
372
+ remove_stopwords,
373
+ remove_punctuation,
374
+ remove_extra_spaces,
375
+ to_lowercase
376
+ ])
377
+
378
+ # Gunakan
379
+ result = custom_pipe("Halooo emg siapa yg nanya? 😀 <a href='#'>link</a> @user #trending 123")
380
+ print(result)
381
+ ```
382
+
383
+ ### Penggunaan Fungsi Individual
384
+
385
+ ```python
386
+ from src.preprocessing import (
387
+ remove_mentions,
388
+ remove_hashtags,
389
+ remove_numbers,
390
+ remove_punctuation,
391
+ remove_extra_spaces,
392
+ to_lowercase
393
+ )
394
+
395
+ # Gunakan fungsi individual
396
+ text = "Halo @user123 #trending! Saya berumur 25 tahun..."
397
+ text = remove_mentions(text) # "Halo #trending! Saya berumur 25 tahun..."
398
+ text = remove_hashtags(text) # "Halo ! Saya berumur 25 tahun..."
399
+ text = remove_numbers(text) # "Halo ! Saya berumur tahun..."
400
+ text = remove_punctuation(text) # "Halo Saya berumur tahun"
401
+ text = remove_extra_spaces(text) # "Halo Saya berumur tahun"
402
+ text = to_lowercase(text) # "halo saya berumur tahun"
403
+ ```
404
+
405
+ ## Requirements
406
+
407
+ - Python 3.7+
408
+ - datasets
409
+ - requests
410
+ - rich (untuk output yang menarik)
411
+
412
+ Untuk fitur tambahan:
413
+
414
+ - Sastrawi (untuk stemming): `pip install Sastrawi`
415
+ - pyspellchecker (untuk spell correction): `pip install pyspellchecker`
416
+
417
+ ## Testing
418
+
419
+ Untuk menjalankan semua test:
420
+
421
+ ```bash
422
+ pytest tests/
423
+ ```
424
+
425
+ ## Demo
426
+
427
+ Untuk melihat demo lengkap library:
428
+
429
+ ```bash
430
+ python main.py
431
+ ```
432
+
433
+ ## Directory Structure
434
+
435
+ ```
436
+ nahiarhdNLP/
437
+ ├── main.py # Demo aplikasi
438
+ ├── requirements.txt # Dependencies
439
+ ├── README.md # Dokumentasi
440
+ ├── src/
441
+ │ ├── __init__.py # Main module
442
+ │ ├── preprocessing/ # Modul preprocessing
443
+ │ │ ├── __init__.py # Export semua fungsi
444
+ │ │ ├── utils.py # Fungsi wrapper utama
445
+ │ │ ├── cleaning/ # Pembersihan teks
446
+ │ │ ├── normalization/ # Normalisasi teks
447
+ │ │ ├── linguistic/ # Pemrosesan linguistik
448
+ │ │ └── tokenization/ # Tokenisasi
449
+ │ └── mydatasets/ # Dataset loader
450
+ └── tests/ # Unit tests
451
+ ```
452
+
453
+ ## Kontribusi
454
+
455
+ Kontribusi sangat diterima! Silakan fork repository ini, buat branch baru, dan submit pull request.
456
+
457
+ ## License
458
+
459
+ MIT License
460
+
461
+ ## Acknowledgments
462
+
463
+ - Dataset stopwords dari HuggingFace
464
+ - Dataset emoji dari HuggingFace
465
+ - Dataset slang dari HuggingFace
466
+ - Sastrawi untuk stemming bahasa Indonesia