nahiarhdNLP 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nahiarhdnlp-1.0.0/LICENSE +21 -0
- nahiarhdnlp-1.0.0/MANIFEST.in +11 -0
- nahiarhdnlp-1.0.0/PKG-INFO +466 -0
- nahiarhdnlp-1.0.0/README.md +419 -0
- nahiarhdnlp-1.0.0/pyproject.toml +97 -0
- nahiarhdnlp-1.0.0/requirements.txt +7 -0
- nahiarhdnlp-1.0.0/setup.cfg +4 -0
- nahiarhdnlp-1.0.0/setup.py +4 -0
- nahiarhdnlp-1.0.0/src/__init__.py +17 -0
- nahiarhdnlp-1.0.0/src/mydatasets/__init__.py +0 -0
- nahiarhdnlp-1.0.0/src/mydatasets/loaders.py +22 -0
- nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/PKG-INFO +466 -0
- nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/SOURCES.txt +35 -0
- nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/dependency_links.txt +1 -0
- nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/requires.txt +20 -0
- nahiarhdnlp-1.0.0/src/nahiarhdNLP.egg-info/top_level.txt +2 -0
- nahiarhdnlp-1.0.0/src/preprocessing/__init__.py +74 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/__init__.py +0 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/README.md +8 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/__init__.py +0 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/kamus.txt +30871 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/kata_dasar_kbbi.csv +28526 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/main.py +44 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/stemmer.py +280 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/teks.txt +1 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/indonesia_spellchecker/wordlist.json +1 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/spell_corrector.py +44 -0
- nahiarhdnlp-1.0.0/src/preprocessing/cleaning/text_cleaner.py +243 -0
- nahiarhdnlp-1.0.0/src/preprocessing/linguistic/__init__.py +0 -0
- nahiarhdnlp-1.0.0/src/preprocessing/linguistic/stemmer.py +23 -0
- nahiarhdnlp-1.0.0/src/preprocessing/linguistic/stopwords.py +325 -0
- nahiarhdnlp-1.0.0/src/preprocessing/normalization/__init__.py +0 -0
- nahiarhdnlp-1.0.0/src/preprocessing/normalization/emoji.py +217 -0
- nahiarhdnlp-1.0.0/src/preprocessing/normalization/slang.py +175 -0
- nahiarhdnlp-1.0.0/src/preprocessing/tokenization/__init__.py +0 -0
- nahiarhdnlp-1.0.0/src/preprocessing/tokenization/tokenizer.py +10 -0
- nahiarhdnlp-1.0.0/src/preprocessing/utils.py +577 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Raihan Hidayatullah Djunaedi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
include README.md
|
|
2
|
+
include LICENSE
|
|
3
|
+
include requirements.txt
|
|
4
|
+
include pyproject.toml
|
|
5
|
+
recursive-include src *.py
|
|
6
|
+
recursive-include src *.txt
|
|
7
|
+
recursive-include src *.csv
|
|
8
|
+
recursive-include src *.json
|
|
9
|
+
recursive-include src *.md
|
|
10
|
+
recursive-exclude * __pycache__
|
|
11
|
+
recursive-exclude * *.py[co]
|
|
@@ -0,0 +1,466 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nahiarhdNLP
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Advanced Indonesian Natural Language Processing Library
|
|
5
|
+
Author-email: Raihan Hidayatullah Djunaedi <raihanhd.dev@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/nahiarhdNLP/nahiarhdNLP
|
|
8
|
+
Project-URL: Documentation, https://nahiarhdNLP.readthedocs.io
|
|
9
|
+
Project-URL: Repository, https://github.com/nahiarhdNLP/nahiarhdNLP
|
|
10
|
+
Project-URL: Issues, https://github.com/nahiarhdNLP/nahiarhdNLP/issues
|
|
11
|
+
Keywords: nlp,indonesian,natural-language-processing,text-processing,bahasa-indonesia
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
24
|
+
Classifier: Operating System :: OS Independent
|
|
25
|
+
Requires-Python: >=3.8
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: pandas>=1.3.0
|
|
29
|
+
Requires-Dist: fsspec>=2021.10.1
|
|
30
|
+
Requires-Dist: huggingface_hub>=0.10.0
|
|
31
|
+
Requires-Dist: sastrawi>=1.0.1
|
|
32
|
+
Requires-Dist: datasets>=2.0.0
|
|
33
|
+
Requires-Dist: rich>=12.0.0
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
37
|
+
Requires-Dist: black>=22.0.0; extra == "dev"
|
|
38
|
+
Requires-Dist: isort>=5.10.0; extra == "dev"
|
|
39
|
+
Requires-Dist: flake8>=4.0.0; extra == "dev"
|
|
40
|
+
Requires-Dist: mypy>=0.991; extra == "dev"
|
|
41
|
+
Requires-Dist: pre-commit>=2.17.0; extra == "dev"
|
|
42
|
+
Provides-Extra: docs
|
|
43
|
+
Requires-Dist: sphinx>=4.0.0; extra == "docs"
|
|
44
|
+
Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
|
|
45
|
+
Requires-Dist: myst-parser>=0.17.0; extra == "docs"
|
|
46
|
+
Dynamic: license-file
|
|
47
|
+
|
|
48
|
+
# nahiarhdNLP
|
|
49
|
+
|
|
50
|
+
[](https://badge.fury.io/py/nahiarhdNLP)
|
|
51
|
+
[](https://pypi.org/project/nahiarhdNLP/)
|
|
52
|
+
[](https://github.com/nahiarhdNLP/nahiarhdNLP/actions)
|
|
53
|
+
[](https://github.com/nahiarhdNLP/nahiarhdNLP/actions)
|
|
54
|
+
[](https://codecov.io/gh/nahiarhdNLP/nahiarhdNLP)
|
|
55
|
+
[](https://github.com/psf/black)
|
|
56
|
+
|
|
57
|
+
**Bahasa** | [English](README.md)
|
|
58
|
+
|
|
59
|
+
nahiarhdNLP adalah library python advanced yang bertujuan untuk memudahkan proyek NLP anda dengan fitur-fitur yang selalu up-to-date.
|
|
60
|
+
|
|
61
|
+
## Installation
|
|
62
|
+
|
|
63
|
+
nahiarhdNLP dapat diinstall dengan mudah dengan menggunakan pip:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
$ pip install nahiarhdNLP
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Atau clone repository ini:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
git clone https://github.com/nahiarhdNLP/nahiarhdNLP.git
|
|
73
|
+
cd nahiarhdNLP
|
|
74
|
+
pip install -r requirements.txt
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Preprocessing
|
|
78
|
+
|
|
79
|
+
Modul `nahiarhdNLP.preprocessing` menyediakan beberapa fungsi umum untuk menyiapkan dan melakukan transformasi terhadap data teks mentah untuk digunakan pada konteks tertentu.
|
|
80
|
+
|
|
81
|
+
### Generics
|
|
82
|
+
|
|
83
|
+
#### remove_html
|
|
84
|
+
|
|
85
|
+
Menghapus html tag yang terdapat di dalam teks
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
>>> from src.preprocessing import remove_html
|
|
89
|
+
>>> remove_html("website <a href='https://google.com'>google</a>")
|
|
90
|
+
"website google"
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
#### remove_url
|
|
94
|
+
|
|
95
|
+
Menghapus url yang terdapat di dalam teks
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
>>> from src.preprocessing import remove_url
|
|
99
|
+
>>> remove_url("retrieved from https://gist.github.com/gruber/8891611")
|
|
100
|
+
"retrieved from "
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
#### remove_stopwords
|
|
104
|
+
|
|
105
|
+
Stopwords merupakan kata yang diabaikan dalam pemrosesan dan biasanya disimpan di dalam stop lists. Stop list ini berisi daftar kata umum yang mempunyai fungsi tapi tidak mempunyai arti.
|
|
106
|
+
|
|
107
|
+
Menghapus stopwords yang terdapat di dalam teks. List stopwords bahasa Indonesia didapatkan dari dataset HuggingFace.
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
>>> from src.preprocessing import remove_stopwords
|
|
111
|
+
>>> remove_stopwords("siapa yang suruh makan?!!")
|
|
112
|
+
"suruh makan?!!"
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
#### replace_slang
|
|
116
|
+
|
|
117
|
+
Mengganti kata gaul (slang) menjadi kata formal tanpa mengubah makna dari kata tersebut. List kata gaul (slang words) bahasa Indonesian didapatkan dari dataset HuggingFace.
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
>>> from src.preprocessing import replace_slang
|
|
121
|
+
>>> replace_slang("emg siapa yg nanya?")
|
|
122
|
+
"memang siapa yang bertanya?"
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
#### replace_word_elongation
|
|
126
|
+
|
|
127
|
+
Word elongation adalah tindakan untuk menambahkan huruf ke kata, biasanya di akhir kata.
|
|
128
|
+
|
|
129
|
+
Menghandle word elongation:
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
>>> from src.preprocessing import replace_word_elongation
|
|
133
|
+
>>> replace_word_elongation("kenapaaa?")
|
|
134
|
+
"kenapaa?"
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Fungsi Pembersihan Individual
|
|
138
|
+
|
|
139
|
+
#### remove_mentions
|
|
140
|
+
|
|
141
|
+
Menghapus mentions (@username) dari teks.
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
>>> from src.preprocessing import remove_mentions
|
|
145
|
+
>>> remove_mentions("Halo @user123 dan @admin, apa kabar?")
|
|
146
|
+
"Halo dan , apa kabar?"
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
#### remove_hashtags
|
|
150
|
+
|
|
151
|
+
Menghapus hashtags (#tag) dari teks.
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
>>> from src.preprocessing import remove_hashtags
|
|
155
|
+
>>> remove_hashtags("Hari ini #senin #libur #weekend")
|
|
156
|
+
"Hari ini"
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
#### remove_numbers
|
|
160
|
+
|
|
161
|
+
Menghapus angka dari teks.
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
>>> from src.preprocessing import remove_numbers
|
|
165
|
+
>>> remove_numbers("Saya berumur 25 tahun dan punya 3 anak")
|
|
166
|
+
"Saya berumur tahun dan punya anak"
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
#### remove_punctuation
|
|
170
|
+
|
|
171
|
+
Menghapus tanda baca dari teks.
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
>>> from src.preprocessing import remove_punctuation
|
|
175
|
+
>>> remove_punctuation("Halo, apa kabar?! Semoga sehat selalu...")
|
|
176
|
+
"Halo apa kabar Semoga sehat selalu"
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
#### remove_extra_spaces
|
|
180
|
+
|
|
181
|
+
Menghapus spasi berlebih dari teks.
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
>>> from src.preprocessing import remove_extra_spaces
|
|
185
|
+
>>> remove_extra_spaces("Halo dunia yang indah")
|
|
186
|
+
"Halo dunia yang indah"
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
#### remove_special_chars
|
|
190
|
+
|
|
191
|
+
Menghapus karakter khusus yang bukan alfanumerik atau spasi.
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
>>> from src.preprocessing import remove_special_chars
|
|
195
|
+
>>> remove_special_chars("Halo @#$%^&*() dunia!!!")
|
|
196
|
+
"Halo () dunia!!!"
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
#### remove_whitespace
|
|
200
|
+
|
|
201
|
+
Membersihkan karakter whitespace (tab, newline, dll).
|
|
202
|
+
|
|
203
|
+
```python
|
|
204
|
+
>>> from src.preprocessing import remove_whitespace
|
|
205
|
+
>>> remove_whitespace("Halo\n\tdunia\r\nyang indah")
|
|
206
|
+
"Halo dunia yang indah"
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
#### to_lowercase
|
|
210
|
+
|
|
211
|
+
Mengubah teks menjadi huruf kecil.
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
>>> from src.preprocessing import to_lowercase
|
|
215
|
+
>>> to_lowercase("HALO Dunia Yang INDAH")
|
|
216
|
+
"halo dunia yang indah"
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### Emoji
|
|
220
|
+
|
|
221
|
+
Preproses teks yang mengandung emoji.
|
|
222
|
+
|
|
223
|
+
#### emoji_to_words
|
|
224
|
+
|
|
225
|
+
Mengubah emoji yang berada dalam sebuah teks menjadi kata-kata yang sesuai dengan emoji tersebut.
|
|
226
|
+
|
|
227
|
+
```python
|
|
228
|
+
>>> from src.preprocessing import emoji_to_words
|
|
229
|
+
>>> emoji_to_words("emoji 😀😁")
|
|
230
|
+
"emoji wajah_gembira wajah_gembira_dengan_mata_bahagia"
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
#### words_to_emoji
|
|
234
|
+
|
|
235
|
+
Mengubah kata-kata dengan kode emoji menjadi emoji.
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
>>> from src.preprocessing import words_to_emoji
|
|
239
|
+
>>> words_to_emoji("emoji wajah_gembira")
|
|
240
|
+
"emoji 😀"
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
### Pipelining
|
|
244
|
+
|
|
245
|
+
Membuat pipeline dari sequence fungsi preprocessing:
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
>>> from src.preprocessing import pipeline, replace_word_elongation, replace_slang
|
|
249
|
+
>>> pipe = pipeline([replace_word_elongation, replace_slang])
|
|
250
|
+
>>> pipe("Knp emg gk mw makan kenapaaa???")
|
|
251
|
+
"Kenapa memang tidak mau makan kenapa??"
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
### Preprocessing All-in-One
|
|
255
|
+
|
|
256
|
+
Fungsi `preprocess` menyediakan preprocessing lengkap dengan berbagai opsi:
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
>>> from src.preprocessing import preprocess
|
|
260
|
+
>>> preprocess("Halooo emg siapa yg nanya? 😀")
|
|
261
|
+
"halo wajah_gembira"
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
Dengan opsi kustomisasi:
|
|
265
|
+
|
|
266
|
+
```python
|
|
267
|
+
>>> from src.preprocessing import preprocess
|
|
268
|
+
>>> preprocess(
|
|
269
|
+
... "Halooo emg siapa yg nanya? 😀",
|
|
270
|
+
... remove_html_tags=True,
|
|
271
|
+
... remove_urls=True,
|
|
272
|
+
... remove_stopwords_flag=True,
|
|
273
|
+
... replace_slang_flag=True,
|
|
274
|
+
... replace_elongation=True,
|
|
275
|
+
... convert_emoji=True,
|
|
276
|
+
... to_lowercase=True
|
|
277
|
+
... )
|
|
278
|
+
"halo wajah_gembira"
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
## Fungsi Tambahan
|
|
282
|
+
|
|
283
|
+
### Tokenization
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
>>> from src.preprocessing import tokenize
|
|
287
|
+
>>> tokenize("Saya suka makan nasi")
|
|
288
|
+
['Saya', 'suka', 'makan', 'nasi']
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
### Text Cleaning
|
|
292
|
+
|
|
293
|
+
```python
|
|
294
|
+
>>> from src.preprocessing import clean_text
|
|
295
|
+
>>> clean_text("Halooo!!! @user #trending https://example.com 😀")
|
|
296
|
+
"haloo!!"
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
### Spell Correction
|
|
300
|
+
|
|
301
|
+
```python
|
|
302
|
+
>>> from src.preprocessing import correct_spelling
|
|
303
|
+
>>> correct_spelling("sya suka mkn nasi")
|
|
304
|
+
"saya suka makan nasi"
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
### Stemming
|
|
308
|
+
|
|
309
|
+
```python
|
|
310
|
+
>>> from src.preprocessing import stem_text
|
|
311
|
+
>>> stem_text("bermain-main dengan senang")
|
|
312
|
+
"main dengan senang"
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
## Advanced Usage
|
|
316
|
+
|
|
317
|
+
### Menggunakan Kelas Langsung
|
|
318
|
+
|
|
319
|
+
Jika Anda memerlukan kontrol lebih lanjut, Anda dapat menggunakan kelas-kelas secara langsung:
|
|
320
|
+
|
|
321
|
+
```python
|
|
322
|
+
from src.preprocessing import TextCleaner, StopwordRemover, SlangNormalizer
|
|
323
|
+
|
|
324
|
+
# Inisialisasi dengan opsi kustom
|
|
325
|
+
cleaner = TextCleaner(
|
|
326
|
+
remove_urls=True,
|
|
327
|
+
remove_mentions=True,
|
|
328
|
+
remove_hashtags=True,
|
|
329
|
+
lowercase=True
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
stopword_remover = StopwordRemover(language="indonesian")
|
|
333
|
+
slang_normalizer = SlangNormalizer(language="indonesian")
|
|
334
|
+
|
|
335
|
+
# Gunakan
|
|
336
|
+
text = "Halooo @user ini contoh teks!!! https://example.com"
|
|
337
|
+
cleaned = cleaner.clean(text)
|
|
338
|
+
no_stopwords = stopword_remover.remove_stopwords(cleaned)
|
|
339
|
+
formal = slang_normalizer.normalize(no_stopwords)
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
### Pipeline Kustom
|
|
343
|
+
|
|
344
|
+
```python
|
|
345
|
+
from src.preprocessing import pipeline
|
|
346
|
+
from src.preprocessing import (
|
|
347
|
+
remove_html,
|
|
348
|
+
remove_url,
|
|
349
|
+
replace_word_elongation,
|
|
350
|
+
emoji_to_words,
|
|
351
|
+
replace_slang,
|
|
352
|
+
remove_stopwords,
|
|
353
|
+
# Fungsi-fungsi individual
|
|
354
|
+
remove_mentions,
|
|
355
|
+
remove_hashtags,
|
|
356
|
+
remove_numbers,
|
|
357
|
+
remove_punctuation,
|
|
358
|
+
remove_extra_spaces,
|
|
359
|
+
to_lowercase
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
# Buat pipeline kustom
|
|
363
|
+
custom_pipe = pipeline([
|
|
364
|
+
remove_html,
|
|
365
|
+
remove_url,
|
|
366
|
+
remove_mentions,
|
|
367
|
+
remove_hashtags,
|
|
368
|
+
remove_numbers,
|
|
369
|
+
replace_word_elongation,
|
|
370
|
+
emoji_to_words,
|
|
371
|
+
replace_slang,
|
|
372
|
+
remove_stopwords,
|
|
373
|
+
remove_punctuation,
|
|
374
|
+
remove_extra_spaces,
|
|
375
|
+
to_lowercase
|
|
376
|
+
])
|
|
377
|
+
|
|
378
|
+
# Gunakan
|
|
379
|
+
result = custom_pipe("Halooo emg siapa yg nanya? 😀 <a href='#'>link</a> @user #trending 123")
|
|
380
|
+
print(result)
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
### Penggunaan Fungsi Individual
|
|
384
|
+
|
|
385
|
+
```python
|
|
386
|
+
from src.preprocessing import (
|
|
387
|
+
remove_mentions,
|
|
388
|
+
remove_hashtags,
|
|
389
|
+
remove_numbers,
|
|
390
|
+
remove_punctuation,
|
|
391
|
+
remove_extra_spaces,
|
|
392
|
+
to_lowercase
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
# Gunakan fungsi individual
|
|
396
|
+
text = "Halo @user123 #trending! Saya berumur 25 tahun..."
|
|
397
|
+
text = remove_mentions(text) # "Halo #trending! Saya berumur 25 tahun..."
|
|
398
|
+
text = remove_hashtags(text) # "Halo ! Saya berumur 25 tahun..."
|
|
399
|
+
text = remove_numbers(text) # "Halo ! Saya berumur tahun..."
|
|
400
|
+
text = remove_punctuation(text) # "Halo Saya berumur tahun"
|
|
401
|
+
text = remove_extra_spaces(text) # "Halo Saya berumur tahun"
|
|
402
|
+
text = to_lowercase(text) # "halo saya berumur tahun"
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
## Requirements
|
|
406
|
+
|
|
407
|
+
- Python 3.7+
|
|
408
|
+
- datasets
|
|
409
|
+
- requests
|
|
410
|
+
- rich (untuk output yang menarik)
|
|
411
|
+
|
|
412
|
+
Untuk fitur tambahan:
|
|
413
|
+
|
|
414
|
+
- Sastrawi (untuk stemming): `pip install Sastrawi`
|
|
415
|
+
- pyspellchecker (untuk spell correction): `pip install pyspellchecker`
|
|
416
|
+
|
|
417
|
+
## Testing
|
|
418
|
+
|
|
419
|
+
Untuk menjalankan semua test:
|
|
420
|
+
|
|
421
|
+
```bash
|
|
422
|
+
pytest tests/
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
## Demo
|
|
426
|
+
|
|
427
|
+
Untuk melihat demo lengkap library:
|
|
428
|
+
|
|
429
|
+
```bash
|
|
430
|
+
python main.py
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
## Directory Structure
|
|
434
|
+
|
|
435
|
+
```
|
|
436
|
+
nahiarhdNLP/
|
|
437
|
+
├── main.py # Demo aplikasi
|
|
438
|
+
├── requirements.txt # Dependencies
|
|
439
|
+
├── README.md # Dokumentasi
|
|
440
|
+
├── src/
|
|
441
|
+
│ ├── __init__.py # Main module
|
|
442
|
+
│ ├── preprocessing/ # Modul preprocessing
|
|
443
|
+
│ │ ├── __init__.py # Export semua fungsi
|
|
444
|
+
│ │ ├── utils.py # Fungsi wrapper utama
|
|
445
|
+
│ │ ├── cleaning/ # Pembersihan teks
|
|
446
|
+
│ │ ├── normalization/ # Normalisasi teks
|
|
447
|
+
│ │ ├── linguistic/ # Pemrosesan linguistik
|
|
448
|
+
│ │ └── tokenization/ # Tokenisasi
|
|
449
|
+
│ └── mydatasets/ # Dataset loader
|
|
450
|
+
└── tests/ # Unit tests
|
|
451
|
+
```
|
|
452
|
+
|
|
453
|
+
## Kontribusi
|
|
454
|
+
|
|
455
|
+
Kontribusi sangat diterima! Silakan fork repository ini, buat branch baru, dan submit pull request.
|
|
456
|
+
|
|
457
|
+
## License
|
|
458
|
+
|
|
459
|
+
MIT License
|
|
460
|
+
|
|
461
|
+
## Acknowledgments
|
|
462
|
+
|
|
463
|
+
- Dataset stopwords dari HuggingFace
|
|
464
|
+
- Dataset emoji dari HuggingFace
|
|
465
|
+
- Dataset slang dari HuggingFace
|
|
466
|
+
- Sastrawi untuk stemming bahasa Indonesia
|