kreyolib 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kreyolib-0.1.0/LICENSE +28 -0
- kreyolib-0.1.0/MANIFEST.in +1 -0
- kreyolib-0.1.0/PKG-INFO +191 -0
- kreyolib-0.1.0/README.md +152 -0
- kreyolib-0.1.0/pyproject.toml +128 -0
- kreyolib-0.1.0/setup.cfg +4 -0
- kreyolib-0.1.0/src/kreyolib/__init__.py +0 -0
- kreyolib-0.1.0/src/kreyolib/_debug.py +43 -0
- kreyolib-0.1.0/src/kreyolib/tagger/data/firstnames.txt +88035 -0
- kreyolib-0.1.0/src/kreyolib/tagger/data/pos_tagger.joblib +0 -0
- kreyolib-0.1.0/src/kreyolib.egg-info/PKG-INFO +191 -0
- kreyolib-0.1.0/src/kreyolib.egg-info/SOURCES.txt +15 -0
- kreyolib-0.1.0/src/kreyolib.egg-info/dependency_links.txt +1 -0
- kreyolib-0.1.0/src/kreyolib.egg-info/requires.txt +11 -0
- kreyolib-0.1.0/src/kreyolib.egg-info/top_level.txt +1 -0
- kreyolib-0.1.0/tests/test_normalize.py +70 -0
- kreyolib-0.1.0/tests/test_pos_tagging.py +72 -0
kreyolib-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, AyitiDev
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
recursive-include src/kreyolib *.txt *.csv *.joblib
|
kreyolib-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kreyolib
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A software library for Haitian Creole (Kreyòl Ayisyen) natural language processing, text normalization, and localization. --- Yon bibliyotèk lojisyèl pou tretman lang natirèl, nòmalizasyon tèks, ak lokalizasyon an Kreyòl Ayisyen.
|
|
5
|
+
Author-email: AyitiDev <116703678+speedyk-005@users.noreply.github.com>
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/AyitiDev/kreyolib
|
|
8
|
+
Project-URL: Repository, https://github.com/AyitiDev/kreyolib
|
|
9
|
+
Project-URL: Issues, https://github.com/AyitiDev/kreyolib/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/AyitiDev/kreyolib/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: haitian-creole,kreyol,nlp,natural-language-processing,text-processing,tokenization,pos-tagging,linguistics,haiti,ayiti,open-source
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Text Processing
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: regex>=2024.11.6
|
|
29
|
+
Requires-Dist: ftfy<7.0,>=6.2.0
|
|
30
|
+
Requires-Dist: nltk<4.0,>=3.9
|
|
31
|
+
Requires-Dist: joblib<2.0,>=1.3.0
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest>=8.3.5; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest-cov>=6.2.1; extra == "dev"
|
|
35
|
+
Requires-Dist: pre-commit>=4.4.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pydoc-markdown<5.0,>=4.8.0; extra == "dev"
|
|
37
|
+
Requires-Dist: ruff>=0.14.14; extra == "dev"
|
|
38
|
+
Dynamic: license-file
|
|
39
|
+
|
|
40
|
+
<div align="center">
|
|
41
|
+
<img src="https://raw.githubusercontent.com/speedyk-005/kreyolib/main/kreyolib_logo.png" alt="Kreyolib Logo" width="625"/>
|
|
42
|
+
<p><i>"Kreyòl merite zouti tou · Kreyòl deserves tools too"</i></p>
|
|
43
|
+
</div>
|
|
44
|
+
|
|
45
|
+
[](https://www.python.org/downloads/)
|
|
46
|
+
[](https://pypi.org/project/kreyolib)
|
|
47
|
+
[](https://coveralls.io/github/AyitiDev/kreyolib?branch=main)
|
|
48
|
+
[](https://github.com/AyitiDev/kreyolib)
|
|
49
|
+
[](https://github.com/AyitiDev/kreyolib/actions)
|
|
50
|
+
[](https://github.com/AyitiDev/kreyolib/actions/workflows/lint.yml)
|
|
51
|
+
[](https://www.codefactor.io/repository/github/ayitidev/kreyolib)
|
|
52
|
+
[](https://opensource.org/licenses/BSD-3-Clause)
|
|
53
|
+
[](https://github.com/AyitiDev)
|
|
54
|
+
[](https://badges.frapsoft.com/os/v2/open-source.svg)
|
|
55
|
+
[](CONTRIBUTING.md)
|
|
56
|
+
|
|
57
|
+
> If you like this project, a star ⭐️ would mean a lot :)
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## Overview / Apèsi
|
|
62
|
+
|
|
63
|
+
Haitian Creole is spoken by millions of people, but it still lacks many of the language resources and tools available for larger languages. This project aims to build an open-source Ayiti NLP ecosystem focused on creating useful Natural Language Processing tools for Haitian Creole.
|
|
64
|
+
|
|
65
|
+
Most NLP progress has focused on high-resource languages, while Haitian Creole remains underrepresented. Creating better tools for Haitian Creole can help preserve the language, improve accessibility, and allow more Haitian developers and researchers to build AI applications.
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Installation / Enstalasyon
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pip install kreyolib -U
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## Usage / Itilizasyon
|
|
78
|
+
|
|
79
|
+
### Normalization [API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibnormalize)
|
|
80
|
+
|
|
81
|
+
#### Orthography ([API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibnormalizeorthographystandardize_text))
|
|
82
|
+
|
|
83
|
+
Standardizes chat slang, archaic spellings, clitics, and article usage into modern IPN orthography.
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from kreyolib.normalize.orthography import standardize_text
|
|
87
|
+
|
|
88
|
+
# -- Chat slang & abbreviations --
|
|
89
|
+
standardize_text("Bjr! Mw tap tann ou sou ban an, svp cheri ou knn c fèt mwen jodi a.")
|
|
90
|
+
# Bonjou! Mwen tap tann ou sou ban an, silvouplè cheri ou konn se fèt mwen jodi a.
|
|
91
|
+
|
|
92
|
+
# -- Article correction --
|
|
93
|
+
standardize_text("Mwen chita sou ban a. Mwen ap manje bannann la ki te sou tab lan.")
|
|
94
|
+
# Mwen chita sou ban an. Mwen ap manje bannann nan ki te sou tab la.
|
|
95
|
+
|
|
96
|
+
# -- Capitalization fixes --
|
|
97
|
+
standardize_text("mwen renmen bondye")
|
|
98
|
+
# Mwen renmen Bondye
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
With `aggressive=True`, older historical variations and non-standard spellings are folded in as well:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
standardize_text("Nan lé monn mouin té pèdu nan péché; Min Jézu té sové-m.", aggressive=True)
|
|
105
|
+
# Nan le mond mwen te pèdi nan peche; Men Jezi te sovem.
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
#### Contractions ([API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibnormalizecontractionsexpand_contractions))
|
|
109
|
+
|
|
110
|
+
Expands colloquial clitics (`m'ap`, `y'ap`, `n'`) into formal standalone words.
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from kreyolib.normalize.contractions import expand_contractions
|
|
114
|
+
|
|
115
|
+
expand_contractions("M'ap ale lakay nou paske yap tann nou pou n' al travay.")
|
|
116
|
+
# Mwen ap ale lakay nou paske yo ap tann nou pou nou al travay.
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
#### Diacritics ([API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibnormalizediacriticsstrip_diacritics))
|
|
120
|
+
|
|
121
|
+
Removes accent marks — useful for search indexes or legacy systems that expect plain ASCII.
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from kreyolib.normalize.diacritics import strip_diacritics
|
|
125
|
+
|
|
126
|
+
strip_diacritics("Abèy yo ap vole sou òganizasyon an lè yo ale nan fèt la.")
|
|
127
|
+
# Abey yo ap vole sou oganizasyon an le yo ale nan fet la.
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### Advanced Models & Intelligence
|
|
131
|
+
|
|
132
|
+
#### POS Tagger ([API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibtaggerpostag))
|
|
133
|
+
|
|
134
|
+
Built on Universal Dependencies treebanks (Autogramm, Adolphe) with custom preprocessing and French-based proper noun handling. See the [Tagger Source Code](https://github.com/AyitiDev/kreyolib/tree/main/src/kreyolib/tagger) for training details.
|
|
135
|
+
|
|
136
|
+
Tag a raw sentence:
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
from kreyolib.tagger.pos import tag
|
|
140
|
+
|
|
141
|
+
tag("Map vini demen nan maten pou n al travay ansanm.")
|
|
142
|
+
# [('M', 'PRON'), ('ap', 'AUX'), ('vini', 'VERB'), ('demen', 'NOUN'), ('nan', 'ADP'),
|
|
143
|
+
# ('maten', 'NOUN'), ('pou', 'ADP'), ('n', 'PRON'), ('al', 'VERB'), ('travay', 'VERB'),
|
|
144
|
+
# ('ansanm', 'ADV'), ('.', 'PUNCT')]
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Or a pre-tokenized list:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
tag(["Mwen", "rele", "Jan", ",", "e", "mwen", "abite", "Okay", "."])
|
|
151
|
+
# [('Mwen', 'PRON'), ('rele', 'VERB'), ('Jan', 'PROPN'), (',', 'PUNCT'), ('e', 'CCONJ'),
|
|
152
|
+
# ('mwen', 'PRON'), ('abite', 'VERB'), ('Okay', 'NOUN'), ('.', 'PUNCT')]
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
---
|
|
156
|
+
|
|
157
|
+
## Roadmap & Progress / Plan Travay
|
|
158
|
+
|
|
159
|
+
- [ ] **1. Normalization & Preprocessing**
|
|
160
|
+
- [x] Text standardization and modernization
|
|
161
|
+
- [x] Contraction expansion
|
|
162
|
+
- [x] Diacritics remover
|
|
163
|
+
- [ ] Date, number, and text formatters
|
|
164
|
+
- [ ] **2. Corpus & Datasets**
|
|
165
|
+
- [x] Stop words and chat/informal abbreviations
|
|
166
|
+
- [ ] Sentences, words, and chat abbreviation maps
|
|
167
|
+
- [ ] Emoji maps with short Kreyòl description values
|
|
168
|
+
- [ ] **3. Advanced Models & Intelligence**
|
|
169
|
+
- [x] Part-of-Speech (POS) tagging engine
|
|
170
|
+
- [ ] Named Entity Recognition for Haitian entities
|
|
171
|
+
- [ ] Lexicon-based sentiment analysis engine
|
|
172
|
+
- [ ] Next-word predictor
|
|
173
|
+
- [ ] Regex and rule-based intent matching chatbot
|
|
174
|
+
- [ ] **4. Tokenization & Segmentation**
|
|
175
|
+
- [ ] Word tokenizer
|
|
176
|
+
- [ ] Sentence boundary splitter
|
|
177
|
+
- [ ] Social media, tweet, mention, and hashtag tokenization
|
|
178
|
+
- [ ] Subword tokenization via Byte-Pair Encoding and rules
|
|
179
|
+
- [ ] **5. Phonetics & Syntax**
|
|
180
|
+
- [ ] Text-to-phonetics and IPA generation
|
|
181
|
+
- [ ] CV syllabification engine
|
|
182
|
+
- [ ] **6. Spelling & Error Correction**
|
|
183
|
+
- [ ] Levenshtein distance and spell-checking engine
|
|
184
|
+
- [ ] **7. Core Architecture & Pipeline**
|
|
185
|
+
- [ ] Sequential execution pipeline runner
|
|
186
|
+
|
|
187
|
+
### How People Can Contribute
|
|
188
|
+
|
|
189
|
+
For ways to contribute, see [Contributing Guide](https://github.com/AyitiDev/kreyolib/blob/main/CONTRIBUTING.md).
|
|
190
|
+
|
|
191
|
+
> This is a community-driven project to give Haitian Creole a stronger place in the AI and NLP ecosystem.
|
kreyolib-0.1.0/README.md
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/speedyk-005/kreyolib/main/kreyolib_logo.png" alt="Kreyolib Logo" width="625"/>
|
|
3
|
+
<p><i>"Kreyòl merite zouti tou · Kreyòl deserves tools too"</i></p>
|
|
4
|
+
</div>
|
|
5
|
+
|
|
6
|
+
[](https://www.python.org/downloads/)
|
|
7
|
+
[](https://pypi.org/project/kreyolib)
|
|
8
|
+
[](https://coveralls.io/github/AyitiDev/kreyolib?branch=main)
|
|
9
|
+
[](https://github.com/AyitiDev/kreyolib)
|
|
10
|
+
[](https://github.com/AyitiDev/kreyolib/actions)
|
|
11
|
+
[](https://github.com/AyitiDev/kreyolib/actions/workflows/lint.yml)
|
|
12
|
+
[](https://www.codefactor.io/repository/github/ayitidev/kreyolib)
|
|
13
|
+
[](https://opensource.org/licenses/BSD-3-Clause)
|
|
14
|
+
[](https://github.com/AyitiDev)
|
|
15
|
+
[](https://badges.frapsoft.com/os/v2/open-source.svg)
|
|
16
|
+
[](CONTRIBUTING.md)
|
|
17
|
+
|
|
18
|
+
> If you like this project, a star ⭐️ would mean a lot :)
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## Overview / Apèsi
|
|
23
|
+
|
|
24
|
+
Haitian Creole is spoken by millions of people, but it still lacks many of the language resources and tools available for larger languages. This project aims to build an open-source Ayiti NLP ecosystem focused on creating useful Natural Language Processing tools for Haitian Creole.
|
|
25
|
+
|
|
26
|
+
Most NLP progress has focused on high-resource languages, while Haitian Creole remains underrepresented. Creating better tools for Haitian Creole can help preserve the language, improve accessibility, and allow more Haitian developers and researchers to build AI applications.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Installation / Enstalasyon
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install kreyolib -U
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## Usage / Itilizasyon
|
|
39
|
+
|
|
40
|
+
### Normalization [API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibnormalize)
|
|
41
|
+
|
|
42
|
+
#### Orthography ([API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibnormalizeorthographystandardize_text))
|
|
43
|
+
|
|
44
|
+
Standardizes chat slang, archaic spellings, clitics, and article usage into modern IPN orthography.
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from kreyolib.normalize.orthography import standardize_text
|
|
48
|
+
|
|
49
|
+
# -- Chat slang & abbreviations --
|
|
50
|
+
standardize_text("Bjr! Mw tap tann ou sou ban an, svp cheri ou knn c fèt mwen jodi a.")
|
|
51
|
+
# Bonjou! Mwen tap tann ou sou ban an, silvouplè cheri ou konn se fèt mwen jodi a.
|
|
52
|
+
|
|
53
|
+
# -- Article correction --
|
|
54
|
+
standardize_text("Mwen chita sou ban a. Mwen ap manje bannann la ki te sou tab lan.")
|
|
55
|
+
# Mwen chita sou ban an. Mwen ap manje bannann nan ki te sou tab la.
|
|
56
|
+
|
|
57
|
+
# -- Capitalization fixes --
|
|
58
|
+
standardize_text("mwen renmen bondye")
|
|
59
|
+
# Mwen renmen Bondye
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
With `aggressive=True`, older historical variations and non-standard spellings are folded in as well:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
standardize_text("Nan lé monn mouin té pèdu nan péché; Min Jézu té sové-m.", aggressive=True)
|
|
66
|
+
# Nan le mond mwen te pèdi nan peche; Men Jezi te sovem.
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
#### Contractions ([API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibnormalizecontractionsexpand_contractions))
|
|
70
|
+
|
|
71
|
+
Expands colloquial clitics (`m'ap`, `y'ap`, `n'`) into formal standalone words.
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from kreyolib.normalize.contractions import expand_contractions
|
|
75
|
+
|
|
76
|
+
expand_contractions("M'ap ale lakay nou paske yap tann nou pou n' al travay.")
|
|
77
|
+
# Mwen ap ale lakay nou paske yo ap tann nou pou nou al travay.
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
#### Diacritics ([API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibnormalizediacriticsstrip_diacritics))
|
|
81
|
+
|
|
82
|
+
Removes accent marks — useful for search indexes or legacy systems that expect plain ASCII.
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from kreyolib.normalize.diacritics import strip_diacritics
|
|
86
|
+
|
|
87
|
+
strip_diacritics("Abèy yo ap vole sou òganizasyon an lè yo ale nan fèt la.")
|
|
88
|
+
# Abey yo ap vole sou oganizasyon an le yo ale nan fet la.
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Advanced Models & Intelligence
|
|
92
|
+
|
|
93
|
+
#### POS Tagger ([API](https://github.com/AyitiDev/kreyolib/blob/main/API_REFERENCES.md#kreyolibtaggerpostag))
|
|
94
|
+
|
|
95
|
+
Built on Universal Dependencies treebanks (Autogramm, Adolphe) with custom preprocessing and French-based proper noun handling. See the [Tagger Source Code](https://github.com/AyitiDev/kreyolib/tree/main/src/kreyolib/tagger) for training details.
|
|
96
|
+
|
|
97
|
+
Tag a raw sentence:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from kreyolib.tagger.pos import tag
|
|
101
|
+
|
|
102
|
+
tag("Map vini demen nan maten pou n al travay ansanm.")
|
|
103
|
+
# [('M', 'PRON'), ('ap', 'AUX'), ('vini', 'VERB'), ('demen', 'NOUN'), ('nan', 'ADP'),
|
|
104
|
+
# ('maten', 'NOUN'), ('pou', 'ADP'), ('n', 'PRON'), ('al', 'VERB'), ('travay', 'VERB'),
|
|
105
|
+
# ('ansanm', 'ADV'), ('.', 'PUNCT')]
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Or a pre-tokenized list:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
tag(["Mwen", "rele", "Jan", ",", "e", "mwen", "abite", "Okay", "."])
|
|
112
|
+
# [('Mwen', 'PRON'), ('rele', 'VERB'), ('Jan', 'PROPN'), (',', 'PUNCT'), ('e', 'CCONJ'),
|
|
113
|
+
# ('mwen', 'PRON'), ('abite', 'VERB'), ('Okay', 'NOUN'), ('.', 'PUNCT')]
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
## Roadmap & Progress / Plan Travay
|
|
119
|
+
|
|
120
|
+
- [ ] **1. Normalization & Preprocessing**
|
|
121
|
+
- [x] Text standardization and modernization
|
|
122
|
+
- [x] Contraction expansion
|
|
123
|
+
- [x] Diacritics remover
|
|
124
|
+
- [ ] Date, number, and text formatters
|
|
125
|
+
- [ ] **2. Corpus & Datasets**
|
|
126
|
+
- [x] Stop words and chat/informal abbreviations
|
|
127
|
+
- [ ] Sentences, words, and chat abbreviation maps
|
|
128
|
+
- [ ] Emoji maps with short Kreyòl description values
|
|
129
|
+
- [ ] **3. Advanced Models & Intelligence**
|
|
130
|
+
- [x] Part-of-Speech (POS) tagging engine
|
|
131
|
+
- [ ] Named Entity Recognition for Haitian entities
|
|
132
|
+
- [ ] Lexicon-based sentiment analysis engine
|
|
133
|
+
- [ ] Next-word predictor
|
|
134
|
+
- [ ] Regex and rule-based intent matching chatbot
|
|
135
|
+
- [ ] **4. Tokenization & Segmentation**
|
|
136
|
+
- [ ] Word tokenizer
|
|
137
|
+
- [ ] Sentence boundary splitter
|
|
138
|
+
- [ ] Social media, tweet, mention, and hashtag tokenization
|
|
139
|
+
- [ ] Subword tokenization via Byte-Pair Encoding and rules
|
|
140
|
+
- [ ] **5. Phonetics & Syntax**
|
|
141
|
+
- [ ] Text-to-phonetics and IPA generation
|
|
142
|
+
- [ ] CV syllabification engine
|
|
143
|
+
- [ ] **6. Spelling & Error Correction**
|
|
144
|
+
- [ ] Levenshtein distance and spell-checking engine
|
|
145
|
+
- [ ] **7. Core Architecture & Pipeline**
|
|
146
|
+
- [ ] Sequential execution pipeline runner
|
|
147
|
+
|
|
148
|
+
### How People Can Contribute
|
|
149
|
+
|
|
150
|
+
For ways to contribute, see [Contributing Guide](https://github.com/AyitiDev/kreyolib/blob/main/CONTRIBUTING.md).
|
|
151
|
+
|
|
152
|
+
> This is a community-driven project to give Haitian Creole a stronger place in the AI and NLP ecosystem.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "kreyolib"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "A software library for Haitian Creole (Kreyòl Ayisyen) natural language processing, text normalization, and localization. --- Yon bibliyotèk lojisyèl pou tretman lang natirèl, nòmalizasyon tèks, ak lokalizasyon an Kreyòl Ayisyen."
|
|
5
|
+
authors = [
|
|
6
|
+
{ name = "AyitiDev", email = "116703678+speedyk-005@users.noreply.github.com" }
|
|
7
|
+
]
|
|
8
|
+
|
|
9
|
+
license = "BSD-3-Clause"
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Programming Language :: Python :: 3.10",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Programming Language :: Python :: 3.14",
|
|
23
|
+
"Topic :: Text Processing",
|
|
24
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
25
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
26
|
+
"Typing :: Typed"
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
keywords = [
|
|
30
|
+
"haitian-creole", "kreyol", "nlp", "natural-language-processing",
|
|
31
|
+
"text-processing", "tokenization", "pos-tagging", "linguistics",
|
|
32
|
+
"haiti", "ayiti", "open-source"
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
dependencies = [
|
|
36
|
+
"regex>=2024.11.6",
|
|
37
|
+
"ftfy>=6.2.0,<7.0",
|
|
38
|
+
"nltk>=3.9,<4.0",
|
|
39
|
+
"joblib>=1.3.0,<2.0",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[project.optional-dependencies]
|
|
43
|
+
dev = [
|
|
44
|
+
"pytest>=8.3.5",
|
|
45
|
+
"pytest-cov>=6.2.1",
|
|
46
|
+
"pre-commit>=4.4.0",
|
|
47
|
+
"pydoc-markdown>=4.8.0,<5.0",
|
|
48
|
+
"ruff>=0.14.14",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[project.urls]
|
|
52
|
+
Homepage = "https://github.com/AyitiDev/kreyolib"
|
|
53
|
+
Repository = "https://github.com/AyitiDev/kreyolib"
|
|
54
|
+
Issues = "https://github.com/AyitiDev/kreyolib/issues"
|
|
55
|
+
Changelog = "https://github.com/AyitiDev/kreyolib/blob/main/CHANGELOG.md"
|
|
56
|
+
|
|
57
|
+
[build-system]
|
|
58
|
+
requires = [
|
|
59
|
+
"setuptools>=64",
|
|
60
|
+
"wheel",
|
|
61
|
+
]
|
|
62
|
+
build-backend = "setuptools.build_meta"
|
|
63
|
+
|
|
64
|
+
[tool.setuptools]
|
|
65
|
+
package-dir = {"" = "src"}
|
|
66
|
+
packages = ["kreyolib"]
|
|
67
|
+
include-package-data = true
|
|
68
|
+
|
|
69
|
+
[tool.pytest.ini_options]
|
|
70
|
+
minversion = "7.4"
|
|
71
|
+
addopts = "--cov=kreyolib --cov-report=term-missing:skip-covered --durations=10"
|
|
72
|
+
testpaths = ["tests"]
|
|
73
|
+
python_files = "test_*.py"
|
|
74
|
+
|
|
75
|
+
[tool.coverage.run]
|
|
76
|
+
omit = ["*/cli.py"]
|
|
77
|
+
|
|
78
|
+
[tool.ruff]
|
|
79
|
+
target-version = "py310"
|
|
80
|
+
line-length = 95
|
|
81
|
+
|
|
82
|
+
[tool.ruff.lint]
|
|
83
|
+
select = [
|
|
84
|
+
"A", # Built-in shadowing and variable shadowing checks
|
|
85
|
+
"ARG", # Unused function and method arguments
|
|
86
|
+
"B", # Likely bugs and dangerous patterns (flake8-bugbear)
|
|
87
|
+
"BLE", # Blind exception handling (e.g. bare except)
|
|
88
|
+
"C90", # McCabe complexity checks
|
|
89
|
+
"E", # pycodestyle errors (PEP 8 violations)
|
|
90
|
+
"ERA", # Commented-out dead code detection
|
|
91
|
+
"F", # Pyflakes logical and static analysis errors
|
|
92
|
+
"FBT", # Boolean positional argument "traps"
|
|
93
|
+
"FIX", # TODO, FIXME, XXX comment detection
|
|
94
|
+
"FURB", # Refactoring and modernization suggestions (refurb)
|
|
95
|
+
"I", # Import sorting and organization (isort)
|
|
96
|
+
"ICN", # Import alias naming conventions
|
|
97
|
+
"INP", # Missing __init__.py package checks
|
|
98
|
+
"N", # PEP 8 naming conventions (pep8-naming)
|
|
99
|
+
"PERF", # Performance anti-patterns
|
|
100
|
+
"PGH", # Pygrep-hooks rules and miscellaneous checks
|
|
101
|
+
"PIE", # Unnecessary or non-pythonic code patterns
|
|
102
|
+
"PL", # Pylint-derived checks
|
|
103
|
+
"PT", # Pytest style and correctness checks
|
|
104
|
+
"PTH", # Prefer pathlib over os/path utilities
|
|
105
|
+
"Q", # Quote style consistency
|
|
106
|
+
"RET", # Return statement consistency and simplification
|
|
107
|
+
"RSE", # Raise statement improvements
|
|
108
|
+
"RUF", # Ruff-specific rules
|
|
109
|
+
"S", # Security issues and insecure code patterns (bandit)
|
|
110
|
+
"SLF", # Access to private members
|
|
111
|
+
"SIM", # Code simplification opportunities
|
|
112
|
+
"TC", # Type-checking block optimizations
|
|
113
|
+
"TCH", # Move typing-only imports into TYPE_CHECKING blocks
|
|
114
|
+
"TID", # Import restriction and tidiness rules
|
|
115
|
+
"T10", # Debugger breakpoint detection
|
|
116
|
+
"UP", # Python syntax and API modernization (pyupgrade)
|
|
117
|
+
"W", # pycodestyle warnings
|
|
118
|
+
]
|
|
119
|
+
ignore = ["PLW2901", "PLW0603", "COM812", "D203", "D213", "RUF001", "RUF003", "RUF012", "S311", "FURB167", "PLC0206", "PLC0415", "PERF203"]
|
|
120
|
+
|
|
121
|
+
[tool.ruff.lint.per-file-ignores]
|
|
122
|
+
"__init__.py" = ["F403", "I001"]
|
|
123
|
+
"cli.py" = ["PLR0913", "PLR0917", "UP045", "FBT001", "FBT002", "PTH123", "S603"]
|
|
124
|
+
"src/kreyolib/tagger/pos.py" = ["PLR2004"]
|
|
125
|
+
"tests/*" = ["E501", "S101", "PLR2004", "Q0", "ERA001", "SLF001", "PT006", "ARG002", "S101"]
|
|
126
|
+
|
|
127
|
+
[tool.ruff.format]
|
|
128
|
+
exclude = [".git", ".venv", "build", "dist"]
|
kreyolib-0.1.0/setup.cfg
ADDED
|
File without changes
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import difflib
|
|
2
|
+
import os
|
|
3
|
+
import shutil
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
RESET = "\033[0m"
|
|
7
|
+
DIM = "\033[2m"
|
|
8
|
+
GREEN = "\033[92m"
|
|
9
|
+
RED = "\033[91m"
|
|
10
|
+
BG = "\033[48;5;234m"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _colorize_line(line: str, width: int) -> str:
|
|
14
|
+
if line.startswith("+"):
|
|
15
|
+
return f"{GREEN}{BG}{line.ljust(width)}{RESET}"
|
|
16
|
+
if line.startswith("-"):
|
|
17
|
+
return f"{RED}{BG}{line.ljust(width)}{RESET}"
|
|
18
|
+
if line.startswith("@@"):
|
|
19
|
+
return f"{DIM}{BG}{line.ljust(width)}{RESET}"
|
|
20
|
+
return f"{BG}{line.ljust(width)}{RESET}"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def print_rich_diff(text1: str, text2: str) -> None:
|
|
24
|
+
"""Renders a colorized diff of two strings using ANSI escape codes."""
|
|
25
|
+
diff = list(
|
|
26
|
+
difflib.unified_diff(
|
|
27
|
+
[line + "\n" for line in text1.splitlines()],
|
|
28
|
+
[line + "\n" for line in text2.splitlines()],
|
|
29
|
+
fromfile="before",
|
|
30
|
+
tofile="after",
|
|
31
|
+
)
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
if not diff:
|
|
35
|
+
print(f"{DIM}No differences found.{RESET}")
|
|
36
|
+
return
|
|
37
|
+
|
|
38
|
+
if os.environ.get("NO_COLOR") or not sys.stdout.isatty():
|
|
39
|
+
sys.stdout.write("".join(diff))
|
|
40
|
+
return
|
|
41
|
+
|
|
42
|
+
width = shutil.get_terminal_size().columns
|
|
43
|
+
sys.stdout.write("".join(_colorize_line(line.rstrip("\n"), width) + "\n" for line in diff))
|