tokmor 1.2.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokmor/__init__.py +77 -0
- tokmor/api.py +194 -0
- tokmor/assets.py +365 -0
- tokmor/base.py +238 -0
- tokmor/brahmic.py +516 -0
- tokmor/cjk.py +497 -0
- tokmor/domain/__init__.py +11 -0
- tokmor/domain/sentiment.py +198 -0
- tokmor/factory.py +394 -0
- tokmor/indic.py +289 -0
- tokmor/inventory.py +51 -0
- tokmor/legacy_api.py +143 -0
- tokmor/lemma_store.py +102 -0
- tokmor/lookup_keys.py +145 -0
- tokmor/models/domain/sentiment/en.json +54 -0
- tokmor/models/domain/sentiment/ko.json +52 -0
- tokmor/models/seg_lexicon/km_wordfreq.pkl +0 -0
- tokmor/models/seg_lexicon/km_wordlist.pkl +0 -0
- tokmor/models/seg_lexicon/lo_wordfreq.pkl +0 -0
- tokmor/models/seg_lexicon/lo_wordlist.pkl +0 -0
- tokmor/models/seg_lexicon/my_wordfreq.pkl +0 -0
- tokmor/models/seg_lexicon/my_wordlist.pkl +0 -0
- tokmor/models/seg_lexicon/th_wordfreq.pkl +0 -0
- tokmor/models/seg_lexicon/th_wordlist.pkl +0 -0
- tokmor/models/seg_lexicon/zh_extra_dict.json +35 -0
- tokmor/models/seg_lexicon/zh_wordfreq.pkl +0 -0
- tokmor/morphology/__init__.py +395 -0
- tokmor/morphology/advanced_base.py +472 -0
- tokmor/morphology/arabic_advanced.py +247 -0
- tokmor/morphology/chinese.py +736 -0
- tokmor/morphology/chinese_advanced.py +425 -0
- tokmor/morphology/english.py +315 -0
- tokmor/morphology/english_advanced.py +560 -0
- tokmor/morphology/french_advanced.py +237 -0
- tokmor/morphology/german_advanced.py +343 -0
- tokmor/morphology/hindi_advanced.py +258 -0
- tokmor/morphology/japanese.py +417 -0
- tokmor/morphology/japanese_advanced.py +589 -0
- tokmor/morphology/korean.py +534 -0
- tokmor/morphology/korean_advanced.py +603 -0
- tokmor/morphology/russian_advanced.py +217 -0
- tokmor/morphology/spanish_advanced.py +226 -0
- tokmor/morphology/templates/__init__.py +32 -0
- tokmor/morphology/templates/arabic_script_template.py +162 -0
- tokmor/morphology/templates/brahmic_template.py +181 -0
- tokmor/morphology/templates/cyrillic_template.py +168 -0
- tokmor/morphology/templates/latin_template.py +235 -0
- tokmor/morphology/templates/other_scripts_template.py +475 -0
- tokmor/morphology/thai_native.py +274 -0
- tokmor/morphology/tier2.py +477 -0
- tokmor/morphology/tier3.py +449 -0
- tokmor/morphology/tier4.py +410 -0
- tokmor/morphology/unified.py +855 -0
- tokmor/morphology/universal_fallback.py +398 -0
- tokmor/ner_prep.py +747 -0
- tokmor/offline.py +89 -0
- tokmor/preprocess.py +80 -0
- tokmor/resources.py +288 -0
- tokmor/routing.py +147 -0
- tokmor/rtl.py +309 -0
- tokmor/schema.py +17 -0
- tokmor/sns_tags.py +281 -0
- tokmor/space_based.py +272 -0
- tokmor/token_quality.py +1185 -0
- tokmor/unified_tokens.py +228 -0
- tokmor-1.2.9.dist-info/METADATA +103 -0
- tokmor-1.2.9.dist-info/RECORD +70 -0
- tokmor-1.2.9.dist-info/WHEEL +5 -0
- tokmor-1.2.9.dist-info/licenses/LICENSE +22 -0
- tokmor-1.2.9.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tier 3 Languages - Regional Languages
|
|
3
|
+
======================================
|
|
4
|
+
|
|
5
|
+
20 languages: no, da, fi, hu, bg, hr, sr, sk, sl, lt, lv, et, ca, eu, gl, cy, ga, is, mt, sq, mk
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .templates.latin_template import LatinScriptAnalyzer
|
|
9
|
+
from .templates.cyrillic_template import CyrillicScriptAnalyzer
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# =============================================================================
|
|
13
|
+
# Norwegian (no)
|
|
14
|
+
# =============================================================================
|
|
15
|
+
class NorwegianAnalyzer(LatinScriptAnalyzer):
|
|
16
|
+
LANG_CODE = "no"
|
|
17
|
+
LANG_NAME = "Norwegian"
|
|
18
|
+
VERB_INFINITIVE_SUFFIXES = ['e']
|
|
19
|
+
NOUN_PLURAL_SUFFIXES = ['er', 'ene', 'a', 'ene']
|
|
20
|
+
|
|
21
|
+
def _build_base_dictionary(self):
|
|
22
|
+
self.function_words = {
|
|
23
|
+
'en': 'DET', 'ei': 'DET', 'et': 'DET', 'den': 'DET', 'det': 'DET', 'de': 'DET',
|
|
24
|
+
'jeg': 'PRON', 'du': 'PRON', 'han': 'PRON', 'hun': 'PRON', 'vi': 'PRON', 'de': 'PRON',
|
|
25
|
+
'i': 'PREP', 'på': 'PREP', 'til': 'PREP', 'fra': 'PREP', 'med': 'PREP', 'av': 'PREP',
|
|
26
|
+
'og': 'CONJ', 'eller': 'CONJ', 'men': 'CONJ', 'at': 'CONJ',
|
|
27
|
+
'ikke': 'NEG', 'ja': 'ADV', 'nei': 'ADV', 'veldig': 'ADV',
|
|
28
|
+
}
|
|
29
|
+
self.irregular_verbs = {
|
|
30
|
+
'er': 'være', 'var': 'være', 'vært': 'være',
|
|
31
|
+
'har': 'ha', 'hadde': 'ha', 'hatt': 'ha',
|
|
32
|
+
'går': 'gå', 'gikk': 'gå', 'gått': 'gå',
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# =============================================================================
|
|
37
|
+
# Danish (da)
|
|
38
|
+
# =============================================================================
|
|
39
|
+
class DanishAnalyzer(LatinScriptAnalyzer):
|
|
40
|
+
LANG_CODE = "da"
|
|
41
|
+
LANG_NAME = "Danish"
|
|
42
|
+
VERB_INFINITIVE_SUFFIXES = ['e']
|
|
43
|
+
NOUN_PLURAL_SUFFIXES = ['er', 'e', 'ene']
|
|
44
|
+
|
|
45
|
+
def _build_base_dictionary(self):
|
|
46
|
+
self.function_words = {
|
|
47
|
+
'en': 'DET', 'et': 'DET', 'den': 'DET', 'det': 'DET', 'de': 'DET',
|
|
48
|
+
'jeg': 'PRON', 'du': 'PRON', 'han': 'PRON', 'hun': 'PRON', 'vi': 'PRON', 'de': 'PRON',
|
|
49
|
+
'i': 'PREP', 'på': 'PREP', 'til': 'PREP', 'fra': 'PREP', 'med': 'PREP', 'af': 'PREP',
|
|
50
|
+
'og': 'CONJ', 'eller': 'CONJ', 'men': 'CONJ', 'at': 'CONJ',
|
|
51
|
+
'ikke': 'NEG', 'ja': 'ADV', 'nej': 'ADV', 'meget': 'ADV',
|
|
52
|
+
}
|
|
53
|
+
self.irregular_verbs = {
|
|
54
|
+
'er': 'være', 'var': 'være', 'været': 'være',
|
|
55
|
+
'har': 'have', 'havde': 'have', 'haft': 'have',
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# =============================================================================
|
|
60
|
+
# Finnish (fi)
|
|
61
|
+
# =============================================================================
|
|
62
|
+
class FinnishAnalyzer(LatinScriptAnalyzer):
|
|
63
|
+
LANG_CODE = "fi"
|
|
64
|
+
LANG_NAME = "Finnish"
|
|
65
|
+
VERB_INFINITIVE_SUFFIXES = ['a', 'ä', 'da', 'dä', 'ta', 'tä', 'la', 'lä', 'na', 'nä', 'ra', 'rä']
|
|
66
|
+
# 동사 인칭 어미 (현재형)
|
|
67
|
+
VERB_PARTICIPLE_SUFFIXES = [
|
|
68
|
+
'n', 't', # 1인칭/2인칭 단수
|
|
69
|
+
'mme', 'tte', # 1인칭/2인칭 복수
|
|
70
|
+
'vat', 'vät', # 3인칭 복수
|
|
71
|
+
'en', 'et', # 부정형
|
|
72
|
+
'nut', 'nyt', 'neet', # 과거분사
|
|
73
|
+
'isi', 'isin', # 조건형
|
|
74
|
+
]
|
|
75
|
+
NOUN_PLURAL_SUFFIXES = ['t']
|
|
76
|
+
|
|
77
|
+
def _build_base_dictionary(self):
|
|
78
|
+
self.function_words = {
|
|
79
|
+
'minä': 'PRON', 'sinä': 'PRON', 'hän': 'PRON', 'me': 'PRON', 'te': 'PRON', 'he': 'PRON',
|
|
80
|
+
'-ssa': 'PSP', '-ssä': 'PSP', '-sta': 'PSP', '-stä': 'PSP', '-lla': 'PSP', '-llä': 'PSP',
|
|
81
|
+
'ja': 'CONJ', 'tai': 'CONJ', 'mutta': 'CONJ', 'että': 'CONJ',
|
|
82
|
+
'ei': 'NEG', 'kyllä': 'ADV', 'hyvin': 'ADV', 'paljon': 'ADV',
|
|
83
|
+
}
|
|
84
|
+
self.irregular_verbs = {
|
|
85
|
+
'on': 'olla', 'olen': 'olla', 'olet': 'olla', 'olemme': 'olla', 'olette': 'olla', 'ovat': 'olla',
|
|
86
|
+
# 일반 동사 활용형
|
|
87
|
+
'menen': 'mennä', 'menet': 'mennä', 'menee': 'mennä', 'menemme': 'mennä', 'menevät': 'mennä',
|
|
88
|
+
'tulen': 'tulla', 'tulet': 'tulla', 'tulee': 'tulla', 'tulemme': 'tulla', 'tulevat': 'tulla',
|
|
89
|
+
'teen': 'tehdä', 'teet': 'tehdä', 'tekee': 'tehdä', 'teemme': 'tehdä', 'tekevät': 'tehdä',
|
|
90
|
+
'näen': 'nähdä', 'näet': 'nähdä', 'näkee': 'nähdä', 'näemme': 'nähdä', 'näkevät': 'nähdä',
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# =============================================================================
|
|
95
|
+
# Hungarian (hu)
|
|
96
|
+
# =============================================================================
|
|
97
|
+
class HungarianAnalyzer(LatinScriptAnalyzer):
|
|
98
|
+
LANG_CODE = "hu"
|
|
99
|
+
LANG_NAME = "Hungarian"
|
|
100
|
+
VERB_INFINITIVE_SUFFIXES = ['ni']
|
|
101
|
+
# 헝가리어 동사 인칭 어미 (부정형)
|
|
102
|
+
VERB_PARTICIPLE_SUFFIXES = [
|
|
103
|
+
'ok', 'ek', 'ök', # 1인칭 단수
|
|
104
|
+
'sz', 'ol', 'el', 'öl', # 2인칭 단수
|
|
105
|
+
'unk', 'ünk', # 1인칭 복수
|
|
106
|
+
'tok', 'tek', 'tök', # 2인칭 복수
|
|
107
|
+
'nak', 'nek', # 3인칭 복수
|
|
108
|
+
]
|
|
109
|
+
NOUN_PLURAL_SUFFIXES = ['k', 'ok', 'ek', 'ök', 'ak']
|
|
110
|
+
|
|
111
|
+
def _build_base_dictionary(self):
|
|
112
|
+
self.function_words = {
|
|
113
|
+
'a': 'DET', 'az': 'DET', 'egy': 'DET',
|
|
114
|
+
'én': 'PRON', 'te': 'PRON', 'ő': 'PRON', 'mi': 'PRON', 'ti': 'PRON', 'ők': 'PRON',
|
|
115
|
+
'-ban': 'PSP', '-ben': 'PSP', '-ra': 'PSP', '-re': 'PSP', '-ból': 'PSP', '-ből': 'PSP',
|
|
116
|
+
'és': 'CONJ', 'vagy': 'CONJ', 'de': 'CONJ', 'hogy': 'CONJ',
|
|
117
|
+
'nem': 'NEG', 'igen': 'ADV', 'nagyon': 'ADV', 'jól': 'ADV',
|
|
118
|
+
'haza': 'ADV', 'itt': 'ADV', 'ott': 'ADV',
|
|
119
|
+
}
|
|
120
|
+
self.irregular_verbs = {
|
|
121
|
+
# van (be)
|
|
122
|
+
'van': 'van', 'vagyok': 'van', 'vagy': 'van', 'vagyunk': 'van', 'vagytok': 'van', 'vannak': 'van',
|
|
123
|
+
'volt': 'van', 'voltam': 'van', 'voltál': 'van', 'voltunk': 'van', 'voltak': 'van',
|
|
124
|
+
# megy (go)
|
|
125
|
+
'megy': 'megy', 'megyek': 'megy', 'mész': 'megy', 'megyünk': 'megy', 'mentek': 'megy', 'mennek': 'megy',
|
|
126
|
+
'ment': 'megy', 'mentem': 'megy', 'mentél': 'megy', 'mentünk': 'megy',
|
|
127
|
+
# jön (come)
|
|
128
|
+
'jön': 'jön', 'jövök': 'jön', 'jössz': 'jön', 'jövünk': 'jön', 'jöttök': 'jön', 'jönnek': 'jön',
|
|
129
|
+
'jött': 'jön', 'jöttem': 'jön', 'jöttél': 'jön', 'jöttünk': 'jön',
|
|
130
|
+
# dolgozik (work)
|
|
131
|
+
'dolgozik': 'dolgozik', 'dolgozom': 'dolgozik', 'dolgozol': 'dolgozik', 'dolgozunk': 'dolgozik',
|
|
132
|
+
'dolgoztok': 'dolgozik', 'dolgoznak': 'dolgozik', 'dolgozott': 'dolgozik',
|
|
133
|
+
# lát (see)
|
|
134
|
+
'lát': 'lát', 'látok': 'lát', 'látsz': 'lát', 'látunk': 'lát', 'láttok': 'lát', 'látnak': 'lát',
|
|
135
|
+
# ad (give)
|
|
136
|
+
'ad': 'ad', 'adok': 'ad', 'adsz': 'ad', 'adunk': 'ad', 'adtok': 'ad', 'adnak': 'ad',
|
|
137
|
+
# csinál (do)
|
|
138
|
+
'csinál': 'csinál', 'csinálok': 'csinál', 'csinálsz': 'csinál', 'csinálunk': 'csinál',
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# =============================================================================
|
|
143
|
+
# Bulgarian (bg)
|
|
144
|
+
# =============================================================================
|
|
145
|
+
class BulgarianAnalyzer(CyrillicScriptAnalyzer):
|
|
146
|
+
LANG_CODE = "bg"
|
|
147
|
+
LANG_NAME = "Bulgarian"
|
|
148
|
+
|
|
149
|
+
def _build_base_dictionary(self):
|
|
150
|
+
self.function_words = {
|
|
151
|
+
'аз': 'PRON', 'ти': 'PRON', 'той': 'PRON', 'тя': 'PRON', 'то': 'PRON',
|
|
152
|
+
'ние': 'PRON', 'вие': 'PRON', 'те': 'PRON',
|
|
153
|
+
'в': 'PREP', 'на': 'PREP', 'с': 'PREP', 'от': 'PREP', 'за': 'PREP', 'до': 'PREP',
|
|
154
|
+
'и': 'CONJ', 'или': 'CONJ', 'но': 'CONJ', 'че': 'CONJ',
|
|
155
|
+
'не': 'NEG', 'да': 'ADV', 'много': 'ADV', 'добре': 'ADV',
|
|
156
|
+
}
|
|
157
|
+
self.irregular_verbs = {
|
|
158
|
+
'съм': 'съм', 'си': 'съм', 'е': 'съм', 'сме': 'съм', 'сте': 'съм', 'са': 'съм',
|
|
159
|
+
'бях': 'съм', 'беше': 'съм', 'бяха': 'съм',
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# =============================================================================
|
|
164
|
+
# Croatian (hr)
|
|
165
|
+
# =============================================================================
|
|
166
|
+
class CroatianAnalyzer(LatinScriptAnalyzer):
|
|
167
|
+
LANG_CODE = "hr"
|
|
168
|
+
LANG_NAME = "Croatian"
|
|
169
|
+
VERB_INFINITIVE_SUFFIXES = ['ti', 'ći']
|
|
170
|
+
|
|
171
|
+
def _build_base_dictionary(self):
|
|
172
|
+
self.function_words = {
|
|
173
|
+
'ja': 'PRON', 'ti': 'PRON', 'on': 'PRON', 'ona': 'PRON', 'ono': 'PRON',
|
|
174
|
+
'mi': 'PRON', 'vi': 'PRON', 'oni': 'PRON',
|
|
175
|
+
'u': 'PREP', 'na': 'PREP', 's': 'PREP', 'iz': 'PREP', 'za': 'PREP', 'od': 'PREP',
|
|
176
|
+
'i': 'CONJ', 'ili': 'CONJ', 'ali': 'CONJ', 'da': 'CONJ',
|
|
177
|
+
'ne': 'NEG', 'da': 'ADV', 'vrlo': 'ADV', 'dobro': 'ADV',
|
|
178
|
+
}
|
|
179
|
+
self.irregular_verbs = {
|
|
180
|
+
'sam': 'biti', 'si': 'biti', 'je': 'biti', 'smo': 'biti', 'ste': 'biti', 'su': 'biti',
|
|
181
|
+
'bio': 'biti', 'bila': 'biti', 'bilo': 'biti',
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# =============================================================================
|
|
186
|
+
# Serbian (sr) - Cyrillic
|
|
187
|
+
# =============================================================================
|
|
188
|
+
class SerbianAnalyzer(CyrillicScriptAnalyzer):
|
|
189
|
+
LANG_CODE = "sr"
|
|
190
|
+
LANG_NAME = "Serbian"
|
|
191
|
+
VERB_INFINITIVE_SUFFIX = 'ти'
|
|
192
|
+
|
|
193
|
+
def _build_base_dictionary(self):
|
|
194
|
+
self.function_words = {
|
|
195
|
+
'ја': 'PRON', 'ти': 'PRON', 'он': 'PRON', 'она': 'PRON', 'оно': 'PRON',
|
|
196
|
+
'ми': 'PRON', 'ви': 'PRON', 'они': 'PRON',
|
|
197
|
+
'у': 'PREP', 'на': 'PREP', 'с': 'PREP', 'из': 'PREP', 'за': 'PREP', 'од': 'PREP',
|
|
198
|
+
'и': 'CONJ', 'или': 'CONJ', 'али': 'CONJ', 'да': 'CONJ',
|
|
199
|
+
'не': 'NEG', 'да': 'ADV', 'врло': 'ADV', 'добро': 'ADV',
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
# =============================================================================
|
|
204
|
+
# Slovak (sk)
|
|
205
|
+
# =============================================================================
|
|
206
|
+
class SlovakAnalyzer(LatinScriptAnalyzer):
|
|
207
|
+
LANG_CODE = "sk"
|
|
208
|
+
LANG_NAME = "Slovak"
|
|
209
|
+
VERB_INFINITIVE_SUFFIXES = ['ť', 'sť']
|
|
210
|
+
|
|
211
|
+
def _build_base_dictionary(self):
|
|
212
|
+
self.function_words = {
|
|
213
|
+
'ja': 'PRON', 'ty': 'PRON', 'on': 'PRON', 'ona': 'PRON', 'ono': 'PRON',
|
|
214
|
+
'my': 'PRON', 'vy': 'PRON', 'oni': 'PRON',
|
|
215
|
+
'v': 'PREP', 'na': 'PREP', 's': 'PREP', 'z': 'PREP', 'do': 'PREP', 'od': 'PREP',
|
|
216
|
+
'a': 'CONJ', 'alebo': 'CONJ', 'ale': 'CONJ', 'že': 'CONJ',
|
|
217
|
+
'nie': 'NEG', 'áno': 'ADV', 'veľmi': 'ADV', 'dobre': 'ADV',
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# =============================================================================
|
|
222
|
+
# Slovenian (sl)
|
|
223
|
+
# =============================================================================
|
|
224
|
+
class SlovenianAnalyzer(LatinScriptAnalyzer):
|
|
225
|
+
LANG_CODE = "sl"
|
|
226
|
+
LANG_NAME = "Slovenian"
|
|
227
|
+
VERB_INFINITIVE_SUFFIXES = ['ti', 'či']
|
|
228
|
+
|
|
229
|
+
def _build_base_dictionary(self):
|
|
230
|
+
self.function_words = {
|
|
231
|
+
'jaz': 'PRON', 'ti': 'PRON', 'on': 'PRON', 'ona': 'PRON', 'ono': 'PRON',
|
|
232
|
+
'mi': 'PRON', 'vi': 'PRON', 'oni': 'PRON',
|
|
233
|
+
'v': 'PREP', 'na': 'PREP', 's': 'PREP', 'iz': 'PREP', 'za': 'PREP', 'od': 'PREP',
|
|
234
|
+
'in': 'CONJ', 'ali': 'CONJ', 'ampak': 'CONJ', 'da': 'CONJ',
|
|
235
|
+
'ne': 'NEG', 'ja': 'ADV', 'zelo': 'ADV', 'dobro': 'ADV',
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# =============================================================================
|
|
240
|
+
# Lithuanian (lt)
|
|
241
|
+
# =============================================================================
|
|
242
|
+
class LithuanianAnalyzer(LatinScriptAnalyzer):
|
|
243
|
+
LANG_CODE = "lt"
|
|
244
|
+
LANG_NAME = "Lithuanian"
|
|
245
|
+
VERB_INFINITIVE_SUFFIXES = ['ti']
|
|
246
|
+
|
|
247
|
+
def _build_base_dictionary(self):
|
|
248
|
+
self.function_words = {
|
|
249
|
+
'aš': 'PRON', 'tu': 'PRON', 'jis': 'PRON', 'ji': 'PRON',
|
|
250
|
+
'mes': 'PRON', 'jūs': 'PRON', 'jie': 'PRON', 'jos': 'PRON',
|
|
251
|
+
'ir': 'CONJ', 'arba': 'CONJ', 'bet': 'CONJ', 'kad': 'CONJ',
|
|
252
|
+
'ne': 'NEG', 'taip': 'ADV', 'labai': 'ADV', 'gerai': 'ADV',
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
# =============================================================================
|
|
257
|
+
# Latvian (lv)
|
|
258
|
+
# =============================================================================
|
|
259
|
+
class LatvianAnalyzer(LatinScriptAnalyzer):
|
|
260
|
+
LANG_CODE = "lv"
|
|
261
|
+
LANG_NAME = "Latvian"
|
|
262
|
+
VERB_INFINITIVE_SUFFIXES = ['t']
|
|
263
|
+
|
|
264
|
+
def _build_base_dictionary(self):
|
|
265
|
+
self.function_words = {
|
|
266
|
+
'es': 'PRON', 'tu': 'PRON', 'viņš': 'PRON', 'viņa': 'PRON',
|
|
267
|
+
'mēs': 'PRON', 'jūs': 'PRON', 'viņi': 'PRON',
|
|
268
|
+
'un': 'CONJ', 'vai': 'CONJ', 'bet': 'CONJ', 'ka': 'CONJ',
|
|
269
|
+
'ne': 'NEG', 'jā': 'ADV', 'ļoti': 'ADV', 'labi': 'ADV',
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
# =============================================================================
|
|
274
|
+
# Estonian (et)
|
|
275
|
+
# =============================================================================
|
|
276
|
+
class EstonianAnalyzer(LatinScriptAnalyzer):
|
|
277
|
+
LANG_CODE = "et"
|
|
278
|
+
LANG_NAME = "Estonian"
|
|
279
|
+
VERB_INFINITIVE_SUFFIXES = ['da', 'ta', 'ma']
|
|
280
|
+
|
|
281
|
+
def _build_base_dictionary(self):
|
|
282
|
+
self.function_words = {
|
|
283
|
+
'mina': 'PRON', 'sina': 'PRON', 'tema': 'PRON',
|
|
284
|
+
'meie': 'PRON', 'teie': 'PRON', 'nemad': 'PRON',
|
|
285
|
+
'ja': 'CONJ', 'või': 'CONJ', 'aga': 'CONJ', 'et': 'CONJ',
|
|
286
|
+
'ei': 'NEG', 'jah': 'ADV', 'väga': 'ADV', 'hästi': 'ADV',
|
|
287
|
+
}
|
|
288
|
+
self.irregular_verbs = {
|
|
289
|
+
'on': 'olema', 'olen': 'olema', 'oled': 'olema', 'oleme': 'olema', 'olete': 'olema',
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
# =============================================================================
|
|
294
|
+
# Catalan (ca)
|
|
295
|
+
# =============================================================================
|
|
296
|
+
class CatalanAnalyzer(LatinScriptAnalyzer):
|
|
297
|
+
LANG_CODE = "ca"
|
|
298
|
+
LANG_NAME = "Catalan"
|
|
299
|
+
VERB_INFINITIVE_SUFFIXES = ['ar', 'er', 're', 'ir']
|
|
300
|
+
|
|
301
|
+
def _build_base_dictionary(self):
|
|
302
|
+
self.function_words = {
|
|
303
|
+
'el': 'DET', 'la': 'DET', 'els': 'DET', 'les': 'DET', 'un': 'DET', 'una': 'DET',
|
|
304
|
+
'jo': 'PRON', 'tu': 'PRON', 'ell': 'PRON', 'ella': 'PRON', 'nosaltres': 'PRON', 'ells': 'PRON',
|
|
305
|
+
'i': 'CONJ', 'o': 'CONJ', 'però': 'CONJ', 'que': 'CONJ',
|
|
306
|
+
'no': 'NEG', 'sí': 'ADV', 'molt': 'ADV', 'bé': 'ADV',
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
# =============================================================================
|
|
311
|
+
# Basque (eu)
|
|
312
|
+
# =============================================================================
|
|
313
|
+
class BasqueAnalyzer(LatinScriptAnalyzer):
|
|
314
|
+
LANG_CODE = "eu"
|
|
315
|
+
LANG_NAME = "Basque"
|
|
316
|
+
|
|
317
|
+
def _build_base_dictionary(self):
|
|
318
|
+
self.function_words = {
|
|
319
|
+
'ni': 'PRON', 'zu': 'PRON', 'hura': 'PRON', 'gu': 'PRON', 'zuek': 'PRON', 'haiek': 'PRON',
|
|
320
|
+
'eta': 'CONJ', 'edo': 'CONJ', 'baina': 'CONJ',
|
|
321
|
+
'ez': 'NEG', 'bai': 'ADV', 'oso': 'ADV', 'ongi': 'ADV',
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
# =============================================================================
|
|
326
|
+
# Galician (gl)
|
|
327
|
+
# =============================================================================
|
|
328
|
+
class GalicianAnalyzer(LatinScriptAnalyzer):
|
|
329
|
+
LANG_CODE = "gl"
|
|
330
|
+
LANG_NAME = "Galician"
|
|
331
|
+
VERB_INFINITIVE_SUFFIXES = ['ar', 'er', 'ir']
|
|
332
|
+
|
|
333
|
+
def _build_base_dictionary(self):
|
|
334
|
+
self.function_words = {
|
|
335
|
+
'o': 'DET', 'a': 'DET', 'os': 'DET', 'as': 'DET', 'un': 'DET', 'unha': 'DET',
|
|
336
|
+
'eu': 'PRON', 'ti': 'PRON', 'el': 'PRON', 'ela': 'PRON', 'nós': 'PRON', 'eles': 'PRON',
|
|
337
|
+
'e': 'CONJ', 'ou': 'CONJ', 'pero': 'CONJ', 'que': 'CONJ',
|
|
338
|
+
'non': 'NEG', 'si': 'ADV', 'moi': 'ADV', 'ben': 'ADV',
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
# =============================================================================
|
|
343
|
+
# Welsh (cy)
|
|
344
|
+
# =============================================================================
|
|
345
|
+
class WelshAnalyzer(LatinScriptAnalyzer):
|
|
346
|
+
LANG_CODE = "cy"
|
|
347
|
+
LANG_NAME = "Welsh"
|
|
348
|
+
|
|
349
|
+
def _build_base_dictionary(self):
|
|
350
|
+
self.function_words = {
|
|
351
|
+
'y': 'DET', 'yr': 'DET', "'r": 'DET',
|
|
352
|
+
'fi': 'PRON', 'ti': 'PRON', 'fe': 'PRON', 'hi': 'PRON', 'ni': 'PRON', 'nhw': 'PRON',
|
|
353
|
+
'a': 'CONJ', 'neu': 'CONJ', 'ond': 'CONJ', 'bod': 'CONJ',
|
|
354
|
+
'ddim': 'NEG', 'ie': 'ADV', 'na': 'ADV', 'iawn': 'ADV',
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# =============================================================================
|
|
359
|
+
# Irish (ga)
|
|
360
|
+
# =============================================================================
|
|
361
|
+
class IrishAnalyzer(LatinScriptAnalyzer):
|
|
362
|
+
LANG_CODE = "ga"
|
|
363
|
+
LANG_NAME = "Irish"
|
|
364
|
+
|
|
365
|
+
def _build_base_dictionary(self):
|
|
366
|
+
self.function_words = {
|
|
367
|
+
'an': 'DET', 'na': 'DET',
|
|
368
|
+
'mé': 'PRON', 'tú': 'PRON', 'sé': 'PRON', 'sí': 'PRON', 'muid': 'PRON', 'siad': 'PRON',
|
|
369
|
+
'agus': 'CONJ', 'nó': 'CONJ', 'ach': 'CONJ', 'go': 'CONJ',
|
|
370
|
+
'ní': 'NEG', 'níl': 'NEG', 'sea': 'ADV', 'an-': 'ADV',
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
# =============================================================================
|
|
375
|
+
# Icelandic (is)
|
|
376
|
+
# =============================================================================
|
|
377
|
+
class IcelandicAnalyzer(LatinScriptAnalyzer):
|
|
378
|
+
LANG_CODE = "is"
|
|
379
|
+
LANG_NAME = "Icelandic"
|
|
380
|
+
VERB_INFINITIVE_SUFFIXES = ['a']
|
|
381
|
+
|
|
382
|
+
def _build_base_dictionary(self):
|
|
383
|
+
self.function_words = {
|
|
384
|
+
'ég': 'PRON', 'þú': 'PRON', 'hann': 'PRON', 'hún': 'PRON', 'við': 'PRON', 'þeir': 'PRON',
|
|
385
|
+
'og': 'CONJ', 'eða': 'CONJ', 'en': 'CONJ', 'að': 'CONJ',
|
|
386
|
+
'ekki': 'NEG', 'já': 'ADV', 'mjög': 'ADV', 'vel': 'ADV',
|
|
387
|
+
}
|
|
388
|
+
self.irregular_verbs = {
|
|
389
|
+
'er': 'vera', 'ert': 'vera', 'erum': 'vera', 'eruð': 'vera', 'eru': 'vera',
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
# =============================================================================
|
|
394
|
+
# Maltese (mt)
|
|
395
|
+
# =============================================================================
|
|
396
|
+
class MalteseAnalyzer(LatinScriptAnalyzer):
|
|
397
|
+
LANG_CODE = "mt"
|
|
398
|
+
LANG_NAME = "Maltese"
|
|
399
|
+
|
|
400
|
+
def _build_base_dictionary(self):
|
|
401
|
+
self.function_words = {
|
|
402
|
+
'il-': 'DET', 'l-': 'DET',
|
|
403
|
+
'jien': 'PRON', 'int': 'PRON', 'huwa': 'PRON', 'hija': 'PRON', 'aħna': 'PRON', 'huma': 'PRON',
|
|
404
|
+
'u': 'CONJ', 'jew': 'CONJ', 'iżda': 'CONJ', 'li': 'CONJ',
|
|
405
|
+
'le': 'NEG', 'ma': 'NEG', 'iva': 'ADV', 'ħafna': 'ADV',
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
# =============================================================================
|
|
410
|
+
# Albanian (sq)
|
|
411
|
+
# =============================================================================
|
|
412
|
+
class AlbanianAnalyzer(LatinScriptAnalyzer):
|
|
413
|
+
LANG_CODE = "sq"
|
|
414
|
+
LANG_NAME = "Albanian"
|
|
415
|
+
|
|
416
|
+
def _build_base_dictionary(self):
|
|
417
|
+
self.function_words = {
|
|
418
|
+
'unë': 'PRON', 'ti': 'PRON', 'ai': 'PRON', 'ajo': 'PRON', 'ne': 'PRON', 'ata': 'PRON',
|
|
419
|
+
'dhe': 'CONJ', 'ose': 'CONJ', 'por': 'CONJ', 'që': 'CONJ',
|
|
420
|
+
'nuk': 'NEG', 'jo': 'NEG', 'po': 'ADV', 'shumë': 'ADV', 'mirë': 'ADV',
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
# =============================================================================
|
|
425
|
+
# Macedonian (mk)
|
|
426
|
+
# =============================================================================
|
|
427
|
+
class MacedonianAnalyzer(CyrillicScriptAnalyzer):
|
|
428
|
+
LANG_CODE = "mk"
|
|
429
|
+
LANG_NAME = "Macedonian"
|
|
430
|
+
|
|
431
|
+
def _build_base_dictionary(self):
|
|
432
|
+
self.function_words = {
|
|
433
|
+
'јас': 'PRON', 'ти': 'PRON', 'тој': 'PRON', 'таа': 'PRON', 'тоа': 'PRON',
|
|
434
|
+
'ние': 'PRON', 'вие': 'PRON', 'тие': 'PRON',
|
|
435
|
+
'во': 'PREP', 'на': 'PREP', 'со': 'PREP', 'од': 'PREP', 'за': 'PREP',
|
|
436
|
+
'и': 'CONJ', 'или': 'CONJ', 'но': 'CONJ', 'дека': 'CONJ',
|
|
437
|
+
'не': 'NEG', 'да': 'ADV', 'многу': 'ADV', 'добро': 'ADV',
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
# Export all
|
|
442
|
+
__all__ = [
|
|
443
|
+
'NorwegianAnalyzer', 'DanishAnalyzer', 'FinnishAnalyzer', 'HungarianAnalyzer',
|
|
444
|
+
'BulgarianAnalyzer', 'CroatianAnalyzer', 'SerbianAnalyzer', 'SlovakAnalyzer',
|
|
445
|
+
'SlovenianAnalyzer', 'LithuanianAnalyzer', 'LatvianAnalyzer', 'EstonianAnalyzer',
|
|
446
|
+
'CatalanAnalyzer', 'BasqueAnalyzer', 'GalicianAnalyzer', 'WelshAnalyzer',
|
|
447
|
+
'IrishAnalyzer', 'IcelandicAnalyzer', 'MalteseAnalyzer', 'AlbanianAnalyzer',
|
|
448
|
+
'MacedonianAnalyzer',
|
|
449
|
+
]
|