sonatoki 0.1.4__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sonatoki-0.1.4 → sonatoki-0.2.0}/PKG-INFO +1 -3
- {sonatoki-0.1.4 → sonatoki-0.2.0}/pyproject.toml +1 -4
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/Cleaners.py +25 -2
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/Configs.py +6 -6
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/Filters.py +77 -35
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/Preprocessors.py +18 -7
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/Scorers.py +2 -15
- sonatoki-0.2.0/src/sonatoki/Tokenizers.py +139 -0
- sonatoki-0.2.0/src/sonatoki/constants.py +67 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/ilo.py +0 -12
- sonatoki-0.2.0/src/sonatoki/utils.py +90 -0
- sonatoki-0.2.0/tests/test_cleaners.py +36 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/tests/test_filters.py +35 -13
- {sonatoki-0.1.4 → sonatoki-0.2.0}/tests/test_ilo.py +0 -1
- {sonatoki-0.1.4 → sonatoki-0.2.0}/tests/test_scorers.py +2 -2
- sonatoki-0.2.0/tests/test_tokenize.py +123 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/tests/tokenize_cases/tokenize_sentences_tok.yml +25 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/tests/tokenize_cases/tokenize_words_tok.yml +76 -0
- sonatoki-0.1.4/src/sonatoki/Tokenizers.py +0 -76
- sonatoki-0.1.4/src/sonatoki/constants.py +0 -67
- sonatoki-0.1.4/tests/test_cleaners.py +0 -26
- sonatoki-0.1.4/tests/test_tokenize.py +0 -111
- sonatoki-0.1.4/tests/tokenize_cases/tokenize_sentences.yml +0 -7
- sonatoki-0.1.4/tests/tokenize_cases/tokenize_words.yml +0 -15
- {sonatoki-0.1.4 → sonatoki-0.2.0}/LICENSE +0 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/README.md +0 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/__init__.py +0 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/__main__.py +0 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/linku.json +0 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/src/sonatoki/sandbox.json +0 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/tests/__init__.py +0 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/tests/test_preprocessors.py +0 -0
- {sonatoki-0.1.4 → sonatoki-0.2.0}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: sonatoki
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.2.0
|
4
4
|
Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
|
5
5
|
Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
|
6
6
|
License: AGPL-3.0-or-later
|
@@ -8,8 +8,6 @@ Requires-Python: >=3.8
|
|
8
8
|
Requires-Dist: unidecode>=1.3.6
|
9
9
|
Requires-Dist: regex>=2023.12.25
|
10
10
|
Requires-Dist: typing-extensions>=4.11.0
|
11
|
-
Requires-Dist: nltk>=3.8.1; extra == "nltk"
|
12
|
-
Provides-Extra: nltk
|
13
11
|
Description-Content-Type: text/markdown
|
14
12
|
|
15
13
|
# sona toki
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[project]
|
2
2
|
name = "sonatoki"
|
3
|
-
version = "0.
|
3
|
+
version = "0.2.0"
|
4
4
|
description = "ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?"
|
5
5
|
authors = [
|
6
6
|
{ name = "jan Kekan San (@gregdan3)", email = "gregory.danielson3@gmail.com" },
|
@@ -17,9 +17,6 @@ readme = "README.md"
|
|
17
17
|
text = "AGPL-3.0-or-later"
|
18
18
|
|
19
19
|
[project.optional-dependencies]
|
20
|
-
nltk = [
|
21
|
-
"nltk>=3.8.1",
|
22
|
-
]
|
23
20
|
|
24
21
|
[build-system]
|
25
22
|
requires = [
|
@@ -23,7 +23,7 @@ class RegexCleaner(Cleaner):
|
|
23
23
|
return re.sub(cls.pattern, cls.replace, token)
|
24
24
|
|
25
25
|
|
26
|
-
class ConsecutiveDuplicates(
|
26
|
+
class ConsecutiveDuplicates(Cleaner):
|
27
27
|
"""Remove consecutive duplicates from an input string, ignoring case.
|
28
28
|
|
29
29
|
The first match of any 2+ will become `\\1`, preserving initial case.
|
@@ -35,8 +35,31 @@ class ConsecutiveDuplicates(RegexCleaner):
|
|
35
35
|
This may be undesirable for moraic scripts like Hiragana, where `わわ` would be
|
36
36
|
incorrectly reduced to `わ`. This does preserve phonotactic validity, though."""
|
37
37
|
|
38
|
+
@classmethod
|
39
|
+
@override
|
40
|
+
def clean(cls, token: str) -> str:
|
41
|
+
if not token:
|
42
|
+
return token
|
43
|
+
|
44
|
+
output = token[0]
|
45
|
+
|
46
|
+
last_output = output.lower() # ignore case in comparison
|
47
|
+
for i in range(1, len(token)):
|
48
|
+
cur_char = token[i].lower()
|
49
|
+
if cur_char == last_output:
|
50
|
+
continue
|
51
|
+
output += token[i] # preserve case of string
|
52
|
+
last_output = cur_char
|
53
|
+
return output
|
54
|
+
|
55
|
+
|
56
|
+
class ConsecutiveDuplicatesRe(RegexCleaner):
|
57
|
+
"""Reference implementation for `ConsecutiveDuplicates`."""
|
58
|
+
|
38
59
|
pattern = re.compile(r"(.)\1+", flags=re.IGNORECASE)
|
39
60
|
replace = r"\1"
|
40
61
|
|
41
62
|
|
42
|
-
__all__ = [
|
63
|
+
__all__ = [
|
64
|
+
"ConsecutiveDuplicates",
|
65
|
+
]
|
@@ -21,7 +21,7 @@ from sonatoki.Filters import (
|
|
21
21
|
)
|
22
22
|
from sonatoki.Scorers import Number, Scorer, PassFail, SoftScaling, SoftPassFail
|
23
23
|
from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
|
24
|
-
from sonatoki.Tokenizers import Tokenizer,
|
24
|
+
from sonatoki.Tokenizers import Tokenizer, WordTokenizer
|
25
25
|
from sonatoki.Preprocessors import (
|
26
26
|
URLs,
|
27
27
|
Preprocessor,
|
@@ -49,7 +49,7 @@ BaseConfig: IloConfig = {
|
|
49
49
|
"scoring_filters": [],
|
50
50
|
"scorer": PassFail,
|
51
51
|
"passing_score": 0.8,
|
52
|
-
"word_tokenizer":
|
52
|
+
"word_tokenizer": WordTokenizer,
|
53
53
|
}
|
54
54
|
|
55
55
|
|
@@ -70,11 +70,11 @@ TelegramConfig: IloConfig = deepcopy(PrefConfig)
|
|
70
70
|
ForumConfig: IloConfig = deepcopy(PrefConfig)
|
71
71
|
|
72
72
|
__all__ = [
|
73
|
-
"IloConfig",
|
74
73
|
"BaseConfig",
|
75
|
-
"PrefConfig",
|
76
|
-
"LazyConfig",
|
77
74
|
"DiscordConfig",
|
78
|
-
"TelegramConfig",
|
79
75
|
"ForumConfig",
|
76
|
+
"IloConfig",
|
77
|
+
"LazyConfig",
|
78
|
+
"PrefConfig",
|
79
|
+
"TelegramConfig",
|
80
80
|
]
|
@@ -1,26 +1,30 @@
|
|
1
1
|
# STL
|
2
|
+
import re
|
2
3
|
from abc import ABC, abstractmethod
|
3
4
|
from typing import Set
|
4
5
|
from functools import lru_cache as cache # cache comes in 3.9
|
5
6
|
|
6
7
|
# PDM
|
7
|
-
import regex
|
8
|
+
import regex
|
8
9
|
from typing_extensions import override
|
9
10
|
|
10
11
|
# LOCAL
|
11
12
|
from sonatoki.constants import (
|
12
13
|
VOWELS,
|
14
|
+
NIMI_PU,
|
15
|
+
ALPHABET,
|
16
|
+
ALLOWABLES,
|
13
17
|
CONSONANTS,
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
18
|
+
NIMI_LINKU,
|
19
|
+
NIMI_PU_ALE,
|
20
|
+
POSIX_PUNCT,
|
21
|
+
UNICODE_PUNCT,
|
22
|
+
NIMI_LINKU_ALE,
|
23
|
+
ALL_PUNCT_RANGES,
|
24
|
+
NIMI_LINKU_SANDBOX,
|
21
25
|
)
|
22
26
|
|
23
|
-
|
27
|
+
regex.DEFAULT_VERSION = regex.VERSION1
|
24
28
|
|
25
29
|
|
26
30
|
class Filter(ABC):
|
@@ -41,7 +45,17 @@ class RegexFilter(Filter):
|
|
41
45
|
return not not re.fullmatch(cls.pattern, token)
|
42
46
|
|
43
47
|
|
44
|
-
class
|
48
|
+
class Regex1Filter(Filter):
|
49
|
+
pattern: "regex.Pattern[str]"
|
50
|
+
|
51
|
+
@classmethod
|
52
|
+
@override
|
53
|
+
@cache(maxsize=None)
|
54
|
+
def filter(cls, token: str) -> bool:
|
55
|
+
return not not regex.fullmatch(cls.pattern, token)
|
56
|
+
|
57
|
+
|
58
|
+
class MemberFilter(Filter):
|
45
59
|
tokens: Set[str]
|
46
60
|
|
47
61
|
@classmethod
|
@@ -51,8 +65,18 @@ class SetFilter(Filter):
|
|
51
65
|
return token.lower() in cls.tokens
|
52
66
|
|
53
67
|
|
54
|
-
class
|
55
|
-
tokens
|
68
|
+
class SubsetFilter(Filter):
|
69
|
+
tokens: Set[str]
|
70
|
+
|
71
|
+
@classmethod
|
72
|
+
@override
|
73
|
+
@cache(maxsize=None)
|
74
|
+
def filter(cls, token: str) -> bool:
|
75
|
+
return set(token.lower()).issubset(cls.tokens)
|
76
|
+
|
77
|
+
|
78
|
+
class Miscellaneous(MemberFilter):
|
79
|
+
tokens = set(ALLOWABLES)
|
56
80
|
|
57
81
|
|
58
82
|
class ProperName(Filter):
|
@@ -70,26 +94,28 @@ class ProperName(Filter):
|
|
70
94
|
@cache(maxsize=None)
|
71
95
|
def filter(cls, token: str) -> bool:
|
72
96
|
return token == token.capitalize()
|
97
|
+
# TODO: If the token is in a script which doesn't have a case distinction,
|
98
|
+
# this will errantly match.
|
73
99
|
|
74
100
|
|
75
|
-
class NimiPu(
|
76
|
-
tokens =
|
101
|
+
class NimiPu(MemberFilter):
|
102
|
+
tokens = set(NIMI_PU)
|
77
103
|
|
78
104
|
|
79
|
-
class NimiPuAle(
|
80
|
-
tokens =
|
105
|
+
class NimiPuAle(MemberFilter):
|
106
|
+
tokens = set(NIMI_PU_ALE)
|
81
107
|
|
82
108
|
|
83
|
-
class NimiLinku(
|
84
|
-
tokens =
|
109
|
+
class NimiLinku(MemberFilter):
|
110
|
+
tokens = set(NIMI_LINKU)
|
85
111
|
|
86
112
|
|
87
|
-
class NimiLinkuAle(
|
88
|
-
tokens =
|
113
|
+
class NimiLinkuAle(MemberFilter):
|
114
|
+
tokens = set(NIMI_LINKU_ALE)
|
89
115
|
|
90
116
|
|
91
|
-
class NimiLinkuSandbox(
|
92
|
-
tokens =
|
117
|
+
class NimiLinkuSandbox(MemberFilter):
|
118
|
+
tokens = set(NIMI_LINKU_SANDBOX)
|
93
119
|
|
94
120
|
|
95
121
|
class Phonotactic(RegexFilter):
|
@@ -122,13 +148,12 @@ class Syllabic(RegexFilter):
|
|
122
148
|
)
|
123
149
|
|
124
150
|
|
125
|
-
class Alphabetic(
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
return set(token.lower()).issubset(ALPHABET_SET)
|
151
|
+
class Alphabetic(SubsetFilter):
|
152
|
+
tokens = set(ALPHABET)
|
153
|
+
|
154
|
+
|
155
|
+
class AlphabeticRe(RegexFilter):
|
156
|
+
pattern = re.compile(rf"[{ALPHABET}]+", flags=re.IGNORECASE)
|
132
157
|
|
133
158
|
|
134
159
|
class Numeric(Filter):
|
@@ -147,18 +172,35 @@ class Numeric(Filter):
|
|
147
172
|
return msg.isnumeric()
|
148
173
|
|
149
174
|
|
150
|
-
class Punctuation(
|
151
|
-
|
175
|
+
class Punctuation(SubsetFilter):
|
176
|
+
"""Identify whether a token is entirely punctuation. Fastest implementation."""
|
177
|
+
|
178
|
+
tokens = set(POSIX_PUNCT + UNICODE_PUNCT)
|
179
|
+
|
180
|
+
|
181
|
+
class PunctuationRe(RegexFilter):
|
182
|
+
"""Faster implementation of `PunctuationRe1`.
|
183
|
+
Goes out of date compared to the `regex` library if UNICODE_PUNCT is not updated."""
|
184
|
+
|
185
|
+
pattern = re.compile(rf"[{ALL_PUNCT_RANGES}]+")
|
186
|
+
|
187
|
+
|
188
|
+
class PunctuationRe1(Regex1Filter):
|
189
|
+
"""Reference implementation for identifying tokens made entirely of punctuation."""
|
190
|
+
|
191
|
+
pattern = regex.compile(r"[\p{Punctuation}\p{posix_punct}]+")
|
152
192
|
|
153
193
|
|
154
194
|
__all__ = [
|
155
|
-
"
|
195
|
+
"Alphabetic",
|
156
196
|
"NimiLinku",
|
157
197
|
"NimiLinkuAle",
|
198
|
+
"NimiLinkuSandbox",
|
199
|
+
"NimiPu",
|
200
|
+
"NimiPuAle",
|
201
|
+
"Numeric",
|
158
202
|
"Phonotactic",
|
159
|
-
"Syllabic",
|
160
|
-
"Alphabetic",
|
161
203
|
"ProperName",
|
162
204
|
"Punctuation",
|
163
|
-
"
|
205
|
+
"Syllabic",
|
164
206
|
]
|
@@ -17,13 +17,14 @@ It is up to the user to order them appropriately.
|
|
17
17
|
"""
|
18
18
|
|
19
19
|
# STL
|
20
|
+
import re
|
20
21
|
from abc import ABC, abstractmethod
|
21
22
|
|
22
23
|
# PDM
|
23
|
-
import regex
|
24
|
+
import regex
|
24
25
|
from typing_extensions import override
|
25
26
|
|
26
|
-
|
27
|
+
regex.DEFAULT_VERSION = regex.VERSION1
|
27
28
|
|
28
29
|
|
29
30
|
class Preprocessor(ABC):
|
@@ -43,6 +44,16 @@ class RegexPreprocessor(Preprocessor):
|
|
43
44
|
return re.sub(cls.pattern, cls.replace, msg)
|
44
45
|
|
45
46
|
|
47
|
+
class Regex1Preprocessor(Preprocessor):
|
48
|
+
pattern: "regex.Pattern[str]"
|
49
|
+
replace: str = " "
|
50
|
+
|
51
|
+
@classmethod
|
52
|
+
@override
|
53
|
+
def process(cls, msg: str) -> str:
|
54
|
+
return regex.sub(cls.pattern, cls.replace, msg)
|
55
|
+
|
56
|
+
|
46
57
|
"""
|
47
58
|
The following classes are Ignorables.
|
48
59
|
|
@@ -146,17 +157,17 @@ class AllQuotes(RegexPreprocessor):
|
|
146
157
|
|
147
158
|
|
148
159
|
__all__ = [
|
160
|
+
"AllQuotes",
|
149
161
|
"AngleBracketObject",
|
162
|
+
"ArrowQuote",
|
163
|
+
"Backticks",
|
150
164
|
"DiscordChannels",
|
165
|
+
"DiscordEmotes",
|
151
166
|
"DiscordMentions",
|
152
167
|
"DiscordSpecial",
|
153
|
-
"DiscordEmotes",
|
154
|
-
"SingleQuotes",
|
155
168
|
"DoubleQuotes",
|
156
|
-
"ArrowQuote",
|
157
|
-
"AllQuotes",
|
158
|
-
"Backticks",
|
159
169
|
"Reference",
|
170
|
+
"SingleQuotes",
|
160
171
|
"Spoilers",
|
161
172
|
"URLs",
|
162
173
|
]
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# STL
|
2
2
|
import math
|
3
|
-
import logging
|
4
3
|
from abc import ABC, abstractmethod
|
5
4
|
from typing import Dict, List, Type, Union
|
6
5
|
|
@@ -10,8 +9,6 @@ from typing_extensions import override
|
|
10
9
|
# LOCAL
|
11
10
|
from sonatoki.Filters import Filter
|
12
11
|
|
13
|
-
LOG = logging.getLogger(__name__)
|
14
|
-
|
15
12
|
Number = Union[int, float]
|
16
13
|
Weights = Dict[str, Number]
|
17
14
|
|
@@ -37,12 +34,7 @@ class PassFail(Scorer):
|
|
37
34
|
def score_token(cls, token: str, filters: List[Type[Filter]]) -> Number:
|
38
35
|
for f in filters:
|
39
36
|
if f.filter(token):
|
40
|
-
|
41
|
-
LOG.debug(
|
42
|
-
"%12s.%s('%s') = %.2f", cls.__name__, f.__name__, token, score
|
43
|
-
)
|
44
|
-
return score
|
45
|
-
LOG.debug("%12s('%s') = 0.00", cls.__name__, token)
|
37
|
+
return 1
|
46
38
|
return 0
|
47
39
|
|
48
40
|
@classmethod
|
@@ -86,12 +78,7 @@ class Scaling(Scorer):
|
|
86
78
|
def score_token(cls, token: str, filters: List[Type[Filter]], scale: int):
|
87
79
|
for i, f in enumerate(filters):
|
88
80
|
if f.filter(token):
|
89
|
-
|
90
|
-
LOG.debug(
|
91
|
-
"%12s.%s('%s') = %.2f", cls.__name__, f.__name__, token, score
|
92
|
-
)
|
93
|
-
return score
|
94
|
-
LOG.debug("%12s('%s') = 0.00", cls.__name__, token)
|
81
|
+
return scale - i
|
95
82
|
return 0
|
96
83
|
|
97
84
|
@classmethod
|
@@ -0,0 +1,139 @@
|
|
1
|
+
# STL
|
2
|
+
import re
|
3
|
+
from abc import ABC, abstractmethod
|
4
|
+
from typing import Set, List
|
5
|
+
|
6
|
+
# PDM
|
7
|
+
import regex
|
8
|
+
from typing_extensions import override
|
9
|
+
|
10
|
+
# LOCAL
|
11
|
+
from sonatoki.utils import regex_escape
|
12
|
+
from sonatoki.constants import (
|
13
|
+
POSIX_PUNCT,
|
14
|
+
UNICODE_PUNCT,
|
15
|
+
SENTENCE_PUNCT,
|
16
|
+
ALL_PUNCT_RANGES,
|
17
|
+
)
|
18
|
+
|
19
|
+
regex.DEFAULT_VERSION = regex.VERSION1
|
20
|
+
|
21
|
+
|
22
|
+
class Tokenizer(ABC):
|
23
|
+
@classmethod
|
24
|
+
@abstractmethod
|
25
|
+
def tokenize(cls, s: str) -> List[str]: ...
|
26
|
+
|
27
|
+
|
28
|
+
class SetTokenizer(Tokenizer):
|
29
|
+
delimiters: Set[str]
|
30
|
+
|
31
|
+
|
32
|
+
class RegexTokenizer(Tokenizer):
|
33
|
+
pattern: "re.Pattern[str]"
|
34
|
+
|
35
|
+
@classmethod
|
36
|
+
@override
|
37
|
+
def tokenize(cls, s: str) -> List[str]:
|
38
|
+
return [clean for word in re.split(cls.pattern, s) if (clean := word.strip())]
|
39
|
+
|
40
|
+
|
41
|
+
class Regex1Tokenizer(Tokenizer):
|
42
|
+
pattern: "regex.Pattern[str]"
|
43
|
+
|
44
|
+
@classmethod
|
45
|
+
@override
|
46
|
+
def tokenize(cls, s: str) -> List[str]:
|
47
|
+
return [
|
48
|
+
clean for word in regex.split(cls.pattern, s) if (clean := word.strip())
|
49
|
+
]
|
50
|
+
|
51
|
+
|
52
|
+
class WordTokenizer(SetTokenizer):
|
53
|
+
delimiters = set(POSIX_PUNCT + UNICODE_PUNCT)
|
54
|
+
|
55
|
+
@classmethod
|
56
|
+
@override
|
57
|
+
def tokenize(cls, s: str) -> List[str]:
|
58
|
+
if not s:
|
59
|
+
return []
|
60
|
+
|
61
|
+
tokens: List[str] = []
|
62
|
+
|
63
|
+
last_match = 0
|
64
|
+
last_membership = s[0] in cls.delimiters
|
65
|
+
for i, char in enumerate(s):
|
66
|
+
mem = char in cls.delimiters
|
67
|
+
if mem == last_membership:
|
68
|
+
continue
|
69
|
+
|
70
|
+
match = s[last_match:i].split()
|
71
|
+
# TODO: kinda sucks? what about unicode whitespace?
|
72
|
+
last_match = i
|
73
|
+
last_membership = mem
|
74
|
+
[tokens.append(t) for t in match if t]
|
75
|
+
|
76
|
+
match = s[last_match:].strip().split()
|
77
|
+
if match:
|
78
|
+
tokens.extend(match)
|
79
|
+
|
80
|
+
return tokens
|
81
|
+
|
82
|
+
|
83
|
+
class WordTokenizerRe(RegexTokenizer):
|
84
|
+
pattern = re.compile(rf"""([{ALL_PUNCT_RANGES}]+|\s+)""")
|
85
|
+
|
86
|
+
|
87
|
+
class WordTokenizerRe1(Regex1Tokenizer):
|
88
|
+
"""Reference implementation for WorkTokenizer."""
|
89
|
+
|
90
|
+
pattern = regex.compile(r"""([\p{posix_punct}\p{Punctuation}]+|\s+)""")
|
91
|
+
|
92
|
+
|
93
|
+
class SentTokenizer(SetTokenizer):
|
94
|
+
delimiters = set(SENTENCE_PUNCT + "\n") # regex does \n with a flag
|
95
|
+
|
96
|
+
@classmethod
|
97
|
+
@override
|
98
|
+
def tokenize(cls, s: str) -> List[str]:
|
99
|
+
if not s:
|
100
|
+
return []
|
101
|
+
|
102
|
+
tokens: List[str] = []
|
103
|
+
last_match = 0
|
104
|
+
for i, char in enumerate(s):
|
105
|
+
if char not in cls.delimiters:
|
106
|
+
continue
|
107
|
+
|
108
|
+
match = s[last_match : i + 1].strip()
|
109
|
+
last_match = i + 1 # newlines can strip but idc
|
110
|
+
if not match:
|
111
|
+
continue
|
112
|
+
tokens.append(match)
|
113
|
+
|
114
|
+
match = s[last_match:].strip()
|
115
|
+
if match:
|
116
|
+
tokens.append(match)
|
117
|
+
|
118
|
+
return tokens
|
119
|
+
|
120
|
+
|
121
|
+
class SentTokenizerRe(RegexTokenizer):
|
122
|
+
pattern = re.compile(
|
123
|
+
rf"""(?<=[{regex_escape(SENTENCE_PUNCT)}])|$""", flags=re.MULTILINE
|
124
|
+
)
|
125
|
+
# TODO: are <> or {} that common as *sentence* delims? [] are already a stretch
|
126
|
+
# TODO: do the typography characters matter?
|
127
|
+
# NOTE: | / and , are *not* sentence delimiters for my purpose
|
128
|
+
|
129
|
+
|
130
|
+
class SentTokenizerRe1(Regex1Tokenizer):
|
131
|
+
pattern = regex.compile(
|
132
|
+
rf"""(?<=[{regex_escape(SENTENCE_PUNCT)}]|$)""", flags=regex.MULTILINE
|
133
|
+
)
|
134
|
+
|
135
|
+
|
136
|
+
__all__ = [
|
137
|
+
"WordTokenizer",
|
138
|
+
"SentTokenizer",
|
139
|
+
]
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# STL
|
2
|
+
import json
|
3
|
+
from typing import Dict, List
|
4
|
+
from pathlib import Path
|
5
|
+
|
6
|
+
# LOCAL
|
7
|
+
from sonatoki.utils import find_unicode_ranges
|
8
|
+
|
9
|
+
# `\p{Punctuation}` character class
|
10
|
+
UNICODE_PUNCT = r"""!"#%&'()*+,-./:;<=>?@[\]^_`{|}~¡¦§¨©«¬®¯°±´¶·¸»¿×÷˂˃˄˅˒˓˔˕˖˗˘˙˚˛˜˝˞˟˥˦˧˨˩˪˫˭˯˰˱˲˳˴˵˶˷˸˹˺˻˼˽˾˿͵;΄΅·϶҂՚՛՜՝՞՟։֊֍֎־׀׃׆׳״؆؇؈؉؊،؍؎؏؛؝؞؟٪٫٬٭۔۞۩۽۾܀܁܂܃܄܅܆܇܈܉܊܋܌܍߶߷߸߹࠰࠱࠲࠳࠴࠵࠶࠷࠸࠹࠺࠻࠼࠽࠾࡞࢈।॥॰৺৽੶૰୰௳௴௵௶௷௸௺౷౿಄൏൹෴๏๚๛༁༂༃༄༅༆༇༈༉༊་༌།༎༏༐༑༒༓༔༕༖༗༚༛༜༝༞༟༴༶༸༺༻༼༽྅྾྿࿀࿁࿂࿃࿄࿅࿇࿈࿉࿊࿋࿌࿎࿏࿐࿑࿒࿓࿔࿕࿖࿗࿘࿙࿚၊။၌၍၎၏႞႟჻፠፡።፣፤፥፦፧፨᎐᎑᎒᎓᎔᎕᎖᎗᎘᎙᐀᙭᙮᚛᚜᛫᛬᛭᜵᜶។៕៖៘៙៚᠀᠁᠂᠃᠄᠅᠆᠇᠈᠉᠊᥀᥄᥅᧞᧟᧠᧡᧢᧣᧤᧥᧦᧧᧨᧩᧪᧫᧬᧭᧮᧯᧰᧱᧲᧳᧴᧵᧶᧷᧸᧹᧺᧻᧼᧽᧾᧿᨞᨟᪠᪡᪢᪣᪤᪥᪦᪨᪩᪪᪫᪬᪭᭚᭛᭜᭝᭞᭟᭠᭡᭢᭣᭤᭥᭦᭧᭨᭩᭪᭴᭵᭶᭷᭸᭹᭺᭻᭼᭽᭾᯼᯽᯾᯿᰻᰼᰽᰾᰿᱾᱿᳀᳁᳂᳃᳄᳅᳆᳇᳓᾽᾿῀῁῍῎῏῝῞῟῭΅`´῾‐‑‒–—―‖‗‘’‚‛“”„‟†‡•‣․‥…‧‰‱′″‴‵‶‷‸‹›※‼‽‾‿⁀⁁⁂⁃⁄⁅⁆⁇⁈⁉⁊⁋⁌⁍⁎⁏⁐⁑⁒⁓⁔⁕⁖⁗⁘⁙⁚⁛⁜⁝⁞⁺⁻⁼⁽⁾₊₋₌₍₎℀℁℃℄℅℆℈℉℔№℗℘℞℟℠℡™℣℥℧℩℮℺℻⅀⅁⅂⅃⅄⅊⅋⅌⅍⅏↊↋←↑→↓↔↕↖↗↘↙↚↛↜↝↞↟↠↡↢↣↤↥↦↧↨↩↪↫↬↭↮↯↰↱↲↳↴↵↶↷↸↹↺↻↼↽↾↿⇀⇁⇂⇃⇄⇅⇆⇇⇈⇉⇊⇋⇌⇍⇎⇏⇐⇑⇒⇓⇔⇕⇖⇗⇘⇙⇚⇛⇜⇝⇞⇟⇠⇡⇢⇣⇤⇥⇦⇧⇨⇩⇪⇫⇬⇭⇮⇯⇰⇱⇲⇳⇴⇵⇶⇷⇸⇹⇺⇻⇼⇽⇾⇿∀∁∂∃∄∅∆∇∈∉∊∋∌∍∎∏∐∑−∓∔∕∖∗∘∙√∛∜∝∞∟∠∡∢∣∤∥∦∧∨∩∪∫∬∭∮∯∰∱∲∳∴∵∶∷∸∹∺∻∼∽∾∿≀≁≂≃≄≅≆≇≈≉≊≋≌≍≎≏≐≑≒≓≔≕≖≗≘≙≚≛≜≝≞≟≠≡≢≣≤≥≦≧≨≩≪≫≬≭≮≯≰≱≲≳≴≵≶≷≸≹≺≻≼≽≾≿⊀⊁⊂⊃⊄⊅⊆⊇⊈⊉⊊⊋⊌⊍⊎⊏⊐⊑⊒⊓⊔⊕⊖⊗⊘⊙⊚⊛⊜⊝⊞⊟⊠⊡⊢⊣⊤⊥⊦⊧⊨⊩⊪⊫⊬⊭⊮⊯⊰⊱⊲⊳⊴⊵⊶⊷⊸⊹⊺⊻⊼⊽⊾⊿⋀⋁⋂⋃⋄⋅⋆⋇⋈⋉⋊⋋⋌⋍⋎⋏⋐⋑⋒⋓⋔⋕⋖⋗⋘⋙⋚⋛⋜⋝⋞⋟⋠⋡⋢⋣⋤⋥⋦⋧⋨⋩⋪⋫⋬⋭⋮⋯⋰⋱⋲⋳⋴⋵⋶⋷⋸⋹⋺⋻⋼⋽⋾⋿⌀⌁⌂⌃⌄⌅⌆⌇⌈⌉⌊⌋⌌⌍⌎⌏⌐⌑⌒⌓⌔⌕⌖⌗⌘⌙⌚⌛⌜⌝⌞⌟⌠⌡⌢⌣⌤⌥⌦⌧⌨〈〉⌫⌬⌭⌮⌯⌰⌱⌲⌳⌴⌵⌶⌷⌸⌹⌺⌻⌼⌽⌾⌿⍀⍁⍂⍃⍄⍅⍆⍇⍈⍉⍊⍋⍌⍍⍎⍏⍐⍑⍒⍓⍔⍕⍖⍗⍘⍙⍚⍛⍜⍝⍞⍟⍠⍡⍢⍣⍤⍥⍦⍧⍨⍩⍪⍫⍬⍭⍮⍯⍰⍱⍲⍳⍴⍵⍶⍷⍸⍹⍺⍻⍼⍽⍾⍿⎀⎁⎂⎃⎄⎅⎆⎇⎈⎉⎊⎋⎌⎍⎎⎏⎐⎑⎒⎓⎔⎕⎖⎗⎘⎙⎚⎛⎜⎝⎞⎟⎠⎡⎢⎣⎤⎥⎦⎧⎨⎩⎪⎫⎬⎭⎮⎯⎰⎱⎲⎳⎴⎵⎶⎷⎸⎹⎺⎻⎼⎽⎾⎿⏀⏁⏂⏃⏄⏅⏆⏇⏈⏉⏊⏋⏌⏍⏎⏏⏐⏑⏒⏓⏔⏕⏖⏗⏘⏙⏚⏛⏜⏝⏞⏟⏠⏡⏢⏣⏤⏥⏦⏧⏨⏩⏪⏫⏬⏭⏮⏯⏰⏱⏲⏳⏴⏵⏶⏷⏸⏹⏺⏻⏼⏽⏾⏿␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟␠␡␢␣␥␦⑀⑁⑂⑃⑄⑅⑆⑇⑈⑉⑊⒜⒝⒞⒟⒠⒡⒢⒣⒤⒥⒦⒧⒨⒩⒪⒫⒬⒭⒮⒯⒰⒱⒲⒳⒴⒵─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟■□▢▣▤▥▦▧▨▩▪▫▬▭▮▯▰▱▲△▴▵▶▷▸▹►▻▼▽▾▿◀◁◂◃◄◅◆◇◈◉◊○◌◍◎●◐◑◒◓◔◕◖◗◘◙◚◛◜◝◞◟◠◡◢◣◤◥◦◧◨◩◪◫◬◭◮◯◰◱◲◳◴◵◶◷◸◹◺◻◼◽◾◿☀☁☂☃☄★☆☇☈☉☊☋☌☍☎☏☐☑☒☓☔☕☖☗☘☙☚☛☜☝☞☟☠☡☢☣☤☥☦☧☨☩☪☫☬☭☮☯☰☱☲☳☴☵☶☷☸☹☺☻☼☽☾☿♀♁♂♃♄♅♆♇♈♉♊♋♌♍♎♏♐♑♒♓♔♕♖♗♘♙♚♛♜♝♞♟♠♡♢♣♤♥♦♧♨♩♪♫♬♭♮♯♰♱♲♳♴♵♶♷♸♹♺♻♼♽♾♿⚀⚁⚂⚃⚄⚅⚆⚇⚈⚉⚊⚋⚌⚍⚎⚏⚐⚑⚒⚓⚔⚕⚖⚗⚘⚙⚚⚛⚜⚝⚞⚟⚠⚡⚢⚣⚤⚥⚦⚧⚨⚩⚪⚫⚬⚭⚮⚯⚰⚱⚲⚳⚴⚵⚶⚷⚸⚹⚺⚻⚼⚽⚾⚿⛀⛁⛂⛃⛄⛅⛆⛇⛈⛉⛊⛋⛌⛍⛎⛏⛐⛑⛒⛓⛔⛕⛖⛗⛘⛙⛚⛛⛜⛝⛞⛟⛠⛡⛢⛣⛤⛥⛦⛧⛨⛩⛪⛫⛬⛭⛮⛯⛰⛱⛲⛳⛴⛵⛶⛷⛸⛹⛺⛻⛼⛽⛾⛿✀✁✂✃✄✅✆✇✈✉✊✋✌✍✎✏✐✑✒✓✔✕✖✗✘✙✚✛✜✝✞✟✠✡✢✣✤✥✦✧✨✩✪✫✬✭✮✯✰✱✲✳✴✵✶✷✸✹✺✻✼✽✾✿❀❁❂❃❄❅❆❇❈❉❊❋❌❍❎❏❐❑❒❓❔❕❖❗❘❙❚❛❜❝❞❟❠❡❢❣❤❥❦❧❨❩❪❫❬❭❮❯❰❱❲❳❴❵➔➕➖➗➘➙➚➛➜➝➞➟➠➡➢➣➤➥➦➧➨➩➪➫➬➭➮➯➰➱➲➳➴➵➶➷➸➹➺➻➼➽➾➿⟀⟁⟂⟃⟄⟅⟆⟇⟈⟉⟊⟋⟌⟍⟎⟏⟐⟑⟒⟓⟔⟕⟖⟗⟘⟙⟚⟛⟜⟝⟞⟟⟠⟡⟢⟣⟤⟥⟦⟧⟨⟩⟪⟫⟬⟭⟮⟯⟰⟱⟲⟳⟴⟵⟶⟷⟸⟹⟺⟻⟼⟽⟾⟿⠀⠁⠂⠃⠄⠅⠆⠇⠈⠉⠊⠋⠌⠍⠎⠏⠐⠑⠒⠓⠔⠕⠖⠗⠘⠙⠚⠛⠜⠝⠞⠟⠠⠡⠢⠣⠤⠥⠦⠧⠨⠩⠪⠫⠬⠭⠮⠯⠰⠱⠲⠳⠴⠵⠶⠷⠸⠹⠺⠻⠼⠽⠾⠿⡀⡁⡂⡃⡄⡅⡆⡇⡈⡉⡊⡋⡌⡍⡎⡏⡐⡑⡒⡓⡔⡕⡖⡗⡘⡙⡚⡛⡜⡝⡞⡟⡠⡡⡢⡣⡤⡥⡦⡧⡨⡩⡪⡫⡬⡭⡮⡯⡰⡱⡲⡳⡴⡵⡶⡷⡸⡹⡺⡻⡼⡽⡾⡿⢀⢁⢂⢃⢄⢅⢆⢇⢈⢉⢊⢋⢌⢍⢎⢏⢐⢑⢒⢓⢔⢕⢖⢗⢘⢙⢚⢛⢜⢝⢞⢟⢠⢡⢢⢣⢤⢥⢦⢧⢨⢩⢪⢫⢬⢭⢮⢯⢰⢱⢲⢳⢴⢵⢶⢷⢸⢹⢺⢻⢼⢽⢾⢿⣀⣁⣂⣃⣄⣅⣆⣇⣈⣉⣊⣋⣌⣍⣎⣏⣐⣑⣒⣓⣔⣕⣖⣗⣘⣙⣚⣛⣜⣝⣞⣟⣠⣡⣢⣣⣤⣥⣦⣧⣨⣩⣪⣫⣬⣭⣮⣯⣰⣱⣲⣳⣴⣵⣶⣷⣸⣹⣺⣻⣼⣽⣾⣿⤀⤁⤂⤃⤄⤅⤆⤇⤈⤉⤊⤋⤌⤍⤎⤏⤐⤑⤒⤓⤔⤕⤖⤗⤘⤙⤚⤛⤜⤝⤞⤟⤠⤡⤢⤣⤤⤥⤦⤧⤨⤩⤪⤫⤬⤭⤮⤯⤰⤱⤲⤳⤴⤵⤶⤷⤸⤹⤺⤻⤼⤽⤾⤿⥀⥁⥂⥃⥄⥅⥆⥇⥈⥉⥊⥋⥌⥍⥎⥏⥐⥑⥒⥓⥔⥕⥖⥗⥘⥙⥚⥛⥜⥝⥞⥟⥠⥡⥢⥣⥤⥥⥦⥧⥨⥩⥪⥫⥬⥭⥮⥯⥰⥱⥲⥳⥴⥵⥶⥷⥸⥹⥺⥻⥼⥽⥾⥿⦀⦁⦂⦃⦄⦅⦆⦇⦈⦉⦊⦋⦌⦍⦎⦏⦐⦑⦒⦓⦔⦕⦖⦗⦘⦙⦚⦛⦜⦝⦞⦟⦠⦡⦢⦣⦤⦥⦦⦧⦨⦩⦪⦫⦬⦭⦮⦯⦰⦱⦲⦳⦴⦵⦶⦷⦸⦹⦺⦻⦼⦽⦾⦿⧀⧁⧂⧃⧄⧅⧆⧇⧈⧉⧊⧋⧌⧍⧎⧏⧐⧑⧒⧓⧔⧕⧖⧗⧘⧙⧚⧛⧜⧝⧞⧟⧠⧡⧢⧣⧤⧥⧦⧧⧨⧩⧪⧫⧬⧭⧮⧯⧰⧱⧲⧳⧴⧵⧶⧷⧸⧹⧺⧻⧼⧽⧾⧿⨀⨁⨂⨃⨄⨅⨆⨇⨈⨉⨊⨋⨌⨍⨎⨏⨐⨑⨒⨓⨔⨕⨖⨗⨘⨙⨚⨛⨜⨝⨞⨟⨠⨡⨢⨣⨤⨥⨦⨧⨨⨩⨪⨫⨬⨭⨮⨯⨰⨱⨲⨳⨴⨵⨶⨷⨸⨹⨺⨻⨼⨽⨾⨿⩀⩁⩂⩃⩄⩅⩆⩇⩈⩉⩊⩋⩌⩍⩎⩏⩐⩑⩒⩓⩔⩕⩖⩗⩘⩙⩚⩛⩜⩝⩞⩟⩠⩡⩢⩣⩤⩥⩦⩧⩨⩩⩪⩫⩬⩭⩮⩯⩰⩱⩲⩳⩴⩵⩶⩷⩸⩹⩺⩻⩼⩽⩾⩿⪀⪁⪂⪃⪄⪅⪆⪇⪈⪉⪊⪋⪌⪍⪎⪏⪐⪑⪒⪓⪔⪕⪖⪗⪘⪙⪚⪛⪜⪝⪞⪟⪠⪡⪢⪣⪤⪥⪦⪧⪨⪩⪪⪫⪬⪭⪮⪯⪰⪱⪲⪳⪴⪵⪶⪷⪸⪹⪺⪻⪼⪽⪾⪿⫀⫁⫂⫃⫄⫅⫆⫇⫈⫉⫊⫋⫌⫍⫎⫏⫐⫑⫒⫓⫔⫕⫖⫗⫘⫙⫚⫛⫝̸⫝⫞⫟⫠⫡⫢⫣⫤⫥⫦⫧⫨⫩⫪⫫⫬⫭⫮⫯⫰⫱⫲⫳⫴⫵⫶⫷⫸⫹⫺⫻⫼⫽⫾⫿⬀⬁⬂⬃⬄⬅⬆⬇⬈⬉⬊⬋⬌⬍⬎⬏⬐⬑⬒⬓⬔⬕⬖⬗⬘⬙⬚⬛⬜⬝⬞⬟⬠⬡⬢⬣⬤⬥⬦⬧⬨⬩⬪⬫⬬⬭⬮⬯⬰⬱⬲⬳⬴⬵⬶⬷⬸⬹⬺⬻⬼⬽⬾⬿⭀⭁⭂⭃⭄⭅⭆⭇⭈⭉⭊⭋⭌⭍⭎⭏⭐⭑⭒⭓⭔⭕⭖⭗⭘⭙⭚⭛⭜⭝⭞⭟⭠⭡⭢⭣⭤⭥⭦⭧⭨⭩⭪⭫⭬⭭⭮⭯⭰⭱⭲⭳⭶⭷⭸⭹⭺⭻⭼⭽⭾⭿⮀⮁⮂⮃⮄⮅⮆⮇⮈⮉⮊⮋⮌⮍⮎⮏⮐⮑⮒⮓⮔⮕⮗⮘⮙⮚⮛⮜⮝⮞⮟⮠⮡⮢⮣⮤⮥⮦⮧⮨⮩⮪⮫⮬⮭⮮⮯⮰⮱⮲⮳⮴⮵⮶⮷⮸⮹⮺⮻⮼⮽⮾⮿⯀⯁⯂⯃⯄⯅⯆⯇⯈⯉⯊⯋⯌⯍⯎⯏⯐⯑⯒⯓⯔⯕⯖⯗⯘⯙⯚⯛⯜⯝⯞⯟⯠⯡⯢⯣⯤⯥⯦⯧⯨⯩⯪⯫⯬⯭⯮⯯⯰⯱⯲⯳⯴⯵⯶⯷⯸⯹⯺⯻⯼⯽⯾⯿⳥⳦⳧⳨⳩⳪⳹⳺⳻⳼⳾⳿⵰⸀⸁⸂⸃⸄⸅⸆⸇⸈⸉⸊⸋⸌⸍⸎⸏⸐⸑⸒⸓⸔⸕⸖⸗⸘⸙⸚⸛⸜⸝⸞⸟⸠⸡⸢⸣⸤⸥⸦⸧⸨⸩⸪⸫⸬⸭⸮⸰⸱⸲⸳⸴⸵⸶⸷⸸⸹⸺⸻⸼⸽⸾⸿⹀⹁⹂⹃⹄⹅⹆⹇⹈⹉⹊⹋⹌⹍⹎⹏⹐⹑⹒⹓⹔⹕⹖⹗⹘⹙⹚⹛⹜⹝⺀⺁⺂⺃⺄⺅⺆⺇⺈⺉⺊⺋⺌⺍⺎⺏⺐⺑⺒⺓⺔⺕⺖⺗⺘⺙⺛⺜⺝⺞⺟⺠⺡⺢⺣⺤⺥⺦⺧⺨⺩⺪⺫⺬⺭⺮⺯⺰⺱⺲⺳⺴⺵⺶⺷⺸⺹⺺⺻⺼⺽⺾⺿⻀⻁⻂⻃⻄⻅⻆⻇⻈⻉⻊⻋⻌⻍⻎⻏⻐⻑⻒⻓⻔⻕⻖⻗⻘⻙⻚⻛⻜⻝⻞⻟⻠⻡⻢⻣⻤⻥⻦⻧⻨⻩⻪⻫⻬⻭⻮⻯⻰⻱⻲⻳⼀⼁⼂⼃⼄⼅⼆⼇⼈⼉⼊⼋⼌⼍⼎⼏⼐⼑⼒⼓⼔⼕⼖⼗⼘⼙⼚⼛⼜⼝⼞⼟⼠⼡⼢⼣⼤⼥⼦⼧⼨⼩⼪⼫⼬⼭⼮⼯⼰⼱⼲⼳⼴⼵⼶⼷⼸⼹⼺⼻⼼⼽⼾⼿⽀⽁⽂⽃⽄⽅⽆⽇⽈⽉⽊⽋⽌⽍⽎⽏⽐⽑⽒⽓⽔⽕⽖⽗⽘⽙⽚⽛⽜⽝⽞⽟⽠⽡⽢⽣⽤⽥⽦⽧⽨⽩⽪⽫⽬⽭⽮⽯⽰⽱⽲⽳⽴⽵⽶⽷⽸⽹⽺⽻⽼⽽⽾⽿⾀⾁⾂⾃⾄⾅⾆⾇⾈⾉⾊⾋⾌⾍⾎⾏⾐⾑⾒⾓⾔⾕⾖⾗⾘⾙⾚⾛⾜⾝⾞⾟⾠⾡⾢⾣⾤⾥⾦⾧⾨⾩⾪⾫⾬⾭⾮⾯⾰⾱⾲⾳⾴⾵⾶⾷⾸⾹⾺⾻⾼⾽⾾⾿⿀⿁⿂⿃⿄⿅⿆⿇⿈⿉⿊⿋⿌⿍⿎⿏⿐⿑⿒⿓⿔⿕⿰⿱⿲⿳⿴⿵⿶⿷⿸⿹⿺⿻、。〃〄〈〉《》「」『』【】〒〓〔〕〖〗〘〙〚〛〜〝〞〟〠〰〶〷〽〾〿゛゜゠・㆐㆑㆖㆗㆘㆙㆚㆛㆜㆝㆞㆟㇀㇁㇂㇃㇄㇅㇆㇇㇈㇉㇊㇋㇌㇍㇎㇏㇐㇑㇒㇓㇔㇕㇖㇗㇘㇙㇚㇛㇜㇝㇞㇟㇠㇡㇢㇣㈀㈁㈂㈃㈄㈅㈆㈇㈈㈉㈊㈋㈌㈍㈎㈏㈐㈑㈒㈓㈔㈕㈖㈗㈘㈙㈚㈛㈜㈝㈞㈪㈫㈬㈭㈮㈯㈰㈱㈲㈳㈴㈵㈶㈷㈸㈹㈺㈻㈼㈽㈾㈿㉀㉁㉂㉃㉄㉅㉆㉇㉐㉠㉡㉢㉣㉤㉥㉦㉧㉨㉩㉪㉫㉬㉭㉮㉯㉰㉱㉲㉳㉴㉵㉶㉷㉸㉹㉺㉻㉼㉽㉾㉿㊊㊋㊌㊍㊎㊏㊐㊑㊒㊓㊔㊕㊖㊗㊘㊙㊚㊛㊜㊝㊞㊟㊠㊡㊢㊣㊤㊥㊦㊧㊨㊩㊪㊫㊬㊭㊮㊯㊰㋀㋁㋂㋃㋄㋅㋆㋇㋈㋉㋊㋋㋌㋍㋎㋏㋐㋑㋒㋓㋔㋕㋖㋗㋘㋙㋚㋛㋜㋝㋞㋟㋠㋡㋢㋣㋤㋥㋦㋧㋨㋩㋪㋫㋬㋭㋮㋯㋰㋱㋲㋳㋴㋵㋶㋷㋸㋹㋺㋻㋼㋽㋾㋿㌀㌁㌂㌃㌄㌅㌆㌇㌈㌉㌊㌋㌌㌍㌎㌏㌐㌑㌒㌓㌔㌕㌖㌗㌘㌙㌚㌛㌜㌝㌞㌟㌠㌡㌢㌣㌤㌥㌦㌧㌨㌩㌪㌫㌬㌭㌮㌯㌰㌱㌲㌳㌴㌵㌶㌷㌸㌹㌺㌻㌼㌽㌾㌿㍀㍁㍂㍃㍄㍅㍆㍇㍈㍉㍊㍋㍌㍍㍎㍏㍐㍑㍒㍓㍔㍕㍖㍗㍘㍙㍚㍛㍜㍝㍞㍟㍠㍡㍢㍣㍤㍥㍦㍧㍨㍩㍪㍫㍬㍭㍮㍯㍰㍱㍲㍳㍴㍵㍶㍷㍸㍹㍺㍻㍼㍽㍾㍿㎀㎁㎂㎃㎄㎅㎆㎇㎈㎉㎊㎋㎌㎍㎎㎏㎐㎑㎒㎓㎔㎕㎖㎗㎘㎙㎚㎛㎜㎝㎞㎟㎠㎡㎢㎣㎤㎥㎦㎧㎨㎩㎪㎫㎬㎭㎮㎯㎰㎱㎲㎳㎴㎵㎶㎷㎸㎹㎺㎻㎼㎽㎾㎿㏀㏁㏂㏃㏄㏅㏆㏇㏈㏉㏊㏋㏌㏍㏎㏏㏐㏑㏒㏓㏔㏕㏖㏗㏘㏙㏚㏛㏜㏝㏞㏟㏠㏡㏢㏣㏤㏥㏦㏧㏨㏩㏪㏫㏬㏭㏮㏯㏰㏱㏲㏳㏴㏵㏶㏷㏸㏹㏺㏻㏼㏽㏾㏿䷀䷁䷂䷃䷄䷅䷆䷇䷈䷉䷊䷋䷌䷍䷎䷏䷐䷑䷒䷓䷔䷕䷖䷗䷘䷙䷚䷛䷜䷝䷞䷟䷠䷡䷢䷣䷤䷥䷦䷧䷨䷩䷪䷫䷬䷭䷮䷯䷰䷱䷲䷳䷴䷵䷶䷷䷸䷹䷺䷻䷼䷽䷾䷿꒐꒑꒒꒓꒔꒕꒖꒗꒘꒙꒚꒛꒜꒝꒞꒟꒠꒡꒢꒣꒤꒥꒦꒧꒨꒩꒪꒫꒬꒭꒮꒯꒰꒱꒲꒳꒴꒵꒶꒷꒸꒹꒺꒻꒼꒽꒾꒿꓀꓁꓂꓃꓄꓅꓆꓾꓿꘍꘎꘏꙳꙾꛲꛳꛴꛵꛶꛷꜀꜁꜂꜃꜄꜅꜆꜇꜈꜉꜊꜋꜌꜍꜎꜏꜐꜑꜒꜓꜔꜕꜖꜠꜡꞉꞊꠨꠩꠪꠫꠶꠷꠹꡴꡵꡶꡷꣎꣏꣸꣹꣺꣼꤮꤯꥟꧁꧂꧃꧄꧅꧆꧇꧈꧉꧊꧋꧌꧍꧞꧟꩜꩝꩞꩟꩷꩸꩹꫞꫟꫰꫱꭛꭪꭫꯫﬩﮲﮳﮴﮵﮶﮷﮸﮹﮺﮻﮼﮽﮾﮿﯀﯁﯂﴾﴿﵀﵁﵂﵃﵄﵅﵆﵇﵈﵉﵊﵋﵌﵍﵎﵏﷏﷽﷾﷿︐︑︒︓︔︕︖︗︘︙︰︱︲︳︴︵︶︷︸︹︺︻︼︽︾︿﹀﹁﹂﹃﹄﹅﹆﹇﹈﹉﹊﹋﹌﹍﹎﹏﹐﹑﹒﹔﹕﹖﹗﹘﹙﹚﹛﹜﹝﹞﹟﹠﹡﹢﹣﹤﹥﹦﹨﹪﹫!"#%&'()*+,-./:;<=>?@[\]^_`{|}~⦅⦆。「」、・¬ ̄¦│←↑→↓■○�𐄀𐄁𐄂𐄷𐄸𐄹𐄺𐄻𐄼𐄽𐄾𐄿𐅹𐅺𐅻𐅼𐅽𐅾𐅿𐆀𐆁𐆂𐆃𐆄𐆅𐆆𐆇𐆈𐆉𐆌𐆍𐆎𐆐𐆑𐆒𐆓𐆔𐆕𐆖𐆗𐆘𐆙𐆚𐆛𐆜𐆠𐇐𐇑𐇒𐇓𐇔𐇕𐇖𐇗𐇘𐇙𐇚𐇛𐇜𐇝𐇞𐇟𐇠𐇡𐇢𐇣𐇤𐇥𐇦𐇧𐇨𐇩𐇪𐇫𐇬𐇭𐇮𐇯𐇰𐇱𐇲𐇳𐇴𐇵𐇶𐇷𐇸𐇹𐇺𐇻𐇼𐎟𐏐𐕯𐡗𐡷𐡸𐤟𐤿𐩐𐩑𐩒𐩓𐩔𐩕𐩖𐩗𐩘𐩿𐫈𐫰𐫱𐫲𐫳𐫴𐫵𐫶𐬹𐬺𐬻𐬼𐬽𐬾𐬿𐮙𐮚𐮛𐮜𐺭𐽕𐽖𐽗𐽘𐽙𐾆𐾇𐾈𐾉𑁇𑁈𑁉𑁊𑁋𑁌𑁍𑂻𑂼𑂾𑂿𑃀𑃁𑅀𑅁𑅂𑅃𑅴𑅵𑇅𑇆𑇇𑇈𑇍𑇛𑇝𑇞𑇟𑈸𑈹𑈺𑈻𑈼𑈽𑊩𑑋𑑌𑑍𑑎𑑏𑑚𑑛𑑝𑓆𑗁𑗂𑗃𑗄𑗅𑗆𑗇𑗈𑗉𑗊𑗋𑗌𑗍𑗎𑗏𑗐𑗑𑗒𑗓𑗔𑗕𑗖𑗗𑙁𑙂𑙃𑙠𑙡𑙢𑙣𑙤𑙥𑙦𑙧𑙨𑙩𑙪𑙫𑙬𑚹𑜼𑜽𑜾𑜿𑠻𑥄𑥅𑥆𑧢𑨿𑩀𑩁𑩂𑩃𑩄𑩅𑩆𑪚𑪛𑪜𑪞𑪟𑪠𑪡𑪢𑬀𑬁𑬂𑬃𑬄𑬅𑬆𑬇𑬈𑬉𑱁𑱂𑱃𑱄𑱅𑱰𑱱𑻷𑻸𑽃𑽄𑽅𑽆𑽇𑽈𑽉𑽊𑽋𑽌𑽍𑽎𑽏𑿕𑿖𑿗𑿘𑿙𑿚𑿛𑿜𑿡𑿢𑿣𑿤𑿥𑿦𑿧𑿨𑿩𑿪𑿫𑿬𑿭𑿮𑿯𑿰𑿱𑿿𒑰𒑱𒑲𒑳𒑴𒿱𒿲𖩮𖩯𖫵𖬷𖬸𖬹𖬺𖬻𖬼𖬽𖬾𖬿𖭄𖭅𖺗𖺘𖺙𖺚𖿢𛲜𛲟𜽐𜽑𜽒𜽓𜽔𜽕𜽖𜽗𜽘𜽙𜽚𜽛𜽜𜽝𜽞𜽟𜽠𜽡𜽢𜽣𜽤𜽥𜽦𜽧𜽨𜽩𜽪𜽫𜽬𜽭𜽮𜽯𜽰𜽱𜽲𜽳𜽴𜽵𜽶𜽷𜽸𜽹𜽺𜽻𜽼𜽽𜽾𜽿𜾀𜾁𜾂𜾃𜾄𜾅𜾆𜾇𜾈𜾉𜾊𜾋𜾌𜾍𜾎𜾏𜾐𜾑𜾒𜾓𜾔𜾕𜾖𜾗𜾘𜾙𜾚𜾛𜾜𜾝𜾞𜾟𜾠𜾡𜾢𜾣𜾤𜾥𜾦𜾧𜾨𜾩𜾪𜾫𜾬𜾭𜾮𜾯𜾰𜾱𜾲𜾳𜾴𜾵𜾶𜾷𜾸𜾹𜾺𜾻𜾼𜾽𜾾𜾿𜿀𜿁𜿂𜿃𝀀𝀁𝀂𝀃𝀄𝀅𝀆𝀇𝀈𝀉𝀊𝀋𝀌𝀍𝀎𝀏𝀐𝀑𝀒𝀓𝀔𝀕𝀖𝀗𝀘𝀙𝀚𝀛𝀜𝀝𝀞𝀟𝀠𝀡𝀢𝀣𝀤𝀥𝀦𝀧𝀨𝀩𝀪𝀫𝀬𝀭𝀮𝀯𝀰𝀱𝀲𝀳𝀴𝀵𝀶𝀷𝀸𝀹𝀺𝀻𝀼𝀽𝀾𝀿𝁀𝁁𝁂𝁃𝁄𝁅𝁆𝁇𝁈𝁉𝁊𝁋𝁌𝁍𝁎𝁏𝁐𝁑𝁒𝁓𝁔𝁕𝁖𝁗𝁘𝁙𝁚𝁛𝁜𝁝𝁞𝁟𝁠𝁡𝁢𝁣𝁤𝁥𝁦𝁧𝁨𝁩𝁪𝁫𝁬𝁭𝁮𝁯𝁰𝁱𝁲𝁳𝁴𝁵𝁶𝁷𝁸𝁹𝁺𝁻𝁼𝁽𝁾𝁿𝂀𝂁𝂂𝂃𝂄𝂅𝂆𝂇𝂈𝂉𝂊𝂋𝂌𝂍𝂎𝂏𝂐𝂑𝂒𝂓𝂔𝂕𝂖𝂗𝂘𝂙𝂚𝂛𝂜𝂝𝂞𝂟𝂠𝂡𝂢𝂣𝂤𝂥𝂦𝂧𝂨𝂩𝂪𝂫𝂬𝂭𝂮𝂯𝂰𝂱𝂲𝂳𝂴𝂵𝂶𝂷𝂸𝂹𝂺𝂻𝂼𝂽𝂾𝂿𝃀𝃁𝃂𝃃𝃄𝃅𝃆𝃇𝃈𝃉𝃊𝃋𝃌𝃍𝃎𝃏𝃐𝃑𝃒𝃓𝃔𝃕𝃖𝃗𝃘𝃙𝃚𝃛𝃜𝃝𝃞𝃟𝃠𝃡𝃢𝃣𝃤𝃥𝃦𝃧𝃨𝃩𝃪𝃫𝃬𝃭𝃮𝃯𝃰𝃱𝃲𝃳𝃴𝃵𝄀𝄁𝄂𝄃𝄄𝄅𝄆𝄇𝄈𝄉𝄊𝄋𝄌𝄍𝄎𝄏𝄐𝄑𝄒𝄓𝄔𝄕𝄖𝄗𝄘𝄙𝄚𝄛𝄜𝄝𝄞𝄟𝄠𝄡𝄢𝄣𝄤𝄥𝄦𝄩𝄪𝄫𝄬𝄭𝄮𝄯𝄰𝄱𝄲𝄳𝄴𝄵𝄶𝄷𝄸𝄹𝄺𝄻𝄼𝄽𝄾𝄿𝅀𝅁𝅂𝅃𝅄𝅅𝅆𝅇𝅈𝅉𝅊𝅋𝅌𝅍𝅎𝅏𝅐𝅑𝅒𝅓𝅔𝅕𝅖𝅗𝅘𝅙𝅚𝅛𝅜𝅝𝅗𝅥𝅘𝅥𝅘𝅥𝅮𝅘𝅥𝅯𝅘𝅥𝅰𝅘𝅥𝅱𝅘𝅥𝅲𝅪𝅫𝅬𝆃𝆄𝆌𝆍𝆎𝆏𝆐𝆑𝆒𝆓𝆔𝆕𝆖𝆗𝆘𝆙𝆚𝆛𝆜𝆝𝆞𝆟𝆠𝆡𝆢𝆣𝆤𝆥𝆦𝆧𝆨𝆩𝆮𝆯𝆰𝆱𝆲𝆳𝆴𝆵𝆶𝆷𝆸𝆹𝆺𝆹𝅥𝆺𝅥𝆹𝅥𝅮𝆺𝅥𝅮𝆹𝅥𝅯𝆺𝅥𝅯𝇁𝇂𝇃𝇄𝇅𝇆𝇇𝇈𝇉𝇊𝇋𝇌𝇍𝇎𝇏𝇐𝇑𝇒𝇓𝇔𝇕𝇖𝇗𝇘𝇙𝇚𝇛𝇜𝇝𝇞𝇟𝇠𝇡𝇢𝇣𝇤𝇥𝇦𝇧𝇨𝇩𝇪𝈀𝈁𝈂𝈃𝈄𝈅𝈆𝈇𝈈𝈉𝈊𝈋𝈌𝈍𝈎𝈏𝈐𝈑𝈒𝈓𝈔𝈕𝈖𝈗𝈘𝈙𝈚𝈛𝈜𝈝𝈞𝈟𝈠𝈡𝈢𝈣𝈤𝈥𝈦𝈧𝈨𝈩𝈪𝈫𝈬𝈭𝈮𝈯𝈰𝈱𝈲𝈳𝈴𝈵𝈶𝈷𝈸𝈹𝈺𝈻𝈼𝈽𝈾𝈿𝉀𝉁𝉅𝌀𝌁𝌂𝌃𝌄𝌅𝌆𝌇𝌈𝌉𝌊𝌋𝌌𝌍𝌎𝌏𝌐𝌑𝌒𝌓𝌔𝌕𝌖𝌗𝌘𝌙𝌚𝌛𝌜𝌝𝌞𝌟𝌠𝌡𝌢𝌣𝌤𝌥𝌦𝌧𝌨𝌩𝌪𝌫𝌬𝌭𝌮𝌯𝌰𝌱𝌲𝌳𝌴𝌵𝌶𝌷𝌸𝌹𝌺𝌻𝌼𝌽𝌾𝌿𝍀𝍁𝍂𝍃𝍄𝍅𝍆𝍇𝍈𝍉𝍊𝍋𝍌𝍍𝍎𝍏𝍐𝍑𝍒𝍓𝍔𝍕𝍖𝛁𝛛𝛻𝜕𝜵𝝏𝝯𝞉𝞩𝟃𝠀𝠁𝠂𝠃𝠄𝠅𝠆𝠇𝠈𝠉𝠊𝠋𝠌𝠍𝠎𝠏𝠐𝠑𝠒𝠓𝠔𝠕𝠖𝠗𝠘𝠙𝠚𝠛𝠜𝠝𝠞𝠟𝠠𝠡𝠢𝠣𝠤𝠥𝠦𝠧𝠨𝠩𝠪𝠫𝠬𝠭𝠮𝠯𝠰𝠱𝠲𝠳𝠴𝠵𝠶𝠷𝠸𝠹𝠺𝠻𝠼𝠽𝠾𝠿𝡀𝡁𝡂𝡃𝡄𝡅𝡆𝡇𝡈𝡉𝡊𝡋𝡌𝡍𝡎𝡏𝡐𝡑𝡒𝡓𝡔𝡕𝡖𝡗𝡘𝡙𝡚𝡛𝡜𝡝𝡞𝡟𝡠𝡡𝡢𝡣𝡤𝡥𝡦𝡧𝡨𝡩𝡪𝡫𝡬𝡭𝡮𝡯𝡰𝡱𝡲𝡳𝡴𝡵𝡶𝡷𝡸𝡹𝡺𝡻𝡼𝡽𝡾𝡿𝢀𝢁𝢂𝢃𝢄𝢅𝢆𝢇𝢈𝢉𝢊𝢋𝢌𝢍𝢎𝢏𝢐𝢑𝢒𝢓𝢔𝢕𝢖𝢗𝢘𝢙𝢚𝢛𝢜𝢝𝢞𝢟𝢠𝢡𝢢𝢣𝢤𝢥𝢦𝢧𝢨𝢩𝢪𝢫𝢬𝢭𝢮𝢯𝢰𝢱𝢲𝢳𝢴𝢵𝢶𝢷𝢸𝢹𝢺𝢻𝢼𝢽𝢾𝢿𝣀𝣁𝣂𝣃𝣄𝣅𝣆𝣇𝣈𝣉𝣊𝣋𝣌𝣍𝣎𝣏𝣐𝣑𝣒𝣓𝣔𝣕𝣖𝣗𝣘𝣙𝣚𝣛𝣜𝣝𝣞𝣟𝣠𝣡𝣢𝣣𝣤𝣥𝣦𝣧𝣨𝣩𝣪𝣫𝣬𝣭𝣮𝣯𝣰𝣱𝣲𝣳𝣴𝣵𝣶𝣷𝣸𝣹𝣺𝣻𝣼𝣽𝣾𝣿𝤀𝤁𝤂𝤃𝤄𝤅𝤆𝤇𝤈𝤉𝤊𝤋𝤌𝤍𝤎𝤏𝤐𝤑𝤒𝤓𝤔𝤕𝤖𝤗𝤘𝤙𝤚𝤛𝤜𝤝𝤞𝤟𝤠𝤡𝤢𝤣𝤤𝤥𝤦𝤧𝤨𝤩𝤪𝤫𝤬𝤭𝤮𝤯𝤰𝤱𝤲𝤳𝤴𝤵𝤶𝤷𝤸𝤹𝤺𝤻𝤼𝤽𝤾𝤿𝥀𝥁𝥂𝥃𝥄𝥅𝥆𝥇𝥈𝥉𝥊𝥋𝥌𝥍𝥎𝥏𝥐𝥑𝥒𝥓𝥔𝥕𝥖𝥗𝥘𝥙𝥚𝥛𝥜𝥝𝥞𝥟𝥠𝥡𝥢𝥣𝥤𝥥𝥦𝥧𝥨𝥩𝥪𝥫𝥬𝥭𝥮𝥯𝥰𝥱𝥲𝥳𝥴𝥵𝥶𝥷𝥸𝥹𝥺𝥻𝥼𝥽𝥾𝥿𝦀𝦁𝦂𝦃𝦄𝦅𝦆𝦇𝦈𝦉𝦊𝦋𝦌𝦍𝦎𝦏𝦐𝦑𝦒𝦓𝦔𝦕𝦖𝦗𝦘𝦙𝦚𝦛𝦜𝦝𝦞𝦟𝦠𝦡𝦢𝦣𝦤𝦥𝦦𝦧𝦨𝦩𝦪𝦫𝦬𝦭𝦮𝦯𝦰𝦱𝦲𝦳𝦴𝦵𝦶𝦷𝦸𝦹𝦺𝦻𝦼𝦽𝦾𝦿𝧀𝧁𝧂𝧃𝧄𝧅𝧆𝧇𝧈𝧉𝧊𝧋𝧌𝧍𝧎𝧏𝧐𝧑𝧒𝧓𝧔𝧕𝧖𝧗𝧘𝧙𝧚𝧛𝧜𝧝𝧞𝧟𝧠𝧡𝧢𝧣𝧤𝧥𝧦𝧧𝧨𝧩𝧪𝧫𝧬𝧭𝧮𝧯𝧰𝧱𝧲𝧳𝧴𝧵𝧶𝧷𝧸𝧹𝧺𝧻𝧼𝧽𝧾𝧿𝨷𝨸𝨹𝨺𝩭𝩮𝩯𝩰𝩱𝩲𝩳𝩴𝩶𝩷𝩸𝩹𝩺𝩻𝩼𝩽𝩾𝩿𝪀𝪁𝪂𝪃𝪅𝪆𝪇𝪈𝪉𝪊𝪋𞅏𞥞𞥟𞲬𞴮𞻰𞻱🀀🀁🀂🀃🀄🀅🀆🀇🀈🀉🀊🀋🀌🀍🀎🀏🀐🀑🀒🀓🀔🀕🀖🀗🀘🀙🀚🀛🀜🀝🀞🀟🀠🀡🀢🀣🀤🀥🀦🀧🀨🀩🀪🀫🀰🀱🀲🀳🀴🀵🀶🀷🀸🀹🀺🀻🀼🀽🀾🀿🁀🁁🁂🁃🁄🁅🁆🁇🁈🁉🁊🁋🁌🁍🁎🁏🁐🁑🁒🁓🁔🁕🁖🁗🁘🁙🁚🁛🁜🁝🁞🁟🁠🁡🁢🁣🁤🁥🁦🁧🁨🁩🁪🁫🁬🁭🁮🁯🁰🁱🁲🁳🁴🁵🁶🁷🁸🁹🁺🁻🁼🁽🁾🁿🂀🂁🂂🂃🂄🂅🂆🂇🂈🂉🂊🂋🂌🂍🂎🂏🂐🂑🂒🂓🂠🂡🂢🂣🂤🂥🂦🂧🂨🂩🂪🂫🂬🂭🂮🂱🂲🂳🂴🂵🂶🂷🂸🂹🂺🂻🂼🂽🂾🂿🃁🃂🃃🃄🃅🃆🃇🃈🃉🃊🃋🃌🃍🃎🃏🃑🃒🃓🃔🃕🃖🃗🃘🃙🃚🃛🃜🃝🃞🃟🃠🃡🃢🃣🃤🃥🃦🃧🃨🃩🃪🃫🃬🃭🃮🃯🃰🃱🃲🃳🃴🃵🄍🄎🄏🄐🄑🄒🄓🄔🄕🄖🄗🄘🄙🄚🄛🄜🄝🄞🄟🄠🄡🄢🄣🄤🄥🄦🄧🄨🄩🄪🄫🄬🄭🄮🄯🅊🅋🅌🅍🅎🅏🅪🅫🅬🅭🅮🅯🆊🆋🆌🆍🆎🆏🆐🆑🆒🆓🆔🆕🆖🆗🆘🆙🆚🆛🆜🆝🆞🆟🆠🆡🆢🆣🆤🆥🆦🆧🆨🆩🆪🆫🆬🆭🇦🇧🇨🇩🇪🇫🇬🇭🇮🇯🇰🇱🇲🇳🇴🇵🇶🇷🇸🇹🇺🇻🇼🇽🇾🇿🈀🈁🈂🈐🈑🈒🈓🈔🈕🈖🈗🈘🈙🈚🈛🈜🈝🈞🈟🈠🈡🈢🈣🈤🈥🈦🈧🈨🈩🈪🈫🈬🈭🈮🈯🈰🈱🈲🈳🈴🈵🈶🈷🈸🈹🈺🈻🉀🉁🉂🉃🉄🉅🉆🉇🉈🉐🉑🉠🉡🉢🉣🉤🉥🌀🌁🌂🌃🌄🌅🌆🌇🌈🌉🌊🌋🌌🌍🌎🌏🌐🌑🌒🌓🌔🌕🌖🌗🌘🌙🌚🌛🌜🌝🌞🌟🌠🌡🌢🌣🌤🌥🌦🌧🌨🌩🌪🌫🌬🌭🌮🌯🌰🌱🌲🌳🌴🌵🌶🌷🌸🌹🌺🌻🌼🌽🌾🌿🍀🍁🍂🍃🍄🍅🍆🍇🍈🍉🍊🍋🍌🍍🍎🍏🍐🍑🍒🍓🍔🍕🍖🍗🍘🍙🍚🍛🍜🍝🍞🍟🍠🍡🍢🍣🍤🍥🍦🍧🍨🍩🍪🍫🍬🍭🍮🍯🍰🍱🍲🍳🍴🍵🍶🍷🍸🍹🍺🍻🍼🍽🍾🍿🎀🎁🎂🎃🎄🎅🎆🎇🎈🎉🎊🎋🎌🎍🎎🎏🎐🎑🎒🎓🎔🎕🎖🎗🎘🎙🎚🎛🎜🎝🎞🎟🎠🎡🎢🎣🎤🎥🎦🎧🎨🎩🎪🎫🎬🎭🎮🎯🎰🎱🎲🎳🎴🎵🎶🎷🎸🎹🎺🎻🎼🎽🎾🎿🏀🏁🏂🏃🏄🏅🏆🏇🏈🏉🏊🏋🏌🏍🏎🏏🏐🏑🏒🏓🏔🏕🏖🏗🏘🏙🏚🏛🏜🏝🏞🏟🏠🏡🏢🏣🏤🏥🏦🏧🏨🏩🏪🏫🏬🏭🏮🏯🏰🏱🏲🏳🏴🏵🏶🏷🏸🏹🏺🏻🏼🏽🏾🏿🐀🐁🐂🐃🐄🐅🐆🐇🐈🐉🐊🐋🐌🐍🐎🐏🐐🐑🐒🐓🐔🐕🐖🐗🐘🐙🐚🐛🐜🐝🐞🐟🐠🐡🐢🐣🐤🐥🐦🐧🐨🐩🐪🐫🐬🐭🐮🐯🐰🐱🐲🐳🐴🐵🐶🐷🐸🐹🐺🐻🐼🐽🐾🐿👀👁👂👃👄👅👆👇👈👉👊👋👌👍👎👏👐👑👒👓👔👕👖👗👘👙👚👛👜👝👞👟👠👡👢👣👤👥👦👧👨👩👪👫👬👭👮👯👰👱👲👳👴👵👶👷👸👹👺👻👼👽👾👿💀💁💂💃💄💅💆💇💈💉💊💋💌💍💎💏💐💑💒💓💔💕💖💗💘💙💚💛💜💝💞💟💠💡💢💣💤💥💦💧💨💩💪💫💬💭💮💯💰💱💲💳💴💵💶💷💸💹💺💻💼💽💾💿📀📁📂📃📄📅📆📇📈📉📊📋📌📍📎📏📐📑📒📓📔📕📖📗📘📙📚📛📜📝📞📟📠📡📢📣📤📥📦📧📨📩📪📫📬📭📮📯📰📱📲📳📴📵📶📷📸📹📺📻📼📽📾📿🔀🔁🔂🔃🔄🔅🔆🔇🔈🔉🔊🔋🔌🔍🔎🔏🔐🔑🔒🔓🔔🔕🔖🔗🔘🔙🔚🔛🔜🔝🔞🔟🔠🔡🔢🔣🔤🔥🔦🔧🔨🔩🔪🔫🔬🔭🔮🔯🔰🔱🔲🔳🔴🔵🔶🔷🔸🔹🔺🔻🔼🔽🔾🔿🕀🕁🕂🕃🕄🕅🕆🕇🕈🕉🕊🕋🕌🕍🕎🕏🕐🕑🕒🕓🕔🕕🕖🕗🕘🕙🕚🕛🕜🕝🕞🕟🕠🕡🕢🕣🕤🕥🕦🕧🕨🕩🕪🕫🕬🕭🕮🕯🕰🕱🕲🕳🕴🕵🕶🕷🕸🕹🕺🕻🕼🕽🕾🕿🖀🖁🖂🖃🖄🖅🖆🖇🖈🖉🖊🖋🖌🖍🖎🖏🖐🖑🖒🖓🖔🖕🖖🖗🖘🖙🖚🖛🖜🖝🖞🖟🖠🖡🖢🖣🖤🖥🖦🖧🖨🖩🖪🖫🖬🖭🖮🖯🖰🖱🖲🖳🖴🖵🖶🖷🖸🖹🖺🖻🖼🖽🖾🖿🗀🗁🗂🗃🗄🗅🗆🗇🗈🗉🗊🗋🗌🗍🗎🗏🗐🗑🗒🗓🗔🗕🗖🗗🗘🗙🗚🗛🗜🗝🗞🗟🗠🗡🗢🗣🗤🗥🗦🗧🗨🗩🗪🗫🗬🗭🗮🗯🗰🗱🗲🗳🗴🗵🗶🗷🗸🗹🗺🗻🗼🗽🗾🗿😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😔😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😤😥😦😧😨😩😪😫😬😭😮😯😰😱😲😳😴😵😶😷😸😹😺😻😼😽😾😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏🙐🙑🙒🙓🙔🙕🙖🙗🙘🙙🙚🙛🙜🙝🙞🙟🙠🙡🙢🙣🙤🙥🙦🙧🙨🙩🙪🙫🙬🙭🙮🙯🙰🙱🙲🙳🙴🙵🙶🙷🙸🙹🙺🙻🙼🙽🙾🙿🚀🚁🚂🚃🚄🚅🚆🚇🚈🚉🚊🚋🚌🚍🚎🚏🚐🚑🚒🚓🚔🚕🚖🚗🚘🚙🚚🚛🚜🚝🚞🚟🚠🚡🚢🚣🚤🚥🚦🚧🚨🚩🚪🚫🚬🚭🚮🚯🚰🚱🚲🚳🚴🚵🚶🚷🚸🚹🚺🚻🚼🚽🚾🚿🛀🛁🛂🛃🛄🛅🛆🛇🛈🛉🛊🛋🛌🛍🛎🛏🛐🛑🛒🛓🛔🛕🛖🛗🛜🛝🛞🛟🛠🛡🛢🛣🛤🛥🛦🛧🛨🛩🛪🛫🛬🛰🛱🛲🛳🛴🛵🛶🛷🛸🛹🛺🛻🛼🜀🜁🜂🜃🜄🜅🜆🜇🜈🜉🜊🜋🜌🜍🜎🜏🜐🜑🜒🜓🜔🜕🜖🜗🜘🜙🜚🜛🜜🜝🜞🜟🜠🜡🜢🜣🜤🜥🜦🜧🜨🜩🜪🜫🜬🜭🜮🜯🜰🜱🜲🜳🜴🜵🜶🜷🜸🜹🜺🜻🜼🜽🜾🜿🝀🝁🝂🝃🝄🝅🝆🝇🝈🝉🝊🝋🝌🝍🝎🝏🝐🝑🝒🝓🝔🝕🝖🝗🝘🝙🝚🝛🝜🝝🝞🝟🝠🝡🝢🝣🝤🝥🝦🝧🝨🝩🝪🝫🝬🝭🝮🝯🝰🝱🝲🝳🝴🝵🝶🝻🝼🝽🝾🝿🞀🞁🞂🞃🞄🞅🞆🞇🞈🞉🞊🞋🞌🞍🞎🞏🞐🞑🞒🞓🞔🞕🞖🞗🞘🞙🞚🞛🞜🞝🞞🞟🞠🞡🞢🞣🞤🞥🞦🞧🞨🞩🞪🞫🞬🞭🞮🞯🞰🞱🞲🞳🞴🞵🞶🞷🞸🞹🞺🞻🞼🞽🞾🞿🟀🟁🟂🟃🟄🟅🟆🟇🟈🟉🟊🟋🟌🟍🟎🟏🟐🟑🟒🟓🟔🟕🟖🟗🟘🟙🟠🟡🟢🟣🟤🟥🟦🟧🟨🟩🟪🟫🟰🠀🠁🠂🠃🠄🠅🠆🠇🠈🠉🠊🠋🠐🠑🠒🠓🠔🠕🠖🠗🠘🠙🠚🠛🠜🠝🠞🠟🠠🠡🠢🠣🠤🠥🠦🠧🠨🠩🠪🠫🠬🠭🠮🠯🠰🠱🠲🠳🠴🠵🠶🠷🠸🠹🠺🠻🠼🠽🠾🠿🡀🡁🡂🡃🡄🡅🡆🡇🡐🡑🡒🡓🡔🡕🡖🡗🡘🡙🡠🡡🡢🡣🡤🡥🡦🡧🡨🡩🡪🡫🡬🡭🡮🡯🡰🡱🡲🡳🡴🡵🡶🡷🡸🡹🡺🡻🡼🡽🡾🡿🢀🢁🢂🢃🢄🢅🢆🢇🢐🢑🢒🢓🢔🢕🢖🢗🢘🢙🢚🢛🢜🢝🢞🢟🢠🢡🢢🢣🢤🢥🢦🢧🢨🢩🢪🢫🢬🢭🢰🢱🤀🤁🤂🤃🤄🤅🤆🤇🤈🤉🤊🤋🤌🤍🤎🤏🤐🤑🤒🤓🤔🤕🤖🤗🤘🤙🤚🤛🤜🤝🤞🤟🤠🤡🤢🤣🤤🤥🤦🤧🤨🤩🤪🤫🤬🤭🤮🤯🤰🤱🤲🤳🤴🤵🤶🤷🤸🤹🤺🤻🤼🤽🤾🤿🥀🥁🥂🥃🥄🥅🥆🥇🥈🥉🥊🥋🥌🥍🥎🥏🥐🥑🥒🥓🥔🥕🥖🥗🥘🥙🥚🥛🥜🥝🥞🥟🥠🥡🥢🥣🥤🥥🥦🥧🥨🥩🥪🥫🥬🥭🥮🥯🥰🥱🥲🥳🥴🥵🥶🥷🥸🥹🥺🥻🥼🥽🥾🥿🦀🦁🦂🦃🦄🦅🦆🦇🦈🦉🦊🦋🦌🦍🦎🦏🦐🦑🦒🦓🦔🦕🦖🦗🦘🦙🦚🦛🦜🦝🦞🦟🦠🦡🦢🦣🦤🦥🦦🦧🦨🦩🦪🦫🦬🦭🦮🦯🦰🦱🦲🦳🦴🦵🦶🦷🦸🦹🦺🦻🦼🦽🦾🦿🧀🧁🧂🧃🧄🧅🧆🧇🧈🧉🧊🧋🧌🧍🧎🧏🧐🧑🧒🧓🧔🧕🧖🧗🧘🧙🧚🧛🧜🧝🧞🧟🧠🧡🧢🧣🧤🧥🧦🧧🧨🧩🧪🧫🧬🧭🧮🧯🧰🧱🧲🧳🧴🧵🧶🧷🧸🧹🧺🧻🧼🧽🧾🧿🨀🨁🨂🨃🨄🨅🨆🨇🨈🨉🨊🨋🨌🨍🨎🨏🨐🨑🨒🨓🨔🨕🨖🨗🨘🨙🨚🨛🨜🨝🨞🨟🨠🨡🨢🨣🨤🨥🨦🨧🨨🨩🨪🨫🨬🨭🨮🨯🨰🨱🨲🨳🨴🨵🨶🨷🨸🨹🨺🨻🨼🨽🨾🨿🩀🩁🩂🩃🩄🩅🩆🩇🩈🩉🩊🩋🩌🩍🩎🩏🩐🩑🩒🩓🩠🩡🩢🩣🩤🩥🩦🩧🩨🩩🩪🩫🩬🩭🩰🩱🩲🩳🩴🩵🩶🩷🩸🩹🩺🩻🩼🪀🪁🪂🪃🪄🪅🪆🪇🪈🪐🪑🪒🪓🪔🪕🪖🪗🪘🪙🪚🪛🪜🪝🪞🪟🪠🪡🪢🪣🪤🪥🪦🪧🪨🪩🪪🪫🪬🪭🪮🪯🪰🪱🪲🪳🪴🪵🪶🪷🪸🪹🪺🪻🪼🪽🪿🫀🫁🫂🫃🫄🫅🫎🫏🫐🫑🫒🫓🫔🫕🫖🫗🫘🫙🫚🫛🫠🫡🫢🫣🫤🫥🫦🫧🫨🫰🫱🫲🫳🫴🫵🫶🫷🫸🬀🬁🬂🬃🬄🬅🬆🬇🬈🬉🬊🬋🬌🬍🬎🬏🬐🬑🬒🬓🬔🬕🬖🬗🬘🬙🬚🬛🬜🬝🬞🬟🬠🬡🬢🬣🬤🬥🬦🬧🬨🬩🬪🬫🬬🬭🬮🬯🬰🬱🬲🬳🬴🬵🬶🬷🬸🬹🬺🬻🬼🬽🬾🬿🭀🭁🭂🭃🭄🭅🭆🭇🭈🭉🭊🭋🭌🭍🭎🭏🭐🭑🭒🭓🭔🭕🭖🭗🭘🭙🭚🭛🭜🭝🭞🭟🭠🭡🭢🭣🭤🭥🭦🭧🭨🭩🭪🭫🭬🭭🭮🭯🭰🭱🭲🭳🭴🭵🭶🭷🭸🭹🭺🭻🭼🭽🭾🭿🮀🮁🮂🮃🮄🮅🮆🮇🮈🮉🮊🮋🮌🮍🮎🮏🮐🮑🮒🮔🮕🮖🮗🮘🮙🮚🮛🮜🮝🮞🮟🮠🮡🮢🮣🮤🮥🮦🮧🮨🮩🮪🮫🮬🮭🮮🮯🮰🮱🮲🮳🮴🮵🮶🮷🮸🮹🮺🮻🮼🮽🮾🮿🯀🯁🯂🯃🯄🯅🯆🯇🯈🯉🯊"""
|
11
|
+
# https://www.compart.com/en/unicode/category
|
12
|
+
# https://unicode.org/Public/UNIDATA/UnicodeData.txt
|
13
|
+
|
14
|
+
# `\p{posix_punct}` character class
|
15
|
+
POSIX_PUNCT = r"""-!"#$%&'()*+,./:;<=>?@[\]^_`{|}~"""
|
16
|
+
ALL_PUNCT_RANGES = "".join(find_unicode_ranges(POSIX_PUNCT + UNICODE_PUNCT))
|
17
|
+
SENTENCE_PUNCT = """.?!:;'"()[-]“”·…"""
|
18
|
+
|
19
|
+
|
20
|
+
LINKU = Path(__file__).resolve().parent / Path("linku.json")
|
21
|
+
SANDBOX = Path(__file__).resolve().parent / Path("sandbox.json")
|
22
|
+
|
23
|
+
VOWELS = "aeiou"
|
24
|
+
CONSONANTS = "jklmnpstw"
|
25
|
+
ALPHABET = VOWELS + CONSONANTS
|
26
|
+
|
27
|
+
LANGUAGE = "english" # for NLTK
|
28
|
+
|
29
|
+
"""Commonly occurring strings which are some kind of valid Toki Pona or external token"""
|
30
|
+
ALLOWABLES = {
|
31
|
+
"cw", # Content Warning
|
32
|
+
"x", # ala
|
33
|
+
"y", # anu
|
34
|
+
"kxk", # ken ala ken
|
35
|
+
"wxw", # wile ala wile
|
36
|
+
}
|
37
|
+
|
38
|
+
with open(LINKU) as f:
|
39
|
+
linku: Dict[str, Dict[str, str]] = json.loads(f.read())
|
40
|
+
NIMI_PU: List[str] = [d["word"] for d in linku.values() if d["book"] == "pu"]
|
41
|
+
NIMI_PU_ALE: List[str] = NIMI_PU + ["namako", "kin", "oko"]
|
42
|
+
NIMI_LINKU: List[str] = [
|
43
|
+
d["word"] for d in linku.values() if d["usage_category"] in ["core", "common"]
|
44
|
+
]
|
45
|
+
NIMI_LINKU_ALE: List[str] = [d["word"] for d in linku.values()]
|
46
|
+
|
47
|
+
with open(SANDBOX) as f:
|
48
|
+
sandbox: Dict[str, Dict[str, str]] = json.loads(f.read())
|
49
|
+
NIMI_LINKU_SANDBOX: List[str] = [d["word"] for d in sandbox.values()]
|
50
|
+
|
51
|
+
del linku
|
52
|
+
del sandbox
|
53
|
+
|
54
|
+
__all__ = [
|
55
|
+
"ALPHABET",
|
56
|
+
"CONSONANTS",
|
57
|
+
"NIMI_LINKU",
|
58
|
+
"NIMI_LINKU_ALE",
|
59
|
+
"NIMI_LINKU_SANDBOX",
|
60
|
+
"NIMI_PU",
|
61
|
+
"NIMI_PU_ALE",
|
62
|
+
"VOWELS",
|
63
|
+
"UNICODE_PUNCT",
|
64
|
+
"ALLOWABLES",
|
65
|
+
"POSIX_PUNCT",
|
66
|
+
"",
|
67
|
+
]
|