sonatoki 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

sonatoki/syllabic.txt

@@ -0,0 +1,297 @@
+alamo
+alan
+alike
+alone
+ama
+amen
+ami
+amin
+amino
+amo
+amuse
+ana
+ane
+ani
+anise
+anon
+antelope
+antena
+anti
+ape
+apetite
+apolo
+asasin
+asasinate
+asimilate
+asinine
+asume
+ate
+awake
+awaken
+awe
+awesome
+awoke
+eliminate
+elite
+elope
+enema
+eta
+ewe
+iluminate
+imense
+imitate
+imune
+inmate
+insane
+insulin
+intake
+intense
+intimate
+into
+isolate
+jake
+jane
+japan
+jese
+jetison
+jin
+joke
+juke
+kale
+kane
+kapa
+keno
+kilo
+kimono
+kite
+kiten
+kiwi
+lake
+lama
+lame
+lane
+late
+latino
+lemon
+leno
+lese
+lesen
+leson
+like
+likewise
+lima
+lime
+limo
+lin
+line
+linen
+lite
+lone
+lonesome
+lose
+losen
+lote
+loto
+lowe
+lulu
+luna
+make
+male
+man
+mana
+manate
+manila
+manipulate
+mano
+masa
+mason
+mate
+matine
+melon
+memento
+memo
+men
+mensa
+menu
+mesa
+meta
+mike
+mile
+milo
+mime
+mina
+mine
+mini
+minute
+misile
+misuse
+mite
+miten
+mojo
+mola
+mole
+moma
+momento
+mon
+mono
+monson
+monte
+mope
+mose
+mote
+moto
+mule
+mumu
+muse
+mutilate
+muton
+name
+namesake
+nan
+nana
+nine
+nineten
+ninja
+nite
+nome
+nominate
+nomine
+non
+none
+nonsense
+nope
+nose
+note
+nuke
+nun
+ole
+omelete
+omen
+one
+onto
+opose
+oposite
+ose
+oto
+otoman
+pajama
+pale
+palete
+palomino
+panama
+pane
+papa
+pate
+paten
+pele
+pen
+pene
+peninsula
+petite
+pike
+pile
+pin
+pina
+pinata
+pine
+pinto
+pipe
+pipeline
+poke
+pole
+polen
+polite
+polo
+polute
+ponton
+popa
+pope
+pose
+potato
+puke
+pun
+sake
+saki
+salami
+sale
+salina
+saline
+salon
+salute
+same
+sane
+santo
+satelite
+satin
+semen
+semi
+sen
+senate
+senile
+sense
+sepuku
+sesame
+simulate
+sine
+site
+sole
+solo
+some
+sometime
+son
+sonata
+sulen
+sumo
+sumon
+sun
+sunken
+suntan
+supose
+take
+taken
+takin
+tale
+tame
+tape
+tate
+tato
+ten
+tense
+tiki
+tile
+time
+timeline
+tin
+titan
+toke
+token
+tomato
+tome
+ton
+tone
+tote
+tule
+tuna
+tune
+tuti
+tutu
+unite
+unlike
+unsen
+unto
+unwise
+upon
+use
+wake
+waken
+wala
+wanton
+win
+wine
+wipe
+wise
+woke
+woken
+woman
+women
+won

sonatoki/utils.py

@@ -1,5 +1,4 @@
 # STL
-import re
 import itertools
 from typing import Set, List, TypeVar, Iterable
 
@@ -87,58 +86,3 @@ def overlapping_ntuples(iterable: Iterable[T], n: int) -> Iterable[T]:
 
     # ends when any iter is empty; all groups will be same size
     return zip(*teed)
-
-
-if __name__ == "__main__":
-    """Helper script to fetch UNICODE_PUNCT in constants.py."""
-
-    PUNCT_CATEGORIES = {
-        "Pc",
-        "Pd",
-        "Pe",
-        "Pf",
-        "Pi",
-        "Po",
-        "Ps",
-        "Sm",
-        "Sk",
-        "Sc",
-        "So",
-    }
-    # Connector, Dash, Close (end), Final, Initial, Other, Open (sOpen), Math, Modifier (kModifier), Currency, Other
-
-    # NOTE: UnicodeData.txt lists character ranges if there would be many characters.
-    # (e.g. CJK Ideograph, First at 4E00 and CJK Ideograph, Last at 9FFF).
-    # This does not apply to any currently defined punctuation category.
-
-    EXCEPTION_RANGES = re.compile(r"""[Ⓐ-ⓩ🄰-🅉🅐-🅩🅰-🆉]+""")
-    # These groups are in Symbol other (So) but are not part of `\p{Punctuation}`
-    # NOTE: There are many characters which look like writing characters but are not. Examples:
-    # - kangxi radicals from ⺀ to ⿕ which are for demonstration
-    # - circled katakana from  to ㋾ which... shouldn't be in \p{Punctuation} but oh well
-
-    def is_punctuation(data: List[str]):
-        return data[2] in PUNCT_CATEGORIES
-
-    def get_character(data: List[str]):
-        return chr(int(data[0], 16))
-
-    def is_exception(c: str):
-        return not not re.fullmatch(EXCEPTION_RANGES, c)
-
-    # http://www.unicode.org/Public/UNIDATA/UnicodeData.txt
-    unicode_punctuation = ""
-    with open("UnicodeData.txt", "r") as f:
-        for line in f:
-            data = line.split(";")
-            if not is_punctuation(data):
-                continue
-
-            char = get_character(data)
-            if is_exception(char):
-                continue
-
-            unicode_punctuation += char
-
-    with open("UnicodePunctuation.txt", "w") as f:
-        _ = f.write(unicode_punctuation)

{sonatoki-0.4.0.dist-info → sonatoki-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.4.0
+Version: 0.5.1
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later
@@ -8,6 +8,7 @@ Requires-Python: >=3.8
 Requires-Dist: unidecode>=1.3.6
 Requires-Dist: regex>=2023.12.25
 Requires-Dist: typing-extensions>=4.11.0
+Requires-Dist: emoji>=2.12.1
 Description-Content-Type: text/markdown
 
 # sona toki

sonatoki-0.5.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+sonatoki-0.5.1.dist-info/METADATA,sha256=gj5B_q10R5l-w0jEuzFY2035qzp9tpmBQ-sZ0q73zXE,6370
+sonatoki-0.5.1.dist-info/WHEEL,sha256=SOP-4bEE0jbVaCHQGVvF08uWxk5rcSsfEybvoQVHlD8,90
+sonatoki-0.5.1.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
+sonatoki/Configs.py,sha256=HHaSAA7hus7aY6Xy-3fNlbzMwk3wJO0HrjTssg8P78M,4291
+sonatoki/Filters.py,sha256=nVSmw5M4sEYA_8KI1fI53rMHkd9KO6yWbKfdxxExxN8,11700
+sonatoki/Preprocessors.py,sha256=zuu-6SLqFgk88vfSnYlyZjZrzoZQ56U_1SFXoxThQDQ,5628
+sonatoki/Scorers.py,sha256=LRQLgXKTU2VqhkMHFPVxyVt83DXf85_zrpDGk4ThU24,3811
+sonatoki/Tokenizers.py,sha256=qFaA1-v-wjKMihtEJMeZpi3m4cSkJQgWhGhL-w0VgPE,4236
+sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sonatoki/__main__.py,sha256=6n4kUF80APl6a0jV46h_ncHNuQbrLpZ_nAmiNAakiag,5673
+sonatoki/alphabetic.txt,sha256=duyqAKilD2vLIr75RShCIAnktNJcGeEoQIk18V6czmg,11702
+sonatoki/constants.py,sha256=a3OjhtH2Jp6RDot1NE-PrQfm2VzfM850b-qipFLnjS4,18868
+sonatoki/ilo.py,sha256=PWZa202Q4h7IjnLxmfgT93iAPJL7dqJbA97L9kQDPiA,5658
+sonatoki/linku.json,sha256=FLsaESG01rQ88OU8HvwOUl_P9qtGykJ1X-1xoMVDkKA,295077
+sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sonatoki/sandbox.json,sha256=3BpCEjw-kB4z7DJAJ2UrE1YuFIe3knat8qi1iYuAIq4,83555
+sonatoki/syllabic.txt,sha256=HnqY4TrZ3tPcHah3TsvG9F9gjMrnAGdJ8hHJNHyyUPc,1712
+sonatoki/utils.py,sha256=sT5xLMEj0aLpy8GP92HKblJU1Wt1m8NUlMgCFWB32xQ,2265
+sonatoki-0.5.1.dist-info/RECORD,,

sonatoki-0.4.0.dist-info/RECORD

@@ -1,18 +0,0 @@
-sonatoki-0.4.0.dist-info/METADATA,sha256=Z89tIHyGG9RRAgcr_3E4XW2IMX9NyT9mawcCeMQfXPU,6341
-sonatoki-0.4.0.dist-info/WHEEL,sha256=SOP-4bEE0jbVaCHQGVvF08uWxk5rcSsfEybvoQVHlD8,90
-sonatoki-0.4.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
-sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
-sonatoki/Configs.py,sha256=tOeJSlYXMBHbRPBxERGWGT5AjvCxNb3ZGu8GA4BYve4,4034
-sonatoki/Filters.py,sha256=mpJBl-YPMF-Yl6mKFXf0D6DwkPR6H424RlvrkSeh4Dc,10714
-sonatoki/Preprocessors.py,sha256=nvAzxpWP9WwT6gOCKcuiz5F8xYDdKIt9bOVUvy9o-G0,4459
-sonatoki/Scorers.py,sha256=LRQLgXKTU2VqhkMHFPVxyVt83DXf85_zrpDGk4ThU24,3811
-sonatoki/Tokenizers.py,sha256=So5_Tu6J98MD3yVcwB_X3lw2uMG0TN6XHcTbQjFCu5Q,4254
-sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sonatoki/__main__.py,sha256=6xc-wIrrFo9wTyn4zRQNAmqwmJBtVvCMwV-CrM-hueA,82
-sonatoki/constants.py,sha256=wH3iR32-Ic7vSkrMjAZIvmIysTtkJ-KBVU5zv3Oamqs,12656
-sonatoki/ilo.py,sha256=7KwTZgczzU2gbhC69yZbxtpTHy_fGtg_MnG_bDpiSxM,5639
-sonatoki/linku.json,sha256=fm4-dks5s9x1bs7q82GNngAedVCWilMPCQ_o-j35QL0,270950
-sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sonatoki/sandbox.json,sha256=zPtZgJ_CpJa-2Den0gTNlk52f-YEwFVcjMarQXeeu5U,77563
-sonatoki/utils.py,sha256=L984aXxvzfJaZ6GSWRKs7LweOGZYTLK11CdAhpLQr0g,4067
-sonatoki-0.4.0.dist-info/RECORD,,