sonatoki 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

sonatoki/Configs.py

@@ -73,6 +73,7 @@ PrefConfig: IloConfig = {
     "scoring_filters": [
         Or(NimiLinkuCore, NimiLinkuCommon, NimiUCSUR, Miscellaneous),
         And(LongSyllabic, Not(FalsePosSyllabic)),
+        # NOTE: These are allowed to pass name and alphabetic below, because they *could* be wrong
         LongProperName,
         LongAlphabetic,
     ],

sonatoki/Filters.py

@@ -351,6 +351,10 @@ class Or:
         else:
             other_filters.extend(member_filters)
 
+        if len(other_filters) == 1:  # we only had member filters
+            # TODO: this sucks?
+            return other_filters[0]
+
         filter = cls.__generic_filter(*other_filters)
         return filter
 

sonatoki/constants.py

@@ -519,8 +519,10 @@ ALLOWABLES = {
     "kxk",  # ken ala ken
     "wxw",  # wile ala wile
     "msa",
+    "anusem",
 }
 
+# NOTE: This is being tracked manually rather than fetched from syllabics.txt until I am convinced that solution is appropriate
 FALSE_POS_SYLLABIC = {
     # ordered by frequency in previous TPT data
     "like",
@@ -540,6 +542,7 @@ FALSE_POS_SYLLABIC = {
     "man",
     # "son",  # sona typo?
     "joke",
+    # pon would go here
     "so",
     "ten",
     "make",
@@ -548,11 +551,14 @@ FALSE_POS_SYLLABIC = {
     # "aka" # in sandbox
     "into",
     "in",
+    "no",
     "some",
+    # "papa",
     "on",
     "me",
     "ipa",
     "sun",
+    "mine",
     "sense",
     "none",
     "meme",
@@ -561,28 +567,101 @@ FALSE_POS_SYLLABIC = {
     "mon",
     "take",
     "luna",
-    "anti",
     "elo",
+    "japanese",
     "an",
+    "anti",
     "win",
     "won",
-    "we",
+    "we",  # word in sandbox
     "men",
     "ton",
     "woke",
+    "sen",  # seen
+    "se",  # see
     "semi",
     "male",
+    # "pen",  # borderline
+    "woman",
+    "line",
+    "meta",
+    "mini",
+    "sine",
+    # "min",  # borderline
+    "oposite",
+    "anime",
+    "potato",
+    # "japan",
+    "nose",
+    "kilo",
+    "alone",
+    "minute",
+    "late",
+    "women",
+    "leson",
+    "amen",
+    "tote",
+    "lame",
+    "online",
+    "tone",
+    "ate",
+    "mile",
+    "melon",
+    "tense",
+    "nonsense",
+    "nine",
+    "emo",
+    "unlike",
+    "lone",
+    # manual additions
+    "alike",
+    "amuse",
+    "antelope",
+    "antena",
+    "apetite",
+    "asasin",
+    "asasinate",
+    "asinine",
+    "asinine",
+    "asume",
+    "atone",
+    "awake",
+    "awaken",
+    "eliminate",
+    "elite",
+    "misuse",
+    "emanate",
+    "iluminate",
+    "imense",
+    "imitate",
+    "insane",
+    "insolate",
+    "insulate",
+    "intense",
+    "lemon",
+    "manipulate",
 }
 
 FALSE_POS_ALPHABETIC: Set[str] = {
     "t",
     "is",
+    "as",
     "not",
+    "link",
+    "wait",
     "lol",
+    "new",
     "also",
     "isn",  # TODO: tokenizer....
     "mean",
     "means",
+    "it",
+    "moment",
+    "its",
+    "lmao",
+    "new",
+    "wel",
+    "makes",
 }
 
 UCSUR_RANGES = [

{sonatoki-0.5.0.dist-info → sonatoki-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.5.0
+Version: 0.5.1
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.5.0.dist-info → sonatoki-0.5.1.dist-info}/RECORD

@@ -1,20 +1,20 @@
-sonatoki-0.5.0.dist-info/METADATA,sha256=ytLxR8WxNEourLjtJhT6TmuaeeKXM1bdoSkn0tw7umc,6370
-sonatoki-0.5.0.dist-info/WHEEL,sha256=SOP-4bEE0jbVaCHQGVvF08uWxk5rcSsfEybvoQVHlD8,90
-sonatoki-0.5.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki-0.5.1.dist-info/METADATA,sha256=gj5B_q10R5l-w0jEuzFY2035qzp9tpmBQ-sZ0q73zXE,6370
+sonatoki-0.5.1.dist-info/WHEEL,sha256=SOP-4bEE0jbVaCHQGVvF08uWxk5rcSsfEybvoQVHlD8,90
+sonatoki-0.5.1.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
 sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
-sonatoki/Configs.py,sha256=gsAJB_SRBzUFTsdgOSqJ5k2WvHk_VuI_PBuXbC3z530,4192
-sonatoki/Filters.py,sha256=Hh5CIMTtJIDK5zEIpDZ-02zujpCT44frMC3VVAlTW0I,11565
+sonatoki/Configs.py,sha256=HHaSAA7hus7aY6Xy-3fNlbzMwk3wJO0HrjTssg8P78M,4291
+sonatoki/Filters.py,sha256=nVSmw5M4sEYA_8KI1fI53rMHkd9KO6yWbKfdxxExxN8,11700
 sonatoki/Preprocessors.py,sha256=zuu-6SLqFgk88vfSnYlyZjZrzoZQ56U_1SFXoxThQDQ,5628
 sonatoki/Scorers.py,sha256=LRQLgXKTU2VqhkMHFPVxyVt83DXf85_zrpDGk4ThU24,3811
 sonatoki/Tokenizers.py,sha256=qFaA1-v-wjKMihtEJMeZpi3m4cSkJQgWhGhL-w0VgPE,4236
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/__main__.py,sha256=6n4kUF80APl6a0jV46h_ncHNuQbrLpZ_nAmiNAakiag,5673
 sonatoki/alphabetic.txt,sha256=duyqAKilD2vLIr75RShCIAnktNJcGeEoQIk18V6czmg,11702
-sonatoki/constants.py,sha256=ZITC1_MVWBdeNH2vzJfxNNv8vlUSUXtSrZkGyGaFthM,17626
+sonatoki/constants.py,sha256=a3OjhtH2Jp6RDot1NE-PrQfm2VzfM850b-qipFLnjS4,18868
 sonatoki/ilo.py,sha256=PWZa202Q4h7IjnLxmfgT93iAPJL7dqJbA97L9kQDPiA,5658
 sonatoki/linku.json,sha256=FLsaESG01rQ88OU8HvwOUl_P9qtGykJ1X-1xoMVDkKA,295077
 sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/sandbox.json,sha256=3BpCEjw-kB4z7DJAJ2UrE1YuFIe3knat8qi1iYuAIq4,83555
 sonatoki/syllabic.txt,sha256=HnqY4TrZ3tPcHah3TsvG9F9gjMrnAGdJ8hHJNHyyUPc,1712
 sonatoki/utils.py,sha256=sT5xLMEj0aLpy8GP92HKblJU1Wt1m8NUlMgCFWB32xQ,2265
-sonatoki-0.5.0.dist-info/RECORD,,
+sonatoki-0.5.1.dist-info/RECORD,,