sonatoki 0.6.1__py3-none-any.whl → 0.6.3__py3-none-any.whl

sonatoki/Configs.py

@@ -1,6 +1,6 @@
 # STL
 from copy import deepcopy
-from typing import Set, List, Type, TypedDict, cast
+from typing import List, Type, TypedDict
 
 # PDM
 from typing_extensions import NotRequired
@@ -12,13 +12,11 @@ from sonatoki.Filters import (
     Not,
     Filter,
     Numeric,
-    Syllabic,
     NimiUCSUR,
     Alphabetic,
     NimiKuLili,
     NimiKuSuli,
     ProperName,
-    Phonotactic,
     Punctuation,
     LongSyllabic,
     Miscellaneous,
@@ -44,6 +42,34 @@ from sonatoki.Preprocessors import (
     AngleBracketObject,
 )
 
+__DICT_PHONOMATCHES = {
+    # Sandbox words are removed from the CorpusConfig if they appear more frequently in English than Toki Pona by a factor of at least 3.
+    # In this case, all of these appear more often in English by a factor of at least 10.
+    "aka",  # also known as
+    "an",  # article
+    "api",  # API
+    "i",  # 1st person
+    "kana",  # japanese script
+    "me",  # 1st person singular, english
+    "ne",  # "no" in several languages
+    "nu",  # "new" in english, "now" in dutch
+    "se",  # spanish particle, english "see"
+    "take",  # acquire, perhaps forcefully or without permission
+    "ten",  # 10
+    "to",  # to, too
+    "je",  # 1st person pronoun, french
+    "u",  # no u
+    "we",  # 1st person plural, english
+    "wi",  # wii and discussions of syllables
+    "sole",  # singular, of shoe
+    # unexplored candidates for removal
+    # "omen",  # ominous
+    # "papa",  # father
+    # "lo",  # "lo" and "loo"
+    # "ewe",  # sheep
+    # "pa",  # father- eh?
+}
+
 
 class IloConfig(TypedDict):
     preprocessors: List[Type[Preprocessor]]
@@ -92,8 +118,8 @@ CorpusConfig: IloConfig = {
             NimiLinkuCore,
             NimiLinkuCommon,
             NimiLinkuUncommon,
-            NimiLinkuObscure,
-            NimiLinkuSandbox,
+            NimiLinkuObscure(sub=__DICT_PHONOMATCHES),
+            NimiLinkuSandbox(sub=__DICT_PHONOMATCHES),
             NimiUCSUR,
             Miscellaneous,
         ),
@@ -104,40 +130,6 @@ CorpusConfig: IloConfig = {
     "scorer": SoftScaling,
     "passing_score": 0.8,
 }
-
-# TODO: create a mechanism to omit tokens from a filter with more granularity
-__corpus_tokens_dict: Set[str] = cast(
-    Set[str],
-    CorpusConfig["scoring_filters"][
-        0
-    ].tokens,  # pyright: ignore[reportAttributeAccessIssue]
-)
-__corpus_tokens_dict -= {
-    # Sandbox words are removed from the CorpusConfig if they appear more frequently in English than Toki Pona by a factor of at least 3.
-    # In this case, all of these appear more often in English by a factor of at least 10.
-    "aka",  # also known as
-    "an",  # article
-    "api",  # API
-    "i",  # 1st person
-    "kana",  # japanese script
-    "me",  # 1st person
-    "ne",  # "no" in several languages
-    "nu",  # "new", now in dutch
-    "se",  # spanish particle, "see"
-    "take",  # acquire, perhaps forcefully or without permission
-    "ten",  # 10
-    "to",  # to, too
-    "u",  # no u
-    "we",  # 1st person plural
-    "wi",  # wii and discussions of syllables
-    "sole",  # singular, of shoe
-    # unexplored candidates for removal
-    # "omen",  # ominous
-    # "papa",  # father
-    # "lo",  # "lo" and "loo"
-    # "ewe",  # sheep
-    # "pa",  # father- eh?
-}
 """Mimics the previous implementation of ilo pi toki pona taso."""
 LazyConfig: IloConfig = {
     "preprocessors": [Emoji, Backticks, URLs, AngleBracketObject, Reference],

sonatoki/Filters.py

@@ -1,7 +1,8 @@
 # STL
 import re
 from abc import ABC, abstractmethod
-from typing import Set, List, Type
+from copy import deepcopy
+from typing import Set, List, Type, Optional
 from functools import lru_cache as cache  # cache comes in 3.9
 
 # PDM
@@ -101,6 +102,20 @@ class MemberFilter(Filter):
     def filter(cls, token: str) -> bool:
         return token.lower() in cls.tokens
 
+    def __new__(
+        cls, add: Optional[Set[str]] = None, sub: Optional[Set[str]] = None
+    ) -> Type[Filter]:
+        parent_tokens = deepcopy(cls.tokens)
+        if add:
+            parent_tokens = parent_tokens.union(add)
+        if sub:
+            parent_tokens -= sub
+
+        class AnonMemberFilter(MemberFilter):
+            tokens = parent_tokens
+
+        return AnonMemberFilter
+
 
 class SubsetFilter(Filter):
     tokens: Set[str]

sonatoki/constants.py

@@ -501,7 +501,7 @@ ALL_PUNCT_RANGES_STR = "".join(find_unicode_ranges(ALL_PUNCT))
 SENTENCE_PUNCT = """.?!:;()[-]·•…"""
 # NOTE: quotes were previously included, but in TP they are *not* reliably sentence boundaries
 
-INTRA_WORD_PUNCT = """-'"""
+INTRA_WORD_PUNCT = """-'’"""
 
 
 LINKU = Path(__file__).resolve().parent / Path("linku.json")

{sonatoki-0.6.1.dist-info → sonatoki-0.6.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.6.1
+Version: 0.6.3
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.6.1.dist-info → sonatoki-0.6.3.dist-info}/RECORD

@@ -1,20 +1,20 @@
-sonatoki-0.6.1.dist-info/METADATA,sha256=lHSEJsJlmARsWbMBZujSafV4R5JdgXhguqdR3YLUg_c,6517
-sonatoki-0.6.1.dist-info/WHEEL,sha256=rSwsxJWe3vzyR5HCwjWXQruDgschpei4h_giTm0dJVE,90
-sonatoki-0.6.1.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki-0.6.3.dist-info/METADATA,sha256=AWtjziHObR8LdeB-QwIXaqWe-k8YQj9C0yDpa1_Y0Q0,6517
+sonatoki-0.6.3.dist-info/WHEEL,sha256=rSwsxJWe3vzyR5HCwjWXQruDgschpei4h_giTm0dJVE,90
+sonatoki-0.6.3.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
 sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
-sonatoki/Configs.py,sha256=RD6YUYW45pwIFx8ebJgGs5PhIhL9sjn_VqIg4zf3VUE,5697
-sonatoki/Filters.py,sha256=nVSmw5M4sEYA_8KI1fI53rMHkd9KO6yWbKfdxxExxN8,11700
+sonatoki/Configs.py,sha256=l0OTEpbq6_IcNburV5pPTzRxsQ7_UCIugGv02adT8R8,5550
+sonatoki/Filters.py,sha256=3daBdOagJtkb4Qx6p5F2cCUd21FfMIY62UDWrR6Jj2Q,12131
 sonatoki/Preprocessors.py,sha256=nN6xL6mvVAnWZjSNW8CaeLm8x4kK3dCoB-1WYqi0ANU,5763
 sonatoki/Scorers.py,sha256=LRQLgXKTU2VqhkMHFPVxyVt83DXf85_zrpDGk4ThU24,3811
 sonatoki/Tokenizers.py,sha256=8lpC70bzXOpHyhVr5bmqpYKmdmQvJdf7X5-Icc9RRCw,5040
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/__main__.py,sha256=6n4kUF80APl6a0jV46h_ncHNuQbrLpZ_nAmiNAakiag,5673
 sonatoki/alphabetic.txt,sha256=duyqAKilD2vLIr75RShCIAnktNJcGeEoQIk18V6czmg,11702
-sonatoki/constants.py,sha256=FDZYYfst_hYPJ8Fhc9KDceFgOwOci4hUIPN9SnCBzFw,19209
+sonatoki/constants.py,sha256=mPbU-X9PNzelOHVZn-8ZqR_ewKYNjDA6lj2XQpnuoRw,19212
 sonatoki/ilo.py,sha256=PWZa202Q4h7IjnLxmfgT93iAPJL7dqJbA97L9kQDPiA,5658
 sonatoki/linku.json,sha256=d72Dvht-a4gBmdqLLI8mElvo83zSpbxDmxJj05hOudM,295413
 sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/sandbox.json,sha256=44csrQDaVtV-n8OyewabX1J9MmUFCsPct5C8E5Xuc58,140197
 sonatoki/syllabic.txt,sha256=HnqY4TrZ3tPcHah3TsvG9F9gjMrnAGdJ8hHJNHyyUPc,1712
 sonatoki/utils.py,sha256=sT5xLMEj0aLpy8GP92HKblJU1Wt1m8NUlMgCFWB32xQ,2265
-sonatoki-0.6.1.dist-info/RECORD,,
+sonatoki-0.6.3.dist-info/RECORD,,