sonatoki 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

sonatoki/Preprocessors.py

@@ -121,6 +121,9 @@ class ArrowQuote(RegexPreprocessor):
 
 
 __all__ = [
+    "DiscordChannels",
+    "DiscordMentions",
+    "DiscordSpecial",
     "DiscordEmotes",
     "SingleQuotes",
     "DoubleQuotes",

sonatoki/Scorers.py

@@ -16,6 +16,13 @@ Number = Union[int, float]
 Weights = Dict[str, Number]
 
 
+def sigmoid(n: int) -> Number:
+    return 1 / (1 + math.exp(-(0.30 * (n - 1))))
+    # n-1 makes sigmoid(1) == 0.5
+    # 0.30 softens scaling in favor of short input
+    # return n / (1+abs(n))   # too weak in 0.7+
+
+
 class Scorer(ABC):
     @classmethod
     @abstractmethod
@@ -27,7 +34,7 @@ class PassFail(Scorer):
     """The token passes any filter or fails all of them, scoring 1 or 0 respectively."""
 
     @classmethod
-    def __score(cls, token: str, filters: List[Type[Filter]]) -> Number:
+    def score_token(cls, token: str, filters: List[Type[Filter]]) -> Number:
         for f in filters:
             if f.filter(token):
                 score = 1
@@ -47,10 +54,27 @@ class PassFail(Scorer):
         total_score = 0
         len_tokens = len(tokens)
         for token in tokens:
-            total_score += cls.__score(token, filters)
+            total_score += cls.score_token(token, filters)
         return total_score / len_tokens if len_tokens else 0
 
 
+class SoftPassFail(PassFail):
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        if not tokens:
+            return 1
+
+        total_score = 0
+        len_tokens = len(tokens)
+        for token in tokens:
+            total_score += cls.score_token(token, filters)
+
+        percentage = total_score / len_tokens if len_tokens else 0
+        percentage **= sigmoid(len_tokens)
+        return percentage
+
+
 class Scaling(Scorer):
     """
     The sooner a token matches a filter, the higher its score.
@@ -91,13 +115,6 @@ class SoftScaling(Scaling):
     For example, a single token scoring 0.64 will now score 0.8.
     """
 
-    @staticmethod
-    def sigmoid(n: int) -> Number:
-        return 1 / (1 + math.exp(-(0.30 * (n - 1))))
-        # n-1 makes sigmoid(1) == 0.5
-        # 0.30 softens scaling in favor of short input
-        # return n / (1+abs(n))   # too weak in 0.7+
-
     @classmethod
     @override
     def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
@@ -113,11 +130,11 @@ class SoftScaling(Scaling):
             total_score += cls.score_token(token, filters, len_filters)
 
         percentage = total_score / max_score if max_score else 0
-        percentage **= cls.sigmoid(len_tokens)
+        percentage **= sigmoid(len_tokens)
         return percentage
 
 
 class Logarithmic(Scorer): ...
 
 
-__all__ = ["PassFail", "Scaling", "SoftScaling"]
+__all__ = ["PassFail", "SoftPassFail", "Scaling", "SoftScaling"]

sonatoki/Tokenizers.py

@@ -4,6 +4,8 @@ from typing import List, Callable
 # PDM
 import regex as re
 
+# TODO: Entire module should be reworked to match the class scheme of the rest of the module, imo
+
 try:
     # PDM
     import nltk
@@ -15,18 +17,14 @@ except ImportError as e:
 
 LANGUAGE = "english"  # for NLTK
 
-SENT_DELIMS_RE = r"""(.*?[.?!;:])|(.+?$)"""
-SENT_DELIMS_RE = re.compile(SENT_DELIMS_RE)
-
-SENT_DELIMS_TOK = r"""(.*?[.?!;:-])|(.+?$)"""
-SENT_DELIMS_TOK = re.compile(SENT_DELIMS_TOK)
-
-
-WORD_DELIMS_RE = r"""\s+|(?=[.?!;:'"-])"""
-WORD_DELIMS_RE = re.compile(WORD_DELIMS_RE)
+SENT_DELIMS_RE = re.compile(r"""(.*?[.?!;:])|(.+?$)""")
+SENT_DELIMS_TOK = re.compile(r"""(?<=[.?!:;·…“”"'()\[\]\-]|$)""")
+# TODO: are <> or {} that common as *sentence* delims? [] are already a stretch
+# TODO: do the typography characters matter?
+# NOTE: | / and , are *not* sentence delimiters for my purpose
 
-WORD_DELIMS_TOK = r"([\p{Punctuation}\p{posix_punct}]+|\s+)"
-WORD_DELIMS_TOK = re.compile(WORD_DELIMS_TOK)
+WORD_DELIMS_RE = re.compile(r"""\s+|(?=[.?!;:'"-])""")
+WORD_DELIMS_TOK = re.compile(r"([\p{Punctuation}\p{posix_punct}]+|\s+)")
 
 Tokenizer = Callable[[str], List[str]]
 
@@ -53,11 +51,7 @@ def word_tokenize_re(s: str) -> List[str]:
 
 
 def sent_tokenize_tok(s: str) -> List[str]:
-    return [
-        clean
-        for sent in re.findall(SENT_DELIMS_TOK, s)
-        if (clean := sent[0].strip() or sent[1].strip())
-    ]
+    return [clean for sent in re.split(SENT_DELIMS_TOK, s) if (clean := sent.strip())]
 
 
 def word_tokenize_tok(s: str) -> List[str]:

{sonatoki-0.1.1.dist-info → sonatoki-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.1.1
+Version: 0.1.2
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.1.1.dist-info → sonatoki-0.1.2.dist-info}/RECORD

@@ -1,14 +1,14 @@
-sonatoki-0.1.1.dist-info/METADATA,sha256=gUp7IAG1ZmBarZnwyxHt6RmP8GX0tahsaG1qkp047GM,5020
-sonatoki-0.1.1.dist-info/WHEEL,sha256=vnE8JVcI2Wz7GRKorsPArnBdnW2SWKWGow5gu5tHlRU,90
-sonatoki-0.1.1.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki-0.1.2.dist-info/METADATA,sha256=3ccNKi5ODyxbkBvihKJ-XyXEDF5mzO0AzB0myhCMNMM,5020
+sonatoki-0.1.2.dist-info/WHEEL,sha256=vnE8JVcI2Wz7GRKorsPArnBdnW2SWKWGow5gu5tHlRU,90
+sonatoki-0.1.2.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
 sonatoki/Cleaners.py,sha256=gTZ9dSsnvKVUtxM_ECSZ-_2heh--nD5A9dCQR1ATb1c,1160
 sonatoki/Filters.py,sha256=yzhYF79GX03cOwlR_-B8SPMQPZv4UpAPytH0fQwBE70,4093
-sonatoki/Preprocessors.py,sha256=G2up2jKKSrHQtTQWYNWH_fkjgroL45ZeajVn1KUECt8,3431
-sonatoki/Scorers.py,sha256=twuFGqcIg6UTaeBVb1SmKenldC3hj1s97m6zyM_HXjg,3678
-sonatoki/Tokenizers.py,sha256=epOG3jZHI3MSO_L_6Z3zsSkexDEMLVzA2ARg6EnPMO0,1628
+sonatoki/Preprocessors.py,sha256=OhjKcH_nlDznT1Y-ssdU1xH6_xbNrPDQEh8D7H3UTcs,3499
+sonatoki/Scorers.py,sha256=V293DBiupBiujzuc4yMrKOAiuNTLltIsiCzIAlLeokA,4129
+sonatoki/Tokenizers.py,sha256=lFvYS9SXij6gIk-CatKyFQtbRV0ska9FAgynEObxd-A,1741
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/__main__.py,sha256=6xc-wIrrFo9wTyn4zRQNAmqwmJBtVvCMwV-CrM-hueA,82
 sonatoki/constants.py,sha256=h5rbCfu9YF76BsjQYud5d2wq1HODY05zOaw0Ir1cwjo,1320
 sonatoki/ilo.py,sha256=h3TYoqrjHxMGhB8ZJLVijVzy1AVCeWJk5x0q-bs4JMc,4278
 sonatoki/linku.json,sha256=MdFuFRIHniPDUVxKEKuUg1KyzPVgcCj4ZeyvburCwD0,270928
-sonatoki-0.1.1.dist-info/RECORD,,
+sonatoki-0.1.2.dist-info/RECORD,,