sonatoki 0.1.1__tar.gz → 0.1.2__tar.gz

{sonatoki-0.1.1 → sonatoki-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.1.1
+Version: 0.1.2
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.1.1 → sonatoki-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sonatoki"
-version = "0.1.1"
+version = "0.1.2"
 description = "ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?"
 authors = [
     { name = "jan Kekan San (@gregdan3)", email = "gregory.danielson3@gmail.com" },

{sonatoki-0.1.1 → sonatoki-0.1.2}/src/sonatoki/Preprocessors.py

@@ -121,6 +121,9 @@ class ArrowQuote(RegexPreprocessor):
 
 
 __all__ = [
+    "DiscordChannels",
+    "DiscordMentions",
+    "DiscordSpecial",
     "DiscordEmotes",
     "SingleQuotes",
     "DoubleQuotes",

{sonatoki-0.1.1 → sonatoki-0.1.2}/src/sonatoki/Scorers.py

@@ -16,6 +16,13 @@ Number = Union[int, float]
 Weights = Dict[str, Number]
 
 
+def sigmoid(n: int) -> Number:
+    return 1 / (1 + math.exp(-(0.30 * (n - 1))))
+    # n-1 makes sigmoid(1) == 0.5
+    # 0.30 softens scaling in favor of short input
+    # return n / (1+abs(n))   # too weak in 0.7+
+
+
 class Scorer(ABC):
     @classmethod
     @abstractmethod
@@ -27,7 +34,7 @@ class PassFail(Scorer):
     """The token passes any filter or fails all of them, scoring 1 or 0 respectively."""
 
     @classmethod
-    def __score(cls, token: str, filters: List[Type[Filter]]) -> Number:
+    def score_token(cls, token: str, filters: List[Type[Filter]]) -> Number:
         for f in filters:
             if f.filter(token):
                 score = 1
@@ -47,10 +54,27 @@ class PassFail(Scorer):
         total_score = 0
         len_tokens = len(tokens)
         for token in tokens:
-            total_score += cls.__score(token, filters)
+            total_score += cls.score_token(token, filters)
         return total_score / len_tokens if len_tokens else 0
 
 
+class SoftPassFail(PassFail):
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        if not tokens:
+            return 1
+
+        total_score = 0
+        len_tokens = len(tokens)
+        for token in tokens:
+            total_score += cls.score_token(token, filters)
+
+        percentage = total_score / len_tokens if len_tokens else 0
+        percentage **= sigmoid(len_tokens)
+        return percentage
+
+
 class Scaling(Scorer):
     """
     The sooner a token matches a filter, the higher its score.
@@ -91,13 +115,6 @@ class SoftScaling(Scaling):
     For example, a single token scoring 0.64 will now score 0.8.
     """
 
-    @staticmethod
-    def sigmoid(n: int) -> Number:
-        return 1 / (1 + math.exp(-(0.30 * (n - 1))))
-        # n-1 makes sigmoid(1) == 0.5
-        # 0.30 softens scaling in favor of short input
-        # return n / (1+abs(n))   # too weak in 0.7+
-
     @classmethod
     @override
     def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
@@ -113,11 +130,11 @@ class SoftScaling(Scaling):
             total_score += cls.score_token(token, filters, len_filters)
 
         percentage = total_score / max_score if max_score else 0
-        percentage **= cls.sigmoid(len_tokens)
+        percentage **= sigmoid(len_tokens)
         return percentage
 
 
 class Logarithmic(Scorer): ...
 
 
-__all__ = ["PassFail", "Scaling", "SoftScaling"]
+__all__ = ["PassFail", "SoftPassFail", "Scaling", "SoftScaling"]

{sonatoki-0.1.1 → sonatoki-0.1.2}/src/sonatoki/Tokenizers.py

@@ -4,6 +4,8 @@ from typing import List, Callable
 # PDM
 import regex as re
 
+# TODO: Entire module should be reworked to match the class scheme of the rest of the module, imo
+
 try:
     # PDM
     import nltk
@@ -15,18 +17,14 @@ except ImportError as e:
 
 LANGUAGE = "english"  # for NLTK
 
-SENT_DELIMS_RE = r"""(.*?[.?!;:])|(.+?$)"""
-SENT_DELIMS_RE = re.compile(SENT_DELIMS_RE)
-
-SENT_DELIMS_TOK = r"""(.*?[.?!;:-])|(.+?$)"""
-SENT_DELIMS_TOK = re.compile(SENT_DELIMS_TOK)
-
-
-WORD_DELIMS_RE = r"""\s+|(?=[.?!;:'"-])"""
-WORD_DELIMS_RE = re.compile(WORD_DELIMS_RE)
+SENT_DELIMS_RE = re.compile(r"""(.*?[.?!;:])|(.+?$)""")
+SENT_DELIMS_TOK = re.compile(r"""(?<=[.?!:;·…“”"'()\[\]\-]|$)""")
+# TODO: are <> or {} that common as *sentence* delims? [] are already a stretch
+# TODO: do the typography characters matter?
+# NOTE: | / and , are *not* sentence delimiters for my purpose
 
-WORD_DELIMS_TOK = r"([\p{Punctuation}\p{posix_punct}]+|\s+)"
-WORD_DELIMS_TOK = re.compile(WORD_DELIMS_TOK)
+WORD_DELIMS_RE = re.compile(r"""\s+|(?=[.?!;:'"-])""")
+WORD_DELIMS_TOK = re.compile(r"([\p{Punctuation}\p{posix_punct}]+|\s+)")
 
 Tokenizer = Callable[[str], List[str]]
 
@@ -53,11 +51,7 @@ def word_tokenize_re(s: str) -> List[str]:
 
 
 def sent_tokenize_tok(s: str) -> List[str]:
-    return [
-        clean
-        for sent in re.findall(SENT_DELIMS_TOK, s)
-        if (clean := sent[0].strip() or sent[1].strip())
-    ]
+    return [clean for sent in re.split(SENT_DELIMS_TOK, s) if (clean := sent.strip())]
 
 
 def word_tokenize_tok(s: str) -> List[str]:

sonatoki-0.1.2/tests/test_ilo.py

@@ -0,0 +1,185 @@
+# PDM
+import pytest
+
+# LOCAL
+from sonatoki.ilo import Ilo
+from sonatoki.Filters import (
+    Numerics,
+    Syllabic,
+    NimiLinku,
+    Alphabetic,
+    ProperName,
+    Punctuations,
+)
+from sonatoki.Scorers import SoftScaling, SoftPassFail
+from sonatoki.Cleaners import ConsecutiveDuplicates
+from sonatoki.Tokenizers import word_tokenize_tok
+from sonatoki.Preprocessors import URLs
+
+
+@pytest.fixture
+def ilo():
+    ilo = Ilo(
+        preprocessors=[URLs],
+        ignoring_filters=[Numerics, Punctuations],
+        scoring_filters=[NimiLinku, Syllabic, ProperName, Alphabetic],
+        cleaners=[ConsecutiveDuplicates],
+        scorer=SoftScaling,
+        tokenizer=word_tokenize_tok,
+        passing_score=0.8,
+    )
+    # ilo.logging_threshold = 0.8
+    return ilo
+
+
+@pytest.fixture()
+def lazy_ilo():
+    ilo = Ilo(
+        preprocessors=[URLs],
+        ignoring_filters=[Numerics, Punctuations],
+        scoring_filters=[Alphabetic, ProperName],
+        cleaners=[ConsecutiveDuplicates],
+        scorer=SoftPassFail,
+        tokenizer=word_tokenize_tok,
+        passing_score=0.8,
+    )
+    # ilo.logging_threshold = 0.8
+    return ilo
+
+
+ALL_VALID = [
+    "mi unpa e mama sina",
+    "mama sina li lon seme? mi wile toki tawa ona",
+    "sina sike pakala",
+    "    sina    seme     e     mi     ?",
+    "AAAAAAAAAAA",
+    "muuuu MUUU muUuUuU",
+    "wawa mute. " * 10,
+]
+
+IGNORABLES = [
+    "",
+    " ",
+    "2+2=5",
+    "kiwen moli 42",
+    "https://mun.la/sona",
+    "https://example.com/",
+    "mi wile e ni: <https://example.com> li pona",
+    "lipu https://example.com li kama pona",
+    "...",
+    " ⟨·⟩, a",
+    "·····",
+]
+
+SYLLABIC_MATCHES = [
+    "ni li tenpo penpo",
+    "sipisi",
+    "walawa malama walama malama mupi",
+    "mi sona ala e nimi sunopatikuna",
+    "kalama wuwojiti li pana e sona",
+    "jan Awaja en jan Alasali en jan Akesinu li pona",  # syllables match before names here
+]
+
+ALPHABETIC_MATCHES = [
+    "mi mtue o kama sona",
+    "mi mute o kma son",
+    "ni li tptpt",
+    "mi wile pana lon sptp",
+    "tmo tawa mi li pona mute la mi kepeken ona lon tenpo mute",
+    "mi pakla lon nimi pi mute lili, taso ale li pona tan ni: mi toki mute",
+]
+
+NAME_MATCHES = [
+    "musi Homestuck li ike tawa mi",
+    "ilo Google li sona ala e nimi Emoticon la mi wile utala e ona",
+    "toki Kanse li lon",
+    "toki Lojban li nasa e lawa mi",
+]
+
+SOME_INVALID = ["kulupu xerox li ike", "mi tawa ma ohio"]
+
+
+EXCESSIVE_SYLLABICS = [
+    "manama manama namana namana majani makala",
+]
+
+EXCESSIVE_ALPHABETICS = [
+    "21st",  # candidate for xfails?
+    "tok",
+    "mut",
+    "mtue",
+    "I wait, I sulk, as a tool I make stoops to ineptness.",
+    "aaa i non-saw usa's most multiple element-set. it's as asinine as in `e`-less speak",
+    "mi pakla ln tepo mtue ls mi kn ala tok poan aun seem",
+    "so, to atone like papa—an awesome anon (no-name) sin man—i ate an asinine lemon-limelike tomato jalapeno isotope. 'nonsense!' amen. note to Oman: take mine katana to imitate a ninja in pantomime. atomise one nuke? 'insane misuse!' same. likewise, Susan, awaken a pepino melon in a linen pipeline. (penile) emanate semen. joke: manipulate a tame toneme to elope online tonite",
+]
+
+EXCESSIVE_NAMES = [
+    "I Want To Evade The Filter",
+    "If You Do This The Bot Can't See You",
+    "This Is A Statement In Perfect Toki Pona, I Guarantee",
+]
+
+NON_MATCHES = [
+    "bong",
+    "super bruh moment 64",
+    "homestuck",
+    "homestuck Homestuck",
+]
+
+XFAILS = [
+    "lete li ike x.x",  # emoticon should not be a problem
+]
+
+
+@pytest.mark.parametrize(
+    "text",
+    ALL_VALID
+    + SYLLABIC_MATCHES
+    + ALPHABETIC_MATCHES
+    + NAME_MATCHES
+    + SOME_INVALID
+    + IGNORABLES,
+)
+def test_known_good(ilo: Ilo, lazy_ilo: Ilo, text: str):
+    assert ilo.is_toki_pona(text), text
+
+
+@pytest.mark.parametrize(
+    "text", EXCESSIVE_SYLLABICS + EXCESSIVE_ALPHABETICS + EXCESSIVE_NAMES + NON_MATCHES
+)
+def test_known_bad(ilo: Ilo, text: str):
+    assert not ilo.is_toki_pona(text), text
+
+
+@pytest.mark.parametrize(
+    "text",
+    ALL_VALID
+    + SYLLABIC_MATCHES
+    + ALPHABETIC_MATCHES
+    + NAME_MATCHES
+    + SOME_INVALID
+    + IGNORABLES,
+)
+def test_known_good_lazy(lazy_ilo: Ilo, text: str):
+    assert lazy_ilo.is_toki_pona(text), text
+    # assumption: lazy ilo should pass anything the more strict ilo does
+
+
+@pytest.mark.parametrize("text", NON_MATCHES)
+def test_known_bad_lazy(lazy_ilo: Ilo, text: str):
+    assert not lazy_ilo.is_toki_pona(text), text
+
+
+@pytest.mark.parametrize(
+    "text", EXCESSIVE_SYLLABICS + EXCESSIVE_ALPHABETICS + EXCESSIVE_NAMES
+)
+def test_weakness_of_lazy(lazy_ilo: Ilo, text: str):
+    # NOTE: This is demonstrative, not preferential
+    assert lazy_ilo.is_toki_pona(text), text
+
+
+@pytest.mark.xfail
+@pytest.mark.parametrize("text", XFAILS)
+def test_known_xfails(ilo: Ilo, text: str):
+    assert ilo.is_toki_pona(text)

sonatoki-0.1.2/tests/tokenize_cases/tokenize_sentences_tok.yml

@@ -0,0 +1,37 @@
+---
+- name: "basic1"
+  input: "mu. mu."
+  output:
+    - "mu."
+    - "mu."
+- name: "basic2"
+  input: "mu! mu!"
+  output:
+    - "mu!"
+    - "mu!"
+- name: "basic3"
+  input: "mu? mu?"
+  output:
+    - "mu?"
+    - "mu?"
+- name: "basic4"
+  input: "mi mu. mi wawa."
+  output:
+    - "mi mu."
+    - "mi wawa."
+- name: "dash"
+  input: "mi sona ala e ni- sina seme a"
+  output:
+    - "mi sona ala e ni-"
+    - "sina seme a"
+- name: "comma"
+  input: "mi mu tawa sina, mi wawa e sina."
+  output:
+    - "mi mu tawa sina, mi wawa e sina."
+- name: "quotes"
+  input: "toki li tan kulupu Kuko li ni: 'o ike ala!'"
+  output: # expected; we split on right of all sentence-ending puncts
+    - "toki li tan kulupu Kuko li ni:"
+    - "'"
+    - "o ike ala!"
+    - "'"

{sonatoki-0.1.1 → sonatoki-0.1.2}/tests/tokenize_cases/tokenize_words.yml

@@ -1,19 +1,15 @@
 ---
 - name: "basic"
   input: "mi mu mute tawa sina."
-  should_be_equal: true
 - name: "spoilers"
   input: "||ni li toki len.||"
-  should_be_equal: true
   xfail: true # lookbehind for . breaks it
 - name: "quotes"
   input: "toki li tan kulupu Kuko li ni: 'o ike ala!'"
-  should_be_equal: true
   xfail: true
 - name: periods every word
   input: "mi.unpa.e.mama.sina"
   xfail: true # lookbehind for . breaks it
 - name: "url"
   input: "https://mun.la/sona/"
-  should_be_equal: true
   xfail: true # i have no idea how to emulate the : behavior

{sonatoki-0.1.1 → sonatoki-0.1.2}/tests/tokenize_cases/tokenize_words_tok.yml

@@ -73,6 +73,18 @@
     - "are"
     - "boring"
     - "'"
+- name: periods every word
+  input: "mi.unpa.e.mama.sina"
+  output:
+    - "mi"
+    - "."
+    - "unpa"
+    - "."
+    - "e"
+    - "."
+    - "mama"
+    - "."
+    - "sina"
 - name: "discovered case 1"
   input: "***__U T A L A__   __M U N__***"
   output:

sonatoki-0.1.1/tests/test_ilo.py

@@ -1,73 +0,0 @@
-# LOCAL
-from sonatoki.ilo import Ilo
-from sonatoki.Filters import (
-    Numerics,
-    Syllabic,
-    NimiLinku,
-    Alphabetic,
-    ProperName,
-    Punctuations,
-)
-from sonatoki.Scorers import Scaling, SoftScaling
-from sonatoki.Cleaners import ConsecutiveDuplicates
-from sonatoki.Tokenizers import word_tokenize_tok
-from sonatoki.Preprocessors import (
-    URLs,
-    DiscordEmotes,
-    DiscordSpecial,
-    DiscordChannels,
-    DiscordMentions,
-)
-
-
-def test_constructor():
-    ilo = Ilo(
-        preprocessors=[
-            URLs,
-            DiscordEmotes,
-            DiscordMentions,
-            DiscordChannels,
-            DiscordSpecial,
-        ],
-        ignoring_filters=[Numerics, Punctuations],
-        scoring_filters=[NimiLinku, Syllabic, ProperName, Alphabetic],
-        cleaners=[ConsecutiveDuplicates],
-        scorer=SoftScaling,
-        tokenizer=word_tokenize_tok,
-        passing_score=0.8,
-    )
-    # ilo._logging_threshold = 0.8
-    assert ilo.is_toki_pona("mi unpa e mama sina")
-    # toki pona
-    assert ilo.is_toki_pona("mama sina li lon seme? mi wile toki tawa ona")
-    assert ilo.is_toki_pona("sina sike pakala")
-    # names
-    assert ilo.is_toki_pona("musi Homestuck li ike tawa mi")
-    # typoes
-    assert ilo.is_toki_pona("mi mtue o kama sona")
-    assert ilo.is_toki_pona("mi mute o kma son")
-    # phonotactically valid
-    assert ilo.is_toki_pona("ni li tenpo penpo")
-    # alphabetically valid
-    assert ilo.is_toki_pona("ni li tptpt")
-    # a single
-    assert ilo.is_toki_pona("sipisi")
-
-    # soft scaling with syllablic filter at 2/4 will pass up to 5 syllablic words
-    assert ilo.is_toki_pona("walawa malama walama malama mupi")
-    # but fail 6 or more
-    assert not ilo.is_toki_pona("manama manama namana namana majani makala")
-
-    # TODO: should soft scaling save an alphabetically valid single word?
-    assert not ilo.is_toki_pona("tok")
-    assert not ilo.is_toki_pona("mtue")
-
-    # just english
-    assert not ilo.is_toki_pona("bong")
-    assert not ilo.is_toki_pona("super bruh moment 64")
-    # all names
-    assert not ilo.is_toki_pona("I Want To Evade The Filter")
-    # all alphabetic
-    assert not ilo.is_toki_pona(
-        "aaa i non-saw usa's most multiple element-set. it's as asinine as in `e`-less speak"
-    )

sonatoki-0.1.1/tests/tokenize_cases/tokenize_sentences_tok.yml

@@ -1,20 +0,0 @@
----
-- name: "basic"
-  input: "mi mu. mi wawa."
-  output:
-    - "mi mu."
-    - "mi wawa."
-  should_be_equal: true
-- name: "dash"
-  input: "mi sona ala e ni- sina seme a"
-  output:
-    - "mi sona ala e ni-"
-    - "sina seme a"
-  should_be_equal: true
-- name: "quotes"
-  input: "toki li tan kulupu Kuko li ni: 'o ike ala!'"
-  output:
-    - "toki li tan kulupu Kuko li ni:"
-    - "'o ike ala!'"
-  should_be_equal: true
-  xfail: true