ts-tokenizer 0.2.2__tar.gz → 0.2.3__tar.gz

{ts_tokenizer-0.2.2/ts_tokenizer.egg-info → ts_tokenizer-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ts_tokenizer
-Version: 0.2.2
+Version: 0.2.3
 Summary: TS Tokenizer is a hybrid (lexicon-based and rule-based) tokenizer designed specifically for tokenizing Turkish texts.
 Home-page: https://github.com/tanerim/ts_tokenizer
 Author: Taner Sezer

ts_tokenizer-0.2.3/tests/test_abbr_apostrophe_suffix.py

@@ -0,0 +1,30 @@
+from ts_tokenizer.tokenizer import tokenize
+
+
+def tagged_lines(text):
+    return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
+
+
+def test_candidate_abbr_with_apostrophe_suffix():
+    assert tagged_lines("md.'ye") == [
+        ("md.", "Abbr"),
+        ("'", "Punc"),
+        ("ye", "Valid_Word"),
+    ]
+
+
+def test_sentence_fragment_with_ordinal_and_abbr_suffix():
+    assert tagged_lines("2. md.'ye") == [
+        ("2.", "Ordinal_Number"),
+        ("md.", "Abbr"),
+        ("'", "Punc"),
+        ("ye", "Valid_Word"),
+    ]
+
+
+def test_existing_abbr_with_apostrophe_suffix():
+    assert tagged_lines("Prof.'a") == [
+        ("Prof.", "Abbr"),
+        ("'", "Punc"),
+        ("a", "OOV"),
+    ]

ts_tokenizer-0.2.3/tests/test_abbreviation_lookup.py

@@ -0,0 +1,13 @@
+from ts_tokenizer.tokenizer import tokenize
+
+
+def tagged_lines(text):
+    return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
+
+
+def test_uppercase_abbreviation_from_abbr_list():
+    assert tagged_lines("YHT") == [("YHT", "Abbr")]
+
+
+def test_existing_abbreviation_behavior_is_preserved():
+    assert tagged_lines("TBMM'ye") == [("TBMM'ye", "Apostrophed")]

ts_tokenizer-0.2.3/tests/test_currency_suffix.py

@@ -0,0 +1,17 @@
+from ts_tokenizer.tokenizer import tokenize
+
+
+def tagged_lines(text):
+    return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
+
+
+def test_currency_suffix_forms_are_preserved():
+    assert tagged_lines("₺350'lik") == [("₺350'lik", "Currency")]
+    assert tagged_lines("350₺'lik") == [("350₺'lik", "Currency")]
+    assert tagged_lines("$100'luk") == [("$100'luk", "Currency")]
+    assert tagged_lines("100$'lık") == [("100$'lık", "Currency")]
+
+
+def test_plain_currency_forms_remain_unchanged():
+    assert tagged_lines("350₺") == [("350₺", "Currency")]
+    assert tagged_lines("₺350") == [("₺350", "Currency")]

ts_tokenizer-0.2.3/tests/test_symbolic_marks.py

@@ -0,0 +1,43 @@
+from ts_tokenizer.tokenizer import tokenize
+
+
+def tagged_lines(text):
+    return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
+
+
+def test_registered_mark_allows_underscored_words():
+    assert tagged_lines("yeni_firma®") == [
+        ("yeni_firma®", "Registered"),
+    ]
+    assert tagged_lines("eski_Firma®") == [
+        ("eski_Firma®", "Registered"),
+    ]
+
+
+def test_existing_symbolic_mark_behavior_is_preserved():
+    assert tagged_lines("tscorpus®") == [("tscorpus®", "Registered")]
+    assert tagged_lines("abc™") == [("abc™", "Trademark")]
+    assert tagged_lines("©tscorpus") == [("©tscorpus", "Copyright")]
+
+
+def test_symbolic_marks_with_trailing_punctuation_are_preserved():
+    assert tagged_lines("™kuruldu.") == [
+        ("™kuruldu", "Trademark"),
+        (".", "Punc"),
+    ]
+    assert tagged_lines("kapandı©.") == [
+        ("kapandı©", "Copyright"),
+        (".", "Punc"),
+    ]
+
+
+def test_mixed_sentence_with_symbolic_marks():
+    assert tagged_lines("yeni_firma® ™kuruldu. eskiFirma® kapandı©. ®yepyeni") == [
+        ("yeni_firma®", "Registered"),
+        ("™kuruldu", "Trademark"),
+        (".", "Punc"),
+        ("eskiFirma®", "Registered"),
+        ("kapandı©", "Copyright"),
+        (".", "Punc"),
+        ("®yepyeni", "Registered"),
+    ]

{ts_tokenizer-0.2.2 → ts_tokenizer-0.2.3}/ts_tokenizer/__init__.py

@@ -1,6 +1,6 @@
 from importlib import import_module
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 
 _EXPORT_MAP = {
     "tokenize": (".tokenizer", "tokenize"),

{ts_tokenizer-0.2.2 → ts_tokenizer-0.2.3}/ts_tokenizer/data/abbr_list.txt

@@ -1128,4 +1128,5 @@ ltd.
 s.s.c.b.
 http
 https
-ftp
+ftp
+YHT

{ts_tokenizer-0.2.2 → ts_tokenizer-0.2.3}/ts_tokenizer/data/cand_abbr_list.txt

@@ -2137,3 +2137,4 @@ Zn
 zool.
 Zr
 ZRV
+YHT

{ts_tokenizer-0.2.2 → ts_tokenizer-0.2.3}/ts_tokenizer/data/currency_symbols.txt

@@ -6,6 +6,7 @@ $
 ₽
 ₩
 ₺
+₺
 ₫
 ₦
 ₪

{ts_tokenizer-0.2.2 → ts_tokenizer-0.2.3}/ts_tokenizer/data.py

@@ -1,6 +1,8 @@
 import os
 from typing import FrozenSet
 
+from .char_fix import CharFix
+
 base_dir = os.path.dirname(os.path.abspath(__file__))
 data_dir = os.path.join(base_dir, 'data')
 
@@ -8,7 +10,8 @@ data_dir = os.path.join(base_dir, 'data')
 
 emoticons = set(line.strip() for line in open(os.path.join(data_dir, 'emoticons.txt')))
 smileys = set(line.strip() for line in open(os.path.join(data_dir, 'smileys.txt')))
-abbrs = set(line.strip() for line in open(os.path.join(data_dir, 'abbr_list.txt')))
+abbrs = set(CharFix.tr_lowercase(line.strip()) for line in open(os.path.join(data_dir, 'abbr_list.txt')))
+candidate_abbrs = set(line.strip() for line in open(os.path.join(data_dir, 'cand_abbr_list.txt')))
 word_list = set(line.strip() for line in open(os.path.join(data_dir, 'TS_Corpus_Turkish_Word_List.txt')))
 exception_words = set(line.strip() for line in open(os.path.join(data_dir, 'exceptions.txt')))
 eng_words = set(line.strip() for line in open(os.path.join(data_dir, 'eng_word_list.txt')))
@@ -30,6 +33,10 @@ class LocalData:
     def abbrs():
         return abbrs
 
+    @staticmethod
+    def candidate_abbrs():
+        return candidate_abbrs
+
     @staticmethod
     def word_list():
         return word_list

{ts_tokenizer-0.2.2 → ts_tokenizer-0.2.3}/ts_tokenizer/token_handler.py

@@ -23,6 +23,7 @@ REGEX_PATTERNS = {
     "hour": re.compile(r"^(0[0-9]|1[0-9]|2[0-3])[:.][0-5][0-9]$"),
     "hour_suffix": re.compile(r"^(0[0-9]|1[0-9]|2[0-3])[:.][0-5][0-9](?:'te|'de|'da|'den|'dan|'ten|'tan|'deki|'daki)$"),
     "number_suffix": re.compile(r"^\d+(?:[.,]\d+)*'[a-zA-ZıiİüÜçÇöÖşŞğĞ]+$"),
+    "currency_suffix": re.compile(rf"^(?:[{re.escape(''.join(LocalData.currency_symbols()))}]\d{{1,3}}(?:[.,]\d{{3}})*([.,]\d+)?|\d{{1,3}}(?:[.,]\d{{3}})*([.,]\d+)?[{re.escape(''.join(LocalData.currency_symbols()))}])'[a-zA-ZıiİüÜçÇöÖşŞğĞ]+$"),
     #"hour_12": re.compile(r"^(0[0-9]|1[0-9]|2[0-3])[:.][0-5][0-9]([AP]M)$"),
     "hour_12": re.compile(r"^(0[1-9]|1[0-2])[:.][0-5][0-9](AM|PM)$"),
     "percentage_numbers_initial": re.compile(r'^%\d{1,3}(?:[.,]\d+)?$'),
@@ -37,9 +38,9 @@ REGEX_PATTERNS = {
     "in_parenthesis": re.compile(r'^[(\[{]+[^()\[\]{}]*[)\]}]+}$'),
     "numbered_title": re.compile(r'^\((\d{1,2})\)|^\[(\d{1,2})\]|^{(\d{1,2})\}'),
     "in_quotes": re.compile(r'^[\'"][^\'"]*[\'"]$'),
-    "copyright": re.compile(r'(^©[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9]+$)|(^[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9]+©$)'),
-    "registered": re.compile(r'(^®[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9]+$)|(^[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9]+®$)'),
-    "trade_mark": re.compile(r'(^™[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9]+$)|(^[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9]+™$)'),
+    "copyright": re.compile(r'(^©[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9_-]+$)|(^[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9_-]+©$)'),
+    "registered": re.compile(r'(^®[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9_-]+$)|(^[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9_-]+®$)'),
+    "trade_mark": re.compile(r'(^™[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9_-]+$)|(^[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9_-]+™$)'),
     "bullet_list": re.compile(r'^•[a-zA-ZıiİüÜçÇöÖşŞğĞ0-9]+$'),
     "three_or_more": re.compile(r'^([{}])\1{{2,}}$'.format(re.escape(string.punctuation))),
     "roman_number": re.compile(r'^(M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3}))\.?$'),
@@ -187,6 +188,31 @@ class TokenPreProcess:
             return [("\\", "Punc"), (word[1], "Punc")] + processed_content
         return None
 
+    @staticmethod
+    @apply_charfix
+    def is_abbr_with_apostrophe_suffix(word: str) -> list:
+        match = re.fullmatch(r"([A-Za-zÇĞİÖŞÜçğıöşü]+\.?)'([A-Za-zÇĞİÖŞÜçğıöşü]+)", word)
+        if not match:
+            return None
+
+        abbr_part = match.group(1)
+        suffix_part = match.group(2)
+        normalized_abbr = CharFix.tr_lowercase(abbr_part)
+
+        if not abbr_part.endswith("."):
+            return None
+
+        if normalized_abbr not in LocalData.abbrs() and abbr_part not in LocalData.candidate_abbrs():
+            return None
+
+        processed_suffix = TokenProcessor.process_token(suffix_part)
+        if isinstance(processed_suffix, tuple):
+            processed_suffix = [processed_suffix]
+        elif not isinstance(processed_suffix, list):
+            processed_suffix = [(suffix_part, "OOV")]
+
+        return [(abbr_part, "Abbr"), ("'", "Punc")] + processed_suffix
+
     @staticmethod
     def is_numbered_title(word: str) -> list:
         # Check if the word matches the "numbered_title" regex pattern
@@ -292,6 +318,19 @@ class TokenPreProcess:
 
         return None
 
+    @staticmethod
+    @apply_charfix
+    def is_currency_suffix(word: str) -> list:
+        result = check_regex(word, "currency_suffix")
+        if not result:
+            return None
+
+        currency_part, suffix_part = word.rsplit("'", 1)
+        if TokenPreProcess.is_currency(currency_part) and suffix_part:
+            return [(word, "Currency")]
+
+        return None
+
     @staticmethod
     @apply_charfix
     def is_percentage_numbers(word: str) -> list:
@@ -430,6 +469,27 @@ class TokenPreProcess:
         result = check_regex(word, "trade_mark")
         return [(result, "Trademark")] if result else None
 
+    @staticmethod
+    @apply_charfix
+    def is_marked_with_trailing_punc(word: str) -> list:
+        for suffix_len in range(1, len(word)):
+            trailing_punc = word[-suffix_len:]
+            core = word[:-suffix_len]
+
+            if not core or not all(char in puncs for char in trailing_punc):
+                continue
+
+            for checker in (
+                TokenPreProcess.is_copyright,
+                TokenPreProcess.is_registered,
+                TokenPreProcess.is_trademark,
+            ):
+                marked = checker(core)
+                if marked:
+                    return marked + [(char, "Punc") for char in trailing_punc]
+
+        return None
+
     @staticmethod
     def is_currency(word: str) -> list:
         result = check_regex(word, "currency")
@@ -1166,6 +1226,7 @@ regex = [
     TokenPreProcess.is_date,
     TokenPreProcess.is_hour,
     TokenPreProcess.is_hour_suffix,
+    TokenPreProcess.is_currency_suffix,
     TokenPreProcess.is_number_suffix,
     TokenPreProcess.is_number,
     TokenPreProcess.is_mention,
@@ -1173,6 +1234,7 @@ regex = [
     TokenPreProcess.is_in_quotes,
     TokenPreProcess.is_escaped_opening_quote,
     TokenPreProcess.is_opening_quote,
+    TokenPreProcess.is_abbr_with_apostrophe_suffix,
     TokenPreProcess.is_apostrophed,
     TokenPreProcess.is_numbered_title,
     TokenPreProcess.is_parenthesized_with_trailing_colon,
@@ -1181,6 +1243,7 @@ regex = [
     TokenPreProcess.is_registered,
     TokenPreProcess.is_copyright,
     TokenPreProcess.is_trademark,
+    TokenPreProcess.is_marked_with_trailing_punc,
     TokenPreProcess.is_bullet_list,
     TokenPreProcess.is_roman_number,
     TokenPreProcess.is_percentage_numbers_chars,

{ts_tokenizer-0.2.2 → ts_tokenizer-0.2.3/ts_tokenizer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ts_tokenizer
-Version: 0.2.2
+Version: 0.2.3
 Summary: TS Tokenizer is a hybrid (lexicon-based and rule-based) tokenizer designed specifically for tokenizing Turkish texts.
 Home-page: https://github.com/tanerim/ts_tokenizer
 Author: Taner Sezer

{ts_tokenizer-0.2.2 → ts_tokenizer-0.2.3}/ts_tokenizer.egg-info/SOURCES.txt

@@ -3,7 +3,11 @@ MANIFEST.in
 README.md
 pyproject.toml
 setup.py
+tests/test_abbr_apostrophe_suffix.py
+tests/test_abbreviation_lookup.py
+tests/test_currency_suffix.py
 tests/test_opening_quote_tokenization.py
+tests/test_symbolic_marks.py
 ts_tokenizer/__init__.py
 ts_tokenizer/char_fix.py
 ts_tokenizer/cli.py