ts-tokenizer 0.2.1__tar.gz → 0.2.2__tar.gz

{ts_tokenizer-0.2.1/ts_tokenizer.egg-info → ts_tokenizer-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ts_tokenizer
-Version: 0.2.1
+Version: 0.2.2
 Summary: TS Tokenizer is a hybrid (lexicon-based and rule-based) tokenizer designed specifically for tokenizing Turkish texts.
 Home-page: https://github.com/tanerim/ts_tokenizer
 Author: Taner Sezer
@@ -71,6 +71,12 @@ pip install --upgrade ts-tokenizer
 ```
 ---
 
+To force upgrade:
+```bash
+pip install --upgrade --no-cache-dir --force-reinstall ts-tokenizer
+```
+---
+
 To remove package, use:
 ```bash
 pip uninstall ts-tokenizer

{ts_tokenizer-0.2.1 → ts_tokenizer-0.2.2}/README.md

@@ -31,6 +31,12 @@ pip install --upgrade ts-tokenizer
 ```
 ---
 
+To force upgrade:
+```bash
+pip install --upgrade --no-cache-dir --force-reinstall ts-tokenizer
+```
+---
+
 To remove package, use:
 ```bash
 pip uninstall ts-tokenizer

ts_tokenizer-0.2.2/tests/test_opening_quote_tokenization.py

@@ -0,0 +1,56 @@
+from ts_tokenizer.tokenizer import tokenize
+
+
+def tagged_lines(text):
+    return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
+
+
+def test_opening_double_quote_attached_to_word():
+    assert tagged_lines('"Expert') == [
+        ('"', "Punc"),
+        ("Expert", "English_Word"),
+    ]
+
+
+def test_opening_single_quote_attached_to_word():
+    assert tagged_lines("'Expert") == [
+        ("'", "Punc"),
+        ("Expert", "English_Word"),
+    ]
+
+
+def test_escaped_opening_single_quote_attached_to_word():
+    assert tagged_lines("\\'Expert") == [
+        ("\\", "Punc"),
+        ("'", "Punc"),
+        ("Expert", "English_Word"),
+    ]
+
+
+def test_quote_followed_by_space_remains_unchanged():
+    assert tagged_lines('" Expert') == [
+        ('"', "Punc"),
+        ("Expert", "English_Word"),
+    ]
+
+
+def test_smiley_inside_word_is_not_split():
+    assert tagged_lines("Expertxp") == [
+        ("Expertxp", "OOV"),
+    ]
+
+
+def test_word_followed_by_smiley_is_preserved():
+    assert tagged_lines("Expert:)") == [
+        ("Expert", "English_Word"),
+        (":)", "Smiley"),
+    ]
+
+
+def test_quoted_word_followed_by_smiley_is_preserved():
+    assert tagged_lines('"Expert:)"') == [
+        ('"', "Punc"),
+        ("Expert", "English_Word"),
+        (":)", "Smiley"),
+        ('"', "Punc"),
+    ]

{ts_tokenizer-0.2.1 → ts_tokenizer-0.2.2}/ts_tokenizer/__init__.py

@@ -1,6 +1,6 @@
 from importlib import import_module
 
-__version__ = "0.2.1"
+__version__ = "0.2.2"
 
 _EXPORT_MAP = {
     "tokenize": (".tokenizer", "tokenize"),

{ts_tokenizer-0.2.1 → ts_tokenizer-0.2.2}/ts_tokenizer/data/smileys.txt

@@ -196,7 +196,6 @@ XD
 (:::X:::)
 :-x
 x-p
-xp
 x-P
 xP
 :-þ

{ts_tokenizer-0.2.1 → ts_tokenizer-0.2.2}/ts_tokenizer/token_handler.py

@@ -153,6 +153,40 @@ class TokenPreProcess:
                 processed_content = [processed_content]
             return [(initial_quotes, "Punc")] + processed_content + [(final_quotes, "Punc")] if result else None
 
+    @staticmethod
+    @apply_charfix
+    def is_opening_quote(word: str) -> list:
+        if len(word) > 1 and word[0] in {'"', "'"}:
+            content = word[1:]
+            if not content:
+                return None
+
+            processed_content = TokenProcessor.process_token(content)
+            if isinstance(processed_content, tuple):
+                processed_content = [processed_content]
+            elif not isinstance(processed_content, list):
+                processed_content = [(content, "OOV")]
+
+            return [(word[0], "Punc")] + processed_content
+        return None
+
+    @staticmethod
+    @apply_charfix
+    def is_escaped_opening_quote(word: str) -> list:
+        if len(word) > 2 and word[0] == "\\" and word[1] in {'"', "'"}:
+            content = word[2:]
+            if not content:
+                return None
+
+            processed_content = TokenProcessor.process_token(content)
+            if isinstance(processed_content, tuple):
+                processed_content = [processed_content]
+            elif not isinstance(processed_content, list):
+                processed_content = [(content, "OOV")]
+
+            return [("\\", "Punc"), (word[1], "Punc")] + processed_content
+        return None
+
     @staticmethod
     def is_numbered_title(word: str) -> list:
         # Check if the word matches the "numbered_title" regex pattern
@@ -491,8 +525,36 @@ class TokenPreProcess:
         if not any(is_smiley for _, is_smiley in segments):
             return None
 
+        filtered_segments = []
+        for index, (segment, is_smiley) in enumerate(segments):
+            if not is_smiley:
+                filtered_segments.append((segment, False))
+                continue
+
+            if segment.isalnum():
+                filtered_segments.append((segment, False))
+                continue
+
+            prev_char = segments[index - 1][0][-1] if index > 0 and segments[index - 1][0] else ""
+            next_char = segments[index + 1][0][0] if index + 1 < len(segments) and segments[index + 1][0] else ""
+
+            if prev_char.isalnum() and next_char.isalnum():
+                filtered_segments.append((segment, False))
+            else:
+                filtered_segments.append((segment, True))
+
+        if not any(is_smiley for _, is_smiley in filtered_segments):
+            return None
+
+        merged_segments = []
+        for segment, is_smiley in filtered_segments:
+            if merged_segments and not is_smiley and not merged_segments[-1][1]:
+                merged_segments[-1] = (merged_segments[-1][0] + segment, False)
+            else:
+                merged_segments.append((segment, is_smiley))
+
         tokens = []
-        for segment, is_smiley in segments:
+        for segment, is_smiley in merged_segments:
             if is_smiley:
                 tokens.append((segment, "Smiley"))
                 continue
@@ -1109,6 +1171,8 @@ regex = [
     TokenPreProcess.is_mention,
     TokenPreProcess.is_hashtag,
     TokenPreProcess.is_in_quotes,
+    TokenPreProcess.is_escaped_opening_quote,
+    TokenPreProcess.is_opening_quote,
     TokenPreProcess.is_apostrophed,
     TokenPreProcess.is_numbered_title,
     TokenPreProcess.is_parenthesized_with_trailing_colon,

{ts_tokenizer-0.2.1 → ts_tokenizer-0.2.2/ts_tokenizer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ts_tokenizer
-Version: 0.2.1
+Version: 0.2.2
 Summary: TS Tokenizer is a hybrid (lexicon-based and rule-based) tokenizer designed specifically for tokenizing Turkish texts.
 Home-page: https://github.com/tanerim/ts_tokenizer
 Author: Taner Sezer
@@ -71,6 +71,12 @@ pip install --upgrade ts-tokenizer
 ```
 ---
 
+To force upgrade:
+```bash
+pip install --upgrade --no-cache-dir --force-reinstall ts-tokenizer
+```
+---
+
 To remove package, use:
 ```bash
 pip uninstall ts-tokenizer

{ts_tokenizer-0.2.1 → ts_tokenizer-0.2.2}/ts_tokenizer.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ MANIFEST.in
 README.md
 pyproject.toml
 setup.py
+tests/test_opening_quote_tokenization.py
 ts_tokenizer/__init__.py
 ts_tokenizer/char_fix.py
 ts_tokenizer/cli.py