pre-bpe-morph-tr 0.1.0__tar.gz

pre_bpe_morph_tr-0.1.0/LICENSE.md

@@ -0,0 +1,32 @@
+Python port of the framework,
+
+Copyright 2020 Loodos
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+=======================================================================
+Original framework written in Java,
+
+Copyright 2018 Ahmet A. Akın, Mehmet D. Akın
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

pre_bpe_morph_tr-0.1.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: pre_bpe_morph_tr
+Version: 0.1.0
+License-File: LICENSE.md
+Requires-Dist: antlr4-python3-runtime==4.8
+Requires-Dist: numpy>=1.19.0
+Dynamic: license-file

pre_bpe_morph_tr-0.1.0/README.md

@@ -0,0 +1,19 @@
+This project is derived from the Python port of Zemberek by Loodos, which is itself based on the original Zemberek Java project by Ahmet A. Akın and Mehmet D. Akın.
+
+# Goal of pre_bpe_morph
+This package recognizes verbs, nouns, and named entities and their suffixes. Before BPE, this package removes suffixes and replaces them with their respective special tokens for identifiability. It also precedes a word type (Verb/Noun/NamedEntity) before any set of suffixes. The goal is to simplify Turkish grammar rules for small language models.
+
+## Usage
+```python
+from pre_bpe_morph_tr import MorphTokenizer
+tokenizer=MorphTokenizer("<|", "|>")
+
+tokenizer.tokenize("gülüveriniz")
+# response: ['g', 'ü', 'l', '<|Verb|>', '<|Hastily|>', '<|Req|>']
+
+tokenizer.detokenize(['k', 'o', 'ş', '<|Verb|>', '<|Fut|>', '<|A1sg|>'])
+# response: "koşacağım"
+```
+
+## Developed for language model
+Since this is developed for LM training, I tried to avoid preprocessing text (like converting "hal" to "hâl", or lowercasing words). The text should remain unchanged when encoded and then decoded.

pre_bpe_morph_tr-0.1.0/custom_tokenizer/detokenizer.py

@@ -0,0 +1,95 @@
+from typing import List
+from collections import OrderedDict
+
+from custom_tokenizer.word_generator import CustomWordGenerator
+
+class MorphDetokenizer:
+    def __init__(self, tk_start: str = "<|", tk_end: str = "|>", cache_limit: int = 100):
+        self.tk_start = tk_start
+        self.tk_end = tk_end
+        self.tk_start_len = len(tk_start)
+        self.tk_end_len = len(tk_end)
+        self.pos_tags = {"Noun", "Verb", "NamedEntity"}
+        self.generator = CustomWordGenerator()
+        self.generation_cache = OrderedDict()
+        self.cache_limit = cache_limit
+
+    def _is_special_token(self, token: str) -> bool:
+        return token.startswith(self.tk_start) and token.endswith(self.tk_end)
+
+    def _strip_token(self, token: str) -> str:
+        return token[self.tk_start_len : -self.tk_end_len]
+
+    def _is_word_char(self, token: str) -> bool:
+        return token.isalnum()
+
+    def _collect_suffixes(self, tokens: List[str], start_idx: int) -> tuple[List[str], int]:
+        """Collect all contiguous suffix tokens starting at start_idx."""
+        suffixes = []
+        n = len(tokens)
+        i = start_idx
+        while i < n and self._is_special_token(tokens[i]):
+            suffix_inner = self._strip_token(tokens[i])
+            # If we hit a word type POS tag, stop collecting suffixes
+            if suffix_inner in self.pos_tags:
+                break
+            suffixes.append(suffix_inner)
+            i += 1
+        return suffixes, i
+
+    def _reconstruct_and_append(self, word_type: str, suffixes: List[str], current_chars: List[str], result_parts: List[str]):
+        """Reconstruct the word from the root and suffixes and append it to result_parts."""
+        root = "".join(current_chars)
+        cache_key = (root, word_type, tuple(suffixes))
+        
+        if cache_key in self.generation_cache:
+            reconstructed = self.generation_cache[cache_key]
+            self.generation_cache.move_to_end(cache_key)
+        else:
+            reconstructed = self.generator.generate_word(root, word_type, suffixes)
+            self.generation_cache[cache_key] = reconstructed
+            if len(self.generation_cache) > self.cache_limit:
+                self.generation_cache.popitem(last=False)
+            
+        result_parts.append(reconstructed)
+
+    def detokenize(self, tokens: List[str]) -> str:
+        result_parts = []
+        current_chars = []
+        
+        i = 0
+        n = len(tokens)
+        while i < n:
+            token = tokens[i]
+            
+            if self._is_special_token(token):
+                inner = self._strip_token(token)
+                
+                if inner in self.pos_tags:
+                    # Case 1: POS tag followed by suffixes
+                    suffixes, i = self._collect_suffixes(tokens, i + 1)
+                    self._reconstruct_and_append(inner, suffixes, current_chars, result_parts)
+                    current_chars = []
+                else:
+                    # Case 2: Suffix token without a preceding POS tag
+                    suffixes, i = self._collect_suffixes(tokens, i + 1)
+                    suffixes.insert(0, inner)
+                    self._reconstruct_and_append("Noun", suffixes, current_chars, result_parts)
+                    current_chars = []
+            else:
+                # Normal character, whitespace, or punctuation
+                if self._is_word_char(token):
+                    current_chars.append(token)
+                else:
+                    if current_chars:
+                        result_parts.append("".join(current_chars))
+                        current_chars = []
+                    result_parts.append(token)
+                i += 1
+                
+        # Append any remaining characters
+        if current_chars:
+            result_parts.append("".join(current_chars))
+            
+        return "".join(result_parts)
+

pre_bpe_morph_tr-0.1.0/custom_tokenizer/dictionary.py

@@ -0,0 +1,130 @@
+from zemberek.morphology.morphotactics.turkish_morphotactics import TurkishMorphotactics, get_morpheme_map
+
+# Trigger class definition to ensure side-effects populate the map
+_ = TurkishMorphotactics
+
+morpheme_map = get_morpheme_map()
+
+idx2morpheme_id =[
+    "Root",
+    "Noun",
+    "Adj",
+    "Verb",
+    "Pron",
+    "Adv",
+    "Conj",
+    "Punc",
+    "Ques",
+    "Postp",
+    "Det",
+    "Num",
+    "Dup",
+    "Interj",
+    "A1sg",
+    "A2sg",
+    "A3sg",
+    "A1pl",
+    "A2pl",
+    "A3pl",
+    "Pnon",
+    "P1sg",
+    "P2sg",
+    "P3sg",
+    "P1pl",
+    "P2pl",
+    "P3pl",
+    "Nom",
+    "Dat",
+    "Acc",
+    "Abl",
+    "Loc",
+    "Ins",
+    "Gen",
+    "Equ",
+    "Dim",
+    "Ness",
+    "With",
+    "Without",
+    "Related",
+    "JustLike",
+    "Rel",
+    "Agt",
+    "Become",
+    "Acquire",
+    "Ly",
+    "Caus",
+    "Recip",
+    "Reflex",
+    "Able",
+    "Pass",
+    "Inf1",
+    "Inf2",
+    "Inf3",
+    "ActOf",
+    "PastPart",
+    "NarrPart",
+    "FutPart",
+    "PresPart",
+    "AorPart",
+    "NotState",
+    "FeelLike",
+    "EverSince",
+    "Repeat",
+    "Almost",
+    "Hastily",
+    "Stay",
+    "Start",
+    "AsIf",
+    "While",
+    "When",
+    "SinceDoingSo",
+    "AsLongAs",
+    "ByDoingSo",
+    "Adamantly",
+    "AfterDoingSo",
+    "WithoutHavingDoneSo",
+    "WithoutBeingAbleToHaveDoneSo",
+    "Zero",
+    "Cop",
+    "Neg",
+    "Unable",
+    "Pres",
+    "Past",
+    "Narr",
+    "Cond",
+    "Prog1",
+    "Prog2",
+    "Aor",
+    "Fut",
+    "Imp",
+    "Req",
+    "Opt",
+    "Desr",
+    "Neces",
+    "Intrj"
+]
+
+morpheme_id2idx = {
+    morpheme_id: idx for idx, morpheme_id in enumerate(idx2morpheme_id)
+}
+
+idx2morpheme = {
+    idx: morpheme_map[morpheme_id] for idx, morpheme_id in enumerate(idx2morpheme_id)
+}
+
+wordPos_ids = {
+    "Root",
+    "Noun",
+    "Adj",
+    "Verb",
+    "Pron",
+    "Adv",
+    "Conj",
+    "Punc",
+    "Ques",
+    "Postp",
+    "Det",
+    "Num",
+    "Dup",
+    "Interj",
+}

pre_bpe_morph_tr-0.1.0/custom_tokenizer/morph_tokenizer.py

@@ -0,0 +1,144 @@
+from typing import List
+from zemberek.morphology.analysis.word_analysis import WordAnalysis
+from zemberek import TurkishSentenceExtractor
+from custom_tokenizer.utils import get_morphology, match_capitilization, is_morph_analysis_ok
+
+from zemberek.core.turkish import PrimaryPos, SecondaryPos
+from custom_tokenizer.detokenizer import MorphDetokenizer
+from zemberek.core.turkish.root_attribute import RootAttribute
+
+class MorphTokenizer:
+    def __init__(self, tk_start, tk_end):
+        self.tk_start = tk_start
+        self.tk_end = tk_end
+        self.morphology = get_morphology()
+        self.extractor = TurkishSentenceExtractor()
+        self.special_token = lambda s: tk_start + s + tk_end
+
+    def tokenize(self, text: str) -> List[str]:
+        sentences = self.extractor.from_paragraph(text)
+        
+        all_tokens = []
+        current_pos = 0
+        
+        for i, sentence_text in enumerate(sentences):
+            # Find the sentence in the original text to capture preceding whitespace/newlines
+            start_idx = text.find(sentence_text, current_pos)
+            prefix = text[current_pos:start_idx]
+            if prefix:
+                all_tokens.extend(list(prefix))
+            
+            # Tokenize the sentence itself
+            all_tokens.extend(self.__tokenize_sentence(sentence_text))
+            
+            current_pos = start_idx + len(sentence_text)
+            
+        # Add any trailing whitespace after the last sentence
+        trailing = text[current_pos:]
+        if trailing:
+            all_tokens.extend(list(trailing))
+            
+        return all_tokens
+    
+    def detokenize(self, tokens: List[str]) -> str:
+        detokenizer = MorphDetokenizer(tk_start=self.tk_start, tk_end=self.tk_end)
+        return detokenizer.detokenize(tokens)
+    
+    def __tokenize_sentence(self, sentence: str) -> List[str]:
+        after = self.morphology.analyze_sentence(sentence)
+
+        whitespaces = self.__collect_whitespaces(sentence, after)
+
+        words = []
+        for word_analysis in after:
+            tokens = self.__get_word_tokens(word_analysis)
+            words.append(tokens)
+            
+        split_by_words = self.__reconstruct_sentence(words, whitespaces)
+        return [token for word in split_by_words for token in word]
+
+    def __get_word_type(self, item, original_surface: str) -> str:
+        if item.primary_pos == PrimaryPos.Verb:
+            return "Verb"
+
+        # Check for numeric words recognized by Zemberek
+        numeric_secondary_pos = {
+            SecondaryPos.Cardinal, SecondaryPos.Clock, SecondaryPos.Date,
+            SecondaryPos.Ordinal, SecondaryPos.Percentage, SecondaryPos.Ratio,
+            SecondaryPos.Real, SecondaryPos.Distribution, SecondaryPos.Range
+        }
+        if item.secondary_pos in numeric_secondary_pos:
+            return "Noun"
+
+        # NamedEntity is anything with an apostrophe (that isn't a verb or numeric)
+        if "'" in original_surface:
+            return "NamedEntity"
+            
+        if item.primary_pos not in [PrimaryPos.Unknown, PrimaryPos.Punctuation]:
+            return "Noun"
+        return None
+
+    def __get_word_tokens(self, word_analysis: WordAnalysis) -> List[str]:
+        original_surface = word_analysis.inp
+        
+        if not is_morph_analysis_ok(original_surface) or not word_analysis.analysis_results:
+            return list(original_surface) # declare word as unknown
+            
+        best = word_analysis.analysis_results[0]
+        item = best.item
+            
+        word_type = self.__get_word_type(item, original_surface)
+        
+        tokens = []
+        suffixes = []
+        for i, m_data in enumerate(best.morpheme_data_list):
+            if i == 0:
+                # Use the lemma/root form for the stem rather than the surface allomorph
+                if (not item.is_unknown()) and (not RootAttribute.CompoundP3sg in item.attributes):
+                    stem = item.normalized_lemma()
+                else:
+                    stem = m_data.surface
+                if "'" in original_surface and not item.is_unknown():
+                    stem = original_surface.split("'")[0]
+                tokens.extend(list(match_capitilization(original_surface, stem)))
+            elif len(m_data.surface) > 0:
+                suffixes.append(self.special_token(m_data.morpheme.id_))
+        
+        if len(suffixes) > 0:
+            if word_type:
+                tokens.append(self.special_token(word_type))
+            tokens.extend(suffixes)
+        else:
+            return list(original_surface)
+        return tokens
+
+    def __reconstruct_sentence(self, words: List[List[str]], whitespaces: List[str]) -> List[List[str]]:
+        sentence = []
+        for i in range(len(words) + len(whitespaces)):
+            if i % 2 == 1:
+                if i // 2 < len(words):
+                    sentence.append(words[i // 2])
+            else:
+                if i // 2 < len(whitespaces) and len(whitespaces[i // 2]) > 0:
+                    sentence.append(list(whitespaces[i // 2]))
+        return sentence
+    
+    def __collect_whitespaces(self, sentence: str, analyses: List[WordAnalysis]) -> List[str]:
+        whitespaces = []
+        current_pos = 0
+        
+        for wa in analyses:
+            original_surface = wa.inp
+            # Find the start of this word in the original sentence
+            start_idx = sentence.find(original_surface, current_pos)
+            
+            # The gap before this word
+            whitespaces.append(sentence[current_pos:start_idx])
+            
+            # Move current_pos past the word
+            current_pos = start_idx + len(original_surface)
+            
+        # Add the trailing characters (if any)
+        whitespaces.append(sentence[current_pos:])
+        
+        return whitespaces

pre_bpe_morph_tr-0.1.0/custom_tokenizer/utils.py

@@ -0,0 +1,44 @@
+from zemberek import TurkishMorphology
+from zemberek.core.turkish.turkish_alphabet import TurkishAlphabet
+import logging
+
+_morphotactics = None
+_morphology = None
+
+def get_morphotactics():
+    global _morphotactics
+    if _morphotactics is None:
+        from zemberek.morphology.lexicon.root_lexicon import RootLexicon
+        from zemberek.morphology.morphotactics.turkish_morphotactics import TurkishMorphotactics
+        lexicon = RootLexicon.get_default()
+        _morphotactics = TurkishMorphotactics(lexicon)
+    return _morphotactics
+
+def get_morphology():
+    global _morphology
+    if _morphology is None:
+        _morphology = TurkishMorphology.create_with_defaults()
+    return _morphology
+
+def match_capitilization(ref: str, target: str) -> str:
+    if not ref or not target:
+        return target
+    
+    alphabet = TurkishAlphabet.INSTANCE
+    
+    if ref.isupper() and len(ref) > 1:
+        return target.translate(alphabet.upper_map).upper()
+        
+    if ref[0].isupper():
+        return target[0].translate(alphabet.upper_map).upper() + target[1:]
+
+    return target[0].translate(alphabet.lower_map).lower() + target[1:]
+
+def is_morph_analysis_ok(word: str) -> bool:
+    parts = word[1:].split("'")
+    trailing = parts[-1]
+    if len(parts) > 2: return False
+    if word.isupper(): return len(parts) == 1
+    for c in trailing:
+        if c.isupper(): return False
+    return True

pre_bpe_morph_tr-0.1.0/custom_tokenizer/word_generator.py

@@ -0,0 +1,267 @@
+from custom_tokenizer.utils import get_morphotactics, match_capitilization
+from zemberek.morphology.morphotactics.morpheme import Morpheme
+from zemberek.morphology.lexicon.dictionary_item import DictionaryItem
+from zemberek.morphology.morphotactics.stem_transition import StemTransition
+from zemberek.core.turkish.root_attribute import RootAttribute
+from zemberek.core.turkish import PhoneticAttribute
+from zemberek.morphology.analysis.attributes_helper import AttributesHelper
+from zemberek.core.turkish.primary_pos import PrimaryPos
+from zemberek.core.turkish.secondary_pos import SecondaryPos
+from zemberek.core.turkish.turkish_alphabet import TurkishAlphabet
+from zemberek.morphology.generator.word_generator import WordGenerator
+from custom_tokenizer.dictionary import morpheme_map
+from zemberek.morphology.analysis.tr.pronunciation_guesser import PronunciationGuesser
+import logging
+from typing import List, Union, Literal, Set
+
+class CustomWordGenerator:
+    def __init__(self):
+        self.guesser = PronunciationGuesser()
+        self.alphabet = TurkishAlphabet.INSTANCE
+        self.morphotactics = get_morphotactics()
+
+    def generate_word(self, root: str, word_type: Literal["Noun", "Verb", "NamedEntity"], suffixes: Union[List[Morpheme], List[str]]) -> str:
+        """Generate a word form using Zemberek's WordGenerator.
+        
+        Handles dictionary items, unknown words, and NamedEntity special logic.
+        """
+        if not suffixes:
+            return root
+
+        suffix_objs = [s if isinstance(s, Morpheme) else morpheme_map[s] for s in suffixes]
+        p_pos, s_pos = self._get_pos_enums(word_type)
+        
+        lexicon = self.morphotactics.get_root_lexicon()
+        matching_items = self._find_lexicon_items(root, word_type, p_pos, lexicon)
+        candidates = self._get_stem_candidates(root, matching_items, word_type, p_pos, s_pos)
+
+        generator = WordGenerator(self.morphotactics)
+        results = generator.generate(morphemes=tuple(suffix_objs), candidates=tuple(candidates))
+        
+        if results:
+            return self._apply_post_processing(root, results[0].surface, word_type)
+
+        forced_result = self.force_suffixes_on_word(root, word_type == "NamedEntity", suffix_objs)
+        return match_capitilization(root, forced_result)
+
+    def _get_pos_enums(self, word_type: str) -> tuple[PrimaryPos, SecondaryPos]:
+        if word_type in ["Noun", "NamedEntity"]:
+            p_pos = PrimaryPos.Noun
+        elif word_type == "Adj":
+            p_pos = PrimaryPos.Adjective
+        else:  # Verb and others
+            p_pos = PrimaryPos.Verb
+        
+        s_pos = SecondaryPos.ProperNoun if word_type == "NamedEntity" else SecondaryPos.None_
+        return p_pos, s_pos
+
+    def _filter_lexicon_items(self, items: List[DictionaryItem], word_type: str, p_pos: PrimaryPos = None) -> List[DictionaryItem]:
+        if p_pos is not None:
+            items = [item for item in items if item.primary_pos == p_pos]
+        else:
+            items = [item for item in items if item.primary_pos != PrimaryPos.Verb]
+
+        if word_type == "Noun":
+            items = [item for item in items if item.secondary_pos != SecondaryPos.ProperNoun]
+        elif word_type == "NamedEntity":
+            proper_items = [item for item in items if item.secondary_pos == SecondaryPos.ProperNoun]
+            if proper_items:
+                items = proper_items
+
+        return items
+
+    def _find_lexicon_items(self, root: str, word_type: str, p_pos: PrimaryPos, lexicon) -> List[DictionaryItem]:
+        lex_key = root
+        if word_type == "Verb":
+            lex_key = self._add_Inf1_suffix(root)
+        
+        items = self._filter_lexicon_items(lexicon.item_map.get(lex_key, []), word_type, p_pos)
+        
+        if not items and (root.istitle() or root.isupper()):
+            alt_lex_key = root.translate(self.alphabet.lower_map).lower()
+            if word_type == "Verb":
+                alt_lex_key = self._add_Inf1_suffix(alt_lex_key)
+            items = self._filter_lexicon_items(lexicon.item_map.get(alt_lex_key, []), word_type, p_pos)
+
+        # If still no items found, try any available POS as a fallback
+        # (but exclude Verb to avoid inappropriate morphological rules)
+        if not items:
+            items = self._filter_lexicon_items(lexicon.item_map.get(lex_key, []), word_type)
+            if not items and (root.istitle() or root.isupper()):
+                alt_lex_key = root.translate(self.alphabet.lower_map).lower()
+                if word_type == "Verb":
+                    alt_lex_key = self._add_Inf1_suffix(alt_lex_key)
+                items = self._filter_lexicon_items(lexicon.item_map.get(alt_lex_key, []), word_type)
+
+        return items
+
+    def _get_stem_candidates(self, root: str, items: List[DictionaryItem], word_type: str, p_pos: PrimaryPos, s_pos: SecondaryPos) -> List[StemTransition]:
+        candidates = []
+        has_vowel = any(self.alphabet.is_vowel(c) for c in root)
+        for item in items:
+            if word_type == "NamedEntity" and not has_vowel:
+                # For vowel-less NamedEntities (abbreviations), ensure they have phonetic attributes
+                # even if using a dictionary item, as some dictionary entries might be missing them.
+                start_state = self.morphotactics.noun_S
+                phonetic_attrs = self._get_phonetic_attributes(root)
+                candidates.append(StemTransition(root, item, phonetic_attrs, start_state))
+            else:
+                candidates.extend(self.morphotactics.stem_transitions.get_transitions_for_item(item))
+
+        if not candidates:
+            candidates.append(self._create_stem_transition(root, p_pos, s_pos))
+        return candidates
+
+    def _turkish_lower(self, s: str) -> str:
+        return s.translate(self.alphabet.lower_map).lower()
+
+    def _apply_post_processing(self, root: str, generated_surface: str, word_type: str) -> str:
+        is_named_entity = word_type == "NamedEntity"
+        is_number = self.alphabet.contains_digit(root)
+        root_lower = self._turkish_lower(root)
+        generated_lower = self._turkish_lower(generated_surface)
+        
+        if (is_named_entity or is_number) and generated_lower != root_lower:
+            if generated_lower.startswith(root_lower):
+                suffix = generated_lower[len(root_lower):]
+                return root + "'" + suffix
+        
+        return match_capitilization(root, generated_surface)
+
+    def force_suffixes_on_word(self, root: str, is_named_entity: bool, suffixes: List[Morpheme]) -> str:
+        logging.warning(
+            f"Couldn't add suffixes: {[s.id_ for s in suffixes]} to \"{root}\" "
+            f"via zemberek's own method. Deploying work around"
+        )
+        
+        generator = WordGenerator(self.morphotactics)
+        current_surface = root
+        apostrophe_added = False
+        
+        for suffix in suffixes:
+            if suffix.id_ == "Rel":
+                current_surface += "ki"
+                continue
+
+            generated_surface = self._try_force_generate_suffix(
+                current_surface, suffix, is_named_entity, apostrophe_added, generator
+            )
+            
+            if generated_surface:
+                current_surface, apostrophe_added = self._update_forced_surface(
+                    current_surface, generated_surface, is_named_entity, apostrophe_added
+                )
+            else:
+                logging.error(f"Could not generate suffix {suffix.id_} for {current_surface}")
+                
+        return current_surface
+
+    def _try_force_generate_suffix(self, current_surface: str, suffix: Morpheme, is_named_entity: bool, apostrophe_added: bool, generator: WordGenerator) -> Union[str, None]:
+        possible_pos = self._get_primary_pos_for_suffix(suffix)
+        for p_pos in possible_pos:
+            s_pos = SecondaryPos.ProperNoun if is_named_entity and not apostrophe_added else SecondaryPos.None_
+            candidate = self._create_stem_transition(current_surface, p_pos, s_pos)
+            results = generator.generate(morphemes=(suffix,), candidates=(candidate,))
+            if results:
+                return results[0].surface
+        return None
+
+    def _update_forced_surface(self, current_surface: str, generated_surface: str, is_named_entity: bool, apostrophe_added: bool) -> tuple[str, bool]:
+        if is_named_entity and not apostrophe_added and generated_surface != current_surface:
+            current_lower = self._turkish_lower(current_surface)
+            generated_lower = self._turkish_lower(generated_surface)
+            if generated_lower.startswith(current_lower):
+                suffix_surface = generated_lower[len(current_lower):]
+                return f"{current_surface}'{suffix_surface}", True
+            else:
+                return generated_surface, False
+        return generated_surface, apostrophe_added
+    
+    def _get_primary_pos_for_suffix(self, morpheme: Morpheme) -> List[PrimaryPos]:
+        m_id = morpheme.id_
+        
+        noun_suffixes = {
+            "Pnon", "P1sg", "P2sg", "P3sg", "P1pl", "P2pl", "P3pl", "Nom", "Dat", "Acc", "Abl", "Loc", "Ins", "Gen", "Equ",
+            "Dim", "Ness", "With", "Without", "Related", "JustLike", "Rel", "Agt", "Become", "Acquire", "Zero", "Root",
+            "A1sg", "A2sg", "A3sg", "A1pl", "A2pl", "A3pl", "Past", "Narr", "Cond", "Cop", "Noun"
+        }
+        
+        adjective_suffixes = {
+            "Ly", "AsIf", "Agt", "JustLike", "Become", "Acquire"
+        }
+        
+        verb_suffixes = {
+            "Caus", "Recip", "Reflex", "Able", "Pass", "Neg", "Unable", "Pres", "Prog1", "Prog2", "Aor", "Fut", "Imp", "Req",
+            "Opt", "Desr", "Neces", "Inf1", "Inf2", "Inf3", "ActOf", "PastPart", "NarrPart", "FutPart", "PresPart", "AorPart",
+            "NotState", "FeelLike", "EverSince", "Repeat", "Almost", "Hastily", "Stay", "Start", "AsIf", "While",
+            "When", "SinceDoingSo", "AsLongAs", "ByDoingSo", "Adamantly", "AfterDoingSo", "WithoutHavingDoneSo",
+            "WithoutBeingAbleToHaveDoneSo", "A1sg", "A2sg", "A3sg", "A1pl", "A2pl", "A3pl", "Past", "Narr", "Cond", "Cop", "Verb",
+            "Intrj"
+        }
+
+        results = []
+        if m_id in noun_suffixes: results.append(PrimaryPos.Noun)
+        if m_id in adjective_suffixes: results.append(PrimaryPos.Adjective)
+        if m_id in verb_suffixes: results.append(PrimaryPos.Verb)
+        
+        if results: return results
+
+        return [PrimaryPos.Unknown]
+
+    def _is_single_syllable(self, word: str) -> bool:
+        from zemberek.core.turkish.turkish_alphabet import TurkishAlphabet
+        vowel_count = sum(1 for char in word if TurkishAlphabet.INSTANCE.is_vowel(char))
+        return vowel_count == 1
+
+    def _add_Inf1_suffix(self, verb: str) -> str:
+        alphabet =TurkishAlphabet()
+        is_frontal = alphabet.get_last_vowel(verb).is_frontal()
+        sfx = "mek" if is_frontal else "mak"
+        return verb+sfx
+
+    def _create_stem_transition(self, root: str, p_pos: PrimaryPos, s_pos: SecondaryPos = SecondaryPos.None_) -> StemTransition:
+        attributes = set()
+        if p_pos == PrimaryPos.Verb:
+            if self._is_single_syllable(root):
+                attributes.add(RootAttribute.Aorist_A)
+            else:
+                attributes.add(RootAttribute.Aorist_I)
+        
+        dummy_item = DictionaryItem(root, root, p_pos, s_pos, attributes=attributes)
+        
+        # Determine the appropriate start state based on PrimaryPos
+        if p_pos == PrimaryPos.Verb:
+            start_state = self._get_verb_root_state(root)
+        elif p_pos == PrimaryPos.Adjective:
+            start_state = self.morphotactics.adjectiveRoot_ST
+        else:  # Noun and others
+            start_state = self.morphotactics.noun_S
+        
+        phonetic_attrs = self._get_phonetic_attributes(root)
+        res = StemTransition(root, dummy_item, phonetic_attrs, start_state)
+        return res
+
+    def _get_verb_root_state(self, root: str):
+        root_lower = root.translate(self.alphabet.lower_map).lower()
+        if root_lower in {"di", "yi", "de", "ye"}:
+            return self.morphotactics.vDeYeRoot_S
+        # Unknown verb roots should use the general verb root state.
+        # Vowel-ending stems without dictionary evidence should not be assumed
+        # to be vowel-drop verbs, otherwise suffixes like Fut fail.
+        return self.morphotactics.verbRoot_S
+
+    def _get_phonetic_attributes(self, root: str) -> Set[PhoneticAttribute]:
+        has_vowel = any(self.alphabet.is_vowel(c) for c in root)
+        
+        if not has_vowel and len(root) > 0:
+            # Letter names are usually defined for lowercase letters in the guesser
+            normalized_root = root.translate(self.alphabet.lower_map).lower()
+            pronunciation = self.guesser.to_turkish_letter_pronunciations(normalized_root)
+            if pronunciation:
+                return AttributesHelper.get_morphemic_attributes(pronunciation)
+        
+        if self.alphabet.contains_digit(root):
+            pronunciation = self.guesser.to_turkish_letter_pronunciation_with_digit(root)
+            return AttributesHelper.get_morphemic_attributes(pronunciation)
+            
+        return AttributesHelper.get_morphemic_attributes(root)

pre_bpe_morph_tr-0.1.0/pre_bpe_morph_tr.egg-info/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: pre_bpe_morph_tr
+Version: 0.1.0
+License-File: LICENSE.md
+Requires-Dist: antlr4-python3-runtime==4.8
+Requires-Dist: numpy>=1.19.0
+Dynamic: license-file

pre_bpe_morph_tr-0.1.0/pre_bpe_morph_tr.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+LICENSE.md
+README.md
+pyproject.toml
+custom_tokenizer/__init__.py
+custom_tokenizer/detokenizer.py
+custom_tokenizer/dictionary.py
+custom_tokenizer/morph_tokenizer.py
+custom_tokenizer/utils.py
+custom_tokenizer/word_generator.py
+pre_bpe_morph_tr.egg-info/PKG-INFO
+pre_bpe_morph_tr.egg-info/SOURCES.txt
+pre_bpe_morph_tr.egg-info/dependency_links.txt
+pre_bpe_morph_tr.egg-info/requires.txt
+pre_bpe_morph_tr.egg-info/top_level.txt
+tests/test_decode_and_encode.py
+tests/test_generation.py
+tests/test_morph_tokenizer.py
+tests/test_sentence_tokenizer.py
+zemberek/__init__.py
+zemberek/examples.py

pre_bpe_morph_tr-0.1.0/pre_bpe_morph_tr.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pre_bpe_morph_tr-0.1.0/pre_bpe_morph_tr.egg-info/requires.txt

@@ -0,0 +1,2 @@
+antlr4-python3-runtime==4.8
+numpy>=1.19.0

pre_bpe_morph_tr-0.1.0/pre_bpe_morph_tr.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+custom_tokenizer
+zemberek

pre_bpe_morph_tr-0.1.0/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["setuptools<82.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pre_bpe_morph_tr"
+version = "0.1.0"
+dependencies = [
+    "antlr4-python3-runtime==4.8",
+    "numpy>=1.19.0",
+]
+
+[tool.setuptools]
+packages = ["zemberek", "custom_tokenizer"]

pre_bpe_morph_tr-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pre_bpe_morph_tr-0.1.0/tests/test_decode_and_encode.py

@@ -0,0 +1,147 @@
+import sys
+from pathlib import Path
+import logging
+
+# Add the parent directory to sys.path to allow importing from the root
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+
+import pandas as pd
+from custom_tokenizer.morph_tokenizer import MorphTokenizer
+
+# Configure logging to hide noisy library messages
+logging.basicConfig(level=logging.ERROR)
+
+def run_decode_encode_test():
+    xlsx_path = Path(__file__).parent / "TurkishTweets.xlsx"
+    
+    if not xlsx_path.exists():
+        print(f"Error: Could not find TurkishTweets.xlsx at {xlsx_path}")
+        sys.exit(1)
+        
+    print(f"Reading Excel file: {xlsx_path.name}...")
+    try:
+        df = pd.read_excel(xlsx_path)
+    except Exception as e:
+        print(f"Error reading Excel file: {e}")
+        print("Please ensure that 'pandas' and 'openpyxl' are installed in your environment.")
+        print("To install them, run:")
+        print("  ./.venv/bin/pip install pandas openpyxl")
+        sys.exit(1)
+
+    # Dynamically find the tweet column (case-insensitive check for 'tweet')
+    tweet_col = None
+    for col in df.columns:
+        if "tweet" in str(col).lower():
+            tweet_col = col
+            break
+
+    if tweet_col is None:
+        tweet_col = df.columns[0]
+        print(f"Could not find a column named 'Tweets'. Using the first column: '{tweet_col}'")
+    else:
+        print(f"Using column: '{tweet_col}'")
+
+    print("Initializing MorphTokenizer...")
+    tokenizer = MorphTokenizer("<|", "|>")
+    
+    mismatches = []
+    total_count = 0
+    success_count = 0
+
+    print(f"Processing {len(df)} rows for the encode-decode round-trip test...")
+
+    for idx, row in df.iterrows():
+        original_tweet = row[tweet_col]
+        if pd.isna(original_tweet):
+            continue
+            
+        original_tweet = str(original_tweet)
+        total_count += 1
+        
+        try:
+            # Tokenize (Encode)
+            tokens = tokenizer.tokenize(original_tweet)
+            # Detokenize (Decode)
+            decoded_tweet = tokenizer.detokenize(tokens)
+            
+            if decoded_tweet == original_tweet:
+                success_count += 1
+            else:
+                mismatches.append({
+                    "row": idx + 2,  # 1-based index, account for header row (+2)
+                    "original": original_tweet,
+                    "decoded": decoded_tweet,
+                    "tokens": tokens
+                })
+        except Exception as e:
+            mismatches.append({
+                "row": idx + 2,
+                "original": original_tweet,
+                "error": str(e)
+            })
+
+    # Output test summary
+    print("\n" + "="*40)
+    print("           TEST RESULTS SUMMARY           ")
+    print("="*40)
+    print(f"Total tweets processed:  {total_count}")
+    print(f"Successful round-trips:  {success_count}")
+    print(f"Failed round-trips:      {len(mismatches)}")
+    if total_count > 0:
+        success_rate = (success_count / total_count) * 100
+        print(f"Success Rate:            {success_rate:.2f}%")
+    print("="*40)
+
+    # Print first few failures if any exist
+    if mismatches:
+        print(f"\nShowing up to the first 10 mismatches out of {len(mismatches)} total failures:")
+        for i, m in enumerate(mismatches[:10]):
+            print(f"\nMismatch #{i+1} (Excel Row {m['row']}):")
+            print(f"  Original:  {repr(m['original'])}")
+            if "error" in m:
+                print(f"  Error:     {m['error']}")
+            else:
+                print(f"  Decoded:   {repr(m['decoded'])}")
+                print(f"  Tokens:    {m['tokens']}")
+    else:
+        print("\nAll tweets successfully passed the encode-decode round-trip test!")
+
+    # Write all failures to log file
+    logs_dir = Path(__file__).resolve().parent / "logs"
+    logs_dir.mkdir(parents=True, exist_ok=True)
+    log_file_path = logs_dir / "failed_tests.log"
+    
+    from datetime import datetime
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    
+    with open(log_file_path, "w", encoding="utf-8") as f:
+        f.write("="*80 + "\n")
+        f.write(f"ENCODE-DECODE ROUND-TRIP TEST LOG - {timestamp}\n")
+        f.write("="*80 + "\n")
+        f.write(f"Total tweets processed:  {total_count}\n")
+        f.write(f"Successful round-trips:  {success_count}\n")
+        f.write(f"Failed round-trips:      {len(mismatches)}\n")
+        if total_count > 0:
+            success_rate = (success_count / total_count) * 100
+            f.write(f"Success Rate:            {success_rate:.2f}%\n")
+        f.write("="*80 + "\n\n")
+        
+        if mismatches:
+            f.write("DETAILED FAILURES:\n")
+            f.write("-"*80 + "\n")
+            for idx, m in enumerate(mismatches):
+                f.write(f"Failure #{idx+1} (Excel Row {m['row']}):\n")
+                f.write(f"  Original:  {repr(m['original'])}\n")
+                if "error" in m:
+                    f.write(f"  Error:     {m['error']}\n")
+                else:
+                    f.write(f"  Decoded:   {repr(m['decoded'])}\n")
+                    f.write(f"  Tokens:    {m['tokens']}\n")
+                f.write("-"*80 + "\n")
+        else:
+            f.write("All tests passed successfully! No mismatches found.\n")
+
+    print(f"\nFailing tests and details written to: {log_file_path}")
+
+if __name__ == "__main__":
+    run_decode_encode_test()

pre_bpe_morph_tr-0.1.0/tests/test_generation.py

@@ -0,0 +1,83 @@
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).parent.parent))
+
+from custom_tokenizer.word_generator import CustomWordGenerator
+import logging
+
+# Configure logging to show errors but avoid noise
+logging.basicConfig(level=logging.ERROR)
+generator = CustomWordGenerator()
+
+def test_generation(root, pos, suffix_ids, expected):
+    result = generator.generate_word(root, pos, suffix_ids)
+
+    output=""
+    if result != expected: output+="\033[31m"
+    output+=f"Root: {root} ({pos}), Suffixes: {suffix_ids} -> Result: {result}, Expected: {expected}"
+    print(output)
+    if result != expected: print("\033[0m", end="")
+
+if __name__ == "__main__":
+    print("Starting generation tests...")
+    
+    # Noun Cases (Dictionary Match)
+    test_generation("elma", "Noun", ["A3pl", "Dat"], "elmalara")
+    test_generation("Burun", "Noun", ["A3sg", "P1pl"], "Burnumuz")
+    test_generation("buRun", "Noun", ["A3sg", "P1pl"], "buRunumuz")
+    test_generation("hak", "Noun", ["Dat"], "hakka")
+    test_generation("burun", "Noun", ["Gen"], "burnun")
+    
+    # Verb Cases (Dictionary Match)
+    test_generation("kaç", "Verb", ["Prog1", "A1sg"], "kaçıyorum")
+    test_generation("koş", "Verb", ["Aor", "A1pl"], "koşarız")
+    test_generation("gel", "Verb", ["AorPart"], "gelir")
+    test_generation("seyret", "Verb", ["Aor", "AsIf"], "seyredercesine")
+    
+    # POS Inference
+    test_generation("at", "Noun", ["Dat"], "ata") 
+    test_generation("at", "Verb", ["Fut", "Narr", "A1sg"], "atacakmışım")
+
+    # Aggressive POS Inference
+    test_generation("elma", "Verb", ["Fut", "Narr", "A1sg"], "elmayacakmışım")
+    test_generation("gel", "Noun", ["P1pl", "Gen"], "gelimizin")
+
+    # Pronoun
+    test_generation("biz", "Noun", ["A3pl"], "bizler")
+
+    # Named Entity (Proper Noun)
+    test_generation("Çıtçıt", "NamedEntity", ["Dat"], "Çıtçıt'a")
+    test_generation("Bürokratistan", "NamedEntity", ["Loc", "Rel", "A3pl", "Gen"], "Bürokratistan'dakilerin")
+    test_generation("Ahmet", "NamedEntity", ["Dat"], "Ahmet'e")
+    test_generation("Ayşe", "Noun", ["Gen"], "Ayşenin")
+    test_generation("Tüik", "NamedEntity", ["Dat"], "Tüik'e")
+
+    # Unknown Root
+    test_generation("bloop", "Noun", ["Dat"], "bloopa")
+    test_generation("bloop", "Verb", ["Prog1"], "bloopuyor")
+
+    # Broken Generation
+    test_generation("kap", "Noun", ["Prog1", "Dim", "A3pl"], "kapıyorcuklar")
+    test_generation("kitap", "Noun", ["Loc", "Rel", "Gen", "Aor", "Almost"], "kitaptakininireyaz")
+
+    # All Caps
+    test_generation("KAÇ", "Verb", ["Prog2", "A2pl"], "KAÇMAKTASINIZ")
+    test_generation("HAK", "Noun", ["P2sg"], "HAKKIN")
+    test_generation("TÜİK", "NamedEntity", ["P2sg"], "TÜİK'in")
+    test_generation("TÜK", "NamedEntity", ["P2sg"], "TÜK'ün")
+
+    # Numbers
+    test_generation("11", "Noun", ["Loc"], "11'de")
+    test_generation("12.00", "Noun", ["Loc"], "12.00'da")
+    test_generation("örnek2", "Noun", ["Gen"], "örnek2'nin") # Not supported by zemberek analyzer
+    test_generation("3/4", "Noun", ["Gen"], "3/4'ün") # Not supported by zemberek analyzer
+
+    # No vowels
+    test_generation("kg", "NamedEntity", ["Ness"], "kg'lik")
+    test_generation("TDK", "NamedEntity", ["Acc"], "TDK'yı")
+    test_generation("TMNB", "NamedEntity", ["Acc"], "TMNB'yi")
+    test_generation("z", "NamedEntity", ["Acc"], "z'yi")
+    test_generation("z", "Noun", ["Acc"], "zyi")
+    test_generation("k", "Verb", ["Fut"], "kyacak")
+
+    print("\nAll tests completed!")

pre_bpe_morph_tr-0.1.0/tests/test_morph_tokenizer.py

@@ -0,0 +1,60 @@
+import sys
+from pathlib import Path
+import logging
+from typing import List
+sys.path.append(str(Path(__file__).parent.parent))
+import time
+
+from custom_tokenizer.morph_tokenizer import MorphTokenizer
+
+logging.basicConfig(level=logging.ERROR)
+
+def test_generation(string: str, tokenizer: MorphTokenizer, expected: List[str]):
+    t0 = time.time()
+    tokens = tokenizer.tokenize(string)
+    t1 = time.time()
+
+    output="COMPLETED IN " + str(t1-t0) + " SECONDS\n"
+    if tokens != expected:
+        output+="\033[31m"
+        output+=f"String: {string} ->\n Result: {tokens}\n Expected: {expected}\033[0m\n"
+    
+    # Test detokenization
+    detokenized = tokenizer.detokenize(tokens)
+    if detokenized != string:
+        output+="\033[31m"
+        output+=f"Detokenization failed!\nOriginal: {repr(string)}\nDetokenized: {repr(detokenized)}\033[0m\n"
+    else:
+        output+="\033[32mDetokenization successful matches original string!\033[0m\n"
+    print(output)
+
+if __name__ == "__main__":
+    tokenizer = MorphTokenizer("<|", "|>")
+    print("initialized!")
+    start_time = time.time()
+    paragraph=(
+        "Ayşeyi, Ahmet'i Veli'yi ve ghim'i eve çağırmış."
+        " burnu *Burnu havada kitapçımız 2 kglik yani kg'lik TR'li eşyayı Yerismi'ye getirirmiş."
+        "\n\tYarın içinse AYŞE'Yİ ve ANNEM'i eVe çağırDI.   "
+        "TDK'ye, UNKNOWN'a, TÜİK'e ve ALİ'ye göre olan hjKŞFh şeyler diyorlar.  "
+        " Oysa Annem, BABAM ve   kardeşlerime\ngöre\tdoğru olandır."
+        "Onlar 11'de veya 12.00'da burada olur."
+        "Sonra level2'nin 3/4'ü biter."
+    )
+    expected = [
+        'A', 'y', 'ş', 'e', '<|Noun|>', '<|Acc|>', ',', ' ', 'A', 'h', 'm', 'e', 't', '<|NamedEntity|>', '<|Acc|>', ' ', 'V', 'e', 'l', 'i', '<|NamedEntity|>', '<|Acc|>', ' ', 'v', 'e', ' ', 'g', 'h', 'i', 'm', '<|NamedEntity|>', '<|Acc|>', ' ', 'e', 'v', '<|Noun|>', '<|Dat|>', ' ', 'ç', 'a', 'ğ', 'ı', 'r', '<|Verb|>', '<|Narr|>', '.',
+        ' ', 'b', 'u', 'r', 'u', 'n', '<|Noun|>', '<|P3sg|>', ' ', '*', 'B', 'u', 'r', 'u', 'n', '<|Noun|>', '<|Acc|>', ' ', 'h', 'a', 'v', 'a', '<|Noun|>', '<|Loc|>', ' ', 'k', 'i', 't', 'a', 'p', '<|Noun|>', '<|Agt|>', '<|P1pl|>', ' ', '2', ' ', 'k', 'g', '<|Noun|>', '<|Ness|>', ' ', 'y', 'a', 'n', 'i', ' ', 'k', 'g', '<|NamedEntity|>', '<|Ness|>', ' ', 'T', 'R', '<|NamedEntity|>', '<|With|>', ' ', 'e', 'ş', 'y', 'a', '<|Noun|>', '<|Acc|>', ' ', 'Y', 'e', 'r', 'i', 's', 'm', 'i', '<|NamedEntity|>', '<|Dat|>', ' ', 'g', 'e', 't', 'i', 'r', '<|Verb|>', '<|Aor|>', '<|Narr|>', '.',
+        '\n', '\t', 'Y', 'a', 'r', 'ı', 'n', ' ', 'i', 'ç', 'i', 'n', '<|Noun|>', '<|Cond|>', ' ', 'A', 'Y', 'Ş', 'E', "'", 'Y', 'İ', ' ', 'v', 'e', ' ', 'A', 'N', 'N', 'E', 'M', '<|NamedEntity|>', '<|Acc|>', ' ', 'e', 'V', 'e', ' ', 'ç', 'a', 'ğ', 'ı', 'r', 'D', 'I', '.', ' ', ' ', ' ',
+        'T', 'D', 'K', "'", 'y', 'e', ',', ' ', 'U', 'N', 'K', 'N', 'O', 'W', 'N', '<|NamedEntity|>', '<|Dat|>', ',', ' ', 'T', 'Ü', 'İ', 'K', '<|NamedEntity|>', '<|Dat|>', ' ', 'v', 'e', ' ', 'A', 'L', 'İ', '<|NamedEntity|>', '<|Dat|>', ' ', 'g', 'ö', 'r', 'e', ' ', 'o', 'l', '<|Verb|>', '<|PresPart|>', ' ', 'h', 'j', 'K', 'Ş', 'F', 'h', ' ', 'ş', 'e', 'y', '<|Noun|>', '<|A3pl|>', ' ', 'd', 'e', '<|Verb|>', '<|Prog1|>', '<|A3pl|>', '.',
+        ' ', ' ', ' ', 'O', 'y', 's', 'a', ' ', 'A', 'n', 'n', 'e', '<|Noun|>', '<|P1sg|>', ',', ' ', 'B', 'A', 'B', 'A', '<|Noun|>', '<|P1sg|>', ' ', 'v', 'e', ' ', ' ', ' ', 'k', 'a', 'r', 'd', 'e', 'ş', '<|Noun|>', '<|A3pl|>', '<|P1sg|>', '<|Dat|>', '\n', 'g', 'ö', 'r', 'e', '\t', 'd', 'o', 'ğ', 'r', 'u', ' ', 'o', 'l', '<|Verb|>', '<|PresPart|>', '<|Cop|>', '.',
+        'O', '<|Noun|>', '<|A3pl|>', ' ', '1', '1', '<|Noun|>', '<|Loc|>', ' ', 'v', 'e', 'y', 'a', ' ', '1', '2', '.', '0', '0', '<|Noun|>', '<|Loc|>', ' ', 'b', 'u', 'r', 'a', '<|Noun|>', '<|Loc|>', ' ', 'o', 'l', '<|Verb|>', '<|Aor|>', '.',
+        'S', 'o', 'n', 'r', 'a', ' ', 'l', 'e', 'v', 'e', 'l', '2', "'", 'n', 'i', 'n', ' ', '3', '/', '4', "'", 'ü', ' ', 'b', 'i', 't', '<|Verb|>', '<|Aor|>', '.'
+    ]
+    test_generation(paragraph, tokenizer, expected)
+
+    test_generation("Jazz bir kediydi. Arkadaşları vardı: Pamuk, Minnoş ve Tekir. Onlar dans etmeyi çok severdi. Bir gün, zor bir dans öğrendiler. Her gün dans ettiler. Sabah, öğle ve akşam.\n\nİlk başlarda çok zorlandılar. Ayakları karıştı, düştüler ve güldüler. Ama pes etmediler. Her gün daha iyi oldular. Jazz, Pamuk, Minnoş ve Tekir birlikte çalıştılar.\n\nSonunda, dansı öğrendiler! Çok mutluydular. Şimdi dans etmeyi biliyorlardı. Dans ederken zıpladılar, döndüler ve kahkaha attılar.\n\nArtık her zaman dans ediyorlardı. Parkta, bahçede ve evde. Jazz ve arkadaşları dans etmeyi çok seviyorlardı!\n", tokenizer, [])
+    test_generation("haftasonu vakti", tokenizer, ['h',  'a',  'f',  't',  'a',  's',  'o',  'n',  '<|Noun|>',  '<|Acc|>',  ' ',  'v',  'a',  'k',  'i', 't', '<|Noun|>',  '<|Acc|>'])
+
+    test_generation("Geldiler. Ama pes etmediler.", tokenizer, ['G', 'e', 'l', '<|Verb|>', '<|Past|>', '<|A3pl|>', '.', ' ', 'A', 'm', 'a', ' ', 'p', 'e', 's', ' ', 'e', 't', '<|Verb|>', '<|Neg|>', '<|Past|>', '<|A3pl|>', '.'])
+    test_generation(".burnumuzun ", tokenizer, ['.', 'b', 'u', 'r', 'u', 'n', '<|Noun|>', '<|P1pl|>', '<|Gen|>', ' '])
+    print("All tests completed!")

pre_bpe_morph_tr-0.1.0/tests/test_sentence_tokenizer.py

@@ -0,0 +1,35 @@
+import sys
+from pathlib import Path
+import logging
+from typing import List
+sys.path.append(str(Path(__file__).parent.parent))
+
+from custom_tokenizer.morph_tokenizer import MorphTokenizer
+
+logging.basicConfig(level=logging.ERROR)
+
+def test_generation(sentence: str, tokenizer: MorphTokenizer, expected: List[str]):
+    tokens = tokenizer._MorphTokenizer__tokenize_sentence(sentence)
+
+    output=""
+    if tokens != expected:
+        output+="\033[31m"
+        output+=f"Sentence: {sentence} ->\n Result: {tokens}\n Expected: {expected}\033[0m"
+        print(output)
+
+if __name__ == "__main__":
+    tokenizer = MorphTokenizer("<|", "|>")
+    print("initialized!")
+    test_generation("Ayşeyi, Ahmet'i Veli'yi ve ghim'i eve çağırdı.", tokenizer, ['A', 'y', 'ş', 'e', '<|Noun|>', '<|Acc|>', ',', ' ', 'A', 'h', 'm', 'e', 't', '<|NamedEntity|>', '<|Acc|>', ' ', 'V', 'e', 'l', 'i', '<|NamedEntity|>', '<|Acc|>', ' ', 'v', 'e', ' ', 'g', 'h', 'i', 'm', '<|NamedEntity|>', '<|Acc|>', ' ', 'e', 'v', '<|Noun|>', '<|Dat|>', ' ', 'ç', 'a', 'ğ', 'ı', 'r', '<|Verb|>', '<|Past|>', '.'])
+    test_generation("kitapçı 2 kglik yani kg'lik TR'li eşya getirirmiş", tokenizer, ['k', 'i', 't', 'a', 'p', '<|Noun|>', '<|Agt|>', ' ', '2', ' ', 'k', 'g', '<|Noun|>', '<|Ness|>', ' ', 'y', 'a', 'n', 'i', ' ', 'k', 'g', '<|NamedEntity|>', '<|Ness|>', ' ', 'T', 'R', '<|NamedEntity|>', '<|With|>', ' ', 'e', 'ş', 'y', 'a', ' ', 'g', 'e', 't', 'i', 'r', '<|Verb|>', '<|Aor|>', '<|Narr|>'])
+    test_generation("Burnu burnu havadadır onun.", tokenizer, ['B', 'u', 'r', 'u', 'n', '<|Noun|>', '<|P3sg|>', ' ', 'b', 'u', 'r', 'u', 'n', '<|Noun|>', '<|P3sg|>', ' ', 'h', 'a', 'v', 'a', '<|Noun|>', '<|Loc|>', '<|Cop|>', ' ', 'o', '<|Noun|>', '<|Gen|>', '.'])
+    test_generation("Yerisimi'ye gelirmiş.", tokenizer, ['Y', 'e', 'r', 'i', 's', 'i', 'm', 'i', '<|NamedEntity|>', '<|Dat|>', ' ', 'g', 'e', 'l', '<|Verb|>', '<|Aor|>', '<|Narr|>', '.'])
+    test_generation("AYŞE'Yİ ve ANNEM'i eVe çağırDI", tokenizer, ['A', 'Y', 'Ş', 'E', "'", 'Y', 'İ', ' ', 'v', 'e', ' ', 'A', 'N', 'N', 'E', 'M', '<|NamedEntity|>', '<|Acc|>', ' ', 'e', 'V', 'e', ' ', 'ç', 'a', 'ğ', 'ı', 'r', 'D', 'I'])
+    test_generation("TDK'ye, UNKNOWN'a, TÜİK'e ve ALİ'ye göre olan hjKŞFh şeyler", tokenizer, ['T', 'D', 'K', "'", 'y', 'e', ',', ' ', 'U', 'N', 'K', 'N', 'O', 'W', 'N', '<|NamedEntity|>', '<|Dat|>', ',', ' ', 'T', 'Ü', 'İ', 'K', '<|NamedEntity|>', '<|Dat|>', ' ', 'v', 'e', ' ', 'A', 'L', 'İ', '<|NamedEntity|>', '<|Dat|>', ' ', 'g', 'ö', 'r', 'e', ' ', 'o', 'l', '<|Verb|>', '<|PresPart|>', ' ', 'h', 'j', 'K', 'Ş', 'F', 'h', ' ', 'ş', 'e', 'y', '<|Noun|>', '<|A3pl|>'])
+    test_generation("Annem, BABAM ve   kardeşlerime\ngöre\tdoğru olandır. ", tokenizer, ['A', 'n', 'n', 'e', '<|Noun|>', '<|P1sg|>', ',', ' ', 'B', 'A', 'B', 'A', '<|Noun|>', '<|P1sg|>', ' ', 'v', 'e', ' ', ' ', ' ', 'k', 'a', 'r', 'd', 'e', 'ş', '<|Noun|>', '<|A3pl|>', '<|P1sg|>', '<|Dat|>', '\n', 'g', 'ö', 'r', 'e', '\t', 'd', 'o', 'ğ', 'r', 'u', ' ', 'o', 'l', '<|Verb|>', '<|PresPart|>', '<|Cop|>', '.', ' '])
+    test_generation("Annem 11'de veya 12.00'da burada olur.", tokenizer, ['A', 'n', 'n', 'e', '<|Noun|>', '<|P1sg|>', ' ', '1', '1', '<|Noun|>', '<|Loc|>', ' ', 'v', 'e', 'y', 'a', ' ', '1', '2', '.', '0', '0', '<|Noun|>', '<|Loc|>', ' ', 'b', 'u', 'r', 'a', '<|Noun|>', '<|Loc|>', ' ', 'o', 'l', '<|Verb|>', '<|Aor|>', '.'])
+    test_generation("18ini de alıp 6'lı ayırdık", tokenizer, ['1',  '8',  'i',  'n',  'i',  ' ',  'd',  'e',  ' ',  'a',  'l',  '<|Verb|>',  '<|AfterDoingSo|>',  ' ',  '6',  '<|Noun|>',  '<|With|>',  ' ',  'a',  'y',  'ı',  'r',  '<|Verb|>',  '<|Past|>',  '<|A1pl|>'])
+
+    # unsupported 👇
+    test_generation("Örnek2'nin 3/4'ü oldu.", tokenizer, ['Ö', 'r', 'n', 'e', 'k', '2', "'", 'n', 'i', 'n', ' ', '3', '/', '4', '<|Noun|>', '<|Acc|>', ' ', 'o', 'l', '<|Verb|>', '<|Past|>', '.'])
+    print("All tests completed!")

pre_bpe_morph_tr-0.1.0/zemberek/__init__.py

@@ -0,0 +1,22 @@
+from .morphology import TurkishMorphology
+from .normalization import TurkishSentenceNormalizer, TurkishSpellChecker
+from .tokenization import TurkishSentenceExtractor, TurkishTokenizer
+import warnings
+
+# Suppress the pkg_resources deprecation warning from zemberek dependencies
+warnings.filterwarnings("ignore", category=UserWarning, module="pkg_resources")
+
+import logging
+import sys
+
+__version__ = '0.2.3'
+
+root = logging.getLogger()
+root.setLevel(logging.INFO)
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s\nMsg: %(message)s\n')
+handler.setFormatter(formatter)
+root.addHandler(handler)
+

pre_bpe_morph_tr-0.1.0/zemberek/examples.py

@@ -0,0 +1,102 @@
+import time
+import logging
+
+from zemberek import (
+    TurkishSpellChecker,
+    TurkishSentenceNormalizer,
+    TurkishSentenceExtractor,
+    TurkishMorphology,
+    TurkishTokenizer
+)
+
+logger = logging.getLogger(__name__)
+
+examples = ["Yrn okua gidicem",
+            "Tmm, yarin havuza giricem ve aksama kadar yaticam :)",
+            "ah aynen ya annemde fark ettı siz evinizden cıkmayın diyo",
+            "gercek mı bu? Yuh! Artık unutulması bile beklenmiyo",
+            "Hayır hayat telaşm olmasa alacam buraları gökdelen dikicem.",
+            "yok hocam kesınlıkle oyle birşey yok",
+            "herseyi soyle hayatında olmaması gerek bence boyle ınsanların falan baskı yapıyosa",
+            "email adresim zemberek_python@loodos.com",
+            "Kredi başvrusu yapmk istiyrum.",
+            "Bankanizin hesp blgilerini ogrenmek istyorum."]
+
+morphology = TurkishMorphology.create_with_defaults()
+
+# SENTENCE NORMALIZATION
+start = time.time()
+normalizer = TurkishSentenceNormalizer(morphology)
+logger.info(f"Normalization instance created in: {time.time() - start} s")
+
+start = time.time()
+for example in examples:
+    print(example)
+    print(normalizer.normalize(example), "\n")
+logger.info(f"Sentences normalized in: {time.time() - start} s")
+
+start = time.time()
+sc = TurkishSpellChecker(morphology)
+logger.info(f"Spell checker instance created in: {time.time() - start} s")
+
+
+# SPELLING SUGGESTION
+li = ["okuyablirim", "tartısıyor", "Ankar'ada", "knlıca", "yapablrim", "kıredi", "geldm", "geliyom", "aldm", "asln"]
+start = time.time()
+for word in li:
+    print(word + " = " + ' '.join(sc.suggest_for_word(word)))
+logger.info(f"Spells checked in: {time.time() - start} s")
+
+
+# SENTENCE BOUNDARY DETECTION
+start = time.time()
+extractor = TurkishSentenceExtractor()
+print("Extractor instance created in: ", time.time() - start, " s")
+
+text = "İnsanoğlu aslında ne para ne sevgi ne kariyer ne şöhret ne de çevre ile sonsuza dek mutlu olabilecek bir " \
+       "yapıya sahiptir. Dış kaynaklardan gelebilecek bu mutluluklar sadece belirli bir zaman için insanı mutlu " \
+       "kılıyor. Kişi bu kaynakları elde ettiği zaman belirli bir dönem için kendini iyi hissediyor, ancak alışma " \
+       "dönemine girdiği andan itibaren bu iyilik hali hızla tükeniyor. Mutlu olma sanatının özü bu değildir. Gerçek " \
+       "mutluluk, kişinin her türlü olaya ve duruma karşı kendini pozitif tutarak mutlu hissedebilmesi halidir. Bu " \
+       "davranış şeklini edinen insan, zor günlerde güçlü, mutlu günlerde zevk alan biri olur ve mutluluğu kalıcı " \
+       "kılar. "
+
+start = time.time()
+sentences = extractor.from_paragraph(text)
+print(f"Sentences separated in {time.time() - start}s")
+
+for sentence in sentences:
+    print(sentence)
+print("\n")
+
+# SINGLE WORD MORPHOLOGICAL ANALYSIS
+results = morphology.analyze("kalemin")
+for result in results:
+    print(result)
+print("\n")
+
+# SENTENCE ANALYSIS AND DISAMBIGUATION
+
+sentence = "Yarın kar yağacak."
+analysis = morphology.analyze_sentence(sentence)
+after = morphology.disambiguate(sentence, analysis)
+
+print("\nBefore disambiguation")
+for e in analysis:
+    print(f"Word = {e.inp}")
+    for s in e:
+        print(s.format_string())
+
+print("\nAfter disambiguation")
+for s in after.best_analysis():
+    print(s.format_string())
+
+# TOKENIZATION
+tokenizer = TurkishTokenizer.DEFAULT
+
+tokens = tokenizer.tokenize("Saat 12:00.")
+for token in tokens:
+    print('Content = ', token.content)
+    print('Type = ', token.type_.name)
+    print('Start = ', token.start)
+    print('Stop = ', token.end, '\n')