tokmor 1.2.9__py3-none-any.whl

tokmor/rtl.py

@@ -0,0 +1,309 @@
+"""
+RTL Tokenizer
+=============
+
+오른쪽에서 왼쪽으로 쓰는 언어용 토크나이저
+아랍어, 히브리어, 페르시아어, 우르두어
+"""
+
+import re
+import unicodedata
+from typing import List
+from .base import BaseTokenizer, Token, TokenizerResult, MorphologicalAnalyzer
+
+
+class RTLTokenizer(BaseTokenizer):
+    """
+    RTL 언어 토크나이저
+
+    아랍어, 히브리어, 페르시아어, 우르두어 등
+    """
+
+    SUPPORTED_LANGUAGES = {'ar', 'he', 'fa', 'ur', 'yi', 'ps'}
+
+    # Unicode ranges
+    ARABIC = '\u0600-\u06ff'
+    ARABIC_SUPPLEMENT = '\u0750-\u077f'
+    ARABIC_EXT_A = '\u08a0-\u08ff'
+    ARABIC_PRESENTATION_A = '\ufb50-\ufdff'
+    ARABIC_PRESENTATION_B = '\ufe70-\ufeff'
+    HEBREW = '\u0590-\u05ff'
+
+    def __init__(self, lang: str, use_morphology: bool = False):
+        super().__init__(lang, use_morphology)
+        self._setup_patterns()
+
+    def _setup_patterns(self):
+        """언어별 패턴 설정"""
+        if self.lang in ('he', 'yi'):  # Hebrew and Yiddish use Hebrew script
+            self._script_pattern = re.compile(f'[{self.HEBREW}]+')
+        else:  # ar, fa, ur, ps
+            self._script_pattern = re.compile(
+                f'[{self.ARABIC}{self.ARABIC_SUPPLEMENT}'
+                f'{self.ARABIC_EXT_A}{self.ARABIC_PRESENTATION_A}'
+                f'{self.ARABIC_PRESENTATION_B}]+'
+            )
+
+        self._latin_pattern = re.compile(r'[a-zA-Z0-9]+')
+
+    def _init_morphology(self):
+        """형태소 분석기 초기화"""
+        if self.lang == 'ar':
+            self._morphology_analyzer = ArabicMorphologyAnalyzer()
+        elif self.lang == 'he':
+            self._morphology_analyzer = HebrewMorphologyAnalyzer()
+
+    def tokenize(self, text: str) -> TokenizerResult:
+        """RTL 토크나이징"""
+        text = self.clean_text(text)
+        if not text:
+            return TokenizerResult(tokens=[], text=text, lang=self.lang)
+
+        # 형태소 분석 사용 시
+        if self.use_morphology and self._morphology_analyzer:
+            if self._morphology_analyzer.is_available():
+                tokens = self._morphology_analyzer.analyze(text)
+                return TokenizerResult(
+                    tokens=tokens,
+                    text=text,
+                    lang=self.lang,
+                    morphology_used=True
+                )
+
+        tokens: List[Token] = []
+
+        # RTL 스크립트 토큰
+        for match in self._script_pattern.finditer(text):
+            tokens.append(Token(
+                text=match.group(),
+                start=match.start(),
+                end=match.end(),
+            ))
+
+        # 라틴/숫자
+        for match in self._latin_pattern.finditer(text):
+            overlaps = any(
+                t.start <= match.start() < t.end or t.start < match.end() <= t.end
+                for t in tokens
+            )
+            if not overlaps:
+                tokens.append(Token(
+                    text=match.group(),
+                    start=match.start(),
+                    end=match.end(),
+                ))
+
+        tokens.sort(key=lambda t: t.start)
+
+        # Preserve any remaining non-space spans (emoji, punctuation, symbols, etc.).
+        # Previously, RTL tokenization dropped these completely, which is harmful for SNS and general preprocessing.
+        if tokens:
+            out2: List[Token] = []
+            i = 0
+            j = 0
+            n = len(text)
+            toks = tokens
+            while i < n:
+                ch = text[i]
+                if ch.isspace():
+                    i += 1
+                    continue
+                # Advance token pointer to the first token that could overlap/appear after i
+                while j < len(toks) and toks[j].end <= i:
+                    j += 1
+                if j < len(toks) and toks[j].start <= i < toks[j].end:
+                    # Inside an existing token
+                    out2.append(toks[j])
+                    i = toks[j].end
+                    continue
+                # Uncovered non-space segment until whitespace or next token start
+                next_start = toks[j].start if j < len(toks) else n
+                k = i
+                while k < n and (not text[k].isspace()) and k < next_start:
+                    k += 1
+                if k > i:
+                    out2.append(Token(text=text[i:k], start=i, end=k))
+                i = k
+            # De-duplicate by (start,end,text) and sort
+            seen = set()
+            dedup: List[Token] = []
+            for t in out2:
+                key = (t.start, t.end, t.text)
+                if key in seen:
+                    continue
+                seen.add(key)
+                dedup.append(t)
+            tokens = sorted(dedup, key=lambda t: t.start)
+
+        # Drop standalone combining marks (diacritics) that appear as separate tokens in noisy corpora.
+        # Example: "سويا ً" where "ً" is a combining mark separated by whitespace.
+        def _is_mark_only(s: str) -> bool:
+            return bool(s) and all(unicodedata.category(ch) in {"Mn", "Mc", "Me"} for ch in s)
+
+        # Also strip *leading* combining marks that sometimes appear due to corpus spacing noise:
+        # e.g., "دون َوقوع" -> token "َوقوع" should become "وقوع".
+        cleaned: List[Token] = []
+        for t in tokens:
+            s = t.text
+            i = 0
+            while i < len(s) and unicodedata.category(s[i]) in {"Mn", "Mc", "Me"}:
+                i += 1
+            if i > 0:
+                s2 = s[i:]
+                if not s2:
+                    continue
+                cleaned.append(Token(text=s2, start=t.start + i, end=t.end, lemma=t.lemma, pos=t.pos, features=t.features))
+            else:
+                cleaned.append(t)
+
+        tokens = [t for t in cleaned if not _is_mark_only(t.text)]
+
+        # Safety: never return empty tokens for non-empty input
+        if not tokens:
+            for m in re.finditer(r"\S+", text):
+                tokens.append(Token(text=m.group(), start=m.start(), end=m.end()))
+
+        return TokenizerResult(
+            tokens=tokens,
+            text=text,
+            lang=self.lang,
+            morphology_used=False
+        )
+
+
+class ArabicMorphologyAnalyzer(MorphologicalAnalyzer):
+    """
+    아랍어 형태소 분석기
+
+    Backends:
+    - camel_tools
+    - pyarabic
+    """
+
+    def __init__(self, backend: str = 'auto'):
+        self.backend = backend
+        self._analyzer = None
+        self._backend_name = None
+        self._init_analyzer()
+
+    def _init_analyzer(self):
+        """분석기 초기화"""
+        # camel_tools 시도
+        if self.backend in ('auto', 'camel'):
+            try:
+                from camel_tools.morphology.analyzer import Analyzer
+                from camel_tools.morphology.database import MorphologyDB
+                db = MorphologyDB.builtin_db()
+                self._analyzer = Analyzer(db)
+                self._backend_name = 'camel'
+                return
+            except (ImportError, Exception):
+                pass
+
+        # pyarabic 시도
+        if self.backend in ('auto', 'pyarabic'):
+            try:
+                import pyarabic.araby as araby
+                self._analyzer = araby
+                self._backend_name = 'pyarabic'
+                return
+            except ImportError:
+                pass
+
+    def is_available(self) -> bool:
+        return self._analyzer is not None
+
+    def analyze(self, text: str) -> List[Token]:
+        """형태소 분석"""
+        if not self._analyzer:
+            return []
+
+        tokens = []
+
+        if self._backend_name == 'pyarabic':
+            # pyarabic는 단어 분리만 지원
+            words = self._analyzer.tokenize(text)
+            offset = 0
+            for word in words:
+                idx = text.find(word, offset)
+                if idx >= 0:
+                    tokens.append(Token(
+                        text=word,
+                        start=idx,
+                        end=idx + len(word),
+                        lemma=word,
+                    ))
+                    offset = idx + len(word)
+
+        return tokens
+
+
+class HebrewMorphologyAnalyzer(MorphologicalAnalyzer):
+    """
+    히브리어 형태소 분석기
+
+    Backends:
+    - hebrew_tokenizer
+    """
+
+    def __init__(self, backend: str = 'auto'):
+        self.backend = backend
+        self._analyzer = None
+        self._backend_name = None
+        self._init_analyzer()
+
+    def _init_analyzer(self):
+        """분석기 초기화"""
+        try:
+            from hebrew_tokenizer import tokenize as heb_tokenize
+            self._analyzer = heb_tokenize
+            self._backend_name = 'hebrew_tokenizer'
+        except ImportError:
+            pass
+
+    def is_available(self) -> bool:
+        return self._analyzer is not None
+
+    def analyze(self, text: str) -> List[Token]:
+        """형태소 분석"""
+        if not self._analyzer:
+            return []
+
+        tokens = []
+        try:
+            for token_type, token_text, _, start, end in self._analyzer(text):
+                tokens.append(Token(
+                    text=token_text,
+                    start=start,
+                    end=end,
+                    pos=token_type,
+                ))
+        except Exception:
+            pass
+
+        return tokens
+
+
+# 언어별 특화 클래스
+class ArabicTokenizer(RTLTokenizer):
+    """아랍어 특화 토크나이저"""
+    SUPPORTED_LANGUAGES = {'ar'}
+
+    def __init__(self, use_morphology: bool = False):
+        super().__init__('ar', use_morphology)
+
+
+class HebrewTokenizer(RTLTokenizer):
+    """히브리어 특화 토크나이저"""
+    SUPPORTED_LANGUAGES = {'he'}
+
+    def __init__(self, use_morphology: bool = False):
+        super().__init__('he', use_morphology)
+
+
+class PersianTokenizer(RTLTokenizer):
+    """페르시아어 특화 토크나이저"""
+    SUPPORTED_LANGUAGES = {'fa'}
+
+    def __init__(self, use_morphology: bool = False):
+        super().__init__('fa', use_morphology)

tokmor/schema.py

@@ -0,0 +1,17 @@
+"""
+TokMor output schema versioning
+==============================
+
+We keep a small, explicit schema_version to avoid breaking downstream clients
+when adding/changing fields in JSON outputs.
+"""
+
+from __future__ import annotations
+
+# Increment only on breaking output changes (field removal/rename/type change).
+SCHEMA_VERSION: int = 1
+
+
+
+
+

tokmor/sns_tags.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Optional
+
+
+_RX_LETTERS: Dict[str, re.Pattern] = {
+    "latin": re.compile(r"[A-Za-z]"),
+    "cyrillic": re.compile(r"[\u0400-\u04FF]"),
+    "arabic": re.compile(r"[\u0600-\u06FF]"),
+    "hebrew": re.compile(r"[\u0590-\u05FF]"),
+    "devanagari": re.compile(r"[\u0900-\u097F]"),
+    "thai": re.compile(r"[\u0E00-\u0E7F]"),
+}
+
+_VOWELS: Dict[str, set[str]] = {
+    # Keep minimal & conservative; this is *not* linguistic correctness, just a keysmash heuristic.
+    "latin": set("aeiouy"),
+    "cyrillic": set("аеёиоуыэюя"),
+    # Arabic/Hebrew: abjads; skip vowel heuristics (too many false positives).
+    "devanagari": set("अआइईउऊएऐओऔऋॠ"),
+    "thai": set("ะาิีึืุูเแโใไำ"),
+}
+
+
+def _script_of_token(t: str) -> Optional[str]:
+    if not t:
+        return None
+    best = None
+    best_cnt = 0
+    for name, rx in _RX_LETTERS.items():
+        cnt = len(rx.findall(t))
+        if cnt > best_cnt:
+            best = name
+            best_cnt = cnt
+    if not best or best_cnt <= 0:
+        return None
+    # Require majority; otherwise treat as mixed and don't attempt keysmash detection.
+    if best_cnt / max(1, len(t)) < 0.7:
+        return None
+    return best
+
+
+def _looks_like_keysmash_generic(token: str) -> bool:
+    """
+    Conservative heuristic for "keyboard smash / garble" tokens across major scripts.
+
+    Goal: produce a neutral DISCOURSE_MARKER/OTHER hint, NOT language understanding.
+    """
+    t = token or ""
+    if len(t) < 8 or len(t) > 40:
+        return False
+    if not t.isalpha():
+        return False
+
+    script = _script_of_token(t)
+    if not script:
+        return False
+
+    # Random-looking: high unique-char ratio (e.g., asdfghjkl, фывапролдж).
+    uniq = len(set(t.lower()))
+    uniq_ratio = uniq / len(t)
+    if uniq_ratio < 0.6:
+        return False
+
+    # Avoid tagging real-ish words: keep script-specific conservatism.
+    vowels = _VOWELS.get(script)
+    # Cyrillic keysmash commonly contains vowels; don't use vowel ratio there.
+    if script == "cyrillic":
+        vowels = None
+    if script == "latin":
+        # Latin false positives are costly (real words). Be *very* strict:
+        # Only treat as keysmash when vowel count is essentially zero.
+        v = sum(1 for ch in t.lower() if ch in _VOWELS["latin"])
+        if v != 0:
+            return False
+        if len(t) < 8 or len(t) > 20:
+            return False
+    elif vowels:
+        v = sum(1 for ch in t.lower() if ch in vowels)
+        # Latin/Devanagari/Thai: keysmash tends to be vowel-poor.
+        if v / len(t) > 0.28:
+            return False
+    else:
+        # For abjads, we don't have a safe vowel heuristic; keep stricter thresholds.
+        # Allow slightly shorter tokens, but require higher randomness.
+        if len(t) < 9:
+            return False
+        if uniq_ratio < 0.75:
+            return False
+
+    # Low bigram repetition -> more "random typing" than a real word.
+    bigrams = [t[i : i + 2].lower() for i in range(len(t) - 1)]
+    bg_ratio = len(set(bigrams)) / max(1, len(bigrams))
+    if bg_ratio < 0.78:
+        return False
+
+    # Extra safety tightening for scripts where false positives are riskier.
+    if script in {"cyrillic"}:
+        # Cyrillic "keysmash" often includes vowels; rely more on randomness + length.
+        if len(t) < 9 or uniq_ratio < 0.7 or bg_ratio < 0.82:
+            return False
+    if script in {"arabic", "hebrew"}:
+        # Abjads: allow slightly shorter, but only when it's extremely "random looking".
+        if len(t) < 8:
+            return False
+        if uniq_ratio < 0.85 or bg_ratio < 0.9:
+            return False
+
+    return True
+
+
+def classify_sns_token(token: str, *, lang: str) -> Optional[Dict[str, Any]]:
+    """
+    Classify SNS discourse markers (NOT POS tagging).
+
+    Output is a small, deterministic hint object:
+      {"class": "DISCOURSE_MARKER", "subtype": "...", "intensity": int}
+
+    This is intentionally minimal and language-agnostic where possible.
+    """
+    if not token:
+        return None
+
+    t = token
+    ll = (lang or "").lower().replace("_", "-")
+    tl = t.lower()
+
+    # Punctuation/ellipsis-only runs (SNS intensity / stance markers)
+    # Keep conservative: require 2+ chars and only specific punctuation.
+    if len(t) >= 2:
+        if re.fullmatch(r"[!！]{2,}", t):
+            return {"class": "DISCOURSE_MARKER", "subtype": "EMPHASIS", "intensity": len(t)}
+        if re.fullmatch(r"[?？]{2,}", t):
+            return {"class": "DISCOURSE_MARKER", "subtype": "SURPRISE", "intensity": len(t)}
+        if re.fullmatch(r"[!?！？]{2,}", t) and ("!" in t or "！" in t) and ("?" in t or "？" in t):
+            return {"class": "DISCOURSE_MARKER", "subtype": "SURPRISE", "intensity": len(t)}
+        if re.fullmatch(r"(?:\.{3,}|…{2,})", t):
+            return {"class": "DISCOURSE_MARKER", "subtype": "HESITATION", "intensity": len(t)}
+        if re.fullmatch(r"[~～]{2,}", t):
+            return {"class": "DISCOURSE_MARKER", "subtype": "SOFTENING", "intensity": len(t)}
+
+    # Emoji sadness / laughter (very common)
+    if any(ch in t for ch in ("😂", "🤣")):
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": 1}
+    if any(ch in t for ch in ("😭", "😢", "🥲")):
+        return {"class": "DISCOURSE_MARKER", "subtype": "SADNESS", "intensity": 1}
+    if any(ch in t for ch in ("😡", "🤬")):
+        return {"class": "DISCOURSE_MARKER", "subtype": "ANGER", "intensity": 1}
+    if any(ch in t for ch in ("❤️", "❤", "💕", "💖", "😍")):
+        return {"class": "DISCOURSE_MARKER", "subtype": "AFFECTION", "intensity": 1}
+
+    # Global keysmash / garble (conservative). Must come early so downstream can ignore it.
+    if _looks_like_keysmash_generic(t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "OTHER", "intensity": len(t)}
+
+    # Hangul/Jamo-based markers (often appear in mixed-language SNS too, so allow globally)
+    if re.fullmatch(r"[ㅋㅎ]{2,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+    # Korean laughter syllable repetition (conservative)
+    if re.fullmatch(r"(?:하){2,}", t) or re.fullmatch(r"(?:헤){2,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t) // 1}
+    if re.fullmatch(r"[ㅠㅜ]{2,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "SADNESS", "intensity": len(t)}
+    if re.fullmatch(r"ㄷ{2,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "SURPRISE", "intensity": len(t)}
+    if re.fullmatch(r"ㅇ{2,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "AFFIRM", "intensity": len(t)}
+    # Hangul Jamo "keysmash"/garbling (e.g., ㅣ마ㅓㅣ넣ㄹ아이고) – treat as discourse noise.
+    # This is NOT spell correction; it's a neutral hint to help downstream ignore unusable tokens.
+    if len(t) >= 4:
+        jamo = re.findall(r"[\u3131-\u3163]", t)  # ㄱ-ㅎ,ㅏ-ㅣ
+        if len(jamo) >= 3:
+            # require presence of vowel jamo to avoid tagging pure consonant runs already handled above
+            has_vowel = any("\u314f" <= ch <= "\u3163" for ch in jamo)  # ㅏ..ㅣ
+            if has_vowel:
+                ratio = len(jamo) / max(1, len(t))
+                if ratio >= 0.3:
+                    return {"class": "DISCOURSE_MARKER", "subtype": "OTHER", "intensity": len(t)}
+    if ll == "ko":
+        if tl in {"ㄹㅇ"}:
+            return {"class": "DISCOURSE_MARKER", "subtype": "EMPHASIS", "intensity": 1}
+        if tl in {"ㅇㅋ", "오케이"}:
+            return {"class": "DISCOURSE_MARKER", "subtype": "AFFIRM", "intensity": 1}
+        # common swear abbreviations (keep minimal)
+        if tl in {"ㅅㅂ", "ㅆㅂ", "ㅈㄴ"}:
+            return {"class": "DISCOURSE_MARKER", "subtype": "SWEAR", "intensity": 1}
+
+    # English-ish / global roman markers
+    if tl in {"lol", "lmao", "rofl"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": 1}
+    if tl in {"haha", "hahaha", "hehe", "hehehe"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+    if tl in {"xd", "x-d", "x_d"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": 1}
+    if tl in {"wtf", "omg"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "SWEAR" if tl == "wtf" else "SURPRISE", "intensity": 1}
+    if tl in {"ok", "okay", "k", "kk", "yes", "yep", "yeah"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "AFFIRM", "intensity": 1}
+    if tl in {"nope", "nah"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "NEGATION", "intensity": 1}
+
+    # Simple ASCII emoticons (conservative)
+    if tl in {":)", ":-)", ":d", ":-d", ";)", ";-)", ":(", ":-(", ":'(", ":'-(", "t_t", ";_;"}:
+        subtype = "LAUGHTER" if "d" in tl else "SADNESS" if "(" in tl or "_" in tl or ";" in tl else "OTHER"
+        if tl in {";)", ";-)"}:
+            subtype = "SOFTENING"
+        return {"class": "DISCOURSE_MARKER", "subtype": subtype, "intensity": 1}
+
+    # Chinese/Japanese common laughter markers
+    if tl in {"www"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+    if ll.startswith("ja"):
+        # Katakana laughter (ハハハ...)
+        if re.fullmatch(r"ハ{2,}", t):
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+    if ll.startswith("zh"):
+        # Very conservative: only pure repetition tokens commonly used as laughter in Chinese.
+        # (Avoid semantic words; do not attempt to classify content tokens.)
+        if re.fullmatch(r"[哈呵嘿嘻]{2,}", t):
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+    if any(ch in t for ch in ("笑",)):
+        # very conservative: only when token is exactly "(笑)" or "笑"
+        if t in {"笑", "（笑）", "(笑)"}:
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": 1}
+
+    # Chinese numeric slang (conservative)
+    if tl in {"666", "2333"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "PRAISE" if tl == "666" else "LAUGHTER", "intensity": len(t)}
+    if tl in {"233", "23333"}:
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+
+    # Thai: "55555" (ha-ha-ha) is extremely common SNS laughter
+    if ll == "th" and re.fullmatch(r"5{3,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+
+    # Arabic-script laughter: ههههه / هاهاها (conservative)
+    if re.fullmatch(r"[ه]{2,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+    # Arabic/Persian internet laughter: خخخ...
+    if re.fullmatch(r"[خ]{2,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t)}
+    if re.fullmatch(r"(?:ها){2,}", t):
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(t) // 2}
+    # Arabic-script "lol" transliteration: لول
+    if t == "لول":
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": 1}
+
+    # Cyrillic laughter (conservative): ха+ / ахаха
+    tl_cyr = tl
+    if re.fullmatch(r"[а-яё]+", tl_cyr) and (re.fullmatch(r"(?:ха){2,}", tl_cyr) or re.fullmatch(r"(?:ах){2,}а?", tl_cyr)):
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": max(1, len(tl_cyr) // 2)}
+    # Cyrillic "lol" transliteration: лол
+    if tl_cyr == "лол":
+        return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": 1}
+
+    # Latin-script regional laughter (conservative)
+    if re.fullmatch(r"[a-z]+", tl):
+        if re.fullmatch(r"(?:ha){2,}h?", tl) or re.fullmatch(r"(?:he){2,}e?", tl):
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": max(1, len(tl) // 2)}
+        if re.fullmatch(r"(?:ja){2,}a?", tl) or re.fullmatch(r"(?:je){2,}e?", tl):
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": max(1, len(tl) // 2)}
+        if re.fullmatch(r"k{4,}", tl):  # pt-BR "kkkkk"
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": len(tl)}
+        if re.fullmatch(r"(?:wk){2,}", tl) or re.fullmatch(r"(?:wkwk){1,}", tl):  # id "wkwk"
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": max(1, len(tl) // 2)}
+        if tl in {"mdr", "ptdr"}:  # fr
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": 1}
+        if tl in {"rsrs"}:  # pt-BR
+            return {"class": "DISCOURSE_MARKER", "subtype": "LAUGHTER", "intensity": 1}
+
+        # Elongated latin words (e.g., "noooo", "soooo") are usually emphasis/stance markers.
+        # Keep conservative: require >=4 chars and a 3+ repetition of the same letter.
+        # NOTE: must come AFTER laughter patterns (e.g., "kkkkk" is laughter in pt-BR).
+        if len(tl) >= 4 and re.search(r"([a-z])\1{2,}", tl):
+            # Avoid mis-tagging common real words like "coffee" (double letters only).
+            return {"class": "DISCOURSE_MARKER", "subtype": "EMPHASIS", "intensity": 1}
+
+    return None
+
+