tokmor 1.2.9__py3-none-any.whl

tokmor/cjk.py

@@ -0,0 +1,497 @@
+"""
+CJK Tokenizer
+=============
+
+중국어, 일본어, 한국어 토크나이저
+외부 라이브러리 없이 자체 구현만 사용
+"""
+
+import re
+import unicodedata
+from typing import List, Optional
+from .base import BaseTokenizer, Token, TokenizerResult, MorphologicalAnalyzer
+
+
+class CJKTokenizer(BaseTokenizer):
+    """
+    CJK 기본 토크나이저
+
+    한자/한글/가나 연속 청크 기반
+    자체 형태소 분석기 사용 (외부 라이브러리 불필요)
+    """
+
+    SUPPORTED_LANGUAGES = {'zh', 'ja', 'ko'}
+
+    # Unicode ranges
+    CJK_UNIFIED = '\u4e00-\u9fff'      # CJK Unified Ideographs
+    CJK_EXT_A = '\u3400-\u4dbf'        # CJK Extension A
+    CJK_EXT_B = '\U00020000-\U0002a6df'  # CJK Extension B
+    HIRAGANA = '\u3040-\u309f'
+    KATAKANA = '\u30a0-\u30ff'
+    HANGUL_SYLLABLES = '\uac00-\ud7af'
+    HANGUL_JAMO = '\u1100-\u11ff'
+    HANGUL_COMPAT = '\u3130-\u318f'
+
+    def __init__(self, lang: str, use_morphology: bool = True, *, zh_join_dates: Optional[bool] = None):
+        """
+        Args:
+            lang: 언어 코드 (ko, ja, zh)
+            use_morphology: 형태소 분석 사용 (기본 True)
+        """
+        # IMPORTANT: BaseTokenizer.__init__ may call _init_morphology() which uses _zh_join_dates.
+        # So set this BEFORE calling super().__init__().
+        self._zh_join_dates: Optional[bool] = zh_join_dates if lang == "zh" else None
+        super().__init__(lang, use_morphology)
+        self._setup_patterns()
+        self._native_analyzer = None
+        self._init_native_analyzer()
+
+    def _setup_patterns(self):
+        """언어별 패턴 설정"""
+        if self.lang == 'ko':
+            # 한국어: 한글 + 한자
+            self._script_pattern = re.compile(
+                f'[{self.HANGUL_SYLLABLES}{self.HANGUL_JAMO}{self.HANGUL_COMPAT}'
+                f'{self.CJK_UNIFIED}{self.CJK_EXT_A}]+'
+            )
+        elif self.lang == 'ja':
+            # 일본어: 한자 + 히라가나 + 가타카나
+            self._script_pattern = re.compile(
+                f'[{self.CJK_UNIFIED}{self.CJK_EXT_A}'
+                f'{self.HIRAGANA}{self.KATAKANA}]+'
+            )
+        else:  # zh
+            # 중국어: 한자
+            self._script_pattern = re.compile(
+                f'[{self.CJK_UNIFIED}{self.CJK_EXT_A}]+'
+            )
+
+        # 라틴/숫자 패턴
+        self._latin_pattern = re.compile(r'[a-zA-Z0-9]+')
+
+    def _init_native_analyzer(self):
+        """자체 형태소 분석기 초기화"""
+        try:
+            if self.lang == 'ko':
+                from .morphology.korean import KoreanAnalyzer
+                self._native_analyzer = KoreanAnalyzer()
+            elif self.lang == 'ja':
+                from .morphology.japanese import JapaneseAnalyzer
+                self._native_analyzer = JapaneseAnalyzer()
+            elif self.lang == 'zh':
+                from .morphology.chinese import ChineseAnalyzer
+                self._native_analyzer = ChineseAnalyzer(join_dates=self._zh_join_dates)
+        except ImportError:
+            self._native_analyzer = None
+
+    def _init_morphology(self):
+        """형태소 분석기 초기화"""
+        if self.lang == 'ko':
+            self._morphology_analyzer = KoreanMorphologyAnalyzer()
+        elif self.lang == 'ja':
+            self._morphology_analyzer = JapaneseMorphologyAnalyzer()
+        elif self.lang == 'zh':
+            self._morphology_analyzer = ChineseMorphologyAnalyzer(join_dates=self._zh_join_dates)
+
+    def tokenize(self, text: str) -> TokenizerResult:
+        """CJK 토크나이징"""
+        text = self.clean_text(text)
+        if not text:
+            return TokenizerResult(tokens=[], text=text, lang=self.lang)
+
+        # 1. 자체 형태소 분석기
+        # NOTE:
+        # - For Korean, native analyzer tends to produce morpheme-level splits.
+        #   When use_morphology=False we should keep surface tokenization (eojeol-like).
+        # - For Japanese/Chinese, native analyzer is effectively the word segmenter,
+        #   so we keep using it regardless.
+        # Only use the native analyzer when the text contains the target script.
+        # This avoids pathological per-character output on mislabeled corpora (e.g., Arabic lines in ja/zh files).
+        has_target = bool(self._script_pattern.search(text))
+        use_native = bool(self._native_analyzer) and has_target and (self.lang != "ko" or self.use_morphology)
+        if use_native:
+            try:
+                morphemes = self._native_analyzer.analyze(text)
+                tokens = []
+                for m in morphemes:
+                    tokens.append(Token(
+                        text=m.surface,
+                        start=m.start,
+                        end=m.end,
+                        lemma=m.lemma,
+                        pos=m.pos,
+                    ))
+                tokens.sort(key=lambda t: t.start)
+                tokens = self._postprocess_marks_and_numbers(tokens)
+                # Apply token-quality rules early for CJK (defensive).
+                # These same rules are also applied globally in TokenizerResult.__post_init__,
+                # but doing it here avoids edge cases where upstream CJK segmentation
+                # emits an over-merged chunk that should be split before returning.
+                try:
+                    from .token_quality import apply_token_quality
+                    tokens = apply_token_quality(tokens, lang=self.lang, text=text)  # type: ignore[assignment]
+                except Exception:
+                    pass
+                return TokenizerResult(
+                    tokens=tokens,
+                    text=text,
+                    lang=self.lang,
+                    morphology_used=self.use_morphology
+                )
+            except Exception:
+                pass  # fallback to chunk-based
+
+        # 2. 기본: 스크립트 청크 기반 (fallback)
+        tokens = []
+
+        # CJK 청크
+        for match in self._script_pattern.finditer(text):
+            tokens.append(Token(
+                text=match.group(),
+                start=match.start(),
+                end=match.end(),
+            ))
+
+        # 라틴/숫자
+        for match in self._latin_pattern.finditer(text):
+            # 이미 추출된 범위와 겹치지 않는지 확인
+            overlaps = any(
+                t.start <= match.start() < t.end or t.start < match.end() <= t.end
+                for t in tokens
+            )
+            if not overlaps:
+                tokens.append(Token(
+                    text=match.group(),
+                    start=match.start(),
+                    end=match.end(),
+                ))
+
+        # 위치순 정렬
+        tokens.sort(key=lambda t: t.start)
+
+        tokens = self._postprocess_marks_and_numbers(tokens)
+        try:
+            from .token_quality import apply_token_quality
+            tokens = apply_token_quality(tokens, lang=self.lang, text=text)  # type: ignore[assignment]
+        except Exception:
+            pass
+
+        # Safety: never return empty tokens for non-empty input.
+        # This can happen when the input contains neither CJK script chunks nor latin/digits
+        # (e.g., mislabeled corpus lines). Fall back to whitespace tokens.
+        if not tokens:
+            for m in re.finditer(r"\S+", text):
+                tokens.append(Token(text=m.group(), start=m.start(), end=m.end()))
+
+        return TokenizerResult(
+            tokens=tokens,
+            text=text,
+            lang=self.lang,
+            morphology_used=False
+        )
+
+    def _postprocess_marks_and_numbers(self, tokens: List[Token]) -> List[Token]:
+        """
+        Postprocess for robustness / quality:
+        - Merge standalone combining marks into previous token when contiguous (e.g., variation selectors in JA).
+        - Merge contiguous digit runs (ASCII/fullwidth/etc) into a single token (esp. JA dates/numbers).
+        - Merge contiguous ASCII alnum runs (e.g., "F" "I" "R" "E" -> "FIRE") when contiguous.
+        """
+        if not tokens:
+            return tokens
+
+        def _is_mark_only(s: str) -> bool:
+            return bool(s) and all(unicodedata.category(ch) in {"Mn", "Mc", "Me"} for ch in s)
+
+        def _starts_with_mark(s: str) -> bool:
+            return bool(s) and unicodedata.category(s[0]) in {"Mn", "Mc", "Me"}
+
+        def _is_all_digits(s: str) -> bool:
+            return bool(s) and all(ch.isdigit() for ch in s)
+
+        def _is_ascii_alnum(s: str) -> bool:
+            return bool(s) and s.isascii() and all(ch.isalnum() for ch in s)
+
+        # pass 1: simple adjacent merges (marks/digits/ascii)
+        out: List[Token] = []
+        for t in tokens:
+            if not out:
+                out.append(t)
+                continue
+            prev = out[-1]
+
+            # merge contiguous combining marks (variation selectors, dakuten marks, etc.)
+            if (t.start == prev.end) and (t.text and (_is_mark_only(t.text) or _starts_with_mark(t.text))):
+                prev.text += t.text
+                prev.end = t.end
+                continue
+
+            # merge contiguous digit runs (including full-width digits)
+            if (t.start == prev.end) and _is_all_digits(prev.text) and _is_all_digits(t.text):
+                prev.text += t.text
+                prev.end = t.end
+                continue
+
+            # merge contiguous ASCII alnum runs (common in JA/KO/ZH corpora)
+            if (t.start == prev.end) and _is_ascii_alnum(prev.text) and _is_ascii_alnum(t.text):
+                prev.text += t.text
+                prev.end = t.end
+                continue
+
+            out.append(t)
+
+        # pass 2 (JA): merge common numeric patterns to avoid token explosions in statistics/news lines
+        if self.lang == "ja" and len(out) >= 3:
+            merged: List[Token] = []
+            i = 0
+            JA_NUM_UNITS = {"年", "月", "日", "代", "人", "件", "話", "歳"}
+            DEC_SEPS = {".", "．", "・"}
+            PERCENTS = {"%", "％"}
+            while i < len(out):
+                t = out[i]
+                # digits + unit (e.g., 20 + 代 -> 20代)
+                if i + 1 < len(out) and (out[i].end == out[i + 1].start) and _is_all_digits(out[i].text) and (out[i + 1].text in JA_NUM_UNITS):
+                    merged.append(Token(text=out[i].text + out[i + 1].text, start=out[i].start, end=out[i + 1].end))
+                    i += 2
+                    continue
+                # decimal: digits + sep + digits (+ percent)
+                if i + 2 < len(out):
+                    a, b, c = out[i], out[i + 1], out[i + 2]
+                    if (a.end == b.start) and (b.end == c.start) and _is_all_digits(a.text) and (b.text in DEC_SEPS) and _is_all_digits(c.text):
+                        txt = a.text + b.text + c.text
+                        end = c.end
+                        j = i + 3
+                        if j < len(out) and (out[j].start == end) and (out[j].text in PERCENTS):
+                            txt += out[j].text
+                            end = out[j].end
+                            j += 1
+                        merged.append(Token(text=txt, start=a.start, end=end))
+                        i = j
+                        continue
+                # digits + percent (e.g., 10 + ％ -> 10％)
+                if i + 1 < len(out) and (out[i].end == out[i + 1].start) and _is_all_digits(out[i].text) and (out[i + 1].text in PERCENTS):
+                    merged.append(Token(text=out[i].text + out[i + 1].text, start=out[i].start, end=out[i + 1].end))
+                    i += 2
+                    continue
+                merged.append(t)
+                i += 1
+            out = merged
+
+        # pass 3 (ZH): merge contiguous ASCII digit token + common CJK unit char
+        # e.g., 10 + 亿 -> 10亿, 2025 + 年 -> 2025年
+        if self.lang == "zh" and len(out) >= 2:
+            merged2: List[Token] = []
+            i = 0
+            ZH_NUM_UNITS = {"年", "月", "日", "号", "亿", "万", "元", "％", "%", "度", "岁"}
+            while i < len(out):
+                t = out[i]
+                if i + 1 < len(out):
+                    a, b = out[i], out[i + 1]
+                    if (a.end == b.start) and _is_all_digits(a.text) and (b.text in ZH_NUM_UNITS):
+                        merged2.append(Token(text=a.text + b.text, start=a.start, end=b.end))
+                        i += 2
+                        continue
+                    # If the next token starts with a unit char (e.g., '亿人'), split it:
+                    # 10 + 亿人 -> 10亿 + 人
+                    if (a.end == b.start) and _is_all_digits(a.text) and b.text and (b.text[0] in ZH_NUM_UNITS) and len(b.text) > 1:
+                        unit = b.text[0]
+                        rest = b.text[1:]
+                        merged2.append(Token(text=a.text + unit, start=a.start, end=b.start + 1))
+                        merged2.append(Token(text=rest, start=b.start + 1, end=b.end))
+                        i += 2
+                        continue
+                merged2.append(t)
+                i += 1
+            out = merged2
+
+        return out
+
+    def extract_ngrams(self, text: str, min_n: int = 2, max_n: int = 8) -> List[str]:
+        """N-gram 추출"""
+        ngrams = []
+
+        for match in self._script_pattern.finditer(text):
+            chunk = match.group()
+            for n in range(min_n, min(max_n + 1, len(chunk) + 1)):
+                for i in range(len(chunk) - n + 1):
+                    ngrams.append(chunk[i:i+n])
+
+        return ngrams
+
+
+# =============================================================================
+# 형태소 분석기 (자체 구현만 사용)
+# =============================================================================
+
+class KoreanMorphologyAnalyzer(MorphologicalAnalyzer):
+    """한국어 형태소 분석기 (자체 구현)"""
+
+    def __init__(self):
+        self._analyzer = None
+        self._init_analyzer()
+
+    def _init_analyzer(self):
+        """분석기 초기화"""
+        try:
+            from .morphology.korean import KoreanAnalyzer
+            self._analyzer = KoreanAnalyzer()
+        except ImportError:
+            self._analyzer = None
+
+    def is_available(self) -> bool:
+        return self._analyzer is not None
+
+    def analyze(self, text: str) -> List[Token]:
+        """형태소 분석"""
+        if not self._analyzer:
+            return []
+
+        tokens = []
+        result = self._analyzer.analyze(text)
+        for morph in result:
+            tokens.append(Token(
+                text=morph.surface,
+                start=morph.start,
+                end=morph.end,
+                lemma=morph.lemma,
+                pos=morph.pos,
+            ))
+        return tokens
+
+    def nouns(self, text: str) -> List[str]:
+        """명사 추출"""
+        if not self._analyzer:
+            return []
+        result = self._analyzer.analyze(text)
+        return [m.surface for m in result if m.pos.startswith('N')]
+
+
+class JapaneseMorphologyAnalyzer(MorphologicalAnalyzer):
+    """일본어 형태소 분석기 (자체 구현)"""
+
+    def __init__(self):
+        self._analyzer = None
+        self._init_analyzer()
+
+    def _init_analyzer(self):
+        """분석기 초기화"""
+        try:
+            from .morphology.japanese import JapaneseAnalyzer
+            self._analyzer = JapaneseAnalyzer()
+        except ImportError:
+            self._analyzer = None
+
+    def is_available(self) -> bool:
+        return self._analyzer is not None
+
+    def analyze(self, text: str) -> List[Token]:
+        """형태소 분석"""
+        if not self._analyzer:
+            return []
+
+        tokens = []
+        result = self._analyzer.analyze(text)
+        for morph in result:
+            tokens.append(Token(
+                text=morph.surface,
+                start=morph.start,
+                end=morph.end,
+                lemma=morph.lemma,
+                pos=morph.pos,
+            ))
+        return tokens
+
+
+class ChineseMorphologyAnalyzer(MorphologicalAnalyzer):
+    """중국어 형태소 분석기 (자체 구현)"""
+
+    def __init__(self, join_dates: Optional[bool] = None):
+        self._analyzer = None
+        self._join_dates = join_dates
+        self._init_analyzer()
+
+    def _init_analyzer(self):
+        """분석기 초기화"""
+        try:
+            from .morphology.chinese import ChineseAnalyzer
+            self._analyzer = ChineseAnalyzer(join_dates=self._join_dates)
+        except ImportError:
+            self._analyzer = None
+
+    def is_available(self) -> bool:
+        return self._analyzer is not None
+
+    def analyze(self, text: str) -> List[Token]:
+        """형태소 분석"""
+        if not self._analyzer:
+            return []
+
+        tokens = []
+        result = self._analyzer.analyze(text)
+        for morph in result:
+            tokens.append(Token(
+                text=morph.surface,
+                start=morph.start,
+                end=morph.end,
+                lemma=morph.lemma,
+                pos=morph.pos,
+            ))
+        return tokens
+
+
+# =============================================================================
+# 언어별 특화 토크나이저
+# =============================================================================
+
+class KoreanTokenizer(CJKTokenizer):
+    """한국어 특화 토크나이저"""
+
+    SUPPORTED_LANGUAGES = {'ko'}
+
+    def __init__(self, use_morphology: bool = True):
+        """
+        Args:
+            use_morphology: 형태소 분석 사용 (기본 True)
+        """
+        super().__init__('ko', use_morphology)
+
+    def _init_morphology(self):
+        self._morphology_analyzer = KoreanMorphologyAnalyzer()
+
+    def nouns(self, text: str) -> List[str]:
+        """명사 추출"""
+        if self._morphology_analyzer and self._morphology_analyzer.is_available():
+            return self._morphology_analyzer.nouns(text)
+        return []
+
+
+class JapaneseTokenizer(CJKTokenizer):
+    """일본어 특화 토크나이저"""
+
+    SUPPORTED_LANGUAGES = {'ja'}
+
+    def __init__(self, use_morphology: bool = True):
+        """
+        Args:
+            use_morphology: 형태소 분석 사용 (기본 True)
+        """
+        super().__init__('ja', use_morphology)
+
+    def _init_morphology(self):
+        self._morphology_analyzer = JapaneseMorphologyAnalyzer()
+
+
+class ChineseTokenizer(CJKTokenizer):
+    """중국어 특화 토크나이저"""
+
+    SUPPORTED_LANGUAGES = {'zh'}
+
+    def __init__(self, use_morphology: bool = True, *, zh_join_dates: Optional[bool] = None):
+        """
+        Args:
+            use_morphology: 형태소 분석 사용 (기본 True)
+        """
+        super().__init__('zh', use_morphology, zh_join_dates=zh_join_dates)
+
+    def _init_morphology(self):
+        self._morphology_analyzer = ChineseMorphologyAnalyzer(join_dates=self._zh_join_dates)

tokmor/domain/__init__.py

@@ -0,0 +1,11 @@
+"""
+Domain lexicons (small, optional)
+================================
+
+TokMor core focuses on tokenization/morphology. Domain lexicons are small,
+optional add-ons that can be shipped in a data pack (`TOKMOR_DATA_DIR`) or as
+tiny bundled assets.
+"""
+
+from .sentiment import load_sentiment_lexicon, sentiment_hint  # noqa: F401
+

tokmor/domain/sentiment.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Tuple
+
+from ..factory import detect_language
+from ..preprocess import normalize_text
+from ..resources import data_dir
+
+
+Polarity = Literal["pos", "neg", "neu"]
+
+
+def _domain_sentiment_path(lang: str) -> Path:
+    # Prefer external assets under TOKMOR_DATA_DIR, fallback to bundled models.
+    #
+    # IMPORTANT:
+    # - `resources.data_dir()` returns TOKMOR_DATA_DIR when set, which would hide bundled
+    #   seed lexicons if we only looked there.
+    # - So we explicitly check external first, then the package-bundled models dir.
+    from .. import resources
+
+    l = (lang or "").lower().replace("_", "-")
+    rel = Path("domain") / "sentiment" / f"{l}.json"
+
+    env = resources.data_dir() / rel
+    bundled = Path(__file__).resolve().parents[1] / "models" / rel  # tokmor/models/...
+
+    if env.exists():
+        return env
+    return bundled
+
+
+@dataclass(frozen=True)
+class SentimentLexicon:
+    lang: str
+    pos: Set[str]
+    neg: Set[str]
+    negators: Set[str]
+    intensifiers: Set[str]
+    diminishers: Set[str]
+
+
+def _as_set(xs: Any) -> Set[str]:
+    if not xs:
+        return set()
+    out: Set[str] = set()
+    for x in xs if isinstance(xs, list) else []:
+        s = str(x).strip()
+        if s:
+            out.add(s)
+    return out
+
+
+@lru_cache(maxsize=32)
+def load_sentiment_lexicon(lang: str) -> Optional[SentimentLexicon]:
+    """
+    Load a small sentiment lexicon for a language.
+
+    Returns None if the lexicon does not exist.
+    """
+    # Allow disabling domain lexicons entirely (e.g., minimal deployments).
+    try:
+        import os
+
+        v = (os.getenv("TOKMOR_DISABLE_DOMAIN_LEXICONS", "") or "").strip().lower()
+        if v in {"1", "true", "yes", "y", "on"}:
+            return None
+    except Exception:
+        pass
+    l = (lang or "").lower().replace("_", "-")
+    if not l:
+        return None
+    p = _domain_sentiment_path(l)
+    if not p.exists():
+        return None
+
+    obj = json.loads(p.read_text(encoding="utf-8", errors="ignore"))
+    return SentimentLexicon(
+        lang=str(obj.get("lang") or l),
+        pos=_as_set(obj.get("pos")),
+        neg=_as_set(obj.get("neg")),
+        negators=_as_set(obj.get("negators")),
+        intensifiers=_as_set(obj.get("intensifiers")),
+        diminishers=_as_set(obj.get("diminishers")),
+    )
+
+
+def _normalize_token_for_match(lang: str, tok: str) -> str:
+    s = (tok or "").strip()
+    if not s:
+        return ""
+    if lang.startswith("en"):
+        return s.lower()
+    return s
+
+
+def _iter_surface_tokens_for_sentiment(text: str, *, lang: str, sns: bool) -> List[str]:
+    # Use ner_preprocess surfaces to avoid morpheme splits where possible.
+    from ..ner_prep import ner_preprocess as _ner_preprocess
+
+    out = _ner_preprocess(
+        text,
+        lang=lang,
+        sns=bool(sns),
+        morphology=None,
+        include_token_hints=False,
+        include_function_word_hints=False,
+        drop_function_words=False,  # keep negators like "not", "안"
+        include_pos4_hints=False,
+        use_surfaces=True,
+    )
+    return [str(x) for x in (out.get("ner_input_tokens") or []) if str(x).strip()]
+
+
+def sentiment_hint(
+    text: str,
+    *,
+    lang: str = "auto",
+    sns: bool = True,
+    window_negate: int = 1,
+) -> Dict[str, Any]:
+    """
+    Best-effort sentiment hint (ko/en seed).
+
+    This is intentionally simple and deterministic:
+    - lexicon match on surface tokens
+    - optional 1-token negation inversion ("not good", "안 좋아")
+    - optional intensifier/diminisher multiplier
+    """
+    if lang == "auto":
+        text_norm = normalize_text(text, sns=bool(sns))
+        lang = detect_language(text_norm)
+
+    lex = load_sentiment_lexicon(lang)
+    if lex is None:
+        return {
+            "lang": lang,
+            "supported": False,
+            "polarity": "neu",
+            "score": 0.0,
+            "hits": [],
+        }
+
+    toks = _iter_surface_tokens_for_sentiment(text, lang=lang, sns=bool(sns))
+    norm = [_normalize_token_for_match(lex.lang, t) for t in toks]
+
+    hits: List[Dict[str, Any]] = []
+    score = 0.0
+
+    def _mult(i: int) -> float:
+        # Look one token back for degree modifiers.
+        if i - 1 >= 0 and norm[i - 1] in lex.intensifiers:
+            return 1.5
+        if i - 1 >= 0 and norm[i - 1] in lex.diminishers:
+            return 0.5
+        return 1.0
+
+    def _is_negated(i: int) -> bool:
+        for j in range(max(0, i - int(window_negate)), i):
+            if norm[j] in lex.negators:
+                return True
+        return False
+
+    for i, t in enumerate(norm):
+        if not t:
+            continue
+        w = _mult(i)
+        if t in lex.pos:
+            s = (1.0 * w)
+            if _is_negated(i):
+                s = -s
+            score += s
+            hits.append({"token": toks[i], "match": "pos", "weight": s})
+        elif t in lex.neg:
+            s = (-1.0 * w)
+            if _is_negated(i):
+                s = -s
+            score += s
+            hits.append({"token": toks[i], "match": "neg", "weight": s})
+
+    pol: Polarity = "neu"
+    if score > 0.25:
+        pol = "pos"
+    elif score < -0.25:
+        pol = "neg"
+
+    return {
+        "lang": lex.lang,
+        "supported": True,
+        "polarity": pol,
+        "score": float(score),
+        "hits": hits,
+    }
+