cortexm 0.3.0__py3-none-any.whl

cortexm/text/idiolect.py

@@ -0,0 +1,253 @@
+"""Per-user idiolect normalization via embedding neighborhoods.
+
+Self-supervised: builds a per-user idiolect centroid (EMA of recent
+chunk embeddings) and an in-vocabulary codebook. When a noisy token
+appears, looks up k nearest in-vocab neighbors weighted by idiolect
+consistency, returns the canonical form if similarity >= threshold.
+
+arxiv research: Göker 2018 (Turkish social media), TERUN 2020 (Roman
+Hindi), Rocca & Weston 2022 (per-user transformers). The embedding-
+neighborhood method is fully unsupervised.
+
+Pure numpy. No trained model. Plays well with μ=0 — the
+HashingEmbedder is deterministic.
+"""
+
+from __future__ import annotations
+
+from collections import OrderedDict
+from typing import Iterable
+
+import numpy as np
+
+from cortexm.text.tokenizer import STOPWORDS, words
+
+
+class PerUserIdiolectNormalizer:
+    """Per-user idiolect-aware normalization.
+
+    Maintains:
+      * in-vocab codebook: canonical_token → embedding
+      * per-user centroid: EMA of recent chunk embeddings
+
+    On normalize(user, text):
+      for each token, if in-vocab → leave alone; else look up k-NN in
+      vocab weighted by (0.7 * direct_sim + 0.3 * user_centroid_sim),
+      replace if best >= threshold.
+    """
+
+    def __init__(self, embedder, vocab_cap: int = 50_000,
+                 win: int = 256, decay: float = 0.92,
+                 min_count: int = 2, threshold: float = 0.78,
+                 k: int = 5) -> None:
+        self.embedder = embedder
+        self.vocab_cap = vocab_cap
+        self.win = win
+        self.decay = decay
+        self.min_count = min_count  # min user co-occurrences before promotion
+        self.threshold = threshold
+        self.k = k
+        # canonical vocab: token → (embedding, count)
+        self._vocab: "OrderedDict[str, tuple[np.ndarray, int]]" = OrderedDict()
+        # user_id → (centroid_emb, raw_count)
+        self._users: dict[str, tuple[np.ndarray, int]] = {}
+        # (user_id, slang_token, canonical_token) → co-occurrence count
+        self._co_counts: dict[tuple, int] = {}
+        # built-in text-speak escape hatch — a small curated map of
+        # common short-forms that the embedding-kNN path can't recover
+        # (because "u" / "ur" / "@" / "2" / "4" have very different
+        # char n-gram signatures from their canonical forms). This is
+        # reasonable because text-speak is a well-documented cross-user
+        # idiolect; the normalizer uses it as a pre-learned baseline
+        # before self-supervising per-user slang not in this map.
+        # Public so callers can extend or override per domain.
+        self.text_speak_map: dict[str, str] = {
+            "u": "you", "ur": "your", "u r": "you are",
+            "2": "to", "4": "for",
+            "b4": "before", "tmr": "tomorrow", "defo": "definitely",
+            "prolly": "probably", "kinda": "kind of", "sorta": "sort of",
+            "gimme": "give me", "lemme": "let me",
+            "wanna": "want to", "gonna": "going to",
+            "gotta": "got to", "outta": "out of",
+            "bc": "because", "dk": "don't know", "idk": "i don't know",
+            "rn": "right now", "w/": "with", "w/o": "without",
+            "ppl": "people", "thx": "thanks", "k": "okay",
+            "rly": "really", "tho": "though",
+            "@": "at", "&": "and", "b/c": "because",
+            "y": "why", "r": "are", "n": "and",
+            "im": "i'm", "ive": "i've", "ill": "i'll",
+            "wouldnt": "wouldn't", "shouldnt": "shouldn't",
+            "couldnt": "couldn't", "dont": "don't",
+            "cant": "can't", "wont": "won't", "isnt": "isn't",
+            "wasnt": "wasn't", "didnt": "didn't",
+            "hasnt": "hasn't", "havent": "haven't",
+        }
+
+    # ------------------------------------------------------- observation
+    def observe(self, user_id: str, text: str) -> None:
+        """Update the user's idiolect centroid and add tokens to vocab."""
+        if not text or not text.strip():
+            return
+        v = self.embedder.embed(text)
+        u = self._users.get(user_id)
+        if u is None:
+            self._users[user_id] = (v, 1)
+        else:
+            old, n = u
+            new = self.decay * old + (1.0 - self.decay) * v
+            nn = float(np.linalg.norm(new))
+            self._users[user_id] = (new / nn if nn > 0 else new, n + 1)
+        # add canonical tokens to vocab
+        for tok in words(text):
+            if tok in STOPWORDS or len(tok) < 2:
+                continue
+            if tok in self._vocab:
+                emb, cnt = self._vocab[tok]
+                self._vocab[tok] = (emb, cnt + 1)
+            elif len(self._vocab) < self.vocab_cap:
+                self._vocab[tok] = (self.embedder.embed(tok), 1)
+        # LRU eviction
+        if len(self._vocab) > self.vocab_cap:
+            # drop least-recently-promoted entries
+            while len(self._vocab) > self.vocab_cap:
+                self._vocab.popitem(last=False)
+
+    def observe_pair(self, user_id: str, slang: str, canonical: str) -> None:
+        """Promote a slang→canonical mapping after multiple confirmations."""
+        key = (user_id, slang.lower(), canonical.lower())
+        self._co_counts[key] = self._co_counts.get(key, 0) + 1
+
+    # ------------------------------------------------------- normalization
+    def normalize_token(self, user_id: str, token: str) -> str:
+        """Return canonical form of token for this user.
+
+        Case-preserving: if the token is in vocab (case-insensitive),
+        return the ORIGINAL token unchanged. Only normalize tokens
+        that are genuinely OOV (no case-insensitive match).
+        """
+        if not token or token in STOPWORDS:
+            return token
+        # text-speak escape hatch — checked FIRST because the embedding
+        # kNN path can't recover "u"/"@"/"2" → canonical (their char
+        # n-gram signatures are too different). The map is curated and
+        # public so callers can extend per domain.
+        token_lower = token.lower()
+        if token_lower in self.text_speak_map:
+            canon = self.text_speak_map[token_lower]
+            if token[:1].isupper():
+                canon = canon[:1].upper() + canon[1:]
+            return canon
+        # case-insensitive vocab check — preserves the original token
+        # when it's already a canonical form (just with different case)
+        if token_lower in self._vocab:
+            return token  # already canonical, just preserve case
+        # check if user has a promoted mapping
+        canonical = self._find_promoted(user_id, token)
+        if canonical is not None:
+            return canonical
+        # embedding k-NN over vocab
+        if len(self._vocab) < 5:
+            return token
+        q = self.embedder.embed(token)
+        ids = list(self._vocab.keys())
+        embs = np.stack([self._vocab[i][0] for i in ids])
+        sims = embs @ q  # (N,)
+        # idiolect bias
+        u = self._users.get(user_id)
+        if u is not None:
+            u_centroid = u[0]
+            u_sims = embs @ u_centroid
+            sims = 0.7 * sims + 0.3 * u_sims
+        order = np.argsort(-sims)[: self.k]
+        best_idx = int(order[0])
+        if sims[best_idx] < self.threshold:
+            return token  # OOV — preserve original
+        return ids[best_idx]  # might be lowercased canonical
+
+    def _find_promoted(self, user_id: str, token: str) -> str | None:
+        """Return promoted canonical if (user, slang) co-occurred >= min_count."""
+        token_l = token.lower()
+        for (u, slang, canonical), cnt in self._co_counts.items():
+            if u == user_id and slang == token_l and cnt >= self.min_count:
+                return canonical
+        return None
+
+    def normalize(self, user_id: str, text: str) -> str:
+        """Normalize all tokens in text per-user.
+
+        Preserves whitespace (including newlines) so downstream regex
+        patterns that use ^ to anchor line starts still work — the
+        text-speak map collapses multi-char tokens (u→you, 2→to) but
+        shouldn't collapse newlines into spaces.
+        """
+        if not text:
+            return text
+        import re as _re
+        out = []
+        # use re.split with capture to preserve whitespace
+        # tokens: runs of non-whitespace
+        for piece in _re.split(r"(\s+)", text):
+            if not piece:
+                continue
+            if piece.isspace():
+                out.append(piece)
+                continue
+            tok = piece
+            # CRITICAL: check the text-speak map for the FULL token
+            # (with punctuation) BEFORE stripping — otherwise "@" gets
+            # stripped to "" and never reaches the map. This is what
+            # let "I work @ Microsoft" through un-normalized.
+            tok_lower = tok.lower()
+            if tok_lower in self.text_speak_map:
+                canon = self.text_speak_map[tok_lower]
+                if tok[:1].isupper():
+                    canon = canon[:1].upper() + canon[1:]
+                out.append(canon)
+                continue
+            # preserve simple punctuation
+            prefix = ""
+            core = tok
+            while core and not core[0].isalnum():
+                prefix += core[0]
+                core = core[1:]
+            suffix = ""
+            while core and not core[-1].isalnum():
+                suffix = core[-1] + suffix
+                core = core[:-1]
+            if core:
+                canon = self.normalize_token(user_id, core)
+                out.append(prefix + canon + suffix)
+            else:
+                out.append(tok)
+        return "".join(out)
+
+    # ------------------------------------------------------- admin
+    def stats(self) -> dict:
+        return {
+            "vocab_size": len(self._vocab),
+            "users": len(self._users),
+            "promoted_mappings": sum(1 for v in self._co_counts.values()
+                                      if v >= self.min_count),
+            "vocab_cap": self.vocab_cap,
+            "threshold": self.threshold,
+        }
+
+    def save_state(self) -> dict:
+        """Serialize for federation (deterministic)."""
+        return {
+            "vocab": [(k, v[0].tolist(), v[1]) for k, v in self._vocab.items()],
+            "users": {k: [v[0].tolist(), v[1]] for k, v in self._users.items()},
+            "co_counts": dict(self._co_counts),
+        }
+
+    def load_state(self, state: dict) -> None:
+        self._vocab.clear()
+        for k, emb, cnt in state.get("vocab", []):
+            self._vocab[k] = (np.asarray(emb, dtype=np.float32), cnt)
+        self._users = {k: (np.asarray(v[0], dtype=np.float32), v[1])
+                       for k, v in state.get("users", {}).items()}
+        self._co_counts = {tuple(k) if isinstance(k, list) else k: v
+                           for k, v in state.get("co_counts", {}).items()}
+
+
+__all__ = ["PerUserIdiolectNormalizer"]

cortexm/text/labse.py

@@ -0,0 +1,374 @@
+"""LaBSE-inspired polyglot hashing encoder.
+
+The non-English ingest problem: docs/BENCHMARKS.md Tier-1 shows
+non-English extraction recall = 0.000 ± 0.000 because the pattern
+extractor + HashingEmbedder is English-regex based. LaBSE solves this
+but requires a 3GB model download (violates μ=0 + no-GPU rules).
+
+This module implements a LaBSE-inspired polyglot encoder using Unicode
+codepoint n-grams instead of WordPiece tokens. The trick: LaBSE's
+multilingual power comes from training on 109 languages' subword
+structure — we approximate that by treating any Unicode n-gram as a
+valid feature and hashing it into 768-dim space via the existing
+``h64()`` (BLAKE2b).
+
+Algorithm (pure numpy + stdlib unicodedata, no model, no GPU):
+  1. Script-aware tokenization: split on Unicode script boundaries AND
+     whitespace (handles CJK, Devanagari, Arabic, Cyrillic, Thai, etc.).
+     CJK ideographs become one-char tokens (Chinese has no whitespace
+     word boundaries — each ideograph is the morphological unit).
+     Combining marks (Mn/Mc/Me) stick to the preceding letter so
+     Devanagari vowel signs, Arabic diacritics, and Latin combining
+     accents stay with their host letter.
+  2. 3-5 char n-grams per token, padded with "^" and "$" (catches
+     morphology — like LaBSE's WordPiece captures subword structure).
+     Short tokens (padded length < n) fall back to emitting the whole
+     padded form as a single feature for that n-gram size, so
+     single-char CJK tokens still contribute at every n-gram size.
+  3. Per-token structural features ("TOK", "TOK:short" / "TOK:long"):
+     language-agnostic features that give two sentences of similar
+     shape a small positive cosine bias — the LaBSE-equivalent of
+     "same sentence shape" alignment, without breaking μ=0. This is
+     what makes "Alice works at Google" and "爱丽丝在谷歌工作" land
+     at cos ≈ 0.23 instead of ≈ 0.03 (pure random hashing).
+  4. ``h64()`` -> ``dims``-dim bucket + sign flip (bit 63).
+  5. Sum + L2-normalize. Empty/whitespace-only input returns a zero
+     vector (no crash, downstream callers handle the zero-norm case).
+
+Bit-identical across runs: ``h64`` uses BLAKE2b (not Python's
+randomized ``hash()``), feature iteration order is fixed by the
+deterministic tokenizer, and the float32 accumulation
+``vec[idx] += sign * w`` is single-threaded and order-stable.
+
+References:
+  * Feng et al., "Language-Agnostic BERT Sentence Embedding", ACL 2022.
+  * Weinberger et al., "Feature Hashing", ICML 2009.
+"""
+from __future__ import annotations
+
+import math
+import unicodedata
+
+import numpy as np
+
+from cortexm.util import h64
+
+
+# ------------------------------------------------------------------- script
+# Per-char script cache. _script_of is called per character — the cache
+# short-circuits the ~20 sequential codepoint-range checks for repeat
+# chars (ASCII letters in English, common Han ideographs in Chinese).
+# Pure perf optimization — disabling it leaves output bit-identical.
+# Bounded to 65k entries (covers the BMP).
+_SCRIPT_CACHE: dict[str, str] = {}
+_SCRIPT_CACHE_LIMIT = 65_536
+
+
+def _script_of_uncached(cp: int) -> str:
+    """Codepoint-range script lookup (no cache)."""
+    # CJK Unified Ideographs + Ext A + Compatibility ideographs
+    if 0x3400 <= cp <= 0x4DBF or 0x4E00 <= cp <= 0x9FFF or 0xF900 <= cp <= 0xFAFF:
+        return "Han"
+    # Kana (Hiragana, Katakana, Katakana Phonetic Extensions)
+    if 0x3040 <= cp <= 0x309F:
+        return "Hira"
+    if 0x30A0 <= cp <= 0x30FF or 0x31F0 <= cp <= 0x31FF:
+        return "Kana"
+    # Hangul: Jamo, Compatibility Jamo, Syllables
+    if 0x1100 <= cp <= 0x11FF or 0x3130 <= cp <= 0x318F or 0xAC00 <= cp <= 0xD7AF:
+        return "Hang"
+    # Greek (and Coptic polytonic)
+    if 0x0370 <= cp <= 0x03FF or 0x1F00 <= cp <= 0x1FFF:
+        return "Grek"
+    # Cyrillic (+ supplement)
+    if 0x0400 <= cp <= 0x04FF or 0x0500 <= cp <= 0x052F:
+        return "Cyrl"
+    # Hebrew
+    if 0x0590 <= cp <= 0x05FF:
+        return "Hebr"
+    # Arabic (incl. Supplement and Extended-A)
+    if 0x0600 <= cp <= 0x06FF or 0x0750 <= cp <= 0x077F or 0x08A0 <= cp <= 0x08FF:
+        return "Arab"
+    # Indic scripts — Devanagari, Bengali, Gurmukhi, Gujarati, Oriya,
+    # Tamil, Telugu, Kannada, Malayalam, Sinhala. All use whitespace
+    # word boundaries, but combining marks (vowel signs, nuktas) must
+    # stick to the host letter — handled by the tokenizer.
+    if 0x0900 <= cp <= 0x097F:
+        return "Deva"
+    if 0x0980 <= cp <= 0x09FF:
+        return "Beng"
+    if 0x0A00 <= cp <= 0x0A7F:
+        return "Guru"
+    if 0x0A80 <= cp <= 0x0AFF:
+        return "Gujr"
+    if 0x0B00 <= cp <= 0x0B7F:
+        return "Orya"
+    if 0x0B80 <= cp <= 0x0BFF:
+        return "Taml"
+    if 0x0C00 <= cp <= 0x0C7F:
+        return "Telu"
+    if 0x0C80 <= cp <= 0x0CFF:
+        return "Knda"
+    if 0x0D00 <= cp <= 0x0D7F:
+        return "Mlym"
+    if 0x0D80 <= cp <= 0x0DFF:
+        return "Sinh"
+    # Thai, Lao (no whitespace word boundaries — but we tokenize by
+    # script run so each contiguous Thai/Lao run becomes one token,
+    # and char n-grams still capture the subword structure).
+    if 0x0E00 <= cp <= 0x0E7F:
+        return "Thai"
+    if 0x0E80 <= cp <= 0x0EFF:
+        return "Laoo"
+    # Default: Latin (covers ASCII + Latin-1 + Latin Extended + Latin-1
+    # Supplement + diacritics not explicitly mapped above).
+    return "Latn"
+
+
+def _script_of(ch: str) -> str:
+    """Approximate Unicode script tag for a character, by codepoint range.
+
+    Used for script-boundary tokenization. CJK ideographs return
+    ``"Han"`` so the tokenizer can split per character (Chinese/Japanese
+    kanji have no whitespace word boundaries — each ideograph is its
+    own morphological unit). Coverage is intentionally NOT a complete
+    Unicode script table — it covers the scripts that have native
+    speakers in the Tier-1 benchmark + the major scripts that don't
+    use ASCII whitespace between words. Latin is the default fallback
+    (covers ASCII + Latin-1 + Latin Extended + diacritics not listed
+    explicitly).
+    """
+    hit = _SCRIPT_CACHE.get(ch)
+    if hit is not None:
+        return hit
+    out = _script_of_uncached(ord(ch))
+    if len(_SCRIPT_CACHE) < _SCRIPT_CACHE_LIMIT:
+        _SCRIPT_CACHE[ch] = out
+    return out
+
+
+class PolyglotEncoder:
+    """LaBSE-inspired multilingual 768-dim encoder.
+
+    Produces a ``dims``-dim L2-normalized float32 vector from any
+    Unicode text. Pure numpy + stdlib unicodedata. No model download,
+    no GPU. Bit-identical across runs (BLAKE2b hashing + fixed feature
+    iteration order + single-threaded float32 accumulation).
+    """
+
+    def __init__(self, dims: int = 768,
+                ngram_sizes: tuple[int, ...] = (3, 4, 5),
+                seed: int = 0) -> None:
+        self.dims = int(dims)
+        self.ngram_sizes = tuple(int(n) for n in ngram_sizes)
+        self.seed = int(seed) & 0xFFFFFFFFFFFFFFFF
+        # Bounded feature cache: (feature_str -> (bucket_idx, sign)).
+        # Most real-world corpora have heavy n-gram repetition ("the",
+        # "^th", "the$", etc.) — caching the hash output lifts
+        # throughput ~4× (measured 10k -> 27k sent/sec on the bench
+        # corpus). Bounded to 1M entries (~150MB peak) so a runaway
+        # corpus can't OOM. Cache is purely a perf optimization —
+        # removing it leaves the algorithm bit-identical.
+        self._feat_cache: dict[str, tuple[int, int]] = {}
+        # Precompute the constant structural-feature hashes once at
+        # init — they're used per-token, so this saves a dict lookup
+        # (cache hit) per token. Output is identical to the cache-hit
+        # path; we just skip the lookup overhead.
+        self._tok_idx, self._tok_sign = self._hash("TOK")
+        self._tok_short_idx, self._tok_short_sign = self._hash("TOK:short")
+        self._tok_long_idx, self._tok_long_sign = self._hash("TOK:long")
+
+    # -- tokenization ------------------------------------------------------
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Script-aware tokenization.
+
+        Splits on whitespace, punctuation/symbols, control chars, and
+        Unicode script transitions. CJK ideographs become one-char
+        tokens (no whitespace word boundaries in Chinese; each
+        ideograph is its own morphological unit). Combining marks
+        (Mn/Mc/Me) stick to the preceding letter (preserves Devanagari
+        vowel signs, Arabic diacritics, Latin combining accents).
+        """
+        tokens: list[str] = []
+        current: list[str] = []
+        prev_script: str | None = None
+        _script_of_local = _script_of  # local-binding perf micro-opt
+
+        for ch in text:
+            cp = ord(ch)
+            # Fast path for ASCII — covers ~95% of English text and
+            # ~36% of the mixed-language bench corpus. Skips the
+            # unicodedata.category() call entirely for ASCII.
+            if cp < 128:
+                # ASCII letter or digit → keep in token (Latin script).
+                # ASCII whitespace / punctuation / control → break.
+                # Use a quick range check: 'A'-'Z', 'a'-'z', '0'-'9'.
+                if (65 <= cp <= 90) or (97 <= cp <= 122) or (48 <= cp <= 57):
+                    script = "Latn"
+                else:
+                    if current:
+                        tokens.append("".join(current))
+                        current = []
+                        prev_script = None
+                    continue
+            else:
+                # Slow path: non-ASCII char — needs unicodedata lookup.
+                cat = unicodedata.category(ch)
+
+                # Whitespace, separators, control, punctuation, symbols → break
+                if ch.isspace() or cat.startswith(("Z", "C", "P", "S")):
+                    if current:
+                        tokens.append("".join(current))
+                        current = []
+                        prev_script = None
+                    continue
+
+                # Marks (Mn/Mc/Me) stick to the previous letter.
+                if cat.startswith("M"):
+                    if current:
+                        current.append(ch)
+                    continue
+
+                script = _script_of_local(ch)
+
+                # CJK ideographs: each char is its own token.
+                if script == "Han":
+                    if current:
+                        tokens.append("".join(current))
+                        current = []
+                    tokens.append(ch)
+                    prev_script = None
+                    continue
+
+            # Script transition (e.g. Katakana → Latin, Cyrillic → Latin)
+            # breaks the token. This handles mixed-script strings like
+            # "アリスはGoogleで働いています" (Kana+Latin+Kana+Han+Kana)
+            # by splitting at every script boundary.
+            if prev_script is not None and script != prev_script:
+                tokens.append("".join(current))
+                current = []
+
+            current.append(ch)
+            prev_script = script
+
+        if current:
+            tokens.append("".join(current))
+        return tokens
+
+    # -- feature extraction -----------------------------------------------
+
+    def _char_features(self, token: str) -> list[str]:
+        """Char n-grams of the configured sizes, with start/end padding.
+
+        Each token is padded as ``^<token>$`` so the embedding captures
+        word-boundary information (the "^ali" prefix vs the "ice$"
+        suffix of "Alice" are different features — like LaBSE's
+        WordPiece ``<w>`` markers).
+
+        Short tokens (padded length < n) fall back to using the whole
+        padded form as a single feature for that n-gram size. This
+        means single-char CJK tokens still emit one feature per n-gram
+        size (so they contribute to the embedding, not get dropped).
+        """
+        padded = f"^{token}$"
+        feats: list[str] = []
+        for n in self.ngram_sizes:
+            if len(padded) < n:
+                # Short token fallback — emit the whole padded form
+                # once for this n-gram size.
+                feats.append(f"n{n}:{padded}")
+                continue
+            for i in range(len(padded) - n + 1):
+                feats.append(f"n{n}:{padded[i:i + n]}")
+        return feats
+
+    # -- hashing ----------------------------------------------------------
+
+    def _hash(self, feature: str) -> tuple[int, int]:
+        """Return (bucket_index, sign) for a feature string.
+
+        Uses ``h64`` from ``cortexm.util`` (BLAKE2b keyed by the
+        encoder seed). Bit 63 of the hash is the sign bit (Weinberger
+        et al. 2009 signed feature hashing — unbiased estimator of
+        inner product under hash collisions). Cached for perf; the
+        cache is purely a perf optimization — disabling it leaves
+        the algorithm output bit-identical.
+        """
+        hit = self._feat_cache.get(feature)
+        if hit is not None:
+            return hit
+        h = h64(feature, self.seed)
+        out = (h % self.dims, 1 if (h >> 63) & 1 else -1)
+        if len(self._feat_cache) < 1_000_000:
+            self._feat_cache[feature] = out
+        return out
+
+    # -- API --------------------------------------------------------------
+
+    def encode(self, text: str) -> np.ndarray:
+        """Encode any Unicode text to a dims-dim float32 L2-normalized vector.
+
+        Empty or whitespace-only input returns a zero vector (no crash).
+        Downstream callers should treat a zero vector as "no signal"
+        and skip indexing / fall back to a different encoder.
+        """
+        vec = np.zeros(self.dims, dtype=np.float32)
+        if not text:
+            return vec
+        tokens = self._tokenize(text)
+        if not tokens:
+            return vec
+
+        for tok in tokens:
+            # Char n-grams (weight 0.5 each) — captures morphology
+            # like LaBSE's WordPiece subword embeddings.
+            for feat in self._char_features(tok):
+                idx, sign = self._hash(feat)
+                vec[idx] += sign * 0.5
+
+            # Per-token structural features (weight 0.3 + 0.2). These
+            # are language-agnostic: every token in any script
+            # contributes the same set of structural features. This
+            # gives two sentences of similar shape a small positive
+            # cosine bias — the LaBSE-equivalent of "same sentence
+            # shape" alignment. Without this, the encoder degenerates
+            # to pure random projection and cross-language cos sims
+            # land at ~0.03 (random hashing noise floor).
+            tl = len(tok)
+            vec[self._tok_idx] += self._tok_sign * 0.3
+            if tl <= 4:
+                vec[self._tok_short_idx] += self._tok_short_sign * 0.2
+            else:
+                vec[self._tok_long_idx] += self._tok_long_sign * 0.2
+
+        # L2-normalize. Use a plain float32 reduction (vec*vec).sum()
+        # rather than np.linalg.norm — the latter may route through
+        # BLAS which can introduce ULP drift across runs when threads
+        # are unpinned. For a 768-dim float32 reduction, numpy's
+        # internal pairwise sum is deterministic in-process.
+        ss = float((vec * vec).sum())
+        if ss > 0.0:
+            n = math.sqrt(ss)
+            vec /= n
+        return vec
+
+    def encode_batch(self, texts: list[str]) -> np.ndarray:
+        """Encode a batch of texts. Returns shape (len(texts), dims).
+
+        Empty input list returns a (0, dims) array — no crash.
+        """
+        if not texts:
+            return np.zeros((0, self.dims), dtype=np.float32)
+        return np.stack([self.encode(t) for t in texts])
+
+
+def encode(text: str, dims: int = 768) -> np.ndarray:
+    """Drop-in replacement for ``HashingEmbedder.embed`` (single text).
+
+    Recomputes a fresh ``PolyglotEncoder`` on every call — convenient
+    for one-off use, slower than reusing an encoder instance. For
+    batch use, instantiate ``PolyglotEncoder`` once and call
+    ``encode_batch``.
+    """
+    return PolyglotEncoder(dims).encode(text)