glitchlings 0.4.3__cp312-cp312-macosx_11_0_universal2.whl → 0.4.5__cp312-cp312-macosx_11_0_universal2.whl

glitchlings/lexicon/wordnet.py

@@ -13,21 +13,17 @@ from ._cache import CacheSnapshot
 
 
 class _LemmaProtocol(Protocol):
-    def name(self) -> str:
-        ...
+    def name(self) -> str: ...
 
 
 class _SynsetProtocol(Protocol):
-    def lemmas(self) -> Sequence[_LemmaProtocol]:
-        ...
+    def lemmas(self) -> Sequence[_LemmaProtocol]: ...
 
 
 class _WordNetResource(Protocol):
-    def synsets(self, word: str, pos: str | None = None) -> Sequence[_SynsetProtocol]:
-        ...
+    def synsets(self, word: str, pos: str | None = None) -> Sequence[_SynsetProtocol]: ...
 
-    def ensure_loaded(self) -> None:
-        ...
+    def ensure_loaded(self) -> None: ...
 
 
 WordNetCorpusReaderFactory = Callable[[Any, Any], _WordNetResource]

glitchlings/util/hokey_generator.py

@@ -0,0 +1,144 @@
+"""Hokey expressive lengthening generator."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .stretch_locator import StretchSite, apply_stretch, find_stretch_site
+from .stretchability import RandomLike, StretchabilityAnalyzer, StretchabilityFeatures
+
+
+@dataclass(slots=True)
+class HokeyConfig:
+    rate: float = 0.3
+    extension_min: int = 2
+    extension_max: int = 5
+    base_p: float = 0.45
+    word_length_threshold: int = 6
+
+
+@dataclass(slots=True)
+class StretchEvent:
+    token_index: int
+    original: str
+    stretched: str
+    repeats: int
+    site: StretchSite
+    score: float
+    features: StretchabilityFeatures
+
+
+class NegativeBinomialSampler:
+    """Sample stretch lengths from a clipped negative binomial distribution."""
+
+    def __init__(self, base_p: float = 0.45) -> None:
+        self.base_p = base_p
+
+    def sample(
+        self,
+        rng: RandomLike,
+        *,
+        intensity: float,
+        minimum: int,
+        maximum: int,
+    ) -> int:
+        minimum = max(0, int(minimum))
+        maximum = max(minimum, int(maximum))
+        if maximum == 0:
+            return 0
+        if maximum == minimum:
+            return maximum
+
+        r = max(1, int(round(1 + 2 * intensity)))
+        adjusted_p = self.base_p / (1.0 + 0.75 * max(0.0, intensity))
+        adjusted_p = max(0.05, min(0.95, adjusted_p))
+        failures = sum(self._geometric_sample(rng, adjusted_p) for _ in range(r))
+        extra = minimum + failures
+        return max(minimum, min(maximum, extra))
+
+    @staticmethod
+    def _geometric_sample(rng: RandomLike, p: float) -> int:
+        count = 0
+        while rng.random() > p:
+            count += 1
+        return count
+
+
+class HokeyGenerator:
+    """Full expressive lengthening pipeline."""
+
+    def __init__(
+        self,
+        analyzer: StretchabilityAnalyzer | None = None,
+        sampler: NegativeBinomialSampler | None = None,
+    ) -> None:
+        self.analyzer = analyzer or StretchabilityAnalyzer()
+        self.sampler = sampler or NegativeBinomialSampler()
+
+    def generate(
+        self,
+        text: str,
+        *,
+        rng: RandomLike,
+        config: HokeyConfig,
+    ) -> tuple[str, list[StretchEvent]]:
+        if not text:
+            return text, []
+
+        if config.base_p != self.sampler.base_p:
+            self.sampler.base_p = config.base_p
+
+        tokens = self.analyzer.tokenise(text)
+        candidates = self.analyzer.analyse_tokens(tokens)
+        selected = self.analyzer.select_candidates(candidates, rate=config.rate, rng=rng)
+        if not selected:
+            return text, []
+
+        token_strings = [token.text for token in tokens]
+        events: list[StretchEvent] = []
+
+        for candidate in selected:
+            token_idx = candidate.token.index
+            original = token_strings[token_idx]
+            site = find_stretch_site(original)
+            if site is None:
+                continue
+
+            intensity = min(1.5, candidate.features.intensity() + 0.35 * candidate.score)
+            alpha_count = sum(1 for ch in original if ch.isalpha())
+            if config.word_length_threshold > 0 and alpha_count > config.word_length_threshold * 2:
+                continue
+            if config.word_length_threshold > 0 and alpha_count > config.word_length_threshold:
+                excess = alpha_count - config.word_length_threshold
+                intensity = intensity / (1.0 + 0.35 * excess)
+                if candidate.score < 0.35 and excess >= 2:
+                    continue
+            intensity = max(0.05, intensity)
+
+            repeats = self.sampler.sample(
+                rng,
+                intensity=intensity,
+                minimum=config.extension_min,
+                maximum=config.extension_max,
+            )
+            if repeats <= 0:
+                continue
+
+            stretched_word = apply_stretch(original, site, repeats)
+            token_strings[token_idx] = stretched_word
+            events.append(
+                StretchEvent(
+                    token_index=token_idx,
+                    original=original,
+                    stretched=stretched_word,
+                    repeats=repeats,
+                    site=site,
+                    score=candidate.score,
+                    features=candidate.features,
+                )
+            )
+
+        return "".join(token_strings), events
+
+
+__all__ = ["HokeyGenerator", "HokeyConfig", "StretchEvent", "NegativeBinomialSampler"]

glitchlings/util/stretch_locator.py

@@ -0,0 +1,140 @@
+"""Identify where expressive stretches should occur within a token."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable
+
+VOWELS = set("aeiouyAEIOUY")
+SONORANTS = set("rlmnwyhRLMNWYH")
+SIBILANTS = set("sSzZxXcCjJ") | {"sh", "Sh", "sH", "SH", "zh", "Zh"}
+DIGRAPHS = {
+    "aa",
+    "ae",
+    "ai",
+    "ay",
+    "ee",
+    "ei",
+    "ey",
+    "ie",
+    "io",
+    "oa",
+    "oe",
+    "oi",
+    "oo",
+    "ou",
+    "ua",
+    "ue",
+    "ui",
+    "ya",
+    "yo",
+    "yu",
+}
+
+
+@dataclass(slots=True)
+class StretchSite:
+    """Location of a stretchable grapheme."""
+
+    start: int
+    end: int
+    category: str
+
+    def unit(self, token: str) -> str:
+        return token[self.start : self.end]
+
+
+def _alpha_indices(token: str) -> list[int]:
+    return [idx for idx, char in enumerate(token) if char.isalpha()]
+
+
+def _vowel_clusters(token: str, indices: Iterable[int]) -> list[tuple[int, int]]:
+    clusters: list[tuple[int, int]] = []
+    start: int | None = None
+    prev_idx: int | None = None
+    for idx in indices:
+        char = token[idx]
+        if char in VOWELS:
+            if start is None:
+                start = idx
+            elif prev_idx is not None and idx != prev_idx + 1:
+                clusters.append((start, prev_idx + 1))
+                start = idx
+        else:
+            if start is not None:
+                clusters.append((start, idx))
+                start = None
+        prev_idx = idx
+    if start is not None and prev_idx is not None:
+        clusters.append((start, prev_idx + 1))
+    return clusters
+
+
+def find_stretch_site(token: str) -> StretchSite | None:
+    """Return the most suitable stretch site for ``token``."""
+
+    alpha_indices = _alpha_indices(token)
+    if not alpha_indices:
+        return None
+
+    lower = token.lower()
+    clusters = _vowel_clusters(lower, alpha_indices)
+    candidates: list[tuple[int, StretchSite]] = []
+
+    # Sibilant/sonorant coda extension (yes -> yesss, hmm -> hmmmm)
+    last_idx = alpha_indices[-1]
+    last_char = lower[last_idx]
+    if len(alpha_indices) >= 2:
+        prev_char = lower[alpha_indices[-2]]
+    else:
+        prev_char = ""
+    has_multi_vowel = any(
+        (end - start >= 2) and not (lower[start] == "y" and start == 0) for start, end in clusters
+    )
+    if last_char in {"s", "z"} and prev_char in VOWELS and not has_multi_vowel:
+        candidates.append((5, StretchSite(last_idx, last_idx + 1, "coda")))
+    elif last_char in SONORANTS and prev_char in VOWELS and not has_multi_vowel:
+        candidates.append((4, StretchSite(last_idx, last_idx + 1, "coda")))
+    elif not clusters:
+        candidates.append((2, StretchSite(last_idx, last_idx + 1, "consonant")))
+
+    # CVCe pattern (cute -> cuuute)
+    if lower.endswith("e") and len(alpha_indices) >= 3:
+        final_letter = alpha_indices[-1]
+        if token[final_letter].lower() == "e":
+            c_idx = alpha_indices[-2]
+            v_idx = alpha_indices[-3]
+            if token[c_idx].lower() not in VOWELS and token[v_idx].lower() in VOWELS:
+                candidates.append((4, StretchSite(v_idx, v_idx + 1, "cvce")))
+
+    for cluster in clusters:
+        start, end = cluster
+        substring = lower[start:end]
+        category = "vowel"
+        if any(substring[i : i + 2] in DIGRAPHS for i in range(max(0, len(substring) - 1))):
+            category = "digraph"
+        priority = 3 if cluster == clusters[-1] else 2
+        candidates.append((priority, StretchSite(start, end, category)))
+
+    if not candidates:
+        return None
+
+    candidates.sort(key=lambda item: (item[0], item[1].end - item[1].start, -item[1].start))
+    return candidates[-1][1]
+
+
+def apply_stretch(token: str, site: StretchSite, repeats: int) -> str:
+    """Return ``token`` with ``repeats`` extra copies of the grapheme at ``site``."""
+
+    if repeats <= 0:
+        return token
+    chars = list(token)
+    stretched: list[str] = []
+    for idx, char in enumerate(chars):
+        stretched.append(char)
+        if site.start <= idx < site.end:
+            stretched.append(char * repeats)
+    return "".join(stretched)
+
+
+__all__ = ["StretchSite", "find_stretch_site", "apply_stretch"]

glitchlings/util/stretchability.py

@@ -0,0 +1,375 @@
+"""Stretchability scoring and candidate selection for Hokey."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from importlib import resources
+from typing import Any, Protocol, Sequence, TypedDict, cast
+
+# Regexes reused across the module
+TOKEN_REGEX = re.compile(r"\w+|\W+")
+ALPHA_REGEX = re.compile(r"[A-Za-z]")
+EMOJI_REGEX = re.compile(r"[\U0001F300-\U0001FAFF]")
+CLAUSE_PUNCTUATION = {".", "?", "!", ";"}
+
+
+class HokeyAssets(TypedDict):
+    lexical_prior: dict[str, float]
+    interjections: list[str]
+    intensifiers: list[str]
+    evaluatives: list[str]
+    positive_lexicon: list[str]
+    negative_lexicon: list[str]
+
+
+class RandomLike(Protocol):
+    """Interface for RNGs that expose ``random()``."""
+
+    def random(self) -> float: ...
+
+
+# Lexical prior probabilities and pragmatic lexica shared with the Rust fast path.
+def _load_assets() -> HokeyAssets:
+    with (
+        resources.files("glitchlings.data")
+        .joinpath("hokey_assets.json")
+        .open("r", encoding="utf-8") as payload
+    ):
+        data: Any = json.load(payload)
+    return cast(HokeyAssets, data)
+
+
+_ASSETS = _load_assets()
+LEXICAL_PRIOR: dict[str, float] = {
+    token: float(score) for token, score in _ASSETS["lexical_prior"].items()
+}
+
+# Pragmatic lexica for POS/discourse cues
+INTERJECTIONS = frozenset(_ASSETS["interjections"])
+INTENSIFIERS = frozenset(_ASSETS["intensifiers"])
+EVALUATIVES = frozenset(_ASSETS["evaluatives"])
+POSITIVE_LEXICON = frozenset(_ASSETS["positive_lexicon"])
+NEGATIVE_LEXICON = frozenset(_ASSETS["negative_lexicon"])
+
+VOWELS = set("aeiouy")
+SONORANT_CODAS = set("rlmnwyh")
+SIBILANT_CODAS = {"s", "z", "x", "c", "j", "sh", "zh"}
+DIGRAPHS = {
+    "aa",
+    "ae",
+    "ai",
+    "ay",
+    "ee",
+    "ei",
+    "ey",
+    "ie",
+    "oa",
+    "oe",
+    "oi",
+    "oo",
+    "ou",
+    "ue",
+    "ui",
+}
+
+MAX_CANDIDATES_PER_CLAUSE = 4
+MIN_SCORE_THRESHOLD = 0.18
+
+
+@dataclass(slots=True)
+class TokenInfo:
+    text: str
+    start: int
+    end: int
+    is_word: bool
+    clause_index: int
+    preceding_punct: str
+    following_punct: str
+    index: int
+
+    @property
+    def normalised(self) -> str:
+        return self.text.lower()
+
+
+@dataclass(slots=True)
+class StretchabilityFeatures:
+    lexical: float
+    pos: float
+    sentiment: float
+    phonotactic: float
+    context: float
+    sentiment_swing: float
+
+    def intensity(self) -> float:
+        """Map features to an intensity scalar in [0, 1.5]."""
+        emphasis = 0.6 * self.context + 0.4 * self.sentiment_swing
+        return max(0.0, min(1.5, 0.5 * (self.lexical + self.phonotactic) + emphasis))
+
+
+@dataclass(slots=True)
+class StretchCandidate:
+    token: TokenInfo
+    score: float
+    features: StretchabilityFeatures
+
+
+class StretchabilityAnalyzer:
+    """Compute stretchability scores and select candidates."""
+
+    def __init__(
+        self,
+        *,
+        lexical_prior: dict[str, float] | None = None,
+        weights: tuple[float, float, float, float, float] = (0.32, 0.18, 0.14, 0.22, 0.14),
+    ) -> None:
+        self.lexical_prior = lexical_prior or LEXICAL_PRIOR
+        self.weights = weights
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+    def tokenise(self, text: str) -> list[TokenInfo]:
+        """Tokenise text preserving separator tokens."""
+        return self._tokenise(text)
+
+    def analyse(self, text: str) -> list[StretchCandidate]:
+        if not text:
+            return []
+        tokens = self._tokenise(text)
+        return self.analyse_tokens(tokens)
+
+    def analyse_tokens(self, tokens: Sequence[TokenInfo]) -> list[StretchCandidate]:
+        candidates: list[StretchCandidate] = []
+        for idx, token in enumerate(tokens):
+            if not token.is_word:
+                continue
+            if self._excluded(token, tokens, idx):
+                continue
+
+            features = self._compute_features(token, tokens, idx)
+            score = self._composite_score(features)
+            if score < MIN_SCORE_THRESHOLD:
+                continue
+            candidates.append(StretchCandidate(token=token, score=score, features=features))
+        return candidates
+
+    def select_candidates(
+        self,
+        candidates: Sequence[StretchCandidate],
+        *,
+        rate: float,
+        rng: RandomLike,
+    ) -> list[StretchCandidate]:
+        if not candidates or rate <= 0:
+            return []
+
+        grouped: dict[int, list[StretchCandidate]] = {}
+        for candidate in candidates:
+            grouped.setdefault(candidate.token.clause_index, []).append(candidate)
+
+        selected: list[StretchCandidate] = []
+        total_expected = max(0, min(len(candidates), int(round(len(candidates) * rate))))
+
+        for clause_index in sorted(grouped):
+            clause_candidates = sorted(
+                grouped[clause_index], key=lambda c: (-c.score, c.token.start)
+            )
+            clause_candidates = clause_candidates[:MAX_CANDIDATES_PER_CLAUSE]
+            clause_quota = max(
+                0, min(len(clause_candidates), int(round(len(clause_candidates) * rate)))
+            )
+
+            provisional: list[StretchCandidate] = []
+            for candidate in clause_candidates:
+                probability = min(1.0, rate * (0.35 + 0.65 * candidate.score))
+                if rng.random() < probability:
+                    provisional.append(candidate)
+                if len(provisional) >= clause_quota:
+                    break
+
+            if len(provisional) < clause_quota:
+                leftovers = [c for c in clause_candidates if c not in provisional]
+                needed = clause_quota - len(provisional)
+                provisional.extend(leftovers[:needed])
+
+            selected.extend(provisional)
+
+        if len(selected) < total_expected:
+            remaining = [c for c in candidates if c not in selected]
+            remaining.sort(key=lambda c: (-c.score, c.token.start))
+            selected.extend(remaining[: total_expected - len(selected)])
+
+        # Keep deterministic order by position
+        selected.sort(key=lambda c: c.token.start)
+        return selected
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+    def _tokenise(self, text: str) -> list[TokenInfo]:
+        tokens: list[TokenInfo] = []
+        clause_index = 0
+        matches = list(TOKEN_REGEX.finditer(text))
+        for idx, match in enumerate(matches):
+            token_text = match.group(0)
+            is_word = bool(ALPHA_REGEX.search(token_text)) and token_text.strip().isalnum()
+            preceding = matches[idx - 1].group(0) if idx > 0 else ""
+            following = matches[idx + 1].group(0) if idx + 1 < len(matches) else ""
+            tokens.append(
+                TokenInfo(
+                    text=token_text,
+                    start=match.start(),
+                    end=match.end(),
+                    is_word=is_word,
+                    clause_index=clause_index,
+                    preceding_punct=preceding,
+                    following_punct=following,
+                    index=idx,
+                )
+            )
+            if any(ch in CLAUSE_PUNCTUATION for ch in token_text):
+                clause_index += 1
+        return tokens
+
+    def _excluded(self, token: TokenInfo, tokens: Sequence[TokenInfo], index: int) -> bool:
+        text = token.text
+        normalised = token.normalised
+        if sum(ch.isalpha() for ch in text) < 2:
+            return True
+        if any(ch.isdigit() for ch in text):
+            return True
+        lowered = normalised
+        if "http" in lowered or "www" in lowered or "//" in lowered:
+            return True
+        if any(symbol in text for symbol in {"#", "@", "&", "{", "}", "<", ">"}):
+            return True
+        if "_" in text:
+            return True
+        if "/" in text or "\\" in text:
+            return True
+
+        # Heuristic proper noun check: Title case mid-clause counts as proper noun
+        if text[:1].isupper() and text[1:].islower():
+            previous_clause_start = index == 0
+            if not previous_clause_start:
+                for prior in reversed(tokens[:index]):
+                    stripped = prior.text.strip()
+                    if not stripped:
+                        continue
+                    if stripped[-1] in CLAUSE_PUNCTUATION:
+                        previous_clause_start = True
+                    break
+            if not previous_clause_start:
+                return True
+        return False
+
+    def _compute_features(
+        self, token: TokenInfo, tokens: Sequence[TokenInfo], index: int
+    ) -> StretchabilityFeatures:
+        lexical = self.lexical_prior.get(token.normalised, 0.12)
+        pos_score = self._pos_score(token)
+        sentiment_score, sentiment_swing = self._sentiment(tokens, index)
+        phon_score = self._phonotactic(token.normalised)
+        context_score = self._contextual(token, tokens, index)
+        return StretchabilityFeatures(
+            lexical=lexical,
+            pos=pos_score,
+            sentiment=sentiment_score,
+            phonotactic=phon_score,
+            context=context_score,
+            sentiment_swing=sentiment_swing,
+        )
+
+    def _composite_score(self, features: StretchabilityFeatures) -> float:
+        lex_w, pos_w, sent_w, phon_w, ctx_w = self.weights
+        weighted = (
+            lex_w * features.lexical
+            + pos_w * features.pos
+            + sent_w * features.sentiment
+            + phon_w * features.phonotactic
+            + ctx_w * features.context
+        )
+        total_weight = sum(self.weights)
+        score = weighted / total_weight if total_weight else 0.0
+        return max(0.0, min(1.0, score))
+
+    # ------------------------------------------------------------------
+    # Feature helpers
+    # ------------------------------------------------------------------
+    def _pos_score(self, token: TokenInfo) -> float:
+        normalised = token.normalised
+        if normalised in INTERJECTIONS:
+            return 0.95
+        if normalised in INTENSIFIERS:
+            return 0.85
+        if normalised in EVALUATIVES:
+            return 0.7
+        if normalised.endswith("ly"):
+            return 0.55
+        if token.text.isupper() and len(token.text) > 1:
+            return 0.65
+        return 0.3
+
+    def _sentiment(self, tokens: Sequence[TokenInfo], index: int) -> tuple[float, float]:
+        window = [tok for tok in tokens[max(0, index - 2) : index + 3] if tok.is_word]
+        if not window:
+            return 0.5, 0.0
+        pos_hits = sum(1 for tok in window if tok.normalised in POSITIVE_LEXICON)
+        neg_hits = sum(1 for tok in window if tok.normalised in NEGATIVE_LEXICON)
+        total = len(window)
+        balance = (pos_hits - neg_hits) / total
+        sentiment_score = 0.5 + 0.5 * max(-1.0, min(1.0, balance))
+        swing = abs(balance)
+        return sentiment_score, swing
+
+    def _phonotactic(self, normalised: str) -> float:
+        if not any(ch in VOWELS for ch in normalised):
+            return 0.0
+        score = 0.25
+        if any(normalised.endswith(c) for c in SONORANT_CODAS):
+            score += 0.2
+        if any(normalised.endswith(c) for c in SIBILANT_CODAS):
+            score += 0.18
+        if any(digraph in normalised for digraph in DIGRAPHS):
+            score += 0.22
+        if re.search(r"[aeiouy]{2,}", normalised):
+            score += 0.22
+        if re.search(r"(.)(?!\1)(.)\1", normalised):
+            score += 0.08
+        return max(0.0, min(1.0, score))
+
+    def _contextual(self, token: TokenInfo, tokens: Sequence[TokenInfo], index: int) -> float:
+        score = 0.2
+        before = token.preceding_punct
+        after = token.following_punct
+        token_text = token.text
+        if after and after.count("!") >= 1:
+            score += 0.25
+        if after and after.count("?") >= 1:
+            score += 0.2
+        if before and before.count("!") >= 2:
+            score += 0.2
+        if after and ("!!" in after or "??" in after):
+            score += 0.15
+        if token_text.isupper() and len(token_text) > 1:
+            score += 0.25
+        if EMOJI_REGEX.search(before or "") or EMOJI_REGEX.search(after or ""):
+            score += 0.15
+        # Clause-final emphasis
+        if index + 1 < len(tokens):
+            trailing = tokens[index + 1].text
+            if any(p in trailing for p in {"!!!", "??", "?!"}):
+                score += 0.2
+        return max(0.0, min(1.0, score))
+
+
+__all__ = [
+    "StretchabilityAnalyzer",
+    "StretchCandidate",
+    "StretchabilityFeatures",
+    "TokenInfo",
+    "RandomLike",
+]