glitchlings 0.4.4__cp313-cp313-manylinux_2_28_x86_64.whl

glitchlings/lexicon/__init__.py

@@ -0,0 +1,192 @@
+"""Lexicon abstractions and default backend resolution helpers."""
+
+from __future__ import annotations
+
+import random
+from abc import ABC, abstractmethod
+from hashlib import blake2s
+from pathlib import Path
+from typing import Callable, Iterable
+
+from glitchlings.config import get_config
+
+from ._cache import CacheEntries, CacheSnapshot
+
+
+class Lexicon(ABC):
+    """Abstract interface describing synonym lookup backends.
+
+    Parameters
+    ----------
+    seed:
+        Optional integer used to derive deterministic random number generators
+        for synonym sampling. Identical seeds guarantee reproducible results for
+        the same word/part-of-speech queries.
+
+    """
+
+    def __init__(self, *, seed: int | None = None) -> None:
+        self._seed = seed
+
+    @property
+    def seed(self) -> int | None:
+        """Return the current base seed used for deterministic sampling."""
+        return self._seed
+
+    def reseed(self, seed: int | None) -> None:
+        """Update the base seed driving deterministic synonym sampling."""
+        self._seed = seed
+
+    def _derive_rng(self, word: str, pos: str | None) -> random.Random:
+        """Return an RNG derived from the base seed, word, and POS tag."""
+        seed_material = blake2s(digest_size=8)
+        seed_material.update(word.lower().encode("utf8"))
+        if pos is not None:
+            seed_material.update(pos.lower().encode("utf8"))
+        seed_repr = "None" if self._seed is None else str(self._seed)
+        seed_material.update(seed_repr.encode("utf8"))
+        derived_seed = int.from_bytes(seed_material.digest(), "big", signed=False)
+        return random.Random(derived_seed)
+
+    def _deterministic_sample(
+        self, values: Iterable[str], *, limit: int, word: str, pos: str | None
+    ) -> list[str]:
+        """Return up to ``limit`` values sampled deterministically."""
+        if limit <= 0:
+            return []
+
+        items = list(values)
+        if len(items) <= limit:
+            return items
+
+        rng = self._derive_rng(word, pos)
+        indices = rng.sample(range(len(items)), k=limit)
+        indices.sort()
+        return [items[index] for index in indices]
+
+    @abstractmethod
+    def get_synonyms(self, word: str, pos: str | None = None, n: int = 5) -> list[str]:
+        """Return up to ``n`` synonyms for ``word`` constrained by ``pos``."""
+
+    def supports_pos(self, pos: str | None) -> bool:
+        """Return ``True`` when the backend can service ``pos`` queries."""
+        return True
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial representation
+        return f"{self.__class__.__name__}(seed={self._seed!r})"
+
+
+class LexiconBackend(Lexicon):
+    """Extended lexicon interface that supports cache persistence."""
+
+    Cache = CacheEntries
+
+    @classmethod
+    @abstractmethod
+    def load_cache(cls, path: str | Path) -> CacheSnapshot:
+        """Return a validated cache snapshot loaded from ``path``."""
+
+    @abstractmethod
+    def save_cache(self, path: str | Path | None = None) -> Path | None:
+        """Persist the backend cache to ``path`` and return the destination."""
+
+
+from .metrics import (  # noqa: E402
+    compare_lexicons,
+    coverage_ratio,
+    mean_cosine_similarity,
+    synonym_diversity,
+)
+from .vector import VectorLexicon, build_vector_cache  # noqa: E402
+
+_WordNetLexicon: type[LexiconBackend] | None
+try:  # pragma: no cover - optional dependency
+    from .wordnet import WordNetLexicon as _WordNetLexicon
+except (
+    ImportError,
+    ModuleNotFoundError,
+    AttributeError,
+):  # pragma: no cover - triggered when nltk unavailable
+    _WordNetLexicon = None
+
+WordNetLexicon: type[LexiconBackend] | None = _WordNetLexicon
+
+
+_BACKEND_FACTORIES: dict[str, Callable[[int | None], Lexicon | None]] = {}
+
+
+def register_backend(name: str, factory: Callable[[int | None], Lexicon | None]) -> None:
+    """Register ``factory`` for ``name`` so it can be selected via config."""
+    normalized = name.lower()
+    _BACKEND_FACTORIES[normalized] = factory
+
+
+def unregister_backend(name: str) -> None:
+    """Remove a previously registered backend."""
+    _BACKEND_FACTORIES.pop(name.lower(), None)
+
+
+def available_backends() -> list[str]:
+    """Return the names of registered lexicon factories."""
+    return sorted(_BACKEND_FACTORIES)
+
+
+def _vector_backend(seed: int | None) -> Lexicon | None:
+    config = get_config()
+    cache_path = config.lexicon.vector_cache
+    if cache_path is None:
+        return None
+    if not cache_path.exists():
+        return None
+    return VectorLexicon(cache_path=cache_path, seed=seed)
+
+
+def _wordnet_backend(seed: int | None) -> Lexicon | None:  # pragma: no cover - optional
+    if WordNetLexicon is None:
+        return None
+    try:
+        lexicon = WordNetLexicon(seed=seed)
+    except RuntimeError:
+        return None
+    return lexicon
+
+
+register_backend("vector", _vector_backend)
+register_backend("wordnet", _wordnet_backend)
+
+
+def get_default_lexicon(seed: int | None = None) -> Lexicon:
+    """Return the first available lexicon according to configuration priority."""
+    config = get_config()
+    attempts: list[str] = []
+    for name in config.lexicon.priority:
+        factory = _BACKEND_FACTORIES.get(name.lower())
+        if factory is None:
+            attempts.append(f"{name} (unknown)")
+            continue
+        lexicon = factory(seed)
+        if lexicon is not None:
+            return lexicon
+        attempts.append(f"{name} (unavailable)")
+    attempted = ", ".join(attempts) or "<none>"
+    raise RuntimeError(
+        "No lexicon backends available; configure lexicon.priority with at least one "
+        f"working backend. Attempts: {attempted}."
+    )
+
+
+__all__ = [
+    "Lexicon",
+    "LexiconBackend",
+    "VectorLexicon",
+    "WordNetLexicon",
+    "build_vector_cache",
+    "compare_lexicons",
+    "coverage_ratio",
+    "mean_cosine_similarity",
+    "synonym_diversity",
+    "get_default_lexicon",
+    "register_backend",
+    "unregister_backend",
+    "available_backends",
+]

glitchlings/lexicon/_cache.py

@@ -0,0 +1,110 @@
+"""Shared cache helpers for lexicon backends."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from hashlib import blake2s
+from pathlib import Path
+from typing import Mapping, Sequence, cast
+
+CacheEntries = dict[str, list[str]]
+
+
+@dataclass(frozen=True)
+class CacheSnapshot:
+    """Materialised cache data and its integrity checksum."""
+
+    entries: CacheEntries
+    checksum: str | None = None
+
+
+def _normalise_entries(payload: Mapping[str, object]) -> CacheEntries:
+    """Convert raw cache payloads into canonical mapping form."""
+    entries: CacheEntries = {}
+    for key, values in payload.items():
+        if not isinstance(key, str):
+            raise RuntimeError("Synonym cache keys must be strings.")
+        if not isinstance(values, Sequence):
+            raise RuntimeError("Synonym cache values must be sequences of strings.")
+        entries[key] = [str(value) for value in values]
+    return entries
+
+
+def _canonical_json(entries: Mapping[str, Sequence[str]]) -> str:
+    """Return a deterministic JSON serialisation for ``entries``."""
+    serialisable = {key: list(values) for key, values in sorted(entries.items())}
+    return json.dumps(serialisable, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+
+
+def compute_checksum(entries: Mapping[str, Sequence[str]]) -> str:
+    """Return a BLAKE2s checksum for ``entries``."""
+    digest = blake2s(_canonical_json(entries).encode("utf8"), digest_size=16)
+    return digest.hexdigest()
+
+
+def load_cache(path: Path) -> CacheSnapshot:
+    """Load a cache from ``path`` and verify its checksum if present."""
+    if not path.exists():
+        return CacheSnapshot(entries={}, checksum=None)
+
+    with path.open("r", encoding="utf8") as handle:
+        payload_obj = json.load(handle)
+
+    checksum: str | None = None
+    entries_payload: Mapping[str, object]
+
+    if not isinstance(payload_obj, Mapping):
+        raise RuntimeError("Synonym cache payload must be a mapping of strings to lists.")
+
+    payload = cast(Mapping[str, object], payload_obj)
+
+    if "__meta__" in payload and "entries" in payload:
+        meta_obj = payload["__meta__"]
+        entries_obj = payload["entries"]
+        if not isinstance(entries_obj, Mapping):
+            raise RuntimeError("Synonym cache entries must be stored as a mapping.")
+        entries_payload = cast(Mapping[str, object], entries_obj)
+        if isinstance(meta_obj, Mapping):
+            raw_checksum = meta_obj.get("checksum")
+            if raw_checksum is not None and not isinstance(raw_checksum, str):
+                raise RuntimeError("Synonym cache checksum must be a string when provided.")
+            checksum = raw_checksum if isinstance(raw_checksum, str) else None
+        else:
+            raise RuntimeError("Synonym cache metadata must be a mapping.")
+    else:
+        entries_payload = payload  # legacy format without metadata
+
+    entries = _normalise_entries(entries_payload)
+    if checksum is not None:
+        expected = compute_checksum(entries)
+        if checksum != expected:
+            raise RuntimeError(
+                "Synonym cache checksum mismatch; the cache file appears to be corrupted."
+            )
+
+    return CacheSnapshot(entries=entries, checksum=checksum)
+
+
+def write_cache(path: Path, entries: Mapping[str, Sequence[str]]) -> CacheSnapshot:
+    """Persist ``entries`` to ``path`` with checksum metadata."""
+    serialisable: CacheEntries = {
+        key: list(values) for key, values in sorted(entries.items())
+    }
+    checksum = compute_checksum(serialisable)
+    payload = {
+        "__meta__": {
+            "checksum": checksum,
+            "entries": len(serialisable),
+        },
+        "entries": serialisable,
+    }
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    with path.open("w", encoding="utf8") as handle:
+        json.dump(payload, handle, ensure_ascii=False, indent=2, sort_keys=True)
+
+    return CacheSnapshot(entries=serialisable, checksum=checksum)
+
+
+__all__ = ["CacheEntries", "CacheSnapshot", "compute_checksum", "load_cache", "write_cache"]

glitchlings/lexicon/data/default_vector_cache.json

@@ -0,0 +1,82 @@
+{
+  "alpha": [
+    "beta",
+    "gamma",
+    "delta"
+  ],
+  "beta": [
+    "alpha",
+    "gamma",
+    "delta"
+  ],
+  "delta": [
+    "alpha",
+    "beta",
+    "gamma"
+  ],
+  "fast": [
+    "rapid",
+    "swift",
+    "speedy",
+    "brisk"
+  ],
+  "gamma": [
+    "alpha",
+    "beta",
+    "delta"
+  ],
+  "happy": [
+    "glad",
+    "joyful",
+    "content",
+    "upbeat"
+  ],
+  "quick": [
+    "swift",
+    "rapid",
+    "speedy",
+    "nimble"
+  ],
+  "quickly": [
+    "swiftly",
+    "rapidly",
+    "promptly",
+    "speedily"
+  ],
+  "sing": [
+    "croon",
+    "serenade",
+    "vocalize",
+    "perform"
+  ],
+  "slow": [
+    "sluggish",
+    "leisurely",
+    "unhurried",
+    "gradual"
+  ],
+  "songs": [
+    "tracks",
+    "melodies",
+    "ballads",
+    "tunes"
+  ],
+  "text": [
+    "passage",
+    "copy",
+    "script",
+    "narrative"
+  ],
+  "they": [
+    "those people",
+    "those individuals",
+    "the group",
+    "those folks"
+  ],
+  "words": [
+    "terms",
+    "phrases",
+    "lexicon",
+    "vocabulary"
+  ]
+}

glitchlings/lexicon/metrics.py

@@ -0,0 +1,162 @@
+"""Utility helpers for evaluating lexicon coverage and quality."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable, Mapping, Sequence
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:  # pragma: no cover - typing hint only
+    from . import Lexicon
+
+
+def _unique_synonyms(
+    lexicon: "Lexicon",
+    word: str,
+    *,
+    pos: str | None,
+    sample_size: int,
+) -> list[str]:
+    """Return unique synonym candidates excluding the original token."""
+    collected: list[str] = []
+    seen: set[str] = set()
+    source = word.lower()
+    for synonym in lexicon.get_synonyms(word, pos=pos, n=sample_size):
+        normalized = synonym.lower()
+        if normalized == source:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        collected.append(synonym)
+    return collected
+
+
+def synonym_diversity(
+    lexicon: "Lexicon",
+    words: Iterable[str],
+    *,
+    pos: str | None = None,
+    sample_size: int = 5,
+) -> float:
+    """Return the mean unique-synonym count for ``words`` using ``lexicon``."""
+    totals = []
+    for word in words:
+        synonyms = _unique_synonyms(lexicon, word, pos=pos, sample_size=sample_size)
+        totals.append(len(synonyms))
+    if not totals:
+        return 0.0
+    return sum(totals) / len(totals)
+
+
+def coverage_ratio(
+    lexicon: "Lexicon",
+    words: Iterable[str],
+    *,
+    pos: str | None = None,
+    sample_size: int = 5,
+    min_synonyms: int = 3,
+) -> float:
+    """Return the fraction of ``words`` with at least ``min_synonyms`` candidates."""
+    total = 0
+    hits = 0
+    for word in words:
+        total += 1
+        synonyms = _unique_synonyms(lexicon, word, pos=pos, sample_size=sample_size)
+        if len(synonyms) >= min_synonyms:
+            hits += 1
+    if total == 0:
+        return 0.0
+    return hits / total
+
+
+def _cosine_similarity(vector_a: Sequence[float], vector_b: Sequence[float]) -> float:
+    dot = 0.0
+    norm_a = 0.0
+    norm_b = 0.0
+    for value_a, value_b in zip(vector_a, vector_b):
+        dot += value_a * value_b
+        norm_a += value_a * value_a
+        norm_b += value_b * value_b
+    magnitude = math.sqrt(norm_a) * math.sqrt(norm_b)
+    if magnitude == 0.0:
+        return 0.0
+    return dot / magnitude
+
+
+def mean_cosine_similarity(
+    lexicon: "Lexicon",
+    embeddings: Mapping[str, Sequence[float]],
+    words: Iterable[str],
+    *,
+    pos: str | None = None,
+    sample_size: int = 5,
+) -> float:
+    """Return the mean cosine similarity between each word and its candidates."""
+    total = 0.0
+    count = 0
+    for word in words:
+        source_vector = embeddings.get(word)
+        if source_vector is None:
+            continue
+        synonyms = _unique_synonyms(lexicon, word, pos=pos, sample_size=sample_size)
+        for synonym in synonyms:
+            synonym_vector = embeddings.get(synonym)
+            if synonym_vector is None:
+                continue
+            total += _cosine_similarity(source_vector, synonym_vector)
+            count += 1
+    if count == 0:
+        return 0.0
+    return total / count
+
+
+def compare_lexicons(
+    baseline: "Lexicon",
+    candidate: "Lexicon",
+    words: Iterable[str],
+    *,
+    pos: str | None = None,
+    sample_size: int = 5,
+    min_synonyms: int = 3,
+    embeddings: Mapping[str, Sequence[float]] | None = None,
+) -> dict[str, float]:
+    """Return comparative coverage and diversity statistics for two lexicons."""
+    stats = {
+        "baseline_diversity": synonym_diversity(baseline, words, pos=pos, sample_size=sample_size),
+        "candidate_diversity": synonym_diversity(
+            candidate, words, pos=pos, sample_size=sample_size
+        ),
+        "baseline_coverage": coverage_ratio(
+            baseline,
+            words,
+            pos=pos,
+            sample_size=sample_size,
+            min_synonyms=min_synonyms,
+        ),
+        "candidate_coverage": coverage_ratio(
+            candidate,
+            words,
+            pos=pos,
+            sample_size=sample_size,
+            min_synonyms=min_synonyms,
+        ),
+    }
+
+    if embeddings is not None:
+        stats["baseline_similarity"] = mean_cosine_similarity(
+            baseline, embeddings, words, pos=pos, sample_size=sample_size
+        )
+        stats["candidate_similarity"] = mean_cosine_similarity(
+            candidate, embeddings, words, pos=pos, sample_size=sample_size
+        )
+
+    return stats
+
+
+__all__ = [
+    "compare_lexicons",
+    "coverage_ratio",
+    "mean_cosine_similarity",
+    "synonym_diversity",
+]