spell-exploder 0.1.0__py3-none-any.whl

spell_exploder/core/entropy.py

@@ -0,0 +1,203 @@
+"""
+Entropy-based measures for text analysis.
+
+Provides Shannon entropy computation, windowed entropy collapse (measuring
+local redundancy relative to a document), and multiscale collapse curves
+that summarize redundancy structure across multiple window sizes.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+
+import numpy as np
+
+
+# ---------------------------------------------------------------------------
+# Shannon entropy
+# ---------------------------------------------------------------------------
+
+def shannon_entropy(counter: Counter) -> float:
+    """
+    Compute Shannon entropy (in bits) from a frequency counter.
+
+    Parameters
+    ----------
+    counter : Counter
+        Token → count mapping.
+
+    Returns
+    -------
+    float
+        Entropy in bits.  Returns ``0.0`` for an empty counter.
+
+    Notes
+    -----
+    .. math::
+        H = -\\sum_{i} p_i \\log_2 p_i
+    """
+    total = sum(counter.values())
+    if total == 0:
+        return 0.0
+    probs = np.array(list(counter.values()), dtype=float) / total
+    # Mask zeros to avoid log(0)
+    probs = probs[probs > 0]
+    return float(-(probs * np.log2(probs)).sum())
+
+
+# ---------------------------------------------------------------------------
+# Windowed entropy collapse
+# ---------------------------------------------------------------------------
+
+def window_collapse(
+    tokens: list[str],
+    win_size: int = 250,
+) -> list[float]:
+    """
+    Compute per-window *entropy collapse* values for non-overlapping windows.
+
+    Entropy collapse for a window is defined as the normalized deficit
+    of the window's entropy relative to the whole-document entropy:
+
+    .. math::
+        \\text{collapse}_w = \\frac{H_{\\text{doc}} - H_w}{H_{\\text{doc}}}
+
+    A value near 1 means the window is highly redundant (low local variety);
+    a value near 0 means the window is as diverse as the full document.
+
+    Parameters
+    ----------
+    tokens : list[str]
+        Full document token sequence.
+    win_size : int
+        Non-overlapping window width (in tokens).
+
+    Returns
+    -------
+    list[float]
+        One collapse value per window.  Tail chunks shorter than 2 tokens
+        are dropped.
+    """
+    h_doc = shannon_entropy(Counter(tokens))
+    if h_doc == 0:
+        return []
+
+    collapses: list[float] = []
+    for start in range(0, len(tokens), win_size):
+        chunk = tokens[start : start + win_size]
+        if len(chunk) < 2:
+            continue
+        h_chunk = shannon_entropy(Counter(chunk))
+        collapses.append((h_doc - h_chunk) / h_doc)
+
+    return collapses
+
+
+# ---------------------------------------------------------------------------
+# Multiscale collapse
+# ---------------------------------------------------------------------------
+
+_DEFAULT_WIN_SIZES: tuple[int, ...] = (25, 50, 100, 250, 500)
+
+
+def multiscale_collapse_curve(
+    tokens: list[str],
+    win_sizes: tuple[int, ...] = _DEFAULT_WIN_SIZES,
+) -> list[dict]:
+    """
+    Compute mean and max entropy collapse at multiple window sizes.
+
+    Parameters
+    ----------
+    tokens : list[str]
+        Full document token sequence.
+    win_sizes : tuple[int, ...]
+        Window widths to evaluate.
+
+    Returns
+    -------
+    list[dict]
+        One dict per window size with keys:
+        ``win_size``, ``n_windows``, ``mean_collapse``, ``max_collapse``.
+    """
+    curve: list[dict] = []
+    for w in win_sizes:
+        cs = window_collapse(tokens, win_size=w)
+        curve.append({
+            "win_size": int(w),
+            "n_windows": len(cs),
+            "mean_collapse": float(np.mean(cs)) if cs else float("nan"),
+            "max_collapse": float(np.max(cs)) if cs else float("nan"),
+        })
+    return curve
+
+
+def summarize_multiscale_collapse(
+    curve: list[dict],
+    x_scale: str = "log",
+) -> dict:
+    """
+    Summarize a multiscale collapse curve into scalar metrics.
+
+    Parameters
+    ----------
+    curve : list[dict]
+        Output of :func:`multiscale_collapse_curve`.
+    x_scale : str
+        ``"log"`` for log-scaled x-axis (window size) in AUC integration,
+        ``"linear"`` for raw window sizes.
+
+    Returns
+    -------
+    dict
+        ``collapse_auc`` — trapezoidal area under the mean-collapse curve.
+        ``collapse_auc_norm`` — AUC divided by x-range (average collapse across scales).
+        ``peak_win_size`` — window size with highest mean collapse.
+        ``peak_mean_collapse`` — that maximum value.
+    """
+    pts = [
+        (d["win_size"], d["mean_collapse"])
+        for d in curve
+        if not math.isnan(d["mean_collapse"])
+    ]
+
+    nan_result = dict(
+        collapse_auc=float("nan"),
+        collapse_auc_norm=float("nan"),
+        peak_win_size=None,
+        peak_mean_collapse=float("nan"),
+    )
+
+    if len(pts) == 0:
+        return nan_result
+
+    if len(pts) == 1:
+        w, m = pts[0]
+        return dict(
+            collapse_auc=0.0,
+            collapse_auc_norm=float(m),
+            peak_win_size=int(w),
+            peak_mean_collapse=float(m),
+        )
+
+    xs = np.array(
+        [math.log(w) if x_scale == "log" else float(w) for w, _ in pts],
+        dtype=float,
+    )
+    ys = np.array([m for _, m in pts], dtype=float)
+
+    # np.trapezoid in numpy 2.x; np.trapz in earlier versions
+    _trapz = getattr(np, "trapezoid", None) or np.trapz
+    auc = float(_trapz(ys, xs))
+    x_range = float(xs.max() - xs.min())
+    auc_norm = float(auc / x_range) if x_range > 0 else float(np.mean(ys))
+
+    peak_win, peak_mean = max(pts, key=lambda t: t[1])
+
+    return dict(
+        collapse_auc=auc,
+        collapse_auc_norm=auc_norm,
+        peak_win_size=int(peak_win),
+        peak_mean_collapse=float(peak_mean),
+    )

spell_exploder/core/information.py

@@ -0,0 +1,179 @@
+"""
+Information-theoretic measures for text analysis.
+
+Mutual information, channel capacity (Shannon–Hartley analogue),
+and Jensen–Shannon divergence — with N-text generalizations.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+
+import numpy as np
+from scipy.spatial.distance import jensenshannon
+
+
+# ---------------------------------------------------------------------------
+# Mutual information
+# ---------------------------------------------------------------------------
+
+def mutual_information(
+    joint: Counter,
+    marginal_x: Counter,
+    marginal_y: Counter,
+    n: int | None = None,
+) -> float:
+    """
+    Compute mutual information I(X; Y) from joint and marginal counters.
+
+    .. math::
+        I(X; Y) = \\sum_{x, y} p(x, y) \\log_2 \\frac{p(x, y)}{p(x)\\,p(y)}
+
+    Parameters
+    ----------
+    joint : Counter
+        Mapping of ``(x, y)`` pairs → counts.
+    marginal_x : Counter
+        Mapping of ``x`` → counts.
+    marginal_y : Counter
+        Mapping of ``y`` → counts.
+    n : int or None
+        Total number of observations.  When ``None``, the sum of *joint*
+        values is used.
+
+    Returns
+    -------
+    float
+        Mutual information in bits.  Returns ``0.0`` when *n* is 0.
+    """
+    if n is None:
+        n = sum(joint.values())
+    if n == 0:
+        return 0.0
+
+    mi = 0.0
+    for (x, y), count_xy in joint.items():
+        p_xy = count_xy / n
+        p_x = marginal_x[x] / n
+        p_y = marginal_y[y] / n
+        if p_xy > 0 and p_x > 0 and p_y > 0:
+            mi += p_xy * math.log2(p_xy / (p_x * p_y))
+
+    return mi
+
+
+# ---------------------------------------------------------------------------
+# Channel capacity (Shannon–Hartley analogue)
+# ---------------------------------------------------------------------------
+
+def channel_capacity(signal: float, noise: float) -> float:
+    """
+    Shannon–Hartley channel capacity with unit bandwidth.
+
+    .. math::
+        C = \\log_2(1 + S/N)
+
+    Parameters
+    ----------
+    signal : float
+        Signal power (e.g. token frequency).
+    noise : float
+        Noise power (e.g. total other-token frequency).
+
+    Returns
+    -------
+    float
+        Channel capacity in bits.
+    """
+    sn_ratio = signal / noise if noise > 0 else signal
+    sn_ratio = max(0.0, sn_ratio)
+    return math.log2(1 + sn_ratio)
+
+
+# ---------------------------------------------------------------------------
+# Jensen–Shannon divergence
+# ---------------------------------------------------------------------------
+
+def js_divergence_from_counters(
+    c1: Counter,
+    c2: Counter,
+) -> float:
+    """
+    Jensen–Shannon *divergence* between two frequency counters.
+
+    JS divergence is the square of the JS distance returned by
+    ``scipy.spatial.distance.jensenshannon``.
+
+    Parameters
+    ----------
+    c1, c2 : Counter
+        Token frequency counters.
+
+    Returns
+    -------
+    float
+        JS divergence.  Ranges from 0 (identical) to ``ln(2) ≈ 0.693``
+        for fully disjoint distributions (scipy uses natural log internally).
+        Returns ``NaN`` if either counter is empty.
+    """
+    vocab = sorted(set(c1.keys()) | set(c2.keys()))
+    p = np.array([c1.get(t, 0) for t in vocab], dtype=float)
+    q = np.array([c2.get(t, 0) for t in vocab], dtype=float)
+
+    if p.sum() <= 0 or q.sum() <= 0:
+        return float("nan")
+
+    js_dist = jensenshannon(p, q)
+    return float(js_dist ** 2)
+
+
+def js_distance_from_counters(c1: Counter, c2: Counter) -> float:
+    """
+    Jensen–Shannon *distance* (the square root of JS divergence).
+
+    Parameters
+    ----------
+    c1, c2 : Counter
+        Token frequency counters.
+
+    Returns
+    -------
+    float
+        JS distance in [0, 1].
+    """
+    vocab = sorted(set(c1.keys()) | set(c2.keys()))
+    p = np.array([c1.get(t, 0) for t in vocab], dtype=float)
+    q = np.array([c2.get(t, 0) for t in vocab], dtype=float)
+
+    if p.sum() <= 0 or q.sum() <= 0:
+        return float("nan")
+
+    return float(jensenshannon(p, q))
+
+
+def js_divergence_matrix(counters: list[Counter]) -> np.ndarray:
+    """
+    Compute the pairwise JS divergence matrix for *N* frequency counters.
+
+    Parameters
+    ----------
+    counters : list[Counter]
+        One frequency counter per document.
+
+    Returns
+    -------
+    np.ndarray
+        Symmetric N×N matrix where entry ``[i, j]`` is the JS divergence
+        between documents *i* and *j*.  Diagonal is 0.
+    """
+    n = len(counters)
+    mat = np.zeros((n, n), dtype=float)
+
+    for i in range(n):
+        for j in range(i + 1, n):
+            d = js_divergence_from_counters(counters[i], counters[j])
+            mat[i, j] = d
+            mat[j, i] = d
+
+    return mat

spell_exploder/core/nlp.py

@@ -0,0 +1,107 @@
+"""
+Shared NLP infrastructure: spaCy model management and tokenization.
+
+Models are loaded lazily (never at import time) and cached so that repeated
+calls with the same configuration reuse the same ``spacy.Language`` instance.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import spacy
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Model cache
+# ---------------------------------------------------------------------------
+_model_cache: dict[str, spacy.Language] = {}
+
+
+def get_nlp(
+    model_name: str = "en_core_web_sm",
+    disable: list[str] | None = None,
+) -> spacy.Language:
+    """
+    Load (and cache) a spaCy model.
+
+    Parameters
+    ----------
+    model_name : str
+        Any installed spaCy model name (e.g. ``"en_core_web_sm"``).
+    disable : list[str] or None
+        Pipeline components to disable (e.g. ``["ner"]``).
+
+    Returns
+    -------
+    spacy.Language
+        The loaded (or cached) pipeline.
+
+    Raises
+    ------
+    spell_exploder.exceptions.ModelNotLoadedError
+        If the requested model is not installed.
+    """
+    import spacy
+    from spell_exploder.exceptions import ModelNotLoadedError
+
+    disable = disable or []
+    cache_key = f"{model_name}|{','.join(sorted(disable))}"
+
+    if cache_key not in _model_cache:
+        try:
+            logger.debug("Loading spaCy model %r (disable=%s)", model_name, disable)
+            _model_cache[cache_key] = spacy.load(model_name, disable=disable)
+        except OSError as exc:
+            raise ModelNotLoadedError(
+                f"spaCy model '{model_name}' is not installed. "
+                f"Run: python -m spacy download {model_name}"
+            ) from exc
+
+    return _model_cache[cache_key]
+
+
+def clear_model_cache() -> None:
+    """Remove all cached models (useful in tests or to reclaim memory)."""
+    _model_cache.clear()
+
+
+# ---------------------------------------------------------------------------
+# Tokenization
+# ---------------------------------------------------------------------------
+
+def tokenize(
+    text: str,
+    model_name: str = "en_core_web_sm",
+    nlp: spacy.Language | None = None,
+) -> list[str]:
+    """
+    Tokenize *text* into lowercase lemmas, keeping only alphabetic tokens
+    and discarding stop-words.
+
+    This is the canonical tokenizer used across Spellcaster for
+    entropy, information-theoretic, and frequency-based analyses.
+
+    Parameters
+    ----------
+    text : str
+        Raw input text.
+    model_name : str
+        spaCy model to use (ignored when *nlp* is provided).
+    nlp : spacy.Language or None
+        Pre-loaded pipeline.  When ``None``, a model is loaded via
+        :func:`get_nlp` with the parser and NER disabled for speed.
+
+    Returns
+    -------
+    list[str]
+        Ordered list of lowercase lemma strings.
+    """
+    if nlp is None:
+        nlp = get_nlp(model_name, disable=["parser", "ner"])
+
+    doc = nlp(text.lower())
+    return [t.lemma_ for t in doc if t.is_alpha and not t.is_stop]

spell_exploder/exceptions.py

@@ -0,0 +1,25 @@
+"""Spellcaster exception hierarchy."""
+
+
+class SpellcasterError(Exception):
+    """Base exception for all spell_exploder errors."""
+
+
+class InsufficientDataError(SpellcasterError):
+    """Raised when input data is too small or empty for meaningful analysis."""
+
+
+class ModelNotLoadedError(SpellcasterError):
+    """Raised when a required NLP model is not available."""
+
+
+class OptionalDependencyError(SpellcasterError):
+    """Raised when an optional dependency is required but not installed."""
+
+    def __init__(self, package: str, extra: str):
+        self.package = package
+        self.extra = extra
+        super().__init__(
+            f"'{package}' is required for this feature. "
+            f"Install it with: pip install spell-exploder[{extra}]"
+        )

spell_exploder/extractors/__init__.py

@@ -0,0 +1,35 @@
+"""
+NLP extraction modules for Spellcaster.
+
+* :mod:`.action_frames` — Verb-centred action frame extraction.
+* :mod:`.noun_dependencies` — Schema–valence noun dependency triples.
+* :mod:`.sentence_parser` — Sentence segmentation with POS tags.
+"""
+
+from spell_exploder.extractors.action_frames import (
+    ActionFrame,
+    extract_action_frames,
+    make_hashable_frame,
+)
+from spell_exploder.extractors.noun_dependencies import (
+    NounDependency,
+    extract_noun_dependencies,
+)
+from spell_exploder.extractors.sentence_parser import (
+    DEFAULT_ABBREVIATIONS,
+    ParsedSentence,
+    parse_sentences,
+    split_sentences_simple,
+)
+
+__all__ = [
+    "ActionFrame",
+    "extract_action_frames",
+    "make_hashable_frame",
+    "NounDependency",
+    "extract_noun_dependencies",
+    "ParsedSentence",
+    "parse_sentences",
+    "split_sentences_simple",
+    "DEFAULT_ABBREVIATIONS",
+]

spell_exploder/extractors/action_frames.py

@@ -0,0 +1,133 @@
+"""
+Action frame extraction from text.
+
+An *action frame* is a verb-centred structure capturing who did what
+to whom, extracted from spaCy dependency parses.  Each frame records
+the verb lemma together with its nominal subjects, objects, and other
+syntactic dependents.
+"""
+
+from __future__ import annotations
+
+from typing import Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import spacy
+
+from spell_exploder.core.nlp import get_nlp
+
+
+# ---------------------------------------------------------------------------
+# Public types
+# ---------------------------------------------------------------------------
+
+# An action frame is a plain dict for now; typed as TypedDict for clarity.
+# Using a dict keeps it JSON-serializable and easy to work with in pandas.
+ActionFrame = dict[str, Any]
+"""
+Keys
+----
+verb : str
+    Verb lemma.
+subjects : list[str]
+    Nominal-subject lemmas (``nsubj``, ``nsubjpass``).
+objects : list[str]
+    Object / complement lemmas (``dobj``, ``pobj``, ``attr``,
+    ``ccomp``, ``xcomp``).
+other_deps : list[tuple[str, str]]
+    ``(dep_label, lemma)`` for all other children.
+"""
+
+# Dependency labels grouped by role
+_SUBJECT_DEPS = frozenset({"nsubj", "nsubjpass"})
+_OBJECT_DEPS = frozenset({"dobj", "pobj", "attr", "ccomp", "xcomp"})
+
+
+# ---------------------------------------------------------------------------
+# Extraction
+# ---------------------------------------------------------------------------
+
+def extract_action_frames(
+    text: str,
+    *,
+    nlp: spacy.Language | None = None,
+    model_name: str = "en_core_web_sm",
+) -> list[ActionFrame]:
+    """
+    Extract verb-centred action frames from *text*.
+
+    Parameters
+    ----------
+    text : str
+        Raw input text.
+    nlp : spacy.Language or None
+        Pre-loaded pipeline (must include the ``parser`` component).
+        When ``None``, a model is loaded via :func:`~spell_exploder.core.nlp.get_nlp`
+        with NER disabled for speed.
+    model_name : str
+        spaCy model name (used only when *nlp* is ``None``).
+
+    Returns
+    -------
+    list[ActionFrame]
+        One dict per verb token found in the text.
+
+    Examples
+    --------
+    >>> frames = extract_action_frames("The cat chased the mouse.")
+    >>> frames[0]["verb"]
+    'chase'
+    >>> frames[0]["subjects"]
+    ['cat']
+    >>> frames[0]["objects"]
+    ['mouse']
+    """
+    if nlp is None:
+        nlp = get_nlp(model_name, disable=["ner"])
+
+    doc = nlp(text)
+    frames: list[ActionFrame] = []
+
+    for tok in doc:
+        if tok.pos_ != "VERB":
+            continue
+
+        subjects = [
+            c.lemma_ for c in tok.children
+            if c.dep_.startswith("nsubj")
+        ]
+        objects = [
+            c.lemma_ for c in tok.children
+            if c.dep_ in _OBJECT_DEPS
+        ]
+        other_deps = [
+            (c.dep_, c.lemma_) for c in tok.children
+            if c.dep_ not in _SUBJECT_DEPS and c.dep_ not in _OBJECT_DEPS
+        ]
+
+        frames.append({
+            "verb": tok.lemma_,
+            "subjects": subjects,
+            "objects": objects,
+            "other_deps": other_deps,
+        })
+
+    return frames
+
+
+def make_hashable_frame(frame: ActionFrame) -> tuple:
+    """
+    Convert an action frame dict into a hashable tuple suitable for
+    counting in a :class:`~collections.Counter`.
+
+    Returns
+    -------
+    tuple
+        ``(verb, sorted_subjects, sorted_objects, sorted_other_deps)``
+    """
+    return (
+        frame["verb"],
+        tuple(sorted(frame["subjects"])),
+        tuple(sorted(frame["objects"])),
+        tuple(sorted(frame["other_deps"])),
+    )