pycorpdiff 0.1.0a0__py3-none-any.whl

pycorpdiff/semantic/alignment.py

@@ -0,0 +1,53 @@
+"""Vector-space alignment for diachronic embeddings.
+
+Reference
+---------
+Hamilton, W. L., Leskovec, J., & Jurafsky, D. (2016). Diachronic word
+embeddings reveal statistical laws of semantic change. In *Proceedings
+of ACL 2016*.
+
+Schönemann, P. H. (1966). A generalized solution of the orthogonal
+Procrustes problem. *Psychometrika*, 31(1), 1-10.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import numpy.typing as npt
+
+
+def procrustes_align(
+    source: npt.NDArray[np.float64],
+    target: npt.NDArray[np.float64],
+) -> npt.NDArray[np.float64]:
+    """Return ``source`` rotated to best match ``target`` via orthogonal Procrustes.
+
+    Implements Schönemann's closed-form solution: ``R = U V^T`` where
+    ``U Σ V^T = SVD(source^T target)``. The returned matrix is the
+    rotated source, not the rotation operator itself.
+
+    Both matrices must have the same shape ``(n, d)``. Rotation
+    preserves vector norms — if ``source`` is L2-normalised the
+    output rows are still on the unit sphere.
+
+    Use this when the two embedding spaces were trained independently
+    (e.g. Hamilton-style diachronic word2vec). Modern shared-model
+    encoders like SBERT already live in a common space, so alignment
+    is unnecessary.
+    """
+    if source.shape != target.shape:
+        raise ValueError(
+            f"source and target must have the same shape; got {source.shape} vs {target.shape}"
+        )
+    # Rank-deficient inputs (e.g. identical rows from a HashEmbedder
+    # encoding identical strings) make matmul emit a numpy 2.2.x
+    # divide-by-zero RuntimeWarning that isn't a real numerical
+    # problem — the SVD below handles the singular case correctly.
+    # Suppress the warning locally so strict filterwarnings doesn't
+    # promote it to an error.
+    with np.errstate(divide="ignore", invalid="ignore", over="ignore", under="ignore"):
+        m = source.T @ target
+        u, _, vt = np.linalg.svd(m, full_matrices=False)
+        rotation = u @ vt
+        rotated: npt.NDArray[np.float64] = source @ rotation
+    return rotated

pycorpdiff/semantic/embed.py

@@ -0,0 +1,84 @@
+"""Embedder protocol, a lazy SBERT default, and a deterministic test helper.
+
+The :class:`Embedder` protocol is the package's plug point for vector
+representations. Anything implementing
+``encode(terms: Sequence[str]) -> np.ndarray`` of shape ``(n, d)``
+satisfies it — a thin wrapper around gensim KeyedVectors, a HuggingFace
+pipeline, your own trained vectors, etc.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+import numpy as np
+import numpy.typing as npt
+
+
+@runtime_checkable
+class Embedder(Protocol):
+    """Anything callable that maps a sequence of strings to a 2-D vector array."""
+
+    def encode(self, terms: Sequence[str]) -> npt.NDArray[np.float64]: ...
+
+
+@dataclass
+class SBERTEmbedder:
+    """Default :class:`Embedder` backed by sentence-transformers.
+
+    sentence-transformers is in the optional ``semantic`` extra and is
+    imported lazily on first call to :meth:`encode` so the base
+    install does not pull torch transitively. The default model
+    ``all-MiniLM-L6-v2`` is ~22 MB; for non-English corpora prefer one
+    of the multilingual options (e.g. ``paraphrase-multilingual-MiniLM-L12-v2``).
+    """
+
+    model_name: str = "all-MiniLM-L6-v2"
+    # ``Any`` because sentence_transformers isn't a base dependency — the
+    # real type is :class:`sentence_transformers.SentenceTransformer` but
+    # we can't import it at module load time without breaking the
+    # "base install stays light" contract.
+    _model: Any = field(default=None, init=False, repr=False, compare=False)
+
+    def encode(self, terms: Sequence[str]) -> npt.NDArray[np.float64]:
+        if self._model is None:
+            try:
+                from sentence_transformers import SentenceTransformer
+            except ImportError as exc:
+                raise ImportError(
+                    "SBERTEmbedder requires sentence-transformers. "
+                    "Install with: pip install 'pycorpdiff[semantic]'"
+                ) from exc
+            object.__setattr__(self, "_model", SentenceTransformer(self.model_name))
+        vectors = self._model.encode(list(terms), convert_to_numpy=True)
+        return np.asarray(vectors, dtype=np.float64)
+
+
+@dataclass(frozen=True)
+class HashEmbedder:
+    """Deterministic seed-derived embedder for testing and offline demos.
+
+    Maps each input string to a vector by seeding a per-string RNG with
+    a SHA-256 digest of the string. Same input always yields the same
+    vector; different inputs yield uncorrelated unit vectors. This
+    isn't useful for *semantic* analysis — there's no signal beyond the
+    string equality — but it's perfect for verifying that the orchestrators
+    (averaging, alignment, neighborhood drift) wire up correctly without
+    paying the cost of a real model.
+    """
+
+    dim: int = 32
+
+    def encode(self, terms: Sequence[str]) -> npt.NDArray[np.float64]:
+        out = np.zeros((len(terms), self.dim), dtype=np.float64)
+        for i, term in enumerate(terms):
+            digest = hashlib.sha256(term.encode("utf-8")).digest()
+            seed = int.from_bytes(digest[:8], "big") & 0xFFFFFFFF
+            rng = np.random.default_rng(seed=seed)
+            v = rng.standard_normal(self.dim)
+            n = np.linalg.norm(v)
+            out[i] = v / n if n > 0 else v
+        return out

pycorpdiff/semantic/shift.py

@@ -0,0 +1,224 @@
+"""Semantic shift and neighborhood drift between corpora.
+
+The default strategy is *averaged contextual embeddings*: for each
+occurrence of the target term in a corpus, encode its surrounding
+window as a sentence, then average across occurrences. The
+corpus-specific representation that comes out is what we compare.
+
+This works with any shared-space embedder (SBERT, multilingual SBERT,
+HuggingFace encoders). For Hamilton-style independently-trained
+embeddings, supply ``align="procrustes"`` to rotate the source space
+onto the target space before comparison.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+import numpy as np
+import pandas as pd
+
+from ..corpus import Corpus, CorpusSlice
+from ..stats import cosine_similarity
+from .embed import Embedder, SBERTEmbedder
+
+AlignmentKind = Literal["none", "procrustes"]
+
+
+def _window_texts(
+    corpus: Corpus | CorpusSlice, target: str, window: int
+) -> list[str]:
+    """Extract every window around ``target`` as a space-joined string."""
+    docs_tokens = corpus.tokens()
+    out: list[str] = []
+    for tokens in docs_tokens:
+        for i, tok in enumerate(tokens):
+            if tok != target:
+                continue
+            lo = max(0, i - window)
+            hi = min(len(tokens), i + window + 1)
+            out.append(" ".join(tokens[lo:hi]))
+    return out
+
+
+def _centroid(vectors: np.ndarray[Any, Any]) -> np.ndarray[Any, Any]:
+    out: np.ndarray[Any, Any] = vectors.mean(axis=0)
+    return out
+
+
+def semantic_shift(
+    a: Corpus | CorpusSlice,
+    b: Corpus | CorpusSlice,
+    target: str | list[str],
+    embedder: Embedder | None = None,
+    window: int = 5,
+    align: AlignmentKind = "none",
+) -> pd.DataFrame:
+    """Embedding-space displacement of target term(s) between corpora.
+
+    Parameters
+    ----------
+    a, b
+        The two corpora (or slices) to compare.
+    target
+        One or more target terms.
+    embedder
+        Anything satisfying :class:`Embedder`. Defaults to
+        :class:`SBERTEmbedder` (lazy-loaded sentence-transformers).
+        Pass :class:`HashEmbedder` for deterministic offline demos.
+    window
+        Tokens of context on each side of the target.
+    align
+        ``"none"`` (default) assumes both corpora are embedded with the
+        same model and skips alignment. ``"procrustes"`` rotates the
+        source's window-vector matrix onto the target's via orthogonal
+        Procrustes — appropriate when the embedder produces independent
+        per-corpus spaces.
+
+    Returns
+    -------
+    pandas.DataFrame
+        One row per target term with ``cosine_similarity``,
+        ``cosine_distance``, ``n_contexts_a``, ``n_contexts_b``.
+    """
+    targets = [target] if isinstance(target, str) else list(target)
+    if embedder is None:
+        embedder = SBERTEmbedder()
+
+    rows: list[dict[str, object]] = []
+    for tgt in targets:
+        wins_a = _window_texts(a, tgt, window=window)
+        wins_b = _window_texts(b, tgt, window=window)
+        if not wins_a or not wins_b:
+            rows.append(
+                {
+                    "target": tgt,
+                    "cosine_similarity": float("nan"),
+                    "cosine_distance": float("nan"),
+                    "n_contexts_a": len(wins_a),
+                    "n_contexts_b": len(wins_b),
+                }
+            )
+            continue
+
+        vecs_a = np.asarray(embedder.encode(wins_a), dtype=np.float64)
+        vecs_b = np.asarray(embedder.encode(wins_b), dtype=np.float64)
+
+        if align == "procrustes":
+            # Procrustes wants two matrices of the same shape. Pad / truncate
+            # the smaller side to make this well-defined; downstream we only
+            # care about the rotated centroid.
+            from .alignment import procrustes_align
+
+            n = min(len(vecs_a), len(vecs_b))
+            vecs_a = procrustes_align(vecs_a[:n], vecs_b[:n])
+            vecs_b = vecs_b[:n]
+
+        centroid_a = _centroid(vecs_a)
+        centroid_b = _centroid(vecs_b)
+        sim = cosine_similarity(centroid_a, centroid_b)
+        rows.append(
+            {
+                "target": tgt,
+                "cosine_similarity": sim,
+                "cosine_distance": 1.0 - sim,
+                "n_contexts_a": len(wins_a),
+                "n_contexts_b": len(wins_b),
+            }
+        )
+    return pd.DataFrame(rows)
+
+
+def neighborhood_drift(
+    a: Corpus | CorpusSlice,
+    b: Corpus | CorpusSlice,
+    target: str,
+    k: int = 10,
+    embedder: Embedder | None = None,
+    window: int = 5,
+    min_count: int = 2,
+) -> pd.DataFrame:
+    """Compare the top-k contextual neighbours of ``target`` in each corpus.
+
+    For each candidate term that appears in the target's windows (in
+    either corpus) at least ``min_count`` times, computes a contextual
+    centroid in each corpus and ranks by cosine similarity to the
+    target's own centroid. Returns the union of the two top-k sets with
+    similarity in each corpus and a signed drift score.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Columns ``neighbor``, ``sim_a``, ``sim_b``, ``rank_a``,
+        ``rank_b``, ``drift = sim_a - sim_b``, ``status``
+        (``"shared"`` / ``"gained_in_a"`` / ``"lost_in_a"``).
+    """
+    if embedder is None:
+        embedder = SBERTEmbedder()
+
+    wins_a = _window_texts(a, target, window=window)
+    wins_b = _window_texts(b, target, window=window)
+    if not wins_a:
+        raise ValueError(f"target {target!r} not found in corpus a")
+    if not wins_b:
+        raise ValueError(f"target {target!r} not found in corpus b")
+
+    target_vec_a = np.asarray(embedder.encode(wins_a), dtype=np.float64).mean(axis=0)
+    target_vec_b = np.asarray(embedder.encode(wins_b), dtype=np.float64).mean(axis=0)
+
+    # Candidate vocabulary: words appearing in any window in either corpus,
+    # excluding the target itself.
+    candidates: set[str] = set()
+    for w in wins_a + wins_b:
+        candidates.update(w.split())
+    candidates.discard(target)
+
+    sims_a: dict[str, float] = {}
+    sims_b: dict[str, float] = {}
+    for cand in candidates:
+        cand_wins_a = _window_texts(a, cand, window=window)
+        cand_wins_b = _window_texts(b, cand, window=window)
+        if len(cand_wins_a) >= min_count:
+            cv_a = np.asarray(embedder.encode(cand_wins_a), dtype=np.float64).mean(axis=0)
+            sims_a[cand] = cosine_similarity(target_vec_a, cv_a)
+        if len(cand_wins_b) >= min_count:
+            cv_b = np.asarray(embedder.encode(cand_wins_b), dtype=np.float64).mean(axis=0)
+            sims_b[cand] = cosine_similarity(target_vec_b, cv_b)
+
+    top_a = sorted(sims_a.items(), key=lambda x: -x[1])[:k]
+    top_b = sorted(sims_b.items(), key=lambda x: -x[1])[:k]
+    rank_a = {term: i + 1 for i, (term, _) in enumerate(top_a)}
+    rank_b = {term: i + 1 for i, (term, _) in enumerate(top_b)}
+
+    union = set(rank_a) | set(rank_b)
+    rows: list[dict[str, object]] = []
+    for term in union:
+        s_a = sims_a.get(term, float("nan"))
+        s_b = sims_b.get(term, float("nan"))
+        in_a = term in rank_a
+        in_b = term in rank_b
+        if in_a and in_b:
+            status = "shared"
+        elif in_a:
+            status = "gained_in_a"
+        else:
+            status = "lost_in_a"
+        rows.append(
+            {
+                "neighbor": term,
+                "sim_a": s_a,
+                "sim_b": s_b,
+                "rank_a": rank_a.get(term),
+                "rank_b": rank_b.get(term),
+                "drift": (s_a if not np.isnan(s_a) else 0.0)
+                - (s_b if not np.isnan(s_b) else 0.0),
+                "status": status,
+            }
+        )
+    return (
+        pd.DataFrame(rows)
+        .assign(_abs=lambda d: d["drift"].abs())
+        .sort_values("_abs", ascending=False, kind="stable")
+        .drop(columns="_abs")
+        .reset_index(drop=True)
+    )

pycorpdiff/semantic/trajectory.py

@@ -0,0 +1,166 @@
+"""Multi-period semantic trajectories of target terms.
+
+Where :func:`semantic_shift` answers "how did the meaning of *target*
+differ between two corpora", :func:`semantic_trajectory` answers "how
+did the meaning of *target* evolve across a sequence of time periods".
+The algorithm is the per-period extension of averaged contextual
+embeddings:
+
+1. Slice the corpus by time period (yearly / quarterly / monthly).
+2. For each period, extract every KWIC window around the target.
+3. Encode each window via the embedder; average into a per-period
+   centroid.
+4. Pick a baseline period (the first populated period by default) and
+   report cosine similarity / distance from each period's centroid to
+   the baseline.
+
+The output is a tidy DataFrame ready to plot — periods on x, distance
+on y, one line per target if you tracked multiple at once.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ..corpus import Corpus, CorpusSlice
+from ..stats import cosine_similarity
+from .embed import Embedder, SBERTEmbedder
+from .shift import _window_texts
+
+
+def semantic_trajectory(
+    corpus: Corpus | CorpusSlice,
+    target: str | list[str],
+    time_col: str = "date",
+    freq: str = "Y",
+    embedder: Embedder | None = None,
+    window: int = 5,
+    baseline_period: str | pd.Period | None = None,
+) -> pd.DataFrame:
+    """Track each target's contextual centroid across time periods.
+
+    Parameters
+    ----------
+    corpus
+        A :class:`Corpus` or :class:`CorpusSlice` with a parseable
+        ``time_col``.
+    target
+        One or more target terms. For a single string, the output
+        carries one row per period; for a list, one row per
+        (period, term) pair.
+    time_col
+        Column carrying the document date.
+    freq
+        Pandas offset alias for the bucketing (``"Y"``, ``"Q"``, ``"M"``,
+        ``"W"``, ``"D"``).
+    embedder
+        Anything satisfying :class:`Embedder`. Defaults to
+        :class:`SBERTEmbedder` (lazy-loaded sentence-transformers).
+        Pass :class:`HashEmbedder` for deterministic offline demos.
+    window
+        Tokens of context on each side of the target.
+    baseline_period
+        Which period to anchor the trajectory at. Strings (e.g.
+        ``"2015"``) are coerced to :class:`pandas.Period` with the
+        ``freq`` above. Defaults to the *first populated* period for
+        each target — so two targets with different period coverage
+        each get their own appropriate baseline.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Columns ``period``, ``target``, ``n_contexts``,
+        ``similarity_to_baseline``, ``distance_from_baseline``.
+        Periods with no occurrences of the target carry NaN
+        similarity/distance and ``n_contexts == 0``.
+    """
+    if embedder is None:
+        embedder = SBERTEmbedder()
+
+    targets = [target] if isinstance(target, str) else list(target)
+    temporal = corpus.by_time(time_col, freq)
+    periods = temporal.periods()
+
+    # First pass: per-target, per-period centroids + context counts.
+    per_target: dict[str, dict[pd.Period, np.ndarray[Any, Any] | None]] = {}
+    per_target_counts: dict[str, dict[pd.Period, int]] = {}
+    for tgt in targets:
+        centroids: dict[pd.Period, np.ndarray[Any, Any] | None] = {}
+        counts: dict[pd.Period, int] = {}
+        for period, slice_ in temporal.iter_slices():
+            windows = _window_texts(slice_, tgt, window=window)
+            counts[period] = len(windows)
+            if not windows:
+                centroids[period] = None
+                continue
+            vecs = np.asarray(embedder.encode(windows), dtype=np.float64)
+            centroids[period] = vecs.mean(axis=0)
+        per_target[tgt] = centroids
+        per_target_counts[tgt] = counts
+
+    # Second pass: choose baseline, compute similarities, emit rows.
+    rows: list[dict[str, object]] = []
+    for tgt in targets:
+        centroids = per_target[tgt]
+        populated = [p for p in periods if centroids[p] is not None]
+        if not populated:
+            if baseline_period is not None:
+                raise ValueError(
+                    f"baseline_period={baseline_period!r} has no contexts "
+                    f"for target {tgt!r}"
+                )
+            # Target never appears anywhere — emit NaN rows for completeness.
+            for period in periods:
+                rows.append(
+                    {
+                        "period": period,
+                        "target": tgt,
+                        "n_contexts": 0,
+                        "similarity_to_baseline": float("nan"),
+                        "distance_from_baseline": float("nan"),
+                    }
+                )
+            continue
+
+        if baseline_period is None:
+            baseline = populated[0]
+        else:
+            baseline = (
+                pd.Period(baseline_period, freq=freq)
+                if isinstance(baseline_period, str)
+                else baseline_period
+            )
+            if baseline not in centroids or centroids[baseline] is None:
+                raise ValueError(
+                    f"baseline_period={baseline_period!r} has no contexts "
+                    f"for target {tgt!r}"
+                )
+
+        baseline_vec = centroids[baseline]
+        assert baseline_vec is not None  # for mypy
+        for period in periods:
+            vec = centroids[period]
+            if vec is None:
+                sim = float("nan")
+                dist = float("nan")
+            else:
+                sim = cosine_similarity(baseline_vec, vec)
+                dist = 1.0 - sim
+            rows.append(
+                {
+                    "period": period,
+                    "target": tgt,
+                    "n_contexts": per_target_counts[tgt][period],
+                    "similarity_to_baseline": sim,
+                    "distance_from_baseline": dist,
+                }
+            )
+
+    return (
+        pd.DataFrame(rows)
+        .sort_values(["target", "period"], kind="stable")
+        .reset_index(drop=True)
+    )

pycorpdiff/stats.py

@@ -0,0 +1,69 @@
+"""Small statistical utilities used across the package.
+
+Right now this is just the Wilson score interval — pycorpdiff prefers
+Wilson over Wald for proportion CIs because Wald collapses badly near
+``p = 0`` or ``p = 1``, which is exactly where temporal trajectories of
+rare terms live.
+
+Reference
+---------
+Wilson, E. B. (1927). Probable inference, the law of succession, and
+statistical inference. *Journal of the American Statistical
+Association*, 22(158), 209-212.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import numpy.typing as npt
+from scipy.special import ndtri
+
+
+def cosine_similarity(
+    a: npt.NDArray[np.float64], b: npt.NDArray[np.float64]
+) -> float:
+    """Cosine similarity between two 1-D vectors.
+
+    Returns 0 when either vector is the zero vector — the geometric
+    definition is undefined there, and zero is the conservative
+    "no relationship" default rather than NaN.
+    """
+    na = float(np.linalg.norm(a))
+    nb = float(np.linalg.norm(b))
+    if na == 0.0 or nb == 0.0:
+        return 0.0
+    return float(np.dot(a, b) / (na * nb))
+
+
+def wilson_ci(
+    x: int | npt.NDArray[np.int64],
+    n: int | npt.NDArray[np.int64],
+    confidence: float = 0.95,
+) -> tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]:
+    """Wilson score interval for a binomial proportion ``x / n``.
+
+    Returns ``(lower, upper)`` as float arrays (or scalars cast to
+    0-d arrays). When ``n == 0`` the interval is undefined and both
+    bounds come back as NaN. ``confidence`` must be in ``(0, 1)``;
+    defaults to 0.95 (z ≈ 1.96).
+    """
+    if not 0 < confidence < 1:
+        raise ValueError(f"confidence must be in (0, 1); got {confidence}")
+    # Two-sided z for the requested level: Φ^{-1}((1 + c) / 2).
+    z = float(ndtri((1.0 + confidence) / 2.0))
+
+    x_arr = np.asarray(x, dtype=np.float64)
+    n_arr = np.asarray(n, dtype=np.float64)
+
+    with np.errstate(divide="ignore", invalid="ignore"):
+        p = np.where(n_arr > 0, x_arr / n_arr, np.nan)
+        z2 = z * z
+        denom = 1.0 + z2 / n_arr
+        center = (p + z2 / (2.0 * n_arr)) / denom
+        margin = z * np.sqrt(p * (1.0 - p) / n_arr + z2 / (4.0 * n_arr * n_arr)) / denom
+
+    lower = np.where(n_arr > 0, center - margin, np.nan)
+    upper = np.where(n_arr > 0, center + margin, np.nan)
+    # Clip to [0, 1] — Wilson's interval is bounded mathematically, but
+    # roundoff at p ≈ 0 / p ≈ 1 can spill a hair past.
+    return np.clip(lower, 0.0, 1.0), np.clip(upper, 0.0, 1.0)

pycorpdiff/temporal/__init__.py

@@ -0,0 +1,15 @@
+"""Temporal slicing, rolling-window analysis, changepoint detection."""
+
+from __future__ import annotations
+
+from .changepoint import detect_changepoints
+from .its import interrupted_time_series
+from .slicing import TemporalCorpus, Tracker, track
+
+__all__ = [
+    "TemporalCorpus",
+    "Tracker",
+    "detect_changepoints",
+    "interrupted_time_series",
+    "track",
+]