pycorpdiff 0.1.0a0__py3-none-any.whl

pycorpdiff/keyness/effect_sizes.py

@@ -0,0 +1,65 @@
+"""Effect-size measures for corpus comparison.
+
+References
+----------
+Hardie, A. (2014). Log Ratio: An informal introduction. Centre for Corpus
+Approaches to Social Science (CASS).
+
+Gabrielatos, C. (2018). Keyness analysis: Nature, metrics and techniques.
+In *Corpus Approaches to Discourse* (pp. 225-258). Routledge.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+
+def log_ratio(
+    counts_a: pd.Series,
+    counts_b: pd.Series,
+    total_a: int,
+    total_b: int,
+    smoothing: float = 0.5,
+) -> pd.Series:
+    """Hardie's LogRatio: ``log2((a+α)/N_a) - log2((b+α)/N_b)``.
+
+    The Laplace ``smoothing`` constant (default 0.5, per Hardie) is added
+    to every count before normalisation so terms absent from one corpus
+    yield a finite (rather than ``±inf``) effect size. Positive LogRatio
+    means the term is more frequent in A; negative means more frequent in B.
+    """
+    if smoothing <= 0:
+        raise ValueError(f"smoothing must be > 0; got {smoothing}")
+
+    terms = counts_a.index.union(counts_b.index)
+    a = counts_a.reindex(terms, fill_value=0).astype(float)
+    b = counts_b.reindex(terms, fill_value=0).astype(float)
+    rate_a = (a + smoothing) / total_a
+    rate_b = (b + smoothing) / total_b
+    return pd.Series(np.log2(rate_a / rate_b), index=terms, name="log_ratio")
+
+
+def percent_diff(
+    counts_a: pd.Series,
+    counts_b: pd.Series,
+    total_a: int,
+    total_b: int,
+) -> pd.Series:
+    """%DIFF — Gabrielatos's normalised percentage difference.
+
+    Computed as ``(rate_a - rate_b) / rate_b * 100`` where rates are per
+    million tokens. The denominator's choice of per-million is convention
+    only; it cancels out of the ratio. Returns ``+inf`` when ``b == 0``
+    and ``a > 0`` (the term is novel in A) — the caller may wish to
+    filter or replace those rows for plotting.
+    """
+    terms = counts_a.index.union(counts_b.index)
+    a = counts_a.reindex(terms, fill_value=0).astype(float)
+    b = counts_b.reindex(terms, fill_value=0).astype(float)
+
+    rate_a = a / total_a * 1_000_000
+    rate_b = b / total_b * 1_000_000
+    with np.errstate(divide="ignore", invalid="ignore"):
+        diff = (rate_a - rate_b) / rate_b * 100.0
+    return pd.Series(diff, index=terms, name="percent_diff")

pycorpdiff/keyness/loglikelihood.py

@@ -0,0 +1,92 @@
+"""Dunning's G² log-likelihood statistic.
+
+Reference
+---------
+Dunning, T. (1993). Accurate methods for the statistics of surprise and
+coincidence. *Computational Linguistics*, 19(1), 61-74.
+
+Notes
+-----
+The G² returned by :func:`log_likelihood` is **signed**: positive when the
+term is overused in corpus A relative to corpus B (i.e. ``a/N_a > b/N_b``)
+and negative when overused in B. This is the convention CASS / Lancaster
+tooling has gravitated toward — it carries direction information without
+needing a separate column. The reported *p*-value uses ``|G²|`` as the
+test statistic; the unsigned form is what's chi-squared distributed.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from scipy.special import xlogy
+from scipy.stats import chi2
+
+
+def log_likelihood(
+    counts_a: pd.Series,
+    counts_b: pd.Series,
+    total_a: int,
+    total_b: int,
+) -> pd.DataFrame:
+    """Compute Dunning G² for every term in the union of input indices.
+
+    ``counts_a`` and ``counts_b`` are aligned on their union; missing
+    terms are imputed as zero. No min-count filtering is applied here —
+    that is the caller's responsibility (see
+    :meth:`pycorpdiff.Comparison.keyness`).
+
+    Parameters
+    ----------
+    counts_a, counts_b
+        Term-frequency series. Index entries are terms; values are
+        non-negative integer counts.
+    total_a, total_b
+        Corpus totals (token counts before any min-count filter). Used
+        for the contingency-table "not-term" cells.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Indexed by term, with columns ``count_a``, ``count_b``,
+        ``expected_a``, ``expected_b``, ``g2`` (signed), ``p_value``.
+    """
+    if total_a <= 0 or total_b <= 0:
+        raise ValueError(f"total_a and total_b must be positive; got {total_a}, {total_b}")
+
+    terms = counts_a.index.union(counts_b.index)
+    a = counts_a.reindex(terms, fill_value=0).astype(np.int64).to_numpy()
+    b = counts_b.reindex(terms, fill_value=0).astype(np.int64).to_numpy()
+
+    obs_sum = a + b
+    total = total_a + total_b
+    expected_a = total_a * obs_sum / total
+    expected_b = total_b * obs_sum / total
+
+    # 2 * sum_i O_i * ln(O_i / E_i), with xlogy giving 0*log(0)=0.
+    unsigned = 2.0 * (
+        xlogy(a, a) - xlogy(a, expected_a) + xlogy(b, b) - xlogy(b, expected_b)
+    )
+    # Mathematically G² >= 0; clip away the tiny negative values that
+    # surface from float roundoff when the two corpora have ~identical rates.
+    unsigned = np.maximum(unsigned, 0.0)
+
+    # Sign by direction of overuse: + when A's rate exceeds B's, else -.
+    a_rate = a / total_a
+    b_rate = b / total_b
+    sign = np.where(a_rate >= b_rate, 1.0, -1.0)
+    signed = sign * unsigned
+
+    p_value = chi2.sf(unsigned, df=1)
+
+    return pd.DataFrame(
+        {
+            "count_a": a,
+            "count_b": b,
+            "expected_a": expected_a,
+            "expected_b": expected_b,
+            "g2": signed,
+            "p_value": p_value,
+        },
+        index=terms,
+    )

pycorpdiff/keyness/multicorpus.py

@@ -0,0 +1,143 @@
+"""N-way keyness — does a term's rate differ across more than two corpora?
+
+The two-corpus :func:`log_likelihood` answers "is this term distinctive
+in A vs B". When you have three or more corpora — three news outlets,
+five parties, ten decades — the natural generalisation is a one-way
+contingency test: for each term, does its observed rate vary across
+corpora more than chance would explain?
+
+The reported G² uses the same marginal-only form as the two-way path
+(:func:`pycorpdiff.keyness.loglikelihood.log_likelihood`), so the N=2
+result agrees exactly with the existing keyness pipeline. Asymptotic
+distribution is χ²(df=N−1).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+import pandas as pd
+from scipy.special import xlogy
+from scipy.stats import chi2
+
+from ..corpus import Corpus, CorpusSlice
+
+
+def keyness_multi(
+    corpora: Sequence[Corpus | CorpusSlice],
+    *,
+    labels: Sequence[str] | None = None,
+    min_count: int = 5,
+    multiple_comparisons: str = "bh",
+    stop_words: set[str] | list[str] | None = None,
+) -> pd.DataFrame:
+    """Test for each term whether its rate varies across ``corpora``.
+
+    For ``N`` corpora and each shared-vocabulary term:
+
+    .. math::
+
+       G^2 = 2 \\sum_{i=1}^{N} O_i \\ln\\!\\left(\\frac{O_i}{E_i}\\right),
+       \\quad E_i = \\frac{(\\sum_j O_j)\\, n_i}{\\sum_j n_j}
+
+    where :math:`O_i` is the term's count in corpus *i* and :math:`n_i`
+    is corpus *i*'s total tokens. Asymptotic distribution is
+    :math:`\\chi^2(\\text{df}=N-1)`.
+
+    Parameters
+    ----------
+    corpora
+        Two or more corpora (or slices) to compare side-by-side.
+    labels
+        Optional labels for the corpora; default ``["corpus_0",
+        "corpus_1", ...]``. Used to name the per-corpus count columns.
+    min_count
+        Drop terms whose total count across all corpora is below this.
+    multiple_comparisons
+        ``"bh"`` (default), ``"bonferroni"``, or ``"none"``.
+    stop_words
+        Iterable of terms to exclude before scoring; same semantics as
+        the two-way :meth:`Comparison.keyness` parameter.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Indexed by term, sorted by G² descending. Columns: per-corpus
+        ``count_<label>``, ``g2``, ``p_value``, and
+        ``p_adjusted`` (unless ``multiple_comparisons="none"``).
+    """
+    if len(corpora) < 2:
+        raise ValueError(f"need at least 2 corpora; got {len(corpora)}")
+
+    if labels is None:
+        labels = [f"corpus_{i}" for i in range(len(corpora))]
+    if len(labels) != len(corpora):
+        raise ValueError(
+            f"labels must have one entry per corpus; "
+            f"got {len(labels)} labels for {len(corpora)} corpora"
+        )
+
+    vocabs = [c.vocab(min_count=1) for c in corpora]
+    totals = np.array([int(v.sum()) for v in vocabs], dtype=np.int64)
+    if (totals == 0).any():
+        which = [labels[i] for i, n in enumerate(totals) if n == 0]
+        raise ValueError(f"empty corpora: {which}")
+
+    all_terms = pd.Index([])
+    for v in vocabs:
+        all_terms = all_terms.union(v.index)
+
+    counts_matrix = np.zeros((len(all_terms), len(corpora)), dtype=np.int64)
+    for j, v in enumerate(vocabs):
+        aligned = v.reindex(all_terms, fill_value=0).astype(np.int64).to_numpy()
+        counts_matrix[:, j] = aligned
+
+    keep_mask = counts_matrix.sum(axis=1) >= min_count
+    if stop_words is not None:
+        stop_set = set(stop_words)
+        keep_mask &= ~all_terms.isin(stop_set)
+    kept_terms = all_terms[keep_mask]
+    kept_counts = counts_matrix[keep_mask]
+
+    if len(kept_terms) == 0:
+        empty = pd.DataFrame(
+            {f"count_{label}": pd.Series(dtype=np.int64) for label in labels}
+        )
+        empty["g2"] = pd.Series(dtype=float)
+        empty["p_value"] = pd.Series(dtype=float)
+        if multiple_comparisons != "none":
+            empty["p_adjusted"] = pd.Series(dtype=float)
+        return empty.rename_axis("term")
+
+    total_per_term = kept_counts.sum(axis=1).astype(np.float64)
+    grand_total = float(totals.sum())
+    expected = total_per_term[:, None] * totals[None, :] / grand_total
+    unsigned = 2.0 * (
+        xlogy(kept_counts, kept_counts) - xlogy(kept_counts, expected)
+    ).sum(axis=1)
+    unsigned = np.maximum(unsigned, 0.0)
+    p_value = chi2.sf(unsigned, df=len(corpora) - 1)
+
+    table = pd.DataFrame(
+        {f"count_{label}": kept_counts[:, j] for j, label in enumerate(labels)},
+        index=kept_terms,
+    )
+    table["g2"] = unsigned
+    table["p_value"] = p_value
+
+    if multiple_comparisons == "bh":
+        from .correction import benjamini_hochberg
+
+        table["p_adjusted"] = benjamini_hochberg(p_value)
+    elif multiple_comparisons == "bonferroni":
+        from .correction import bonferroni
+
+        table["p_adjusted"] = bonferroni(p_value)
+    elif multiple_comparisons != "none":
+        raise ValueError(
+            f"multiple_comparisons must be 'bh', 'bonferroni', or 'none'; "
+            f"got {multiple_comparisons!r}"
+        )
+
+    return table.sort_values("g2", ascending=False).rename_axis("term")

pycorpdiff/keyness/permutation.py

@@ -0,0 +1,154 @@
+"""Permutation-test *p*-values for keyness.
+
+The asymptotic χ²-based *p*-value reported by :func:`log_likelihood`
+relies on the large-sample approximation Dunning warned about in 1993:
+when expected cell counts are small (~< 5), the χ²(df=1) reference
+distribution stops being faithful. A permutation test sidesteps this
+entirely — it builds the null distribution empirically from the data
+itself by repeatedly shuffling document labels and recomputing G².
+
+The standard reference is Westfall & Young (1993), *Resampling-Based
+Multiple Testing*. The "+1/+1" small-sample correction in the *p*-value
+formula is Phipson & Smyth (2010); without it the smallest reportable
+*p* is 0, which is misleading for any finite *B*.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from scipy import sparse
+from scipy.special import xlogy
+
+from ..corpus import Corpus, CorpusSlice
+
+
+def permutation_pvalues(
+    a: Corpus | CorpusSlice,
+    b: Corpus | CorpusSlice,
+    *,
+    terms: pd.Index | list[str] | None = None,
+    n_permutations: int = 999,
+    seed: int | None = None,
+) -> pd.Series:
+    """Compute empirical permutation *p*-values for per-term keyness.
+
+    For each permutation, the documents from both corpora are pooled,
+    randomly relabelled into two groups preserving the original
+    document counts ``|docs(a)|`` and ``|docs(b)|``, and a per-term G²
+    is recomputed against the new marginal counts. The reported
+    *p*-value for each term is the Phipson–Smyth (2010) empirical
+    fraction::
+
+        p = (#perms with |G²_perm| ≥ |G²_observed| + 1) / (B + 1)
+
+    Documents (rather than tokens) are the unit of exchangeability —
+    this matches the null "speeches are exchangeable between frames"
+    that most corpus-linguistic two-sample questions are really
+    testing.
+
+    Parameters
+    ----------
+    a, b
+        The two corpora (or slices) to test.
+    terms
+        Optional restriction to a subset of vocabulary terms. If
+        ``None`` (default) every term in the union of vocabularies is
+        scored.
+    n_permutations
+        Number of label permutations. 999 is the conventional default
+        — coarse enough to be fast, fine enough that the smallest
+        attainable *p* (= 1/1000 = 0.001) is below the usual 0.01
+        screening threshold.
+    seed
+        Optional random seed for reproducibility.
+
+    Returns
+    -------
+    pandas.Series
+        Indexed by term, named ``p_permutation``. Values in [1/(B+1), 1].
+    """
+    if n_permutations < 1:
+        raise ValueError(f"n_permutations must be >= 1; got {n_permutations}")
+
+    dtm_a = a.doc_term_counts(min_count=1)
+    dtm_b = b.doc_term_counts(min_count=1)
+
+    all_terms = (
+        dtm_a.columns.union(dtm_b.columns) if terms is None else pd.Index(list(terms))
+    )
+
+    a_aligned = dtm_a.reindex(columns=all_terms, fill_value=0).astype(np.int64)
+    b_aligned = dtm_b.reindex(columns=all_terms, fill_value=0).astype(np.int64)
+
+    a_matrix = sparse.csr_matrix(a_aligned.to_numpy())
+    b_matrix = sparse.csr_matrix(b_aligned.to_numpy())
+    stacked = sparse.vstack([a_matrix, b_matrix]).tocsr()
+
+    n_docs_a = a_matrix.shape[0]
+    n_docs_total = stacked.shape[0]
+
+    if n_docs_a == 0 or n_docs_a == n_docs_total:
+        raise ValueError(
+            "permutation_pvalues needs at least one document on each side; "
+            f"got |docs(a)|={n_docs_a}, |docs(b)|={n_docs_total - n_docs_a}"
+        )
+
+    doc_lens = np.asarray(stacked.sum(axis=1)).ravel().astype(np.int64)
+    full_counts = np.asarray(stacked.sum(axis=0)).ravel().astype(np.int64)
+    total_n = int(doc_lens.sum())
+
+    if total_n == 0:
+        raise ValueError("permutation_pvalues needs at least one token across both corpora")
+
+    a_marginal = np.asarray(a_matrix.sum(axis=0)).ravel().astype(np.int64)
+    b_marginal = full_counts - a_marginal
+    n_a_observed = int(a_marginal.sum())
+    n_b_observed = total_n - n_a_observed
+    observed_g2 = _g2_unsigned(a_marginal, b_marginal, n_a_observed, n_b_observed)
+    observed_abs = np.abs(observed_g2)
+
+    rng = np.random.default_rng(seed)
+    extreme = np.zeros(len(all_terms), dtype=np.int64)
+
+    for _ in range(n_permutations):
+        perm = rng.permutation(n_docs_total)
+        idx_a = perm[:n_docs_a]
+        # Vectorised marginal sum for the shuffled "a" assignment.
+        marg_a_perm = np.asarray(stacked[idx_a, :].sum(axis=0)).ravel().astype(np.int64)
+        marg_b_perm = full_counts - marg_a_perm
+        n_a_perm = int(doc_lens[idx_a].sum())
+        n_b_perm = total_n - n_a_perm
+        if n_a_perm == 0 or n_b_perm == 0:
+            # Degenerate permutation — all-zero docs landed in one side.
+            # Treat as null contribution of zero (no terms exceed observed).
+            continue
+        g2_perm = _g2_unsigned(marg_a_perm, marg_b_perm, n_a_perm, n_b_perm)
+        extreme += np.abs(g2_perm) >= observed_abs
+
+    p_perm = (extreme + 1.0) / (n_permutations + 1.0)
+    return pd.Series(p_perm, index=all_terms, name="p_permutation")
+
+
+def _g2_unsigned(
+    a: np.ndarray,
+    b: np.ndarray,
+    n_a: int,
+    n_b: int,
+) -> np.ndarray:
+    """Vectorised unsigned G² for a vocabulary of terms.
+
+    Mirrors the math in :func:`pycorpdiff.keyness.loglikelihood.log_likelihood`
+    but operates on already-aligned numpy arrays and skips the sign
+    + p-value bookkeeping. Used internally by the permutation loop
+    where the per-iteration cost dominates.
+    """
+    obs_sum = a + b
+    total = n_a + n_b
+    expected_a = n_a * obs_sum / total
+    expected_b = n_b * obs_sum / total
+    unsigned = 2.0 * (
+        xlogy(a, a) - xlogy(a, expected_a) + xlogy(b, b) - xlogy(b, expected_b)
+    )
+    clipped: np.ndarray = np.maximum(unsigned, 0.0)
+    return clipped