pycorpdiff 0.1.0a0__py3-none-any.whl

pycorpdiff/io/duckdb.py

@@ -0,0 +1,92 @@
+"""Out-of-core corpus querying via DuckDB.
+
+DuckDB is in the optional ``duckdb`` extra. The reader is a thin
+shim that runs a SQL query and projects the result into a pandas
+DataFrame — DuckDB handles the heavy lifting (out-of-core scans of
+parquet, CSV, Arrow tables, SQLite, S3-hosted files) before the data
+ever touches pandas.
+
+Use this when your corpus is too large to fit in pandas comfortably
+but small enough that the rows you actually need fit after filtering.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..corpus import Corpus
+from ..tokenize import Tokenizer
+
+
+def read_duckdb(
+    connection: Any,
+    query: str,
+    text_col: str = "text",
+    id_col: str | None = None,
+    meta_cols: tuple[str, ...] = (),
+    tokenizer: Tokenizer | None = None,
+    params: list[Any] | dict[str, Any] | None = None,
+) -> Corpus:
+    """Run a SQL query against a DuckDB connection and wrap as a :class:`Corpus`.
+
+    Parameters
+    ----------
+    connection
+        A :class:`duckdb.DuckDBPyConnection` (the object returned by
+        ``duckdb.connect(...)``). Pass ``duckdb.connect()`` for an
+        in-memory database, or ``duckdb.connect("path/to/file.duckdb")``
+        for an on-disk one. DuckDB also accepts parquet / CSV / Arrow
+        directly in SQL via ``read_parquet('path')``.
+    query
+        SQL that returns rows; must include the text column named by
+        ``text_col``. Anything you can express in DuckDB SQL is fine —
+        filters, joins, aggregates — the only requirement is that the
+        final SELECT yields one row per document.
+    text_col
+        Name of the column containing document text. Default: ``"text"``.
+    id_col
+        Optional unique-document-id column.
+    meta_cols
+        Tuple of metadata column names to surface for slicing. If empty
+        (the default), every non-text column becomes metadata.
+    tokenizer
+        Optional :class:`Tokenizer`. Defaults to :class:`RegexTokenizer`.
+    params
+        Optional positional or named SQL parameters; forwarded to
+        :meth:`duckdb.DuckDBPyConnection.execute`.
+
+    Returns
+    -------
+    Corpus
+        Whose backing DataFrame is the result of the query.
+
+    Examples
+    --------
+    >>> import duckdb, pycorpdiff as pcd
+    >>> con = duckdb.connect()
+    >>> corpus = pcd.read_duckdb(           # doctest: +SKIP
+    ...     con,
+    ...     "SELECT body AS text, outlet, year FROM read_parquet('news/*.parquet') "
+    ...     "WHERE year >= 2020",
+    ... )
+    """
+    try:
+        import duckdb  # noqa: F401
+    except ImportError as exc:  # pragma: no cover
+        raise ImportError(
+            "read_duckdb requires duckdb. Install with: pip install 'pycorpdiff[duckdb]'"
+        ) from exc
+
+    cursor = connection.execute(query, params) if params is not None else connection.execute(query)
+    df = cursor.df()
+    if text_col not in df.columns:
+        raise ValueError(
+            f"text_col={text_col!r} not found in query result columns "
+            f"{list(df.columns)!r}"
+        )
+
+    from .readers import from_dataframe
+
+    return from_dataframe(
+        df, text_col=text_col, id_col=id_col, meta_cols=meta_cols, tokenizer=tokenizer
+    )

pycorpdiff/io/huggingface.py

@@ -0,0 +1,142 @@
+"""HuggingFace Datasets loader.
+
+The dominant modern source of public text corpora — Hansard mirrors,
+news datasets, social-media archives, academic corpora — is the
+HuggingFace `datasets` hub. This module wraps `datasets.load_dataset`
+in a thin shim that converts the result to a :class:`pycorpdiff.Corpus`.
+
+``datasets`` is heavy (pulls ``pyarrow``, ``fsspec``, ``requests``,
+``aiohttp``), so it lives in the optional ``huggingface`` extra:
+
+    pip install 'pycorpdiff[huggingface]'
+
+Then::
+
+    corpus = pcd.from_huggingface(
+        "stanfordnlp/imdb", split="train",
+        text_col="text", meta_cols=("label",),
+    )
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+import pandas as pd
+
+from ..corpus import Corpus
+from ..io.readers import from_dataframe
+from ..tokenize import Tokenizer
+
+
+def from_huggingface(
+    dataset_id: str,
+    *,
+    split: str = "train",
+    text_col: str = "text",
+    id_col: str | None = None,
+    meta_cols: tuple[str, ...] = (),
+    tokenizer: Tokenizer | None = None,
+    config_name: str | None = None,
+    columns: list[str] | None = None,
+    n_rows: int | None = None,
+    _loader: Callable[..., Any] | None = None,
+    **load_dataset_kwargs: Any,
+) -> Corpus:
+    """Load a HuggingFace dataset and wrap it as a :class:`Corpus`.
+
+    Parameters
+    ----------
+    dataset_id
+        The hub identifier — e.g. ``"stanfordnlp/imdb"``,
+        ``"openwebtext"``, ``"wikitext"``, or any private repo path
+        you have access to.
+    split
+        Which split to materialise — ``"train"`` (default),
+        ``"test"``, ``"validation"``, or a slice expression like
+        ``"train[:1000]"``.
+    text_col
+        Name of the column carrying document text in the dataset.
+    id_col
+        Optional unique-document-id column.
+    meta_cols
+        Tuple of metadata column names to surface for slicing. If
+        empty, every non-text column becomes metadata (matching the
+        :func:`from_dataframe` default).
+    tokenizer
+        Optional :class:`Tokenizer`.
+    config_name
+        HuggingFace's "name" parameter for multi-config datasets
+        (e.g. ``"wikitext-103-v1"`` for the ``wikitext`` dataset).
+    columns
+        Restrict materialisation to a subset of columns — useful when
+        the dataset has many fields you don't need.
+    n_rows
+        Materialise only the first ``n_rows`` documents. Equivalent
+        to passing ``split=f"{split}[:{n_rows}]"``; the explicit
+        parameter is just more discoverable.
+    _loader
+        Internal hook for unit tests; substitutes
+        :func:`datasets.load_dataset`.
+    **load_dataset_kwargs
+        Anything else gets forwarded to ``datasets.load_dataset``.
+
+    Examples
+    --------
+    >>> import pycorpdiff as pcd
+    >>> corpus = pcd.from_huggingface(                                  # doctest: +SKIP
+    ...     "stanfordnlp/imdb", split="train[:1000]",
+    ...     text_col="text", meta_cols=("label",),
+    ... )
+    >>> pos = corpus.slice(label=1); neg = corpus.slice(label=0)        # doctest: +SKIP
+    >>> pcd.compare(pos, neg).keyness().plot()                          # doctest: +SKIP
+    """
+    loader = _loader
+    if loader is None:
+        try:
+            from datasets import load_dataset as _hf_load  # type: ignore[import-not-found]
+        except ImportError as exc:  # pragma: no cover
+            raise ImportError(
+                "from_huggingface requires the `datasets` library. "
+                "Install with: pip install 'pycorpdiff[huggingface]'"
+            ) from exc
+        loader = _hf_load
+
+    effective_split = split if n_rows is None else f"{split}[:{int(n_rows)}]"
+
+    ds = loader(
+        dataset_id,
+        name=config_name,
+        split=effective_split,
+        **load_dataset_kwargs,
+    )
+
+    # The datasets library exposes Arrow Tables; the canonical conversion
+    # to pandas is .to_pandas(), which most dataset objects implement.
+    if hasattr(ds, "to_pandas"):
+        df = ds.to_pandas()
+    elif isinstance(ds, pd.DataFrame):
+        df = ds
+    else:
+        # Last-resort: iterate as dicts.
+        df = pd.DataFrame(list(ds))
+
+    if columns is not None:
+        # Keep just the requested columns (plus text_col if not listed).
+        keep = list(dict.fromkeys([text_col, *columns]))
+        df = df[[c for c in keep if c in df.columns]]
+
+    if text_col not in df.columns:
+        raise ValueError(
+            f"text_col={text_col!r} not found in dataset columns "
+            f"{list(df.columns)!r}"
+        )
+
+    return from_dataframe(
+        df,
+        text_col=text_col,
+        id_col=id_col,
+        meta_cols=meta_cols,
+        tokenizer=tokenizer,
+    )

pycorpdiff/io/readers.py

@@ -0,0 +1,138 @@
+"""Corpus readers — txt, csv, parquet, in-memory DataFrame."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from ..corpus import Corpus
+from ..tokenize import RegexTokenizer, Tokenizer
+
+
+def from_dataframe(
+    df: Any,
+    text_col: str = "text",
+    id_col: str | None = None,
+    meta_cols: tuple[str, ...] = (),
+    tokenizer: Tokenizer | None = None,
+) -> Corpus:
+    """Construct a :class:`Corpus` from an in-memory DataFrame.
+
+    Accepts either a :class:`pandas.DataFrame` or a
+    :class:`polars.DataFrame`. Polars input is converted to pandas
+    internally — the analytical layer is pandas-based, but the
+    constructor is symmetric so polars-native pipelines slot in
+    without explicit conversion.
+    """
+    if isinstance(df, pd.DataFrame):
+        df = df.reset_index(drop=True)
+    # else: Corpus.__post_init__ handles polars → pandas coercion.
+    return Corpus(
+        docs=df,
+        text_col=text_col,
+        id_col=id_col,
+        meta_cols=meta_cols,
+        tokenizer=tokenizer if tokenizer is not None else RegexTokenizer(),
+    )
+
+
+def read_csv(
+    path: str | Path,
+    text_col: str = "text",
+    id_col: str | None = None,
+    meta_cols: tuple[str, ...] = (),
+    tokenizer: Tokenizer | None = None,
+    **read_csv_kwargs: Any,
+) -> Corpus:
+    """Read a CSV file into a :class:`Corpus`.
+
+    Extra keyword arguments are forwarded to :func:`pandas.read_csv`.
+    """
+    df = pd.read_csv(path, **read_csv_kwargs)
+    return from_dataframe(
+        df, text_col=text_col, id_col=id_col, meta_cols=meta_cols, tokenizer=tokenizer
+    )
+
+
+def read_parquet(
+    path: str | Path,
+    text_col: str = "text",
+    id_col: str | None = None,
+    meta_cols: tuple[str, ...] = (),
+    tokenizer: Tokenizer | None = None,
+    use_polars: bool = False,
+    **read_parquet_kwargs: Any,
+) -> Corpus:
+    """Read a parquet file (or directory of parquet files) into a :class:`Corpus`.
+
+    Set ``use_polars=True`` to read via ``polars.read_parquet`` instead
+    of ``pandas.read_parquet`` — polars's parquet reader is often
+    several × faster on large files, particularly when only a subset of
+    columns is materialised. The result is converted to pandas
+    internally; the user-visible Corpus is identical either way.
+    Requires the ``polars`` extra.
+    """
+    if use_polars:
+        try:
+            import polars as pl
+        except ImportError as exc:  # pragma: no cover
+            raise ImportError(
+                "use_polars=True requires polars. Install with: "
+                "pip install 'pycorpdiff[polars]'"
+            ) from exc
+        df_pl = pl.read_parquet(path, **read_parquet_kwargs)
+        return from_dataframe(
+            df_pl, text_col=text_col, id_col=id_col, meta_cols=meta_cols, tokenizer=tokenizer
+        )
+    df = pd.read_parquet(path, **read_parquet_kwargs)
+    return from_dataframe(
+        df, text_col=text_col, id_col=id_col, meta_cols=meta_cols, tokenizer=tokenizer
+    )
+
+
+def read_txt(
+    path: str | Path,
+    encoding: str = "utf-8",
+    one_doc_per: str = "file",
+    tokenizer: Tokenizer | None = None,
+) -> Corpus:
+    """Read a single text file into a :class:`Corpus`.
+
+    Parameters
+    ----------
+    path
+        Path to a UTF-8 text file (override via ``encoding``).
+    one_doc_per
+        ``"file"`` treats the whole file as one document. ``"line"``
+        treats each non-empty line as its own document — useful for
+        per-line corpora like JSONL exports already projected to text,
+        or one-utterance-per-line transcripts.
+    tokenizer
+        Optional :class:`Tokenizer`. Defaults to :class:`RegexTokenizer`.
+
+    Returns
+    -------
+    Corpus
+        Has columns ``text``, ``source`` (the path), and — when
+        ``one_doc_per="line"`` — an integer ``line`` column with the
+        1-based line number so KWIC results can point back at the
+        original file.
+    """
+    if one_doc_per not in ("file", "line"):
+        raise ValueError(
+            f"one_doc_per must be 'file' or 'line'; got {one_doc_per!r}"
+        )
+    text = Path(path).read_text(encoding=encoding)
+    if one_doc_per == "file":
+        df = pd.DataFrame({"text": [text], "source": [str(path)]})
+    else:
+        lines = text.splitlines()
+        rows = [
+            {"text": line, "source": str(path), "line": i + 1}
+            for i, line in enumerate(lines)
+            if line.strip()
+        ]
+        df = pd.DataFrame(rows, columns=["text", "source", "line"])
+    return from_dataframe(df, text_col="text", tokenizer=tokenizer)

pycorpdiff/keyness/__init__.py

@@ -0,0 +1,26 @@
+"""Keyness measures — Dunning log-likelihood, LogRatio, Bayes factor."""
+
+from __future__ import annotations
+
+from .bayes import bayes_factor
+from .chi_squared import chi_squared
+from .correction import benjamini_hochberg, bonferroni
+from .dispersion import dispersion_dp, juilland_d
+from .effect_sizes import log_ratio, percent_diff
+from .loglikelihood import log_likelihood
+from .multicorpus import keyness_multi
+from .permutation import permutation_pvalues
+
+__all__ = [
+    "bayes_factor",
+    "benjamini_hochberg",
+    "bonferroni",
+    "chi_squared",
+    "dispersion_dp",
+    "juilland_d",
+    "keyness_multi",
+    "log_likelihood",
+    "log_ratio",
+    "percent_diff",
+    "permutation_pvalues",
+]

pycorpdiff/keyness/bayes.py

@@ -0,0 +1,50 @@
+"""Bayes factor keyness, BIC-based approximation.
+
+References
+----------
+Wilson, A. (2013). Embracing Bayes factors for key item analysis in
+corpus linguistics. In *New Approaches to the Study of Linguistic
+Variability* (pp. 3-11).
+
+Kass, R. E., & Raftery, A. E. (1995). Bayes factors. *Journal of the
+American Statistical Association*, 90(430), 773-795.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from .loglikelihood import log_likelihood
+
+
+def bayes_factor(
+    counts_a: pd.Series,
+    counts_b: pd.Series,
+    total_a: int,
+    total_b: int,
+) -> pd.Series:
+    """BIC-approximated Bayes factor for each term's frequency difference.
+
+    Uses Wilson's BIC approximation: ``BIC = |G²| - ln(N)`` where ``N``
+    is the total tokens across both corpora and ``G²`` is the unsigned
+    log-likelihood. The Bayes factor is then ``exp(BIC / 2)``.
+
+    Interpret with Kass & Raftery (1995):
+
+    - ``BF > 2``  : positive evidence
+    - ``BF > 6``  : strong evidence
+    - ``BF > 10`` : very strong evidence
+    - ``BF > 100``: decisive evidence
+
+    Very large BF values overflow float64 and surface as ``inf``; that is
+    semantically correct ("evidence is essentially conclusive") and pandas
+    plots / sorts handle it.
+    """
+    terms = counts_a.index.union(counts_b.index)
+    ll_table = log_likelihood(counts_a, counts_b, total_a, total_b)
+    g2_abs = ll_table["g2"].abs()
+    bic = g2_abs - np.log(total_a + total_b)
+    with np.errstate(over="ignore"):
+        bf = np.exp(bic / 2.0)
+    return pd.Series(bf, index=terms, name="bayes_factor")

pycorpdiff/keyness/chi_squared.py

@@ -0,0 +1,94 @@
+"""Pearson's χ² keyness statistic.
+
+The historical alternative to Dunning's G² for 2×2 corpus-comparison
+contingency tables. Both are asymptotically χ²(1)-distributed under
+the null of identical relative frequencies; G² is more robust to
+small expected counts (Dunning 1993), which is why pycorpdiff defaults
+to it. χ² is exposed here for the *test* of the equivalence and for
+researchers replicating older keyness-via-chi-squared studies.
+
+Reference
+---------
+Pearson, K. (1900). On the criterion that a given system of deviations
+from the probable in the case of a correlated system of variables is
+such that it can be reasonably supposed to have arisen from random
+sampling. *Philosophical Magazine*, 50(302), 157–175.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from scipy.stats import chi2
+
+
+def chi_squared(
+    counts_a: pd.Series,
+    counts_b: pd.Series,
+    total_a: int,
+    total_b: int,
+) -> pd.DataFrame:
+    """Compute Pearson χ² for every term in ``counts_a ∪ counts_b``.
+
+    Inputs and conventions mirror
+    :func:`pycorpdiff.keyness.log_likelihood`: caller is responsible for
+    min-count filtering; the returned ``chi_squared`` column is **signed**
+    by the direction of overuse (positive when A's rate exceeds B's),
+    while the *p*-value is computed from ``|χ²|``.
+
+    Parameters
+    ----------
+    counts_a, counts_b
+        Term-frequency series; missing terms imputed as zero on the union.
+    total_a, total_b
+        Corpus totals.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Indexed by term, columns ``count_a``, ``count_b``, ``expected_a``,
+        ``expected_b``, ``chi_squared`` (signed), ``p_value``.
+    """
+    if total_a <= 0 or total_b <= 0:
+        raise ValueError(f"total_a and total_b must be positive; got {total_a}, {total_b}")
+
+    terms = counts_a.index.union(counts_b.index)
+    # Cast to float64 throughout: the 2×2 numerator below is
+    # ``(ad − bc)² · N``, which overflows int64 for any realistic
+    # corpus size (a few hundred occurrences against a million-token
+    # corpus is enough).
+    a = counts_a.reindex(terms, fill_value=0).astype(np.float64).to_numpy()
+    b = counts_b.reindex(terms, fill_value=0).astype(np.float64).to_numpy()
+
+    obs_sum = a + b
+    total = float(total_a + total_b)
+    expected_a = float(total_a) * obs_sum / total
+    expected_b = float(total_b) * obs_sum / total
+
+    # 2×2 closed form: χ² = ((ad − bc)² · N) / ((a+b)(c+d)(a+c)(b+d))
+    # where c = N_a − a (non-term in A), d = N_b − b (non-term in B).
+    c = float(total_a) - a
+    d = float(total_b) - b
+    numerator = (a * d - b * c) ** 2 * total
+    denominator = obs_sum * (c + d) * float(total_a) * float(total_b)
+    with np.errstate(divide="ignore", invalid="ignore"):
+        unsigned = np.where(denominator > 0, numerator / denominator, 0.0)
+    unsigned = np.maximum(unsigned, 0.0)
+
+    a_rate = a / total_a
+    b_rate = b / total_b
+    sign = np.where(a_rate >= b_rate, 1.0, -1.0)
+    signed = sign * unsigned
+    p_value = chi2.sf(unsigned, df=1)
+
+    return pd.DataFrame(
+        {
+            "count_a": a,
+            "count_b": b,
+            "expected_a": expected_a,
+            "expected_b": expected_b,
+            "chi_squared": signed,
+            "p_value": p_value,
+        },
+        index=terms,
+    )

pycorpdiff/keyness/correction.py

@@ -0,0 +1,34 @@
+"""Multiple-comparison correction for keyness *p*-value vectors."""
+
+from __future__ import annotations
+
+import numpy as np
+import numpy.typing as npt
+
+
+def benjamini_hochberg(pvals: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]:
+    """Return Benjamini–Hochberg–adjusted *p*-values.
+
+    For each input *p*, the adjusted value is the minimum over the
+    rank-cumulative ``p_(k) * n / k`` from that rank rightward, clipped
+    to ``[0, 1]``. Order of the input is preserved.
+    """
+    pvals = np.asarray(pvals, dtype=np.float64)
+    n = pvals.size
+    if n == 0:
+        return pvals
+    order = np.argsort(pvals)
+    ranks = np.arange(1, n + 1)
+    raw = pvals[order] * n / ranks
+    # Cumulative minimum from the right enforces monotonicity.
+    monotone = np.minimum.accumulate(raw[::-1])[::-1]
+    monotone = np.clip(monotone, 0.0, 1.0)
+    out = np.empty(n, dtype=np.float64)
+    out[order] = monotone
+    return out
+
+
+def bonferroni(pvals: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]:
+    """Bonferroni-corrected *p*-values: ``min(p * n, 1)`` elementwise."""
+    pvals = np.asarray(pvals, dtype=np.float64)
+    return np.clip(pvals * pvals.size, 0.0, 1.0)

pycorpdiff/keyness/dispersion.py

@@ -0,0 +1,89 @@
+"""Dispersion measures for corpus-comparison sanity checks.
+
+A term can be "key" (significant + large effect) simply because one
+document overuses it. Reporting dispersion alongside keyness lets the
+caller filter out these spurious findings.
+
+References
+----------
+Juilland, A., & Chang-Rodríguez, E. (1964). *Frequency Dictionary of
+Spanish Words*. Mouton.
+
+Gries, S. Th. (2008). Dispersions and adjusted frequencies in corpora.
+*International Journal of Corpus Linguistics*, 13(4), 403-437.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+
+def juilland_d(doc_term_matrix: pd.DataFrame) -> pd.Series:
+    """Juilland's D — a 0..1 dispersion score; higher is more even.
+
+    Assumes the rows of ``doc_term_matrix`` are equally weighted parts
+    (i.e. treats each document as one "part"). For arbitrarily-sized
+    parts, aggregate to fixed-size buckets first.
+
+    Per-term D = ``1 - CV / sqrt(k - 1)``, where CV is the coefficient
+    of variation of the term's per-document relative frequencies and
+    ``k`` is the number of documents. D = 1 means perfectly even
+    spread; D = 0 means concentrated in one document.
+
+    Edge cases: when ``k == 1`` the formula is undefined and we return
+    NaN. When a term's count is zero everywhere the per-document rates
+    are all zero and we return 0 (no spread).
+    """
+    k = len(doc_term_matrix)
+    if k <= 1:
+        return pd.Series(np.nan, index=doc_term_matrix.columns, name="juilland_d")
+
+    counts = doc_term_matrix.to_numpy(dtype=float)  # (k, V)
+    doc_totals = counts.sum(axis=1)  # (k,)
+    # Per-document relative frequencies. Empty documents contribute zero
+    # rate everywhere (avoid divide-by-zero with a safe denominator).
+    safe_totals = np.where(doc_totals > 0, doc_totals, 1.0)
+    rates = counts / safe_totals[:, None]
+    rates = np.where(doc_totals[:, None] > 0, rates, 0.0)
+
+    mean = rates.mean(axis=0)
+    std = rates.std(axis=0, ddof=0)
+    with np.errstate(divide="ignore", invalid="ignore"):
+        cv = np.where(mean > 0, std / mean, 0.0)
+    d = np.where(mean > 0, 1.0 - cv / np.sqrt(k - 1), 0.0)
+    return pd.Series(d, index=doc_term_matrix.columns, name="juilland_d")
+
+
+def dispersion_dp(doc_term_matrix: pd.DataFrame) -> pd.Series:
+    """Gries's DP (Deviation of Proportions) — 0..1; lower is more even.
+
+    For each document ``i`` with size ``s_i`` (in tokens) and target-term
+    count ``c_i``, let ``expected_i = s_i / S`` (the document's share of
+    the corpus) and ``observed_i = c_i / C`` (the document's share of the
+    target's occurrences). Then ``DP = 0.5 * Σ |observed_i - expected_i|``.
+
+    DP = 0 means perfectly even spread; DP near 1 means total
+    concentration. We return the unnormalised form (Gries 2008 §3); the
+    normalised variant ``DPnorm = DP / (1 - min(expected_i))`` is a
+    one-line transformation if needed.
+    """
+    if len(doc_term_matrix) == 0:
+        return pd.Series(dtype=float, name="dispersion_dp")
+
+    doc_sizes = doc_term_matrix.sum(axis=1).astype(float).to_numpy()
+    s_total = doc_sizes.sum()
+    if s_total == 0:
+        return pd.Series(np.nan, index=doc_term_matrix.columns, name="dispersion_dp")
+    expected = doc_sizes / s_total  # shape (k,)
+
+    term_totals = doc_term_matrix.sum(axis=0).astype(float)
+    counts = doc_term_matrix.astype(float).to_numpy()  # shape (k, V)
+
+    with np.errstate(divide="ignore", invalid="ignore"):
+        observed = counts / term_totals.to_numpy()  # shape (k, V), broadcasts
+    # Terms with zero total get observed = NaN/Inf → replace with expected
+    # so |observed - expected| = 0 (a uniformly-absent term has trivial DP).
+    observed = np.where(term_totals.to_numpy() > 0, observed, expected[:, None])
+    dp = 0.5 * np.abs(observed - expected[:, None]).sum(axis=0)
+    return pd.Series(dp, index=doc_term_matrix.columns, name="dispersion_dp")