pycorpdiff 0.1.0a5__tar.gz → 0.1.0a6__tar.gz

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/CHANGELOG.md

@@ -4,7 +4,7 @@ All notable changes to `pycorpdiff` are documented in this file. The format
 follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this
 project adheres to [Semantic Versioning](https://semver.org/).
 
-## [0.1.0a5] — initial release
+## [0.1.0a6] — initial release
 
 The initial public release of `pycorpdiff` — comparative corpus analysis
 for modern Python workflows. Three public verbs (`compare`, `track`,

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/CITATION.cff

@@ -4,7 +4,7 @@ message: >
   entry. GitHub renders a "Cite this repository" widget directly from
   this file.
 title: "pycorpdiff: Comparative Corpus Analysis for Modern Python Workflows"
-version: 0.1.0a5
+version: 0.1.0a6
 date-released: 2026-05-25
 authors:
   - family-names: Turner
@@ -32,7 +32,9 @@ abstract: >
   API. The package targets corpus linguistics, digital humanities,
   computational social science, and discourse analysis research,
   emphasising interpretability, explainability, statistical rigour,
-  and reproducibility.
+  and reproducibility. A bundled synthetic UK-Hansard-style sample
+  ships for offline demonstration; real-data interfaces include
+  fetch_hansard and from_huggingface.
 identifiers:
   - type: url
     value: "https://github.com/jturner-uofl/pycorpdiff"

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pycorpdiff
-Version: 0.1.0a5
+Version: 0.1.0a6
 Summary: Comparative corpus analysis for Python: keyness, collocations, semantic shift, temporal trajectories with changepoints + causal inference.
 Project-URL: Homepage, https://github.com/jturner-uofl/pycorpdiff
 Project-URL: Documentation, https://github.com/jturner-uofl/pycorpdiff
@@ -131,7 +131,7 @@ points — one-line adapters, no plugin registry. The base install pulls
 only `numpy`, `pandas`, `scipy`, and `pyarrow`; everything else is opt-in
 via extras.
 
-> **Status: alpha (0.1.0a5).** Public API is stable for the features
+> **Status: alpha (0.1.0a6).** Public API is stable for the features
 > described below; on PyPI as `pip install pycorpdiff`.
 
 ## The three-layer architecture
@@ -237,7 +237,7 @@ The math agrees with the standard tools — by automated test:
 - **Rayson's LL Wizard** — hand-derived contingency-table reference triples
 - **NLTK** `BigramAssocMeasures` — PMI + t-score to ≤ 1e-12 on every adjacent bigram
 - **Scattertext (Kessler 2017)** — behavioural agreement on the 2012 US Conventions corpus
-- **quanteda (R)** via `rpy2` — byte-for-byte G² agreement (slow tier)
+- **quanteda (R)** via `rpy2` — byte-for-byte G² agreement with `formula="dunning"` (slow tier)
 - **HistWords (Hamilton et al. 2016)** — diachronic cosine displacements on COHA (slow tier)
 
 ## Citation

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/README.md

@@ -35,7 +35,7 @@ points — one-line adapters, no plugin registry. The base install pulls
 only `numpy`, `pandas`, `scipy`, and `pyarrow`; everything else is opt-in
 via extras.
 
-> **Status: alpha (0.1.0a5).** Public API is stable for the features
+> **Status: alpha (0.1.0a6).** Public API is stable for the features
 > described below; on PyPI as `pip install pycorpdiff`.
 
 ## The three-layer architecture
@@ -141,7 +141,7 @@ The math agrees with the standard tools — by automated test:
 - **Rayson's LL Wizard** — hand-derived contingency-table reference triples
 - **NLTK** `BigramAssocMeasures` — PMI + t-score to ≤ 1e-12 on every adjacent bigram
 - **Scattertext (Kessler 2017)** — behavioural agreement on the 2012 US Conventions corpus
-- **quanteda (R)** via `rpy2` — byte-for-byte G² agreement (slow tier)
+- **quanteda (R)** via `rpy2` — byte-for-byte G² agreement with `formula="dunning"` (slow tier)
 - **HistWords (Hamilton et al. 2016)** — diachronic cosine displacements on COHA (slow tier)
 
 ## Citation

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pycorpdiff"
-version = "0.1.0a5"
+version = "0.1.0a6"
 description = "Comparative corpus analysis for Python: keyness, collocations, semantic shift, temporal trajectories with changepoints + causal inference."
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -176,6 +176,8 @@ disallow_any_generics = true
 module = [
     "altair",
     "altair.*",
+    "datasets",
+    "datasets.*",
     "duckdb",
     "duckdb.*",
     "matplotlib",

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/src/pycorpdiff/__init__.py

@@ -14,12 +14,12 @@ Example
 
 >>> import pycorpdiff as pcd
 >>> pcd.__version__
-'0.1.0a5'
+'0.1.0a6'
 """
 
 from __future__ import annotations
 
-__version__ = "0.1.0a5"
+__version__ = "0.1.0a6"
 
 from .collocation.network import NetworkResult, cooccurrence_network
 from .compare import Comparison, compare

pycorpdiff-0.1.0a6/src/pycorpdiff/_backends/pandas.py

@@ -0,0 +1,9 @@
+"""Pandas-backed internals for :class:`pycorpdiff.Corpus`.
+
+Corpus operations route through this module so backend-specific code
+stays out of the public API. The pandas backend is the default and is
+exercised on every install; polars is opt-in via the ``polars`` extra
+and lives in the sibling :mod:`pycorpdiff._backends.polars`.
+"""
+
+from __future__ import annotations

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/src/pycorpdiff/compare.py

@@ -10,6 +10,7 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Literal
 
 from .corpus import Corpus, CorpusSlice
+from .keyness.loglikelihood import LLFormula
 
 if TYPE_CHECKING:
     from .results import (
@@ -46,6 +47,7 @@ class Comparison:
     def keyness(
         self,
         method: KeynessMethod = "log_likelihood",
+        formula: LLFormula = "rayson",
         effect_size: bool = True,
         dispersion: bool = False,
         min_count: int = 5,
@@ -64,6 +66,14 @@ class Comparison:
             sorts by signed Pearson χ². The other modes
             (``"log_ratio"``, ``"bayes_factor"``, ``"percent_diff"``)
             require ``effect_size=True`` and sort by that column.
+        formula
+            Which log-likelihood formulation to use for the G² column.
+            ``"rayson"`` (default) is the 2-cell shortcut matching
+            Rayson's UCREL LL Wizard; ``"dunning"`` is the full 4-cell
+            G² matching NLTK's ``BigramAssocMeasures`` and R's
+            ``quanteda::textstat_keyness(measure="lr")``. See
+            ``docs/statistical-methods.md`` for the math + when they
+            diverge.
         effect_size
             If True (default), also compute LogRatio (Hardie),
             %DIFF (Gabrielatos), and the BIC-approximated Bayes factor.
@@ -131,7 +141,7 @@ class Comparison:
         # G² is always computed (cheap, the default sort column). χ² is
         # computed only when requested — same shape, asymptotically
         # equivalent, no need to pay for both by default.
-        table = log_likelihood(a_kept, b_kept, n_a, n_b)
+        table = log_likelihood(a_kept, b_kept, n_a, n_b, formula=formula)
         if method == "chi_squared":
             chi_table = _chi_squared(a_kept, b_kept, n_a, n_b)
             table["chi_squared"] = chi_table["chi_squared"]
@@ -192,6 +202,7 @@ class Comparison:
             label_a=_corpus_label(self.a),
             label_b=_corpus_label(self.b),
             params={
+                "formula": formula,
                 "effect_size": effect_size,
                 "dispersion": dispersion,
                 "min_count": min_count,

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/src/pycorpdiff/io/huggingface.py

@@ -95,7 +95,7 @@ def from_huggingface(
     loader = _loader
     if loader is None:
         try:
-            from datasets import load_dataset as _hf_load  # type: ignore[import-not-found]
+            from datasets import load_dataset as _hf_load
         except ImportError as exc:  # pragma: no cover
             raise ImportError(
                 "from_huggingface requires the `datasets` library. "

pycorpdiff-0.1.0a6/src/pycorpdiff/keyness/loglikelihood.py

@@ -0,0 +1,149 @@
+"""Log-likelihood-ratio keyness statistic (Dunning, Rayson formulations).
+
+References
+----------
+Dunning, T. (1993). Accurate methods for the statistics of surprise and
+coincidence. *Computational Linguistics*, 19(1), 61-74.
+
+Rayson, P., & Garside, R. (2000). Comparing corpora using frequency
+profiling. In *Proceedings of the Workshop on Comparing Corpora*,
+ACL 2000, pp. 1-6.
+
+Notes
+-----
+Two slightly different log-likelihood-ratio statistics circulate in the
+corpus-linguistics literature for the 2-corpus / 2-state contingency
+table:
+
+- **Rayson** (``formula="rayson"``, default): the 2-cell shortcut
+  ``LL = 2·(O₁·ln(O₁/E₁) + O₂·ln(O₂/E₂))``, summing only the
+  *term-present* cells. This is the formulation behind the
+  UCREL Lancaster LL Wizard (the standard online reference at
+  http://ucrel.lancs.ac.uk/llwizard.html), originally published in
+  Rayson & Garside (2000).
+
+- **Dunning** (``formula="dunning"``): the full 4-cell G²,
+  summing over *all four* cells of the 2×2 contingency table —
+  ``LL = 2·Σᵢ₌₁..₄ Oᵢ·ln(Oᵢ/Eᵢ)``. This is the classical
+  Dunning (1993) likelihood-ratio statistic and the formulation used
+  by NLTK's :class:`BigramAssocMeasures` and R's
+  :func:`quanteda::textstat_keyness(measure="lr")`.
+
+For the canonical Rayson example (12000/1M vs 10000/1M), the two
+formulations give 182.07 (Rayson) and 184.10 (Dunning). For the
+near-symmetric, low-frequency cases that dominate corpus-linguistics
+practice the two are typically within 1-2 % of each other; for highly
+asymmetric corpora or high-frequency terms they diverge more.
+
+The statistic returned by :func:`log_likelihood` is **signed**: positive
+when the term is overused in corpus A relative to corpus B (i.e.
+``a/N_a > b/N_b``) and negative when overused in B. This is the
+convention CASS / Lancaster tooling has gravitated toward — it carries
+direction information without needing a separate column. The reported
+*p*-value uses the unsigned magnitude as the test statistic; both
+formulations are asymptotically χ²₁-distributed under the null.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+import numpy as np
+import pandas as pd
+from scipy.special import xlogy
+from scipy.stats import chi2
+
+LLFormula = Literal["rayson", "dunning"]
+
+
+def log_likelihood(
+    counts_a: pd.Series,
+    counts_b: pd.Series,
+    total_a: int,
+    total_b: int,
+    *,
+    formula: LLFormula = "rayson",
+) -> pd.DataFrame:
+    """Compute the signed log-likelihood-ratio keyness statistic.
+
+    Two formulations are available; see the module docstring for the
+    references and the math.
+
+    ``counts_a`` and ``counts_b`` are aligned on their union; missing
+    terms are imputed as zero. No min-count filtering is applied here —
+    that is the caller's responsibility (see
+    :meth:`pycorpdiff.Comparison.keyness`).
+
+    Parameters
+    ----------
+    counts_a, counts_b
+        Term-frequency series. Index entries are terms; values are
+        non-negative integer counts.
+    total_a, total_b
+        Corpus totals (token counts before any min-count filter). Used
+        for the contingency-table "not-term" cells under
+        ``formula="dunning"``.
+    formula
+        ``"rayson"`` (default; 2-cell shortcut, matches Rayson's LL
+        Wizard) or ``"dunning"`` (full 4-cell G²).
+
+    Returns
+    -------
+    pandas.DataFrame
+        Indexed by term, with columns ``count_a``, ``count_b``,
+        ``expected_a``, ``expected_b``, ``g2`` (signed), ``p_value``.
+    """
+    if total_a <= 0 or total_b <= 0:
+        raise ValueError(f"total_a and total_b must be positive; got {total_a}, {total_b}")
+    if formula not in ("rayson", "dunning"):
+        raise ValueError(
+            f"formula must be 'rayson' or 'dunning'; got {formula!r}"
+        )
+
+    terms = counts_a.index.union(counts_b.index)
+    a = counts_a.reindex(terms, fill_value=0).astype(np.int64).to_numpy()
+    b = counts_b.reindex(terms, fill_value=0).astype(np.int64).to_numpy()
+
+    obs_sum = a + b
+    total = total_a + total_b
+    expected_a = total_a * obs_sum / total
+    expected_b = total_b * obs_sum / total
+
+    # Rayson 2-cell shortcut: only the term-present rows.
+    # LL = 2 · (O₁·ln(O₁/E₁) + O₂·ln(O₂/E₂))
+    contrib = (
+        xlogy(a, a) - xlogy(a, expected_a) + xlogy(b, b) - xlogy(b, expected_b)
+    )
+    if formula == "dunning":
+        # Add the term-absent cells for the full 4-cell G².
+        c = total_a - a  # other tokens in A
+        d = total_b - b  # other tokens in B
+        expected_c = total_a - expected_a
+        expected_d = total_b - expected_b
+        contrib = contrib + (
+            xlogy(c, c) - xlogy(c, expected_c) + xlogy(d, d) - xlogy(d, expected_d)
+        )
+    unsigned = 2.0 * contrib
+    # Mathematically LL >= 0; clip away the tiny negative values that
+    # surface from float roundoff when the two corpora have ~identical rates.
+    unsigned = np.maximum(unsigned, 0.0)
+
+    # Sign by direction of overuse: + when A's rate exceeds B's, else -.
+    a_rate = a / total_a
+    b_rate = b / total_b
+    sign = np.where(a_rate >= b_rate, 1.0, -1.0)
+    signed = sign * unsigned
+
+    p_value = chi2.sf(unsigned, df=1)
+
+    return pd.DataFrame(
+        {
+            "count_a": a,
+            "count_b": b,
+            "expected_a": expected_a,
+            "expected_b": expected_b,
+            "g2": signed,
+            "p_value": p_value,
+        },
+        index=terms,
+    )

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/src/pycorpdiff/results.py

@@ -1,16 +1,22 @@
 """Result dataclasses returned by every public analytical verb.
 
-Every Result implements the same informal contract:
+Each Result implements the relevant subset of an informal six-method
+contract:
 
 - ``.to_df()`` returns a tidy :class:`pandas.DataFrame`.
 - ``.plot(**kw)`` returns an :class:`altair.Chart`.
-- ``.explain(term, n)`` returns a :class:`ConcordanceResult` with
-  evidence for one row of the result.
+- ``.to_html(path=None)`` renders the table as HTML.
+- ``.to_json(path=None)`` renders the table as JSON.
 - ``.summary()`` returns a short human-readable string.
-
-This contract is intentionally a duck-typing convention rather than an
-abstract base class — it keeps Results lightweight and lets them be
-constructed from a plain DataFrame without inheritance gymnastics.
+- ``.explain(term, n)`` returns a :class:`ConcordanceResult` with
+  KWIC evidence for one row of the result. Defined only on
+  comparison-based Results (``KeynessResult``, ``CollocationShiftResult``)
+  where "one row of the result" maps to a target term.
+
+See ``docs/design.md`` for the per-Result method matrix. This contract
+is a duck-typing convention rather than an abstract base class — it
+keeps Results lightweight and lets them be constructed from a plain
+DataFrame without inheritance gymnastics.
 """
 
 from __future__ import annotations

{pycorpdiff-0.1.0a5 → pycorpdiff-0.1.0a6}/tests/integration/test_crossval_quanteda.py

@@ -103,10 +103,20 @@ def _quanteda_keyness(corpus_df: pd.DataFrame) -> pd.DataFrame:
 def test_log_likelihood_matches_quanteda_byte_for_byte(
     fixture_corpus: pcd.Corpus,
 ) -> None:
-    """For every term shared with quanteda, our signed G² agrees to 1e-4."""
+    """For every term shared with quanteda (using formula='dunning'),
+    our signed G² agrees byte-for-byte to ≤ 1e-10.
+
+    quanteda's ``textstat_keyness(measure="lr")`` uses the full 4-cell
+    Dunning G². The Rayson 2-cell shortcut (our default) is a different
+    statistic; comparing like-to-like requires passing ``formula="dunning"``.
+    """
     a = fixture_corpus.slice(frame="A")
     b = fixture_corpus.slice(frame="B")
-    ours = pcd.compare(a, b).keyness(min_count=1).table.set_index("term")["g2"]
+    ours = (
+        pcd.compare(a, b)
+        .keyness(min_count=1, formula="dunning")
+        .table.set_index("term")["g2"]
+    )
 
     quanteda_df = _quanteda_keyness(fixture_corpus.docs.copy())
     theirs = pd.Series(
@@ -123,7 +133,8 @@ def test_log_likelihood_matches_quanteda_byte_for_byte(
         theirs_v = float(theirs[term])
         # quanteda's textstat_keyness uses signed G² with the same
         # convention we do: positive when overused in the target
-        # group. Agreement to 4 decimal places is more than enough.
-        assert math.isclose(ours_v, theirs_v, abs_tol=1e-4), (
+        # group. With matching formulae, the two implementations
+        # should agree to floating-point noise.
+        assert math.isclose(ours_v, theirs_v, abs_tol=1e-10), (
             f"{term}: pycorpdiff={ours_v}, quanteda={theirs_v}"
         )

pycorpdiff-0.1.0a5/src/pycorpdiff/_backends/pandas.py

@@ -1,3 +0,0 @@
-"""Pandas backend shim — placeholder until backend abstractions are needed."""
-
-from __future__ import annotations

pycorpdiff-0.1.0a5/src/pycorpdiff/keyness/loglikelihood.py

@@ -1,92 +0,0 @@
-"""Dunning's G² log-likelihood statistic.
-
-Reference
----------
-Dunning, T. (1993). Accurate methods for the statistics of surprise and
-coincidence. *Computational Linguistics*, 19(1), 61-74.
-
-Notes
------
-The G² returned by :func:`log_likelihood` is **signed**: positive when the
-term is overused in corpus A relative to corpus B (i.e. ``a/N_a > b/N_b``)
-and negative when overused in B. This is the convention CASS / Lancaster
-tooling has gravitated toward — it carries direction information without
-needing a separate column. The reported *p*-value uses ``|G²|`` as the
-test statistic; the unsigned form is what's chi-squared distributed.
-"""
-
-from __future__ import annotations
-
-import numpy as np
-import pandas as pd
-from scipy.special import xlogy
-from scipy.stats import chi2
-
-
-def log_likelihood(
-    counts_a: pd.Series,
-    counts_b: pd.Series,
-    total_a: int,
-    total_b: int,
-) -> pd.DataFrame:
-    """Compute Dunning G² for every term in the union of input indices.
-
-    ``counts_a`` and ``counts_b`` are aligned on their union; missing
-    terms are imputed as zero. No min-count filtering is applied here —
-    that is the caller's responsibility (see
-    :meth:`pycorpdiff.Comparison.keyness`).
-
-    Parameters
-    ----------
-    counts_a, counts_b
-        Term-frequency series. Index entries are terms; values are
-        non-negative integer counts.
-    total_a, total_b
-        Corpus totals (token counts before any min-count filter). Used
-        for the contingency-table "not-term" cells.
-
-    Returns
-    -------
-    pandas.DataFrame
-        Indexed by term, with columns ``count_a``, ``count_b``,
-        ``expected_a``, ``expected_b``, ``g2`` (signed), ``p_value``.
-    """
-    if total_a <= 0 or total_b <= 0:
-        raise ValueError(f"total_a and total_b must be positive; got {total_a}, {total_b}")
-
-    terms = counts_a.index.union(counts_b.index)
-    a = counts_a.reindex(terms, fill_value=0).astype(np.int64).to_numpy()
-    b = counts_b.reindex(terms, fill_value=0).astype(np.int64).to_numpy()
-
-    obs_sum = a + b
-    total = total_a + total_b
-    expected_a = total_a * obs_sum / total
-    expected_b = total_b * obs_sum / total
-
-    # 2 * sum_i O_i * ln(O_i / E_i), with xlogy giving 0*log(0)=0.
-    unsigned = 2.0 * (
-        xlogy(a, a) - xlogy(a, expected_a) + xlogy(b, b) - xlogy(b, expected_b)
-    )
-    # Mathematically G² >= 0; clip away the tiny negative values that
-    # surface from float roundoff when the two corpora have ~identical rates.
-    unsigned = np.maximum(unsigned, 0.0)
-
-    # Sign by direction of overuse: + when A's rate exceeds B's, else -.
-    a_rate = a / total_a
-    b_rate = b / total_b
-    sign = np.where(a_rate >= b_rate, 1.0, -1.0)
-    signed = sign * unsigned
-
-    p_value = chi2.sf(unsigned, df=1)
-
-    return pd.DataFrame(
-        {
-            "count_a": a,
-            "count_b": b,
-            "expected_a": expected_a,
-            "expected_b": expected_b,
-            "g2": signed,
-            "p_value": p_value,
-        },
-        index=terms,
-    )