keyflux 0.1.0__py3-none-any.whl

keyflux/__init__.py

@@ -0,0 +1,30 @@
+"""Keyflux — keyness, rank-turbulence divergence, and allotaxonographs."""
+
+__version__ = "0.1.0"
+
+from keyflux.divergence import Contribution, RTDResult, rtd
+from keyflux.io.corpus import counts_from_text, counts_from_tokens, load_counts
+from keyflux.keyness import (
+    Keyness,
+    KeynessRow,
+    KeywordTable,
+    ReproRecord,
+)
+from keyflux.ranking import RankedList
+from keyflux.viz import allotaxonograph
+
+__all__ = [
+    "Keyness",
+    "KeynessRow",
+    "KeywordTable",
+    "ReproRecord",
+    "RankedList",
+    "rtd",
+    "RTDResult",
+    "Contribution",
+    "allotaxonograph",
+    "counts_from_tokens",
+    "counts_from_text",
+    "load_counts",
+    "__version__",
+]

keyflux/_types.py

@@ -0,0 +1,31 @@
+"""Shared type aliases for the keyflux package."""
+
+from collections import Counter
+from typing import Literal, TypeAlias
+
+Token: TypeAlias = str
+"""A surface or lemmatised word type."""
+
+Count: TypeAlias = int
+"""A raw frequency count for a single type."""
+
+FreqTable: TypeAlias = Counter[str]
+"""Type-to-count mapping for one corpus (focus or reference)."""
+
+Rank: TypeAlias = float
+"""A 1-based rank; float because tied and tied-last ranks are averaged."""
+
+MeasureName: TypeAlias = Literal[
+    "log_likelihood",
+    "log_ratio",
+    "simple_maths",
+    "percent_diff",
+    "chi_square",
+]
+"""Identifier selecting a keyness scoring function."""
+
+Significance: TypeAlias = Literal["ns", "p05", "p01", "p001", "p0001"]
+"""Significance band from a log-likelihood / chi-square statistic (1 d.f.)."""
+
+Direction: TypeAlias = Literal["positive", "negative", "neutral"]
+"""Keyness polarity: over-represented, under-represented, or neither in focus."""

keyflux/datasets/__init__.py

@@ -0,0 +1,104 @@
+"""Tiny bundled corpora and fixtures for docs and tests.
+
+The data lives inline as Python dicts so it is always importable in doctests
+with no package-data or ``importlib.resources`` machinery.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from keyflux.ranking.rankedlist import RankedList
+
+# The jkbren/rank-turbulence-divergence reference example. Both systems rank the
+# same seven elements; rtd at alpha=1.0 is 0.45924793111057804.
+_JKBREN_FOCUS: dict[str, int] = {
+    "a": 20,
+    "e": 14,
+    "c": 8,
+    "b": 7,
+    "f": 4,
+    "g": 2,
+    "d": 1,
+}
+_JKBREN_REFERENCE: dict[str, int] = {
+    "b": 24,
+    "a": 16,
+    "e": 5,
+    "d": 4,
+    "c": 3,
+    "f": 2,
+    "g": 1,
+}
+
+_DEMO_FOCUS: dict[str, int] = {
+    "climate": 42,
+    "carbon": 28,
+    "emissions": 19,
+    "warming": 14,
+    "policy": 11,
+    "the": 320,
+    "of": 180,
+    "and": 165,
+    "to": 150,
+    "energy": 22,
+    "renewable": 9,
+    "global": 17,
+}
+_DEMO_REFERENCE: dict[str, int] = {
+    "market": 40,
+    "stock": 26,
+    "trade": 21,
+    "profit": 13,
+    "policy": 12,
+    "the": 318,
+    "of": 176,
+    "and": 170,
+    "to": 148,
+    "energy": 8,
+    "shares": 15,
+    "global": 16,
+}
+
+
+def load_demo_pair() -> tuple[Counter[str], Counter[str]]:
+    """Return the bundled (focus, reference) demo corpus pair.
+
+    A tiny climate-discourse focus corpus versus a finance-discourse reference
+    corpus, with shared function words and a couple of lockword-like overlaps.
+
+    Returns:
+        ``(focus, reference)`` frequency Counters.
+
+    Examples:
+        >>> focus, reference = load_demo_pair()
+        >>> focus["climate"], reference["market"]
+        (42, 40)
+    """
+    return Counter(_DEMO_FOCUS), Counter(_DEMO_REFERENCE)
+
+
+def load_jkbren_example() -> tuple[RankedList, RankedList]:
+    """Return the jkbren rank-turbulence-divergence regression pair.
+
+    Two ranked lists over the same seven elements. Their rank-turbulence
+    divergence at ``alpha=1.0`` is ``0.45924793111057804`` — the regression
+    anchor from the reference implementation.
+
+    Returns:
+        ``(list1, list2)`` as :class:`keyflux.ranking.rankedlist.RankedList`.
+
+    Examples:
+        >>> from keyflux.divergence import rtd
+        >>> r1, r2 = load_jkbren_example()
+        >>> round(rtd(r1, r2, alpha=1.0).divergence, 6)
+        0.459248
+    """
+    from keyflux.ranking.rankedlist import RankedList
+
+    return (
+        RankedList.from_counts(_JKBREN_FOCUS, label="system 1"),
+        RankedList.from_counts(_JKBREN_REFERENCE, label="system 2"),
+    )

keyflux/divergence/__init__.py

@@ -0,0 +1,5 @@
+"""Divergence: rank-turbulence divergence between two ranked lists."""
+
+from keyflux.divergence.rtd import Contribution, RTDResult, rtd
+
+__all__ = ["rtd", "RTDResult", "Contribution"]

keyflux/divergence/rtd.py

@@ -0,0 +1,213 @@
+r"""Rank-turbulence divergence (Dodds et al. 2020, arXiv:2002.09770).
+
+Rank-turbulence divergence (RTD) compares two ranked lists by how far each type
+moves between them, with a tunable parameter ``alpha``. Small ``alpha`` surfaces
+churn among rare, low-rank types; large ``alpha`` surfaces shifts among common,
+high-rank types. The per-type divergence is
+
+    delta(tau; alpha) = | r1 ** -alpha - r2 ** -alpha | ** (1 / (alpha + 1))
+
+and the total is normalised to [0, 1] by the divergence of the maximally
+disjoint pair, so RTD(x, x) = 0 and disjoint lists approach 1. As ``alpha -> 0``
+the per-type term degenerates to the logarithmic form ``| ln r1 - ln r2 |``,
+which is used directly to avoid dividing by zero.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Literal
+
+if TYPE_CHECKING:
+    from keyflux.ranking.rankedlist import RankedList
+
+ShiftDirection = Literal["system1", "system2", "shared"]
+"""Which system a type is more characteristic of (lower rank = more typical)."""
+
+_ALPHA_ZERO_TOL = 1e-10
+
+
+@dataclass(frozen=True, slots=True)
+class Contribution:
+    """One type's contribution to the total divergence.
+
+    Attributes:
+        type: The type.
+        delta: The raw per-type divergence term (before normalisation).
+        contribution: This type's additive share of ``divergence`` (the shares
+            sum to the total divergence).
+        rank1: The type's rank in the first list (tied-last if absent).
+        rank2: The type's rank in the second list (tied-last if absent).
+        direction: Which list the type leans toward (lower rank wins).
+    """
+
+    type: str
+    delta: float
+    contribution: float
+    rank1: float
+    rank2: float
+    direction: ShiftDirection
+
+
+@dataclass(frozen=True, slots=True)
+class RTDResult:
+    """The result of a rank-turbulence divergence computation.
+
+    Attributes:
+        divergence: The normalised divergence in [0, 1].
+        raw: The un-normalised weighted sum (matches the jkbren reference at
+            ``alpha=1.0`` when ``normalize=False``).
+        alpha: The tuning parameter used.
+        contributions: Per-type contributions, sorted by contribution descending.
+        labels: The two list labels.
+    """
+
+    divergence: float
+    raw: float
+    alpha: float
+    contributions: tuple[Contribution, ...]
+    labels: tuple[str, str]
+
+
+def _direction(rank1: float, rank2: float) -> ShiftDirection:
+    if rank1 < rank2:
+        return "system1"
+    if rank2 < rank1:
+        return "system2"
+    return "shared"
+
+
+def _elements_general(
+    ranks1: list[float],
+    ranks2: list[float],
+    n1: int,
+    n2: int,
+    alpha: float,
+) -> tuple[list[float], float]:
+    """Per-type terms and normalisation for alpha > 0."""
+    exp = 1.0 / (alpha + 1.0)
+    ref1 = (n1 + 0.5 * n2) ** (-alpha)
+    ref2 = (n2 + 0.5 * n1) ** (-alpha)
+    deltas: list[float] = []
+    norm = 0.0
+    for r1, r2 in zip(ranks1, ranks2, strict=True):
+        a1 = r1**-alpha
+        a2 = r2**-alpha
+        deltas.append(abs(a1 - a2) ** exp)
+        norm += abs(a1 - ref1) ** exp + abs(ref2 - a2) ** exp
+    return deltas, norm
+
+
+def _elements_log_limit(
+    ranks1: list[float],
+    ranks2: list[float],
+    n1: int,
+    n2: int,
+) -> tuple[list[float], float]:
+    """Per-type terms and normalisation for the alpha -> 0 logarithmic limit."""
+    ref1 = math.log(n1 + 0.5 * n2)
+    ref2 = math.log(n2 + 0.5 * n1)
+    deltas: list[float] = []
+    norm = 0.0
+    for r1, r2 in zip(ranks1, ranks2, strict=True):
+        l1 = math.log(r1)
+        l2 = math.log(r2)
+        deltas.append(abs(l1 - l2))
+        norm += abs(l1 - ref1) + abs(ref2 - l2)
+    return deltas, norm
+
+
+def rtd(
+    list1: RankedList,
+    list2: RankedList,
+    *,
+    alpha: float = 1.0 / 3.0,
+    normalize: bool = True,
+) -> RTDResult:
+    """Compute the rank-turbulence divergence between two ranked lists.
+
+    Args:
+        list1: The first ranked list.
+        list2: The second ranked list.
+        alpha: Tuning parameter (``>= 0``). Small values surface rare, low-rank
+            churn; large values surface common-word shifts. ``alpha == 0`` uses
+            the logarithmic limit. The default ``1/3`` is the Dodds et al.
+            recommendation for text.
+        normalize: If True, return ``divergence`` scaled to [0, 1]; the ``raw``
+            field always holds the un-normalised sum.
+
+    Returns:
+        An :class:`RTDResult` with the scalar divergence and the sorted per-type
+        contributions, each tagged with its shift direction.
+
+    Raises:
+        ValueError: If ``alpha`` is negative, or either list is empty.
+
+    Contract:
+        - ``rtd(x, x).divergence == 0`` (a list never diverges from itself).
+        - ``0 <= divergence <= 1`` for every input and every ``alpha``.
+        - Symmetric: ``rtd(a, b).divergence == rtd(b, a).divergence``.
+        - The per-type contributions sum to ``divergence``.
+        - Exclusives (present in one list only) are placed at a tied-last rank.
+
+    Examples:
+        >>> from keyflux.ranking.rankedlist import RankedList
+        >>> r1 = RankedList.from_counts({"a": 20, "e": 14, "c": 8, "b": 7,
+        ...                              "f": 4, "g": 2, "d": 1})
+        >>> r2 = RankedList.from_counts({"b": 24, "a": 16, "e": 5, "d": 4,
+        ...                              "c": 3, "f": 2, "g": 1})
+        >>> round(rtd(r1, r2, alpha=1.0).divergence, 6)
+        0.459248
+        >>> rtd(r1, r1, alpha=1.0).divergence
+        0.0
+    """
+    if alpha < 0:
+        msg = f"alpha must be non-negative, got {alpha}."
+        raise ValueError(msg)
+    if not len(list1) or not len(list2):
+        msg = "Both ranked lists must be non-empty."
+        raise ValueError(msg)
+
+    types, ranks1, ranks2 = list1.aligned(list2)
+    n1 = len(list1)
+    n2 = len(list2)
+
+    if alpha < _ALPHA_ZERO_TOL:
+        deltas, norm = _elements_log_limit(ranks1, ranks2, n1, n2)
+        prefactor = 1.0
+    else:
+        deltas, norm = _elements_general(ranks1, ranks2, n1, n2, alpha)
+        prefactor = (alpha + 1.0) / alpha
+
+    raw = prefactor * math.fsum(deltas)
+    normalizer = prefactor * norm
+    divergence = raw / normalizer if normalize and normalizer > 0 else raw
+    if normalize and normalizer <= 0:
+        divergence = 0.0
+
+    share_denom = normalizer if normalize and normalizer > 0 else 1.0
+    contributions = tuple(
+        sorted(
+            (
+                Contribution(
+                    type=t,
+                    delta=d,
+                    contribution=prefactor * d / share_denom,
+                    rank1=r1,
+                    rank2=r2,
+                    direction=_direction(r1, r2),
+                )
+                for t, d, r1, r2 in zip(types, deltas, ranks1, ranks2, strict=True)
+            ),
+            key=lambda c: c.contribution,
+            reverse=True,
+        )
+    )
+    return RTDResult(
+        divergence=divergence,
+        raw=raw,
+        alpha=alpha,
+        contributions=contributions,
+        labels=(list1.label, list2.label),
+    )

keyflux/io/__init__.py

@@ -0,0 +1,5 @@
+"""I/O: build frequency Counters from tokens, text, or count files."""
+
+from keyflux.io.corpus import counts_from_text, counts_from_tokens, load_counts
+
+__all__ = ["counts_from_tokens", "counts_from_text", "load_counts"]

keyflux/io/corpus.py

@@ -0,0 +1,99 @@
+"""Build frequency Counters from tokens, text, or count files.
+
+keyflux is about keyness and rank comparison, not tokenisation. These helpers
+cover the simple cases; for real linguistic tokenisation, pre-tokenise (for
+example with ``kenon.Tokenizer``) and pass the resulting Counter directly.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from collections.abc import Iterable
+from pathlib import Path
+
+_WORD_RE = re.compile(r"\w+", re.UNICODE)
+
+
+def counts_from_tokens(
+    tokens: Iterable[str], *, lowercase: bool = True
+) -> Counter[str]:
+    """Build a frequency Counter from a token iterable.
+
+    Args:
+        tokens: Already-tokenised word types.
+        lowercase: If True, lowercase each token before counting.
+
+    Returns:
+        A Counter mapping each type to its frequency.
+
+    Examples:
+        >>> counts_from_tokens(["The", "cat", "the", "CAT"])
+        Counter({'the': 2, 'cat': 2})
+    """
+    if lowercase:
+        return Counter(t.lower() for t in tokens)
+    return Counter(tokens)
+
+
+def counts_from_text(text: str, *, lowercase: bool = True) -> Counter[str]:
+    """Tokenise a string on word characters, then count.
+
+    Uses a simple word-character regular expression — adequate for demos and
+    tests, not a substitute for linguistic tokenisation.
+
+    Args:
+        text: Raw text to tokenise and count.
+        lowercase: If True, lowercase before counting.
+
+    Returns:
+        A Counter mapping each type to its frequency.
+
+    Examples:
+        >>> counts_from_text("The cat sat. The dog ran.")["the"]
+        2
+    """
+    tokens = _WORD_RE.findall(text.lower() if lowercase else text)
+    return Counter(tokens)
+
+
+def load_counts(path: str | Path) -> Counter[str]:
+    """Read a count file into a Counter.
+
+    Each non-empty line is either ``type<TAB>count`` or a bare ``type`` (counted
+    as one occurrence per line).
+
+    Args:
+        path: Path to the count file.
+
+    Returns:
+        A Counter built from the file.
+
+    Raises:
+        FileNotFoundError: If ``path`` does not exist.
+        ValueError: If a count field is present but not an integer.
+
+    Examples:
+        >>> import tempfile, pathlib
+        >>> p = pathlib.Path(tempfile.mkdtemp()) / "counts.tsv"
+        >>> rows = ["climate" + chr(9) + "30", "carbon" + chr(9) + "12"]
+        >>> _ = p.write_text(chr(10).join(rows))
+        >>> load_counts(p)
+        Counter({'climate': 30, 'carbon': 12})
+    """
+    counts: Counter[str] = Counter()
+    for line in Path(path).read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parts = line.split("\t")
+        if len(parts) == 1:
+            counts[parts[0]] += 1
+        else:
+            word, raw = parts[0], parts[1]
+            try:
+                counts[word] += int(raw)
+            except ValueError as exc:
+                msg = f"Non-integer count {raw!r} for type {word!r}."
+                raise ValueError(msg) from exc
+    return counts

keyflux/keyness/__init__.py

@@ -0,0 +1,43 @@
+"""Keyness: focus-vs-reference keywords, lockwords, and association measures."""
+
+from keyflux.keyness.classify import (
+    Category,
+    classify_direction,
+    classify_row,
+    is_significant,
+    partition,
+)
+from keyflux.keyness.keyness import (
+    Keyness,
+    KeynessRow,
+    KeywordTable,
+    ReproRecord,
+)
+from keyflux.keyness.measures import (
+    chi_square,
+    expected_counts,
+    log_likelihood,
+    log_ratio,
+    percent_diff,
+    significance_band,
+    simple_maths,
+)
+
+__all__ = [
+    "Keyness",
+    "KeynessRow",
+    "KeywordTable",
+    "ReproRecord",
+    "Category",
+    "classify_direction",
+    "classify_row",
+    "is_significant",
+    "partition",
+    "chi_square",
+    "expected_counts",
+    "log_likelihood",
+    "log_ratio",
+    "percent_diff",
+    "significance_band",
+    "simple_maths",
+]