pycorpdiff 0.1.0a0__py3-none-any.whl

pycorpdiff/__init__.py

@@ -0,0 +1,126 @@
+"""pycorpdiff — comparative corpus analysis for modern Python workflows.
+
+The package exposes three public verbs (:func:`compare`, :func:`track`,
+plus the :class:`Corpus` constructor and the I/O ``read_*`` helpers) and
+four families of result objects (:class:`KeynessResult`,
+:class:`CollocationShiftResult`, :class:`SemanticShiftResult`,
+:class:`TemporalTrajectory`).
+
+Layer-1 ingestion utilities are functional in this scaffolding release;
+Layer-2 analytical methods raise :class:`NotImplementedError` until Phase 1
+of the roadmap lands.
+
+Example
+-------
+
+>>> import pycorpdiff as pcd
+>>> pcd.__version__
+'0.1.0a0'
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0a0"
+
+from .collocation.network import NetworkResult, cooccurrence_network
+from .compare import Comparison, compare
+from .corpus import Corpus, CorpusSlice
+from .datasets import (
+    fetch_hansard,
+    fetch_histwords_decade,
+    histwords_cosine_shift,
+    load_hansard_sample,
+)
+from .explain import kwic, representative_docs
+from .io.duckdb import read_duckdb
+from .io.huggingface import from_huggingface
+from .io.readers import from_dataframe, read_csv, read_parquet, read_txt
+from .keyness.multicorpus import keyness_multi
+from .results import (
+    CollocationShiftResult,
+    ConcordanceResult,
+    KeynessResult,
+    SemanticShiftResult,
+    TemporalTrajectory,
+)
+from .semantic.embed import Embedder, HashEmbedder, SBERTEmbedder
+from .semantic.shift import neighborhood_drift
+from .semantic.trajectory import semantic_trajectory
+from .temporal.bocpd import BocpdResult, bocpd
+from .temporal.causal_impact import CausalImpactResult, causal_impact
+from .temporal.forecast import (
+    ForecastResult,
+    forecast_semantic_drift,
+    forecast_trajectory,
+)
+from .temporal.slicing import TemporalCorpus, track
+from .tokenize import NgramTokenizer, RegexTokenizer, Tokenizer
+
+# Convenience aliases for the standalone (non-`.plot()`-delegated)
+# visualisation functions. The full plot family also lives at
+# ``pycorpdiff.viz.*`` — these are surfaced at the root so common
+# patterns like ``pcd.dispersion_plot(corpus, term)`` work without
+# requiring a separate import path. Plots that are only meaningful
+# as ``Result.plot()`` (keyness_volcano, keyness_top_n_bar,
+# collocation_diverging_bar, trajectory_with_ci) stay in ``viz`` only.
+from .viz import (
+    bocpd_plot,
+    causal_impact_plot,
+    dispersion_plot,
+    forecast_plot,
+    network_plot,
+    scattertext_plot,
+    semantic_forecast_plot,
+)
+
+__all__ = [
+    "BocpdResult",
+    "CausalImpactResult",
+    "CollocationShiftResult",
+    "Comparison",
+    "ConcordanceResult",
+    "Corpus",
+    "CorpusSlice",
+    "Embedder",
+    "ForecastResult",
+    "HashEmbedder",
+    "KeynessResult",
+    "NetworkResult",
+    "NgramTokenizer",
+    "RegexTokenizer",
+    "SBERTEmbedder",
+    "SemanticShiftResult",
+    "TemporalCorpus",
+    "TemporalTrajectory",
+    "Tokenizer",
+    "__version__",
+    "bocpd",
+    "bocpd_plot",
+    "causal_impact",
+    "causal_impact_plot",
+    "compare",
+    "cooccurrence_network",
+    "dispersion_plot",
+    "fetch_hansard",
+    "fetch_histwords_decade",
+    "forecast_plot",
+    "forecast_semantic_drift",
+    "forecast_trajectory",
+    "from_dataframe",
+    "from_huggingface",
+    "histwords_cosine_shift",
+    "keyness_multi",
+    "kwic",
+    "load_hansard_sample",
+    "neighborhood_drift",
+    "network_plot",
+    "read_csv",
+    "read_duckdb",
+    "read_parquet",
+    "read_txt",
+    "representative_docs",
+    "scattertext_plot",
+    "semantic_forecast_plot",
+    "semantic_trajectory",
+    "track",
+]

pycorpdiff/_backends/__init__.py

@@ -0,0 +1,3 @@
+"""Backend shims. Pandas is the default; polars is reserved for a later phase."""
+
+from __future__ import annotations

pycorpdiff/_backends/pandas.py

@@ -0,0 +1,3 @@
+"""Pandas backend shim — placeholder until backend abstractions are needed."""
+
+from __future__ import annotations

pycorpdiff/_backends/polars.py

@@ -0,0 +1,3 @@
+"""Polars backend shim — populated when the polars extra is wired up."""
+
+from __future__ import annotations

pycorpdiff/collocation/__init__.py

@@ -0,0 +1,19 @@
+"""Collocation measures and collocation-shift analysis."""
+
+from __future__ import annotations
+
+from .cooccurrence import collocate_counts
+from .measures import logdice, mi_three, pmi, t_score
+from .network import NetworkResult, cooccurrence_network
+from .shift import collocation_shift
+
+__all__ = [
+    "NetworkResult",
+    "collocate_counts",
+    "collocation_shift",
+    "cooccurrence_network",
+    "logdice",
+    "mi_three",
+    "pmi",
+    "t_score",
+]

pycorpdiff/collocation/cooccurrence.py

@@ -0,0 +1,65 @@
+"""Window-based co-occurrence extraction.
+
+Given a tokenized corpus and a target term, walks each document and
+accumulates the count of every other token that appears within ``window``
+positions of the target. Windows never cross document boundaries — each
+document's contexts are isolated, which matches the SketchEngine / NLTK
+convention and avoids spurious cross-doc associations.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Sequence
+
+
+def collocate_counts(
+    docs_tokens: Sequence[Sequence[str]],
+    target: str,
+    window: int = 5,
+) -> tuple[Counter[str], int]:
+    """Return ``(collocate_counter, target_occurrences)`` for ``target``.
+
+    Each occurrence of ``target`` contributes to up to ``2 * window``
+    collocate counts (fewer near document boundaries). The target itself
+    is excluded from its own context window — if the target appears
+    twice within ``window`` of itself, each occurrence still contributes
+    one count to the other.
+
+    Parameters
+    ----------
+    docs_tokens
+        One sequence of tokens per document, in original order.
+    target
+        The pivot term whose context we are accumulating.
+    window
+        Tokens on each side of the target. ``window=5`` gives a
+        ten-token context (five left, five right).
+
+    Returns
+    -------
+    counter
+        Mapping ``collocate -> joint count``.
+    target_occurrences
+        Number of times the target appears across all documents. Used
+        downstream as ``f_x`` (the marginal target count) for
+        association measures.
+    """
+    if window < 1:
+        raise ValueError(f"window must be >= 1; got {window}")
+
+    counter: Counter[str] = Counter()
+    target_n = 0
+    for tokens in docs_tokens:
+        n = len(tokens)
+        for i in range(n):
+            if tokens[i] != target:
+                continue
+            target_n += 1
+            lo = max(0, i - window)
+            hi = min(n, i + window + 1)
+            for j in range(lo, hi):
+                if j == i:
+                    continue
+                counter[tokens[j]] += 1
+    return counter, target_n

pycorpdiff/collocation/measures.py

@@ -0,0 +1,102 @@
+"""Collocation association measures.
+
+All four functions accept the same five-argument shape:
+
+``f_xy``
+    Joint count of (target, collocate) in a window.
+``f_x``
+    Total occurrences of the target in the corpus.
+``f_y``
+    Total occurrences of the collocate in the corpus.
+``n``
+    Total tokens in the corpus (required by every measure except logDice).
+
+Counts may be passed as scalars or pandas Series; broadcasting follows
+NumPy / pandas conventions.
+
+References
+----------
+Rychlý, P. (2008). A lexicographer-friendly association score. In
+*Proceedings of RASLAN 2008*.
+
+Church, K., Gale, W., Hanks, P., & Hindle, D. (1991). Using statistics in
+lexical analysis. In *Lexical Acquisition*, 115-164.
+
+Daille, B. (1994). *Approche mixte pour l'extraction automatique de
+terminologie*. PhD thesis, Université Paris 7.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+
+def logdice(
+    f_xy: pd.Series,
+    f_x: float,
+    f_y: pd.Series,
+) -> pd.Series:
+    """Rychlý's logDice: ``14 + log2(2 · f_xy / (f_x + f_y))``.
+
+    Range-bounded above at 14 (perfect co-occurrence). Robust to corpus
+    size because it never references the total. Values below 0 are
+    typically noise; the practical interesting band is roughly 7..14.
+
+    Zero joint counts yield ``-inf``; pre-smooth ``f_xy`` upstream
+    (e.g. via :func:`pycorpdiff.collocation.collocation_shift`) if you
+    need finite scores across the union of vocabularies.
+    """
+    with np.errstate(divide="ignore", invalid="ignore"):
+        ratio = (2.0 * f_xy) / (f_x + f_y)
+        return pd.Series(14.0 + np.log2(ratio), index=f_xy.index)
+
+
+def pmi(
+    f_xy: pd.Series,
+    f_x: float,
+    f_y: pd.Series,
+    n: int,
+) -> pd.Series:
+    """Pointwise mutual information: ``log2(f_xy · N / (f_x · f_y))``.
+
+    The "association ratio" of Church & Hanks (1990). PMI rewards rare
+    pairs disproportionately — always pair with a frequency floor or
+    use MI³ if rare-pair inflation is a concern.
+    """
+    with np.errstate(divide="ignore", invalid="ignore"):
+        return pd.Series(np.log2((f_xy * n) / (f_x * f_y)), index=f_xy.index)
+
+
+def t_score(
+    f_xy: pd.Series,
+    f_x: float,
+    f_y: pd.Series,
+    n: int,
+) -> pd.Series:
+    """Welch-style t-score: ``(f_xy - E[f_xy]) / sqrt(f_xy)``.
+
+    Where ``E[f_xy] = f_x · f_y / N`` is the count expected under
+    independence. Favours frequent collocates — the inverse of PMI's
+    sparsity bias.
+    """
+    expected = (f_x * f_y) / n
+    with np.errstate(divide="ignore", invalid="ignore"):
+        return pd.Series((f_xy - expected) / np.sqrt(f_xy), index=f_xy.index)
+
+
+def mi_three(
+    f_xy: pd.Series,
+    f_x: float,
+    f_y: pd.Series,
+    n: int,
+) -> pd.Series:
+    """Daille's MI³: ``log2(f_xy³ · N / (f_x · f_y))``.
+
+    Cubes the joint count in the numerator, which empirically downweights
+    PMI's rare-pair bias without t-score's frequency dominance.
+    """
+    with np.errstate(divide="ignore", invalid="ignore"):
+        return pd.Series(
+            np.log2((np.power(f_xy, 3) * n) / (f_x * f_y)), index=f_xy.index
+        )

pycorpdiff/collocation/network.py

@@ -0,0 +1,233 @@
+"""Term co-occurrence networks.
+
+For exploratory work — "what does the discourse *graph* look like?" —
+the natural artefact is a network: nodes are the corpus's most frequent
+terms, edges connect terms that co-occur within a window, and edge
+weights come from a standard association measure (PMI, t-score, MI³).
+
+This is the term-as-vertex visualisation that gephi-style network tools
+have made common in digital humanities; here it lands as a first-class
+:class:`pycorpdiff.collocation.NetworkResult` with the same
+:meth:`to_df` / :meth:`plot` shape as every other Result.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Literal
+
+import numpy as np
+import pandas as pd
+
+from ..corpus import Corpus, CorpusSlice
+from .measures import logdice, mi_three, pmi, t_score
+
+if TYPE_CHECKING:
+    import altair as alt
+
+NetworkMeasure = Literal["PMI", "t_score", "MI3", "logDice"]
+
+
+@dataclass(frozen=True)
+class NetworkResult:
+    """A term co-occurrence network with nodes, edges, and a plot method.
+
+    The two DataFrames are the canonical "long-format" shape every
+    network analytics tool consumes:
+
+    - ``nodes``: index = term, columns = ``count``, ``degree``
+    - ``edges``: columns = ``source``, ``target``, ``cooccur_count``,
+      ``weight``  (the association score), and ``rank`` (0-based, by
+      ``|weight|`` descending).
+    """
+
+    nodes: pd.DataFrame
+    edges: pd.DataFrame
+    measure: NetworkMeasure
+    window: int
+    label: str = ""
+    params: dict[str, object] = field(default_factory=dict)
+
+    def to_df(self) -> pd.DataFrame:
+        """Return the edges as a flat tidy DataFrame (for round-trips)."""
+        return self.edges.copy()
+
+    def summary(self) -> str:
+        return (
+            f"NetworkResult(measure={self.measure}, window={self.window}, "
+            f"nodes={len(self.nodes):,}, edges={len(self.edges):,})"
+        )
+
+    def plot(self, **kw: object) -> alt.Chart:
+        """Render the network as an altair force-directed-style plot."""
+        from ..viz.network import network_plot
+
+        return network_plot(self, **kw)  # type: ignore[arg-type]
+
+
+def cooccurrence_network(
+    corpus: Corpus | CorpusSlice,
+    *,
+    top_n: int = 50,
+    window: int = 5,
+    measure: NetworkMeasure = "PMI",
+    min_count: int = 3,
+    min_cooccur: int = 2,
+    smoothing: float = 0.5,
+) -> NetworkResult:
+    """Build a term co-occurrence network for the ``top_n`` terms.
+
+    Each pair of distinct terms among the ``top_n`` vocabulary is
+    weighted by the chosen association measure on their joint counts
+    within ``window`` tokens of each other inside a document.
+
+    Parameters
+    ----------
+    corpus
+        A :class:`Corpus` or :class:`CorpusSlice`.
+    top_n
+        Vocabulary cap — the ``top_n`` most frequent terms (after
+        ``min_count``) become network nodes.
+    window
+        Symmetric context window for the co-occurrence count.
+    measure
+        Edge-weight association measure.
+    min_count
+        Drop terms below this corpus-wide frequency before picking the
+        top-N.
+    min_cooccur
+        Drop edges with joint count below this. Acts as the network's
+        noise floor.
+    smoothing
+        Laplace constant added to joint and marginal counts before
+        scoring (mirrors :func:`collocation_shift`'s convention so the
+        same measures stay finite on absent pairs).
+
+    Returns
+    -------
+    NetworkResult
+    """
+    if top_n < 2:
+        raise ValueError(f"top_n must be >= 2; got {top_n}")
+    if window < 1:
+        raise ValueError(f"window must be >= 1; got {window}")
+    if smoothing <= 0:
+        raise ValueError(f"smoothing must be > 0; got {smoothing}")
+
+    vocab = corpus.vocab(min_count=min_count).head(top_n)
+    if len(vocab) < 2:
+        raise ValueError(
+            f"need at least 2 terms after min_count={min_count} filter; "
+            f"got {len(vocab)}"
+        )
+
+    keep_set = set(vocab.index)
+    pair_counts: Counter[tuple[str, str]] = Counter()
+
+    for tokens in corpus.tokens():
+        # Pre-filter to in-vocab tokens with original positions.
+        positions = [(i, t) for i, t in enumerate(tokens) if t in keep_set]
+        for k, (i, t_i) in enumerate(positions):
+            for j, t_j in positions[k + 1 :]:
+                if j - i > window:
+                    break  # positions are sorted; rest are further away
+                if t_i == t_j:
+                    continue
+                pair = (t_i, t_j) if t_i < t_j else (t_j, t_i)
+                pair_counts[pair] += 1
+
+    if not pair_counts:
+        return NetworkResult(
+            nodes=vocab.rename("count").to_frame().assign(degree=0),
+            edges=pd.DataFrame(
+                columns=["source", "target", "cooccur_count", "weight", "rank"]
+            ),
+            measure=measure,
+            window=window,
+            label=_corpus_label(corpus),
+            params={"top_n": top_n, "min_count": min_count, "min_cooccur": min_cooccur},
+        )
+
+    n_total = corpus.total_tokens()
+    rows = []
+    for (src, tgt), joint in pair_counts.items():
+        if joint < min_cooccur:
+            continue
+        rows.append(
+            {
+                "source": src,
+                "target": tgt,
+                "cooccur_count": joint,
+                "f_a": int(vocab[src]),
+                "f_b": int(vocab[tgt]),
+            }
+        )
+    if not rows:
+        return NetworkResult(
+            nodes=vocab.rename("count").to_frame().assign(degree=0),
+            edges=pd.DataFrame(
+                columns=["source", "target", "cooccur_count", "weight", "rank"]
+            ),
+            measure=measure,
+            window=window,
+            label=_corpus_label(corpus),
+            params={"top_n": top_n, "min_count": min_count, "min_cooccur": min_cooccur},
+        )
+
+    edges = pd.DataFrame(rows)
+    f_xy_arr = edges["cooccur_count"].to_numpy(dtype=float) + smoothing
+    f_a_arr = edges["f_a"].to_numpy(dtype=float) + smoothing
+    f_b_arr = edges["f_b"].to_numpy(dtype=float) + smoothing
+
+    if measure == "PMI":
+        weight_arr = np.log2((f_xy_arr * n_total) / (f_a_arr * f_b_arr))
+    elif measure == "t_score":
+        expected_arr = (f_a_arr * f_b_arr) / n_total
+        weight_arr = (f_xy_arr - expected_arr) / np.sqrt(f_xy_arr)
+    elif measure == "MI3":
+        weight_arr = np.log2((np.power(f_xy_arr, 3) * n_total) / (f_a_arr * f_b_arr))
+    elif measure == "logDice":
+        weight_arr = 14.0 + np.log2((2.0 * f_xy_arr) / (f_a_arr + f_b_arr))
+    else:
+        raise ValueError(f"unknown measure={measure!r}")
+
+    # Silence the lint warnings on unused-but-validated helper imports.
+    _ = (pmi, t_score, mi_three, logdice)
+
+    edges = edges.drop(columns=["f_a", "f_b"]).assign(weight=weight_arr)
+    edges = edges.sort_values("weight", ascending=False, key=lambda s: s.abs())
+    edges = edges.reset_index(drop=True).assign(rank=lambda d: d.index.astype(int))
+
+    # Degrees (undirected): how many edges touch each node?
+    degree_a = edges.groupby("source").size()
+    degree_b = edges.groupby("target").size()
+    degrees = degree_a.add(degree_b, fill_value=0).astype(int)
+
+    nodes = vocab.rename("count").to_frame()
+    nodes["degree"] = degrees.reindex(nodes.index, fill_value=0).astype(int)
+
+    return NetworkResult(
+        nodes=nodes,
+        edges=edges,
+        measure=measure,
+        window=window,
+        label=_corpus_label(corpus),
+        params={
+            "top_n": top_n,
+            "min_count": min_count,
+            "min_cooccur": min_cooccur,
+            "smoothing": smoothing,
+        },
+    )
+
+
+def _corpus_label(c: Corpus | CorpusSlice) -> str:
+    if isinstance(c, CorpusSlice):
+        return c.label
+    return "corpus"
+
+
+# Public type alias for users to depend on if they want.
+__all__: Sequence[str] = ["NetworkResult", "cooccurrence_network", "NetworkMeasure"]