pycorpdiff 0.1.0a0__py3-none-any.whl

pycorpdiff/corpus.py

@@ -0,0 +1,411 @@
+"""Core ``Corpus`` and ``CorpusSlice`` data structures.
+
+A :class:`Corpus` wraps a :class:`pandas.DataFrame` of documents plus
+metadata. Slicing returns a :class:`CorpusSlice` that shares the
+parent's configuration (text column, tokenizer) but presents a
+boolean-masked view. Both objects are immutable frozen dataclasses;
+mutations produce new objects.
+
+**Polars interop.** A Corpus stores its documents internally as a
+pandas DataFrame because that's what the analytical layer is built on,
+but the constructors and round-trip helpers accept and produce polars
+DataFrames so they slot into polars-native pipelines:
+
+>>> import polars as pl
+>>> df = pl.DataFrame({"text": ["the cat sat"], "outlet": ["A"]})
+>>> corpus = pcd.from_dataframe(df, text_col="text")  # polars → pandas internally
+>>> corpus.to_polars().shape   # round-trip back
+(1, 2)
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field, replace
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import pandas as pd
+
+from .tokenize import RegexTokenizer, Tokenizer
+
+if TYPE_CHECKING:
+    import polars as pl
+
+    from .temporal.slicing import TemporalCorpus
+
+
+def _doc_term_counts(
+    docs: pd.DataFrame,
+    text_col: str,
+    tokenizer: Tokenizer,
+    min_count: int = 1,
+) -> pd.DataFrame:
+    """Build a docs × term integer count matrix.
+
+    The result is dense (``int64``) and indexed by the parent frame's index.
+    For corpora large enough that a dense ``n_docs × |vocab|`` matrix is
+    infeasible (~10⁵ docs × ~10⁵ vocab → ~80 GB int64), use
+    :meth:`Corpus.doc_term_counts_sparse` instead.
+    """
+    counts_per_doc: list[Counter[str]] = [Counter(tokenizer(t)) for t in docs[text_col]]
+    all_terms: list[str] = sorted({term for c in counts_per_doc for term in c})
+    term_to_idx: dict[str, int] = {t: i for i, t in enumerate(all_terms)}
+
+    data = np.zeros((len(counts_per_doc), len(all_terms)), dtype=np.int64)
+    for i, doc_counts in enumerate(counts_per_doc):
+        for term, n in doc_counts.items():
+            data[i, term_to_idx[term]] = n
+
+    df = pd.DataFrame(data, columns=all_terms, index=docs.index)
+    if min_count > 1:
+        df = df.loc[:, df.sum(axis=0) >= min_count]
+    return df
+
+
+def _doc_term_counts_sparse(
+    docs: pd.DataFrame,
+    text_col: str,
+    tokenizer: Tokenizer,
+    min_count: int = 1,
+) -> tuple[Any, list[str]]:
+    """Build the same docs × term counts as :func:`_doc_term_counts` but sparse.
+
+    Returns ``(csr_matrix, vocab)`` — the canonical scikit-learn shape that
+    sklearn's :class:`CountVectorizer` and gensim's matrix utilities both
+    expose. Memory scales with nnz (non-zero cells), not ``n_docs × |vocab|``.
+
+    Computed by accumulating ``(row, col, count)`` triples in ``coo`` form
+    then converting to ``csr``; vocabulary is sorted lexicographically to
+    match :func:`_doc_term_counts` so the two views agree column-for-column.
+    """
+    from scipy.sparse import coo_matrix
+
+    counts_per_doc: list[Counter[str]] = [Counter(tokenizer(t)) for t in docs[text_col]]
+    all_terms: list[str] = sorted({term for c in counts_per_doc for term in c})
+    term_to_idx: dict[str, int] = {t: i for i, t in enumerate(all_terms)}
+
+    rows: list[int] = []
+    cols: list[int] = []
+    vals: list[int] = []
+    for i, doc_counts in enumerate(counts_per_doc):
+        for term, n in doc_counts.items():
+            rows.append(i)
+            cols.append(term_to_idx[term])
+            vals.append(n)
+
+    n_docs = len(counts_per_doc)
+    n_terms = len(all_terms)
+    matrix = coo_matrix(
+        (np.asarray(vals, dtype=np.int64), (rows, cols)),
+        shape=(n_docs, n_terms),
+        dtype=np.int64,
+    ).tocsr()
+
+    if min_count > 1:
+        col_totals = np.asarray(matrix.sum(axis=0)).ravel()
+        keep_mask = col_totals >= min_count
+        matrix = matrix[:, keep_mask]
+        all_terms = [t for t, k in zip(all_terms, keep_mask, strict=True) if k]
+
+    return matrix, all_terms
+
+
+def _coerce_to_pandas(docs: Any) -> pd.DataFrame:
+    """Accept a pandas or polars DataFrame; return a pandas one.
+
+    The analytical layer is pandas-based; this function is the single
+    boundary where polars input gets converted. ``polars.DataFrame``
+    has ``.to_pandas()`` so the conversion is one method call.
+    """
+    if isinstance(docs, pd.DataFrame):
+        return docs
+    # Defer the polars import — it's an optional dep.
+    try:
+        import polars as pl
+    except ImportError:  # pragma: no cover
+        pl = None  # type: ignore[assignment]
+    if pl is not None and isinstance(docs, pl.DataFrame):
+        return docs.to_pandas()
+    raise TypeError(
+        f"docs must be a pandas or polars DataFrame; got {type(docs).__name__}"
+    )
+
+
+@dataclass(frozen=True, eq=False)
+class Corpus:
+    """A corpus of documents with optional metadata columns.
+
+    Parameters
+    ----------
+    docs
+        A DataFrame whose rows are documents. Must contain at least the
+        text column named by ``text_col``. Accepts either
+        :class:`pandas.DataFrame` or :class:`polars.DataFrame`; polars
+        input is converted to pandas internally.
+    text_col
+        Name of the column containing document text.
+    id_col
+        Name of an optional unique-document-id column.
+    meta_cols
+        Tuple of column names treated as metadata available for slicing.
+        If empty (the default), every non-text column is considered
+        metadata.
+    tokenizer
+        A callable conforming to :class:`pycorpdiff.tokenize.Tokenizer`.
+        Defaults to the package's :class:`RegexTokenizer`.
+
+    Hashability
+    -----------
+
+    :class:`Corpus` is hashable with a content-derived hash — two
+    corpora with the same documents, schema, and tokenizer hash the
+    same. The hash uses :func:`pandas.util.hash_pandas_object` so it's
+    fast (O(N) over rows, vectorised) but deterministic. Use a Corpus
+    as a dict key when memoising analyses or building reproducibility
+    caches.
+    """
+
+    docs: pd.DataFrame
+    text_col: str = "text"
+    id_col: str | None = None
+    meta_cols: tuple[str, ...] = ()
+    tokenizer: Tokenizer = field(default_factory=RegexTokenizer)
+
+    def __post_init__(self) -> None:
+        # Coerce polars to pandas if needed; otherwise leave alone.
+        if not isinstance(self.docs, pd.DataFrame):
+            object.__setattr__(self, "docs", _coerce_to_pandas(self.docs))
+        if self.text_col not in self.docs.columns:
+            raise ValueError(
+                f"text_col={self.text_col!r} not found in DataFrame columns "
+                f"{list(self.docs.columns)!r}"
+            )
+        if self.id_col is not None and self.id_col not in self.docs.columns:
+            raise ValueError(
+                f"id_col={self.id_col!r} not found in DataFrame columns "
+                f"{list(self.docs.columns)!r}"
+            )
+
+    def __len__(self) -> int:
+        return len(self.docs)
+
+    def __hash__(self) -> int:
+        """Content-derived hash for cache keys / reproducibility checks.
+
+        Combines a fast vectorised hash of every document row with the
+        corpus configuration (text/id/meta columns + tokenizer repr).
+        Two corpora with identical docs, schema, and tokenizer hash the
+        same; mutating any of those (in a copy — Corpus is frozen)
+        produces a different hash.
+        """
+        row_hash = int(pd.util.hash_pandas_object(self.docs, index=False).sum()) & 0xFFFFFFFFFFFFFFFF
+        return hash(
+            (row_hash, self.text_col, self.id_col, self.meta_cols, repr(self.tokenizer))
+        )
+
+    def __eq__(self, other: object) -> bool:
+        """Two Corpora are equal iff their hashes agree on content + config."""
+        if not isinstance(other, Corpus):
+            return NotImplemented
+        return hash(self) == hash(other)
+
+    @property
+    def metadata_columns(self) -> tuple[str, ...]:
+        """Effective metadata columns — explicit if given, else inferred."""
+        if self.meta_cols:
+            return self.meta_cols
+        return tuple(c for c in self.docs.columns if c != self.text_col)
+
+    def slice(self, **filters: Any) -> CorpusSlice:
+        """Return a :class:`CorpusSlice` filtered on metadata columns.
+
+        Each keyword argument is a column name; the value may be a scalar
+        (exact match) or an iterable (membership). All conditions are
+        combined with logical AND.
+        """
+        mask = pd.Series(True, index=self.docs.index)
+        for col, value in filters.items():
+            if col not in self.docs.columns:
+                raise KeyError(f"slice() got unknown column {col!r}")
+            if isinstance(value, (list, tuple, set, pd.Series)):
+                mask &= self.docs[col].isin(list(value))
+            else:
+                mask &= self.docs[col] == value
+        return CorpusSlice(parent=self, mask=mask, filters=dict(filters))
+
+    def by_time(self, col: str, freq: str = "Y") -> TemporalCorpus:
+        """Return a :class:`TemporalCorpus` indexed by time-period.
+
+        ``col`` must be parseable as datetime; ``freq`` is any pandas
+        offset alias (``"Y"``, ``"Q"``, ``"M"``, ``"W"``, ``"D"``).
+        """
+        from .temporal.slicing import TemporalCorpus  # local import to break cycle
+
+        return TemporalCorpus(parent=self, time_col=col, freq=freq)
+
+    def with_tokenizer(self, tokenizer: Tokenizer) -> Corpus:
+        """Return a copy of the corpus with a different tokenizer."""
+        return replace(self, tokenizer=tokenizer)
+
+    def tokens(self) -> list[list[str]]:
+        """Tokenize every document; return one list of tokens per doc."""
+        return [self.tokenizer(t) for t in self.docs[self.text_col]]
+
+    def doc_term_counts(self, min_count: int = 1) -> pd.DataFrame:
+        """Return a docs × term integer count DataFrame."""
+        return _doc_term_counts(self.docs, self.text_col, self.tokenizer, min_count)
+
+    def doc_term_counts_sparse(self, min_count: int = 1) -> tuple[Any, list[str]]:
+        """Return the docs × term matrix in :class:`scipy.sparse` form.
+
+        Returns ``(matrix, vocab)`` where ``matrix`` is a
+        :class:`scipy.sparse.csr_matrix` of shape ``(n_docs, |vocab|)``
+        and ``vocab`` is the lexicographically-sorted list of column
+        terms. This is the canonical scikit-learn shape, and lets you
+        plug a pycorpdiff corpus directly into anything that expects
+        :class:`~sklearn.feature_extraction.text.CountVectorizer` output.
+
+        For typical analytical work the dense
+        :meth:`doc_term_counts` is fine — but on a corpus with ~100K
+        docs and ~50K vocab the dense int64 matrix is ~40 GB while the
+        sparse CSR is megabytes. Use this when the dense matrix would
+        blow your RAM budget.
+        """
+        return _doc_term_counts_sparse(
+            self.docs, self.text_col, self.tokenizer, min_count
+        )
+
+    def vocab(self, min_count: int = 1) -> pd.Series:
+        """Return a term → total-count Series sorted descending."""
+        counts = self.doc_term_counts(min_count=min_count).sum(axis=0)
+        return counts.rename("count").sort_values(ascending=False)
+
+    def total_tokens(self) -> int:
+        """Total tokens across all documents (before any min_count filter)."""
+        return int(self.doc_term_counts(min_count=1).values.sum())
+
+    def to_polars(self) -> pl.DataFrame:
+        """Return the corpus's documents as a polars DataFrame.
+
+        Requires the ``polars`` extra. The original pandas index is
+        dropped (polars has no concept of a row index); the document
+        ordering is preserved.
+        """
+        try:
+            import polars as pl
+        except ImportError as exc:  # pragma: no cover
+            raise ImportError(
+                "to_polars() requires polars. Install with: pip install 'pycorpdiff[polars]'"
+            ) from exc
+        return pl.from_pandas(self.docs.reset_index(drop=True))
+
+
+@dataclass(frozen=True)
+class CorpusSlice:
+    """A boolean-masked view of a :class:`Corpus`.
+
+    Slices behave like corpora for downstream analytical purposes —
+    they expose the same ``docs``, ``text_col``, ``tokenizer`` surface —
+    but also remember the ``filters`` that produced them, which the
+    :class:`pycorpdiff.compare.Comparison` machinery uses to label
+    plots and result tables.
+    """
+
+    parent: Corpus
+    mask: pd.Series
+    filters: dict[str, Any]
+
+    @property
+    def docs(self) -> pd.DataFrame:
+        return self.parent.docs.loc[self.mask]
+
+    @property
+    def text_col(self) -> str:
+        return self.parent.text_col
+
+    @property
+    def id_col(self) -> str | None:
+        return self.parent.id_col
+
+    @property
+    def tokenizer(self) -> Tokenizer:
+        return self.parent.tokenizer
+
+    def __len__(self) -> int:
+        return int(self.mask.sum())
+
+    @property
+    def label(self) -> str:
+        """A short human-readable label derived from the slice's filters."""
+        if not self.filters:
+            return "slice"
+        return ", ".join(f"{k}={v!r}" for k, v in self.filters.items())
+
+    def tokens(self) -> list[list[str]]:
+        """Tokenize every document in the slice."""
+        return [self.tokenizer(t) for t in self.docs[self.text_col]]
+
+    def doc_term_counts(self, min_count: int = 1) -> pd.DataFrame:
+        return _doc_term_counts(self.docs, self.text_col, self.tokenizer, min_count)
+
+    def doc_term_counts_sparse(self, min_count: int = 1) -> tuple[Any, list[str]]:
+        """Sparse counterpart to :meth:`doc_term_counts`. See
+        :meth:`Corpus.doc_term_counts_sparse` for semantics.
+        """
+        return _doc_term_counts_sparse(
+            self.docs, self.text_col, self.tokenizer, min_count
+        )
+
+    def vocab(self, min_count: int = 1) -> pd.Series:
+        counts = self.doc_term_counts(min_count=min_count).sum(axis=0)
+        return counts.rename("count").sort_values(ascending=False)
+
+    def total_tokens(self) -> int:
+        return int(self.doc_term_counts(min_count=1).values.sum())
+
+    def slice(self, **filters: Any) -> CorpusSlice:
+        """Further filter this slice — produces a new CorpusSlice on the
+        same parent with the masks AND-ed and the filters merged.
+
+        Lets you chain: ``corpus.slice(topic="x").slice(party="y")``.
+        """
+        new_mask = self.mask.copy()
+        merged_filters = dict(self.filters)
+        for col, value in filters.items():
+            if col not in self.parent.docs.columns:
+                raise KeyError(f"slice() got unknown column {col!r}")
+            if isinstance(value, (list, tuple, set, pd.Series)):
+                new_mask &= self.parent.docs[col].isin(list(value))
+            else:
+                new_mask &= self.parent.docs[col] == value
+            merged_filters[col] = value
+        return CorpusSlice(parent=self.parent, mask=new_mask, filters=merged_filters)
+
+    def by_time(self, col: str, freq: str = "Y") -> TemporalCorpus:
+        """Return a TemporalCorpus over the slice's documents only.
+
+        Materialises the slice's masked rows into a fresh Corpus, then
+        delegates to :meth:`Corpus.by_time`. Lets you chain
+        ``corpus.slice(topic="x").by_time("date")``.
+        """
+        from .temporal.slicing import TemporalCorpus  # local import to break cycle
+
+        fresh = Corpus(
+            docs=self.docs.reset_index(drop=True),
+            text_col=self.text_col,
+            id_col=self.id_col,
+            meta_cols=self.parent.meta_cols,
+            tokenizer=self.tokenizer,
+        )
+        return TemporalCorpus(parent=fresh, time_col=col, freq=freq)
+
+    def to_polars(self) -> pl.DataFrame:
+        """Return the slice's documents as a polars DataFrame."""
+        try:
+            import polars as pl
+        except ImportError as exc:  # pragma: no cover
+            raise ImportError(
+                "to_polars() requires polars. Install with: pip install 'pycorpdiff[polars]'"
+            ) from exc
+        return pl.from_pandas(self.docs.reset_index(drop=True))

pycorpdiff/datasets/__init__.py

@@ -0,0 +1,27 @@
+"""Bundled corpora for demonstrations, tutorials, and reproducible tests.
+
+What ships with the package
+---------------------------
+
+- :func:`load_hansard_sample` — a 200-speech synthetic corpus designed
+  to mimic UK Hansard's structure across two decades, four topics, and
+  four parties, with topical language shifts around real-world events
+  (Brexit referendum, COVID-19, the climate-emergency declarations).
+
+The sample is **synthetic** but its structure is realistic enough to
+demo every analytical surface in pycorpdiff. For an actual research
+project users will want the real Hansard archive — see the docstring on
+:func:`load_hansard_sample` for the canonical download paths.
+"""
+
+from __future__ import annotations
+
+from .hansard import fetch_hansard, load_hansard_sample
+from .histwords import fetch_histwords_decade, histwords_cosine_shift
+
+__all__ = [
+    "fetch_hansard",
+    "fetch_histwords_decade",
+    "histwords_cosine_shift",
+    "load_hansard_sample",
+]

pycorpdiff/datasets/_generate_hansard.py

@@ -0,0 +1,221 @@
+"""Deterministically generate the synthetic Hansard-sample parquet.
+
+This script is the source-of-truth for ``hansard_sample.parquet``. It is
+*not* run at import time — the parquet is committed and shipped with the
+package, and :func:`load_hansard_sample` just reads it. The script is
+here so reviewers can verify the sample is reproducible and so we can
+regenerate it when the templates change.
+
+Run with::
+
+    python -m pycorpdiff.datasets._generate_hansard
+
+The output is written to ``src/pycorpdiff/datasets/_data/hansard_sample.parquet``.
+The generator is seeded so output is byte-identical across machines and
+Python versions.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+OPENINGS = [
+    "I rise to address the House on",
+    "I beg leave to bring to the attention of this House the question of",
+    "Mr Speaker, I wish to make a statement concerning",
+    "I am grateful for the opportunity to debate",
+    "The Honourable Members of this House should consider",
+    "Madam Deputy Speaker, I wish to address",
+    "I am pleased to speak on the matter of",
+]
+
+CLOSINGS = [
+    "I commend this motion to the House.",
+    "I urge my Honourable colleagues to support this.",
+    "The Government must take immediate action.",
+    "We owe this much to our constituents.",
+    "This matter brooks no further delay.",
+    "The time for inaction has long passed.",
+    "I beg to move.",
+]
+
+# (topic, period_label) -> list of body templates.
+# The period_label structure encodes the temporal-frame shifts:
+#   - immigration shifts from humanising (pre-2016) to criminalising (post-2016)
+#   - brexit goes through emerging → peak → aftermath
+#   - nhs has a steady frame + crisis spikes in 2010 (austerity) and 2020 (covid)
+#   - climate sharpens scientific → policy → crisis post-2019
+TOPIC_BODIES: dict[tuple[str, str], list[str]] = {
+    ("immigration", "humanising"): [
+        "the immigrant worker arrived with hope and the immigrant family settled with dignity",
+        "the immigrant community contributes to our shared prosperity and shared future",
+        "the immigrant family deserves protection refuge and a clear path to citizenship",
+        "the immigrant worker rights advance through union solidarity and our labour movement",
+        "the immigrant community organised with strength and the immigrant worker spoke with pride",
+        "the immigrant family brings cultural richness and economic vitality to our towns",
+        "the immigrant worker contributes to public services education and our national life",
+        "the immigrant community has thrived and the immigrant family has flourished here",
+    ],
+    ("immigration", "criminalising"): [
+        "the immigrant criminal threat grows and the immigrant invasion of gangs spreads",
+        "the immigrant criminal element alarms residents and our border control has failed",
+        "the immigrant invasion narrative dominates news and immigrant criminal gangs persist",
+        "the immigrant threat has increased and the immigrant crime narrative grew with concern",
+        "the immigrant gangs threaten the border and the immigrant criminal risk grows daily",
+        "the immigrant criminal threat must be confronted and the immigrant invasion halted",
+        "the immigrant criminal gangs operate freely and immigrant invasion routes remain open",
+    ],
+    ("brexit", "emerging"): [
+        "the european question must be addressed and the referendum we promised must be delivered",
+        "our relationship with europe requires renegotiation and reform from this government",
+        "the european union framework no longer serves british interests and british sovereignty",
+        "the european treaties demand renegotiation before the public can give their consent",
+        "the european question divides this house but the people must have their say",
+    ],
+    ("brexit", "peak"): [
+        "the brexit referendum result must be respected and delivered without further delay",
+        "the brexit deal must respect the democratic will of seventeen million leave voters",
+        "the brexit transition must protect british businesses and british workers from harm",
+        "the brexit negotiations require firm leadership and a clear vision for our nation",
+        "the brexit outcome will define this generation and the brexit deal must be honoured",
+        "the brexit settlement requires patience but the brexit mandate is unambiguous",
+    ],
+    ("brexit", "aftermath"): [
+        "the brexit deal has delivered for british sovereignty and british democratic accountability",
+        "the brexit aftermath reveals supply chain disruption and significant economic adjustment",
+        "the brexit transition continues with new opportunities for global trade and partnership",
+        "the brexit dividend has yet to materialise for working families across our nation",
+        "the brexit settlement requires further work on northern ireland and our customs arrangements",
+    ],
+    ("nhs", "normal"): [
+        "the national health service requires sustained investment for our nurses and doctors",
+        "the nhs workforce must be supported with proper funding and training programmes",
+        "the patient care standards must be maintained across all hospitals and trusts",
+        "the nhs provides universal care and the nhs principles remain our foundation",
+        "the national health service belongs to all of us and to our future generations",
+    ],
+    ("nhs", "austerity"): [
+        "the nhs austerity cuts threaten patient care and waiting times grow alarming",
+        "the nhs underfunding creates crises in accident and emergency departments nationwide",
+        "the nhs austerity decisions cost lives and the nhs underfunding harms our communities",
+        "the nhs cuts must be reversed and the nhs funding settlement must be honoured",
+    ],
+    ("nhs", "covid"): [
+        "the nhs response to the pandemic deserves our gratitude and continued support",
+        "the nhs covid crisis demands emergency funding and ventilator capacity now",
+        "the nhs frontline workers face unprecedented pressure and the nhs covid response saves lives",
+        "the nhs covid surge requires us to clap and to legislate for fair pay",
+    ],
+    ("climate", "scientific"): [
+        "the scientific consensus on climate change requires policy response from this government",
+        "the climate models indicate warming trends that demand emissions reduction targets",
+        "the climate science is settled and the climate research is unambiguous in its conclusions",
+        "the climate evidence accumulates and the climate scientists call for action",
+    ],
+    ("climate", "policy"): [
+        "the climate policy framework must align with our paris agreement obligations",
+        "the climate change committee recommends carbon budget reductions for the coming decade",
+        "the climate policy must include just transition for fossil fuel workers and communities",
+        "the climate framework needs revision and the climate policy targets need strengthening",
+    ],
+    ("climate", "crisis"): [
+        "the climate crisis is here now and the climate emergency demands urgent action",
+        "the climate breakdown threatens our coastlines and the climate emergency cannot wait",
+        "the climate crisis requires immediate emissions cuts and the climate emergency is upon us",
+        "the climate disaster unfolds and the climate emergency response must accelerate",
+        "the climate emergency declaration must be backed by the climate action this house owes",
+    ],
+}
+
+# Period predicates for each topic. Each yields a (year) -> period_label
+# mapping that decides which template bucket a speech in that year uses.
+def _immigration_period(year: int) -> str:
+    return "humanising" if year < 2016 else "criminalising"
+
+
+def _brexit_period(year: int) -> str:
+    if year < 2016:
+        return "emerging"
+    if year < 2020:
+        return "peak"
+    return "aftermath"
+
+
+def _nhs_period(year: int) -> str:
+    if 2010 <= year <= 2014:
+        return "austerity"
+    if 2020 <= year <= 2022:
+        return "covid"
+    return "normal"
+
+
+def _climate_period(year: int) -> str:
+    if year < 2011:
+        return "scientific"
+    if year < 2019:
+        return "policy"
+    return "crisis"
+
+
+PERIOD_FOR = {
+    "immigration": _immigration_period,
+    "brexit": _brexit_period,
+    "nhs": _nhs_period,
+    "climate": _climate_period,
+}
+
+PARTIES = ["Labour", "Conservative", "Liberal Democrat", "SNP"]
+TOPICS = ["immigration", "brexit", "nhs", "climate"]
+
+
+def generate(seed: int = 20260522) -> pd.DataFrame:
+    """Return a deterministic 200-speech synthetic Hansard sample."""
+    rng = np.random.default_rng(seed)
+    rows: list[dict[str, object]] = []
+    speech_id = 0
+    for year in range(2005, 2024):
+        # Roughly 10-11 speeches per year. Brexit and immigration get
+        # more airtime in years close to the referendum.
+        n_speeches = 10 + (1 if year in {2016, 2017, 2019} else 0)
+        for _ in range(n_speeches):
+            topic = TOPICS[int(rng.integers(0, len(TOPICS)))]
+            period = PERIOD_FOR[topic](year)
+            body_pool = TOPIC_BODIES[(topic, period)]
+            body = body_pool[int(rng.integers(0, len(body_pool)))]
+            opening = OPENINGS[int(rng.integers(0, len(OPENINGS)))]
+            closing = CLOSINGS[int(rng.integers(0, len(CLOSINGS)))]
+            party = PARTIES[int(rng.integers(0, len(PARTIES)))]
+            month = int(rng.integers(1, 13))
+            day = int(rng.integers(1, 28))
+            rows.append(
+                {
+                    "speech_id": speech_id,
+                    "text": f"{opening} {topic}. {body}. {closing}",
+                    "topic": topic,
+                    "frame": period,
+                    "party": party,
+                    "date": f"{year}-{month:02d}-{day:02d}",
+                    "year": year,
+                }
+            )
+            speech_id += 1
+    return pd.DataFrame(rows)
+
+
+def main() -> None:
+    df = generate()
+    out_path = Path(__file__).parent / "_data" / "hansard_sample.parquet"
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_parquet(out_path, index=False)
+    print(f"wrote {len(df)} speeches to {out_path}")
+    print(f"topic distribution: {df['topic'].value_counts().to_dict()}")
+    print(f"frame distribution: {df['frame'].value_counts().to_dict()}")
+    print(f"party distribution: {df['party'].value_counts().to_dict()}")
+    print(f"year range: {df['year'].min()}–{df['year'].max()}")
+
+
+if __name__ == "__main__":
+    main()