jlink 0.1.0__py3-none-any.whl

jlink/__init__.py

@@ -0,0 +1,19 @@
+"""jlink: record linkage with match rules in plain English, judged by TypeSafe's Jev model.
+
+    import jlink
+    result = jlink.link(left, right, entity="firm", on=["name", "state"],
+                        definition="A parent company and its subsidiary are different firms.")
+    result.links            # the chosen pairs, with probabilities
+    print(result.methods()) # a paragraph for the data appendix
+"""
+
+__version__ = "0.1.0"
+
+from . import block  # noqa: E402
+from .audit import Evaluation, audit_sample, evaluate, score_against_truth  # noqa: E402
+from .judge import judge  # noqa: E402
+from .linker import Linker, Result, link, load  # noqa: E402
+from .resolve import resolve  # noqa: E402
+
+__all__ = ["Linker", "Result", "link", "load", "block", "judge", "resolve", "audit_sample", "evaluate",
+           "score_against_truth", "Evaluation", "__version__"]

jlink/__main__.py

@@ -0,0 +1,3 @@
+from .cli import cli
+
+cli()

jlink/audit.py

@@ -0,0 +1,297 @@
+"""Samples for hand labeling and weighted evidence about linkage quality."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+import pandas as pd
+
+from .fields import check_columns, ids, parse_on
+from .resolve import _columns, _number, _numbers, _pairs, _probabilities
+
+
+def _integer(value: int, name: str, minimum: int) -> int:
+    if isinstance(value, (bool, np.bool_)) or not isinstance(value, (int, np.integer)) or value < minimum:
+        raise ValueError(f"{name} must be a whole number at least {minimum}")
+    return int(value)
+
+
+def _allocation(sizes: np.ndarray, n: int) -> np.ndarray:
+    """Equal allocation, repeatedly redistributing places left by exhausted bins."""
+    counts = np.zeros(len(sizes), dtype=np.int64)
+    remaining = min(n, int(sizes.sum()))
+    while remaining:
+        available = np.flatnonzero(counts < sizes)
+        share, remainder = divmod(remaining, len(available))
+        extra = np.full(len(available), share, dtype=np.int64)
+        extra[:remainder] += 1
+        extra = np.minimum(extra, sizes[available] - counts[available])
+        counts[available] += extra
+        remaining -= int(extra.sum())
+    return counts
+
+
+def _fields(sample: pd.DataFrame, left: pd.DataFrame, right: pd.DataFrame, on,
+            left_id: str | None, right_id: str | None) -> list[str]:
+    fields = parse_on(on)
+    labels = [label for label, _, _ in fields]
+    if len(set(labels)) != len(labels):
+        raise ValueError("on must give each field a different left column name")
+    positions = []
+    for frame, column, side in ((left, left_id, "left"), (right, right_id, "right")):
+        _columns(frame, [], f"the {side} data")
+        values = ids(frame, column, side)
+        if values.isna().any():
+            raise ValueError(f"the {side} data has missing IDs; supply an ID for every record")
+        indexer = values.get_indexer(sample[f"{side}_id"])
+        if (indexer < 0).any():
+            raise ValueError(f"sample column '{side}_id' contains IDs absent from the {side} data")
+        positions.append(indexer)
+    columns = []
+    for label, a, b in fields:
+        check_columns(left, [a], "left")
+        check_columns(right, [b], "right")
+        sample[f"a_{label}"] = left[a].iloc[positions[0]].reset_index(drop=True)
+        sample[f"b_{label}"] = right[b].iloc[positions[1]].reset_index(drop=True)
+        columns.extend([f"a_{label}", f"b_{label}"])
+    return columns
+
+
+def audit_sample(scores: pd.DataFrame, *, n: int = 200,
+                 bins: tuple[float, ...] = (0, 0.05, 0.2, 0.5, 0.8, 0.95, 1.0), seed: int = 0,
+                 left: pd.DataFrame | None = None, right: pd.DataFrame | None = None, on=None,
+                 left_id: str | None = None, right_id: str | None = None) -> pd.DataFrame:
+    """Draw an equally allocated stratified sample, with inverse inclusion weights.
+
+    Nonempty bin categories are retained even when n is too small to sample every bin.
+    Saving to CSV loses unused categories; label every sampled bin before evaluation.
+    """
+    n = _integer(n, "n", 0)
+    seed = _integer(seed, "seed", 0)
+    _pairs(scores, "scores")
+    p = _probabilities(scores)
+    try:
+        edges = np.asarray(bins, dtype=float)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("bins must be increasing numeric boundaries from 0 to 1") from exc
+    if (edges.ndim != 1 or len(edges) < 2 or not np.isfinite(edges).all()
+            or edges[0] != 0 or edges[-1] != 1 or (np.diff(edges) <= 0).any()):
+        raise ValueError("bins must be strictly increasing boundaries starting at 0 and ending at 1")
+    supplied = (left is not None, right is not None, on is not None)
+    if any(supplied) and not all(supplied):
+        raise ValueError("provide left, right and on together to include fields for labeling")
+    frame = scores.loc[p.notna(), ["left_id", "right_id"]].copy().reset_index(drop=True)
+    frame["p"] = p.loc[p.notna()].to_numpy()
+    labels = [f"{'[' if i == 0 else '('}{low:g}, {high:g}]"
+              for i, (low, high) in enumerate(zip(edges[:-1], edges[1:]))]
+    if len(set(labels)) != len(labels):
+        # Preserve distinct labels even for boundaries that differ past six significant digits.
+        labels = [f"{'[' if i == 0 else '('}{low!r}, {high!r}]"
+                  for i, (low, high) in enumerate(zip(edges[:-1].tolist(), edges[1:].tolist()))]
+    frame["bin"] = pd.cut(frame["p"], edges, labels=labels, right=True, include_lowest=True)
+    frame["bin"] = frame["bin"].cat.remove_unused_categories()
+    groups = frame.groupby("bin", observed=True, sort=True).indices
+    counts = _allocation(np.asarray([len(group) for group in groups.values()]), n)
+    rng = np.random.default_rng(seed)
+    samples = []
+    for group, count in zip(groups.values(), counts):
+        if count:
+            part = frame.iloc[rng.choice(group, size=count, replace=False)].copy()
+            part["weight"] = len(group) / int(count)
+            samples.append(part)
+    if samples:
+        sample = pd.concat(samples, ignore_index=True)
+        sample = sample.iloc[rng.permutation(len(sample))].reset_index(drop=True)
+    else:
+        sample = frame.iloc[:0].copy()
+        sample["weight"] = pd.Series(dtype=float)
+    sample["is_match"] = pd.Series(pd.NA, index=sample.index, dtype="object")
+    fields = _fields(sample, left, right, on, left_id, right_id) if all(supplied) else []
+    return sample[["is_match", *fields, "p", "bin", "weight", "left_id", "right_id"]]
+
+
+def _labels(values: pd.Series) -> pd.Series:
+    text = values.astype("string").str.strip().str.casefold()
+    mapping = {"1": 1.0, "1.0": 1.0, "true": 1.0, "y": 1.0, "yes": 1.0,
+               "0": 0.0, "0.0": 0.0, "false": 0.0, "n": 0.0, "no": 0.0}
+    blank = text.isna() | text.eq("").fillna(False)
+    invalid = ~blank & ~text.isin(mapping)
+    if invalid.any():
+        bad = values.loc[invalid].iloc[0]
+        raise ValueError(f"column 'is_match' has {bad!r}; use 1/0, True/False, y/n, yes/no, or blank")
+    return text.map(mapping).astype(float)
+
+
+def _metrics(totals: np.ndarray) -> np.ndarray:
+    tp, linked, true = np.moveaxis(totals, -1, 0)
+    numerator = np.stack([tp, tp, 2 * tp], axis=-1)
+    denominator = np.stack([linked, true, linked + true], axis=-1)
+    return np.divide(numerator, denominator, out=np.full_like(numerator, np.nan), where=denominator > 0)
+
+
+def _bootstrap(contributions: np.ndarray, groups: dict, n_boot: int, seed: int) -> np.ndarray:
+    rng = np.random.default_rng(seed)
+    totals = np.zeros((n_boot, 3))
+    for group in groups.values():
+        size = len(group)
+        if not size:
+            continue
+        # Bound the temporary draws for large audits, while vectorizing over replicates.
+        batch = max(1, 250_000 // size)
+        values = contributions[group]
+        for start in range(0, n_boot, batch):
+            stop = min(n_boot, start + batch)
+            draws = rng.integers(size, size=(stop - start, size))
+            totals[start:stop] += values[draws].sum(axis=1)
+    return _metrics(totals)
+
+
+def _format(value: float) -> str:
+    return "NaN" if np.isnan(value) else f"{value:.4f}"
+
+
+@dataclass
+class Evaluation:
+    """Weighted audit estimates and percentile intervals from a stratified bootstrap."""
+
+    precision: tuple[float, float, float]
+    recall: tuple[float, float, float]
+    f1: tuple[float, float, float]
+    brier: float
+    calibration: pd.DataFrame
+    n_labeled: int
+    n_unlabeled: int
+    threshold: float
+    _notes: tuple[str, ...] = field(default=(), repr=False)
+
+    def summary(self) -> str:
+        """Describe the estimand and explain unavailable estimates."""
+        parts = [f"{self.n_labeled:,} labeled pairs; {self.n_unlabeled:,} blank labels dropped; "
+                 f"threshold {self.threshold:g}."]
+        for name in ("precision", "recall", "f1"):
+            estimate, low, high = getattr(self, name)
+            label = "F1" if name == "f1" else name.capitalize()
+            parts.append(f"{label} {_format(estimate)} (95% CI {_format(low)} to {_format(high)}).")
+        parts.append(f"Weighted Brier score {_format(self.brier)}.")
+        parts.append("Estimates use sampling weights; 95% intervals use a bootstrap within bins.")
+        parts.append("Recall is among candidate pairs and cannot see true matches lost in blocking.")
+        parts.extend(self._notes)
+        return " ".join(parts)
+
+    def to_markdown(self) -> str:
+        """Return appendix tables without requiring pandas' optional tabulate dependency."""
+        lines = ["| Metric | Estimate | 95% interval |", "|:--|--:|:--|"]
+        for name, label in (("precision", "Precision"), ("recall", "Candidate-pair recall"), ("f1", "F1")):
+            estimate, low, high = getattr(self, name)
+            lines.append(f"| {label} | {_format(estimate)} | {_format(low)} to {_format(high)} |")
+        lines.append(f"| Weighted Brier score | {_format(self.brier)} | — |")
+        lines.extend(["", "| Probability bin | Labeled pairs | Weighted mean p | Weighted match rate |",
+                      "|:--|--:|--:|--:|"])
+        for row in self.calibration.itertuples(index=False):
+            label = str(row.bin).replace("|", "\\|").replace("\n", " ").replace("\r", " ")
+            lines.append(f"| {label} | {row.n:,} | {_format(row.mean_p)} | {_format(row.match_rate)} |")
+        lines.extend(["", self.summary()])
+        return "\n".join(lines)
+
+
+def evaluate(labeled: pd.DataFrame, *, threshold: float = 0.5,
+             n_boot: int = 2000, seed: int = 0) -> Evaluation:
+    """Estimate linkage accuracy using inverse inclusion weights and within-bin resampling.
+
+    Entirely unlabeled strata make population-wide estimates unavailable. Partial labeling
+    uses the supplied weights for the available judgments; it assumes missing labels do not
+    introduce selection bias. F1 is 2 TP / (predicted links + true matches).
+    """
+    threshold = _number(threshold, "threshold", probability=True)
+    n_boot = _integer(n_boot, "n_boot", 1)
+    seed = _integer(seed, "seed", 0)
+    _columns(labeled, ["p", "bin", "weight", "is_match"], "labeled data")
+    frame = labeled.reset_index(drop=True).copy()
+    y = _labels(frame["is_match"])
+    keep = y.notna()
+    n_labeled, n_unlabeled = int(keep.sum()), int((~keep).sum())
+    if frame["bin"].isna().any():
+        raise ValueError("column 'bin' must identify a sampling stratum for every pair")
+    strata = (list(frame["bin"].cat.categories) if isinstance(frame["bin"].dtype, pd.CategoricalDtype)
+              else list(pd.unique(frame["bin"])))
+    judged = frame.loc[keep].reset_index(drop=True)
+    p = _probabilities(judged, missing=False).to_numpy()
+    weights = _numbers(judged, "weight").to_numpy()
+    if (weights <= 0).any():
+        raise ValueError("column 'weight' must contain positive sampling weights")
+    if n_labeled:
+        # Ratios are invariant to a common scale; avoid overflow with large population weights.
+        weights = weights / weights.max()
+    truth = y.loc[keep].to_numpy()
+    predicted = p >= threshold
+    contributions = weights[:, None] * np.column_stack([predicted * truth, predicted, truth])
+    totals = contributions.sum(axis=0)
+    groups = judged.groupby("bin", observed=True, sort=False).indices
+    calibration = []
+    empty = []
+    for stratum in strata:
+        group = groups.get(stratum, np.array([], dtype=int))
+        if len(group):
+            mean_p = float(np.average(p[group], weights=weights[group]))
+            rate = float(np.average(truth[group], weights=weights[group]))
+        else:
+            mean_p = rate = float("nan")
+            empty.append(stratum)
+        calibration.append((stratum, len(group), mean_p, rate))
+    table = pd.DataFrame(calibration, columns=["bin", "n", "mean_p", "match_rate"])
+    notes = []
+    estimates = _metrics(totals)
+    intervals = np.full((3, 2), np.nan)
+    brier = float(np.average((p - truth) ** 2, weights=weights)) if n_labeled else float("nan")
+    if not n_labeled:
+        notes.append("No labeled pairs: all estimates and intervals are NaN.")
+    if empty:
+        names = ", ".join(str(value) for value in empty)
+        notes.append(f"No labels in bin(s) {names}: population-wide estimates and intervals are NaN.")
+    if not n_labeled or empty:
+        estimates[:] = np.nan
+        brier = float("nan")
+    else:
+        if totals[1] == 0:
+            notes.append("No predicted links at this threshold: precision and its interval are NaN.")
+        if totals[2] == 0:
+            notes.append("No true matches among labeled pairs: recall and its interval are NaN.")
+        if totals[1] + totals[2] == 0:
+            notes.append("No predicted links or true matches: F1 and its interval are NaN.")
+        boot = _bootstrap(contributions, groups, n_boot, seed)
+        for index, name in enumerate(("precision", "recall", "F1")):
+            finite = np.isfinite(boot[:, index])
+            if np.isfinite(estimates[index]) and finite.any():
+                intervals[index] = np.quantile(boot[finite, index], [0.025, 0.975])
+                if not finite.all():
+                    notes.append(f"{n_boot - int(finite.sum()):,} bootstrap replicates with undefined {name} "
+                                 "were excluded from that interval.")
+        if any(len(group) == 1 for group in groups.values()):
+            notes.append("A bin has only one label; its within-bin uncertainty cannot be estimated "
+                         "by resampling. Label more pairs for reliable intervals.")
+    if n_unlabeled and n_labeled:
+        notes.append("Available labels retain their sampling weights; "
+                     "selective missing labels can bias estimates.")
+    metrics = [tuple(float(value) for value in (estimate, *interval))
+               for estimate, interval in zip(estimates, intervals)]
+    return Evaluation(*metrics, brier, table, n_labeled, n_unlabeled, threshold, tuple(notes))
+
+
+def score_against_truth(links: pd.DataFrame, truth: pd.DataFrame,
+                        candidates: pd.DataFrame | None = None) -> dict:
+    """Compare unique linked pairs with known truth; optionally measure blocking recall."""
+    _pairs(links, "links")
+    _pairs(truth, "truth")
+    index = pd.MultiIndex.from_frame(truth[["left_id", "right_id"]])
+    linked = pd.MultiIndex.from_frame(links[["left_id", "right_id"]])
+    tp = int(linked.isin(index).sum())
+    fp, fn = len(links) - tp, len(truth) - tp
+    precision, recall, f1 = _metrics(np.array([tp, len(links), len(truth)], dtype=float))
+    result = {"precision": float(precision), "recall": float(recall), "f1": float(f1),
+              "tp": tp, "fp": fp, "fn": fn}
+    if candidates is not None:
+        _pairs(candidates, "candidates")
+        proposed = pd.MultiIndex.from_frame(candidates[["left_id", "right_id"]])
+        result["pairs_completeness"] = float(index.isin(proposed).mean()) if len(index) else float("nan")
+    return result

jlink/block.py

@@ -0,0 +1,302 @@
+"""Cheap, deterministic candidate passes before pairs are sent to the judge."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from numbers import Integral, Real
+
+import numpy as np
+import pandas as pd
+from scipy.sparse import csr_matrix
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+from .fields import check_columns, ids, normalize, parse_on, record_text
+
+# Bound even a fully populated sparse product to about 64 MiB (float64 + int32).
+_CHUNK_ROWS = 256
+_PRODUCT_BYTES = 64 * 1024 * 1024
+_PAIR_CHUNK_ROWS = 8192
+_SIM_NGRAMS = (2, 4)
+_IGNORED = frozenset({
+    "and", "of", "the", "for", "inc", "corp", "co", "ltd", "llc", "plc",
+    "company", "corporation", "incorporated", "limited",
+})
+
+
+class Blocker:
+    """A named pass returning pairs of row positions, independent of source IDs."""
+
+    name: str
+
+    def pairs(self, left: pd.DataFrame, right: pd.DataFrame) -> np.ndarray:
+        """Return an int64 array of shape (number of pairs, 2)."""
+        raise NotImplementedError("implement pairs(left, right) for this blocking pass")
+
+
+def _positive_int(value: object, label: str) -> None:
+    if isinstance(value, (bool, np.bool_)) or not isinstance(value, Integral) or value < 1:
+        raise ValueError(f"`{label}` must be a positive integer")
+
+
+def _pass_fields(columns: tuple, name: str | None, kind: str) -> tuple[list, str]:
+    fields = parse_on(columns)
+    if name is None:
+        names = [a if a == b else f"{a}={b}" for _, a, b in fields]
+        name = f"{kind}:{','.join(names)}"
+    if not isinstance(name, str) or not name.strip():
+        raise ValueError("a blocker's `name` must be a nonempty string")
+    return fields, name
+
+
+def _columns(left: pd.DataFrame, right: pd.DataFrame, fields: list) -> tuple[list[str], list[str]]:
+    for side, frame in (("left", left), ("right", right)):
+        if not isinstance(frame, pd.DataFrame):
+            raise ValueError(f"the {side} data must be a pandas DataFrame")
+    a, b = [f[1] for f in fields], [f[2] for f in fields]
+    check_columns(left, a, "left")
+    check_columns(right, b, "right")
+    return a, b
+
+
+def _empty_pairs() -> np.ndarray:
+    return np.empty((0, 2), dtype=np.int64)
+
+
+def exact(*columns: str | tuple[str, str], name: str | None = None) -> Blocker:
+    """Pair records equal in every listed field; incomplete keys never pair."""
+    fields, name = _pass_fields(columns, name, "exact")
+    return _Exact(fields, name)
+
+
+@dataclass
+class _Exact(Blocker):
+    fields: list
+    name: str
+
+    def pairs(self, left: pd.DataFrame, right: pd.DataFrame) -> np.ndarray:
+        a, b = _columns(left, right, self.fields)
+        groups = {}
+        for j, key in enumerate(zip(*(right[c].map(normalize) for c in b))):
+            if all(key):
+                groups.setdefault(key, []).append(j)
+        pairs = []
+        for i, key in enumerate(zip(*(left[c].map(normalize) for c in a))):
+            if all(key):
+                pairs.extend((i, j) for j in groups.get(key, ()))
+        return np.asarray(pairs, dtype=np.int64).reshape(-1, 2)
+
+
+def ngrams(*columns: str | tuple[str, str], k: int = 10, n: tuple[int, int] = (2, 4),
+           min_sim: float = 0.1, name: str | None = None) -> Blocker:
+    """Keep up to k cosine neighbors per left row; ties use right row position."""
+    fields, name = _pass_fields(columns, name, "ngrams")
+    _positive_int(k, "k")
+    if not isinstance(n, tuple) or len(n) != 2:
+        raise ValueError("`n` must be a (minimum, maximum) tuple of positive n-gram lengths")
+    for value in n:
+        _positive_int(value, "n")
+    if n[0] > n[1]:
+        raise ValueError("`n` must have its minimum n-gram length before its maximum")
+    if (isinstance(min_sim, (bool, np.bool_)) or not isinstance(min_sim, Real)
+            or not np.isfinite(min_sim) or not 0 <= min_sim <= 1):
+        raise ValueError("`min_sim` must be a finite number between 0 and 1")
+    return _Ngrams(fields, name, int(k), n, float(min_sim))
+
+
+def _vectors(left: pd.DataFrame, right: pd.DataFrame, a: list[str], b: list[str],
+             n: tuple[int, int]) -> tuple[csr_matrix, csr_matrix]:
+    text = pd.concat([record_text(left, a), record_text(right, b)], ignore_index=True)
+    vectorizer = TfidfVectorizer(analyzer="char_wb", ngram_range=n, norm="l2", dtype=np.float64)
+    try:
+        matrix = vectorizer.fit_transform(text).tocsr()
+    except ValueError as error:
+        if "empty vocabulary" not in str(error):
+            raise
+        matrix = csr_matrix((len(text), 0), dtype=np.float64)
+    # A fixed feature traversal keeps dot-product rounding independent of row chunks.
+    matrix.sort_indices()
+    return matrix[:len(left)], matrix[len(left):]
+
+
+def _top_k(positions: np.ndarray, scores: np.ndarray, k: int) -> np.ndarray:
+    """Partition by score, resolving the entire boundary tie by position."""
+    if len(scores) > k:
+        cutoff = np.partition(scores, len(scores) - k)[len(scores) - k]
+        better = np.flatnonzero(scores > cutoff)
+        tied = np.flatnonzero(scores == cutoff)
+        remaining = k - len(better)
+        if len(tied) > remaining:
+            tied = tied[np.argpartition(positions[tied], remaining - 1)[:remaining]]
+        keep = np.concatenate((better, tied))
+        positions, scores = positions[keep], scores[keep]
+    return positions[np.lexsort((positions, -scores))]
+
+
+def _clip_cosines(scores: np.ndarray) -> np.ndarray:
+    np.clip(scores, 0, 1, out=scores)
+    # Normalized identical vectors can sum to just below one in floating point.
+    scores[np.abs(scores - 1) <= 1e-14] = 1
+    return scores
+
+
+@dataclass
+class _Ngrams(Blocker):
+    fields: list
+    name: str
+    k: int
+    n: tuple[int, int]
+    min_sim: float
+
+    def pairs(self, left: pd.DataFrame, right: pd.DataFrame) -> np.ndarray:
+        a, b = _columns(left, right, self.fields)
+        if left.empty or right.empty:
+            return _empty_pairs()
+        x, y = _vectors(left, right, a, b, self.n)
+        k = min(self.k, len(right))
+        if not x.nnz or not y.nnz:
+            if self.min_sim > 0:
+                return _empty_pairs()
+            return np.column_stack((np.repeat(np.arange(len(left), dtype=np.int64), k),
+                                    np.tile(np.arange(k, dtype=np.int64), len(left))))
+        transpose = y.T.tocsr()
+        rows = max(1, min(_CHUNK_ROWS, _PRODUCT_BYTES // (12 * len(right))))
+        output = np.empty((len(left) * k, 2), dtype=np.int64)
+        count = 0
+        for start in range(0, len(left), rows):
+            product = (x[start:start + rows] @ transpose).tocsr()
+            _clip_cosines(product.data)
+            for local in range(product.shape[0]):
+                lo, hi = product.indptr[local:local + 2]
+                positions, scores = product.indices[lo:hi], product.data[lo:hi]
+                keep = (scores >= self.min_sim) & (scores > 0)
+                positions, scores = positions[keep], scores[keep]
+                chosen = _top_k(positions, scores, k)
+                if self.min_sim == 0 and len(chosen) < k:
+                    # Implicit sparse zeros qualify too, including for missing text.
+                    first = np.arange(k, dtype=np.int64)
+                    zeros = first[~np.isin(first, chosen)][:k - len(chosen)]
+                    chosen = np.concatenate((chosen, zeros))
+                stop = count + len(chosen)
+                output[count:stop, 0] = start + local
+                output[count:stop, 1] = chosen
+                count = stop
+        return output[:count].copy()
+
+
+def initials(column: str | tuple[str, str], min_len: int = 2, name: str | None = None) -> Blocker:
+    """Pair an acronym with a token expansion in either direction, omitting filler words."""
+    fields, name = _pass_fields((column,), name, "initials")
+    _positive_int(min_len, "min_len")
+    return _Initials(fields, name, int(min_len))
+
+
+def _acronym_keys(text: str, min_len: int) -> tuple[str, str]:
+    acronym = text if text.isalpha() and len(text) >= min_len else ""
+    expanded = "".join(token[0] for token in text.split() if token not in _IGNORED)
+    if len(expanded) < min_len:
+        expanded = ""
+    return acronym, expanded
+
+
+@dataclass
+class _Initials(Blocker):
+    fields: list
+    name: str
+    min_len: int
+
+    def pairs(self, left: pd.DataFrame, right: pd.DataFrame) -> np.ndarray:
+        a, b = _columns(left, right, self.fields)
+        acronyms, expansions = {}, {}
+        for j, text in enumerate(right[b[0]].map(normalize)):
+            acronym, expanded = _acronym_keys(text, self.min_len)
+            if acronym:
+                acronyms.setdefault(acronym, []).append(j)
+            if expanded:
+                expansions.setdefault(expanded, []).append(j)
+        pairs = []
+        for i, text in enumerate(left[a[0]].map(normalize)):
+            acronym, expanded = _acronym_keys(text, self.min_len)
+            matches = set(expansions.get(acronym, ())) | set(acronyms.get(expanded, ()))
+            pairs.extend((i, j) for j in sorted(matches))
+        return np.asarray(pairs, dtype=np.int64).reshape(-1, 2)
+
+
+def _checked_pairs(blocker: Blocker, left: pd.DataFrame, right: pd.DataFrame) -> np.ndarray:
+    pairs = blocker.pairs(left, right)
+    if (not isinstance(pairs, np.ndarray) or pairs.ndim != 2 or pairs.shape[1] != 2
+            or not np.issubdtype(pairs.dtype, np.integer)):
+        raise ValueError(f"blocker {blocker.name!r} must return an integer array "
+                         "with shape (number of pairs, 2)")
+    if pairs.size and (np.any(pairs < 0) or np.any(pairs[:, 0] >= len(left))
+                       or np.any(pairs[:, 1] >= len(right))):
+        raise ValueError(f"blocker {blocker.name!r} returned a row position outside the left or right data")
+    return pairs
+
+
+def _pair_similarities(left: pd.DataFrame, right: pd.DataFrame, a: list[str], b: list[str],
+                       pairs: np.ndarray) -> np.ndarray:
+    x, y = _vectors(left, right, a, b, _SIM_NGRAMS)
+    similarities = np.empty(len(pairs), dtype=np.float64)
+    for start in range(0, len(pairs), _PAIR_CHUNK_ROWS):
+        chunk = pairs[start:start + _PAIR_CHUNK_ROWS]
+        similarities[start:start + len(chunk)] = np.asarray(
+            x[chunk[:, 0]].multiply(y[chunk[:, 1]]).sum(axis=1)
+        ).ravel()
+    return _clip_cosines(similarities)
+
+
+def candidates(left: pd.DataFrame, right: pd.DataFrame, *, on: str | list[str | tuple[str, str]],
+               blockers: list[Blocker] | None = None, left_id: str | None = None,
+               right_id: str | None = None, max_pairs: int | None = 5_000_000) -> pd.DataFrame:
+    """Union named passes and score every pair on all `on` fields using 2–4 character grams."""
+    try:
+        fields = parse_on(on)
+    except TypeError as error:
+        raise ValueError("`on` must list column names or (left, right) pairs of column names") from error
+    a, b = _columns(left, right, fields)
+    left_ids, right_ids = ids(left, left_id, "left"), ids(right, right_id, "right")
+    if max_pairs is not None and (isinstance(max_pairs, (bool, np.bool_))
+                                  or not isinstance(max_pairs, Integral) or max_pairs < 0):
+        raise ValueError("`max_pairs` must be a nonnegative integer, or None for no limit")
+    if blockers is None:
+        blockers = [ngrams(*[(a, b) for _, a, b in fields], k=10)]
+    if not isinstance(blockers, list) or any(not isinstance(p, Blocker) for p in blockers):
+        raise ValueError("`blockers` must be a list of Blocker passes, or None for the default n-gram pass")
+    union = {}
+    for pass_number, blocker in enumerate(blockers):
+        if not isinstance(getattr(blocker, "name", None), str) or not blocker.name.strip():
+            raise ValueError("each blocker must have a nonempty string `name`")
+        bit = 1 << pass_number
+        for i, j in _checked_pairs(blocker, left, right):
+            key = (int(i), int(j))
+            union[key] = union.get(key, 0) | bit
+        if max_pairs is not None and len(union) > max_pairs:
+            raise ValueError(f"blocking produced {len(union):,} pairs, exceeding max_pairs={max_pairs:,}; "
+                             "use a smaller `k` or a more selective `exact` pass")
+    pairs = np.asarray(list(union), dtype=np.int64).reshape(-1, 2)
+    labels = {mask: "+".join(p.name for i, p in enumerate(blockers) if mask & (1 << i))
+              for mask in set(union.values())}
+    blocks = [labels[mask] for mask in union.values()]
+    del union
+    sim = _pair_similarities(left, right, a, b, pairs) if len(pairs) else np.empty(0, dtype=float)
+    order = np.lexsort((pairs[:, 1], -sim, pairs[:, 0]))
+    return pd.DataFrame({
+        "left_id": left_ids.take(pairs[:, 0]),
+        "right_id": right_ids.take(pairs[:, 1]),
+        "block": pd.Series(blocks, dtype=object),
+        "sim": sim,
+    }).iloc[order].reset_index(drop=True)
+
+
+def pairs_completeness(candidates: pd.DataFrame, truth: pd.DataFrame) -> float:
+    """Share of distinct truth pairs proposed; NaN when there are no known truth pairs."""
+    columns = ["left_id", "right_id"]
+    for label, frame in (("candidates", candidates), ("truth", truth)):
+        if not isinstance(frame, pd.DataFrame):
+            raise ValueError(f"the {label} data must be a pandas DataFrame")
+        check_columns(frame, columns, label)
+    known = pd.MultiIndex.from_frame(truth[columns].drop_duplicates())
+    if not len(known):
+        return float("nan")
+    proposed = pd.MultiIndex.from_frame(candidates[columns].drop_duplicates())
+    return float(known.isin(proposed).mean())