custmatch 0.1.0__py3-none-any.whl

custmatch/__init__.py

@@ -0,0 +1 @@
+"""Customer matching: the pipeline measured in docs/report/, as a tool. See docs/guide.md."""

custmatch/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

custmatch/blocking.py

@@ -0,0 +1,480 @@
+"""Blocking for the tool: rules, learned rule sets, split oversized blocks, SQL self-join, diagnostics.
+
+What the three leading projects do and custmatch's first blocker did not (docs/guide.md):
+
+  learned rules      dedupe: pick the cheapest set of blocking rules that still covers a target
+                     share (default 95%) of the labelled matches - a greedy weighted set cover
+                     over a library of rules - instead of fixed hand-picked keys
+  split, not drop    Zingg: a block larger than max_block is refined with the next predicate
+                     (first letters of the first name, birth year, street, ...) until it fits,
+                     and dropped only if it still cannot be split - custmatch dropped it outright,
+                     which lost the 173-record London block in Splink's fake_1000
+  SQL self-join      dedupe / Splink: pairs come from a DuckDB self-join on (rule, key) with
+                     DISTINCT, not Python loops over every block
+  diagnostics        Splink: pairs contributed by each rule, its largest blocks, and what was split
+                     or dropped - blocking_report.json in the run's output
+
+core.block (the fixed keys the report measured) is unchanged; this module is used by the tool.
+A rule is a tuple of predicate names; records agree on a rule when every predicate gives the
+same non-empty key.
+"""
+import numpy as np
+import pandas as pd
+
+# predicate name -> (column, transform). Transforms map a cleaned string column to keys.
+_TRANSFORMS = {
+    "exact": lambda s: s,
+    "first3": lambda s: s.str[:3].where(s.str.len() >= 3, ""),
+    "first1": lambda s: s.str[:1],
+    "year": lambda s: s.str.extract(r"(\d{4})", expand=False).fillna(""),
+    "yearmonth": None,   # set below: needs date parsing
+    "soundex": None,
+}
+NAME_FIELDS = ("first_name", "last_name")
+FIXED_RULES = [("last_name:exact", "first_name:first3"), ("last_name:exact", "addr_street:exact"),
+               ("addr_number:exact", "addr_street:exact"), ("first_name:exact", "addr_street:exact"),
+               ("email:exact",), ("phone:exact",)]
+# refinements tried, in order, to split an oversized block
+REFINEMENTS = ["first_name:first3", "date_of_birth:year", "last_name:first3", "addr_street:exact",
+               "first_name:exact", "date_of_birth:exact", "addr_number:exact"]
+
+
+_SOUNDEX = {**dict.fromkeys("bfpv", "1"), **dict.fromkeys("cgjkqsxz", "2"), **dict.fromkeys("dt", "3"),
+            "l": "4", **dict.fromkeys("mn", "5"), "r": "6"}
+
+
+def soundex(word):
+    """American Soundex: the first letter, then up to three digits for the consonant sounds that
+    follow (Robert, Rupert -> R163). Letters with one code next to each other, or separated
+    only by h or w, count once; a vowel, space or hyphen between them counts them twice - as
+    jellyfish.soundex, which this replaces, does."""
+    w = word.lower()
+    first = next((n for n, c in enumerate(w) if "a" <= c <= "z"), None)
+    if first is None:
+        return ""
+    out, last = [], _SOUNDEX.get(w[first], "")
+    for c in w[first + 1:]:
+        code = _SOUNDEX.get(c, "")              # vowels, spaces and hyphens separate: code ""
+        if code and code != last:
+            out.append(code)
+        if c not in "hw":                       # h and w do not separate two equal codes
+            last = code
+    return (w[first].upper() + "".join(out) + "000")[:4]
+
+
+def _soundex(s):
+    return s.map(lambda v: soundex(v) if v and v[0].isalpha() else "")
+
+
+def _yearmonth(s):
+    from .core import parse_dates
+    d = parse_dates(s.to_numpy())
+    return pd.Series([f"{y:04d}{m:02d}" if y >= 0 else "" for y, m, _ in d], index=s.index)
+
+
+class Keys:
+    """Lazily computed predicate keys per record (cleaned strings, "" = no key)."""
+
+    def __init__(self, df, profiles=None):
+        self.df, self.cache = df, {}
+        self.junk = {}
+        for f in ("email", "phone"):
+            if profiles and f in profiles:
+                self.junk[f] = df[f].map(lambda v: profiles[f].get(v, ("personal",))[0] == "junk").to_numpy()
+
+    def get(self, pred):
+        if pred not in self.cache:
+            col, how = pred.split(":")
+            if col not in self.df.columns:
+                self.cache[pred] = np.full(len(self.df), "", dtype=object)
+            else:
+                s = self.df[col].astype(str)
+                if how == "soundex":
+                    k = _soundex(s)
+                elif how == "yearmonth":
+                    k = _yearmonth(s)
+                else:
+                    k = _TRANSFORMS[how](s)
+                k = k.to_numpy(dtype=object)
+                if col in self.junk:
+                    k = np.where(self.junk[col], "", k)
+                self.cache[pred] = k
+        return self.cache[pred]
+
+    def codes(self, pred):
+        """Integer code per record for a predicate; -1 where it has no key."""
+        ck = ("codes", pred)
+        if ck not in self.cache:
+            k = self.get(pred)
+            c, _ = pd.factorize(k)
+            self.cache[ck] = np.where(k == "", -1, c).astype(np.int64)
+        return self.cache[ck]
+
+    def rule(self, rule):
+        """Integer key per record for a rule (a conjunction of predicates); -1 where any
+        predicate has no key. Integer codes, not joined strings: at 1M records joining strings
+        made blocking the slowest stage."""
+        parts = [self.codes(p) for p in rule]
+        empty = np.zeros(len(self.df), bool)
+        for c in parts:
+            empty |= c < 0
+        if len(parts) == 1:
+            key = parts[0]
+        else:
+            key, _ = pd.MultiIndex.from_arrays(parts).factorize()
+            key = key.astype(np.int64)
+        return np.where(empty, -1, key)
+
+    def label(self, rule, i):
+        """Readable key of record i under a rule, for diagnostics."""
+        return " | ".join(str(self.get(p)[i]) for p in rule)
+
+
+def rule_library(df, cfg=None):
+    """Candidate rules over the fields this data actually has."""
+    from .schema import extra_fields
+    # country.py's derived name columns are blocked on through country.blocking_keys, not as rules
+    derived = {"name_script", "first_roots", "name_skeleton", "country", "updated_at"}
+    have = [c for c in df.columns if c not in ("source", "record_id") and c not in derived
+            and (df[c] != "").sum() > 1]
+    single = []
+    for c in have:
+        single.append(f"{c}:exact")
+        if c in NAME_FIELDS or c in ("addr_street",) or c in {n for n, _ in extra_fields(cfg or {})}:
+            single.append(f"{c}:first3")
+        if c in NAME_FIELDS:
+            single.append(f"{c}:soundex")
+        if c == "date_of_birth":
+            single += ["date_of_birth:year", "date_of_birth:yearmonth"]
+    rules = [(p,) for p in single]
+    strong = [p for p in single if p.endswith(":exact") or p.endswith(":soundex")]
+    for a in range(len(single)):
+        for b in range(a + 1, len(single)):
+            pa, pb = single[a], single[b]
+            if pa.split(":")[0] == pb.split(":")[0]:
+                continue
+            if pa in strong or pb in strong:
+                rules.append((pa, pb))
+    rules += fixed_rules(cfg, df)
+    return list(dict.fromkeys(rules))
+
+
+def fixed_rules(cfg=None, df=None):
+    """The fixed keys as rules. A predicate on a field the data never fills is dropped from its
+    rule, so (last_name, addr_street) becomes (last_name) where there is no address - what the
+    report's core.block did implicitly (an empty street was a shared key), and the recall it gave
+    on historical_50k (87.0 vs 84.0 without) depends on it."""
+    from .schema import extra_keys
+    rules = [tuple(r) for r in FIXED_RULES] + [tuple(c if ":" in c else f"{c}:exact" for c in k) for k in extra_keys(cfg or {})]
+    if df is None:
+        return rules
+    filled = {c for c in df.columns if (df[c] != "").any()}
+    out = []
+    for r in rules:
+        kept = tuple(p for p in r if p.split(":")[0] in filled)
+        if kept:
+            out.append(kept)
+    return list(dict.fromkeys(out))
+
+
+def _n2(sizes):
+    sizes = np.asarray(sizes, dtype=np.float64)
+    return float(np.sum(sizes * (sizes - 1) / 2))
+
+
+def learn_rules(df, keys, pos_pairs, cfg=None, target=0.95, max_block=60, max_rules=8,
+                sample=50000, seed=0):
+    """Greedy weighted set cover: repeatedly add the rule covering the most still-uncovered
+    labelled matches per candidate pair it would generate, until `target` of the labelled
+    matches that any rule can cover are covered. Returns (rules, report).
+
+    Like dedupe, it learns on a sample: the records in labelled pairs plus up to `sample`
+    random others, with block sizes scaled up to the full file for the cost - evaluating the
+    whole rule library on a million records took 143 s."""
+    lib = rule_library(df, cfg)
+    n = len(df)
+    rng = np.random.default_rng(seed)
+    in_labels = np.unique(pos_pairs.ravel())
+    others = np.setdiff1d(np.arange(n), in_labels)
+    extra = rng.choice(others, min(sample, len(others)), replace=False) if len(others) else others
+    idx = np.sort(np.concatenate([in_labels, extra]))
+    sub = Keys(df.iloc[idx].reset_index(drop=True), None)
+    sub.junk = {f: v[idx] for f, v in keys.junk.items()}
+    pos_local = np.searchsorted(idx, pos_pairs)
+    i, j = pos_local[:, 0], pos_local[:, 1]
+    scale = n / len(idx)
+    covers, cost = {}, {}
+    for r in lib:
+        k = sub.rule(r)
+        covers[r] = (k[i] == k[j]) & (k[i] >= 0)
+        sizes = np.bincount(k[k >= 0]) * scale if (k >= 0).any() else np.zeros(0)
+        sizes = sizes[sizes > 0]
+        # oversized blocks will be split; charge them roughly as pieces of max_block
+        small, big = sizes[sizes <= max_block], sizes[sizes > max_block]
+        cost[r] = _n2(small) + float(np.sum(np.ceil(big / max_block))) * _n2([max_block])
+    coverable = np.any([covers[r] for r in lib], axis=0) if lib else np.zeros(len(i), bool)
+    need = target * coverable.sum()
+    chosen, covered = [], np.zeros(len(i), bool)
+    while covered.sum() < need and len(chosen) < max_rules:
+        best = max(lib, key=lambda r: ((covers[r] & ~covered).sum() / (cost[r] + 1.0), -cost[r]))
+        if not (covers[best] & ~covered).any():
+            break
+        chosen.append(best)
+        covered |= covers[best]
+    report = {"labelled_matches": int(len(i)), "coverable_by_library": int(coverable.sum()),
+              "covered": int(covered.sum()), "target": target, "library_size": len(lib),
+              "rules": [{"rule": list(r), "covers": int(covers[r].sum()), "est_pairs": int(cost[r])}
+                        for r in chosen]}
+    return chosen, report
+
+
+def _split(keys, idx_block, rule, max_block):
+    """Refine an oversized block with the next predicates; returns (list of record-index arrays,
+    number of pieces dropped because nothing was left to split them on)."""
+    if len(idx_block) <= max_block:
+        return [idx_block], 0
+    for p in REFINEMENTS:
+        if p in rule:
+            continue
+        sub = keys.codes(p)[idx_block]
+        if (sub < 0).all() or len(np.unique(sub)) == 1:
+            continue
+        out, dropped = [], 0
+        # records with no value for this refinement (a first name shorter than three letters,
+        # no birth year) stay together as their own piece rather than being lost
+        for v in np.unique(sub):
+            part = idx_block[sub == v]
+            if len(part) < 2:
+                continue
+            o, d = _split(keys, part, rule + (p,), max_block)
+            out += o
+            dropped += d
+        return out, dropped
+    return [], 1        # nothing left to split on: drop it as a hub
+
+
+def generate(df, keys, rules, max_block=60):
+    """Candidate pairs for the rules (splitting oversized blocks), plus per-rule diagnostics.
+    Returns (pairs int32 array sorted, report)."""
+    rows, report = [], []
+    for n, r in enumerate(rules):
+        k = keys.rule(r)
+        has = np.flatnonzero(k >= 0)
+        kk = k[has]
+        sizes = np.bincount(kk) if len(kk) else np.zeros(0, int)
+        big_codes = np.flatnonzero(sizes > max_block)
+        small = sizes[kk] <= max_block
+        rk, ri = kk[small], has[small]
+        next_key = int(sizes.size)
+        split_blocks, dropped, extra_k, extra_i = 0, 0, [], []
+        for bc in big_codes:
+            parts, d = _split(keys, has[kk == bc], tuple(r), max_block)
+            split_blocks += 1
+            dropped += d
+            for pidx in parts:
+                extra_k.append(np.full(len(pidx), next_key, np.int64))
+                extra_i.append(pidx)
+                next_key += 1
+        rows.append(pd.DataFrame({"rule": n, "key": np.concatenate([rk] + extra_k) if extra_k else rk,
+                                  "rid": np.concatenate([ri] + extra_i) if extra_i else ri}))
+        order = np.argsort(-sizes)[:5]
+        report.append({"rule": list(r), "records_with_key": int(len(has)), "blocks": int((sizes > 0).sum()),
+                       "largest_blocks": [{"key": keys.label(r, int(has[kk == c][0])), "size": int(sizes[c])}
+                                          for c in order if sizes[c] > 0],
+                       "oversized_blocks_split": split_blocks, "blocks_dropped_as_hubs": dropped})
+    table = pd.concat(rows, ignore_index=True) if rows else pd.DataFrame({"rule": [], "key": [], "rid": []})
+    per_rule = _block_pairs(table.rule.to_numpy(np.int64), table.key.to_numpy(np.int64), table.rid.to_numpy(np.int64))
+    first = per_rule.groupby(["i", "j"]).rule.min().reset_index()
+    counts = per_rule.rule.value_counts()
+    new = first.rule.value_counts()
+    for n, entry in enumerate(report):
+        entry["pairs"] = int(counts.get(n, 0))
+        entry["new_pairs"] = int(new.get(n, 0))      # not already produced by an earlier rule
+    pairs = first.sort_values(["i", "j"])[["i", "j"]].to_numpy(dtype=np.int32) if len(first) \
+        else np.zeros((0, 2), np.int32)
+    return pairs, {"rules": report, "candidate_pairs": int(len(pairs)), "max_block": max_block}
+
+
+def _block_pairs(rule, key, rid):
+    """Every pair of records sharing a (rule, key) block: a DataFrame of rule, i < j. Sorted by
+    block, a record pairs with the k-th record after it while that one is still in its block;
+    blocks are capped at max_block, so k stays small and each step is one vectorised pass."""
+    if not len(rid):
+        return pd.DataFrame({"rule": np.zeros(0, np.int64), "i": np.zeros(0, np.int64), "j": np.zeros(0, np.int64)})
+    order = np.lexsort((rid, key, rule))
+    rule, key, rid = rule[order], key[order], rid[order]
+    new = np.r_[True, (rule[1:] != rule[:-1]) | (key[1:] != key[:-1])]
+    starts = np.flatnonzero(new)
+    end = np.repeat(np.r_[starts[1:], len(rid)], np.diff(np.r_[starts, len(rid)]))   # block end per row
+    pos = np.arange(len(rid))
+    out_r, out_i, out_j = [], [], []
+    for k in range(1, int((end - pos).max())):
+        a = pos[pos + k < end]
+        b = a + k
+        out_r.append(rule[a]); out_i.append(np.minimum(rid[a], rid[b])); out_j.append(np.maximum(rid[a], rid[b]))
+    if not out_r:
+        return pd.DataFrame({"rule": np.zeros(0, np.int64), "i": np.zeros(0, np.int64), "j": np.zeros(0, np.int64)})
+    return pd.DataFrame({"rule": np.concatenate(out_r), "i": np.concatenate(out_i), "j": np.concatenate(out_j)})
+
+
+def country_pairs(df, profiles, max_block):
+    """Pairs from country.blocking_keys (first-name root + DOB or surname, email user name, name
+    skeleton), blocks larger than max_block skipped - the same keys core.block uses."""
+    from collections import defaultdict
+    from . import country
+    groups = defaultdict(list)
+    for k, i in country.blocking_keys(df, profiles):
+        groups[k].append(i)
+    out = set()
+    for idx in groups.values():
+        if 2 <= len(idx) <= max_block:
+            idx = sorted(set(idx))
+            out.update((a, b) for x, a in enumerate(idx) for b in idx[x + 1:])
+    return np.array(sorted(out), dtype=np.int32).reshape(-1, 2)
+
+
+def candidates_for(df, profiles, cfg, labelled_pos=None, rules=None):
+    """The tool's candidate pairs. Returns (pairs, hubs_dropped, report, rules_used).
+
+    cfg "blocking": absent -> the fixed keys the report measured (core.block, blocks over
+    max_block dropped); "fixed" -> the same kind of keys as rules, oversized blocks split;
+    "learned" -> rules learned from labelled_pos (falls back to fixed without labels).
+    rules: reuse a saved rule set (incremental matching)."""
+    from .core import block
+    from .schema import extra_keys, max_block as cfg_max_block
+    mode = cfg.get("blocking")
+    mb = cfg_max_block(cfg) or 60
+    if rules is None and not mode:
+        skipped, by_key = [], []
+        pairs, hubs = block(df, profiles, extra_keys=extra_keys(cfg), max_block=cfg_max_block(cfg),
+                            lsh=cfg.get("lsh_blocking"), skipped=skipped, by_key=by_key)
+        # records never compared with anyone because their only blocks were skipped - a record
+        # whose broad block (postcode alone) was skipped but whose narrow one was used is fine
+        compared = np.zeros(len(df), bool)
+        compared[pairs.ravel()] = True
+        records = len({i for _, idx in skipped for i in idx if not compared[i]})
+        top = sorted(skipped, key=lambda s: -len(s[1]))[:5]
+        return pairs, hubs, {"mode": "core", "candidate_pairs": int(len(pairs)), "hubs_dropped": int(hubs),
+                             "skipped_blocks_with_values": len(skipped), "records_only_in_skipped_blocks": records,
+                             "largest_skipped_blocks": [{"key": " / ".join(str(v) for v in k), "records": len(idx)}
+                                                        for k, idx in top],
+                             "pairs_by_key": by_key, "_skipped": skipped}, None
+    if cfg.get("lsh_blocking"):
+        from .io import ConfigError
+        raise ConfigError('"lsh_blocking" works with the default blocking only; remove "blocking": '
+                          f'"{mode}" or "lsh_blocking"')
+    keys = Keys(df, profiles)
+    learn = None
+    if rules is None:
+        if mode == "learned" and labelled_pos is not None and len(labelled_pos):
+            rules, learn = learn_rules(df, keys, labelled_pos, cfg, target=cfg.get("blocking_recall", 0.95),
+                                       max_block=mb)
+        else:
+            rules = fixed_rules(cfg, df)
+    pairs, report = generate(df, keys, rules, max_block=mb)
+    extra = country_pairs(df, profiles, mb)        # country.py keys: nickname + DOB, email user...
+    if len(extra):
+        allp = np.vstack([pairs, extra]) if len(pairs) else extra
+        pairs = np.unique(allp, axis=0)
+    report.update(mode=mode or "saved", learned=learn, country_key_pairs=int(len(extra)))
+    hubs = sum(r["blocks_dropped_as_hubs"] for r in report["rules"])
+    return pairs, hubs, report, [list(r) for r in rules]
+
+
+# ------------------------------------------------------------------ similarity (LSH) blocking
+def lsh_keys(df, spec, seed=0):
+    """Blocking keys from MinHash locality-sensitive hashing of letter pairs - the idea behind
+    anonlink's blocklib, on plain text. Records whose `fields` (joined) share most of their
+    letter pairs get the same signature in at least one band with high probability, so names
+    with typos in both still meet; exact keys miss them. `with` adds a field to every key, to
+    keep the blocks small (a band of common names state-wide is a hub otherwise).
+
+        "lsh_blocking": {"fields": ["first_name", "last_name"], "with": "town",
+                         "bands": 16, "rows": 3}
+
+    With b bands of r rows, two records whose letter-pair sets have Jaccard similarity s share
+    a band with probability 1 - (1 - s^r)^b: for 16 x 3, 0.97 at s = 0.6 and 0.32 at s = 0.2.
+    Returns a list of (record index, key) pairs."""
+    import zlib
+    fields, bands, rows = spec["fields"], int(spec.get("bands", 16)), int(spec.get("rows", 3))
+    text = df[fields[0]].astype(str)
+    for f in fields[1:]:
+        text = text + " " + df[f].astype(str)
+    text = text.str.strip()
+    rng = np.random.default_rng(seed)
+    k = bands * rows
+    p = np.uint64((1 << 61) - 1)
+    a = rng.integers(1, int(p), k, dtype=np.uint64)       # k hash functions (a*x + b) mod p
+    b = rng.integers(0, int(p), k, dtype=np.uint64)
+    other = df[spec["with"]].astype(str).to_numpy() if spec.get("with") else None
+    out = []
+    cache = {}
+    for i, t in enumerate(text):
+        if len(t) < 3 or (other is not None and not other[i]):
+            continue
+        if t not in cache:
+            g = {f"{t[x:x + 2]}" for x in range(len(t) - 1)}
+            ids = np.array([zlib.crc32(s.encode()) for s in g], dtype=np.uint64)
+            # min over the record's letter pairs of k random hash functions
+            sig = ((ids[:, None] * a[None, :] + b[None, :]) % p).min(axis=0)
+            cache[t] = [hash(tuple(sig[x * rows:(x + 1) * rows].tolist())) for x in range(bands)]
+        extra = other[i] if other is not None else ""
+        out += [(i, ("lsh", band, extra, h)) for band, h in enumerate(cache[t])]
+    return out
+
+
+LARGE_BLOCK_SAMPLE = 50         # pairs scored per oversized block
+LARGE_BLOCK_MATCH_SHARE = 0.5   # kept when at least this share of them are matches
+LARGE_BLOCK_MAX = 5000          # records: larger blocks are not checked (12.5 million pairs each)
+
+
+def large_block_check_on(cfg):
+    """On unless "check_large_blocks": false - it found no large entity, and changed nothing, on
+    six person benchmarks, and lifted dedupe's patent-applicant file from 6.93 to 75.88."""
+    return cfg.get("check_large_blocks", True) and not cfg.get("blocking")
+
+
+def saved_run_large_blocks(cfg, report, predict, th, pairs):
+    """For `add` and `explain`: the same check over a saved run's model, so a large entity kept by
+    the run is kept again. Returns `pairs` with the kept blocks' pairs added (sorted, unique)."""
+    skipped = report.pop("_skipped", [])
+    if not (large_block_check_on(cfg) and skipped):
+        return pairs
+    extra, _ = check_large_blocks(skipped, predict, th, budget=max(1_000_000, len(pairs)))
+    return np.unique(np.vstack([pairs, extra]), axis=0).astype(np.int32) if len(extra) else pairs
+
+
+def check_large_blocks(skipped, predict, th, budget, seed=0, share=LARGE_BLOCK_MATCH_SHARE):
+    """Blocks skipped as hubs that are one real entity - an organisation with hundreds of records -
+    rather than junk (a@a.com: unrelated people agreeing on nothing else) or many people (an
+    apartment building). The trained model scores a sample of each block's pairs; a block whose
+    sampled pairs are mostly matches is kept, and all its pairs become candidates, largest-first
+    until `budget` pairs. Returns (new pairs int32 (m, 2), i < j; report rows)."""
+    rng = np.random.default_rng(seed)
+    blocks = [(key, np.unique(np.asarray(idx))) for key, idx in skipped if len(idx) <= LARGE_BLOCK_MAX]
+    if not blocks:
+        return np.zeros((0, 2), np.int32), []
+    samples, owner = [], []
+    for b, (_, idx) in enumerate(blocks):
+        a, c = rng.choice(idx, LARGE_BLOCK_SAMPLE), rng.choice(idx, LARGE_BLOCK_SAMPLE)
+        ok = a != c
+        samples.append(np.c_[np.minimum(a[ok], c[ok]), np.maximum(a[ok], c[ok])])
+        owner.append(np.full(int(ok.sum()), b))
+    p = predict(np.vstack(samples).astype(np.int32))      # one scoring pass for every block
+    owner = np.concatenate(owner)
+    match_share = np.bincount(owner, weights=p > th, minlength=len(blocks)) / np.maximum(
+        np.bincount(owner, minlength=len(blocks)), 1)
+    rows, new, used = [], [], 0
+    for b in np.argsort(-np.array([len(idx) for _, idx in blocks]), kind="stable"):
+        key, idx = blocks[b]
+        n_pairs = len(idx) * (len(idx) - 1) // 2
+        keep = match_share[b] >= share and used + n_pairs <= budget
+        rows.append({"key": " / ".join(str(v) for v in key), "records": int(len(idx)),
+                     "sampled_match_share": round(float(match_share[b]), 3), "kept": bool(keep)})
+        if keep:
+            i, j = np.triu_indices(len(idx), 1)
+            new.append(np.c_[idx[i], idx[j]])
+            used += n_pairs
+    if not new:
+        return np.zeros((0, 2), np.int32), rows
+    return np.unique(np.vstack(new), axis=0).astype(np.int32), rows

custmatch/cli.py

@@ -0,0 +1,124 @@
+"""custmatch - match customer records across sources.
+
+  custmatch sample CONFIG --n 500 --out to_label.csv [--active labels_so_far.csv]
+      random candidate pairs, side by side, for you to label (fill `label` with 1 or 0);
+      --active picks the pairs the forest is least sure of instead
+
+  custmatch run CONFIG --labels labels.csv --out results/ [--previous DIR] [--model DIR]
+      clusters.csv, golden_records.csv, review.csv, history.csv, summary.json;
+      --previous keeps that run's customer IDs; --model reuses its trained model (no labels)
+
+  custmatch blocking CONFIG
+      blocking diagnostics before you label: pairs per rule, largest blocks, splits and drops
+
+  custmatch add NEW_CONFIG --state results/ --out results2/ [--delete deleted.csv]
+      match new or changed records against a previous run's customers without rerunning it,
+      and remove deleted ones (--delete: a CSV of source, record_id); writes changes.csv,
+      merges.csv and history.csv
+
+  custmatch forget RECORDS.csv --state results/ --out results2/
+      an erasure request: remove these records (source, record_id) and rebuild the affected
+      customers without them; delete the old output folders yourself
+
+  custmatch explain --state results/ RECORD_A RECORD_B
+      why two records were or were not matched: blocking, evidence, score, overrides
+
+  custmatch monitor results/2026-09 results/2026-10 ... [--out trend.csv] [--fail-on-drift]
+      match rates, field fill and score drift across outputs, oldest first; exits 3 on drift
+      with --fail-on-drift
+
+CONFIG maps your files and columns onto the pipeline's (see custmatch/io.py and docs/guide.md).
+"""
+import argparse
+import json
+import sys
+
+from . import io, labels, pipeline
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="custmatch", description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    s = sub.add_parser("sample", help="draw candidate pairs for hand-labelling")
+    s.add_argument("config")
+    s.add_argument("--n", type=int, default=500)
+    s.add_argument("--out", required=True)
+    s.add_argument("--seed", type=int, default=0)
+    s.add_argument("--active", metavar="LABELS",
+                   help="instead of random pairs, the ones the forest trained on LABELS is least sure of")
+    s.add_argument("--strategy", choices=["uncertainty", "disagreement"], default="uncertainty",
+                   help="with --active: disagreement also offers pairs blocking missed that the forest calls matches")
+    s.add_argument("--mix", type=float, default=0.0,
+                   help="with --active: share of each batch drawn at random instead (e.g. 0.5)")
+    r = sub.add_parser("run", help="match, cluster, households, golden records")
+    r.add_argument("config")
+    r.add_argument("--labels", help="labelled pairs; optional with --model")
+    r.add_argument("--previous", metavar="DIR", help="a previous run's output: keep its customer IDs")
+    r.add_argument("--model", metavar="DIR", help="reuse that run's trained model and threshold")
+    r.add_argument("--out", required=True)
+    bl = sub.add_parser("blocking", help="blocking diagnostics: pairs per rule, largest blocks, splits")
+    bl.add_argument("config")
+    ad = sub.add_parser("add", help="match new records against a previous run's customers")
+    ad.add_argument("config", help="config for the NEW records only")
+    ad.add_argument("--state", required=True, help="output folder of a previous run or add")
+    ad.add_argument("--out", required=True)
+    ad.add_argument("--delete", metavar="CSV", help="records to remove: columns source, record_id")
+    fg = sub.add_parser("forget", help="erase records: remove them and rebuild their customers")
+    fg.add_argument("records", help="CSV of source, record_id")
+    fg.add_argument("--state", required=True)
+    fg.add_argument("--out", required=True)
+    ex = sub.add_parser("explain", help="why two records were or were not matched")
+    ex.add_argument("a", help="first record, source:record_id")
+    ex.add_argument("b", help="second record, source:record_id")
+    ex.add_argument("--state", required=True, help="output folder of a run or add")
+    mo = sub.add_parser("monitor", help="match rates and drift across outputs, oldest first")
+    mo.add_argument("outputs", nargs="+", help="output folders of runs or adds, in time order")
+    mo.add_argument("--out", help="also write the table to this CSV")
+    mo.add_argument("--fail-on-drift", action="store_true", help="exit 3 when any drift alert fires")
+    a = ap.parse_args(argv)
+    try:
+        if a.cmd == "sample":
+            k = (labels.sample_active(a.config, a.active, a.n, a.out, a.seed, a.mix, a.strategy) if a.active
+                 else labels.sample(a.config, a.n, a.out, a.seed))
+            print(f"wrote {k} pairs to {a.out}: fill the label column with 1 (same person) or 0")
+        elif a.cmd == "blocking":
+            from .blocking import candidates_for
+            from .core import profile_contacts
+            df, cfg = io.load(a.config)
+            # the blocker `run` will use: default, "fixed" or "learned" (which, before labels
+            # exist, falls back to fixed rules)
+            _, _, rep, _ = candidates_for(df, {f: profile_contacts(df, f) for f in ("email", "phone")}, cfg)
+            rep.pop("_skipped", None)
+            print(json.dumps(rep, indent=1, default=str))
+        elif a.cmd == "add":
+            from . import incremental
+            print(json.dumps(incremental.add(a.state, a.config, a.out, delete=a.delete), indent=1))
+        elif a.cmd == "forget":
+            from . import incremental
+            print(json.dumps(incremental.add(a.state, None, a.out, delete=a.records), indent=1))
+        elif a.cmd == "monitor":
+            from . import monitor
+            table, alerts = monitor.compare(a.outputs)
+            print(table.to_string(index=False))
+            print("\n".join(["", "drift alerts:"] + alerts) if alerts else "\nno drift alerts")
+            if a.out:
+                table.to_csv(a.out, index=False)
+            if alerts and a.fail_on_drift:
+                return 3
+        elif a.cmd == "explain":
+            from . import explain
+            print(json.dumps(explain.why(a.state, a.a, a.b), indent=1))
+        else:
+            if a.labels is None and a.model is None:
+                print("custmatch: run needs --labels (or --model to reuse a trained model)", file=sys.stderr)
+                return 2
+            print(json.dumps(pipeline.run(a.config, a.labels, a.out, previous_dir=a.previous, model_dir=a.model), indent=1))
+    except (io.ConfigError, FileNotFoundError) as e:
+        print(f"custmatch: {e}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())