lp2graph 0.3.0__py3-none-any.whl

lp2graph/mining/cluster/operator.py

@@ -0,0 +1,218 @@
+"""The cluster-and-name operator ``CN`` (M3 core).
+
+``CN(E)`` partitions a set of entities ``E`` (each carrying a TF-IDF concept
+vector) into named parts. It is the single primitive the multi-level taxonomy
+passes are built from:
+
+1. Route entities with no concept signal (zero vector) to an explicit
+   ``unassigned`` part (label ``-1``) — every entity ends up in exactly one
+   part.
+2. Cluster the rest in cosine-distance space. Default backend is the
+   deterministic average-linkage core; ``fixed_k`` selects K by silhouette;
+   ``hdbscan`` is used when the optional dependency is installed.
+3. Name each part by aggregated TF-IDF weight: the concept carrying the most
+   summed mass across the part's members becomes its name (collisions get a
+   deterministic numeric suffix).
+
+The whole operator is deterministic given the versioned
+:class:`ClusterConfig`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Literal
+
+from lp2graph.mining.cluster.agglomerative import agglomerative
+from lp2graph.mining.cluster.distance import distance_matrix, is_zero
+from lp2graph.mining.cluster.silhouette import select_k, silhouette_score
+from lp2graph.mining.homologize.vectorize import Vocabulary
+from lp2graph.mining.versions import CLUSTERING_VERSION
+
+#: Cluster id reserved for entities with no concept signal.
+UNASSIGNED = -1
+
+Algorithm = Literal["agglomerative", "fixed_k", "hdbscan"]
+
+
+@dataclass(frozen=True, slots=True)
+class ClusterConfig:
+    """Versioned, deterministic configuration for :func:`CN`."""
+
+    algorithm: Algorithm = "agglomerative"
+    distance_threshold: float = 0.7
+    k: int | None = None
+    k_range: tuple[int, int] = (2, 8)
+    seed: int = 0
+    version: str = CLUSTERING_VERSION
+
+
+@dataclass(frozen=True, slots=True)
+class NamedClustering:
+    """The result of :func:`CN`: a named partition over the input entities.
+
+    ``labels`` is aligned to the input order (``-1`` == unassigned).
+    ``names`` maps each cluster id to its concept name. ``members`` maps each
+    id to the (sorted) input indices it contains. ``silhouette`` is the mean
+    silhouette over the assigned points.
+    """
+
+    labels: tuple[int, ...]
+    names: dict[int, str]
+    members: dict[int, tuple[int, ...]]
+    config: ClusterConfig
+    silhouette: float
+    top_concepts: dict[int, tuple[tuple[str, float], ...]] = field(default_factory=dict)
+
+    def name_of(self, index: int) -> str:
+        """Name of the cluster the entity at ``index`` belongs to."""
+        return self.names[self.labels[index]]
+
+    @property
+    def n_clusters(self) -> int:
+        """Number of real (non-unassigned) clusters."""
+        return len([cid for cid in self.members if cid != UNASSIGNED])
+
+
+def _aggregate_weights(
+    members: Sequence[int], vectors: Sequence[Sequence[float]], vocab: Vocabulary
+) -> list[tuple[str, float]]:
+    totals = [0.0] * len(vocab)
+    for idx in members:
+        v = vectors[idx]
+        for j, val in enumerate(v):
+            if val != 0.0:
+                totals[j] += val
+    weighted = [(vocab.concepts[j], totals[j]) for j in range(len(vocab)) if totals[j] > 0.0]
+    # Sort by descending weight, then concept name for deterministic ties.
+    weighted.sort(key=lambda kv: (-kv[1], kv[0]))
+    return weighted
+
+
+def _name_clusters(
+    members: dict[int, tuple[int, ...]],
+    vectors: Sequence[Sequence[float]],
+    vocab: Vocabulary,
+) -> tuple[dict[int, str], dict[int, tuple[tuple[str, float], ...]]]:
+    names: dict[int, str] = {}
+    tops: dict[int, tuple[tuple[str, float], ...]] = {}
+    used: dict[str, int] = {}
+    # Deterministic order: by cluster id.
+    for cid in sorted(members):
+        if cid == UNASSIGNED:
+            names[cid] = "unassigned"
+            tops[cid] = ()
+            continue
+        weighted = _aggregate_weights(members[cid], vectors, vocab)
+        tops[cid] = tuple(weighted[:5])
+        base = weighted[0][0] if weighted else f"cluster_{cid}"
+        if base in used:
+            used[base] += 1
+            name = f"{base}_{used[base]}"
+        else:
+            used[base] = 1
+            name = base
+        names[cid] = name
+    return names, tops
+
+
+def _hdbscan_labels(dist: Sequence[Sequence[float]], min_cluster_size: int) -> list[int]:
+    import hdbscan  # local import; optional dependency
+
+    clusterer = hdbscan.HDBSCAN(metric="precomputed", min_cluster_size=max(2, min_cluster_size))
+    rows = [[float(x) for x in row] for row in dist]
+    labels = clusterer.fit_predict(rows)
+    return [int(x) for x in labels]
+
+
+def CN(
+    entities: Sequence[object],
+    vectors: Sequence[Sequence[float]],
+    vocab: Vocabulary,
+    config: ClusterConfig | None = None,
+) -> NamedClustering:
+    """Cluster ``entities`` (by ``vectors``) and name the parts.
+
+    ``entities`` is only used for its length and ordering; the partition is
+    computed from ``vectors`` and named from ``vocab``. Returns a
+    deterministic :class:`NamedClustering`.
+    """
+    cfg = config or ClusterConfig()
+    n = len(vectors)
+    if len(entities) != n:
+        raise ValueError("entities and vectors must have the same length")
+
+    labels = [UNASSIGNED] * n
+    assigned = [i for i in range(n) if not is_zero(vectors[i])]
+
+    if assigned:
+        sub_vectors = [vectors[i] for i in assigned]
+        sub_dist = distance_matrix(sub_vectors)
+        sub_labels = _cluster(sub_dist, cfg)
+        # Re-pack sub-cluster ids into a dense 0..K-1 space, ordered by first
+        # appearance, then map back to original indices.
+        remap: dict[int, int] = {}
+        for sub_idx, lab in enumerate(sub_labels):
+            if lab < 0:  # hdbscan noise → unassigned
+                continue
+            if lab not in remap:
+                remap[lab] = len(remap)
+            labels[assigned[sub_idx]] = remap[lab]
+
+    grouped: dict[int, list[int]] = {}
+    for i, lab in enumerate(labels):
+        grouped.setdefault(lab, []).append(i)
+    members: dict[int, tuple[int, ...]] = {
+        cid: tuple(sorted(idxs)) for cid, idxs in grouped.items()
+    }
+
+    names, tops = _name_clusters(members, vectors, vocab)
+
+    real_labels = [labels[i] for i in assigned]
+    if assigned and len({lab for lab in real_labels if lab >= 0}) >= 2:
+        sub_dist = distance_matrix([vectors[i] for i in assigned])
+        sil = silhouette_score(sub_dist, real_labels)
+    else:
+        sil = 0.0
+
+    return NamedClustering(
+        labels=tuple(labels),
+        names=names,
+        members=members,
+        config=cfg,
+        silhouette=sil,
+        top_concepts=tops,
+    )
+
+
+def _cluster(dist: Sequence[Sequence[float]], cfg: ClusterConfig) -> list[int]:
+    n = len(dist)
+    if n == 0:
+        return []
+    if n == 1:
+        return [0]
+    if cfg.algorithm == "hdbscan":
+        return _hdbscan_labels(dist, min_cluster_size=2)
+    if cfg.algorithm == "fixed_k":
+        k_target = cfg.k
+        if k_target is None:
+            k_target, _ = select_k(
+                dist,
+                lambda k: agglomerative(dist, distance_threshold=None, k=k),
+                k_min=cfg.k_range[0],
+                k_max=cfg.k_range[1],
+            )
+        k_target = max(1, min(k_target, n))
+        return list(agglomerative(dist, distance_threshold=None, k=k_target))
+    # default: threshold-based agglomerative
+    return list(agglomerative(dist, distance_threshold=cfg.distance_threshold, k=None))
+
+
+__all__ = [
+    "CN",
+    "UNASSIGNED",
+    "Algorithm",
+    "ClusterConfig",
+    "NamedClustering",
+]

lp2graph/mining/cluster/silhouette.py

@@ -0,0 +1,88 @@
+"""Silhouette score and K selection (M3).
+
+The silhouette coefficient measures how well each point sits in its assigned
+cluster versus the nearest other cluster, in the same cosine-distance space
+the clustering runs in. It serves two roles in the method: it is the
+selection criterion for the fixed-K fallback, and it is reported (alongside
+bootstrap ARI) in the stability report.
+
+Pure Python and deterministic.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Callable, Sequence
+
+
+def silhouette_samples(dist: Sequence[Sequence[float]], labels: Sequence[int]) -> list[float]:
+    """Per-point silhouette values given a distance matrix and labels.
+
+    Points whose cluster has size 1, and all points when there is only one
+    cluster, get a silhouette of 0 (the conventional definition).
+    """
+    n = len(labels)
+    by_cluster: dict[int, list[int]] = defaultdict(list)
+    for i, lab in enumerate(labels):
+        by_cluster[lab].append(i)
+    clusters = sorted(by_cluster)
+    out = [0.0] * n
+    if len(clusters) < 2:
+        return out
+
+    for i in range(n):
+        own = labels[i]
+        own_members = by_cluster[own]
+        if len(own_members) <= 1:
+            out[i] = 0.0
+            continue
+        a = sum(dist[i][j] for j in own_members if j != i) / (len(own_members) - 1)
+        b = float("inf")
+        for lab in clusters:
+            if lab == own:
+                continue
+            members = by_cluster[lab]
+            mean_d = sum(dist[i][j] for j in members) / len(members)
+            if mean_d < b:
+                b = mean_d
+        denom = max(a, b)
+        out[i] = 0.0 if denom == 0.0 else (b - a) / denom
+    return out
+
+
+def silhouette_score(dist: Sequence[Sequence[float]], labels: Sequence[int]) -> float:
+    """Mean silhouette over all points (0.0 for a degenerate partition)."""
+    samples = silhouette_samples(dist, labels)
+    if not samples:
+        return 0.0
+    return sum(samples) / len(samples)
+
+
+def select_k(
+    dist: Sequence[Sequence[float]],
+    cluster_fn: Callable[[int], Sequence[int]],
+    *,
+    k_min: int,
+    k_max: int,
+) -> tuple[int, float]:
+    """Pick the K in ``[k_min, k_max]`` with the highest mean silhouette.
+
+    ``cluster_fn(k)`` must return labels for ``k`` clusters. Ties are broken
+    toward the smaller K (simpler partition). Returns ``(best_k, best_score)``.
+    """
+    n = len(dist)
+    hi = min(k_max, n - 1)
+    lo = max(2, k_min)
+    if hi < lo:
+        return (1, 0.0)
+    best_k = lo
+    best_score = float("-inf")
+    for k in range(lo, hi + 1):
+        score = silhouette_score(dist, cluster_fn(k))
+        if score > best_score:
+            best_score = score
+            best_k = k
+    return (best_k, best_score)
+
+
+__all__ = ["select_k", "silhouette_samples", "silhouette_score"]

lp2graph/mining/cluster/stability.py

@@ -0,0 +1,178 @@
+"""Clustering stability report: silhouette, bootstrap ARI, sensitivity (M3).
+
+A partition is only trustworthy if it survives resampling and small changes
+to the knobs. This module quantifies that:
+
+- **silhouette** — cohesion/separation of the reported partition (re-exported
+  from :mod:`lp2graph.mining.cluster.silhouette`).
+- **bootstrap ARI** — resample the entities with replacement, re-cluster, and
+  measure Adjusted Rand Index against the reference partition restricted to
+  the shared points; report mean and spread.
+- **sensitivity** — re-cluster under alternative configs (a different
+  algorithm, a coarsened vocabulary ``|C|``) and report the ARI to the
+  reference partition.
+
+Determinism: bootstrap resampling uses a seeded :class:`random.Random`, so the
+report is reproducible from ``(seed, versions)``.
+"""
+
+from __future__ import annotations
+
+import random
+from collections import defaultdict
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from lp2graph.mining.cluster.operator import CN, ClusterConfig, NamedClustering
+from lp2graph.mining.homologize.vectorize import Vocabulary, build_vocabulary
+
+
+def adjusted_rand_index(a: Sequence[int], b: Sequence[int]) -> float:
+    """Adjusted Rand Index between two labelings of the same points.
+
+    Returns 1.0 for identical partitions (up to relabeling), ~0.0 for random
+    agreement. Defined as 1.0 for the degenerate all-equal case.
+    """
+    if len(a) != len(b):
+        raise ValueError("labelings must have equal length")
+    n = len(a)
+    if n == 0:
+        return 1.0
+    contingency: dict[tuple[int, int], int] = defaultdict(int)
+    rows: dict[int, int] = defaultdict(int)
+    cols: dict[int, int] = defaultdict(int)
+    for x, y in zip(a, b, strict=True):
+        contingency[(x, y)] += 1
+        rows[x] += 1
+        cols[y] += 1
+
+    def comb2(x: int) -> int:
+        return x * (x - 1) // 2
+
+    sum_comb = sum(comb2(v) for v in contingency.values())
+    sum_rows = sum(comb2(v) for v in rows.values())
+    sum_cols = sum(comb2(v) for v in cols.values())
+    total = comb2(n)
+    expected = (sum_rows * sum_cols) / total if total else 0.0
+    max_index = (sum_rows + sum_cols) / 2.0
+    if max_index == expected:
+        return 1.0
+    return (sum_comb - expected) / (max_index - expected)
+
+
+@dataclass(frozen=True, slots=True)
+class StabilityReport:
+    """Silhouette + bootstrap ARI + sensitivity for one clustering."""
+
+    silhouette: float
+    bootstrap_ari_mean: float
+    bootstrap_ari_min: float
+    bootstrap_ari_max: float
+    n_bootstrap: int
+    sensitivity: dict[str, float]
+    config_version: str
+    seed: int
+
+
+def bootstrap_ari(
+    vectors: Sequence[Sequence[float]],
+    vocab: Vocabulary,
+    reference: NamedClustering,
+    config: ClusterConfig,
+    *,
+    n_bootstrap: int = 25,
+) -> tuple[float, float, float]:
+    """Mean/min/max ARI of bootstrap re-clusterings vs the reference partition.
+
+    Each bootstrap draws ``n`` indices with replacement, re-clusters the
+    unique drawn points, and compares to the reference labels on those same
+    points. Deterministic via ``config.seed``.
+    """
+    n = len(vectors)
+    if n < 3:
+        return (1.0, 1.0, 1.0)
+    rng = random.Random(config.seed)
+    scores: list[float] = []
+    for _ in range(n_bootstrap):
+        drawn = sorted({rng.randrange(n) for _ in range(n)})
+        if len(drawn) < 2:
+            continue
+        sub_vectors = [vectors[i] for i in drawn]
+        sub = CN(drawn, sub_vectors, vocab, config)
+        ref_labels = [reference.labels[i] for i in drawn]
+        scores.append(adjusted_rand_index(ref_labels, list(sub.labels)))
+    if not scores:
+        return (1.0, 1.0, 1.0)
+    return (sum(scores) / len(scores), min(scores), max(scores))
+
+
+def _coarsen_vocabulary(vocab: Vocabulary, keep_every: int) -> Vocabulary:
+    """Drop concepts to shrink ``|C|`` (deterministic stride over sorted axis)."""
+    kept = tuple(c for i, c in enumerate(vocab.concepts) if i % keep_every == 0)
+    return build_vocabulary([{c: 1 for c in kept}], version=vocab.version)
+
+
+def sensitivity(
+    vectors: Sequence[Sequence[float]],
+    vocab: Vocabulary,
+    reference: NamedClustering,
+    config: ClusterConfig,
+) -> dict[str, float]:
+    """ARI of the reference partition against alternative configurations.
+
+    Probes robustness to the clustering algorithm (``fixed_k``) and to a
+    coarsened vocabulary ``|C|`` (every-other concept). Each entry is the ARI
+    between the reference labels and the labels under that variation.
+    """
+    out: dict[str, float] = {}
+
+    alt_algo = ClusterConfig(
+        algorithm="fixed_k",
+        k_range=config.k_range,
+        seed=config.seed,
+        version=config.version,
+    )
+    alt = CN(list(range(len(vectors))), vectors, vocab, alt_algo)
+    out["algorithm=fixed_k"] = adjusted_rand_index(list(reference.labels), list(alt.labels))
+
+    if len(vocab) >= 4:
+        coarse_vocab = _coarsen_vocabulary(vocab, keep_every=2)
+        pos = {c: i for i, c in enumerate(vocab.concepts)}
+        # Re-project vectors onto the coarsened axis (keep matching coordinates).
+        coarse_vectors = [tuple(v[pos[c]] for c in coarse_vocab.concepts) for v in vectors]
+        coarse = CN(list(range(len(vectors))), coarse_vectors, coarse_vocab, config)
+        out["vocabulary=halved"] = adjusted_rand_index(list(reference.labels), list(coarse.labels))
+
+    return out
+
+
+def stability_report(
+    vectors: Sequence[Sequence[float]],
+    vocab: Vocabulary,
+    clustering: NamedClustering,
+    config: ClusterConfig,
+    *,
+    n_bootstrap: int = 25,
+) -> StabilityReport:
+    """Assemble the full stability report for one clustering."""
+    mean, lo, hi = bootstrap_ari(vectors, vocab, clustering, config, n_bootstrap=n_bootstrap)
+    sens = sensitivity(vectors, vocab, clustering, config)
+    return StabilityReport(
+        silhouette=clustering.silhouette,
+        bootstrap_ari_mean=mean,
+        bootstrap_ari_min=lo,
+        bootstrap_ari_max=hi,
+        n_bootstrap=n_bootstrap,
+        sensitivity=sens,
+        config_version=config.version,
+        seed=config.seed,
+    )
+
+
+__all__ = [
+    "StabilityReport",
+    "adjusted_rand_index",
+    "bootstrap_ari",
+    "sensitivity",
+    "stability_report",
+]