keo 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
keo/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """KEO: benchmark of K-means (Lloyd), E-means and O-means."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,25 @@
1
+ # Track A (primary) dataset. Not used by either source paper (see README section 2).
2
+ name: glove100
3
+ description: GloVe-100 word embeddings (ann-benchmarks, angular)
4
+ source:
5
+ type: url
6
+ url: http://ann-benchmarks.com/glove-100-angular.hdf5
7
+ filename: glove-100-angular.hdf5
8
+ format: hdf5
9
+ train_key: train
10
+ eval_key: test
11
+ # Trust-on-first-use: ann-benchmarks publishes no checksum. Pinned from the first download
12
+ # on 2026-11-16 (see docs/ASSUMPTIONS.md A31). Empty string = not pinned yet.
13
+ sha256: "544af1d5e84e112cd4749571dcfd8ca109818a572f850af75a3a09e093a953c4"
14
+ expected_bytes: 485413888
15
+ d: 100
16
+ n_train: 1183514
17
+ n_eval: 10000
18
+ normalize: true # L2-normalise (A24); verified/decided after `keo data-info`
19
+ split:
20
+ val_fraction: 0.5 # provided queries -> val/test (A25)
21
+ seed: 0
22
+ # Experiment defaults (consumed from Phase 6 on)
23
+ k_grid: [4096, 16384, 32768]
24
+ n_subset_grid: [100000, 300000, 1000000, null] # null = full train
25
+ min_points_per_cluster: 8 # keep n/k >= 8 [E App. B.1]
@@ -0,0 +1,18 @@
1
+ # Smoke test: GloVe-100 subset, tiny grid, CPU only, must finish in < 5 minutes (Phase 6 gate).
2
+ # o-means uses cell_size 8 here (k=512 with the default 128 would give nlist = nprobe = 16 = exact search; A42).
3
+ name: smoke
4
+ dataset: glove100
5
+ n: [50000]
6
+ k: [512]
7
+ seeds: [0, 1]
8
+ baseline_iterations: 25
9
+ warmup: true
10
+ train_eval_subset: null
11
+ runs:
12
+ - {algo: lloyd, backend: reference, budgets: [1, 2, 3, 5, 10, 25]}
13
+ - {algo: emeans, backend: reference, budgets: [1, 2]}
14
+ - {algo: omeans, backend: reference, budgets: [1, 2], params: {cell_size: 8}}
15
+ # official implementations: run when available, otherwise listed in skipped.json with the reason
16
+ - {algo: lloyd, backend: official, budgets: [25]}
17
+ - {algo: emeans, backend: official, budgets: [1, 2]}
18
+ - {algo: omeans, backend: official, budgets: [1, 2]}
keo/algos/__init__.py ADDED
@@ -0,0 +1,6 @@
1
+ from keo.algos.base import Budget, Clusterer, FitResult
2
+ from keo.algos.emeans import EMeans
3
+ from keo.algos.lloyd import Lloyd
4
+ from keo.algos.omeans import OMeans
5
+
6
+ __all__ = ["Budget", "Clusterer", "EMeans", "FitResult", "Lloyd", "OMeans"]
File without changes
@@ -0,0 +1,163 @@
1
+ """Stand-alone worker: runs Faiss-based algorithms in their OWN process.
2
+
3
+ Why a separate process: on macOS the torch and faiss wheels each bundle their own libomp; loading both into one
4
+ process aborts ("OMP: Error #15") or segfaults. KMP_DUPLICATE_LIB_OK=TRUE would silence that but can "silently
5
+ produce incorrect results", so we isolate instead (docs/ASSUMPTIONS.md A36).
6
+
7
+ Executed BY PATH (`python -I _faiss_worker.py <workdir>`), never imported, so it must not import torch (except in the
8
+ explicit GPU mode) or the `keo` package. Protocol: <workdir>/params.json {"algo": ..., ...}; optional X.npy, init.npy;
9
+ outputs result.json (+ centroids.npy, labels.npy). Exit code 3 = "algorithm unavailable" (reason in result.json).
10
+
11
+ Modes: kmeans | emeans | omeans | probe | recipe
12
+ """
13
+
14
+ import importlib.util
15
+ import json
16
+ import sys
17
+ import time
18
+ from pathlib import Path
19
+
20
+ import numpy as np
21
+
22
+ UNAVAILABLE = 3
23
+
24
+
25
+ def _write(w: Path, obj: dict) -> None:
26
+ (w / "result.json").write_text(json.dumps(obj, default=str))
27
+
28
+
29
+ def _load(w: Path):
30
+ X = np.ascontiguousarray(np.load(w / "X.npy", mmap_mode="r"), dtype=np.float32)
31
+ init = np.ascontiguousarray(np.load(w / "init.npy"), dtype=np.float32) if (w / "init.npy").exists() else None
32
+ return X, init
33
+
34
+
35
+ def real_gpu_count(faiss) -> int:
36
+ """GPUs usable by faiss. NOTE: the faiss-cpu wheel's `get_num_gpus()` returns 1 (a stub!), so a count is only
37
+ trusted when the real GPU classes exist (found on faiss-cpu 1.16.0: `GpuIndexFlat` is absent)."""
38
+ if not hasattr(faiss, "GpuIndexFlat") or not hasattr(faiss, "get_num_gpus"):
39
+ return 0
40
+ return int(faiss.get_num_gpus())
41
+
42
+
43
+ def run_probe(w: Path, p: dict) -> int:
44
+ import faiss
45
+
46
+ _write(w, {
47
+ "faiss_version": faiss.__version__,
48
+ "has_e_means": importlib.util.find_spec("faiss.contrib.e_means") is not None,
49
+ "has_o_means": importlib.util.find_spec("faiss.contrib.o_means") is not None,
50
+ "num_gpus": real_gpu_count(faiss),
51
+ "raw_get_num_gpus": int(faiss.get_num_gpus()) if hasattr(faiss, "get_num_gpus") else None,
52
+ })
53
+ return 0
54
+
55
+
56
+ def run_recipe(w: Path, p: dict) -> int:
57
+ """The shipped e-means recipe (batch size, alpha0, law) for given (n, k, T, d) -- used to verify our `official` recipe."""
58
+ from faiss.contrib.e_means import base
59
+
60
+ out = []
61
+ for n, k, T, d in p["cases"]:
62
+ law, _ = base._resolve_batch_law("auto", d)
63
+ B, a0 = base._resolve_recipe(n, k, T, law, None, None)
64
+ out.append({"n": n, "k": k, "T": T, "d": d, "law": law, "batch_size": int(B), "alpha0": float(a0)})
65
+ _write(w, {"recipes": out})
66
+ return 0
67
+
68
+
69
+ def run_kmeans(w: Path, p: dict) -> int:
70
+ import faiss
71
+
72
+ X, init = _load(w)
73
+ n, d = X.shape
74
+ if p.get("threads"):
75
+ faiss.omp_set_num_threads(int(p["threads"]))
76
+ use_gpu = bool(p.get("gpu"))
77
+ if use_gpu and real_gpu_count(faiss) == 0:
78
+ _write(w, {"unavailable": "faiss was built without GPU support or no GPU is visible"})
79
+ return UNAVAILABLE
80
+ km = faiss.Kmeans(
81
+ d, int(p["k"]), niter=int(p["niter"]), nredo=1, seed=int(p["seed"]), verbose=False, gpu=use_gpu,
82
+ max_points_per_centroid=2**30, # disable the 256 points/centroid subsampling [E §4.1 fn.1]
83
+ min_points_per_centroid=1, # silence the warning-only <39 points/centroid notice
84
+ )
85
+ t0 = time.perf_counter()
86
+ km.train(X, init_centroids=init)
87
+ fit_s = time.perf_counter() - t0
88
+ np.save(w / "centroids.npy", np.ascontiguousarray(km.centroids, dtype=np.float32))
89
+ _write(w, {"fit_seconds": fit_s, "obj": [float(v) for v in km.obj], "faiss_version": faiss.__version__,
90
+ "omp_threads": faiss.omp_get_max_threads(), "n": int(n)})
91
+ return 0
92
+
93
+
94
+ def run_emeans(w: Path, p: dict) -> int:
95
+ if importlib.util.find_spec("faiss.contrib.e_means") is None:
96
+ _write(w, {"unavailable": "installed faiss has no contrib.e_means (needs faiss >= 1.16.0)"})
97
+ return UNAVAILABLE
98
+ import faiss
99
+ from faiss.contrib.e_means import NumpyEmeans
100
+
101
+ X, init = _load(w)
102
+ n, d = X.shape
103
+ if p.get("threads"):
104
+ faiss.omp_set_num_threads(int(p["threads"]))
105
+ extra = {kk: p[kk] for kk in ("batch_law", "unused_threshold", "batch_size", "alpha0") if p.get(kk) is not None}
106
+ em = NumpyEmeans(d, int(p["k"]), nepochs=float(p["epochs"]), seed=int(p["seed"]), verbose=False, **extra)
107
+ t0 = time.perf_counter()
108
+ centroids, labels = em.train(X, init_centroids=init)
109
+ fit_s = time.perf_counter() - t0
110
+ np.save(w / "centroids.npy", np.ascontiguousarray(centroids, dtype=np.float32))
111
+ if labels is not None:
112
+ np.save(w / "labels.npy", np.asarray(labels, dtype=np.int64))
113
+ rec = {kk: v for kk, v in (em.recipe or {}).items() if isinstance(v, (int, float, str, bool, list, type(None)))}
114
+ _write(w, {"fit_seconds": fit_s, "recipe": rec, "n_splits": int(em.n_splits), "faiss_version": faiss.__version__,
115
+ "omp_threads": faiss.omp_get_max_threads(), "n": int(n)})
116
+ return 0
117
+
118
+
119
+ def run_omeans(w: Path, p: dict) -> int:
120
+ """UNVERIFIED against a real install: written from the o-means project page (`Omeans(d, k, nepochs)`, `.train(x)`,
121
+ `.assign(x)`); the authors' faiss fork has no public URL we could locate. CUDA only."""
122
+ if importlib.util.find_spec("faiss.contrib.o_means") is None:
123
+ _write(w, {"unavailable": "faiss.contrib.o_means not found: it lives in the o-means authors' faiss fork, not in "
124
+ "the released faiss wheel"})
125
+ return UNAVAILABLE
126
+ import faiss # noqa: F401 (import faiss before torch in this process)
127
+ import torch
128
+
129
+ if not torch.cuda.is_available():
130
+ _write(w, {"unavailable": "o-means is a GPU algorithm and no CUDA device is visible"})
131
+ return UNAVAILABLE
132
+ from faiss.contrib.o_means import Omeans
133
+
134
+ X, _ = _load(w)
135
+ n, d = X.shape
136
+ torch.manual_seed(int(p["seed"])) # the page documents no seed/init argument: shared init is NOT possible
137
+ x = torch.from_numpy(X).cuda()
138
+ om = Omeans(d, int(p["k"]), nepochs=float(p["epochs"]))
139
+ torch.cuda.synchronize()
140
+ t0 = time.perf_counter()
141
+ centroids, labels = om.train(x)
142
+ torch.cuda.synchronize()
143
+ fit_s = time.perf_counter() - t0
144
+ np.save(w / "centroids.npy", np.ascontiguousarray(centroids.detach().cpu().numpy(), dtype=np.float32))
145
+ _write(w, {"fit_seconds": fit_s, "n": int(n), "shared_init": False})
146
+ return 0
147
+
148
+
149
+ MODES = {"probe": run_probe, "recipe": run_recipe, "kmeans": run_kmeans, "emeans": run_emeans, "omeans": run_omeans}
150
+
151
+
152
+ def main(workdir: str) -> int:
153
+ w = Path(workdir)
154
+ params = json.loads((w / "params.json").read_text())
155
+ algo = params.get("algo", "kmeans")
156
+ if algo not in MODES:
157
+ _write(w, {"error": f"unknown algo {algo!r}"})
158
+ return 2
159
+ return MODES[algo](w, params)
160
+
161
+
162
+ if __name__ == "__main__":
163
+ sys.exit(main(sys.argv[1]))
@@ -0,0 +1,64 @@
1
+ """Parent-side client for `_faiss_worker.py` (runs it in a separate process; the parent never imports faiss)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import functools
6
+ import importlib.util
7
+ import json
8
+ import subprocess
9
+ import sys
10
+ import tempfile
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ import numpy as np
15
+
16
+ WORKER = Path(__file__).with_name("_faiss_worker.py")
17
+ UNAVAILABLE_EXIT = 3
18
+
19
+
20
+ class AdapterUnavailable(RuntimeError):
21
+ """The requested official implementation cannot run here (missing package, fork, or GPU)."""
22
+
23
+
24
+ def faiss_installed() -> bool:
25
+ return importlib.util.find_spec("faiss") is not None
26
+
27
+
28
+ def run_worker(
29
+ params: dict[str, Any],
30
+ X: np.ndarray | None = None,
31
+ init: np.ndarray | None = None,
32
+ want_labels: bool = False,
33
+ ) -> tuple[dict[str, Any], np.ndarray | None, np.ndarray | None]:
34
+ """Run one worker job. Returns (result dict, centroids or None, labels or None)."""
35
+ if not faiss_installed():
36
+ raise AdapterUnavailable("faiss is not installed (pip install faiss-cpu)")
37
+ with tempfile.TemporaryDirectory(prefix="keo_worker_") as tmp:
38
+ w = Path(tmp)
39
+ if X is not None:
40
+ np.save(w / "X.npy", np.ascontiguousarray(X, dtype=np.float32))
41
+ if init is not None:
42
+ np.save(w / "init.npy", np.ascontiguousarray(init, dtype=np.float32))
43
+ (w / "params.json").write_text(json.dumps(params))
44
+ proc = subprocess.run([sys.executable, "-I", str(WORKER), str(w)], capture_output=True, text=True)
45
+ res: dict[str, Any] = {}
46
+ if (w / "result.json").exists():
47
+ res = json.loads((w / "result.json").read_text())
48
+ if proc.returncode == UNAVAILABLE_EXIT:
49
+ raise AdapterUnavailable(res.get("unavailable", "unavailable"))
50
+ if proc.returncode != 0:
51
+ raise RuntimeError(
52
+ f"worker failed (algo={params.get('algo')}, exit {proc.returncode}): "
53
+ f"{res.get('error') or proc.stderr[-2000:]}"
54
+ )
55
+ cen = np.load(w / "centroids.npy") if (w / "centroids.npy").exists() else None
56
+ lab = np.load(w / "labels.npy") if (want_labels and (w / "labels.npy").exists()) else None
57
+ return res, cen, lab
58
+
59
+
60
+ @functools.lru_cache(maxsize=1)
61
+ def probe() -> dict[str, Any]:
62
+ """Capabilities of the installed faiss (cached per process): version, e_means/o_means presence, GPU count."""
63
+ res, _, _ = run_worker({"algo": "probe"})
64
+ return res
@@ -0,0 +1,84 @@
1
+ """Official backend: Faiss Lloyd k-means (the e-means paper's Lloyd reference, [E App. C]); CPU, or GPU if available.
2
+
3
+ Runs in a SEPARATE PROCESS (see _faiss_worker.py): this parent process must never import faiss, because the
4
+ torch and faiss wheels ship conflicting libomp copies on macOS (A36). Reported fit time is measured inside the
5
+ worker around `Kmeans.train` only, so data serialisation / process start-up are excluded.
6
+
7
+ Faiss's automatic sub-sampling (max 256 points per centroid) is disabled, as in the paper [E §4.1 fn.1].
8
+ `niter` = number of Lloyd iterations (verified: one objective entry per iteration).
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import importlib.metadata
14
+ import time
15
+
16
+ import numpy as np
17
+
18
+ from keo.algos.adapters._worker_client import AdapterUnavailable, faiss_installed, probe, run_worker
19
+ from keo.algos.base import Budget, FitResult
20
+ from keo.algos.init import random_row_init
21
+
22
+
23
+ def is_available(gpu: bool = False) -> tuple[bool, str]:
24
+ """Check availability WITHOUT importing faiss into this process."""
25
+ if not faiss_installed():
26
+ return False, "faiss is not installed (pip install faiss-cpu)"
27
+ try:
28
+ ver = importlib.metadata.version("faiss-cpu")
29
+ except importlib.metadata.PackageNotFoundError:
30
+ ver = "unknown"
31
+ if gpu:
32
+ n = int(probe().get("num_gpus", 0))
33
+ return (n > 0, f"faiss-cpu {ver}; {n} GPU(s) visible to faiss" if n else
34
+ f"faiss-cpu {ver} has no GPU support / no GPU visible (needs faiss-gpu on a CUDA machine)")
35
+ return True, f"faiss-cpu {ver} (isolated worker process)"
36
+
37
+
38
+ class FaissLloyd:
39
+ name = "lloyd"
40
+
41
+ def __init__(self, threads: int | None = None, gpu: bool = False) -> None:
42
+ self.threads, self.gpu = threads, gpu
43
+ self.backend = "official"
44
+ self.variant = "faiss-gpu" if gpu else "faiss-cpu"
45
+
46
+ def fit(
47
+ self,
48
+ X: np.ndarray,
49
+ k: int,
50
+ budget: Budget,
51
+ seed: int,
52
+ init_centroids: np.ndarray | None = None,
53
+ ) -> FitResult:
54
+ ok, why = is_available(self.gpu)
55
+ if not ok:
56
+ raise AdapterUnavailable(why)
57
+ iters = budget.iterations if budget.iterations is not None else int(budget.epochs) # type: ignore[arg-type]
58
+ n, d = X.shape
59
+
60
+ t0 = time.perf_counter()
61
+ if init_centroids is None:
62
+ init_centroids, _ = random_row_init(X, k, seed)
63
+ t_init = time.perf_counter() - t0
64
+
65
+ res, centroids, _ = run_worker(
66
+ {"algo": "kmeans", "k": k, "niter": iters, "seed": seed, "threads": self.threads, "gpu": self.gpu},
67
+ X=X, init=init_centroids,
68
+ )
69
+ assert centroids is not None
70
+ obj = np.asarray(res["obj"], dtype=np.float64) # sum of squared distances at each iteration's assignment
71
+ return FitResult(
72
+ centroids=centroids,
73
+ train_labels=None,
74
+ timings={"init": t_init, "fit": float(res["fit_seconds"]), "finalize": 0.0},
75
+ op_counts={"distance_computations": float(n) * k * iters, "assign_passes": float(iters)},
76
+ diagnostics={
77
+ "init": "given",
78
+ "iterations": iters,
79
+ "train_mse_per_iter": (obj / res["n"]).tolist(),
80
+ "faiss_version": res["faiss_version"],
81
+ "faiss_omp_threads": res["omp_threads"],
82
+ "variant": self.variant,
83
+ },
84
+ )
@@ -0,0 +1,120 @@
1
+ """Official backends shipped by the authors.
2
+
3
+ * `OfficialEmeans` -- `faiss.contrib.e_means.NumpyEmeans` (in faiss-cpu >= 1.16.0). VERIFIED on this machine: the API was
4
+ read from the installed package (constructor, `train(x, init_centroids=...)`, `recipe`), and it runs on CPU.
5
+ * `OfficialOmeans` -- `faiss.contrib.o_means.Omeans` from the o-means authors' faiss fork. NOT verifiable here: no public
6
+ URL for the fork could be found, it is not in the released wheel, and it needs CUDA. Written from the project page
7
+ only (`Omeans(d, k, nepochs)`, `.train(x)`), so it is flagged UNVERIFIED and reports "unavailable" cleanly.
8
+
9
+ Both run in the isolated worker process (A36). Neither is allowed to be mixed with `reference` rows (backend column).
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import time
15
+
16
+ import numpy as np
17
+
18
+ from keo.algos.adapters._worker_client import AdapterUnavailable, faiss_installed, probe, run_worker
19
+ from keo.algos.base import Budget, FitResult
20
+ from keo.algos.init import random_row_init
21
+
22
+
23
+ def emeans_available() -> tuple[bool, str]:
24
+ if not faiss_installed():
25
+ return False, "faiss is not installed (pip install faiss-cpu)"
26
+ p = probe()
27
+ if not p.get("has_e_means"):
28
+ return False, f"faiss {p.get('faiss_version')} has no contrib.e_means (needs >= 1.16.0)"
29
+ return True, f"faiss {p['faiss_version']} contrib.e_means.NumpyEmeans (CPU, isolated worker)"
30
+
31
+
32
+ def omeans_available() -> tuple[bool, str]:
33
+ if not faiss_installed():
34
+ return False, "faiss is not installed"
35
+ p = probe()
36
+ if not p.get("has_o_means"):
37
+ return False, ("faiss.contrib.o_means is not in the installed faiss "
38
+ f"({p.get('faiss_version')}): it lives in the o-means authors' fork (no public URL found); needs CUDA")
39
+ try:
40
+ import torch
41
+
42
+ if not torch.cuda.is_available():
43
+ return False, "o-means is a GPU algorithm and no CUDA device is visible"
44
+ except ImportError:
45
+ return False, "torch is not installed"
46
+ return True, "faiss fork with contrib.o_means + CUDA (UNVERIFIED adapter)"
47
+
48
+
49
+ class OfficialEmeans:
50
+ name = "emeans"
51
+ backend = "official"
52
+ variant = "faiss.contrib.e_means.NumpyEmeans"
53
+
54
+ def __init__(self, threads: int | None = None, batch_law: str = "auto", unused_threshold: float | None = None,
55
+ batch_size: int | None = None, alpha0: float | None = None) -> None:
56
+ self.threads, self.batch_law, self.unused_threshold = threads, batch_law, unused_threshold
57
+ self.batch_size, self.alpha0 = batch_size, alpha0
58
+
59
+ def fit(self, X: np.ndarray, k: int, budget: Budget, seed: int,
60
+ init_centroids: np.ndarray | None = None) -> FitResult:
61
+ ok, why = emeans_available()
62
+ if not ok:
63
+ raise AdapterUnavailable(why)
64
+ if budget.epochs is None:
65
+ raise ValueError("OfficialEmeans needs an epochs budget")
66
+ n, d = X.shape
67
+ t0 = time.perf_counter()
68
+ if init_centroids is None:
69
+ init_centroids, _ = random_row_init(X, k, seed)
70
+ t_init = time.perf_counter() - t0
71
+
72
+ res, centroids, labels = run_worker(
73
+ {"algo": "emeans", "k": k, "epochs": float(budget.epochs), "seed": seed, "threads": self.threads,
74
+ "batch_law": self.batch_law, "unused_threshold": self.unused_threshold,
75
+ "batch_size": self.batch_size, "alpha0": self.alpha0},
76
+ X=X, init=init_centroids, want_labels=True,
77
+ )
78
+ assert centroids is not None
79
+ rec = res["recipe"]
80
+ rows = float(sum(rec.get("epoch_rows", [])))
81
+ dist = rows * k
82
+ return FitResult(
83
+ centroids=centroids,
84
+ train_labels=labels,
85
+ timings={"init": t_init, "fit": float(res["fit_seconds"]), "finalize": 0.0},
86
+ op_counts={"distance_computations": dist, "assign_passes": dist / (float(n) * k)},
87
+ diagnostics={
88
+ "init": "given", "epochs": float(budget.epochs), "n": n, "k": k,
89
+ "batch_size": rec.get("batch_size"), "alpha0": rec.get("alpha0"), "batch_law": rec.get("batch_law"),
90
+ "steps_per_pass": rec.get("steps_per_pass"), "n_updates": rec.get("total_steps"),
91
+ "n_revived_total": res["n_splits"], "recipe_official": rec,
92
+ "faiss_version": res["faiss_version"], "variant": self.variant,
93
+ },
94
+ )
95
+
96
+
97
+ class OfficialOmeans:
98
+ """UNVERIFIED adapter (see module docstring). Cannot share the reference init: the page documents no init argument."""
99
+
100
+ name = "omeans"
101
+ backend = "official"
102
+ variant = "faiss-fork contrib.o_means.Omeans (UNVERIFIED)"
103
+
104
+ def fit(self, X: np.ndarray, k: int, budget: Budget, seed: int,
105
+ init_centroids: np.ndarray | None = None) -> FitResult:
106
+ ok, why = omeans_available()
107
+ if not ok:
108
+ raise AdapterUnavailable(why)
109
+ if budget.epochs is None:
110
+ raise ValueError("OfficialOmeans needs an epochs budget")
111
+ n, d = X.shape
112
+ res, centroids, _ = run_worker({"algo": "omeans", "k": k, "epochs": float(budget.epochs), "seed": seed}, X=X)
113
+ assert centroids is not None
114
+ return FitResult(
115
+ centroids=centroids, train_labels=None,
116
+ timings={"init": 0.0, "fit": float(res["fit_seconds"]), "finalize": 0.0},
117
+ op_counts={}, # the fork does not expose operation counts
118
+ diagnostics={"init": "own (random rows of x; not shared)", "shared_init": False, "variant": self.variant,
119
+ "ignored_init_centroids": init_centroids is not None},
120
+ )
@@ -0,0 +1,76 @@
1
+ """Official backend: Flash-KMeans (Yang et al., arXiv:2603.09229), the GPU Lloyd reference of both papers.
2
+
3
+ UNVERIFIED adapter: the package (PyPI `flash-kmeans`, 0.3.1 in both papers) is not installed here and needs CUDA/Triton.
4
+ Written from its public README (`batch_kmeans_Euclid(x[B,N,D], n_clusters, tol, verbose, ...)` -> (ids, centers, _)) and from
5
+ [E App. C] ("one public-API call, forced to the full count: max_iters=T, tol=-1"). Because parameter names are not
6
+ confirmed, the call passes only keyword arguments that appear in the function's signature, and records which were
7
+ honoured. `flash-kmeans-mlx` (an independent Apple-Silicon port by someone else) is NOT this implementation and is
8
+ deliberately not used.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import importlib.util
14
+ import inspect
15
+ import time
16
+
17
+ import numpy as np
18
+
19
+ from keo.algos.adapters._worker_client import AdapterUnavailable
20
+ from keo.algos.base import Budget, FitResult
21
+ from keo.algos.init import random_row_init
22
+
23
+
24
+ def is_available() -> tuple[bool, str]:
25
+ if importlib.util.find_spec("flash_kmeans") is None:
26
+ return False, "flash_kmeans is not installed (pip install flash-kmeans; needs a CUDA GPU with Triton)"
27
+ try:
28
+ import torch
29
+ except ImportError:
30
+ return False, "torch is not installed"
31
+ if not torch.cuda.is_available():
32
+ return False, "flash_kmeans needs a CUDA GPU; none is visible"
33
+ return True, "flash_kmeans + CUDA (UNVERIFIED adapter)"
34
+
35
+
36
+ class FlashKMeans:
37
+ name = "lloyd"
38
+ backend = "official"
39
+ variant = "flash-kmeans (UNVERIFIED)"
40
+
41
+ def __init__(self, dtype: str = "float32") -> None:
42
+ self.dtype = dtype
43
+
44
+ def fit(self, X: np.ndarray, k: int, budget: Budget, seed: int,
45
+ init_centroids: np.ndarray | None = None) -> FitResult:
46
+ ok, why = is_available()
47
+ if not ok:
48
+ raise AdapterUnavailable(why)
49
+ import flash_kmeans
50
+ import torch
51
+
52
+ iters = budget.iterations if budget.iterations is not None else int(budget.epochs) # type: ignore[arg-type]
53
+ n, d = X.shape
54
+ fn = flash_kmeans.batch_kmeans_Euclid
55
+ sig = inspect.signature(fn).parameters
56
+ wanted = {"n_clusters": k, "max_iters": iters, "tol": -1.0, "verbose": False}
57
+ if init_centroids is not None:
58
+ wanted["init_centroids"] = torch.from_numpy(np.ascontiguousarray(init_centroids))[None].cuda()
59
+ kwargs = {kk: v for kk, v in wanted.items() if kk in sig}
60
+ honoured, ignored = sorted(kwargs), sorted(set(wanted) - set(kwargs))
61
+ torch.manual_seed(seed)
62
+ x = torch.from_numpy(np.ascontiguousarray(X)).to(getattr(torch, self.dtype))[None].cuda()
63
+ torch.cuda.synchronize()
64
+ t0 = time.perf_counter()
65
+ _, centers, *_ = fn(x, **kwargs)
66
+ torch.cuda.synchronize()
67
+ fit_s = time.perf_counter() - t0
68
+ return FitResult(
69
+ centroids=centers[0].float().cpu().numpy(),
70
+ train_labels=None,
71
+ timings={"init": 0.0, "fit": fit_s, "finalize": 0.0},
72
+ op_counts={"distance_computations": float(n) * k * iters, "assign_passes": float(iters)},
73
+ diagnostics={"init": "given" if "init_centroids" in kwargs else "own", "iterations": iters,
74
+ "kwargs_honoured": honoured, "kwargs_ignored_not_in_signature": ignored,
75
+ "shared_init": "init_centroids" in kwargs, "variant": self.variant},
76
+ )
@@ -0,0 +1,57 @@
1
+ """Which official implementations can run on THIS machine, and why not. Never raises."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import asdict, dataclass
6
+
7
+ from keo.algos.adapters import faiss_cpu, faiss_official, flash_kmeans
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class AdapterStatus:
12
+ key: str # stable id used in configs
13
+ algo: str # lloyd | emeans | omeans
14
+ description: str
15
+ available: bool
16
+ reason: str
17
+ verified: bool # True = API read from an installed package on this machine; False = written from docs only
18
+
19
+ def as_dict(self) -> dict:
20
+ return asdict(self)
21
+
22
+
23
+ def _safe(fn, *a) -> tuple[bool, str]:
24
+ try:
25
+ return fn(*a)
26
+ except Exception as e: # availability probing must never take the benchmark down
27
+ return False, f"probe failed: {type(e).__name__}: {e}"
28
+
29
+
30
+ def adapter_status() -> list[AdapterStatus]:
31
+ rows = [
32
+ ("faiss_lloyd_cpu", "lloyd", "Faiss CPU Lloyd k-means", _safe(faiss_cpu.is_available, False), True),
33
+ ("faiss_lloyd_gpu", "lloyd", "Faiss GPU Lloyd k-means", _safe(faiss_cpu.is_available, True), False),
34
+ ("flash_kmeans", "lloyd", "Flash-KMeans (GPU Lloyd)", _safe(flash_kmeans.is_available), False),
35
+ ("official_emeans_cpu", "emeans", "faiss.contrib.e_means.NumpyEmeans (CPU)", _safe(faiss_official.emeans_available), True),
36
+ ("official_omeans", "omeans", "faiss-fork contrib.o_means.Omeans (CUDA)", _safe(faiss_official.omeans_available), False),
37
+ ]
38
+ return [AdapterStatus(k, a, desc, ok, why, ver) for k, a, desc, (ok, why), ver in rows]
39
+
40
+
41
+ def get_adapter(key: str, **kwargs):
42
+ """Instantiate an adapter by key, or raise AdapterUnavailable with the reason."""
43
+ from keo.algos.adapters._worker_client import AdapterUnavailable
44
+
45
+ table = {
46
+ "faiss_lloyd_cpu": lambda: faiss_cpu.FaissLloyd(gpu=False, **kwargs),
47
+ "faiss_lloyd_gpu": lambda: faiss_cpu.FaissLloyd(gpu=True, **kwargs),
48
+ "flash_kmeans": lambda: flash_kmeans.FlashKMeans(**kwargs),
49
+ "official_emeans_cpu": lambda: faiss_official.OfficialEmeans(**kwargs),
50
+ "official_omeans": lambda: faiss_official.OfficialOmeans(**kwargs),
51
+ }
52
+ if key not in table:
53
+ raise KeyError(f"unknown adapter {key!r}; known: {sorted(table)}")
54
+ st = {s.key: s for s in adapter_status()}[key]
55
+ if not st.available:
56
+ raise AdapterUnavailable(f"{key}: {st.reason}")
57
+ return table[key]()
keo/algos/base.py ADDED
@@ -0,0 +1,40 @@
1
+ """Common interface for K / E / O clusterers (reference and official backends)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+ from typing import Any, Protocol
7
+
8
+ import numpy as np
9
+
10
+
11
+ @dataclass(frozen=True)
12
+ class Budget:
13
+ """Training budget. Lloyd uses `iterations`; E/O use `epochs` (may be fractional)."""
14
+
15
+ epochs: float | None = None
16
+ iterations: int | None = None
17
+
18
+ def __post_init__(self) -> None:
19
+ if (self.epochs is None) == (self.iterations is None):
20
+ raise ValueError("Budget needs exactly one of epochs / iterations")
21
+ if self.epochs is not None and self.epochs <= 0:
22
+ raise ValueError("epochs must be > 0")
23
+ if self.iterations is not None and self.iterations < 1:
24
+ raise ValueError("iterations must be >= 1")
25
+
26
+
27
+ @dataclass
28
+ class FitResult:
29
+ centroids: np.ndarray # float32 [k, d]
30
+ train_labels: np.ndarray | None # int64 [n]; may be approximate/stale (documented per algorithm)
31
+ timings: dict[str, float] = field(default_factory=dict) # seconds: init, fit, finalize
32
+ op_counts: dict[str, float] = field(default_factory=dict) # distance_computations, assign_passes
33
+ diagnostics: dict[str, Any] = field(default_factory=dict)
34
+
35
+
36
+ class Clusterer(Protocol):
37
+ name: str # "lloyd" | "emeans" | "omeans"
38
+ backend: str # "reference" | "official"
39
+
40
+ def fit(self, X: np.ndarray, k: int, budget: Budget, seed: int) -> FitResult: ...