linkscluster 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- linkscluster/__init__.py +26 -0
- linkscluster/configs.py +25 -0
- linkscluster/graph.py +115 -0
- linkscluster/links_clusterer.py +519 -0
- linkscluster/subcluster.py +102 -0
- linkscluster/utils.py +154 -0
- linkscluster-0.1.0.dist-info/METADATA +256 -0
- linkscluster-0.1.0.dist-info/RECORD +11 -0
- linkscluster-0.1.0.dist-info/WHEEL +5 -0
- linkscluster-0.1.0.dist-info/licenses/LICENSE +176 -0
- linkscluster-0.1.0.dist-info/top_level.txt +1 -0
linkscluster/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""__init__ file for linkscluster."""
|
|
2
|
+
|
|
3
|
+
from linkscluster import configs
|
|
4
|
+
from linkscluster import graph
|
|
5
|
+
from linkscluster import links_clusterer
|
|
6
|
+
from linkscluster import subcluster
|
|
7
|
+
from linkscluster import utils
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
|
10
|
+
|
|
11
|
+
LinksClusterer = links_clusterer.LinksClusterer
|
|
12
|
+
Subcluster = subcluster.Subcluster
|
|
13
|
+
SubclusterGraph = graph.SubclusterGraph
|
|
14
|
+
|
|
15
|
+
single_threshold = utils.single_threshold
|
|
16
|
+
multi_threshold = utils.multi_threshold
|
|
17
|
+
anisotropic_threshold = utils.anisotropic_threshold
|
|
18
|
+
s = utils.s
|
|
19
|
+
s_tilde = utils.s_tilde
|
|
20
|
+
compute_accuracy = utils.compute_accuracy
|
|
21
|
+
enforce_ordered_labels = utils.enforce_ordered_labels
|
|
22
|
+
l2_normalize = utils.l2_normalize
|
|
23
|
+
|
|
24
|
+
default_links_clusterer = configs.default_links_clusterer
|
|
25
|
+
facenet_clusterer = configs.facenet_clusterer
|
|
26
|
+
ge2e_voice_clusterer = configs.ge2e_voice_clusterer
|
linkscluster/configs.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Example configurations for Links online clustering."""
|
|
2
|
+
|
|
3
|
+
from linkscluster import links_clusterer
|
|
4
|
+
|
|
5
|
+
LinksClusterer = links_clusterer.LinksClusterer
|
|
6
|
+
|
|
7
|
+
# Default configuration for general high-dimensional unit embeddings
|
|
8
|
+
default_links_clusterer = LinksClusterer(
|
|
9
|
+
cluster_similarity_threshold=0.5,
|
|
10
|
+
subcluster_similarity_threshold=0.8,
|
|
11
|
+
pair_similarity_maximum=1.0)
|
|
12
|
+
|
|
13
|
+
# Configuration tuned for 128-dimensional FaceNet CNN face embeddings
|
|
14
|
+
# (Schroff et al., CVPR 2015)
|
|
15
|
+
facenet_clusterer = LinksClusterer(
|
|
16
|
+
cluster_similarity_threshold=0.6,
|
|
17
|
+
subcluster_similarity_threshold=0.85,
|
|
18
|
+
pair_similarity_maximum=0.95)
|
|
19
|
+
|
|
20
|
+
# Configuration tuned for 256-dimensional LSTM voice embeddings
|
|
21
|
+
# (Wan et al., 2017 GE2E loss; Wang et al., 2017 Speaker Diarization with LSTM)
|
|
22
|
+
ge2e_voice_clusterer = LinksClusterer(
|
|
23
|
+
cluster_similarity_threshold=0.55,
|
|
24
|
+
subcluster_similarity_threshold=0.8,
|
|
25
|
+
pair_similarity_maximum=0.9)
|
linkscluster/graph.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Graph data structure for maintaining subcluster connectivity."""
|
|
2
|
+
|
|
3
|
+
import collections
|
|
4
|
+
import typing
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class SubclusterGraph:
|
|
8
|
+
"""Undirected graph representing connections between subclusters."""
|
|
9
|
+
|
|
10
|
+
def __init__(self):
|
|
11
|
+
"""Initialize an empty graph."""
|
|
12
|
+
self.adj: typing.Dict[int, typing.Set[int]] = collections.defaultdict(set)
|
|
13
|
+
|
|
14
|
+
def add_node(self, node_id: int):
|
|
15
|
+
"""Add a node to the graph if not already present."""
|
|
16
|
+
if node_id not in self.adj:
|
|
17
|
+
self.adj[node_id] = set()
|
|
18
|
+
|
|
19
|
+
def remove_node(self, node_id: int):
|
|
20
|
+
"""Remove a node and all its incident edges."""
|
|
21
|
+
if node_id in self.adj:
|
|
22
|
+
for neighbor in list(self.adj[node_id]):
|
|
23
|
+
self.adj[neighbor].discard(node_id)
|
|
24
|
+
del self.adj[node_id]
|
|
25
|
+
|
|
26
|
+
def add_edge(self, u: int, v: int):
|
|
27
|
+
"""Add an undirected edge between u and v."""
|
|
28
|
+
self.adj[u].add(v)
|
|
29
|
+
self.adj[v].add(u)
|
|
30
|
+
|
|
31
|
+
def remove_edge(self, u: int, v: int):
|
|
32
|
+
"""Remove the undirected edge between u and v if present."""
|
|
33
|
+
if u in self.adj:
|
|
34
|
+
self.adj[u].discard(v)
|
|
35
|
+
if v in self.adj:
|
|
36
|
+
self.adj[v].discard(u)
|
|
37
|
+
|
|
38
|
+
def has_edge(self, u: int, v: int) -> bool:
|
|
39
|
+
"""Return True if edge (u, v) exists."""
|
|
40
|
+
return u in self.adj and v in self.adj[u]
|
|
41
|
+
|
|
42
|
+
def neighbors(self, u: int) -> typing.Set[int]:
|
|
43
|
+
"""Return the set of neighbors of node u."""
|
|
44
|
+
return self.adj.get(u, set())
|
|
45
|
+
|
|
46
|
+
def degree(self, u: int) -> int:
|
|
47
|
+
"""Return the degree of node u."""
|
|
48
|
+
return len(self.adj.get(u, set()))
|
|
49
|
+
|
|
50
|
+
def merge_nodes(self, keep_id: int, remove_id: int):
|
|
51
|
+
"""Merge remove_id into keep_id, preserving all neighbor connections."""
|
|
52
|
+
if remove_id not in self.adj:
|
|
53
|
+
return
|
|
54
|
+
remove_neighbors = set(self.adj[remove_id])
|
|
55
|
+
self.remove_node(remove_id)
|
|
56
|
+
for neighbor in remove_neighbors:
|
|
57
|
+
if neighbor != keep_id:
|
|
58
|
+
self.add_edge(keep_id, neighbor)
|
|
59
|
+
|
|
60
|
+
def check_severed(
|
|
61
|
+
self,
|
|
62
|
+
start_id: int,
|
|
63
|
+
target_id: int,
|
|
64
|
+
allowed_nodes: typing.Optional[typing.Set[int]] = None
|
|
65
|
+
) -> typing.Tuple[bool, typing.Set[int]]:
|
|
66
|
+
"""Check if removing an edge disconnected start_id from target_id.
|
|
67
|
+
|
|
68
|
+
Traverses the component of start_id using BFS. If target_id is reached,
|
|
69
|
+
they remain connected. If target_id is unreachable, returns True and the
|
|
70
|
+
set of nodes reachable from start_id.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
start_id: the node to begin BFS from
|
|
74
|
+
target_id: the node we want to test reachability to
|
|
75
|
+
allowed_nodes: optional set of nodes restricting traversal
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
severed: bool, True if start_id cannot reach target_id
|
|
79
|
+
visited: set of nodes reachable from start_id (empty if not severed)
|
|
80
|
+
"""
|
|
81
|
+
visited: typing.Set[int] = set()
|
|
82
|
+
queue = collections.deque([start_id])
|
|
83
|
+
visited.add(start_id)
|
|
84
|
+
|
|
85
|
+
while queue:
|
|
86
|
+
curr = queue.popleft()
|
|
87
|
+
if curr == target_id:
|
|
88
|
+
return False, set()
|
|
89
|
+
for nbr in self.adj.get(curr, ()):
|
|
90
|
+
if nbr not in visited:
|
|
91
|
+
if allowed_nodes is None or nbr in allowed_nodes:
|
|
92
|
+
visited.add(nbr)
|
|
93
|
+
queue.append(nbr)
|
|
94
|
+
|
|
95
|
+
return True, visited
|
|
96
|
+
|
|
97
|
+
def get_component(
|
|
98
|
+
self,
|
|
99
|
+
start_id: int,
|
|
100
|
+
allowed_nodes: typing.Optional[typing.Set[int]] = None
|
|
101
|
+
) -> typing.Set[int]:
|
|
102
|
+
"""Return all nodes in the connected component of start_id."""
|
|
103
|
+
visited: typing.Set[int] = set()
|
|
104
|
+
queue = collections.deque([start_id])
|
|
105
|
+
visited.add(start_id)
|
|
106
|
+
|
|
107
|
+
while queue:
|
|
108
|
+
curr = queue.popleft()
|
|
109
|
+
for nbr in self.adj.get(curr, ()):
|
|
110
|
+
if nbr not in visited:
|
|
111
|
+
if allowed_nodes is None or nbr in allowed_nodes:
|
|
112
|
+
visited.add(nbr)
|
|
113
|
+
queue.append(nbr)
|
|
114
|
+
|
|
115
|
+
return visited
|
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
"""Links: A High-Dimensional Online Clustering Method."""
|
|
2
|
+
|
|
3
|
+
import collections
|
|
4
|
+
import numpy as np
|
|
5
|
+
import typing
|
|
6
|
+
from linkscluster import graph
|
|
7
|
+
from linkscluster import subcluster
|
|
8
|
+
from linkscluster import utils
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LinksClusterer:
|
|
12
|
+
"""Links online clustering algorithm for high-dimensional unit vectors."""
|
|
13
|
+
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
cluster_similarity_threshold: typing.Optional[float] = None,
|
|
17
|
+
subcluster_similarity_threshold: typing.Optional[float] = None,
|
|
18
|
+
pair_similarity_maximum: typing.Optional[float] = None,
|
|
19
|
+
tc: typing.Optional[float] = None,
|
|
20
|
+
ts: typing.Optional[float] = None,
|
|
21
|
+
tp: typing.Optional[float] = None,
|
|
22
|
+
use_theoretical_norm: bool = False,
|
|
23
|
+
normalize_input: bool = True,
|
|
24
|
+
return_online_labels: bool = True,
|
|
25
|
+
unassigned_label: int = -1):
|
|
26
|
+
"""Initialize the LinksClusterer.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
cluster_similarity_threshold: Tc (cos theta_c), similarity threshold for
|
|
30
|
+
determining whether vectors/subclusters belong to the same cluster.
|
|
31
|
+
Must be in (0, 1). Defaults to 0.5 if neither this nor tc is given.
|
|
32
|
+
subcluster_similarity_threshold: Ts, threshold for grouping vectors into
|
|
33
|
+
the same fine-grained subcluster. Must be in (0, 1) and typically
|
|
34
|
+
Ts >= Tc. Defaults to 0.8 if neither this nor ts is given.
|
|
35
|
+
pair_similarity_maximum: Tp, upper limit for pair similarity taking into
|
|
36
|
+
account intra-subcluster correlations and anisotropy. Must be in
|
|
37
|
+
(Tc^2, 1]. Defaults to 1.0 (isotropic) if not specified.
|
|
38
|
+
tc: alias for cluster_similarity_threshold (Tc in paper).
|
|
39
|
+
ts: alias for subcluster_similarity_threshold (Ts in paper).
|
|
40
|
+
tp: alias for pair_similarity_maximum (Tp in paper).
|
|
41
|
+
use_theoretical_norm: if True, centroid normalization uses Eq. 12
|
|
42
|
+
theoretical norm \\sqrt{k^2 cos^2\\theta_c + k sin^2\\theta_c}; if False
|
|
43
|
+
(recommended for arbitrary empirical distributions), normalizes by
|
|
44
|
+
empirical L2 norm to ensure unit length on S^{N-1}.
|
|
45
|
+
normalize_input: if True, input vectors are L2-normalized to unit length.
|
|
46
|
+
return_online_labels: if True, fit_predict returns the online assigned
|
|
47
|
+
cluster IDs at vector arrival time. If False, returns final cluster IDs
|
|
48
|
+
reflecting all subsequent splits and merges.
|
|
49
|
+
unassigned_label: integer label assigned to out-of-sample vectors in
|
|
50
|
+
predict() that do not meet the cluster membership threshold.
|
|
51
|
+
"""
|
|
52
|
+
effective_tc = tc if tc is not None else cluster_similarity_threshold
|
|
53
|
+
effective_ts = ts if ts is not None else subcluster_similarity_threshold
|
|
54
|
+
effective_tp = tp if tp is not None else pair_similarity_maximum
|
|
55
|
+
|
|
56
|
+
self.tc = 0.5 if effective_tc is None else float(effective_tc)
|
|
57
|
+
self.ts = 0.8 if effective_ts is None else float(effective_ts)
|
|
58
|
+
self.tp = 1.0 if effective_tp is None else float(effective_tp)
|
|
59
|
+
|
|
60
|
+
if not (0.0 < self.tc < 1.0):
|
|
61
|
+
raise ValueError(f"tc must be in (0, 1), got {self.tc}")
|
|
62
|
+
if not (0.0 < self.ts < 1.0):
|
|
63
|
+
raise ValueError(f"ts must be in (0, 1), got {self.ts}")
|
|
64
|
+
|
|
65
|
+
self._tc_sq = self.tc ** 2
|
|
66
|
+
if self.tp < self._tc_sq or self.tp > 1.0:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"tp must be in [{self._tc_sq:.4f}, 1.0], got {self.tp}")
|
|
69
|
+
|
|
70
|
+
# Aliases matching paper notation
|
|
71
|
+
self.cluster_similarity_threshold = self.tc
|
|
72
|
+
self.subcluster_similarity_threshold = self.ts
|
|
73
|
+
self.pair_similarity_maximum = self.tp
|
|
74
|
+
self.Tc = self.tc
|
|
75
|
+
self.Ts = self.ts
|
|
76
|
+
self.Tp = self.tp
|
|
77
|
+
|
|
78
|
+
self.use_theoretical_norm = use_theoretical_norm
|
|
79
|
+
self.normalize_input = normalize_input
|
|
80
|
+
self.return_online_labels = return_online_labels
|
|
81
|
+
self.unassigned_label = unassigned_label
|
|
82
|
+
|
|
83
|
+
self._alpha = (1.0 / self._tc_sq) - 1.0
|
|
84
|
+
if self.tp < 1.0:
|
|
85
|
+
self._beta = (self.tp - self._tc_sq) / (1.0 - self._tc_sq)
|
|
86
|
+
else:
|
|
87
|
+
self._beta = 1.0
|
|
88
|
+
|
|
89
|
+
self.reset()
|
|
90
|
+
|
|
91
|
+
def reset(self):
|
|
92
|
+
"""Reset the clusterer state to initial empty condition."""
|
|
93
|
+
self._subclusters: typing.List[subcluster.Subcluster] = []
|
|
94
|
+
self._subcluster_map: typing.Dict[int, subcluster.Subcluster] = {}
|
|
95
|
+
self._centroids: np.ndarray = np.empty((0, 0), dtype=np.float64)
|
|
96
|
+
self._centroid_capacity: int = 0
|
|
97
|
+
|
|
98
|
+
self._graph = graph.SubclusterGraph()
|
|
99
|
+
self._cluster_subclusters: typing.Dict[int, typing.Set[int]] = (
|
|
100
|
+
collections.defaultdict(set))
|
|
101
|
+
|
|
102
|
+
self._next_sc_id: int = 0
|
|
103
|
+
self._next_cluster_id_val: int = 0
|
|
104
|
+
self.n_samples_seen_: int = 0
|
|
105
|
+
self.n_features_in_: typing.Optional[int] = None
|
|
106
|
+
|
|
107
|
+
self._online_labels: typing.List[int] = []
|
|
108
|
+
self.labels_: typing.Optional[np.ndarray] = None
|
|
109
|
+
self.online_labels_: typing.Optional[np.ndarray] = None
|
|
110
|
+
self.final_labels_: typing.Optional[np.ndarray] = None
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def n_clusters_(self) -> int:
|
|
114
|
+
"""Number of active clusters (connected components in the graph)."""
|
|
115
|
+
return len(self._cluster_subclusters)
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def n_subclusters_(self) -> int:
|
|
119
|
+
"""Number of active subclusters."""
|
|
120
|
+
return len(self._subclusters)
|
|
121
|
+
|
|
122
|
+
@property
|
|
123
|
+
def subclusters_(self) -> typing.List[subcluster.Subcluster]:
|
|
124
|
+
"""List of active subclusters."""
|
|
125
|
+
return list(self._subclusters)
|
|
126
|
+
|
|
127
|
+
def _alloc_cluster_id(self) -> int:
|
|
128
|
+
"""Allocate a new unique integer cluster ID."""
|
|
129
|
+
cid = self._next_cluster_id_val
|
|
130
|
+
self._next_cluster_id_val += 1
|
|
131
|
+
return cid
|
|
132
|
+
|
|
133
|
+
def s(self, k: int, k_prime: int = 1) -> float:
|
|
134
|
+
"""Cosine similarity threshold s(k) or s(k, k') (Eqs. 13 & 16 in paper).
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
k: size of the first subcluster
|
|
138
|
+
k_prime: size of the second subcluster (defaults to 1 for s(k))
|
|
139
|
+
|
|
140
|
+
Returns:
|
|
141
|
+
threshold: float threshold value
|
|
142
|
+
"""
|
|
143
|
+
term1 = 1.0 + self._alpha / k
|
|
144
|
+
term2 = 1.0 + self._alpha / k_prime
|
|
145
|
+
return float(1.0 / np.sqrt(term1 * term2))
|
|
146
|
+
|
|
147
|
+
def s_tilde(self, k: int, k_prime: int = 1) -> float:
|
|
148
|
+
"""Anisotropic threshold \\tilde{s}(k, k') or \\tilde{s}(k) (Eqs. 24 & 25).
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
k: size of the first subcluster
|
|
152
|
+
k_prime: size of the second subcluster (defaults to 1 for \\tilde{s}(k))
|
|
153
|
+
|
|
154
|
+
Returns:
|
|
155
|
+
threshold: float interpolated threshold value
|
|
156
|
+
"""
|
|
157
|
+
raw_s = self.s(k, k_prime)
|
|
158
|
+
if self._beta == 1.0:
|
|
159
|
+
return raw_s
|
|
160
|
+
return float(self._tc_sq + self._beta * (raw_s - self._tc_sq))
|
|
161
|
+
|
|
162
|
+
def _ensure_centroid_capacity(self, needed: int, dim: int):
|
|
163
|
+
"""Ensure contiguous centroid matrix has enough capacity."""
|
|
164
|
+
if self._centroid_capacity == 0:
|
|
165
|
+
self._centroid_capacity = max(32, needed)
|
|
166
|
+
self._centroids = np.zeros(
|
|
167
|
+
(self._centroid_capacity, dim), dtype=np.float64)
|
|
168
|
+
elif needed > self._centroid_capacity:
|
|
169
|
+
new_cap = max(self._centroid_capacity * 2, needed)
|
|
170
|
+
new_centroids = np.zeros((new_cap, dim), dtype=np.float64)
|
|
171
|
+
n_active = len(self._subclusters)
|
|
172
|
+
new_centroids[:n_active] = self._centroids[:n_active]
|
|
173
|
+
self._centroids = new_centroids
|
|
174
|
+
self._centroid_capacity = new_cap
|
|
175
|
+
|
|
176
|
+
def _add_subcluster(
|
|
177
|
+
self,
|
|
178
|
+
vector: np.ndarray,
|
|
179
|
+
vector_idx: int,
|
|
180
|
+
cluster_id: int) -> int:
|
|
181
|
+
"""Create a new subcluster and register it."""
|
|
182
|
+
dim = len(vector)
|
|
183
|
+
idx = len(self._subclusters)
|
|
184
|
+
self._ensure_centroid_capacity(idx + 1, dim)
|
|
185
|
+
|
|
186
|
+
sc_id = self._next_sc_id
|
|
187
|
+
self._next_sc_id += 1
|
|
188
|
+
|
|
189
|
+
sc = subcluster.Subcluster(
|
|
190
|
+
subcluster_id=sc_id,
|
|
191
|
+
cluster_id=cluster_id,
|
|
192
|
+
vector=vector,
|
|
193
|
+
vector_idx=vector_idx,
|
|
194
|
+
tc=self.tc,
|
|
195
|
+
index=idx,
|
|
196
|
+
use_theoretical_norm=self.use_theoretical_norm)
|
|
197
|
+
|
|
198
|
+
self._subclusters.append(sc)
|
|
199
|
+
self._subcluster_map[sc_id] = sc
|
|
200
|
+
self._centroids[idx] = sc.centroid
|
|
201
|
+
self._graph.add_node(sc_id)
|
|
202
|
+
self._cluster_subclusters[cluster_id].add(sc_id)
|
|
203
|
+
return sc_id
|
|
204
|
+
|
|
205
|
+
def _remove_subcluster_by_index(self, idx: int):
|
|
206
|
+
"""Remove a subcluster at idx using swap-and-pop for O(1) removal."""
|
|
207
|
+
last_idx = len(self._subclusters) - 1
|
|
208
|
+
sc_to_remove = self._subclusters[idx]
|
|
209
|
+
sc_id = sc_to_remove.subcluster_id
|
|
210
|
+
|
|
211
|
+
if idx != last_idx:
|
|
212
|
+
last_sc = self._subclusters[last_idx]
|
|
213
|
+
self._subclusters[idx] = last_sc
|
|
214
|
+
last_sc.index = idx
|
|
215
|
+
self._centroids[idx] = self._centroids[last_idx]
|
|
216
|
+
|
|
217
|
+
self._subclusters.pop()
|
|
218
|
+
del self._subcluster_map[sc_id]
|
|
219
|
+
|
|
220
|
+
cid = sc_to_remove.cluster_id
|
|
221
|
+
if cid in self._cluster_subclusters:
|
|
222
|
+
self._cluster_subclusters[cid].discard(sc_id)
|
|
223
|
+
if not self._cluster_subclusters[cid]:
|
|
224
|
+
del self._cluster_subclusters[cid]
|
|
225
|
+
|
|
226
|
+
def predict_next(self, x: np.ndarray) -> int:
|
|
227
|
+
"""Assign cluster ID to a new incoming vector in an online stream.
|
|
228
|
+
|
|
229
|
+
Implements Section 3.3 and Section 3.4 of the paper.
|
|
230
|
+
|
|
231
|
+
Args:
|
|
232
|
+
x: 1D or 2D vector of shape (n_features,) or (1, n_features)
|
|
233
|
+
|
|
234
|
+
Returns:
|
|
235
|
+
cluster_id: integer cluster identifier assigned to vector x
|
|
236
|
+
"""
|
|
237
|
+
x = np.asarray(x, dtype=np.float64).flatten()
|
|
238
|
+
if self.n_features_in_ is None:
|
|
239
|
+
self.n_features_in_ = len(x)
|
|
240
|
+
elif len(x) != self.n_features_in_:
|
|
241
|
+
raise ValueError(
|
|
242
|
+
f"Expected vector of dimension {self.n_features_in_}, got {len(x)}")
|
|
243
|
+
|
|
244
|
+
if self.normalize_input:
|
|
245
|
+
x = utils.l2_normalize(x)
|
|
246
|
+
|
|
247
|
+
t = self.n_samples_seen_
|
|
248
|
+
self.n_samples_seen_ += 1
|
|
249
|
+
|
|
250
|
+
num_subclusters = len(self._subclusters)
|
|
251
|
+
if num_subclusters == 0:
|
|
252
|
+
# First vector starts first subcluster and cluster
|
|
253
|
+
cid = self._alloc_cluster_id()
|
|
254
|
+
self._add_subcluster(x, t, cid)
|
|
255
|
+
self._online_labels.append(cid)
|
|
256
|
+
return cid
|
|
257
|
+
|
|
258
|
+
# Eq. 19: J = argmax_j { x . \\hat{\\mu}_j }
|
|
259
|
+
sims = self._centroids[:num_subclusters] @ x
|
|
260
|
+
J = int(np.argmax(sims))
|
|
261
|
+
x_dot_mu_J = float(sims[J])
|
|
262
|
+
sc_J = self._subclusters[J]
|
|
263
|
+
k_J = sc_J.k
|
|
264
|
+
|
|
265
|
+
# Eq. 20: x . \\hat{\\mu}_J >= Ts
|
|
266
|
+
if x_dot_mu_J >= self.Ts:
|
|
267
|
+
# Inequality 20 holds: add x to subcluster J
|
|
268
|
+
assigned_cid = sc_J.cluster_id
|
|
269
|
+
self._online_labels.append(assigned_cid)
|
|
270
|
+
sc_J.add_vector(x, t)
|
|
271
|
+
self._centroids[J] = sc_J.centroid
|
|
272
|
+
|
|
273
|
+
# Updating clusters (Section 3.4)
|
|
274
|
+
self._update_clusters(sc_J.subcluster_id)
|
|
275
|
+
return assigned_cid
|
|
276
|
+
else:
|
|
277
|
+
# Inequality 20 does not hold: start a new subcluster containing just x.
|
|
278
|
+
# Eq. 21: x . \\hat{\\mu}_J >= s(k_J) (anisotropic: \\tilde{s}(k_J))
|
|
279
|
+
thresh_kJ = self.s_tilde(k_J, 1)
|
|
280
|
+
if x_dot_mu_J >= thresh_kJ:
|
|
281
|
+
# Include new subcluster in same cluster as J, add edge (new_sc, J)
|
|
282
|
+
assigned_cid = sc_J.cluster_id
|
|
283
|
+
new_sc_id = self._add_subcluster(x, t, assigned_cid)
|
|
284
|
+
self._graph.add_edge(new_sc_id, sc_J.subcluster_id)
|
|
285
|
+
else:
|
|
286
|
+
# Start new cluster
|
|
287
|
+
assigned_cid = self._alloc_cluster_id()
|
|
288
|
+
self._add_subcluster(x, t, assigned_cid)
|
|
289
|
+
|
|
290
|
+
self._online_labels.append(assigned_cid)
|
|
291
|
+
return assigned_cid
|
|
292
|
+
|
|
293
|
+
def update(self, x: np.ndarray) -> int:
|
|
294
|
+
"""Alias for predict_next."""
|
|
295
|
+
return self.predict_next(x)
|
|
296
|
+
|
|
297
|
+
def _update_clusters(self, start_sc_id: int):
|
|
298
|
+
"""Perform subcluster merging and edge validity checks (Section 3.4).
|
|
299
|
+
|
|
300
|
+
Args:
|
|
301
|
+
start_sc_id: identifier of the updated subcluster i
|
|
302
|
+
"""
|
|
303
|
+
# 1. Subcluster merging:
|
|
304
|
+
# "If this brings it within the subcluster similarity threshold of the
|
|
305
|
+
# centroid of another subcluster currently joined to the first by an edge,
|
|
306
|
+
# then the two are merged. In other words, if \\hat{\\mu}_i . \\hat{\\mu}_j
|
|
307
|
+
# >= Ts, then nodes i and j are replaced with a single node containing the
|
|
308
|
+
# vectors of both, and with the edge connections of both. Since the merging
|
|
309
|
+
# process also results in a new subcluster centroid, this check is
|
|
310
|
+
# continued recursively on affected subclusters."
|
|
311
|
+
i = start_sc_id
|
|
312
|
+
while i in self._subcluster_map:
|
|
313
|
+
sc_i = self._subcluster_map[i]
|
|
314
|
+
merged_any = False
|
|
315
|
+
nbr_ids = list(self._graph.neighbors(i))
|
|
316
|
+
|
|
317
|
+
for j in nbr_ids:
|
|
318
|
+
if j not in self._subcluster_map:
|
|
319
|
+
continue
|
|
320
|
+
sc_j = self._subcluster_map[j]
|
|
321
|
+
# \\hat{\\mu}_i . \\hat{\\mu}_j >= Ts
|
|
322
|
+
mu_i_dot_mu_j = float(np.dot(sc_i.mu_hat, sc_j.mu_hat))
|
|
323
|
+
if mu_i_dot_mu_j >= self.Ts:
|
|
324
|
+
# Merge node j into node i
|
|
325
|
+
sc_i.merge_with(sc_j)
|
|
326
|
+
self._centroids[sc_i.index] = sc_i.centroid
|
|
327
|
+
self._graph.merge_nodes(keep_id=i, remove_id=j)
|
|
328
|
+
self._remove_subcluster_by_index(sc_j.index)
|
|
329
|
+
merged_any = True
|
|
330
|
+
break # Check continued recursively on affected subcluster i
|
|
331
|
+
|
|
332
|
+
if not merged_any:
|
|
333
|
+
break
|
|
334
|
+
|
|
335
|
+
if i not in self._subcluster_map:
|
|
336
|
+
return
|
|
337
|
+
|
|
338
|
+
# 2. Edge validity check:
|
|
339
|
+
# "Next, the edges joining affected nodes are checked for validity.
|
|
340
|
+
# The edge joining subclusters i and j is removed if the following does
|
|
341
|
+
# not continue to hold:
|
|
342
|
+
# \\hat{\\mu}_i . \\hat{\\mu}_j >= s(k_i, k_j) (Eq. 22)"
|
|
343
|
+
sc_i = self._subcluster_map[i]
|
|
344
|
+
nbr_ids = list(self._graph.neighbors(i))
|
|
345
|
+
|
|
346
|
+
for j in nbr_ids:
|
|
347
|
+
if not self._graph.has_edge(i, j):
|
|
348
|
+
continue
|
|
349
|
+
if j not in self._subcluster_map:
|
|
350
|
+
continue
|
|
351
|
+
sc_j = self._subcluster_map[j]
|
|
352
|
+
k_i = sc_i.k
|
|
353
|
+
k_j = sc_j.k
|
|
354
|
+
mu_i_dot_mu_j = float(np.dot(sc_i.mu_hat, sc_j.mu_hat))
|
|
355
|
+
thresh_ij = self.s_tilde(k_i, k_j)
|
|
356
|
+
|
|
357
|
+
if mu_i_dot_mu_j < thresh_ij:
|
|
358
|
+
# Edge (i, j) removed
|
|
359
|
+
self._graph.remove_edge(i, j)
|
|
360
|
+
cluster_nodes = self._cluster_subclusters.get(sc_i.cluster_id, set())
|
|
361
|
+
severed, comp_j = self._graph.check_severed(
|
|
362
|
+
start_id=j, target_id=i, allowed_nodes=cluster_nodes)
|
|
363
|
+
|
|
364
|
+
if severed:
|
|
365
|
+
# "After severing a cluster in two by removing an edge, an attempt is
|
|
366
|
+
# made to re-join the two parts by adding an edge from the affected
|
|
367
|
+
# node to a new partner node that does satisfy inequality 22."
|
|
368
|
+
best_partner = None
|
|
369
|
+
best_partner_sim = -2.0
|
|
370
|
+
for w in comp_j:
|
|
371
|
+
sc_w = self._subcluster_map[w]
|
|
372
|
+
k_w = sc_w.k
|
|
373
|
+
mu_i_dot_mu_w = float(np.dot(sc_i.mu_hat, sc_w.mu_hat))
|
|
374
|
+
thresh_iw = self.s_tilde(k_i, k_w)
|
|
375
|
+
if mu_i_dot_mu_w >= thresh_iw and mu_i_dot_mu_w > best_partner_sim:
|
|
376
|
+
best_partner = w
|
|
377
|
+
best_partner_sim = mu_i_dot_mu_w
|
|
378
|
+
|
|
379
|
+
if best_partner is not None:
|
|
380
|
+
# Re-join parts by adding edge (i, best_partner)
|
|
381
|
+
self._graph.add_edge(i, best_partner)
|
|
382
|
+
else:
|
|
383
|
+
# "If no such partner is found, then the cluster remains permanently
|
|
384
|
+
# split."
|
|
385
|
+
new_cid = self._alloc_cluster_id()
|
|
386
|
+
old_cid = sc_i.cluster_id
|
|
387
|
+
for w in comp_j:
|
|
388
|
+
sc_w = self._subcluster_map[w]
|
|
389
|
+
self._cluster_subclusters[old_cid].discard(w)
|
|
390
|
+
self._cluster_subclusters[new_cid].add(w)
|
|
391
|
+
sc_w.cluster_id = new_cid
|
|
392
|
+
if not self._cluster_subclusters[old_cid]:
|
|
393
|
+
del self._cluster_subclusters[old_cid]
|
|
394
|
+
|
|
395
|
+
def partial_fit(self, X: np.ndarray, y=None) -> "LinksClusterer":
|
|
396
|
+
"""Incrementally cluster a batch of vectors.
|
|
397
|
+
|
|
398
|
+
Args:
|
|
399
|
+
X: numpy array of shape (n_samples, n_features) or (n_features,)
|
|
400
|
+
y: ignored
|
|
401
|
+
|
|
402
|
+
Returns:
|
|
403
|
+
self
|
|
404
|
+
"""
|
|
405
|
+
X = np.asarray(X, dtype=np.float64)
|
|
406
|
+
if X.ndim == 1:
|
|
407
|
+
X = np.expand_dims(X, 0)
|
|
408
|
+
for vec in X:
|
|
409
|
+
self.predict_next(vec)
|
|
410
|
+
return self
|
|
411
|
+
|
|
412
|
+
def predict_stream(
|
|
413
|
+
self,
|
|
414
|
+
stream: typing.Iterable[np.ndarray]) -> typing.Iterator[int]:
|
|
415
|
+
"""Stream vectors one by one and yield predicted cluster IDs."""
|
|
416
|
+
for vec in stream:
|
|
417
|
+
yield self.predict_next(vec)
|
|
418
|
+
|
|
419
|
+
def fit(self, X: np.ndarray, y=None) -> "LinksClusterer":
|
|
420
|
+
"""Fit the Links clusterer on a dataset.
|
|
421
|
+
|
|
422
|
+
Args:
|
|
423
|
+
X: numpy array of shape (n_samples, n_features)
|
|
424
|
+
y: ignored
|
|
425
|
+
|
|
426
|
+
Returns:
|
|
427
|
+
self
|
|
428
|
+
"""
|
|
429
|
+
self.fit_predict(X)
|
|
430
|
+
return self
|
|
431
|
+
|
|
432
|
+
def fit_predict(self, X: np.ndarray, y=None) -> np.ndarray:
|
|
433
|
+
"""Fit the clusterer and return cluster labels for X.
|
|
434
|
+
|
|
435
|
+
Args:
|
|
436
|
+
X: numpy array of shape (n_samples, n_features)
|
|
437
|
+
y: ignored
|
|
438
|
+
|
|
439
|
+
Returns:
|
|
440
|
+
labels: numpy array of shape (n_samples,)
|
|
441
|
+
"""
|
|
442
|
+
self.reset()
|
|
443
|
+
X = np.asarray(X, dtype=np.float64)
|
|
444
|
+
if X.ndim != 2:
|
|
445
|
+
raise ValueError(
|
|
446
|
+
f"X must be 2D array of shape (n_samples, n_features), got {X.shape}")
|
|
447
|
+
if X.shape[0] == 0:
|
|
448
|
+
self.labels_ = np.empty((0,), dtype=int)
|
|
449
|
+
self.online_labels_ = np.empty((0,), dtype=int)
|
|
450
|
+
self.final_labels_ = np.empty((0,), dtype=int)
|
|
451
|
+
return self.labels_
|
|
452
|
+
|
|
453
|
+
for vec in X:
|
|
454
|
+
self.predict_next(vec)
|
|
455
|
+
|
|
456
|
+
self.online_labels_ = np.array(self._online_labels, dtype=int)
|
|
457
|
+
self.final_labels_ = self.get_final_labels()
|
|
458
|
+
|
|
459
|
+
if self.return_online_labels:
|
|
460
|
+
self.labels_ = self.online_labels_.copy()
|
|
461
|
+
else:
|
|
462
|
+
self.labels_ = self.final_labels_.copy()
|
|
463
|
+
|
|
464
|
+
return self.labels_
|
|
465
|
+
|
|
466
|
+
def predict(self, X: np.ndarray) -> np.ndarray:
|
|
467
|
+
"""Predict cluster IDs for X.
|
|
468
|
+
|
|
469
|
+
If the model has not been fitted, calls fit_predict(X). If already fitted,
|
|
470
|
+
assigns each vector to its most similar subcluster's cluster without
|
|
471
|
+
modifying clusterer state.
|
|
472
|
+
|
|
473
|
+
Args:
|
|
474
|
+
X: numpy array of shape (n_samples, n_features) or (n_features,)
|
|
475
|
+
|
|
476
|
+
Returns:
|
|
477
|
+
labels: numpy array of shape (n_samples,)
|
|
478
|
+
"""
|
|
479
|
+
if self.n_samples_seen_ == 0:
|
|
480
|
+
return self.fit_predict(X)
|
|
481
|
+
|
|
482
|
+
X = np.asarray(X, dtype=np.float64)
|
|
483
|
+
if X.ndim == 1:
|
|
484
|
+
X = np.expand_dims(X, 0)
|
|
485
|
+
if self.normalize_input:
|
|
486
|
+
X = utils.l2_normalize(X)
|
|
487
|
+
|
|
488
|
+
num_subclusters = len(self._subclusters)
|
|
489
|
+
if num_subclusters == 0:
|
|
490
|
+
return np.full(X.shape[0], self.unassigned_label, dtype=int)
|
|
491
|
+
|
|
492
|
+
sims = X @ self._centroids[:num_subclusters].T
|
|
493
|
+
best_j = np.argmax(sims, axis=1)
|
|
494
|
+
preds = np.empty(X.shape[0], dtype=int)
|
|
495
|
+
|
|
496
|
+
for idx in range(X.shape[0]):
|
|
497
|
+
j = int(best_j[idx])
|
|
498
|
+
sim = float(sims[idx, j])
|
|
499
|
+
sc = self._subclusters[j]
|
|
500
|
+
thresh = self.s_tilde(sc.k, 1)
|
|
501
|
+
if sim >= thresh or self.unassigned_label is None:
|
|
502
|
+
preds[idx] = sc.cluster_id
|
|
503
|
+
else:
|
|
504
|
+
preds[idx] = self.unassigned_label
|
|
505
|
+
|
|
506
|
+
return preds
|
|
507
|
+
|
|
508
|
+
def get_online_labels(self) -> np.ndarray:
|
|
509
|
+
"""Return the online cluster IDs assigned at vector arrival times."""
|
|
510
|
+
return np.array(self._online_labels, dtype=int)
|
|
511
|
+
|
|
512
|
+
def get_final_labels(self) -> np.ndarray:
|
|
513
|
+
"""Return post-hoc cluster IDs reflecting all subsequent splits and merges.
|
|
514
|
+
"""
|
|
515
|
+
labels = np.zeros(self.n_samples_seen_, dtype=int)
|
|
516
|
+
for sc in self._subclusters:
|
|
517
|
+
for v_idx in sc.vector_indices:
|
|
518
|
+
labels[v_idx] = sc.cluster_id
|
|
519
|
+
return labels
|