vectorphile 0.1.0__py3-none-any.whl

vectorphile/ml/cluster.py

@@ -0,0 +1,218 @@
+"""Clustering algorithms."""
+import numpy as np
+from typing import Optional
+
+
+class KMeans:
+    def __init__(self, n_clusters=8, max_iter=300, tol=1e-4, n_init=10, random_state=None):
+        self.n_clusters = n_clusters
+        self.max_iter = max_iter
+        self.tol = tol
+        self.n_init = n_init
+        self.random_state = random_state
+        self.cluster_centers_: Optional[np.ndarray] = None
+        self.labels_: Optional[np.ndarray] = None
+        self.inertia_: float = np.inf
+
+    def _run_once(self, X, seed):
+        rng = np.random.RandomState(seed)
+        idx = rng.choice(len(X), self.n_clusters, replace=False)
+        centers = X[idx].copy()
+        for _ in range(self.max_iter):
+            dists = np.linalg.norm(X[:, None] - centers[None], axis=2)
+            labels = dists.argmin(axis=1)
+            new_centers = np.array([X[labels == k].mean(axis=0) if (labels == k).any() else centers[k]
+                                    for k in range(self.n_clusters)])
+            if np.max(np.abs(new_centers - centers)) < self.tol:
+                break
+            centers = new_centers
+        inertia = sum(np.sum((X[labels == k] - centers[k])**2) for k in range(self.n_clusters))
+        return centers, labels, float(inertia)
+
+    def fit(self, X) -> 'KMeans':
+        X = np.array(X, dtype=float)
+        if self.random_state is not None:
+            np.random.seed(self.random_state)
+        best_inertia = np.inf
+        for i in range(self.n_init):
+            centers, labels, inertia = self._run_once(X, seed=i)
+            if inertia < best_inertia:
+                best_inertia = inertia
+                self.cluster_centers_ = centers
+                self.labels_ = labels
+                self.inertia_ = inertia
+        return self
+
+    def predict(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float)
+        dists = np.linalg.norm(X[:, None] - self.cluster_centers_[None], axis=2)
+        return dists.argmin(axis=1)
+
+    def fit_predict(self, X) -> np.ndarray:
+        self.fit(X)
+        return self.labels_
+
+    def score(self, X) -> float:
+        return -self.inertia_
+
+
+class DBSCAN:
+    def __init__(self, eps=0.5, min_samples=5, metric='euclidean'):
+        self.eps = eps
+        self.min_samples = min_samples
+        self.metric = metric
+        self.labels_: Optional[np.ndarray] = None
+        self.core_sample_indices_: Optional[np.ndarray] = None
+
+    def fit(self, X) -> 'DBSCAN':
+        X = np.array(X, dtype=float)
+        n = len(X)
+        dists = np.linalg.norm(X[:, None] - X[None], axis=2)
+        labels = np.full(n, -1, dtype=int)
+        visited = np.zeros(n, dtype=bool)
+        cluster_id = 0
+        neighbors = [np.where(dists[i] <= self.eps)[0] for i in range(n)]
+        core = np.array([i for i in range(n) if len(neighbors[i]) >= self.min_samples])
+        self.core_sample_indices_ = core
+
+        def expand(idx, cid):
+            queue = list(neighbors[idx])
+            while queue:
+                q = queue.pop()
+                if not visited[q]:
+                    visited[q] = True
+                    if len(neighbors[q]) >= self.min_samples:
+                        for nb in neighbors[q]:
+                            if not visited[nb]:
+                                queue.append(nb)
+                if labels[q] == -1:
+                    labels[q] = cid
+
+        for i in core:
+            if not visited[i]:
+                visited[i] = True
+                labels[i] = cluster_id
+                expand(i, cluster_id)
+                cluster_id += 1
+        self.labels_ = labels
+        return self
+
+    def fit_predict(self, X) -> np.ndarray:
+        self.fit(X)
+        return self.labels_
+
+
+class AgglomerativeClustering:
+    def __init__(self, n_clusters=2, linkage='ward'):
+        self.n_clusters = n_clusters
+        self.linkage = linkage
+        self.labels_: Optional[np.ndarray] = None
+
+    def fit(self, X) -> 'AgglomerativeClustering':
+        X = np.array(X, dtype=float)
+        n = len(X)
+        clusters = [{i} for i in range(n)]
+
+        def dist(c1, c2):
+            pts1 = X[list(c1)]
+            pts2 = X[list(c2)]
+            if self.linkage == 'ward':
+                merged = np.vstack([pts1, pts2])
+                return len(merged) * np.sum((merged.mean(0) - np.vstack([
+                    np.tile(pts1.mean(0), (len(pts1),1)),
+                    np.tile(pts2.mean(0), (len(pts2),1))
+                ]))**2)
+            d = np.linalg.norm(pts1[:, None] - pts2[None], axis=2)
+            if self.linkage == 'single':
+                return d.min()
+            if self.linkage == 'complete':
+                return d.max()
+            return d.mean()
+
+        while len(clusters) > self.n_clusters:
+            best_d, merge_i, merge_j = np.inf, 0, 1
+            for i in range(len(clusters)):
+                for j in range(i+1, len(clusters)):
+                    d = dist(clusters[i], clusters[j])
+                    if d < best_d:
+                        best_d, merge_i, merge_j = d, i, j
+            clusters[merge_i] = clusters[merge_i] | clusters[merge_j]
+            clusters.pop(merge_j)
+
+        self.labels_ = np.zeros(n, dtype=int)
+        for label, cluster in enumerate(clusters):
+            for idx in cluster:
+                self.labels_[idx] = label
+        return self
+
+    def fit_predict(self, X) -> np.ndarray:
+        self.fit(X)
+        return self.labels_
+
+
+class GaussianMixture:
+    def __init__(self, n_components=1, max_iter=100, tol=1e-3, random_state=None, covariance_type='full'):
+        self.n_components = n_components
+        self.max_iter = max_iter
+        self.tol = tol
+        self.random_state = random_state
+        self.covariance_type = covariance_type
+        self.weights_: Optional[np.ndarray] = None
+        self.means_: Optional[np.ndarray] = None
+        self.covariances_: Optional[np.ndarray] = None
+
+    def _gaussian_pdf(self, X, mean, cov):
+        d = X.shape[1]
+        diff = X - mean
+        try:
+            cov_inv = np.linalg.inv(cov + np.eye(d) * 1e-6)
+            cov_det = np.linalg.det(cov + np.eye(d) * 1e-6)
+        except np.linalg.LinAlgError:
+            cov_inv = np.eye(d)
+            cov_det = 1.0
+        exponent = -0.5 * np.einsum('ni,ij,nj->n', diff, cov_inv, diff)
+        return np.exp(exponent) / (np.sqrt((2*np.pi)**d * np.abs(cov_det)) + 1e-300)
+
+    def fit(self, X) -> 'GaussianMixture':
+        if self.random_state is not None:
+            np.random.seed(self.random_state)
+        X = np.array(X, dtype=float)
+        n, d = X.shape
+        K = self.n_components
+        idx = np.random.choice(n, K, replace=False)
+        self.means_ = X[idx].copy()
+        self.covariances_ = np.array([np.eye(d) for _ in range(K)])
+        self.weights_ = np.ones(K) / K
+        for _ in range(self.max_iter):
+            # E-step
+            resp = np.array([self.weights_[k] * self._gaussian_pdf(X, self.means_[k], self.covariances_[k]) for k in range(K)]).T
+            resp_sum = resp.sum(axis=1, keepdims=True)
+            resp /= (resp_sum + 1e-300)
+            # M-step
+            Nk = resp.sum(axis=0)
+            new_means = (resp.T @ X) / (Nk[:, None] + 1e-300)
+            new_covs = []
+            for k in range(K):
+                diff = X - new_means[k]
+                new_covs.append((resp[:, k:k+1] * diff).T @ diff / (Nk[k] + 1e-300))
+            old_means = self.means_.copy()
+            self.means_ = new_means
+            self.covariances_ = np.array(new_covs)
+            self.weights_ = Nk / n
+            if np.max(np.abs(self.means_ - old_means)) < self.tol:
+                break
+        return self
+
+    def predict(self, X) -> np.ndarray:
+        return self.predict_proba(X).argmax(axis=1)
+
+    def predict_proba(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float)
+        K = self.n_components
+        resp = np.array([self.weights_[k] * self._gaussian_pdf(X, self.means_[k], self.covariances_[k]) for k in range(K)]).T
+        resp_sum = resp.sum(axis=1, keepdims=True)
+        return resp / (resp_sum + 1e-300)
+
+    def fit_predict(self, X) -> np.ndarray:
+        self.fit(X)
+        return self.predict(X)

vectorphile/ml/decomposition.py

@@ -0,0 +1,161 @@
+"""Dimensionality reduction."""
+import numpy as np
+from typing import Optional
+
+
+class PCA:
+    def __init__(self, n_components=None, whiten=False, random_state=None):
+        self.n_components = n_components
+        self.whiten = whiten
+        self.random_state = random_state
+        self.components_: Optional[np.ndarray] = None
+        self.explained_variance_: Optional[np.ndarray] = None
+        self.explained_variance_ratio_: Optional[np.ndarray] = None
+        self.mean_: Optional[np.ndarray] = None
+        self.singular_values_: Optional[np.ndarray] = None
+
+    def fit(self, X) -> 'PCA':
+        X = np.array(X, dtype=float)
+        self.mean_ = X.mean(axis=0)
+        Xc = X - self.mean_
+        U, S, Vt = np.linalg.svd(Xc, full_matrices=False)
+        n = len(X)
+        k = self.n_components or Vt.shape[0]
+        self.components_ = Vt[:k]
+        self.singular_values_ = S[:k]
+        self.explained_variance_ = S[:k]**2 / (n - 1)
+        self.explained_variance_ratio_ = self.explained_variance_ / (S**2 / (n - 1)).sum()
+        return self
+
+    def transform(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float)
+        Xc = X - self.mean_
+        result = Xc @ self.components_.T
+        if self.whiten:
+            result /= np.sqrt(self.explained_variance_) + 1e-12
+        return result
+
+    def fit_transform(self, X) -> np.ndarray:
+        self.fit(X)
+        return self.transform(X)
+
+    def inverse_transform(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float)
+        if self.whiten:
+            X = X * np.sqrt(self.explained_variance_)
+        return X @ self.components_ + self.mean_
+
+
+class TruncatedSVD:
+    def __init__(self, n_components=2, random_state=None):
+        self.n_components = n_components
+        self.random_state = random_state
+        self.components_: Optional[np.ndarray] = None
+        self.explained_variance_: Optional[np.ndarray] = None
+        self.explained_variance_ratio_: Optional[np.ndarray] = None
+        self.singular_values_: Optional[np.ndarray] = None
+
+    def fit(self, X) -> 'TruncatedSVD':
+        X = np.array(X, dtype=float)
+        U, S, Vt = np.linalg.svd(X, full_matrices=False)
+        k = self.n_components
+        self.components_ = Vt[:k]
+        self.singular_values_ = S[:k]
+        total = (S**2).sum()
+        self.explained_variance_ = S[:k]**2 / len(X)
+        self.explained_variance_ratio_ = S[:k]**2 / (total + 1e-12)
+        return self
+
+    def transform(self, X) -> np.ndarray:
+        return np.array(X, dtype=float) @ self.components_.T
+
+    def fit_transform(self, X) -> np.ndarray:
+        self.fit(X)
+        return self.transform(X)
+
+
+class NMF:
+    def __init__(self, n_components=2, max_iter=200, tol=1e-4, random_state=None):
+        self.n_components = n_components
+        self.max_iter = max_iter
+        self.tol = tol
+        self.random_state = random_state
+        self.components_: Optional[np.ndarray] = None
+        self.reconstruction_err_: float = np.inf
+
+    def fit_transform(self, X) -> np.ndarray:
+        if self.random_state is not None:
+            np.random.seed(self.random_state)
+        X = np.array(X, dtype=float)
+        X = np.maximum(X, 0)
+        n, m = X.shape
+        k = self.n_components
+        W = np.random.rand(n, k) + 0.1
+        H = np.random.rand(k, m) + 0.1
+        for _ in range(self.max_iter):
+            WH = W @ H + 1e-10
+            H *= (W.T @ X) / (W.T @ WH)
+            WH = W @ H + 1e-10
+            W *= (X @ H.T) / (WH @ H.T)
+        self.components_ = H
+        self.reconstruction_err_ = float(np.linalg.norm(X - W @ H))
+        return W
+
+    def fit(self, X) -> 'NMF':
+        self.fit_transform(X)
+        return self
+
+    def transform(self, X) -> np.ndarray:
+        X = np.maximum(np.array(X, dtype=float), 0)
+        H = self.components_
+        W, _ = np.linalg.lstsq(H.T, X.T, rcond=None)
+        return np.maximum(W.T, 0)
+
+
+class FastICA:
+    def __init__(self, n_components=None, max_iter=200, tol=1e-4, fun='logcosh', random_state=None):
+        self.n_components = n_components
+        self.max_iter = max_iter
+        self.tol = tol
+        self.fun = fun
+        self.random_state = random_state
+        self.components_: Optional[np.ndarray] = None
+        self.mean_: Optional[np.ndarray] = None
+
+    def _g(self, x):
+        if self.fun == 'logcosh':
+            return np.tanh(x), (1 - np.tanh(x)**2).mean(axis=-1)
+        return x * np.exp(-x**2 / 2), (1 - x**2) * np.exp(-x**2 / 2).mean(axis=-1)
+
+    def fit_transform(self, X) -> np.ndarray:
+        if self.random_state is not None:
+            np.random.seed(self.random_state)
+        X = np.array(X, dtype=float)
+        n, p = X.shape
+        self.mean_ = X.mean(axis=0)
+        X = X - self.mean_
+        _, S, Vt = np.linalg.svd(X, full_matrices=False)
+        k = self.n_components or min(n, p)
+        X_white = (X @ Vt[:k].T) / (S[:k] / np.sqrt(n) + 1e-10)
+        W = np.random.randn(k, k)
+        for i in range(self.max_iter):
+            WX = W @ X_white.T
+            g, g_prime = self._g(WX)
+            W_new = (g @ X_white) / n - g_prime[:, None] * W
+            # Orthogonalization
+            u, _, vt = np.linalg.svd(W_new)
+            W_new = u @ vt
+            if np.max(np.abs(np.abs(np.diag(W_new @ W.T)) - 1)) < self.tol:
+                W = W_new
+                break
+            W = W_new
+        self.components_ = W @ np.diag(1.0 / (S[:k] / np.sqrt(n) + 1e-10)) @ Vt[:k]
+        return X_white @ W.T
+
+    def fit(self, X) -> 'FastICA':
+        self.fit_transform(X)
+        return self
+
+    def transform(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float) - self.mean_
+        return X @ self.components_.T

vectorphile/ml/ensemble.py

@@ -0,0 +1,209 @@
+"""Ensemble methods: Random Forest, Gradient Boosting."""
+import numpy as np
+from typing import Optional
+from vectorphile.ml.tree import DecisionTreeClassifier, DecisionTreeRegressor
+
+
+class RandomForestClassifier:
+    def __init__(self, n_estimators=100, max_depth=None, min_samples_split=2,
+                 min_samples_leaf=1, max_features='sqrt', bootstrap=True, random_state=None):
+        self.n_estimators = n_estimators
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.min_samples_leaf = min_samples_leaf
+        self.max_features = max_features
+        self.bootstrap = bootstrap
+        self.random_state = random_state
+        self.estimators_ = []
+        self.classes_: Optional[np.ndarray] = None
+
+    def fit(self, X, y) -> 'RandomForestClassifier':
+        if self.random_state is not None:
+            np.random.seed(self.random_state)
+        X, y = np.array(X, dtype=float), np.array(y)
+        self.classes_ = np.unique(y)
+        n = len(X)
+        self.estimators_ = []
+        for _ in range(self.n_estimators):
+            if self.bootstrap:
+                idx = np.random.choice(n, n, replace=True)
+                Xb, yb = X[idx], y[idx]
+            else:
+                Xb, yb = X, y
+            tree = DecisionTreeClassifier(
+                max_depth=self.max_depth,
+                min_samples_split=self.min_samples_split,
+                min_samples_leaf=self.min_samples_leaf,
+                max_features=self.max_features,
+            )
+            tree.fit(Xb, yb)
+            self.estimators_.append(tree)
+        return self
+
+    def predict_proba(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float)
+        all_proba = np.array([t.predict_proba(X) for t in self.estimators_])
+        return all_proba.mean(axis=0)
+
+    def predict(self, X) -> np.ndarray:
+        proba = self.predict_proba(X)
+        return self.classes_[np.argmax(proba, axis=1)]
+
+    def score(self, X, y) -> float:
+        return float(np.mean(self.predict(X) == np.array(y)))
+
+
+class RandomForestRegressor:
+    def __init__(self, n_estimators=100, max_depth=None, min_samples_split=2,
+                 min_samples_leaf=1, max_features='sqrt', bootstrap=True, random_state=None):
+        self.n_estimators = n_estimators
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.min_samples_leaf = min_samples_leaf
+        self.max_features = max_features
+        self.bootstrap = bootstrap
+        self.random_state = random_state
+        self.estimators_ = []
+
+    def fit(self, X, y) -> 'RandomForestRegressor':
+        if self.random_state is not None:
+            np.random.seed(self.random_state)
+        X, y = np.array(X, dtype=float), np.array(y, dtype=float)
+        n = len(X)
+        self.estimators_ = []
+        for _ in range(self.n_estimators):
+            if self.bootstrap:
+                idx = np.random.choice(n, n, replace=True)
+                Xb, yb = X[idx], y[idx]
+            else:
+                Xb, yb = X, y
+            tree = DecisionTreeRegressor(
+                max_depth=self.max_depth,
+                min_samples_split=self.min_samples_split,
+                min_samples_leaf=self.min_samples_leaf,
+                max_features=self.max_features,
+            )
+            tree.fit(Xb, yb)
+            self.estimators_.append(tree)
+        return self
+
+    def predict(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float)
+        preds = np.array([t.predict(X) for t in self.estimators_])
+        return preds.mean(axis=0)
+
+    def score(self, X, y) -> float:
+        y = np.array(y, dtype=float)
+        pred = self.predict(X)
+        ss_res = np.sum((y - pred)**2)
+        ss_tot = np.sum((y - y.mean())**2)
+        return float(1 - ss_res / (ss_tot + 1e-12))
+
+
+class GradientBoostingRegressor:
+    def __init__(self, n_estimators=100, learning_rate=0.1, max_depth=3,
+                 min_samples_split=2, subsample=1.0, loss='mse', random_state=None):
+        self.n_estimators = n_estimators
+        self.learning_rate = learning_rate
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.subsample = subsample
+        self.loss = loss
+        self.random_state = random_state
+        self.estimators_ = []
+        self.init_pred_: float = 0.0
+
+    def fit(self, X, y) -> 'GradientBoostingRegressor':
+        if self.random_state is not None:
+            np.random.seed(self.random_state)
+        X, y = np.array(X, dtype=float), np.array(y, dtype=float)
+        self.init_pred_ = float(y.mean())
+        F = np.full(len(y), self.init_pred_)
+        self.estimators_ = []
+        n = len(X)
+        for _ in range(self.n_estimators):
+            residuals = y - F
+            if self.subsample < 1.0:
+                idx = np.random.choice(n, int(n * self.subsample), replace=False)
+                Xs, rs = X[idx], residuals[idx]
+            else:
+                Xs, rs = X, residuals
+            tree = DecisionTreeRegressor(max_depth=self.max_depth, min_samples_split=self.min_samples_split)
+            tree.fit(Xs, rs)
+            update = tree.predict(X)
+            F += self.learning_rate * update
+            self.estimators_.append(tree)
+        return self
+
+    def predict(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float)
+        F = np.full(len(X), self.init_pred_)
+        for tree in self.estimators_:
+            F += self.learning_rate * tree.predict(X)
+        return F
+
+    def score(self, X, y) -> float:
+        y = np.array(y, dtype=float)
+        pred = self.predict(X)
+        ss_res = np.sum((y - pred)**2)
+        ss_tot = np.sum((y - y.mean())**2)
+        return float(1 - ss_res / (ss_tot + 1e-12))
+
+
+class GradientBoostingClassifier:
+    def __init__(self, n_estimators=100, learning_rate=0.1, max_depth=3,
+                 min_samples_split=2, subsample=1.0, random_state=None):
+        self.n_estimators = n_estimators
+        self.learning_rate = learning_rate
+        self.max_depth = max_depth
+        self.min_samples_split = min_samples_split
+        self.subsample = subsample
+        self.random_state = random_state
+        self.estimators_ = []
+        self.classes_: Optional[np.ndarray] = None
+        self.init_pred_: Optional[np.ndarray] = None
+
+    def _softmax(self, z):
+        z = z - z.max(axis=1, keepdims=True)
+        e = np.exp(z)
+        return e / e.sum(axis=1, keepdims=True)
+
+    def fit(self, X, y) -> 'GradientBoostingClassifier':
+        if self.random_state is not None:
+            np.random.seed(self.random_state)
+        X, y = np.array(X, dtype=float), np.array(y)
+        self.classes_ = np.unique(y)
+        K = len(self.classes_)
+        n = len(X)
+        y_ohe = np.zeros((n, K))
+        for i, c in enumerate(self.classes_):
+            y_ohe[:, i] = (y == c).astype(float)
+        F = np.zeros((n, K))
+        self.init_pred_ = np.log(y_ohe.mean(axis=0) + 1e-12)
+        F += self.init_pred_
+        self.estimators_ = []
+        for _ in range(self.n_estimators):
+            probs = self._softmax(F)
+            self.estimators_.append([])
+            for k in range(K):
+                residuals = y_ohe[:, k] - probs[:, k]
+                tree = DecisionTreeRegressor(max_depth=self.max_depth, min_samples_split=self.min_samples_split)
+                tree.fit(X, residuals)
+                F[:, k] += self.learning_rate * tree.predict(X)
+                self.estimators_[-1].append(tree)
+        return self
+
+    def predict_proba(self, X) -> np.ndarray:
+        X = np.array(X, dtype=float)
+        F = np.tile(self.init_pred_, (len(X), 1)).copy()
+        for group in self.estimators_:
+            for k, tree in enumerate(group):
+                F[:, k] += self.learning_rate * tree.predict(X)
+        return self._softmax(F)
+
+    def predict(self, X) -> np.ndarray:
+        proba = self.predict_proba(X)
+        return self.classes_[np.argmax(proba, axis=1)]
+
+    def score(self, X, y) -> float:
+        return float(np.mean(self.predict(X) == np.array(y)))