phyloK 1.0.0__py3-none-any.whl

phyloK/LRMF.py

@@ -0,0 +1,171 @@
+import numpy as np
+
+def check_convergence(x_old, x_new, threshold=1e-6):
+    max_updt = np.linalg.norm(x_new - x_old, ord = np.inf)
+    w_max    = np.linalg.norm(x_new, ord = np.inf)
+    if w_max == 0.0 or max_updt/w_max <= threshold:
+        return True
+    
+    return False
+
+def get_Options(X, nan_value = 0):
+    if np.isnan(nan_value):
+        Oi,Oj = np.where(~np.isnan(X))
+    else:
+        Oi,Oj = np.where(X != nan_value)
+
+    return np.c_[Oi, Oj]
+
+def init_AB(k, shape, rng = np.random.default_rng(12038)):
+    a,b = shape
+
+    A = rng.normal(size = (a,k))
+    B = rng.normal(size = (k,b))
+    return A, B
+
+def ALS_legacy(sample_train, sample_test, 
+                k = 6, lam1 = 0.1, lam2 = 0.1, 
+                get_errors = True, epsilon = 1e-6,
+                max_epochs = 1000, 
+                rng = np.random.default_rng(12038)):
+
+    Oij_test = get_Options(sample_test)
+    Oij = get_Options(sample_train)
+
+    M_test = sample_test[Oij_test[:,0], Oij_test[:,1]]
+
+    Ik  = np.eye(k)
+    A,B = init_AB(k, sample_train.shape, rng = rng)
+    
+    all_errors = []
+    for _ in range(max_epochs):
+        Ac = np.copy(A)
+        Bc = np.copy(B)
+        # row and columns
+        # update
+        for i,j in Oij:
+            # i,j
+            Ok_ = Oij[ Oij[:,0] == i ]
+            Mkj = sample_train[ Ok_[:,0], Ok_[:,1] ]
+            Bj = B[:, Ok_[:,1] ]
+
+            # matrix form of
+            # update rule of A[i:]
+            A[i,:] =  np.linalg.solve(Bj @ Bj.T + lam1*Ik, Bj @ Mkj.T).T
+
+            O_k = Oij[ Oij[:,1] == j ]
+            Mik = sample_train[ O_k[:,0], O_k[:,1] ]    
+            Ai = A[ O_k[:,0],:]
+
+            # matrix form of
+            # update rule of B[:k]
+            B[:,j] = np.linalg.solve( Ai.T @ Ai + lam2*Ik, Ai.T @ Mik )
+
+        if get_errors:
+            # matrix form of
+            # error evaluation
+            AiBj = (A @ B)[Oij_test[:,0], Oij_test[:,1]]
+            # rmse's
+            myrmse = np.sqrt( np.mean( (M_test - AiBj)**2 ) )
+            # print("current RMSE: ", myrmse)
+            all_errors.append( myrmse)
+
+        if check_convergence(Ac, A, epsilon) and\
+           check_convergence(Bc, B, epsilon):
+            break
+    
+    AiBj = (A @ B)[ Oij_test[:,0], Oij_test[:,1] ]
+    final_error = np.sqrt( np.mean( (M_test - AiBj)**2 ) )
+    
+    return A, B, all_errors, final_error
+
+
+def ALS(M_train, 
+        k = 6, lam1 = 0.1, lam2 = 0.1, 
+        epsilon = 1e-6,
+        max_epochs = 1000, 
+        rng = np.random.default_rng(12038),
+        nan_value = 0):
+
+
+    Oij = get_Options(M_train, nan_value = nan_value)
+    Ik  = np.eye(k)
+    A,B = init_AB(k, M_train.shape, rng = rng)
+    
+    for _ in range(max_epochs):
+        Ac = np.copy(A)
+        Bc = np.copy(B)
+        # row and columns
+        # update
+        for i,j in Oij:
+            # i,j
+            Ok_ = Oij[ Oij[:,0] == i ]
+            Mkj = M_train[ Ok_[:,0], Ok_[:,1] ]
+            Bj = B[:, Ok_[:,1] ]
+
+            # matrix form of
+            # update rule of A[i:]
+            A[i,:] =  np.linalg.solve(Bj @ Bj.T + lam1*Ik, Bj @ Mkj.T).T
+
+            O_k = Oij[ Oij[:,1] == j ]
+            Mik = M_train[ O_k[:,0], O_k[:,1] ]    
+            Ai = A[ O_k[:,0],:]
+
+            # matrix form of
+            # update rule of B[:k]
+            B[:,j] = np.linalg.solve( Ai.T @ Ai + lam2*Ik, Ai.T @ Mik )
+
+        if check_convergence(Ac, A, epsilon) and\
+           check_convergence(Bc, B, epsilon):
+            break
+    
+    return A, B
+
+def score(A, B, M_test, nan_value = 0):
+    Oij_test = get_Options(M_test, nan_value = nan_value)
+    M_test = M_test[Oij_test[:,0], Oij_test[:,1]]
+    AiBj = (A @ B)[Oij_test[:,0], Oij_test[:,1]]
+    final_error = np.sqrt( np.mean( (M_test - AiBj)**2 ) )
+    return final_error
+
+
+if __name__ == "__main__":
+
+    train_file = "../data/rate_train.csv"
+    test_file  = "../data/rate_test.csv"
+
+    rng = np.random.default_rng(12038)
+
+    sample_train = np.loadtxt(train_file, delimiter=',')
+    sample_test  = np.loadtxt(test_file , delimiter=',')
+
+    lam1 = 4
+    lam2 = 4
+    k = 5
+
+    (A,B) = ALS(sample_train,
+                k = k, lam1 = lam1, lam2 = lam2, 
+                max_epochs = 100, epsilon = 1e-3, 
+                rng = rng)
+
+    final_error = score(A, B, sample_test)
+    print("Final RMSE: ", final_error) # Final RMSE:  0.8042507213383732
+
+
+    
+    # (A,B,_,_) = ALS_legacy(sample_train, sample_test, 
+    #                     k = k, lam1 = lam1, lam2 = lam2, 
+    #                     max_epochs = 100, epsilon = 1e-3, 
+    #                     get_errors = False,rng = rng,)
+
+
+    # import matplotlib.pyplot as plt
+    # plt.plot(all_errors)
+    # plt.xlabel('Epoch')
+    # plt.ylabel('RMSE')
+    # plt.title('Convergence of ALS')
+    # plt.show()
+
+
+    # with open("./completed_matrix.csv", "w") as f:
+    #     np.savetxt(f, A @ B, delimiter=",", fmt='%1.4f')

phyloK/RBF.py

@@ -0,0 +1,203 @@
+import numpy as np
+
+
+def distance_matrix(a, b, M=None):
+    """
+    l2 norm squared matrix
+
+    Time complexity:
+    If M is full matrix, then O(np^2 + n^2p). Assuming p < n, then O(n^2p).
+    If M is a diagonal or vector, then O(n^2p) regardless of the relationship between p and n.
+    If M is None, then O(n^2p) regardless of the relationship between p and n.
+
+    a: shape (m, p)
+    b: shape (n, p)
+    M: shape (p, p) or None
+    
+    returns: shape (m, n)
+    """
+    # a = X_test[:2, :]
+    # b = X_train[:3, :]
+    # # M = np.array([[2, 0], 
+    # #               [0, 1]])
+    # M = np.diag(M)
+
+    # a = X_train
+    # b = X_test
+    # M = np.eye(a.shape[1])
+    if M is None:
+        # O(pn^2) for the whole matrix
+        return np.linalg.norm(a[:, None, :] - b[None, :, :], axis=-1)**2
+    
+    else:
+        # total time:
+        # O(np + np + mnp + mn) = O(mnp) = O(n^2p) if m = n
+        if len(M.shape) == 1:
+            # O(np)
+            A = ((a[ :, None, :] * M) @ a[:, :, None]).reshape(a.shape[0])
+            # O(mp)
+            B = ((b[ :, None, :] * M) @ b[:, :, None]).reshape(b.shape[0])
+            # O(mpn + pn) = O(mnp)
+            C = (a * M) @ b.T
+            # O(mn)
+            return np.add.outer(A,B) - 2*C
+
+
+        else:
+            # total time:
+            # O(mn + mp^2 + np^2 + mpn + p^2n) 
+            #  = O((m + n)p^2 + mpn)  
+            # if m = n:
+            # O(np^2 + n^2p) = O(npp + nnp) <= O(n^2p) if p < n
+
+            # O(np^2)
+            A = (a[ :, None, :] @ M @ a[:, :, None]).reshape(a.shape[0])
+            # O(mp^2)
+            B = (b[ :, None, :] @ M @ b[:, :, None]).reshape(b.shape[0])
+            # O(mpn + p^2n)
+            C = a @ M @ b.T
+            # O(mn)
+            return np.add.outer(A,B) - 2*C
+
+
+def nan_sqdist(a, b, use_t=False, T=None, k = 3):
+    r"""
+    Assuming a and b have 0 where it is missing.
+    
+    a**2 @ Mbf.T = [ a_1^2    [  |       |
+                      ...       0/1 ... 0/1 
+                     a_m^2 ]     |       | ]
+                                 b1      bn <- presence/absence in b
+                 = [ || a^{1}_1 ||^2  ...   || a^{n}_1 ||^2
+                         
+                     || a^{1}_m ||^2  ...   || a^{n}_m ||^2 ]
+    In the sum, this activates l2 norm for overlapped features.
+    The same applies to (b**2 @ Maf.T)
+
+    a @ b.T = [ a_1^T b_1  ...  a_1^T b_n
+                ...        ...      ...
+                a_m^T b_1  ...  a_m^T b_n ]
+
+    thus a**2 @ Mbf.T + (b**2 @ Maf.T).T - 2 * (a @ b.T):
+
+    = [ ||a^{1}_1||^2 -2a_1^T b_1 +||b^{1}_1||^2  ...  ||a^{n}_1||^2 -2a_1^T b_n +||b^{1}_n||^2
+                                                  ...
+        ||a^{1}_m||^2 -2a_m^T b_1 +||b^{m}_1||^2  ...  ||a^{n}_m||^2 -2a_m^T b_n +||b^{m}_n||^2 ]
+    
+
+    O(n^2p) time complexity.
+    """
+    p = a.shape[1]
+
+    Ma = ~np.isnan(a) # observed masks, shape (m, p), O(mp)
+    Mb = ~np.isnan(b) # shape (n, p), O(np)
+    a0 = np.where(Ma, a, 0.0)  # shape (m, p) zero-filled data, O(mp)
+    b0 = np.where(Mb, b, 0.0)  # shape (n, p) zero-filled data, O(np)
+
+    Maf = Ma.astype(a.dtype)
+    Mbf = Mb.astype(b.dtype)
+
+    n = Maf @ Mbf.T    # overlap counts, (m, n), O(mpn)
+    # R^{m,p} R^{p,n} + (R^{n,p} R^{p,m}).T - 2 R^{m,p} R^{p,n} = R^{m,n}
+    # O(mpn + mpn + mnp) = O(mnp) = O(n^2p) if m = n
+    S = (a0**2) @ Mbf.T + ((b0**2) @ Maf.T).T - 2 * (a0 @ b0.T) 
+
+    with np.errstate(divide="ignore", invalid="ignore"):
+        D = S * (p / n)        # n == 0 -> inf or nan
+
+    D[n == 0] = np.nan
+
+    if sum(n == 0) > 0:
+        k_linear_interpolation(D, k=k, T=T, use_t=use_t)
+
+    return D
+
+def k_linear_interpolation(D, k = 3, T = None, use_t = False):
+    r"""
+    D: distance matrix with NaN values, shape (m, n)
+    k: number of nearest neighbors to use for interpolation
+    T: distance matrix with NaN values, shape (m, n)
+    use_t: if True, use T for finding nearest neighbors instead of D    
+
+
+    O(n^2) algorithm 
+    The algorithm iterates over each row of the distance matrix D, 
+    and for each row, it finds the k nearest neighbors that have non-NaN values. 
+    It then replaces the NaN values in that row with the mean of the k 
+    nearest neighbors' distances.
+    """
+    # k = 2
+    assert k > 0, "k must be a positive integer"
+
+    if use_t:
+        assert T is not None, "T must be provided when use_t is True"
+
+    m,n = D.shape
+    # O(n^2) for the whole matrix
+    for i in range(m):
+        # i = 0
+
+        full_idx = ~np.isnan(D[i,:]) # O(n)
+        if sum(full_idx) == n:
+            continue
+
+        # available neighbors
+        full_idx[i] = False
+
+        if sum(full_idx) == 0:
+            # this means that there exist a single x_ik value at the row i
+            # and there there no more values at column k.
+            raise ValueError("All neighbors are NaN for row {}".format(i))
+
+        if sum(full_idx) <= k:
+            k_neighs_idx = full_idx
+        else:
+            # Get the k nearest neighbors, O(n)
+            if use_t:
+                k_neighs_idx = np.argpartition(T[i, full_idx], k)[:k]
+
+            else:
+                k_neighs_idx = np.argpartition(D[i, full_idx], k)[:k]
+
+        # index with full values
+        full_idx[i] = True
+
+        D[i, ~full_idx] = np.mean(D[i, k_neighs_idx])
+
+def RBF_kernel(a, b, gamma, M = None, nan_adjust = False, k = 3, T = None, use_t = False):
+    """
+    Radial Basis Function
+
+    Time complexity:
+    If M is full matrix, then O(np^2 + n^2p). Assuming p < n, then O(n^2p).
+    If M is a diagonal or vector, then O(n^2p) regardless of the relationship between p and n.
+    If M is None, then O(n^2p) regardless of the relationship between p and n.
+
+
+    a: shape (m, p)\\
+    b: shape (n, p)\\
+    M: shape (p, p) or None\\
+    nan_adjust: if True, then use nan_sqdist to compute the distance matrix, otherwise use distance_matrix\\
+    k: number of nearest neighbors to use for interpolation if nan_adjust is True\\
+    T: distance matrix with NaN values, shape (m, n)\\
+    use_t: if True, use T for finding nearest neighbors instead of D\\
+
+
+    returns: shape (m, n)
+    """         
+    a = np.ascontiguousarray(a, dtype=np.float64)
+    b = np.ascontiguousarray(b, dtype=np.float64)
+    # data sim
+    if nan_adjust:
+        # O(n^2p) for the whole matrix and never uses M.
+        D = nan_sqdist(a, b, use_t=use_t, T=T, k=k)
+
+    else:
+        D = distance_matrix(a, b, M=M)
+    # print("D.shape", D.shape)
+    # print("D", D)
+    D = np.where(D < 1e-10, 0.0, D) # numerical stability
+    tmp_rbf = -gamma * D
+
+    np.exp(tmp_rbf, tmp_rbf) # RBF kernel. Inplace exponentiation
+    return tmp_rbf

phyloK/agop.py

@@ -0,0 +1,240 @@
+import numpy as np
+
+
+def nabla_kw(l, K, X, M, gamma, w, diag = False):
+    r"""
+    Derivative of the kernel matrix with respect to the input data X.
+    \partial k_{\ell, j} w_j / \partial x_{\ell} 
+
+    total time complexity: 
+    O(pnm + mp^2) = O( mp(n + p) ) if diag = False
+    O(pnm)                         O\W
+    """
+
+    # w = W * Du[l,:]
+
+    Dl = (X[l,:] - X).T #O(pn), R^{p,n}
+    Dl_Kl = Dl * K[l,:] #O(pn), R^{p,n}
+    Dl_kl_w = Dl_Kl @ w #O(pnm), R^{p,n} x R^{n,m} = R^{p,m}
+
+    if M is None:
+        return -2. * gamma * Dl_kl_w # O(pm), R^{p,m}
+    
+    else:
+        if not diag:
+            return -2. * gamma * M @ Dl_kl_w # O(mp^2), R^{p,m}
+        else:
+            assert len(M.shape) == 1, "M should be a vector when diag is True"
+            b = -2. * gamma * M[:,None] * Dl_kl_w # O(pm), R^{p,m}
+            # b = -2. * gamma * M * Dl_kl_w # O(pm), R^{p,m} # needed it for obj agop
+            # print("b ", b)
+            return b
+
+def agop(Du, mu, Vu, y, W, lam, X, K, M, gamma, diag = False):
+    """
+    Outer product gradient for recursive feature machines
+
+    Algorithm exact time complexity: 
+    m be the number of outputs, 
+    n be the number of samples, and 
+    p be the number of features.
+
+    If diag = False:
+    O(nmp^2 + nmp(n + p)) = O(nmp(n + p)).  Assuming p < n, then O(mpn^2).
+    
+    
+    If diag = True:
+    O(n^2mp). regardless of the relationship between p and n.
+
+    Du: Derivative of the estimator, shape (n, m)
+    mu: Estimator, shape (n, m)
+    Vu: Variance of the estimator, shape (n, m)
+    y: observed labels, shape (n, m)
+    W: weights, shape (n, m)
+    lam: regularization parameter
+    X: input data, shape (n, p)
+    K: kernel matrix, shape (n, n)
+    M: prev agop matrix, shape (p, p) or shape (p,) if diag = True or None if no previous agop matrix is provided
+    gamma: rbf kernel parameter
+    diag: if True, then M is a vector and the agop matrix is computed as
+          M_new = np.sum(nabla_f_l**2, axis=0) / n
+          otherwise, M is a matrix and the agop matrix is computed as
+          M_new = np.sum(np.outer(nabla_f_l, nabla_f_l), axis=0) / n
+        
+
+    returns: M_new, shape (p, p)
+
+    reference:
+        Supplemntary material of the paper "Mechanism for feature learning in neural networks and backpropagation-free machine learning models"
+    """
+    
+    # mu = lkrr.mu(self.f_curr)
+    # Vu = lkrr.Vu_func(self.f_curr)
+    
+    # lam = lkrr.lam
+    # M = lkrr.M
+    # gamma = lkrr.gamma
+    
+    # K = lkrr.K_train
+    # X = X_train
+    # W = lkrr.alpha[1:]
+    # Du = lkrr.Du
+    # y = lkrr.y
+    # diag = True
+
+    mn = y.shape
+    d1_array = False
+    if len(mn) == 1:
+        n,m = (mn[0], 1)
+        d1_array = True
+
+    else:
+        n,m = mn
+
+    p = X.shape[1]
+
+    if d1_array:
+        W  = W[ :, np.newaxis]
+        Du = Du[:, np.newaxis]
+
+
+    G = np.zeros((p,m,n))
+    # O( nmp(n + p) ) or O( n^2mp ) if diag
+    for l in range(n):
+        # l = 0
+        # O( mp(n + p) ) or O( mnp ) if diag
+        G[:,:,l] = nabla_kw(l, K, X, M, gamma,  W * Du[l,:], diag=diag)
+
+
+    G -= G.mean(axis = 2, keepdims = True)
+
+    if not diag:
+        M_new = np.zeros((p,p), dtype=np.float32)
+        # O(nmp^2)
+        for l in range(n):
+            # O(pm)
+            nabla_f_l = G[:,:,l] # R^{p,m}
+            # O(mp^2)
+            # M_new += np.outer(nabla_f_l, nabla_f_l)
+            M_new += nabla_f_l @ nabla_f_l.T
+    else:
+        M_new = np.zeros(p, dtype=np.float32)
+        # O(nmp)
+        for l in range(n):
+            # O(mp)
+            nabla_f_l = G[:,:,l]
+            # O(mp)
+            M_new += np.sum(nabla_f_l**2, axis = 1)
+
+    return M_new / n
+
+
+def obj_agop(Du, mu, Vu, y, W, lam, X, K, M, gamma, diag = False):
+    """
+    Average gradient outer product from the objective function
+
+    Algorithm exact time complexity: 
+    m be the number of outputs, 
+    n be the number of samples, and 
+    p be the number of features.
+
+    If diag = False:
+    O(nmp^2 + pmn^2 + np^2) = O(mnp^2 + mpn^2) = O(mnp(p + n)).  Assuming p < n, then O(mpn^2).
+    
+    
+    If diag = True:
+    O(nm + mpn^2 + np) = O(mpn^2) regardless of the relationship between p and n.
+
+    Du: Derivative of the estimator, shape (n, m)
+    mu: Estimator, shape (n, m)
+    Vu: Variance of the estimator, shape (n, m)
+    y: observed labels, shape (n, m)
+    W: weights, shape (n, m)
+    lam: regularization parameter
+    X: input data, shape (n, p)
+    K: kernel matrix, shape (n, n)
+    M: prev agop matrix, shape (p, p) or shape (p,) if diag = True or None if no previous agop matrix is provided
+    gamma: rbf kernel parameter
+    diag: if True, then M is a vector and the agop matrix is computed as
+          M_new = np.sum(nabla_f_l**2, axis=0) / n
+          otherwise, M is a matrix and the agop matrix is computed as
+          M_new = np.sum(np.outer(nabla_f_l, nabla_f_l), axis=0) / n
+        
+
+    returns: M_new, shape (p, p)
+
+    reference:
+        Supplemntary material of the paper "Mechanism for feature learning in neural networks and backpropagation-free machine learning models"
+    """
+    # f_curr = lkrr.f_curr
+    # Du = lkrr.Du
+    # mu = self.mu(self.f_curr)
+    # Vu = lkrr.Vu
+    # W = lkrr.alpha[1:]
+    # lam = lkrr.lam
+    # M = lkrr.M
+    # gamma = lkrr.gamma
+    # X = X_train
+    # K = lkrr.K_train
+    # self = lkrr
+    # y = lkrr.y
+    # diag = True
+    # y =  np.random.normal(size = (10,1))
+
+    mn = y.shape
+    d1_array = False
+    if len(mn) == 1:
+        n,m = (mn[0], 1)
+        d1_array = True
+
+    else:
+        n,m = mn
+
+    # Du, mu = self.get_Du_mu_internal(f_curr) # O(nm)
+    # Vu = self.get_Vu_func(f_curr) # O(nm)
+    # yb = y.reshape(n, m, order='F') # O(nm)
+    e = y - mu # O(nm)
+    e /= Vu
+
+    if d1_array:
+        e  = e[ :, np.newaxis]
+        W  = W[ :, np.newaxis]
+        Du = Du[:, np.newaxis]
+
+    c1 = m*n
+    # O(nmp^2 + pmn^2) or O(mpn^2) if diag = True
+    G = np.zeros((n, X.shape[1]))
+    for l in range(n):
+        nabla_f_l = np.zeros(X.shape[1])
+        I_0 = np.ones(n)
+        I_0[l] = 0
+        # O(p^2m + pnm) or O(pnm) if diag = True
+        for j in range(m):
+            w1 = (-1./c1) * ( (W[:,j]*Du[l,j]*e[l,j]) + (W[l,j]*I_0*Du[:,j]*e[:,j]) )   # O(n)
+            w2 = (lam/2.) * W[l,j] * ( W[:,j] + I_0 * W[:,j] ) # O(n)
+            # O(p^2 + pn) or O(pn) if diag = True
+            tmp = nabla_kw(l, K, X, M, gamma,  w1 + w2, diag=diag)
+            # print("tmp: ",tmp)
+            nabla_f_l += tmp
+
+        G[l,:] = nabla_f_l
+
+    G -= G.mean(axis=0)
+    if not diag:
+        M_new = np.zeros((X.shape[1], X.shape[1]), dtype=np.float32)
+        # O(np^2)
+        for l in range(n):
+            # O(p)
+            nabla_f_l = G[l,:]
+            # O(p^2)
+            M_new += np.outer(nabla_f_l, nabla_f_l)
+    else:
+        M_new = np.zeros(X.shape[1], dtype=np.float32)
+        # O(np)
+        for l in range(n):
+            # O(p)
+            nabla_f_l = G[l,:]
+            # O(p)
+            M_new += nabla_f_l**2
+
+    return M_new / n