icol 0.1.6__py3-none-any.whl → 0.7.4__py3-none-any.whl

icol/icol.py

@@ -8,9 +8,10 @@ from itertools import combinations, permutations
 import numpy as np
 import sympy as sp
 
-from sklearn.linear_model import lars_path
+from sklearn.linear_model import lars_path, Ridge, Lars
 from sklearn.preprocessing import PolynomialFeatures
 from sklearn.base import clone
+from sklearn.model_selection import train_test_split
 
 from sklearn.metrics import mean_squared_error
 
@@ -21,10 +22,135 @@ def LL(res):
     n = len(res)
     return n*np.log(np.sum(res**2)/n)
 
+def initialize_ols(D, y, init_idx):
+    """
+    Fit initial OLS solution on selected columns of D.
+    
+    Parameters
+    ----------
+    D : (n, d) ndarray
+        Full dictionary matrix.
+    y : (n,) ndarray
+        Response vector.
+    init_idx : list[int]
+        Indices of columns from D to use initially.
+    
+    Returns
+    -------
+    beta : (p,) ndarray
+        OLS coefficients for selected columns.
+    A_inv : (p, p) ndarray
+        Inverse Gram matrix for selected columns.
+    XT : (p, n) ndarray
+        Transposed design matrix of selected columns.
+    active_idx : list[int]
+        Current indices of D included in the model.
+    """
+    X = D[:, init_idx]
+    A = X.T @ X
+    try: 
+        A_inv = np.linalg.inv(A)
+    except np.linalg.LinAlgError:
+        A_inv = np.linalg.pinv(A)
+    beta = A_inv @ (X.T @ y)
+    XT = X.T
+    return beta, A_inv, XT, list(init_idx)
+
+def sweep_update_from_D(beta, A_inv, XT, active_idx, D, y, new_idx):
+    # Generated with ChatGPT using the commands;
+    # 1. write me a function which takes in an n by p dimension matrix X, for which we already have an OLS solution, beta.
+    #  Additionally, a second input is a data matrix Z with n rows and q columns. 
+    # Add the Z matrix of columns to the OLS solution using SWEEP
+    # 2. Are we also able to efficiently update the gram and its inverse with this procedure for X augmented with Z
+    # 3. Ok, imagine that I need to update my SWEEP solution multiple times.
+    #  Adjust the inputs and return values so that everything can be used again in the next SWEEP update.
+    #  Then update the function to make use of these previous computations
+    # 4. Lets make some changes for the sake of indexing. Imagine that we have a large matrix D, with d columns.
+    # Through some selection procedure we select p of those columns to form an initial OLS solution.
+    # We then iteratively select p new columns and incorporate those into the ols solution using sweep. 
+    # Update the code to reflect this change while also tracking the indices of columns in the original D matrix 
+    # and their mapping to the respective betas.
+
+    """
+    Update OLS solution by adding new columns from D.
+    
+    Parameters
+    ----------
+    beta : (p,) ndarray
+        Current OLS coefficients.
+    A_inv : (p, p) ndarray
+        Inverse Gram matrix for current features.
+    XT : (p, n) ndarray
+        Transposed design matrix for current features.
+    active_idx : list[int]
+        Current indices of columns in D that are in the model.
+    D : (n, d) ndarray
+        Full dictionary matrix.
+    y : (n,) ndarray
+        Response vector.
+    new_idx : list[int]
+        Indices of new columns in D to add.
+    
+    Returns
+    -------
+    beta_new : (p+q,) ndarray
+        Updated OLS coefficients.
+    A_tilde_inv : (p+q, p+q) ndarray
+        Updated inverse Gram matrix.
+    XT_new : (p+q, n) ndarray
+        Updated design matrix transpose.
+    active_idx_new : list[int]
+        Updated indices of active columns in D.
+    """
+    p = beta.shape[0]
+    Z = D[:, new_idx]    # n x q
+    q = Z.shape[1]
+
+    # Cross products
+    B = XT @ Z                # p x q
+    C = Z.T @ Z               # q x q
+    yZ = Z.T @ y              # q x 1
+
+    # Schur complement
+    S = C - B.T @ (A_inv @ B)
+
+    # Solve for new coefficients (numerically stable)
+    rhs = yZ - B.T @ beta
+    try:
+        beta_Z = np.linalg.solve(S, rhs)
+    except np.linalg.LinAlgError:
+        beta_Z = np.linalg.pinv(S) @ rhs
+
+    # Update old coefficients
+    beta_X_new = beta - A_inv @ (B @ beta_Z)
+    beta_new = np.concatenate([beta_X_new, beta_Z])
+
+    # Update Gram inverse
+    try: 
+        S_inv = np.linalg.inv(S)  # small q x q
+    except np.linalg.LinAlgError:
+        S_inv = np.linalg.pinv(S)
+
+    top_left = A_inv + A_inv @ B @ S_inv @ B.T @ A_inv
+    top_right = -A_inv @ B @ S_inv
+    bottom_left = -S_inv @ B.T @ A_inv
+    bottom_right = S_inv
+
+    A_tilde_inv = np.block([
+        [top_left, top_right],
+        [bottom_left, bottom_right]
+    ])
+
+    # Update XT and active indices
+    XT_new = np.vstack([XT, Z.T])
+    active_idx_new = active_idx + list(new_idx)
+
+    return beta_new, A_tilde_inv, XT_new, active_idx_new
+
 IC_DICT = {
     'AIC': lambda res, k: LL(res) + 2*k,
     'HQIC': lambda res, k: LL(res) + np.log(np.log(len(res)))*k,
-    'AIC': lambda res, k: LL(res) + 2*k,
+    'BIC': lambda res, k, n: LL(res) + 2*k*np.log(n),
     'CAIC': lambda res, k: LL(res) + (np.log(len(res))+1)*k,
     'AICc': lambda res, k: LL(res) + 2*k + 2*k*(k+1)/(len(res)-k-1)
 }
@@ -168,7 +294,7 @@ class PolynomialFeaturesICL:
     
     def get_feature_names_out(self):
         return self.PolynomialFeatures.get_feature_names_out()
-    
+
 class BSS:
     def __init__(self):
         pass
@@ -232,13 +358,97 @@ class BSS:
         beta_ret = np.zeros(p)
         beta_ret[list(best_comb)] = beta.reshape(1, -1)
         return beta_ret
-                    
+
+class EfficientAdaptiveLASSO:
+    def __init__(self, gamma=1, fit_intercept=False, default_d=5, rcond=-1, alpha=0):
+        self.gamma = gamma
+        self.fit_intercept = fit_intercept
+        self.default_d = default_d
+        self.rcond=rcond
+        self.alpha=alpha
+        self.A_inv = None
+        self.XT = None
+        self.beta_ols = None
+        self.active_idx = None
+
+    def __str__(self):
+        return ('EffAda' if self.gamma != 0 else '') + ('LASSO') + ('(gamma={0})'.format(self.gamma) if self.gamma != 0 else '')
+    
+    def __repr__(self):
+        return self.__str__()
+    
+    def get_params(self, deep=False):
+        return {'gamma': self.gamma,
+                'fit_intercept': self.fit_intercept,
+                'default_d': self.default_d,
+                'rcond': self.rcond}
+    
+    def set_default_d(self, d):
+        self.default_d = d
+
+    def __call__(self, X, y, d, idx_old = None, idx_new=None, verbose=False):
+
+        self.set_default_d(d)
+        nonancols = np.isnan(X).sum(axis=0)==0
+        noinfcols = np.isinf(X).sum(axis=0)==0
+        valcols = np.logical_and(nonancols, noinfcols)
+        idx_ala = list(idx_new) + list(idx_old)
+
+        if np.abs(self.gamma)<1e-10:
+            beta_ols = np.ones(X.shape[1])
+            w_hat = np.ones(X.shape[1])
+            X_star_star = X.copy()
+        else:
+            X_valcols = X[:, valcols]
+            if not idx_old:
+                self.beta_ols, self.A_inv, self.XT, self.active_idx = initialize_ols(X_valcols, y, init_idx=idx_new)
+            else:
+                self.beta_ols, self.A_inv, self.XT, self.active_idx = sweep_update_from_D(beta = self.beta_ols, A_inv=self.A_inv,
+                                                                                          XT=self.XT, active_idx=self.active_idx, D=X, y=y, 
+                                                                                          new_idx=idx_new)
+
+            w_hat = 1/np.power(np.abs(self.beta_ols), self.gamma)
+            X_star_star = np.zeros_like(X_valcols[:, idx_ala])
+            for j in range(X_star_star.shape[1]): # vectorise
+                X_j = X_valcols[:, j]/w_hat[j]
+                X_star_star[:, j] = X_j
+
+        _, _, coefs, _ = lars_path(X_star_star, y.ravel(), return_n_iter=True, max_iter=d, method='lasso')
+        # alphas, active, coefs = lars_path(X_star_star, y.ravel(), method='lasso')
+        try:           
+            beta_hat_star_star = coefs[:, d]
+        except IndexError: # in the event that a solution with d components cant be found, use the next largest. 
+            beta_hat_star_star = coefs[:, -1]
+
+        beta_hat_star_n_old_new = np.array([beta_hat_star_star[j]/w_hat[j] for j in range(len(beta_hat_star_star))])
+#        beta_hat_star_n = np.zeros(X.shape[1])
+#        beta_hat_star_n[idx_ala] = beta_hat_star_n_old_new
+
+#        beta_hat_star_n[valcols] = beta_hat_star_n_valcol
+#        ret = beta_hat_star_n.reshape(1, -1).squeeze()
+        return beta_hat_star_n_old_new.squeeze()
+    
+    def fit(self, X, y, verbose=False):
+        self.mu = y.mean() if self.fit_intercept else 0            
+        beta = self.__call__(X=X, y=y-self.mu, d=self.default_d, verbose=verbose)
+        self.beta = beta.reshape(-1, 1)
+
+    def predict(self, X):
+        return np.dot(X, self.beta) + self.mu
+    
+    def s_max(self, k, n, p, c1=1, c0=0):
+        if self.gamma==0:
+            return c1*(p/(k**2)) + c0
+        else:
+            return c1*min(np.power(p, 1/2)/k, np.power(p*n, 1/3)/k) + c0
+
 class AdaptiveLASSO:
-    def __init__(self, gamma=1, fit_intercept=False, default_d=5, rcond=-1):
+    def __init__(self, gamma=1, fit_intercept=False, default_d=5, rcond=-1, alpha=0):
         self.gamma = gamma
         self.fit_intercept = fit_intercept
         self.default_d = default_d
         self.rcond=rcond
+        self.alpha=0
 
     def __str__(self):
         return ('Ada' if self.gamma != 0 else '') + ('LASSO') + ('(gamma={0})'.format(self.gamma) if self.gamma != 0 else '')
@@ -255,21 +465,26 @@ class AdaptiveLASSO:
     def set_default_d(self, d):
         self.default_d = d
 
-    def __call__(self, X, y, d, rcond=None, verbose=False):
+    def __call__(self, X, y, d, verbose=False):
 
         self.set_default_d(d)
 
+        nonancols = np.isnan(X).sum(axis=0)==0
+        noinfcols = np.isinf(X).sum(axis=0)==0
+        valcols = np.logical_and(nonancols, noinfcols)
         if np.abs(self.gamma)<1e-10:
             beta_hat = np.ones(X.shape[1])
             w_hat = np.ones(X.shape[1])
             X_star_star = X.copy()
         else:
-            beta_hat, _, _, _ = np.linalg.lstsq(X, y, rcond=self.rcond)
+
+            X_valcols = X[:, valcols]
+            beta_hat, _, _, _ = np.linalg.lstsq(X_valcols, y, rcond=self.rcond)
 
             w_hat = 1/np.power(np.abs(beta_hat), self.gamma)
-            X_star_star = np.zeros_like(X)
+            X_star_star = np.zeros_like(X_valcols)
             for j in range(X_star_star.shape[1]): # vectorise
-                X_j = X[:, j]/w_hat[j]
+                X_j = X_valcols[:, j]/w_hat[j]
                 X_star_star[:, j] = X_j
 
         _, _, coefs, _ = lars_path(X_star_star, y.ravel(), return_n_iter=True, max_iter=d, method='lasso')
@@ -278,7 +493,10 @@ class AdaptiveLASSO:
             beta_hat_star_star = coefs[:, d]
         except IndexError:
             beta_hat_star_star = coefs[:, -1]
-        beta_hat_star_n = np.array([beta_hat_star_star[j]/w_hat[j] for j in range(len(beta_hat_star_star))])
+
+        beta_hat_star_n_valcol = np.array([beta_hat_star_star[j]/w_hat[j] for j in range(len(beta_hat_star_star))])
+        beta_hat_star_n = np.zeros(X.shape[1])
+        beta_hat_star_n[valcols] = beta_hat_star_n_valcol
         return beta_hat_star_n.reshape(1, -1).squeeze()
     
     def fit(self, X, y, verbose=False):
@@ -295,6 +513,27 @@ class AdaptiveLASSO:
         else:
             return c1*min(np.power(p, 1/2)/k, np.power(p*n, 1/3)/k) + c0
 
+class LARS:
+    def __init__(self, default_d=None):
+        self.default_d=default_d
+    
+    def __repr__(self):
+        return 'Lars'
+
+    def __str__(self):
+        return 'Lars'
+
+    def set_default_d(self, default_d):
+        self.default_d = default_d
+
+    def get_params(self, deep=False):
+        return {'default_d': self.default_d}
+
+    def __call__(self, X, y, d, verbose=False):
+        self.lars = Lars(fit_intercept=False, fit_path=False, verbose=verbose, n_nonzero_coefs=d, copy_X=True)
+        self.lars.fit(X, y)
+        return self.lars.coef_
+
 class ThresholdedLeastSquares:
     def __init__(self, default_d=None):
         self.default_d=default_d
@@ -368,38 +607,38 @@ class SIS:
         return best_corr, best_idxs
 
 class ICL:
-    def __init__(self, s, so, d, fit_intercept=True, normalize=True, pool_reset=False, information_criteria=None): #, track_intermediates=False):
+    def __init__(self, s, so, k, fit_intercept=True, normalize=True, pool_reset=False, optimize_k=False, track_intermediates=False):
         self.s = s
         self.sis = SIS(n_sis=s)
         self.so = so
-        self.d = d
+        self.k = k
         self.fit_intercept = fit_intercept
         self.normalize=normalize
         self.pool_reset = pool_reset
-        self.information_criteria = information_criteria if information_criteria in IC_DICT.keys() else None
-        # self.track_intermediates = track_intermediates
-    
+        self.optimize_k = optimize_k
+        self.track_intermediates = track_intermediates
+
     def get_params(self, deep=False):
         return {'s': self.s,
                 'so': self.so,
-                'd': self.d,
+                'k': self.k,
                 'fit_intercept': self.fit_intercept,
                 'normalize': self.normalize,
                 'pool_reset': self.pool_reset,
-                'information_criteria': self.information_criteria
+                'self.optimize_k': self.optimize_k
                 }
 
     def __str__(self):
-        return 'SISSO(n_sis={0}, SO={1}, d={2})'.format(self.s, str(self.so), self.d)
+        return 'ICL(n_sis={0}, SO={1}, k={2})'.format(self.s, str(self.so), self.k)
 
     def __repr__(self, prec=3):
         ret = []
         for i, name in enumerate(self.feature_names_sparse_):
-            ret += [('+' if self.coef_[0, i] > 0 else '') + str(np.round(self.coef_[0, i], prec)) + str(name)]
-        ret += ['+' + str(float(np.round(self.intercept_, prec)))]
+            ret += [('+' if self.coef_[0, i] > 0 else '') + 
+                    str(np.format_float_scientific(self.coef_[0, i], precision=prec, unique=False))
+                      + ' (' + str(name) + ')' + '\n']
+        ret += [('+' if self.intercept_>0 else '') + str(float(np.round(self.intercept_, prec)))]
         return ''.join(ret)
-    
-        # return '+'.join(['{0}({1})'.format(str(np.round(b, 3)), self.feature_names_sparse_[i]) for i, b in enumerate(self.coef_) if np.abs(b) > 0]+[str(self.intercept_)])
      
     def solve_norm_coef(self, X, y):
         n, p = X.shape
@@ -440,32 +679,37 @@ class ICL:
 
         return bad_cols
 
-    def fitting(self, X, y, feature_names=None, verbose=False, track_pool=False, track_intermediates=False):
+    def fitting(self, X, y, feature_names=None, verbose=False, track_pool=False, opt_k = None):
         self.feature_names_ = feature_names
         n,p = X.shape
+        stopping = self.k if opt_k is None else opt_k
+        if verbose: print('Stopping after {0} iterations'.format(stopping))
 
         pool_ = set()
         if track_pool: self.pool = []
-        if track_intermediates: self.intermediates = np.empty(shape=(self.d, 5), dtype=object)
+        if self.optimize_k or self.track_intermediates: self.intermediates = np.empty(shape=(self.k, 5), dtype=object)
+
         res = y
         i = 0
         IC = np.infty
-        cont = True
-        while i < self.d and cont:
+        while i < stopping:
             self.intercept_ = np.mean(res).squeeze()
             if verbose: print('.', end='')
 
             p, sis_i = self.sis(X=X, res=res, pool=list(pool_), verbose=verbose)
+            pool_old = deepcopy(pool_)
             pool_.update(sis_i)
             pool_lst = list(pool_)
-            
             if track_pool: self.pool = pool_lst
-            beta_i = self.so(X=X[:, pool_lst], y=y, d=i+1, verbose=verbose)
+            if str(self.so) == 'EffAdaLASSO(gamma=1)':
+                beta_i = self.so(X=X, y=y, d=i+1, idx_old = list(pool_old), idx_new=sis_i, verbose=verbose)
+            else:
+                beta_i = self.so(X=X[:, pool_lst], y=y, d=i+1, verbose=verbose)
 
             beta = np.zeros(shape=(X.shape[1]))
             beta[pool_lst] = beta_i
 
-            if track_intermediates:
+            if self.optimize_k or self.track_intermediates:
                 idx = np.nonzero(beta)[0]
                 if self.normalize:
                     coef = (beta[idx].reshape(1, -1)*self.b_y/self.b_x[idx].reshape(1, -1))
@@ -474,7 +718,7 @@ class ICL:
                     coef = beta[idx]
                     intercept_ = self.intercept_
                 coef = coef[0]
-                expr = ''.join([('+' if float(c) >= 0 else '') + str(np.round(float(c), 3)) + self.feature_names_[idx][q] for q, c in enumerate(coef)])
+                expr = ''.join([('+' if float(c) >= 0 else '') + str(np.round(float(c), 3)) + str(self.feature_names_[idx][q]) for q, c in enumerate(coef)])
                 if verbose: print('Model after {0} iterations: {1}'.format(i, expr))
 
                 self.intermediates[i, 0] = deepcopy(idx)
@@ -491,14 +735,9 @@ class ICL:
                 pool_ = set(pool_lst)
 
             res = (y.reshape(1, -1) - (np.dot(X, beta).reshape(1, -1)+self.intercept_) ).T
-            if not(self.information_criteria is None):
-                IC_old = IC
-                IC = IC_DICT[self.information_criteria](res=res, k=i+1)
-                if verbose: print('{0}={1}'.format(self.information_criteria, IC))
-                cont = IC < IC_old
 
             i += 1
-        if track_intermediates: self.intermediates = self.intermediates[:, :i]
+        if self.optimize_k or self.track_intermediates: self.intermediates = self.intermediates[:, :i]
             
         if verbose: print()
         
@@ -511,7 +750,7 @@ class ICL:
 
         return self
 
-    def fit(self, X, y, feature_names=None, timer=False, verbose=False, track_pool=False, track_intermediates=False):
+    def fit(self, X, y, val_size=0.1, feature_names=None, timer=False, verbose=False, track_pool=False, random_state=None):
         if verbose: print('removing invalid features')
         self.bad_col = self.filter_invalid_cols(X)
         X_ = np.delete(X, self.bad_col, axis=1)
@@ -522,9 +761,27 @@ class ICL:
         self.solve_norm_coef(X_, y)
         X_, y_ = self.normalize_Xy(X_, y)
 
-        if verbose: print('Fitting SISSO model')
+        if verbose: print('Fitting ICL model')
         if timer: start=time()
-        self.fitting(X=X_, y=y_, feature_names=feature_names_, verbose=verbose, track_pool = track_pool, track_intermediates=track_intermediates)
+        if self.optimize_k == False:
+            self.fitting(X=X_, y=y_, feature_names=feature_names_, verbose=verbose, track_pool = track_pool)
+        else:
+            if verbose: print('Finding optimal model size')
+            X_train, X_val, y_train, y_val = train_test_split(X_, y_, test_size=val_size, random_state=random_state)
+            self.fitting(X=X_train, y=y_train, feature_names=feature_names_, verbose=verbose, track_pool = track_pool)
+            best_k, best_e2 = 0, np.infty
+            for k in range(self.k):
+                idx = self.intermediates[k, 0]
+                coef = self.intermediates[k, 1]
+                inter = self.intermediates[k, 2]
+                X_pred = np.delete(X_val, self.bad_col, axis=1)
+                y_hat = (np.dot(X_pred[:, idx], coef.squeeze()) + inter).reshape(-1, 1)
+                e2_val = rmse(y_hat, y_val)
+                if e2_val < best_e2:
+                    best_k, best_e2 = k+1, e2_val
+            if verbose: print('refitting with k={0}'.format(best_k))
+            self.fitting(X=X_, y=y_, feature_names=feature_names_, verbose=verbose, track_pool = track_pool, opt_k = best_k)
+
         if timer: self.fit_time=time()-start
         if timer and verbose: print(self.fit_time)
 
@@ -544,9 +801,40 @@ class ICL:
         X_ = np.delete(X, self.bad_col, axis=1)
         return (np.dot(X_[:, self.beta_idx_], self.coef_.squeeze()) + self.intercept_).reshape(-1, 1)
 
+    def predict_ensemble(self, X):
+        y_hat = np.zeros(shape=(X.shape[0], self.k))
+        for k in range(self.k):
+            idx = self.intermediates[k, 0]
+            coef = self.intermediates[k, 1]
+            inter = self.intermediates[k, 2]
+            X_pred = np.delete(X, self.bad_col, axis=1)
+            y_hat[:, k]=(np.dot(X_pred[:, idx], coef) + inter).reshape(-1, 1).squeeze()
+        return y_hat
+    
+    def repr_ensemble(self, prec=3):
+        ret = []
+        for k in range(self.k):
+            idx = self.intermediates[k, 0]
+            coef = self.intermediates[k, 1]
+            inter = self.intermediates[k, 2]
+            feat = self.intermediates[k, 3]
+            model_k = []
+            for i, name in enumerate(feat):
+                model_k += [('+' if coef[i] > 0 else '') + 
+                        str(np.format_float_scientific(coef[i], precision=prec, unique=False))
+                        + ' (' + str(name) + ')' + '\n']
+            model_k += [('+' if inter > 0 else '')  + str(float(np.round(inter, prec)))]
+            model_k = ''.join(model_k)
+            ret += [model_k]
+        return ';\n\n'.join(ret)
+
     def score(self, X, y, scorer=rmse):
         return scorer(self.predict(X), y)
 
+    def score_ensemble(self, X, y):
+        y_hat_ens = self.predict_ensemble(X)
+        return np.mean((y_hat_ens - y.reshape(-1,1))**2, axis=0)
+
 class BOOTSTRAP:
     def __init__(self, X, y=None, random_state=None):
         self.X = X
@@ -630,20 +918,12 @@ class FeatureExpansion:
         self.ops = ops
         self.rung = rung
         self.printrate = printrate
-
-    def __call__(self, X, feature_names=None, verbose=False, f=None):
-        if verbose: print('Prepping Symbols')
-        if feature_names is None: feature_names = sp.symbols(' '.join(['x_{0}'.format(i) for i in range(X.shape[1])]))
-        if verbose: print('Performing Feature Expansion')
-        if verbose: print('Estimating the creation of {0} features with duplicates'.format(self.extimate_workload(X=X, max_rung=self.rung)))
-        spnames, names, X_ = self.FE_aux(X=X, feature_names=feature_names, rung=self.rung, max_rung=self.rung, prev_start = -1, verbose=verbose)
-        if verbose: print('Created {0} features, now removing duplicate features'.format(X_.shape[1]))
-        spnames, names, X_ = self.remove_redundant_features(spnames, names, X_)
-        if f:
-            pass
-        return spnames, names, X_
-
-    def remove_redundant_features(self, spnames, names, X_):
+        self.prev_print = 0
+        for i, op in enumerate(self.ops):
+            if type(op) == str:
+                self.ops[i] = (op, range(rung))
+        
+    def remove_redundant_features(self, symbols, names, X):
         sorted_idxs = np.argsort(names)
         for i, idx in enumerate(sorted_idxs):
             if i == 0:
@@ -651,28 +931,83 @@ class FeatureExpansion:
             elif names[idx] != names[sorted_idxs[i-1]]:
                 unique += [idx]
         unique_original_order = np.sort(unique)
-        return spnames[unique_original_order], names[unique_original_order], X_[:, unique_original_order]
-            
-    def extimate_workload(self, X, max_rung):
-        rung = max_rung
-        p = X.shape[1]
-        p_prev = X.shape[1]
-        unary = 0
-        binary = 0
-        for op in self.ops:
-            if OP_DICT[op]['inputs'] == 1:
-                unary += 1
-            elif OP_DICT[op]['inputs'] == 2:
-                binary += 1
-        while rung > 0:
-            new_unary = unary*(p-p_prev) if rung != max_rung else unary*p
-            new_binary = int(binary*(p-p_prev)*(p-1)) if rung != max_rung else int(binary*p*(p-1)/2)
-            p_prev = p
-            p = p + new_unary + new_binary
-            rung -= 1
-        return p
-
-    def FE_aux(self, X, feature_names, prev_start, rung=0, max_rung=0, verbose=False):
+        
+        return symbols[unique_original_order], names[unique_original_order], X[:, unique_original_order]
+    
+    def expand(self, X, names=None, verbose=False, f=None, check_pos=False):
+        n, p = X.shape
+        if (names is None) or (len(names) != p):
+            names = ['x_{0}'.format(i) for i in range(X.shape[1])]
+        
+        if check_pos == False:
+            symbols = sp.symbols(' '.join(name.replace(' ', '.') for name in names))
+        else:
+            symbols = []
+            for i, name in enumerate(names):
+                name = name.replace(' ', '.')
+                if np.all(X[:, i] > 0):
+                    sym = sp.symbols(name, real=True, positive=True)
+                else:
+                    sym = sp.symbols(name, real=True)               
+                symbols.append(sym)
+
+        symbols = np.array(symbols)
+        names = np.array(names)
+        
+        if verbose: print('Estimating the creation of around {0} features'.format(self.estimate_workload(p=p, max_rung=self.rung, verbose=verbose>2)))
+        
+        names, symbols, X = self.expand_aux(X=X, names=names, symbols=symbols, crung=0, prev_p=0, verbose=verbose)
+        if not(f is None):
+            import pandas as pd
+            df = pd.DataFrame(data=X, columns=names)
+            df['y'] = y
+            df.to_csv(f)
+
+        return names, symbols, X
+        
+    def estimate_workload(self, p, max_rung,verbose=False):
+        p0 = 0
+        p1 = p
+        for rung in range(max_rung):
+            if verbose: print('Applying rung {0} expansion'.format(rung))
+            new_u, new_bc, new_bn = 0, 0, 0
+            for (op, rung_range) in self.ops:
+                if rung in rung_range:
+                    if verbose: print('Applying {0} to {1} features will result in approximately '.format(op, p1-p0))
+                    if OP_DICT[op]['inputs'] == 1:
+                        new_u += p1
+                        if verbose: print('{0} new features'.format(p1))
+                    elif OP_DICT[op]['commutative'] == True:
+                        new_bc += (1/2)*(p1 - p0 + 1)*(p0 + p1 + 2)
+                        if verbose: print('{0} new features'.format((1/2)*(p1 - p0 + 1)*(p0 + p1 + 2)))
+                    else:
+                        new_bn += (p1 - p0 + 1)*(p0 + p1 + 2)
+                        if verbose: print('{0} new features'.format((p1 - p0 + 1)*(p0 + p1 + 2)))
+            p0 = p1
+            p1 = p1 + new_u + new_bc + new_bn
+            if verbose: print('For a total of {0} features by rung {1}'.format(p1, rung))
+        return p1
+        
+    def add_new(self, new_names, new_symbols, new_X, new_name, new_symbol, new_X_i, verbose=False):
+        valid = (np.isnan(new_X_i).sum(axis=0) + np.isposinf(new_X_i).sum(axis=0) + np.isneginf(new_X_i).sum(axis=0)) == 0
+        if new_names is None:
+            new_names = np.array(new_name[valid])
+            new_symbols = np.array(new_symbol[valid])
+            new_X = np.array(new_X_i[:, valid])
+        else:
+            new_names = np.concatenate((new_names, new_name[valid]))
+            new_symbols = np.concatenate((new_symbols, new_symbol[valid]))
+            new_X = np.hstack([new_X, new_X_i[:, valid]])
+#        if (verbose > 1) and not(new_names is None) and (len(new_names) % self.printrate == 0): print('Created {0} features so far'.format(len(new_names)))
+        if (verbose > 1) and not(new_names is None) and (len(new_names) - self.prev_print >= self.printrate):
+            self.prev_print = len(new_names)
+            elapsed = np.round(time() - self.start_time, 2)
+            print('Created {0} features so far in {1} seconds'.format(len(new_names),elapsed))
+        return new_names, new_symbols, new_X
+
+    def expand_aux(self, X, names, symbols, crung, prev_p, verbose=False):
+        
+        str_vectorize = np.vectorize(str)
 
         def simplify_nested_powers(expr):
             # Replace (x**n)**(1/n) with x
@@ -693,82 +1028,76 @@ class FeatureExpansion:
                 flatten_pow_chain
             )
         
-        # if rung == max_rung:
-        #     feature_names = np.array(feature_names)
-        #     sympy_names = np.array([str(name) for name in feature_names])
-
-        if rung <= 0:
-            return (np.array(feature_names), 
-                    np.array(
-                        [str(sp.simplify(simplify_nested_powers(name)))for name in feature_names]), 
-                    X)
+        if crung == 0:
+            self.start_time = time()
+            symbols, names, X = self.remove_redundant_features(X=X, names=names, symbols=symbols)
+        if crung==self.rung:
+            if verbose: print('Completed {0} rounds of feature transformations'.format(self.rung))
+            return symbols, names, X
         else:
-            if verbose: print('Creating rung {0} features'.format(max_rung - rung+1)) 
-            new_names = ()
-            for op_key in self.ops:
-                if OP_DICT[op_key]['inputs'] == 1:
-                    for i in range(prev_start, len(feature_names)):
-                        if verbose and ((len(feature_names) + len(new_names)) % self.printrate == 0): print('Created {0} Features'.format(len(feature_names) + len(new_names)))
-                        if len(new_names) == 0:
-                            new_X = OP_DICT[op_key]['op_np'](X[:, i]).reshape(X.shape[0], 1)
-                        else:
-                            new_X = np.hstack([new_X, OP_DICT[op_key]['op_np'](X[:, i]).reshape(X.shape[0], 1)])
-                        new_names += (OP_DICT[op_key]['op'](feature_names[i]), )
-                        if verbose>1: print(new_names[-1])
-                elif OP_DICT[op_key]['inputs'] == 2:
-                    pairings = combinations if OP_DICT[op_key]['commutative'] else permutations
-                    for idx1, idx2 in pairings(range(len(feature_names)), 2): 
-                        if verbose and ((len(feature_names) + len(new_names)) % self.printrate == 0): print('Created {0} Features'.format(len(feature_names) + len(new_names)))
-                        # make sure at least one of the features if from the new features
-                        if idx1 >= prev_start or idx2 >= prev_start: 
-                            new_col = OP_DICT[op_key]['op_np'](X[:, idx1], X[:, idx2]).reshape(X.shape[0], 1).reshape(X.shape[0], 1)
-                            new_X = new_col if len(new_names) == 0 else np.hstack([new_X,new_col])
-                            new_name = OP_DICT[op_key]['op'](feature_names[idx1],feature_names[idx2])
-                            new_names += (new_name, )
-                            if verbose > 1: print(new_name)
-            if new_names == ():
-                return self.FE_aux(X = X, feature_names=feature_names, rung=rung-1, prev_start=len(feature_names), max_rung=max_rung, verbose=verbose)
+            if verbose: print('Applying round {0} of feature transformations'.format(crung+1))
+#            if verbose: print('Estimating the creation of {0} features this iteration'.format(self.estimate_workload(p=X.shape[1], max_rung=1)))
+                
+            new_names, new_symbols, new_X = None, None, None
+            
+            for (op_key, rung_range) in self.ops:
+                if crung in rung_range:
+                    if verbose>1: print('Applying operator {0} to {1} features'.format(op_key, X.shape[1]))
+                    op_params = OP_DICT[op_key]
+                    op_sym, op_np, inputs, comm = op_params['op'], op_params['op_np'], op_params['inputs'], op_params['commutative']
+                    if inputs == 1:
+                        sym_vect = np.vectorize(op_sym)
+                        new_op_symbols = sym_vect(symbols[prev_p:])
+                        new_op_X = op_np(X[:, prev_p:])
+                        new_op_names = str_vectorize(new_op_symbols)
+                        new_names, new_symbols, new_X = self.add_new(new_names=new_names, new_symbols=new_symbols, new_X=new_X, 
+                                                                    new_name=new_op_names, new_symbol=new_op_symbols, new_X_i=new_op_X, verbose=verbose)
+                    elif inputs == 2:
+                        for idx1 in range(prev_p, X.shape[1]):
+                            sym_vect = np.vectorize(lambda idx2: op_sym(symbols[idx1], symbols[idx2]))
+                            idx2 = range(idx1 if comm else X.shape[1])
+                            if len(idx2) > 0:
+                                new_op_symbols = sym_vect(idx2)
+                                new_op_names = str_vectorize(new_op_symbols)
+                                X_i = X[:, idx1]
+                                new_op_X = X_i[:, np.newaxis]*X[:, idx2]                                                
+                                new_names, new_symbols, new_X = self.add_new(new_names=new_names, new_symbols=new_symbols, new_X=new_X, 
+                                                                        new_name=new_op_names, new_symbol=new_op_symbols, new_X_i=new_op_X, verbose=verbose)
+            if not(new_names is None):                
+                names = np.concatenate((names, new_names))
+                symbols = np.concatenate((symbols, new_symbols))
+                prev_p = X.shape[1]
+                X = np.hstack([X, new_X])
             else:
-                return self.FE_aux(X = np.hstack([X, new_X]), feature_names=feature_names+new_names, rung=rung-1, prev_start=len(feature_names), max_rung=max_rung, verbose=verbose)
-
+                prev_p = X.shape[1]
+                
+            if verbose: print('After applying rounds {0} of feature transformations there are {1} features'.format(crung+1, X.shape[1]))
+            if verbose: print('Removing redundant features leaves... ', end='')            
+            symbols, names, X = self.remove_redundant_features(X=X, names=names, symbols=symbols)
+            if verbose: print('{0} features'.format(X.shape[1]))
+
+            return self.expand_aux(X=X, names=names, symbols=symbols, crung=crung+1, prev_p=prev_p, verbose=verbose)
+        
 if __name__ == "__main__":
+    from sklearn.model_selection import train_test_split
     random_state = 0
-    n = 100
-    p = 10
-    rung = 3
-    s = 5
-    d = 4
-
     np.random.seed(random_state)
-    X_train = np.random.normal(size=(n, p))
-
-    y = lambda X: X[:, 0] + 2*X[:, 1]**2 - X[:, 0]*X[:, 1] + 3*X[:, 2]**3
-    y_train = y(X_train)
-
-    # Initialise and fit the ICL model
-    FE = PolynomialFeaturesICL(rung=rung, include_bias=False)
-    so = AdaptiveLASSO(gamma=1, fit_intercept=False)
-    information_criteria='BIC'
-
-    X_train_transformed = FE.fit_transform(X_train, y)
-    feature_names = FE.get_feature_names_out()
-
-    icl = ICL(s=s, so=so, d=d, fit_intercept=True, normalize=True, pool_reset=False, information_criteria=information_criteria)
-    icl.fit(X_train_transformed, y_train, feature_names=feature_names, verbose=True, track_intermediates=True)
-
-    # Compute the train and test error and print the model to verify that we have reproduced the data generating function
-    print(icl)
-    print(icl.__repr__())
-
-    y_hat_train = icl.predict(X_train_transformed)
-
-    print("Train rmse: " + str(rmse(y_hat_train, y_train)))
-
-    X_test = np.random.normal(size=(100*n, p))
-    X_test_transformed = FE.transform(X_test)
-    y_test = y(X_test)
-    y_hat_test = icl.predict(X_test_transformed)
-    print("Test rmse: " + str(rmse(y_hat_test, y_test)))
-    print("k={0}".format(len(icl.coef_[0])))
-
-    # print(icl.intermediates)
+    n, p = 10000, 10
+    X = np.random.random(size=(n,p))
+    y = np.sqrt(X[:, 0]) - np.cbrt(X[:, 0]) + X[:, 0]**3 - np.log(X[:, 0]) + np.sin(X[:, 0]) + 1
+    names = ['X_{0}'.format(i) for i in range(p)]
+
+    rung = 1
+    small = ['sin', 'cos', 'log', 'abs', 'sqrt', 'cbrt', 'sq', 'cb', 'inv']
+    big = ['six_pow', 'exp', 'add', 'mul', 'div', 'abs_diff']
+    small  = [(op, range(rung)) for op in small]
+    big = [(op, range(1)) for op in big]
+    ops = small+big
+
+    FE = FeatureExpansion(rung=rung, ops=ops)
+    Phi_names, Phi_symbols, Phi_ = FE.expand(X=X, names=names, check_pos=True, verbose=True)
+    X_train, X_test, y_train, y_test = train_test_split(Phi_, y, test_size=0.2, random_state=random_state)
+    for i, s in enumerate([5]):
+        icl = ICL(s=s, so=AdaptiveLASSO(gamma=1), k=5, fit_intercept=True, normalize=True, optimize_k=False, track_intermediates=True)
+        icl.fit(X=X_train, y=y_train, feature_names = Phi_names, verbose=False)
+        print(icl.repr_ensemble())

{icol-0.1.6.dist-info → icol-0.7.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: icol
-Version: 0.1.6
+Version: 0.7.4
 Summary: Iterative Correlation Learning implementation
 Author-email: Simon Teshuva <simon.teshuva@gmail.com>
 License: MIT
@@ -13,7 +13,7 @@ Requires-Dist: scikit-learn>=1.2.2
 # icol
 ** Iterative Correlation Learning in Python **
 
-`icol` allows one to fit extremly sparse linear models from very high dimensional datasets in a computationally efficient manner. Given a feature transformation, it can also be used to fit Symbolic Regression models
+`icol` allows one to fit extremly sparse linear models from very high dimensional datasets in a computationally efficient manner. We also include two feature expansion methods, allowing icol to be used as a Symbolic Regression tool.
 
 ---
 

icol-0.7.4.dist-info/RECORD

@@ -0,0 +1,7 @@
+icol/__init__.py,sha256=nnhJPjnFCpho8OB-5q-Mq8J91EeCV_o3KVO-lLC8tQY,173
+icol/icol.py,sha256=59HIf4VKznrTKMVI46iz6eRXGLvvSfbGS1lQoLlJT1c,42179
+icol-0.7.4.dist-info/LICENSE,sha256=aD00NFSvGfojy-IWFmtKpeSg262O0dWzmsfXAaT0xuk,1070
+icol-0.7.4.dist-info/METADATA,sha256=ZE20mOaTldgxJtiMOHyVOsh23VjDIKk8r_Tmo8JHwGM,1977
+icol-0.7.4.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+icol-0.7.4.dist-info/top_level.txt,sha256=OKisIKQUWtt2x-hxR53qbTr2AR3kdeRfTChIdmn2sDY,5
+icol-0.7.4.dist-info/RECORD,,

icol-0.1.6.dist-info/RECORD

@@ -1,7 +0,0 @@
-icol/__init__.py,sha256=nnhJPjnFCpho8OB-5q-Mq8J91EeCV_o3KVO-lLC8tQY,173
-icol/icol.py,sha256=Yh3xf64Z4vjo0aFiHUgTxAhVylNtZbyWHe3_4b6fnN8,28387
-icol-0.1.6.dist-info/LICENSE,sha256=aD00NFSvGfojy-IWFmtKpeSg262O0dWzmsfXAaT0xuk,1070
-icol-0.1.6.dist-info/METADATA,sha256=CexfevglpUbzgZUrINQ5GW38fj1YJsh2_GPwFO00SNs,1960
-icol-0.1.6.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-icol-0.1.6.dist-info/top_level.txt,sha256=OKisIKQUWtt2x-hxR53qbTr2AR3kdeRfTChIdmn2sDY,5
-icol-0.1.6.dist-info/RECORD,,