driada 0.0.1__py3-none-any.whl

dim_reduction/data.py

@@ -0,0 +1,117 @@
+# @title Data class { form-width: "200px" }
+
+import numpy as np
+import matplotlib.pyplot as plt
+import scipy.sparse as sp
+
+from .dr_base import *
+from src.utils.data import correlation_matrix, to_numpy_array, rescale
+from .embedding import Embedding
+from .graph import ProximityGraph
+
+# TODO: refactor this
+def check_data_for_errors(d):
+    sums = np.sum(np.abs(d), axis=0)
+    if len(sums.nonzero()[1]) != d.shape[1]:
+        bad_points = np.where(sums == 0)[1]
+        print('zero points:', bad_points)
+        print(d.A[:, bad_points[0]])
+        raise Exception('Data contains zero points!')
+
+
+class MVData(object):
+    '''
+    Main class for multivariate data storage & processing
+    '''
+
+    def __init__(self,
+                 data,
+                 labels=None,
+                 distmat=None,
+                 rescale_rows=False,
+                 data_name=None,
+                 downsampling=None):
+
+        if downsampling is None:
+            self.ds = 1
+        else:
+            self.ds = int(downsampling)
+
+        self.data = to_numpy_array(data)[:, ::self.ds]
+
+        # TODO: add support for various preprocessing methods (wvt, med_filt, etc.)
+        self.rescale_rows = rescale_rows
+        if self.rescale_rows:
+            for i, row in enumerate(self.data):
+                self.data[i] = rescale(row)
+
+        self.data_name = data_name
+        self.n_dim = self.data.shape[0]
+        self.n_points = self.data.shape[1]
+
+        if labels is None:
+            self.labels = np.zeros(self.n_points)
+        else:
+            self.labels = to_numpy_array(labels)
+
+        self.distmat = distmat
+
+    def median_filter(self, window):
+        from scipy.signal import medfilt
+        d = self.data.A
+
+        new_d = medfilt(d, window)
+
+        self.data = sp.csr_matrix(new_d)
+
+    def corr_mat(self):
+        cm = correlation_matrix(self.data)
+        return cm
+
+    def get_distmat(self, method='euclidean'):
+        raise NotImplementedError('To be implemented soon')
+
+    def get_embedding(self, e_params, g_params=None, m_params=None, kwargs=None):
+        method = e_params['e_method']
+        method_name = e_params['e_method_name']
+
+        if method_name not in EMBEDDING_CONSTRUCTION_METHODS:
+            raise Exception('Unknown embedding construction method!')
+
+        graph = None
+        if method.requires_graph:
+            if g_params is None:
+                raise ValueError(f'Method {method_name} requires proximity graph, but '
+                                 f'graph params were not provided')
+            if g_params['weighted'] and m_params is None:
+                raise ValueError(f'Method {method_name} requires weights for proximity graph, but '
+                                 f'metric params were not provided')
+
+            graph = self.get_proximity_graph(m_params, g_params)
+
+        if method.requires_distmat and self.distmat is None:
+            raise Exception(f'No distmat provided for {method_name} method.'
+                            f' Try constructing it first with get_distmat() method')
+
+        emb = Embedding(self.data, self.distmat, self.labels, e_params, g=graph)
+        emb.build(kwargs=kwargs)
+
+        return emb
+
+    def get_proximity_graph(self, m_params, g_params):
+        if g_params['g_method_name'] not in GRAPH_CONSTRUCTION_METHODS:
+            raise Exception('Unknown graph construction method!')
+
+        graph = ProximityGraph(self.data, m_params, g_params)
+        # print('Graph succesfully constructed')
+        return graph
+
+    def draw_vector(self, num):
+        data = self.data[:, num]
+        plt.matshow(data.reshape(1, self.n_dim))
+        plt.matshow(self.data)
+
+    def draw_row(self, num):
+        data = self.data[num, :]
+        plt.figure(figsize=(12, 10))
+        plt.plot(data)

dim_reduction/dim_reduction.py

@@ -0,0 +1,75 @@
+from .dr_base import *
+from .data import MVData
+from .graph import ProximityGraph
+from .embedding import Embedding
+
+
+# TODO: refactor this
+def dr_series(d,
+              n_jumps,
+              all_metric_params,
+              all_graph_params,
+              all_embedding_params,
+              recalculate_if_error=0):
+
+    print('---------------------------  JUMP 1  --------------------------------')
+    print('Performing jump from dim', d.dim, 'to dim', all_embedding_params['dim'][0], ':')
+    metric_params = dict(zip(all_metric_params.keys(), [val[0] for val in all_metric_params.values()]))
+    graph_params = dict(zip(all_graph_params.keys(), [val[0] for val in all_graph_params.values()]))
+    embedding_params = dict(zip(all_embedding_params.keys(), [val[0] for val in all_embedding_params.values()]))
+    embedding_params['e_method'] = METHODS_DICT[embedding_params['e_method_name']]
+
+    m_params = m_param_filter(metric_params)
+    g_params = g_param_filter(graph_params)
+    e_params = e_param_filter(embedding_params)
+
+    maxiter = 20
+    n_iter = maxiter * int(recalculate_if_error) + 1
+    it = 0
+    success = 0
+    while it < n_iter and success == 0:
+        try:
+            emb = d.get_embedding(m_params, g_params, e_params)
+            success = 1
+        except:
+            it += 1
+            print("Unexpected error:", sys.exc_info()[0])
+            raise
+
+    if it == n_iter:
+        raise Exception('First jump failed after %s attempts' % n_iter)
+
+    if n_jumps > 1:
+        datalist = [d, MVData(emb.coords, labels=d.labels)]
+
+    for i in range(1, n_jumps):
+        print('---------------------------  JUMP ' + str(i + 1) + ' --------------------------------', )
+        print('Performing jump from dim', datalist[i].dim, 'to dim', all_embedding_params['dim'][i], ':')
+        metric_params = dict(zip(all_metric_params.keys(), [val[i] for val in all_metric_params.values()]))
+        graph_params = dict(zip(all_graph_params.keys(), [val[i] for val in all_graph_params.values()]))
+        embedding_params = dict(zip(all_embedding_params.keys(), [val[i] for val in all_embedding_params.values()]))
+        embedding_params['e_method'] = METHODS_DICT[embedding_params['e_method_name']]
+
+        m_params = m_param_filter(metric_params)
+        g_params = g_param_filter(graph_params)
+        e_params = e_param_filter(embedding_params)
+
+        maxiter = 20
+        n_iter = maxiter * (recalculate_if_error) + 1
+        it = 0
+        success = 0
+
+        while it < n_iter and success == 0:
+            try:
+                emb = datalist[i].get_embedding(m_params, g_params, e_params)
+                success = 1
+            except:
+                print('iter', it)
+                it += 1
+
+        if it == n_iter:
+            raise Exception('Jump ', str(i + 1), ' failed after %s attempts' % n_iter)
+
+        datalist.append(MVData(emb.coords, labels=d.labels))
+
+    return emb

dim_reduction/dr_base.py

@@ -0,0 +1,111 @@
+from pynndescent.distances import named_distances
+import sys
+
+class DRMethod(object):
+
+    def __init__(self, is_linear, requires_graph, requires_distmat, nn_based):
+        self.is_linear = is_linear
+        self.requires_graph = requires_graph
+        self.requires_distmat = requires_distmat
+        self.nn_based = nn_based
+
+
+METHODS_DICT = {
+    'pca': DRMethod(1, 0, 0, 0),
+    'le': DRMethod(0, 1, 0, 0),
+    'auto_le': DRMethod(0, 1, 0, 0),
+    'dmaps': DRMethod(0, 1, 0, 0),
+    'auto_dmaps': DRMethod(0, 1, 0, 0),
+    'mds': DRMethod(0, 0, 1, 0),
+    'isomap': DRMethod(0, 1, 0, 0),
+    'lle': DRMethod(0, 1, 0, 0),
+    'hlle': DRMethod(0, 1, 0, 0),
+    'mvu': DRMethod(0, 1, 0, 0),
+    'ae': DRMethod(0, 0, 0, 1),
+    'vae': DRMethod(0, 0, 0, 1),
+    'tsne': DRMethod(0, 0, 0, 0),
+    'umap': DRMethod(0, 1, 0, 0)
+}
+
+GRAPH_CONSTRUCTION_METHODS = ['knn', 'auto_knn', 'eps', 'eknn', 'umap', 'tsne']
+
+EMBEDDING_CONSTRUCTION_METHODS = ['pca',
+                                  'le',
+                                  'auto_le',
+                                  'dmaps',
+                                  'auto_dmaps',
+                                  'mds',
+                                  'isomap',
+                                  'lle',
+                                  'hlle',
+                                  'mvu',
+                                  'ae',
+                                  'vae',
+                                  'tsne',
+                                  'umap']
+
+# TODO: implement random projections
+
+
+def m_param_filter(para):
+    '''
+    This function prunes parameters that are excessive for
+    chosen distance matrix construction method
+    '''
+    name = para['metric_name']
+    appr_keys = ['metric_name']
+
+    if not (para['sigma'] is None):
+        appr_keys.append('sigma')
+
+    if name not in named_distances:
+        if name == 'hyperbolic':
+            # para['metric_name'] = globals()[name]
+            pass
+        else:
+            raise Exception('this custom metric is not implemented!')
+
+    if name == 'minkowski':
+        appr_keys.append('p')
+
+    return {key: para[key] for key in appr_keys}
+
+
+def g_param_filter(para):
+    '''
+    This function prunes parameters that are excessive for
+    chosen graph construction method
+    '''
+    gmethod = para['g_method_name']
+    appr_keys = ['g_method_name', 'max_deleted_nodes', 'weighted', 'dist_to_aff']
+
+    if gmethod in ['knn', 'auto_knn', 'umap']:
+        appr_keys.extend(['nn'])
+
+    elif gmethod == 'eps':
+        appr_keys.extend(['eps', 'eps_min'])
+
+    elif gmethod == 'eknn':
+        appr_keys.extend(['eps', 'eps_min', 'nn'])
+
+    elif gmethod == 'tsne':
+        appr_keys.extend(['perplexity'])
+
+    return {key: para[key] for key in appr_keys}
+
+
+def e_param_filter(para):
+    '''
+    This function prunes parameters that are excessive for the
+    chosen embedding construction method
+    '''
+
+    appr_keys = ['e_method', 'e_method_name', 'dim']
+
+    if para['e_method_name'] == 'umap':
+        appr_keys.append('min_dist')
+
+    if para['e_method_name'] in ['dmaps', 'auto_dmaps']:
+        appr_keys.append('dm_alpha')
+
+    return {key: para[key] for key in appr_keys}