hjxdl 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

hdl/jupyfuncs/dbtools/query_info.py

@@ -0,0 +1,150 @@
+import re
+
+import cirpy
+import pubchempy as pcp
+import molvs as mv
+from psycopg import sql
+
+from .pg import connect_by_infofile
+
+
+def query_from_cir(query_name: str):
+    smiles = None
+    # cas_list = []
+    # name_list = [host=172.20.0.5 dbname=pistachio port=5432 user=postgres password=woshipostgres]
+
+    cas_list = cirpy.resolve(query_name, 'cas')
+    if cas_list is None or not cas_list:
+        cas_list = []
+    if isinstance(cas_list, str):
+        cas_list = [cas_list]
+
+    name_list = cirpy.resolve(query_name, 'names')
+    if name_list is None or not name_list:
+        name_list = []
+    if isinstance(name_list, str):
+        name_list = [name_list]
+
+    smiles = cirpy.resolve(query_name, 'smiles')
+    try:
+        smiles = mv.standardize_smiles(smiles)
+    except Exception as e:
+        print(e)
+
+    return smiles, cas_list, name_list
+
+
+def query_from_pubchem(query_name: str):
+    results = pcp.get_compounds(query_name, 'name')
+    smiles = None
+    name_list = set()
+    cas_list = set()
+
+    if any(results):
+        try:
+            smiles = mv.standardize_smiles(results[0].canonical_smiles)
+        except Exception as e:
+            smiles = results[0].canonical_smiles
+            print(smiles)
+            print(e)
+        for compound in results:
+            name_list.update(set(compound.synonyms))
+            for syn in compound.synonyms:
+                match = re.match('(\d{2,7}-\d\d-\d)', syn)
+                if match:
+                    cas_list.add(match.group(1))
+        
+        cas_list = list(cas_list)
+        name_list = list(name_list)
+
+    return smiles, cas_list, name_list
+
+
+def query_a_compound(
+    query_name: str,
+    connect_info: str,
+    by: str = 'name',
+    log_file: str = './err.log'
+):
+    fei = None
+    found = False  
+
+    if by != 'smiles':
+        query_name = query_name.lower()
+    
+    by = 'name'
+    table = by + '_maps'
+    # query_name = 'adipic acid'
+    query = sql.SQL(
+        "select fei from {table} where {by} = %s"
+    ).format(
+        table=sql.Identifier(table),
+        by=sql.Identifier(by)
+    )
+    conn = connect_by_infofile(connect_info)
+    
+    cur = conn.execute(query, [query_name]).fetchone()
+
+    if cur is not None:
+        fei = cur[0]
+        found = True
+        return fei
+ 
+    if not found:
+        try:
+            smiles, cas_list, name_list = query_from_pubchem(query_name) 
+        except Exception as e:
+            print(e)
+            smiles, cas_list, name_list = None, [], []
+        if smiles is not None:
+            found = True
+        else:
+            try:
+                smiles, cas_list, name_list = query_from_cir(query_name)
+            except Exception as e:
+                print(e)
+                smiles, cas_list, name_list = None, [], []
+            if smiles is not None:
+                found = True
+    
+    if not found:
+        with open(log_file, 'a') as f:
+            f.write(query_name)
+            f.write('\n')
+        return
+        # raise ValueError('给的啥破玩意儿查都查不着！')
+    else:
+        query_compound = sql.SQL(
+            "select fei from compounds where smiles = %s"
+        )
+        cur = conn.execute(query_compound, [smiles]).fetchone()
+        if cur is not None:
+            fei = cur[0]
+        elif any(cas_list):
+            fei = cas_list[0]
+            insert_compounds_sql = sql.SQL(
+                "INSERT INTO compounds (fei, smiles) VALUES (%s, %s) ON CONFLICT (fei) DO NOTHING"
+            )
+            conn.execute(insert_compounds_sql, [fei, smiles])
+        for cas in cas_list:
+            insert_cas_map_sql = sql.SQL(
+                "INSERT INTO cas_maps (fei, cas) VALUES (%s, %s) ON CONFLICT (cas) DO NOTHING"
+            )
+            try:
+                conn.execute(insert_cas_map_sql, [fei, cas])
+            except Exception as e:
+                print(e)
+        for name in name_list:
+            insert_name_map_sql = sql.SQL(
+                "INSERT INTO name_maps (fei, name) VALUES (%s, %s) ON CONFLICT (fei, name) DO NOTHING"
+                # "INSERT INTO name_maps (fei, name) VALUES (%s, %s)"
+            ) 
+            try:
+                conn.execute(insert_name_map_sql, [fei, name.lower()])
+            except Exception as e:
+                print(e)
+
+    conn.commit()
+    conn.close()
+
+    return fei

hdl/jupyfuncs/dl/cp.py

@@ -0,0 +1,54 @@
+"""
+Conformer Prediction for classification Task
+
+As we only consider the predicted values as input, 
+we do not differentiate between transductive and inductive conformers
+"""
+
+import numpy as np
+import pandas as pd
+
+
+class CpClassfier:
+    def __init__(self):
+        self.cal_data = None
+        self.class_num = 0
+
+    def fit_with_data(self, cal_proba, cal_y, class_num=0):
+        """
+        :parm cal_proba: numpy array of shape [n_samples, n_classes]
+                        predicted probability of calibration set
+        :parm cal_y: numpy array of shape [n_samples,]
+                        true label of calibration set
+        """
+
+        if class_num <= 0:
+            print("Get class number for input data.")
+            self.class_num = cal_proba.shape[1]
+        else:
+            self.class_num = class_num
+
+        cal_df = pd.DataFrame(cal_proba)
+        cal_df.columns = ["class_%d"%i for i in range(self.class_num)]
+        cal_df["true_label"] = list(cal_y)
+        self.cal_data = cal_df
+     
+    def fit_with_model(self, func):
+        #TODO: besides calibration data, we can also use our model
+        pass
+
+    def predict_with_proba(self, X_proba):
+        """
+        :parm X_proba: numpy array of shape [n_samples, n_classes]
+                        predicted probabilities of calibration set
+        """
+        cp_proba = []
+        class_lsts = [sorted(self.cal_data[self.cal_data["true_label"] == i]["class_%d"%i]) \
+                        for i in range(self.class_num)]
+        proba_lsts = [X_proba[:, i] for i in range(self.class_num)]
+        for c_lst, p_lst in zip(class_lsts, proba_lsts):
+            c_proba = np.searchsorted(c_lst, p_lst, side='left')/len(c_lst)
+            cp_proba.append(c_proba)
+
+        self.cp_P = np.array(cp_proba).T
+        return self.cp_P

hdl/jupyfuncs/dl/dataframe.py

@@ -0,0 +1,38 @@
+def rm_index(df):
+    """Remove columns with 'Unnamed' in their names from the DataFrame.
+    
+        Args:
+            df (pandas.DataFrame): The input DataFrame.
+    
+        Returns:
+            pandas.DataFrame: DataFrame with columns containing 'Unnamed' removed.
+    """
+    return df.loc[:, ~df.columns.str.match('Unnamed')]
+
+
+def rm_col(df, col_name):
+    """Remove a column from a DataFrame.
+    
+    Args:
+        df (pandas.DataFrame): The input DataFrame.
+        col_name (str): The name of the column to be removed.
+    
+    Returns:
+        pandas.DataFrame: A new DataFrame with the specified column removed.
+    """
+    return df.loc[:, ~df.columns.str.match(col_name)]
+
+
+def shuffle_df(df):
+    """Shuffle the rows of a DataFrame.
+    
+    Args:
+        df (pandas.DataFrame): The input DataFrame to shuffle.
+    
+    Returns:
+        pandas.DataFrame: A new DataFrame with rows shuffled.
+    
+    Example:
+        shuffled_df = shuffle_df(df)
+    """
+    return df.sample(frac=1).reset_index(drop=True)

hdl/jupyfuncs/dl/fp.py

@@ -0,0 +1,49 @@
+from rdkit import Chem
+import numpy as np
+from rdkit.Chem import AllChem
+from rdkit.Chem import MACCSkeys
+
+
+__all__ = [
+    'get_fp',
+]
+
+
+def get_rdnorm_fp(smiles):
+    from descriptastorus.descriptors import rdNormalizedDescriptors
+    generator = rdNormalizedDescriptors.RDKit2DNormalized()
+    features = generator.process(smiles)[1:]
+    arr = np.array(features)
+    return arr
+
+
+def get_maccs_fp(smiles):
+    arr = np.zeros(167)
+    try:
+        mol = Chem.MolFromSmiles(smiles)
+        vec = MACCSkeys.GenMACCSKeys(mol)
+        bv = list(vec.GetOnBits())
+        arr[bv] = 1
+    except Exception as e:
+        print(e)
+    return arr
+
+
+def get_morgan_fp(smiles):
+    mol = Chem.MolFromSmiles(smiles)
+    vec = AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=1024)
+    bv = list(vec.GetOnBits())
+    arr = np.zeros(1024)
+    arr[bv] = 1
+    return arr
+
+
+fp_dict = {
+    'rdnorm': get_rdnorm_fp,
+    'maccs': get_maccs_fp,
+    'morgan': get_morgan_fp
+}
+
+
+def get_fp(smiles, fp='maccs'):
+    return fp_dict[fp](smiles)

hdl/jupyfuncs/dl/list.py

@@ -0,0 +1,20 @@
+def list_diff(listA, listB, mode="intersection"):
+    """Calculate the difference between two lists based on the specified mode.
+    
+        Args:
+            listA (list): The first list.
+            listB (list): The second list.
+            mode (str, optional): The mode to determine the difference. 
+                Possible values are "intersection" (default), "union", or "diff".
+    
+        Returns:
+            list: A list containing the elements based on the specified mode.
+    """
+    if mode == "intersection":
+        ret = list(set(listA).intersection(set(listB)))
+    elif mode == "union":
+        ret = list(set(listA).union(set(listB)))
+    elif mode == "diff":
+        ret = list(set(listB).difference(set(listA)))
+    
+    return ret

hdl/jupyfuncs/dl/model_utils.py

@@ -0,0 +1,97 @@
+import torch
+from torch import nn
+
+
+def save_model(
+    model,
+    save_dir,
+    epoch=0,
+    optimizer=None,
+    loss=None,
+):
+    """Save the model and related training information to a specified directory.
+    
+    Args:
+        model: The model to be saved.
+        save_dir: The directory where the model will be saved.
+        epoch (int): The current epoch number (default is 0).
+        optimizer: The optimizer used for training (default is None).
+        loss: The loss value (default is None).
+    """
+    if isinstance(model, nn.DataParallel):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    if optimizer is None:
+        optim_params = None
+    else:
+        optim_params = optimizer.state_dict()
+    torch.save(
+        {
+            'init_args': model.init_args,
+            'epoch': epoch,
+            'model_state_dict': state_dict,
+            'optimizer_state_dict': optim_params,
+            'loss': loss,
+        },
+        save_dir
+    )
+
+
+def load_model(
+    save_dir,
+    model_class=None,
+    model=None,
+    optimizer=None,
+    train=False,
+):
+    """Load a saved model from the specified directory.
+    
+    Args:
+        save_dir (str): The directory where the model checkpoint is saved.
+        model_class (torch.nn.Module, optional): The class of the model to be loaded. Defaults to None.
+        model (torch.nn.Module, optional): The model to load the state_dict into. Defaults to None.
+        optimizer (torch.optim.Optimizer, optional): The optimizer to load the state_dict into. Defaults to None.
+        train (bool, optional): Whether to set the model to training mode. Defaults to False.
+    
+    Returns:
+        tuple: A tuple containing the loaded model, optimizer, epoch, and loss.
+    """
+    # from .model_dict import MODEL_DICT
+    checkpoint = torch.load(save_dir)
+    if model is None:
+        init_args = checkpoint['init_args']
+        assert model_class is not None
+        model = model_class(**init_args)
+        model.load_state_dict( 
+            checkpoint['model_state_dict'], 
+        )
+    
+    elif isinstance(model, nn.DataParallel):
+        state_dict = checkpoint['model_state_dict']
+        from collections import OrderedDict
+        new_state_dict = OrderedDict()
+
+        for k, v in state_dict.items():
+            if 'module' not in k:
+                k = 'module.' + k
+            else:
+                k = k.replace('features.module.', 'module.features.')
+            new_state_dict[k] = v
+        model.load_state_dict(new_state_dict)
+    else:
+        model.load_state_dict( 
+            checkpoint['model_state_dict'], 
+        )
+    
+    if optimizer is not None:
+        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+    epoch = checkpoint['epoch']
+    loss = checkpoint['loss']
+
+    if train:
+        model.train()
+    else:
+        model.eval()
+
+    return model, optimizer, epoch, loss

hdl/jupyfuncs/dl/tensor.py

@@ -0,0 +1,159 @@
+import typing as t
+from collections import defaultdict
+
+import torch
+import numpy as np
+import pandas as pd
+from scipy.spatial.distance import cdist
+
+def spmmsp(
+    sp1: torch.sparse.Tensor,
+    sp2: torch.sparse.Tensor
+) -> torch.sparse.Tensor:
+    from torch_sparse import spspmm
+    assert sp1.size(-1) == sp2.size(0) and sp1.is_sparse and sp2.is_sparse
+    m = sp1.size(0)
+    k = sp2.size(0)
+    n = sp2.size(-1)
+    indices, values = spspmm(
+        sp1.indices(), sp1.values(),
+        sp2.indices(), sp2.values(),
+        m, k, n
+    )
+    return torch.sparse_coo_tensor(
+        indices,
+        values,
+        torch.Size([m, n])
+    )
+
+
+def label_to_onehot(ls, class_num, missing_label=-1):
+    """
+    example:
+    >>>label_to_onehot([2,3,-1],6,-1)
+    array([[ 0.,  0.,  1.,  0.],
+       [ 0.,  0.,  0.,  1.],
+       [nan, nan, nan, nan]])
+    :param ls:
+    :param class_num:
+    :param missing_label:
+    :return:
+    """
+    if isinstance(ls, torch.Tensor):
+        bool_t = ls == missing_label
+        clamp_t = torch.clamp(ls, min=0)
+        full_tensor = torch.zeros(ls.numel(), class_num)
+        full_tensor = full_tensor.scatter_(1, clamp_t.reshape(-1, 1), 1)
+        full_tensor[bool_t] = 0
+        return full_tensor
+    elif isinstance(ls, t.List):
+        ls = np.array(ls, dtype=np.int32)
+        bool_array = ls == missing_label
+        arr = np.zeros((ls.size, ls.max() + 1))
+        arr[np.arange(ls.size), ls] = 1
+        arr[bool_array] = 0
+        return arr
+    elif not isinstance(ls, t.Iterable):
+        arr = torch.zeros(class_num)
+        if ls != missing_label and not np.isnan(ls) and ls is not None:
+            arr[int(ls)] = 1
+        return arr
+
+
+def onehot_to_label(tensor):
+    if isinstance(tensor, torch.Tensor):
+        return torch.argmax(tensor, dim=-1)
+    elif isinstance(tensor, np.ndarray):
+        return np.argmax(tensor, axis=-1)
+
+
+def label_to_tensor(
+    label,
+    num_classes,
+    missing_label=-1,
+    device=torch.device('cpu')
+):
+    if isinstance(label, t.List) and not any(label):
+        return torch.zeros(num_classes).to(device)
+    elif isinstance(label, t.List) and isinstance(label[0], t.Iterable):
+        max_length = max([len(_l) for _l in label])
+        index = [_l + _l[-1:] * (max_length - len(_l)) for _l in label]
+        tensor_list = []
+#         tensor = torch.zeros(len(label), num_classes, device=device)
+        for _idx in index:
+            _tensor = torch.zeros(num_classes).to(device)
+            _idx = torch.LongTensor(_idx)
+            _tensor = _tensor.scatter(0, _idx, 1)
+            tensor_list.append(_tensor)
+        
+        return torch.vstack(tensor_list).to(device)
+    else:
+        if label == missing_label or np.isnan(label):
+            return torch.zeros(num_classes)
+        tensor = torch.zeros(num_classes).to(device)
+        tensor[int(label)] = 1
+        return tensor
+
+
+def tensor_to_label(tensor, threshold=0.5):
+    label_list, label_dict = [], defaultdict(list)
+    labels = (tensor > threshold).nonzero(as_tuple=False)
+    for label in labels:
+        label_dict[label[0].item()].append(label[1].item())
+    for _, label_value in label_dict.items():
+        label_list.append(label_value)
+    return label_list 
+
+
+def get_dist_matrix(
+    a: np.ndarray, b: np.ndarray
+):
+    return cdist(a, b)
+    # aSumSquare = np.sum(np.square(a), axis=1)
+    # bSumSquare = np.sum(np.square(b), axis=1)
+    # mul = np.dot(a, b.T)
+    # dists = np.sqrt(aSumSquare[:, np.newaxis] + bSumSquare - 2 * mul)
+    # return dists
+
+
+def get_valid_indices(labels):
+    if isinstance(labels, torch.Tensor):
+        nan_indices = torch.isnan(labels)
+        valid_indices = (
+            nan_indices == False
+        ).nonzero(as_tuple=True)[0]
+    else:
+        target_pd = pd.array(labels)
+        nan_indices = pd.isna(target_pd)
+        valid_indices = torch.LongTensor(
+            np.where(nan_indices == False)[0]
+        )
+    return valid_indices
+
+
+def smooth_max(
+    tensor: torch.Tensor,
+    inf_k: int = None,
+    **kwargs
+):
+    if inf_k is None:
+        inf_k = 10
+    max_value = torch.log(
+        torch.sum(
+            torch.exp(tensor * inf_k),
+            **kwargs
+        )
+    ) / inf_k
+    return max_value
+
+
+def list_df(listA, listB):
+    retB = list(set(listA).intersection(set(listB)))
+
+    retC = list(set(listA).union(set(listB)))
+
+    retD = list(set(listB).difference(set(listA)))
+
+    retE = list(set(listA).difference(set(listB)))
+    return retB, retC, retD, retE
+

hdl/jupyfuncs/dl/uncs.py

@@ -0,0 +1,112 @@
+"""UNCERTAINTY SAMPLING
+
+Uncertainty Sampling examples for Active Learning in PyTorch
+
+It contains four Active Learning strategies:
+1. Least Confidence Sampling
+2. Margin of Confidence Sampling
+3. Ratio of Confidence Sampling
+4. Entropy-based Sampling
+
+"""
+from copy import deepcopy
+
+import numpy as np
+
+__all__ = [
+    "get_prob_unc",
+]
+
+
+def least_conf_unc(prob_array: np.ndarray) -> np.ndarray:
+    """Least confidence uncertainty
+
+    .. math::
+        \phi_{L C}(x)=\left(1-P_{\theta}\left(y^{*} \mid x\right)\right) \times \frac{n}{n-1}
+
+    Args:
+        prob_array (np.array): a 1D or 2D array of probabilities
+ 
+    Returns:
+        np.ndarray: the uncertainty value(s)
+    """
+    if prob_array.ndim == 1:
+        indices = prob_array.argmax()
+    else:
+        indices = (
+            np.arange(prob_array.shape[0]),
+            prob_array.argmax(-1)
+        )
+    num_labels = prob_array.shape[-1]
+    uncs = (1 - prob_array[indices]) * (num_labels / (num_labels - 1))
+    return uncs
+
+
+def margin_conf_unc(prob_array: np.ndarray) -> np.ndarray:
+    """The margin confidence uncertainty
+
+    .. math:: 
+        \phi_{M C}(x)=1-\left(P_{\theta}\left(y_{1}^{*} \mid x\right)-P_{\theta}\left(y_{2}^{*} \mid x\right)\right)
+
+    Args:
+        prob_array (np.array): a 1D or 2D probability array from an NN.  
+
+    Returns:
+        np.array: the uncertainty value(s)
+    """
+    probs = deepcopy(prob_array)
+    probs.sort(-1)
+    diffs = probs[..., -1] - probs[..., -2]
+    return 1 - diffs
+
+
+def ratio_conf_unc(prob_array: np.ndarray) -> np.ndarray:
+    """Ratio based uncertainties
+
+    .. math::
+            \phi_{R C}(x)=P_{\theta}\left(y_{2}^{*} \mid x\right) / P_{\theta}\left(y_{1}^{*} \mid x\right)
+
+    Args:
+        prob_array (np.array): a 1D or 2D probability array
+
+    Returns:
+        np.array: the uncertainty value(s)
+    """
+    probs = deepcopy(prob_array)
+    probs.sort(-1)
+    ratio = probs[..., -1] / probs[..., -2]
+    return ratio
+
+
+def entropy_unc(prob_array: np.ndarray) -> np.ndarray:
+    """Entropy based uncertainty
+
+    .. math::
+        \phi_{E N T}(x)=\frac{-\Sigma_{y} P_{\theta}(y \mid x) \log _{2} P_{\theta}(y \mid x)}{\log _{2}(n)}
+
+    Args:
+        prob_array (np.array): a 1D or 2D probability array
+
+    Returns:
+        np.array: the uncertainty value(s)
+    """
+    num_labels = prob_array.shape[-1]
+    log_probs = prob_array * np.log2(prob_array)
+    
+    raw_entropy = 0 - np.sum(log_probs, -1)
+
+    normalized_entropy = raw_entropy / np.log2(num_labels)
+
+    return normalized_entropy
+
+
+unc_dict = {
+    'least': least_conf_unc,
+    'margin': margin_conf_unc,
+    'ratio': ratio_conf_unc,
+    'entropy': entropy_unc,
+}
+
+
+def get_prob_unc(prob_array: np.ndarray, unc: str) -> np.ndarray:
+    return unc_dict[unc](prob_array)