gwaslab 3.4.37__py3-none-any.whl → 3.4.39__py3-none-any.whl

gwaslab/util_in_get_sig.py

@@ -13,8 +13,9 @@ from gwaslab.bd_common_data import get_chr_to_NC
 from gwaslab.bd_common_data import gtf_to_protein_coding
 from gwaslab.bd_download import check_and_download
 from gwaslab.util_ex_gwascatalog import gwascatalog_trait
-
-
+from gwaslab.qc_fix_sumstats import check_dataframe_shape
+from gwaslab.qc_fix_sumstats import start_to
+from gwaslab.qc_fix_sumstats import finished
 # getsig
 # closest_gene
 # annogene
@@ -39,11 +40,27 @@ def getsig(insumstats,
     """
     Extract the lead variants using a sliding window. P or MLOG10P will be used and converted to SCALEDP for sorting. 
     """
+    ##start function with col checking##########################################################
+    _start_line = "extract lead variants"
+    _end_line = "extracting lead variants"
+    _start_cols = [chrom,pos]
+    _start_function = ".get_lead()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=insumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return None
+    ############################################################################################
 
-    if verbose: log.write("Start to extract lead variants...")
-    if verbose: log.write(" -Processing "+str(len(insumstats))+" variants...")
-    if verbose: log.write(" -Significance threshold :", sig_level)
-    if verbose: log.write(" -Sliding window size:", str(windowsizekb) ," kb")
+    log.write(" -Processing "+str(len(insumstats))+" variants...", verbose=verbose)
+    log.write(" -Significance threshold :", sig_level, verbose=verbose)
+    log.write(" -Sliding window size:", str(windowsizekb) ," kb", verbose=verbose)
     
     #load data
     sumstats=insumstats.loc[~insumstats[id].isna(),:].copy()
@@ -73,12 +90,12 @@ def getsig(insumstats,
         sumstats[p] = pd.to_numeric(sumstats[p], errors='coerce')
         sumstats_sig = sumstats.loc[sumstats[p]<sig_level,:].copy()
         sumstats_sig.loc[:,"__SCALEDP"] = pd.to_numeric(sumstats_sig[p], errors='coerce')
-    if verbose:log.write(" -Found "+str(len(sumstats_sig))+" significant variants in total...")
+    log.write(" -Found "+str(len(sumstats_sig))+" significant variants in total...", verbose=verbose)
 
     #sort the coordinates
     sumstats_sig = sumstats_sig.sort_values([chrom,pos])
     if sumstats_sig is None:
-        if verbose:log.write(" -No lead snps at given significance threshold!")
+        log.write(" -No lead snps at given significance threshold!", verbose=verbose)
         return None
     
     #init 
@@ -131,7 +148,7 @@ def getsig(insumstats,
             sig_index_list.append(current_sig_index)
             continue
     
-    if verbose:log.write(" -Identified "+str(len(sig_index_list))+" lead variants!")
+    log.write(" -Identified "+str(len(sig_index_list))+" lead variants!", verbose=verbose)
     
     # drop internal __SCALEDP
     sumstats_sig = sumstats_sig.drop("__SCALEDP",axis=1)
@@ -141,8 +158,8 @@ def getsig(insumstats,
 
     # annotate GENENAME
     if anno is True and len(output)>0:
-        if verbose:log.write(" -Annotating variants using references:{}".format(source))
-        if verbose:log.write(" -Annotating variants using references based on genome build:{}".format(build))
+        log.write(" -Annotating variants using references:{}".format(source), verbose=verbose)
+        log.write(" -Annotating variants using references based on genome build:{}".format(build), verbose=verbose)
         
         output = annogene(
                output,
@@ -155,11 +172,9 @@ def getsig(insumstats,
                source=source,
                verbose=verbose)
         
-    # Finishing
-    if verbose: log.write("Finished extracting lead variants successfully!")
     # drop internal id
     output = output.drop("__ID",axis=1)
-    gc.collect()
+    finished(log,verbose,_end_line)
     return output.copy()
 
 
@@ -234,13 +249,13 @@ def annogene(
            source="ensembl",
            verbose=True):
     
-    if verbose: log.write("Start to annotate variants with nearest gene name(s)...")
+    log.write("Start to annotate variants with nearest gene name(s)...", verbose=verbose)
     output = insumstats.copy()
     
     if source == "ensembl":
         if build=="19":
             #data = EnsemblRelease(75)
-            if verbose:log.write(" -Assigning Gene name using ensembl_hg19_gtf for protein coding genes")
+            log.write(" -Assigning Gene name using ensembl_hg19_gtf for protein coding genes", verbose=verbose)
             #zcat Homo_sapiens.GRCh37.75.gtf.gz| 
             #grep -E 'processed_transcript|protein_coding|_gene' 
             #| gzip >Homo_sapiens.GRCh37.75.processed.chr.gtf.gz     
@@ -260,7 +275,7 @@ def annogene(
                 list(output.apply(lambda x:closest_gene(x,data=data,chrom=chrom,pos=pos,source=source), axis=1)), 
                 index=output.index).values
         elif build=="38":
-            if verbose:log.write(" -Assigning Gene name using ensembl_hg38_gtf for protein coding genes")
+            log.write(" -Assigning Gene name using ensembl_hg38_gtf for protein coding genes", verbose=verbose)
             #gtf_path = check_and_download("ensembl_hg38_gtf_protein_coding")
             gtf_path = check_and_download("ensembl_hg38_gtf")
             gtf_path = gtf_to_protein_coding(gtf_path,log=log,verbose=verbose)
@@ -277,7 +292,7 @@ def annogene(
     
     if source == "refseq":
         if build=="19":
-            if verbose:log.write(" -Assigning Gene name using NCBI refseq latest GRCh37 for protein coding genes")
+            log.write(" -Assigning Gene name using NCBI refseq latest GRCh37 for protein coding genes", verbose=verbose)
             #gtf_path = check_and_download("refseq_hg19_gtf_protein_coding")
             gtf_path = check_and_download("refseq_hg19_gtf")
             gtf_path = gtf_to_protein_coding(gtf_path,log=log,verbose=verbose)
@@ -292,7 +307,7 @@ def annogene(
                 list(output.apply(lambda x:closest_gene(x,data=data,chrom=chrom,pos=pos,source=source,build=build), axis=1)), 
                 index=output.index).values
         elif build=="38":
-            if verbose:log.write(" -Assigning Gene name using NCBI refseq latest GRCh38 for protein coding genes")
+            log.write(" -Assigning Gene name using NCBI refseq latest GRCh38 for protein coding genes", verbose=verbose)
             #gtf_path = check_and_download("refseq_hg38_gtf_protein_coding")
             gtf_path = check_and_download("refseq_hg38_gtf")
             gtf_path = gtf_to_protein_coding(gtf_path,log=log,verbose=verbose)
@@ -306,7 +321,7 @@ def annogene(
             output.loc[:,["LOCATION","GENE"]] = pd.DataFrame(
                 list(output.apply(lambda x:closest_gene(x,data=data,chrom=chrom,pos=pos,source=source,build=build), axis=1)), 
                 index=output.index).values
-    if verbose: log.write("Finished annotating variants with nearest gene name(s) successfully!")
+    log.write("Finished annotating variants with nearest gene name(s) successfully!", verbose=verbose)
     return output
 
 def getnovel(insumstats,
@@ -318,6 +333,8 @@ def getnovel(insumstats,
            known=False,
            efo=False,
            only_novel=False,
+           group_key=None,
+           if_get_lead = True,
            windowsizekb_for_novel=1000,
            windowsizekb=500,
            sig_level=5e-8,
@@ -329,37 +346,48 @@ def getnovel(insumstats,
            gwascatalog_source="NCBI",
            output_known=False,
            verbose=True):
-    if verbose: log.write("Start to check if lead variants are known...")
-    allsig = getsig(insumstats=insumstats,
-           id=id,chrom=chrom,pos=pos,p=p,use_p=use_p,windowsizekb=windowsizekb,sig_level=sig_level,log=log,
-           xymt=xymt,anno=anno,build=build, source=source,verbose=verbose)
-    
-    big_number = 1000000000
-    for i in range(7):
-        if insumstats["POS"].max()*10 >  big_number:
-            big_number = int(big_number * 10)
-        else:
-            break
+    ##start function with col checking##########################################################
+    _start_line = "check if lead variants are known"
+    _end_line = "checking if lead variants are known"
+    _start_cols = [chrom,pos]
+    _start_function = ".get_novel()"
+    _must_args ={}
 
-    # create helper column TCHR+POS for allsig
-    allsig["TCHR+POS"]=allsig[chrom]*big_number + allsig[pos]
+    is_enough_info = start_to(sumstats=insumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return None
+    ############################################################################################
     
+    if if_get_lead == True:
+        allsig = getsig(insumstats=insumstats,
+            id=id,chrom=chrom,pos=pos,p=p,use_p=use_p,windowsizekb=windowsizekb,sig_level=sig_level,log=log,
+            xymt=xymt,anno=anno,build=build, source=source,verbose=verbose)
+    else:
+        allsig = insumstats.copy()
+
+    ############################################################################################
     knownsig = pd.DataFrame()
     if efo != False:
         if type(efo) is not list:
-            if verbose: log.write("Start to retrieve data using EFO: {}...".format(efo))
+            log.write("Start to retrieve data using EFO: {}...".format(efo), verbose=verbose)
             known_Sumstats = gwascatalog_trait(efo,source=gwascatalog_source,sig_level=sig_level,verbose=verbose,log=log)
             knownsig = known_Sumstats.data.copy()
         else:
             knownsig=pd.DataFrame()
-            if verbose: log.write("Start to retrieve data using {} EFOs: {}...".format(len(efo),efo))
+            log.write("Start to retrieve data using {} EFOs: {}...".format(len(efo),efo), verbose=verbose)
             for single_efo in efo:
                 known_Sumstats = gwascatalog_trait(single_efo,source=gwascatalog_source,sig_level=sig_level,verbose=verbose,log=log)
                 known_Sumstats.data["EFOID"] = single_efo
                 knownsig = pd.concat([known_Sumstats.data, knownsig],ignore_index=True)
         knownsig["CHR"] = knownsig["CHR"].astype("Int64")
         knownsig["POS"] = knownsig["POS"].astype("Int64")
-        if verbose: log.write(" -Retrieved {} associations from GWAS catalog.".format(len(knownsig)))
+        log.write(" -Retrieved {} associations from GWAS catalog.".format(len(knownsig)), verbose=verbose)
     if type(known) is pd.DataFrame:
         knownsig_2 = known.copy()
         knownsig = pd.concat([knownsig, knownsig_2],ignore_index=True)
@@ -374,19 +402,230 @@ def getnovel(insumstats,
         knownsig["POS"] = knownsig["POS"].astype("Int64")
         if "SNPID" not in knownsig.columns:
             knownsig["SNPID"] =knownsig["CHR"].astype("string") + ":" + knownsig["POS"].astype("string")
+    
     if len(knownsig)<1:
         raise ValueError("Please input a dataframe of known loci or valid efo code")
-
-    # create helper column TCHR+POS for knownsig
-    knownsig["TCHR+POS"]=knownsig[chrom]*big_number + knownsig[pos]
-    
-    if verbose: log.write(" -Lead variants in known loci:",len(knownsig))
-    if verbose: log.write(" -Checking the minimum distance between identified lead variants and provided known variants...")
     
+    if group_key is not None:
+        if (group_key not in allsig.columns) or (group_key not in knownsig.columns):
+            raise ValueError("Please check if group_key is in both sumstats and list of known associations.")
+
+    # create helper column TCHR+POS for knownsig and all sig
+     ############################################################################################
+    maxpos = insumstats["POS"].max()
+    big_number = determine_big_number(maxpos)
+    knownsig = add_tchr_pos(knownsig, chrom, pos, big_number)
+    allsig = add_tchr_pos(allsig, chrom, pos, big_number)
+    ############################################################################################   
     #sorting
     allsig = allsig.sort_values(by="TCHR+POS",ignore_index=True)
     knownsig = knownsig.sort_values(by="TCHR+POS",ignore_index=True)
+    ############################################################################################
+    if group_key is not None:
+        number_of_groups_allsig = allsig[group_key].nunique()
+        number_of_groups_known = knownsig[group_key].nunique()
+        log.write(" -Number of groups in sumstats:{}".format(number_of_groups_allsig), verbose=verbose)
+        log.write(" -Number of groups in reference:{}".format(number_of_groups_known), verbose=verbose)
+
+    log.write(" -Lead variants in known loci:",len(knownsig), verbose=verbose)
+    log.write(" -Checking the minimum distance between identified lead variants and provided known variants...", verbose=verbose)
+    
+    ############################################################################################
+    if group_key is None:
+        # get distance
+        allsig = determine_distance(allsig, knownsig)
+        # get other info 
+        allsig = fill_meta_info_for_known(allsig, knownsig)
+        ############################################################################################
+        # determine if novel
+        allsig = determine_novel(allsig, windowsizekb_for_novel)
+        # determine location
+        allsig = determine_location(allsig)
+        # if not on same chromosome, distance set to pd.NA
+        allsig = determine_if_same_chromosome(allsig, knownsig, maxpos)
+        ############################################################################################
+    else:
+        #groups1 = set(allsig[group_key].unique())
+        #groups2 = set(knownsig[group_key].unique())
+        #common_group = groups1.intersection(groups2)
+        
+        #allsig_no_group = allsig.loc[~allsig[group_key].isin(common_group),:].copy()
+        allsig_group = pd.DataFrame()
+
+        for key in allsig[group_key].unique():
+            allsig_single_group = allsig.loc[allsig[group_key]==key,:].copy()
+            knownsig_single_group = knownsig.loc[knownsig[group_key]==key,:].copy()
+
+            #if len(allsig_single_group) >0 and len(knownsig_single_group) >0:
+            allsig_single_group = determine_distance(allsig_single_group, knownsig_single_group)
+            # get other info 
+            allsig_single_group = fill_meta_info_for_known(allsig_single_group, knownsig_single_group)
+            
+            # determine if novel
+            allsig_single_group = determine_novel(allsig_single_group, windowsizekb_for_novel)
+            
+            # determine location
+            allsig_single_group = determine_location(allsig_single_group)
+            
+            # if not on same chromosome, distance set to pd.NA
+            allsig_single_group = determine_if_same_chromosome(allsig_single_group, knownsig_single_group, maxpos) 
+            
+            allsig_group = pd.concat([allsig_group, allsig_single_group], ignore_index=True)
+        
+        allsig = allsig_group
+        #pd.concat([allsig_no_group, allsig_group], ignore_index=True)
+        
+    # drop helper column TCHR+POS
+    allsig = allsig.drop(["TCHR+POS"], axis=1)
+    
+    try:
+        allsig = allsig.where(~pd.isna(allsig), pd.NA)
+    except:
+        pass
+
+    log.write(" -Identified ",len(allsig)-sum(allsig["NOVEL"])," known vairants in current sumstats...", verbose=verbose)
+    log.write(" -Identified ",sum(allsig["NOVEL"])," novel vairants in current sumstats...", verbose=verbose)
+    
+    finished(log,verbose,_end_line)
+    
+    # how to return
+    if only_novel is True:
+        if output_known is True:
+            return allsig.loc[allsig["NOVEL"],:], knownsig
+        else:
+            return allsig.loc[allsig["NOVEL"],:]
+    else:
+        if output_known is True:
+            return allsig, knownsig
+        else:
+            return allsig
+##################################################################################################################################################################################################
+
+
+def _check_cis(insumstats,
+           id,
+           chrom,
+           pos,
+           p,
+           use_p=False,
+           known=False,
+           group_key=None,
+           if_get_lead = False,
+           windowsizekb=500,
+           sig_level=5e-8,
+           log=Log(),
+           xymt=["X","Y","MT"],
+           anno=False,
+           build="19",
+           source="ensembl",
+           verbose=True):
+    ##start function with col checking##########################################################
+    _start_line = "check if variants are in cis or trans regions"
+    _end_line = "checking if variants are in cis or trans regions"
+    _start_cols = [chrom,pos, group_key]
+    _start_function = ".check_cis()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=insumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return None
+    ############################################################################################
     
+    if if_get_lead == True:
+        allsig = getsig(insumstats=insumstats,
+            id=id,chrom=chrom,pos=pos,p=p,use_p=use_p,windowsizekb=windowsizekb,sig_level=sig_level,log=log,
+            xymt=xymt,anno=anno,build=build, source=source,verbose=verbose)
+    else:
+        allsig = insumstats.copy()
+
+    ############################################################################################
+    knownsig = pd.DataFrame()
+    if type(known) is pd.DataFrame:
+        knownsig_2 = known.copy()
+        knownsig = pd.concat([knownsig, knownsig_2],ignore_index=True)
+        knownsig["CHR"] = knownsig["CHR"].astype("Int64")
+        knownsig["START"] = knownsig["START"].astype("Int64")
+        knownsig["END"] = knownsig["END"].astype("Int64")
+    elif type(known) is str:
+        knownsig_2 = pd.read_csv(known,sep="\s+",dtype={"CHR":"Int64","POS":"Int64"})
+        knownsig = pd.concat([knownsig, knownsig_2],ignore_index=True)
+        knownsig["CHR"] = knownsig["CHR"].astype("Int64")
+        knownsig["START"] = knownsig["START"].astype("Int64")
+        knownsig["END"] = knownsig["END"].astype("Int64")
+    
+    if len(knownsig)<1:
+        raise ValueError("Please input a dataframe of gene list with GENE, CHR, START, END.")
+    
+    if group_key is not None:
+        if group_key not in knownsig.columns:
+            raise ValueError("Please check if group_key is in both sumstats and list of known associations.")
+
+    ############################################################################################
+    if group_key is not None:
+        number_of_groups_allsig = allsig[group_key].nunique()
+        number_of_groups_known = knownsig[group_key].nunique()
+        log.write(" -Number of groups in sumstats:{}".format(number_of_groups_allsig), verbose=verbose)
+        log.write(" -Number of groups in reference:{}".format(number_of_groups_known), verbose=verbose)
+
+    log.write(" -Checking if variants in cis/trans regions grouped by {}...".format(group_key), verbose=verbose)
+    log.write(" -Window size in kb adding to start and end: {}...".format(windowsizekb), verbose=verbose)
+    ############################################################################################
+    #convert to  a dict
+    reference_dict = {}
+    for index,row in knownsig.iterrows():
+        reference_dict[row[group_key]] = (row["CHR"], row["START"], row["END"] )
+    ############################################################################################
+    try:
+        no_reference_avaialble = allsig.loc[~allsig[group_key].isin(reference_dict.keys()),group_key]
+        if len(no_reference_avaialble)>0:
+            log.write(" -Groups not in reference: {}".format( ",".join(no_reference_avaialble.unique())), verbose=verbose)
+    except:
+        pass
+
+    allsig["CIS/TRANS"] = allsig.apply(lambda x: determine_if_cis(x, group_key,windowsizekb, reference_dict), axis=1)
+    
+    try:
+        allsig = allsig.where(~pd.isna(allsig), pd.NA)
+    except:
+        pass
+    
+    try:
+        number_of_cis = sum(allsig["CIS/TRANS"] == "Cis")
+        number_of_trans = sum(allsig["CIS/TRANS"] == "Trans")
+        number_of_noreference = sum(allsig["CIS/TRANS"] == "NoReference")
+        log.write (" -Number of Cis variants: {}".format(number_of_cis),verbose=verbose)
+        log.write (" -Number of Trans variants: {}".format(number_of_trans),verbose=verbose)
+        log.write (" -Number of NoReference variants: {}".format(number_of_noreference),verbose=verbose)
+    except:
+        pass
+
+    finished(log,verbose,_end_line)
+    
+    return allsig
+
+###################################################################################################################################################################################################
+
+
+def determine_big_number(maxpos, big_number = 1000000000):
+    for i in range(7):
+        if maxpos*10 >  big_number:
+            big_number = int(big_number * 10)
+        else:
+            break
+    return big_number
+
+def add_tchr_pos(df, chrom, pos, big_number):
+    df["TCHR+POS"]=df[chrom]*big_number + df[pos]
+    return df
+
+def fill_meta_info_for_known(allsig, knownsig):
+    if len(allsig)==0 or len(knownsig)==0: return allsig
     if "SNPID" in knownsig.columns:
         knownids=knownsig["SNPID"].values
     if "PUBMEDID" in knownsig.columns:
@@ -395,12 +634,7 @@ def getnovel(insumstats,
         knownauthor=knownsig["AUTHOR"].values
     if "EFOID" in knownsig.columns:
         knownefo=knownsig["EFOID"].values
-    
-    # get distance
-    lambda x:np.min(np.abs(knownsig["TCHR+POS"]-x))
-    allsig["DISTANCE_TO_KNOWN"] = allsig["TCHR+POS"].apply(lambda x:min(knownsig["TCHR+POS"]-x, key=abs))
-    
-    # get other info 
+
     if "SNPID" in knownsig.columns:
         allsig["KNOWN_ID"] = allsig["TCHR+POS"].apply(lambda x:knownids[np.argmin(np.abs(knownsig["TCHR+POS"]-x))])    
     if "PUBMEDID" in knownsig.columns:
@@ -408,20 +642,48 @@ def getnovel(insumstats,
     if "AUTHOR" in knownsig.columns:
         allsig["KNOWN_AUTHOR"] = allsig["TCHR+POS"].apply(lambda x:knownauthor[np.argmin(np.abs(knownsig["TCHR+POS"]-x))])
     if "EFOID" in knownsig.columns:
-        allsig["KNOWN_EFOID"] = allsig["TCHR+POS"].apply(lambda x:knownefo[np.argmin(np.abs(knownsig["TCHR+POS"]-x))])
+        allsig["KNOWN_EFOID"] = allsig["TCHR+POS"].apply(lambda x:knownefo[np.argmin(np.abs(knownsig["TCHR+POS"]-x))])    
+    return allsig
+
+def determine_if_cis(x, group_key,windowsizekb, reference_dict):
+    if x[group_key] in reference_dict.keys():
+        is_same_chr = str(reference_dict[x[group_key]][0]) == str(x["CHR"])
+        is_large_than_start = int(reference_dict[x[group_key]][1]) - windowsizekb*1000 <= x["POS"]
+        is_smaller_than_end = int(reference_dict[x[group_key]][2]) + windowsizekb*1000 >= x["POS"]               
+        
+        if  is_same_chr and is_large_than_start  and is_smaller_than_end:
+            return "Cis"
+        else: 
+            return "Trans"
+    else:
+        return "NoReference"
 
-    # determine if novel
+def determine_distance(allsig, knownsig):
+    if len(allsig)==0: 
+        return allsig
+    if len(knownsig)==0:
+        allsig["DISTANCE_TO_KNOWN"] = pd.NA
+        return allsig
+    allsig["DISTANCE_TO_KNOWN"] = allsig["TCHR+POS"].apply(lambda x:min(knownsig["TCHR+POS"]-x, key=abs))
+    return allsig
+
+def determine_novel(allsig, windowsizekb_for_novel):
+    if len(allsig)==0 or "DISTANCE_TO_KNOWN" not in allsig.columns:
+        return allsig
     allsig["NOVEL"] = allsig["DISTANCE_TO_KNOWN"].abs() > windowsizekb_for_novel*1000
-    
-    # determine location
-    allsig["LOCATION_OF_KNOWN"]="Unknown"
+    allsig.loc[allsig["DISTANCE_TO_KNOWN"].isna(), "NOVEL"] = True
+    return allsig
+
+def determine_location(allsig):
+    allsig["LOCATION_OF_KNOWN"]="NoReference"
     allsig.loc[ allsig["DISTANCE_TO_KNOWN"]== 0,"LOCATION_OF_KNOWN"] = "Same"
     allsig.loc[ allsig["DISTANCE_TO_KNOWN"] > 0 ,"LOCATION_OF_KNOWN"] = "Upstream"
     allsig.loc[ allsig["DISTANCE_TO_KNOWN"] < 0 ,"LOCATION_OF_KNOWN"] = "Downstream"
+    return allsig
 
-    # if not on same chromosome, distance set to pd.NA
-    if sum(allsig["DISTANCE_TO_KNOWN"].abs() > insumstats["POS"].max())>0:
-        not_on_same_chromosome = allsig["DISTANCE_TO_KNOWN"].abs() > insumstats["POS"].max()
+def determine_if_same_chromosome(allsig, knownsig, maxpos):
+    if sum(allsig["DISTANCE_TO_KNOWN"].abs() > maxpos)>0:
+        not_on_same_chromosome = allsig["DISTANCE_TO_KNOWN"].abs() > maxpos
         allsig.loc[ not_on_same_chromosome ,"DISTANCE_TO_KNOWN"] = pd.NA
         allsig.loc[ not_on_same_chromosome ,"LOCATION_OF_KNOWN"] = "NoneOnThisChr"
         if "SNPID" in knownsig.columns:
@@ -432,23 +694,135 @@ def getnovel(insumstats,
             allsig.loc[ not_on_same_chromosome ,"KNOWN_AUTHOR"] = pd.NA
         if "EFOID" in knownsig.columns:
             allsig.loc[ not_on_same_chromosome ,"KNOWN_EFOID"] = pd.NA
+    return allsig
 
-    # drop helper column TCHR+POS
-    allsig = allsig.drop(["TCHR+POS"], axis=1)
+def _check_novel_set(insumstats,
+           id,
+           chrom,
+           pos,
+           p,
+           use_p=False,
+           known=False,
+           group_key=None,
+           snpset="SNPSET",
+           snpid="SNPID",
+           if_get_lead = False,
+           windowsizekb=500,
+           sig_level=5e-8,
+           log=Log(),
+           xymt=["X","Y","MT"],
+           anno=False,
+           build="19",
+           source="ensembl",
+           verbose=True):
+    
+    ##start function with col checking##########################################################
+    _start_line = "check if variant sets are overlapping with those in reference file"
+    _end_line = "checking if variant sets are overlapping with those in reference file"
+    _start_cols = [chrom,pos, group_key]
+    _start_function = ".check_cis()"
+    _must_args ={}
 
-    if verbose: log.write(" -Identified ",len(allsig)-sum(allsig["NOVEL"])," known vairants in current sumstats...")
-    if verbose: log.write(" -Identified ",sum(allsig["NOVEL"])," novel vairants in current sumstats...")
-    if verbose: log.write("Finished checking known or novel successfully!")
-    gc.collect()
+    is_enough_info = start_to(sumstats=insumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return None
+    ############################################################################################
     
-    # how to return
-    if only_novel is True:
-        if output_known is True:
-            return allsig.loc[allsig["NOVEL"],:], knownsig
-        else:
-            return allsig.loc[allsig["NOVEL"],:]
+    if if_get_lead == True:
+        allsig = getsig(insumstats=insumstats,
+            id=id,chrom=chrom,pos=pos,p=p,use_p=use_p,windowsizekb=windowsizekb,sig_level=sig_level,log=log,
+            xymt=xymt,anno=anno,build=build, source=source,verbose=verbose)
     else:
-        if output_known is True:
-            return allsig, knownsig
+        allsig = insumstats.copy()
+
+    ############################################################################################
+    knownsig = pd.DataFrame()
+    if type(known) is pd.DataFrame:
+        knownsig_2 = known.copy()
+        knownsig = pd.concat([knownsig, knownsig_2],ignore_index=True)
+        knownsig[snpid] = knownsig[snpid].astype("string")
+        knownsig[snpset] = knownsig[snpset].astype("string")
+        knownsig[group_key] = knownsig[group_key].astype("string")
+    elif type(known) is str:
+        knownsig_2 = pd.read_csv(known,sep="\s+",dtype={"CHR":"Int64","POS":"Int64"})
+        knownsig = pd.concat([knownsig, knownsig_2],ignore_index=True)
+        knownsig[snpid] = knownsig[snpid].astype("string")
+        knownsig[snpset] = knownsig[snpset].astype("string")
+        knownsig[group_key] = knownsig[group_key].astype("string")
+    
+    if len(knownsig)<1:
+        raise ValueError("Please input a dataframe of gene list with GENE, CHR, START, END.")
+    
+    if group_key is not None:
+        if group_key not in knownsig.columns:
+            raise ValueError("Please check if group_key is in both sumstats and list of known associations.")
+
+    ############################################################################################
+    if group_key is not None:
+        number_of_groups_allsig = allsig[group_key].nunique()
+        number_of_groups_known = knownsig[group_key].nunique()
+        log.write(" -Number of groups in sumstats:{}".format(number_of_groups_allsig), verbose=verbose)
+        log.write(" -Number of groups in reference:{}".format(number_of_groups_known), verbose=verbose)
+
+    log.write(" -Checking if variants in cis/trans regions grouped by {}...".format(group_key), verbose=verbose)
+    
+    ############################################################################################
+    #convert to  a dict
+    reference_dict = {}
+
+    for index,row in knownsig.iterrows():
+        if row[group_key] in reference_dict.keys():
+            if row[snpset] in reference_dict[row[group_key]].keys():
+                reference_dict[row[group_key]][row[snpset]].add(row[snpid])
+            else:
+                reference_dict[row[group_key]][row[snpset]] = set([row[snpid]])
         else:
-            return allsig
+            reference_dict[row[group_key]] = {row[snpset]:set([row[snpid]])}
+    ############################################################################################
+    
+    try:
+        no_reference_avaialble = allsig.loc[~allsig[group_key].isin(reference_dict.keys()),group_key]
+        if len(no_reference_avaialble)>0:
+            log.write(" -Groups not in reference: {}".format( ",".join(no_reference_avaialble)), verbose=verbose)
+    except:
+        pass
+
+    log.write(" -Checking if variants are in reference variant sets...", verbose=verbose)    
+    known_list = allsig.apply(lambda x: check_overlap(x,snpid, group_key,reference_dict), axis=1)
+    
+    allsig["KNOWN_SET"] = known_list.str[0]
+    allsig["KNOWN_VARIANT"] = known_list.str[1]
+
+    back_dict={}
+    for i in allsig[group_key].unique():
+        back_dict[i] ={}
+        for j in allsig.loc[allsig[group_key]==i,snpset].unique():
+            back_dict[i][j] =set()
+            for index, row in allsig.loc[(allsig[group_key]==i) & (allsig[snpset]==j) & (~allsig["KNOWN_SET"].isna()),:].iterrows():
+                back_dict[i][j].add("{}-{}-{}".format(row[group_key], row["KNOWN_SET"],row["KNOWN_VARIANT"]))
+
+    allsig["KNOWN_SET_VARIANT"] = allsig.apply(lambda x: assign_set_variant(x,group_key,snpset,back_dict), axis=1)
+    
+    finished(log,verbose,_end_line)
+    
+    return allsig
+
+def check_overlap(x,snpid, group_key,reference_dict):
+    if x[group_key] in reference_dict.keys():
+        for key, value in reference_dict[x[group_key]].items():
+            if x[snpid] in value:
+                return key, x[snpid]
+    return pd.NA, pd.NA, 
+
+def assign_set_variant(x,group_key,snpset,back_dict):
+    if x[group_key] in back_dict.keys():
+        if x[snpset] in back_dict[x[group_key]].keys():
+            if len(back_dict[x[group_key]][x[snpset]]) >0:
+                return back_dict[x[group_key]][x[snpset]]
+    return pd.NA