gwaslab 3.4.41__py3-none-any.whl → 3.4.43__py3-none-any.whl

gwaslab/hm_harmonize_sumstats.py

@@ -21,8 +21,10 @@ from gwaslab.qc_check_datatype import check_dataframe_shape
 from gwaslab.bd_common_data import get_number_to_chr
 from gwaslab.bd_common_data import get_chr_list
 from gwaslab.bd_common_data import get_chr_to_number
+from gwaslab.bd_common_data import _maketrans
 from gwaslab.g_vchange_status import vchange_status
 from gwaslab.g_version import _get_version
+from gwaslab.cache_manager import CacheManager, PALINDROMIC_INDEL, NON_PALINDROMIC
 
 #rsidtochrpos
 #checkref
@@ -30,6 +32,34 @@ from gwaslab.g_version import _get_version
 #inferstrand
 #parallelecheckaf
 
+### CONSTANTS AND MAPPINGS ###
+
+PADDING_VALUE = 100
+
+# chr(0) should not be used in the mapping dict because it's a reserved value.
+# Instead of starting from chr(1), we start from chr(2) because this could be useful in the future
+# to compute the complementary allele with a simple XOR operation (e.g. 2 ^ 1 = 3, 3 ^ 1 = 2, 4 ^ 1 = 5, 5 ^ 1 = 4, ...)
+MAPPING = {
+    "A": chr(2),
+    "T": chr(3),
+    "C": chr(4),
+    "G": chr(5),
+    "N": chr(6),
+}
+assert all(value != chr(0) for value in MAPPING.values()), "Mapping in the dictionary should not be equal to chr(0). This is a reserved value"
+
+_COMPLEMENTARY_MAPPING = {
+    "A": "T",
+    "C": "G",
+    "G": "C",
+    "T": "A",
+    "N": "N",
+}
+COMPLEMENTARY_MAPPING = {k: MAPPING[v] for k,v in _COMPLEMENTARY_MAPPING.items()}
+
+TRANSLATE_TABLE = _maketrans(MAPPING)
+TRANSLATE_TABLE_COMPL = _maketrans(COMPLEMENTARY_MAPPING)
+
 #20220808
 #################################################################################################################
 
@@ -44,7 +74,7 @@ def rsidtochrpos(sumstats,
     ##start function with col checking##########################################################
     _start_line = "assign CHR and POS using rsIDs"
     _end_line = "assigning CHR and POS using rsIDs"
-    _start_cols = [rsid,chrom,pos]
+    _start_cols = [rsid]
     _start_function = ".rsid_to_chrpos()"
     _must_args ={}
 
@@ -131,7 +161,7 @@ def parallelrsidtochrpos(sumstats, rsid="rsID", chrom="CHR",pos="POS", path=None
     ##start function with col checking##########################################################
     _start_line = "assign CHR and POS using rsIDs"
     _end_line = "assigning CHR and POS using rsIDs"
-    _start_cols = [rsid,chrom,pos]
+    _start_cols = [rsid]
     _start_function = ".rsid_to_chrpos2()"
     _must_args ={}
 
@@ -186,7 +216,7 @@ def parallelrsidtochrpos(sumstats, rsid="rsID", chrom="CHR",pos="POS", path=None
     pool = Pool(n_cores)
     if chrom not in input_columns:
         log.write(" -Initiating CHR ... ",verbose=verbose)
-        sumstats_rs[chrom]=pd.Series(dtype="Int32") 
+        sumstats_rs[chrom]=pd.Series(dtype="Int64") 
         
     if pos not in input_columns:
         log.write(" -Initiating POS ... ",verbose=verbose)
@@ -207,7 +237,7 @@ def parallelrsidtochrpos(sumstats, rsid="rsID", chrom="CHR",pos="POS", path=None
 
     # update CHR and POS using rsID with multiple threads
     sumstats_rs = pd.concat(pool.map(partial(merge_chrpos,all_groups_max=all_groups_max,path=path,build=build,status=status),df_split),ignore_index=True)
-    sumstats_rs.loc[:,["CHR","POS"]] = sumstats_rs.loc[:,["CHR","POS"]].astype("Int64")
+    sumstats_rs[["CHR","POS"]] = sumstats_rs[["CHR","POS"]].astype("Int64")
     del df_split
     gc.collect()
     log.write(" -Merging group data... ",verbose=verbose)
@@ -234,8 +264,8 @@ def parallelrsidtochrpos(sumstats, rsid="rsID", chrom="CHR",pos="POS", path=None
     finished(log, verbose, _end_line)
     return sumstats
 ####################################################################################################################
-#20220426 check if non-effect allele is aligned with reference genome 
-def check_status(row,record):
+# old version
+def _old_check_status(row,record):
     #pos,ea,nea
     # status 
     #0 /  ----->  match
@@ -288,16 +318,14 @@ def check_status(row,record):
                     return status_pre+"5"+status_end 
             # ea !=ref
             return status_pre+"8"+status_end
-        
 
-def checkref(sumstats,ref_path,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="STATUS",chr_dict=get_chr_to_number(),remove=False,verbose=True,log=Log()):
+def oldcheckref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="STATUS",chr_dict=get_chr_to_number(),remove=False,verbose=True,log=Log()):
     ##start function with col checking##########################################################
     _start_line = "check if NEA is aligned with reference sequence"
     _end_line = "checking if NEA is aligned with reference sequence"
     _start_cols = [chrom,pos,ea,nea,status]
     _start_function = ".check_ref()"
     _must_args ={}
-
     is_enough_info = start_to(sumstats=sumstats,
                               log=log,
                               verbose=verbose,
@@ -308,10 +336,10 @@ def checkref(sumstats,ref_path,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="S
                               **_must_args)
     if is_enough_info == False: return sumstats
     ############################################################################################
-    log.write(" -Reference genome FASTA file: "+ ref_path,verbose=verbose)  
+    log.write(" -Reference genome FASTA file: "+ ref_seq,verbose=verbose)  
     log.write(" -Checking records: ", end="",verbose=verbose)  
     chromlist = get_chr_list(add_number=True)
-    records = SeqIO.parse(ref_path, "fasta")
+    records = SeqIO.parse(ref_seq, "fasta")
     for record in records:
         #record = next(records)
         if record is not None:
@@ -323,7 +351,7 @@ def checkref(sumstats,ref_path,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="S
             if i in chromlist:
                 log.write(record_chr," ", end="",show_time=False,verbose=verbose) 
                 to_check_ref = (sumstats[chrom]==i) & (~sumstats[pos].isna()) & (~sumstats[nea].isna()) & (~sumstats[ea].isna())
-                sumstats.loc[to_check_ref,status] = sumstats.loc[to_check_ref,[pos,ea,nea,status]].apply(lambda x:check_status(x,record),axis=1)
+                sumstats.loc[to_check_ref,status] = sumstats.loc[to_check_ref,[pos,ea,nea,status]].apply(lambda x:_old_check_status(x,record),axis=1)
     
     log.write("\n",end="",show_time=False,verbose=verbose) 
         
@@ -360,6 +388,332 @@ def checkref(sumstats,ref_path,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="S
     finished(log, verbose, _end_line)
     return sumstats
 
+#20240320 check if non-effect allele is aligned with reference genome         
+def _fast_check_status(x: pd.DataFrame, record: np.array, starting_positions: np.array):
+    # status 
+    #0 /  ----->  match
+    #1 /  ----->  Flipped Fixed
+    #2 /  ----->  Reverse_complementary Fixed
+    #3 /  ----->  flipped
+    #4 /  ----->  reverse_complementary 
+    #5 / ------>  reverse_complementary + flipped
+    #6 /  ----->  both allele on genome + unable to distinguish
+    #7 /  ----> reverse_complementary + both allele on genome + unable to distinguish
+    #8 / -----> not on ref genome
+    #9 / ------> unchecked
+    if x.empty:
+        return np.array([])
+    
+    # x is expected to be a DataFrame with these columns in that order: ['CHR', 'POS', 'EA', 'NEA', 'STATUS']
+    # In this way, we don't need to specify the columns names
+    _chrom = x.iloc[:, 0]
+    _pos = x.iloc[:, 1]
+    _ea = x.iloc[:, 2]
+    _nea = x.iloc[:, 3]
+    _status = x.iloc[:, 4]
+
+    # position of the status (i.e. x['STATUS']) that will be modified
+    status_flip_idx = 5 
+
+    pos = _pos.values.astype(np.int64) # convert to int64 because they could be of type 'object'
+
+    # Rebase the chromosome numbers to 0-based indexing
+    # e.g. ['1', '2', '4', '2'] -> [0, 1, 2, 1]
+    # This is needed because record is a single 1D array containing all the records for all the selected chromosomes,
+    # so for instance if record contains the records for chr1, chr2, chr4 ([...chr1...chr2...chr4...]), we need to 
+    # rebase the chromosome numbers to 0-based indexing to index the correct record portion when we do starting_positions[chrom]
+    # Note that in x there are only the rows for the same chromosomes for which we have the records in record
+    # (i.e. we don't have rows for chr3 if we don't have the record for chr3). This filtering is done in the caller function
+    _chrom = _chrom.values
+    unique_values, _ = np.unique(_chrom, return_inverse=True) # Get the sorted unique values and their indices
+    chrom = np.searchsorted(unique_values, _chrom) # Replace each value in '_chrom' with its corresponding index in the sorted unique values
+
+    max_len_nea = _nea.str.len().max()
+    max_len_ea = _ea.str.len().max()
+
+
+    # Let's apply the same magic used for the fasta records (check build_fasta_records() for details) to convert the NEA and EA to
+    # a numpy array of integers in a very fast way.
+    # In that case we start from a pd.Series to we can apply some built-in methods.
+    # Also, when doing nea.view('<u4'), each row will be automatically right-padded with zeros to reach the max_len_nea.
+    # For this reason, we then replace the zeros with out padding value
+    # (and that's why the mapping dict can't have chr(0) as a value, otherwise we would have zeros for both padding and a character)
+    # Reshaping is needed because .view('<u4') will create a flattened array    
+    nea = _nea.str.translate(TRANSLATE_TABLE).to_numpy().astype(f'<U{max_len_nea}')
+    nea = nea.view('<u4').reshape(-1, max_len_nea).astype(np.uint8)
+    nea[nea == 0] = PADDING_VALUE # padding value
+
+    # Create a mask holding True at the position of non-padding values
+    mask_nea = nea != PADDING_VALUE
+
+    # Create the reverse complement of NEA
+    # In this case, we manually left-pad the translated string with the padding value, since the padding done by view('<u4') would be right-padded
+    # and that will make hard the reverse operation (because we would have e.g. [2, 2, 4, 100, ..., 100] which will be hard to convert into [4, 2, 2, 100, ..., 100])
+    rev_nea = _nea.str.translate(TRANSLATE_TABLE_COMPL).str.pad(max_len_nea, 'left', chr(PADDING_VALUE)).to_numpy().astype(f'<U{max_len_nea}')
+    rev_nea = rev_nea.view('<u4').reshape(-1, max_len_nea).astype(np.uint8)
+    rev_nea = rev_nea[:, ::-1]
+
+
+    # Let's do everything again for EA
+    ea = _ea.str.translate(TRANSLATE_TABLE).to_numpy().astype(f'<U{max_len_ea}')
+    ea = ea.view('<u4').reshape(-1, max_len_ea).astype(np.uint8)
+    ea[ea == 0] = PADDING_VALUE # padding value
+
+    mask_ea = ea != PADDING_VALUE
+
+    rev_ea = _ea.str.translate(TRANSLATE_TABLE_COMPL).str.pad(max_len_ea, 'left', chr(PADDING_VALUE)).to_numpy().astype(f'<U{max_len_ea}')
+    rev_ea = rev_ea.view('<u4').reshape(-1, max_len_ea).astype(np.uint8)
+    rev_ea = rev_ea[:, ::-1]
+
+
+    # Convert the status (which are integers represented as strings) to a numpy array of integers.
+    # Again, use the same concept as before to do this in a very fast way.
+    # e.g. ["9999999", "9939999", "9929999"] -> [[9, 9, 9, 9, 9, 9, 9], [9, 9, 3, 9, 9, 9, 9], [9, 9, 2, 9, 9, 9, 9]]
+    assert _status.str.len().value_counts().nunique() == 1 # all the status strings should have the same length, let's be sure of that.
+    status_len = len(_status.iloc[0])
+    mapping_status = {str(v): chr(v) for v in range(10)}
+    table_stats = _maketrans(mapping_status)
+    status = _status.str.translate(table_stats).to_numpy().astype(f'<U{status_len}')
+    status = status.view('<u4').reshape(-1, status_len).astype(np.uint8)
+
+
+    # Expand the position to a 2D array and subtract 1 to convert to 0-based indexing
+    # e.g. [2, 21, 46] -> [[1], [20], [45]]
+    pos = np.expand_dims(pos, axis=-1) - 1
+
+    # Create a modified indices array specifying the starting position of each chromosome in the concatenated record array
+    modified_indices = starting_positions[chrom]
+    modified_indices = modified_indices[:, np.newaxis] # Add a new axis to modified_indices to align with the dimensions of pos
+
+    # Create the range of indices: [0, ..., max_len_nea-1]
+    indices_range = np.arange(max_len_nea)
+
+    # Add the range of indices to the starting indices
+    # e.g. pos = [[1], [20], [45]], indices_range = [0, 1, 2], indices = [[1, 2, 3], [20, 21, 22], [45, 46, 47]]
+    indices = pos + indices_range
+
+    # Modify indices to select the correct absolute position in the concatenated record array
+    indices = indices + modified_indices
+
+    # Let's pad the fasta records array because if there is a (pos, chrom) for which (pos+starting_position[chrom]+max_len_nea > len(record) we get out of bounds error.
+    # This basically happens if there is a pos for the last chromosome for which pos+max_len_nea > len(record for that chrom).
+    # This is very unlikely to happen but we should handle this case.
+    record = np.pad(record, (0, max_len_nea), constant_values=PADDING_VALUE)
+    
+    # Index the record array using the computed indices.
+    # Since we use np.take, indices must all have the same length, and this is why we added the padding to NEA
+    # and we create the indices using max_len_nea (long story short, we can't obtain a scattered/ragged array)
+    output_nea = np.take(record, indices)
+
+    # Check if the NEA is equal to the reference sequence at the given position
+    # In a non-matrix way, this is equivalent (for one single element) to:
+    # nea == record[pos-1: pos+len(nea)-1]
+    # where for example:
+    #  a) nea = "AC", record = "ACTG", pos = 1 -> True
+    #  b) nea = "T", record = "ACTG", pos = 3 -> True
+    #  c) nea = "AG", record = "ACTG", pos = 1 -> False
+    # Since we want to do everything in a vectorized way, we will compare the padded NEA with the output 
+    # and then we use the mask to focus only on the non-padded elements
+    # Pseudo example (X represents the padding value):
+    #  nea = ['AC', 'T'], record = 'ACTGAAG', pos = [1, 3]
+    #  -> nea = ['AC', 'TX'], indices = [[1, 2], [3, 4]], mask = [[True, True], [True, False]], output_nea = [['A', 'C'], ['T', 'G']]
+    #  -> nea == output_nea: [[True, True], [True, False]], mask: [[True, True], [True, False]]
+    #  -> nea == output_nea + ~mask: [[True, True], [True, True]]
+    #  -> np.all(nea == output_nea + ~mask, 1): [True, True]
+    nea_eq_ref = np.all((nea == output_nea) + ~mask_nea, 1)
+    rev_nea_eq_ref = np.all((rev_nea == output_nea) + ~mask_nea, 1)
+
+    # Let's do everything again for EA
+    indices_range = np.arange(max_len_ea)
+    indices = pos + indices_range
+    indices = indices + modified_indices
+    output_ea = np.take(record, indices)
+
+    ea_eq_ref = np.all((ea == output_ea) + ~mask_ea, 1)
+    rev_ea_eq_ref = np.all((rev_ea == output_ea) + ~mask_ea, 1)
+
+    masks_max_len = max(mask_nea.shape[1], mask_ea.shape[1])
+
+    len_nea_eq_len_ea = np.all(
+        np.pad(mask_nea, ((0,0),(0, masks_max_len-mask_nea.shape[1])), constant_values=False) == 
+        np.pad(mask_ea, ((0,0),(0, masks_max_len-mask_ea.shape[1])), constant_values=False)
+        , axis=1) # pad masks with False to reach same shape
+    len_rev_nea_eq_rev_len_ea = len_nea_eq_len_ea
+
+    # The following conditions replicates the if-else statements of the original check_status function:
+    # https://github.com/Cloufield/gwaslab/blob/f6b4c4e58a26e5d67d6587141cde27acf9ce2a11/src/gwaslab/hm_harmonize_sumstats.py#L238
+
+    # nea == ref && ea == ref && len(nea) != len(ea)
+    status[nea_eq_ref * ea_eq_ref * ~len_nea_eq_len_ea, status_flip_idx] = 6
+
+    # nea == ref && ea != ref
+    status[nea_eq_ref * ~ea_eq_ref, status_flip_idx] = 0
+
+    # nea != ref && ea == ref
+    status[~nea_eq_ref * ea_eq_ref, status_flip_idx] = 3
+
+    # nea != ref && ea != ref && rev_nea == ref && rev_ea == ref && len(rev_nea) != len(rev_ea)
+    status[~nea_eq_ref * ~ea_eq_ref * rev_nea_eq_ref * rev_ea_eq_ref * ~len_rev_nea_eq_rev_len_ea, status_flip_idx] = 8
+
+    # nea != ref && ea != ref && rev_nea == ref && rev_ea != ref
+    status[~nea_eq_ref * ~ea_eq_ref * rev_nea_eq_ref * ~rev_ea_eq_ref, status_flip_idx] = 4
+
+    # nea != ref && ea != ref && rev_nea != ref && rev_ea == ref
+    status[~nea_eq_ref * ~ea_eq_ref * ~rev_nea_eq_ref * rev_ea_eq_ref, status_flip_idx] = 5
+
+    # nea != ref && ea != ref && rev_nea != ref && rev_ea != ref
+    status[~nea_eq_ref * ~ea_eq_ref * ~rev_nea_eq_ref * ~rev_ea_eq_ref, status_flip_idx] = 8
+
+    # Convert back the (now modified) 2D status array to a numpy array of strings in a very fast way.
+    # Since 'status' is a 2D array of integers ranging from 0 to 9, we can build the integer representation
+    # of each row using the efficent operation below (e.g. [1, 2, 3, 4, 5] -> [12345]).
+    # Then we convert this integer to a string using the f'<U{status.shape[1]}' dtype (e.g. 12345 -> '12345')
+    # The "naive" way would be:
+    #   status_str = [''.join(map(str, l)) for l in status]
+    #   status_arr = np.array(status_str)
+    status_flat = np.sum(status * 10**np.arange(status.shape[1]-1, -1, -1), axis=1)
+    status_arr = status_flat.astype(f'<U{status.shape[1]}')
+
+    return status_arr
+
+
+def check_status(sumstats: pd.DataFrame, fasta_records_dict, log=Log(), verbose=True):
+
+    chrom,pos,ea,nea,status = sumstats.columns
+
+    # First, convert the fasta records to a single numpy array of integers
+    record, starting_positions_dict = build_fasta_records(fasta_records_dict, pos_as_dict=True, log=log, verbose=verbose)
+
+    # In _fast_check_status(), several 2D numpy arrays are created and they are padded to have shape[1] == max_len_nea or max_len_ea
+    # Since most of the NEA and EA strings are short, we perform the check first on the records having short NEA and EA strings,
+    # and then we perform the check on the records having long NEA and EA strings. In this way we can speed up the process (since the 
+    # arrays are smaller) and save memory.
+    max_len = 4 # this is a chosen value, we could compute it using some stats about the length and count of NEA and EA strings
+    condition = (sumstats[nea].str.len() <= max_len) * (sumstats[ea].str.len() <= max_len)
+
+    log.write(f"   -Checking records for ( len(NEA) <= {max_len} and len(EA) <= {max_len} )", verbose=verbose)
+    sumstats_cond = sumstats[condition]
+    starting_pos_cond = np.array([starting_positions_dict[k] for k in sumstats_cond[chrom].unique()])
+    sumstats.loc[condition, status] = _fast_check_status(sumstats_cond, record=record, starting_positions=starting_pos_cond)
+
+    log.write(f"   -Checking records for ( len(NEA) > {max_len} or len(EA) > {max_len} )", verbose=verbose)
+    sumstats_not_cond = sumstats[~condition]
+    starting_not_pos_cond = np.array([starting_positions_dict[k] for k in sumstats_not_cond[chrom].unique()])
+    sumstats.loc[~condition, status] = _fast_check_status(sumstats_not_cond, record=record, starting_positions=starting_not_pos_cond)
+
+    return sumstats[status].values
+        
+
+def checkref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="STATUS",chr_dict=get_chr_to_number(),remove=False,verbose=True,log=Log()):
+    ##start function with col checking##########################################################
+    _start_line = "check if NEA is aligned with reference sequence"
+    _end_line = "checking if NEA is aligned with reference sequence"
+    _start_cols = [chrom,pos,ea,nea,status]
+    _start_function = ".check_ref()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                              log=log,
+                              verbose=verbose,
+                              start_line=_start_line,
+                              end_line=_end_line,
+                              start_cols=_start_cols,
+                              start_function=_start_function,
+                              **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+    log.write(" -Reference genome FASTA file: "+ ref_seq,verbose=verbose)  
+    log.write(" -Loading fasta records:",end="", verbose=verbose)
+    chromlist = get_chr_list(add_number=True)
+    records = SeqIO.parse(ref_seq, "fasta")
+
+    all_records_dict = {}
+    chroms_in_sumstats = sumstats[chrom].unique() # load records from Fasta file only for the chromosomes present in the sumstats
+    for record in records:
+        #record = next(records)
+        if record is not None:
+            record_chr = str(record.id).strip("chrCHR").upper()
+            if record_chr in chr_dict.keys():
+                i = chr_dict[record_chr]
+            else:
+                i = record_chr
+            if (i in chromlist) and (i in chroms_in_sumstats):
+                log.write(record_chr," ", end="",show_time=False,verbose=verbose)
+                all_records_dict.update({i: record})
+    log.write("",show_time=False,verbose=verbose)
+
+    if len(all_records_dict) > 0:
+        log.write(" -Checking records", verbose=verbose)
+        all_records_dict = dict(sorted(all_records_dict.items())) # sort by key in case the fasta records are not already ordered by chromosome
+        to_check_ref = (sumstats[chrom].isin(list(all_records_dict.keys()))) & (~sumstats[pos].isna()) & (~sumstats[nea].isna()) & (~sumstats[ea].isna())
+        sumstats_to_check = sumstats.loc[to_check_ref,[chrom,pos,ea,nea,status]]
+        sumstats.loc[to_check_ref,status] = check_status(sumstats_to_check, all_records_dict, log=log, verbose=verbose)
+        log.write(" -Finished checking records", verbose=verbose) 
+ 
+    sumstats[status] = sumstats[status].astype("string")
+        
+    available_to_check =sum( (~sumstats[pos].isna()) & (~sumstats[nea].isna()) & (~sumstats[ea].isna()))
+    status_0=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[0]\w", case=False, flags=0, na=False))
+    status_3=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[3]\w", case=False, flags=0, na=False))
+    status_4=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[4]\w", case=False, flags=0, na=False))
+    status_5=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[5]\w", case=False, flags=0, na=False))
+    status_6=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[6]\w", case=False, flags=0, na=False))
+    #status_7=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[7]\w", case=False, flags=0, na=False))
+    status_8=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[8]\w", case=False, flags=0, na=False))
+    
+    log.write(" -Variants allele on given reference sequence : ",status_0,verbose=verbose)
+    log.write(" -Variants flipped : ",status_3,verbose=verbose)
+    raw_matching_rate = (status_3+status_0)/available_to_check
+    flip_rate = status_3/available_to_check
+    log.write("  -Raw Matching rate : ","{:.2f}%".format(raw_matching_rate*100),verbose=verbose)
+    if raw_matching_rate <0.8:
+        log.warning("Matching rate is low, please check if the right reference genome is used.")
+    if flip_rate > 0.85 :
+        log.write("  -Flipping variants rate > 0.85, it is likely that the EA is aligned with REF in the original dataset.",verbose=verbose)
+    
+    log.write(" -Variants inferred reverse_complement : ",status_4,verbose=verbose)
+    log.write(" -Variants inferred reverse_complement_flipped : ",status_5,verbose=verbose)
+    log.write(" -Both allele on genome + unable to distinguish : ",status_6,verbose=verbose)
+    #log.write(" -Reverse_complementary + both allele on genome + unable to distinguish: ",status_7)
+    log.write(" -Variants not on given reference sequence : ",status_8,verbose=verbose)
+    
+    if remove is True:
+        sumstats = sumstats.loc[~sumstats["STATUS"].str.match("\w\w\w\w\w[8]\w"),:]
+        log.write(" -Variants not on given reference sequence were removed.",verbose=verbose)
+    
+    finished(log, verbose, _end_line)
+    return sumstats
+
+def build_fasta_records(fasta_records_dict, pos_as_dict=True, log=Log(), verbose=True):
+    log.write("   -Building numpy fasta records from dict", verbose=verbose)
+
+    # Let's do some magic to convert the fasta record to a numpy array of integers in a very fast way.
+    # fasta_record.seq._data is a byte-string, so we can use the bytes.maketrans to apply a translation.
+    # Here we map the bytes to the unicode character representing the desired integer as defined in the mapping dict
+    # (i.e. b'A' -> '\x02', b'T' -> '\x03', b'C' -> '\x04', b'G' -> '\x05', b'N' -> '\x06')
+    # Then, using np.array(... dtype=<U..) we convert the string to a numpy array of unicode characters.
+    # Then, we do a magic with view('<u4') to convert the unicode characters to 4-byte integers, so we obtain the actual integer representation of the characters
+    # Lastly, we cast the array to np.uint8 to convert the 4-byte integers to 1-byte integers to save memory
+    # Full example:
+    # fasta_record.seq._data = b'ACTGN' -> b'\x02\x04\x03\x05\x06' -> np.array(['\x02\x04\x03\x05\x06'], dtype='<U5') -> np.array([2, 4, 3, 5, 6], dtype=uint32) -> np.array([2, 4, 3, 5, 6], dtype=uint8)
+    all_r = []
+    for r in fasta_records_dict.values():
+        r = r.seq._data.translate(TRANSLATE_TABLE)
+        r = np.array([r], dtype=f'<U{len(r)}').view('<u4').astype(np.uint8)
+        all_r.append(r)
+
+    # We've just created a list of numpy arrays, so we can concatenate them to obtain a single numpy array
+    # Then we keep track of the starting position of each record in the concatenated array. This will be useful later
+    # to index the record array depending on the position of the variant and the chromosome
+    records_len = np.array([len(r) for r in all_r])
+    starting_positions = np.cumsum(records_len) - records_len
+    if pos_as_dict:
+        starting_positions = {k: v for k, v in zip(fasta_records_dict.keys(), starting_positions)}
+    record = np.concatenate(all_r)
+    del all_r # free memory
+
+    return record, starting_positions
+
 #######################################################################################################################################
 
 #20220721
@@ -559,6 +913,56 @@ def check_strand_status(chr,start,end,ref,alt,eaf,vcf_reader,alt_freq,status,chr
                 return status_pre+"5"+status_end
     return status_pre+"8"+status_end
 
+def check_strand_status_cache(data,cache,ref_infer=None,ref_alt_freq=None,chr_dict=get_number_to_chr(),trust_cache=True,log=Log(),verbose=True):
+    if not trust_cache:
+        assert ref_infer is not None, "If trust_cache is False, ref_infer must be provided"
+        log.warning("You are not trusting the cache, this will slow down the process. Please consider building a complete cache.")
+
+    if ref_infer is not None and not trust_cache:
+        vcf_reader = VariantFile(ref_infer)
+
+    if isinstance(data, pd.DataFrame):
+        data = data.values
+    
+    in_cache = 0
+    new_statuses = []
+    
+    for i in range(data.shape[0]):
+        _chrom, pos, ref, alt, eaf, status = data[i]
+        chrom = _chrom
+        start = pos - 1
+        end = pos
+        
+        if chr_dict is not None: chrom=chr_dict[chrom]
+        
+        status_pre=status[:6]
+        status_end=""
+        
+        new_status = status_pre+"8"+status_end # default value
+        
+        cache_key = f"{chrom}:{pos}:{ref}:{alt}"
+        if cache_key in cache:
+            in_cache += 1
+            record = cache[cache_key]
+            if record is None:
+                new_status = status_pre+"8"+status_end
+            else:
+                if (record<0.5) and (eaf<0.5):
+                    new_status = status_pre+"1"+status_end
+                elif (record>0.5) and (eaf>0.5):
+                    new_status = status_pre+"1"+status_end
+                else:
+                    new_status = status_pre+"5"+status_end
+        else:
+            if not trust_cache:
+                # If we don't trust the cache as a not complete cache, we should perform the check reading from the VCF file
+                new_status = check_strand_status(_chrom, start, end, ref, alt, eaf, vcf_reader, ref_alt_freq, status, chr_dict)
+        
+        new_statuses.append(new_status)
+        
+    log.write(f"  -Elements in cache: {in_cache}", verbose=verbose)
+    return new_statuses
+
 
 def check_unkonwn_indel(chr,start,end,ref,alt,eaf,vcf_reader,alt_freq,status,chr_dict=get_number_to_chr(),daf_tolerance=0.2):
     ### input : unknown indel, both on genome (xx1[45]x)
@@ -586,6 +990,65 @@ def check_unkonwn_indel(chr,start,end,ref,alt,eaf,vcf_reader,alt_freq,status,chr
 
     return status_pre+"8"+status_end
 
+
+def check_unkonwn_indel_cache(data,cache,ref_infer=None,ref_alt_freq=None,chr_dict=get_number_to_chr(),daf_tolerance=0.2,trust_cache=True,log=Log(),verbose=True):
+    if not trust_cache:
+        assert ref_infer is not None, "If trust_cache is False, ref_infer must be provided"
+        log.warning("You are not trusting the cache, this will slow down the process. Please consider building a complete cache.")
+
+    if ref_infer is not None:
+        vcf_reader = VariantFile(ref_infer)
+
+    if isinstance(data, pd.DataFrame):
+        data = data.values
+    
+    in_cache = 0
+    new_statuses = []
+    
+    for i in range(data.shape[0]):
+        _chrom, pos, ref, alt, eaf, status = data[i]
+        chrom = _chrom
+        
+        if chr_dict is not None: chrom=chr_dict[chrom]
+        start = pos - 1
+        end = pos
+        
+        status_pre=status[:6]
+        status_end=""
+        
+        new_status = status_pre+"8"+status_end # default value
+        
+        cache_key_ref_alt = f"{chrom}:{pos}:{ref}:{alt}"
+        cache_key_alt_ref = f"{chrom}:{pos}:{alt}:{ref}"
+
+        if cache_key_ref_alt in cache:
+            in_cache += 1
+            record = cache[cache_key_ref_alt]
+            if record is None:
+                new_status = status_pre+"8"+status_end
+            else:
+                if  abs(record - eaf)<daf_tolerance:
+                    new_status = status_pre+"3"+status_end
+
+        elif cache_key_alt_ref in cache:
+            in_cache += 1
+            record = cache[cache_key_alt_ref]
+            if record is None:
+                new_status = status_pre+"8"+status_end
+            else:
+                if  abs(record - (1 - eaf))<daf_tolerance:
+                    new_status = status_pre+"6"+status_end
+
+        else:
+            if not trust_cache:
+                # If we don't trust the cache as a not complete cache, we should perform the check reading from the VCF file
+                new_status = check_unkonwn_indel(_chrom, start, end, ref, alt, eaf, vcf_reader, ref_alt_freq, status, chr_dict, daf_tolerance)
+                
+        new_statuses.append(new_status)
+        
+    log.write(f"  -Elements in cache: {in_cache}", verbose=verbose)
+    return new_statuses
+
                                                
 def get_reverse_complementary_allele(a):
     dic = str.maketrans({
@@ -610,16 +1073,40 @@ def check_strand(sumstats,ref_infer,ref_alt_freq=None,chr="CHR",pos="POS",ref="N
     status_part = sumstats.apply(lambda x:check_strand_status(x.iloc[0],x.iloc[1]-1,x.iloc[1],x.iloc[2],x.iloc[3],x.iloc[4],vcf_reader,ref_alt_freq,x.iloc[5],chr_dict),axis=1) 
     return status_part
 
+def check_strand_cache(sumstats,cache,ref_infer,ref_alt_freq=None,chr_dict=get_number_to_chr(),trust_cache=True,log=Log(),verbose=True):
+    assert cache is not None, "Cache must be provided"
+    status_part = check_strand_status_cache(sumstats,cache,ref_infer,ref_alt_freq,chr_dict,trust_cache,log,verbose)
+    return status_part
+
 def check_indel(sumstats,ref_infer,ref_alt_freq=None,chr="CHR",pos="POS",ref="NEA",alt="EA",eaf="EAF",chr_dict=get_number_to_chr(),status="STATUS",daf_tolerance=0.2):
     vcf_reader = VariantFile(ref_infer)
     status_part = sumstats.apply(lambda x:check_unkonwn_indel(x.iloc[0],x.iloc[1]-1,x.iloc[1],x.iloc[2],x.iloc[3],x.iloc[4],vcf_reader,ref_alt_freq,x.iloc[5],chr_dict,daf_tolerance),axis=1)
     return status_part
 
+def check_indel_cache(sumstats,cache,ref_infer,ref_alt_freq=None,chr_dict=get_number_to_chr(),daf_tolerance=0.2,trust_cache=True,log=Log(),verbose=True):
+    assert cache is not None, "Cache must be provided"
+    status_part = check_unkonwn_indel_cache(sumstats,cache,ref_infer,ref_alt_freq,chr_dict,daf_tolerance,trust_cache,log,verbose)
+    return status_part
+
 ##################################################################################################################################################
 
 def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,daf_tolerance=0.20,remove_snp="",mode="pi",n_cores=1,remove_indel="",
                        chr="CHR",pos="POS",ref="NEA",alt="EA",eaf="EAF",status="STATUS",
-                       chr_dict=None,verbose=True,log=Log()):
+                       chr_dict=None,cache_options={},verbose=True,log=Log()):
+    '''
+    Args:
+    cache_options : A dictionary with the following keys:
+        - cache_manager: CacheManager object or None. If any between cache_loader and cache_process is not None, or use_cache is True, a CacheManager object will be created automatically.
+        - trust_cache: bool (optional, default: True). Whether to completely trust the cache or not. Trusting the cache means that any key not found inside the cache will be considered as a missing value even in the VCF file.
+        - cache_loader: Object with a get_cache() method or None.
+        - cache_process: Object with an apply_fn() method or None.
+        - use_cache: bool (optional, default: False). If any of the cache_manager, cache_loader or cache_process is not None, this will be set to True automatically.
+                     If set to True and all between cache_manager, cache_loader and cache_process are None, the cache will be loaded (or built) on the spot.
+
+        The usefulness of a cache_loader or cache_process object is to pass a custom object which already has the cache loaded. This can be useful if the cache is loaded in background in another thread/process while other operations are performed.
+        The cache_manager is a CacheManager object is used to expose the API to interact with the cache.
+    '''
+
     ##start function with col checking##########################################################
     _start_line = "infer strand for palindromic SNPs/align indistinguishable indels"
     _end_line = "inferring strand for palindromic SNPs/align indistinguishable indels"
@@ -642,6 +1129,16 @@ def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,
 
     chr_dict = auto_check_vcf_chr_dict(ref_infer, chr_dict, verbose, log)
     
+    # Setup cache variables
+    cache_manager = cache_options.get("cache_manager", None)
+    if cache_manager is not None:
+        assert isinstance(cache_manager, CacheManager), "cache_manager must be a CacheManager object"
+    trust_cache = cache_options.get("trust_cache", True)
+    cache_loader = cache_options.get("cache_loader", None)
+    cache_process = cache_options.get("cache_process", None)
+    use_cache = any(c is not None for c in [cache_manager, cache_loader, cache_process]) or cache_options.get('use_cache', False)
+    _n_cores = n_cores # backup n_cores
+    
     log.write(" -Field for alternative allele frequency in VCF INFO: {}".format(ref_alt_freq), verbose=verbose)  
 
     if "p" in mode:
@@ -669,16 +1166,30 @@ def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,
         ######################################################################################### 
         if sum(unknow_palindromic_to_check)>0:
             if sum(unknow_palindromic_to_check)<10000: 
-                n_cores=1  
-            
-            #df_split = np.array_split(sumstats.loc[unknow_palindromic_to_check,[chr,pos,ref,alt,eaf,status]], n_cores)
-            df_split = _df_split(sumstats.loc[unknow_palindromic_to_check,[chr,pos,ref,alt,eaf,status]], n_cores)
-            pool = Pool(n_cores)
-            map_func = partial(check_strand,chr=chr,pos=pos,ref=ref,alt=alt,eaf=eaf,status=status,ref_infer=ref_infer,ref_alt_freq=ref_alt_freq,chr_dict=chr_dict) 
-            status_inferred = pd.concat(pool.map(map_func,df_split))
-            sumstats.loc[unknow_palindromic_to_check,status] = status_inferred.values
-            pool.close()
-            pool.join()
+                n_cores=1
+
+            if use_cache and cache_manager is None:
+                cache_manager = CacheManager(base_path=ref_infer, cache_loader=cache_loader, cache_process=cache_process,
+                                             ref_alt_freq=ref_alt_freq, category=PALINDROMIC_INDEL,
+                                             n_cores=_n_cores, log=log, verbose=verbose)
+
+            log.write(" -Starting strand inference for palindromic SNPs...",verbose=verbose)
+            df_to_check = sumstats.loc[unknow_palindromic_to_check,[chr,pos,ref,alt,eaf,status]]
+           
+            if use_cache and cache_manager.cache_len > 0:
+                log.write("  -Using cache for strand inference",verbose=verbose)
+                status_inferred = cache_manager.apply_fn(check_strand_cache, sumstats=df_to_check, ref_infer=ref_infer, ref_alt_freq=ref_alt_freq, chr_dict=chr_dict, trust_cache=trust_cache, log=log, verbose=verbose)
+                sumstats.loc[unknow_palindromic_to_check,status] = status_inferred
+            else:
+                #df_split = np.array_split(df_to_check, n_cores)
+                df_split = _df_split(df_to_check, n_cores)
+                pool = Pool(n_cores)
+                map_func = partial(check_strand,chr=chr,pos=pos,ref=ref,alt=alt,eaf=eaf,status=status,ref_infer=ref_infer,ref_alt_freq=ref_alt_freq,chr_dict=chr_dict) 
+                status_inferred = pd.concat(pool.map(map_func,df_split))
+                sumstats.loc[unknow_palindromic_to_check,status] = status_inferred.values
+                pool.close()
+                pool.join()
+            log.write(" -Finished strand inference.",verbose=verbose)
         else:
             log.warning("No palindromic variants available for checking.")
         #########################################################################################
@@ -729,15 +1240,30 @@ def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,
                          
             if sum(unknow_indel)>0:
                 if sum(unknow_indel)<10000: 
-                    n_cores=1    
-                #df_split = np.array_split(sumstats.loc[unknow_indel, [chr,pos,ref,alt,eaf,status]], n_cores)
-                df_split = _df_split(sumstats.loc[unknow_indel, [chr,pos,ref,alt,eaf,status]], n_cores)
-                pool = Pool(n_cores)
-                map_func = partial(check_indel,chr=chr,pos=pos,ref=ref,alt=alt,eaf=eaf,status=status,ref_infer=ref_infer,ref_alt_freq=ref_alt_freq,chr_dict=chr_dict,daf_tolerance=daf_tolerance) 
-                status_inferred = pd.concat(pool.map(map_func,df_split))
-                sumstats.loc[unknow_indel,status] = status_inferred.values 
-                pool.close()
-                pool.join()
+                    n_cores=1
+
+                if use_cache and cache_manager is None:
+                    cache_manager = CacheManager(base_path=ref_infer, cache_loader=cache_loader, cache_process=cache_process,
+                                                ref_alt_freq=ref_alt_freq, category=PALINDROMIC_INDEL,
+                                                n_cores=_n_cores, log=log, verbose=verbose)
+
+                log.write(" -Starting indistinguishable indel inference...",verbose=verbose)
+                df_to_check = sumstats.loc[unknow_indel,[chr,pos,ref,alt,eaf,status]]
+            
+                if use_cache and cache_manager.cache_len > 0:
+                    log.write("  -Using cache for indel inference",verbose=verbose)
+                    status_inferred = cache_manager.apply_fn(check_indel_cache, sumstats=df_to_check, ref_infer=ref_infer, ref_alt_freq=ref_alt_freq, chr_dict=chr_dict, daf_tolerance=daf_tolerance, trust_cache=trust_cache, log=log, verbose=verbose)
+                    sumstats.loc[unknow_indel,status] = status_inferred
+                else:
+                    #df_split = np.array_split(sumstats.loc[unknow_indel, [chr,pos,ref,alt,eaf,status]], n_cores)
+                    df_split = _df_split(sumstats.loc[unknow_indel, [chr,pos,ref,alt,eaf,status]], n_cores)
+                    pool = Pool(n_cores)
+                    map_func = partial(check_indel,chr=chr,pos=pos,ref=ref,alt=alt,eaf=eaf,status=status,ref_infer=ref_infer,ref_alt_freq=ref_alt_freq,chr_dict=chr_dict,daf_tolerance=daf_tolerance) 
+                    status_inferred = pd.concat(pool.map(map_func,df_split))
+                    sumstats.loc[unknow_indel,status] = status_inferred.values 
+                    pool.close()
+                    pool.join()
+                log.write(" -Finished indistinguishable indel inference.",verbose=verbose)
 
             #########################################################################################