NGSpeciesID 0.3.1__py2.py3-none-any.whl

modules/consensus.py

@@ -0,0 +1,278 @@
+from __future__ import print_function
+import subprocess
+import sys, os
+from sys import stdout
+import re 
+import shutil
+import parasail
+import glob
+import logging
+
+from modules import help_functions
+
+
+def cigar_to_seq(cigar, query, ref):
+    cigar_tuples = []
+    result = re.split(r'[=DXSMI]+', cigar)
+    cig_pos = 0
+    for length in result[:-1]:
+        cig_pos += len(length)
+        type_ = cigar[cig_pos]
+        cig_pos += 1
+        cigar_tuples.append((int(length), type_ ))
+
+    r_index = 0
+    q_index = 0
+    q_aln = []
+    r_aln = []
+    for length_ , type_ in cigar_tuples:
+        if type_ == "=" or type_ == "X":
+            q_aln.append(query[q_index : q_index + length_])
+            r_aln.append(ref[r_index : r_index + length_])
+
+            r_index += length_
+            q_index += length_
+        
+        elif  type_ == "I":
+            # insertion w.r.t. reference
+            r_aln.append('-' * length_)
+            q_aln.append(query[q_index: q_index + length_])
+            #  only query index change
+            q_index += length_
+
+        elif type_ == 'D':
+            # deletion w.r.t. reference
+            r_aln.append(ref[r_index: r_index + length_])
+            q_aln.append('-' * length_)
+            #  only ref index change
+            r_index += length_
+        
+        else:
+            logging.error("Error processing cigar")
+            logging.error(cigar)
+            sys.exit()
+
+    return  "".join([s for s in q_aln]), "".join([s for s in r_aln]), cigar_tuples
+
+
+def parasail_alignment(s1, s2, match_score = 2, mismatch_penalty = -2, opening_penalty = 3, gap_ext = 1):
+    user_matrix = parasail.matrix_create("ACGT", match_score, mismatch_penalty)
+    result = parasail.sg_trace_scan_16(s1, s2, opening_penalty, gap_ext, user_matrix)
+    if result.saturated:
+        logging.warning(f"SATURATED!{len(s1)} {len(s2)}")
+        result = parasail.sg_trace_scan_32(s1, s2, opening_penalty, gap_ext, user_matrix)
+        logging.warning("computed 32 bit instead")
+
+    # difference in how to obtain string from parasail between python v2 and v3... 
+    if sys.version_info[0] < 3:
+        cigar_string = str(result.cigar.decode).decode('utf-8')
+    else:
+        cigar_string = str(result.cigar.decode, 'utf-8')
+    s1_alignment, s2_alignment, cigar_tuples = cigar_to_seq(cigar_string, s1, s2)
+
+    return s1_alignment, s2_alignment, cigar_string, cigar_tuples, result.score
+
+def reverse_complement(string):
+    #rev_nuc = {'A':'T', 'C':'G', 'G':'C', 'T':'A', 'N':'N', 'X':'X'}
+    # Modified for Abyss output
+    rev_nuc = {'A':'T', 'C':'G', 'G':'C', 'T':'A', 'a':'t', 'c':'g', 'g':'c', 't':'a', 'N':'N', 'X':'X', 'n':'n', 'Y':'R', 'R':'Y', 'K':'M', 'M':'K', 'S':'S', 'W':'W', 'B':'V', 'V':'B', 'H':'D', 'D':'H', 'y':'r', 'r':'y', 'k':'m', 'm':'k', 's':'s', 'w':'w', 'b':'v', 'v':'b', 'h':'d', 'd':'h'}
+
+    rev_comp = ''.join([rev_nuc[nucl] for nucl in reversed(string)])
+    return(rev_comp)
+
+def run_spoa(reads, spoa_out_file, spoa_path):
+    with open(spoa_out_file, "w") as output_file:
+        stdout.flush()
+        with open("/dev/null", "w") as null:
+            subprocess.check_call([ spoa_path, reads, "-l", "0", "-r", "0", "-g", "-2"], stdout=output_file, stderr=null)
+        stdout.flush()
+    with open(spoa_out_file, "r") as sof:
+        l = sof.readlines()
+    consensus = l[1].strip()
+    return consensus
+
+def run_medaka(reads_to_center, center_file, outfolder, cores, medaka_model, outfastq=False):
+    medaka_stdout = os.path.join(outfolder, "stdout.txt")
+    with open(medaka_stdout, "w") as output_file:
+        stdout.flush()
+        with open(os.path.join(outfolder, "stderr.txt"), "w") as medaka_stderr:
+            cmd_args = ['medaka_consensus', '-i', reads_to_center, "-d", center_file, "-o", outfolder, "-t", cores]
+            if medaka_model:
+                cmd_args += ["-m", medaka_model]
+            if outfastq:
+                cmd_args += ["-q"]
+            subprocess.check_call(cmd_args, stdout=output_file, stderr=medaka_stderr)
+        stdout.flush()
+
+def run_racon(reads_to_center, center_file, outfolder, cores, racon_iter):
+    racon_stdout = os.path.join(outfolder, "stdout.txt")
+    with open(racon_stdout, "w") as output_file:
+        stdout.flush()
+        for i in range(racon_iter):
+            with open(
+                os.path.join(outfolder, "read_alignments_it_{0}.paf".format(i)), 'w'
+            ) as read_alignments, open(
+                os.path.join(outfolder, "mm2_stderr_it_{0}.txt".format(i)), "w"
+            ) as mm2_stderr, open(
+                os.path.join(outfolder, "racon_stderr_it_{0}.txt".format(i)), "w"
+            ) as racon_stderr, open(
+                os.path.join(outfolder, "racon_polished_it_{0}.fasta".format(i)
+            ), 'w') as racon_polished:
+                subprocess.check_call(['minimap2', '-x', 'map-ont', center_file, reads_to_center], stdout=read_alignments, stderr=mm2_stderr)
+                subprocess.check_call(['racon', reads_to_center, read_alignments.name, center_file], stdout=racon_polished, stderr=racon_stderr)
+            center_file = racon_polished.name
+
+        shutil.copyfile(center_file, os.path.join(outfolder, "consensus.fasta"))
+        stdout.flush()
+
+
+def highest_aln_identity(seq, seq2):
+    # RC
+    seq2_rc = reverse_complement(seq2)
+    seq_aln_rc, seq2_aln_rc, cigar_string_rc, cigar_tuples_rc, alignment_score_rc = parasail_alignment(seq, seq2_rc)
+    nr_mismatching_pos = len([1 for n1, n2 in zip(seq_aln_rc, seq2_aln_rc) if n1 != n2])
+    total_pos_rc = len(seq_aln_rc)
+    aln_identity_rc =  (total_pos_rc - nr_mismatching_pos) / float(total_pos_rc)
+    logging.debug(f"Rec comp orientation identity %: {aln_identity_rc}")
+
+    # FW
+    seq_aln, seq2_aln, cigar_string, cigar_tuples, alignment_score = parasail_alignment(seq, seq2)
+    nr_mismatching_pos = len([1 for n1, n2 in zip(seq_aln, seq2_aln) if n1 != n2])
+    total_pos = len(seq_aln)
+    aln_identity_fw = (total_pos - nr_mismatching_pos) / float(total_pos)  
+    logging.debug(f"Forward orientation identity %: {aln_identity_fw}")
+    aln_identity = max([aln_identity_fw, aln_identity_rc])
+    return aln_identity
+
+
+def detect_reverse_complements(centers, rc_identity_threshold):
+    filtered_centers = []
+    already_removed = set()
+    for i, (nr_reads_in_cl, c_id, seq, reads_path) in enumerate(centers):
+        if type(reads_path) != list:
+            all_reads = [reads_path]
+        else:
+            all_reads = reads_path
+
+        merged_cluster_id = c_id
+        merged_nr_reads = nr_reads_in_cl
+        if c_id in already_removed:
+            logging.debug("has already been merged, skipping")
+            continue
+
+        elif i == len(centers) - 1: # last sequence and it is not in already_removed
+            filtered_centers.append( [merged_nr_reads, c_id, seq, all_reads ] )
+
+        else:
+            for j, (nr_reads_in_cl2, c_id2, seq2, reads_path) in enumerate(centers[i+1:]):
+                aln_identity = highest_aln_identity(seq, seq2)
+                if aln_identity >= rc_identity_threshold:
+                    logging.debug("Detected two consensus sequences with alignment identidy above threshold (from either reverse complement or split clusters). Keeping center with the most read support and merging reads.")
+                    merged_nr_reads += nr_reads_in_cl2
+                    already_removed.add(c_id2)
+
+                    if type(reads_path) != list:
+                        all_reads.append(reads_path)
+                    else:
+                        for rp in reads_path:
+                            all_reads.append(rp)
+
+            filtered_centers.append( [merged_nr_reads, c_id, seq, all_reads] )
+
+    logging.debug(f"{len(filtered_centers)} consensus formed.")
+    return filtered_centers
+
+
+def polish_sequences(centers, args):
+    spoa_ref_location = os.path.join(args.outfolder, "consensus_reference_X.fasta")
+    logging.debug(f"Saving spoa references to files: {spoa_ref_location}")
+    # printing output from spoa and grouping reads
+    if args.medaka:
+        polishing_pattern = os.path.join(args.outfolder, "medaka_cl_id_*")
+    elif args.racon:
+        polishing_pattern = os.path.join(args.outfolder, "racon_cl_id_*")
+
+    for folder in glob.glob(polishing_pattern):
+        shutil.rmtree(folder)
+
+    spoa_pattern = os.path.join(args.outfolder, "consensus_reference_*")
+    for file in glob.glob(spoa_pattern):
+        os.remove(file)
+
+    for i, (nr_reads_in_cluster, c_id, center, all_reads) in enumerate(centers):
+        spoa_center_file = os.path.join(args.outfolder, "consensus_reference_{0}.fasta".format(c_id))
+        with open(spoa_center_file, "w") as f:
+            f.write(">{0}\n{1}\n".format("consensus_cl_id_{0}_total_supporting_reads_{1}".format(c_id, nr_reads_in_cluster), center))
+        
+        nr_reads_used = 0
+        all_reads_file = os.path.join(args.outfolder, "reads_to_consensus_{0}.fastq".format(c_id))
+        with open(all_reads_file, "w") as f:
+            for fasta_file in all_reads: 
+                reads = { acc : (seq, qual) for acc, (seq, qual) in help_functions.readfq(open(fasta_file, 'r'))}
+                for acc, (seq, qual) in reads.items():
+                    acc_tmp = acc.split()[0]
+                    f.write("@{0}\n{1}\n{2}\n{3}\n".format(acc_tmp, seq, "+", qual))
+                    nr_reads_used += 1
+
+        if args.medaka:
+            logging.debug("running medaka on spoa reference {0} using {1} reads for polishing.".format(c_id, nr_reads_used))
+            polishing_outfolder = os.path.join(args.outfolder, "medaka_cl_id_{0}".format(c_id))
+            outfiles = [  # consider all output formats for compatibility with all Medaka versions
+                os.path.join(polishing_outfolder, "consensus.fasta"),
+                os.path.join(polishing_outfolder, "consensus.fastq")
+            ]
+            help_functions.mkdir_p(polishing_outfolder)
+            run_medaka(all_reads_file, spoa_center_file, polishing_outfolder, "1", args.medaka_model, outfastq=args.medaka_fastq)
+            medaka_ref_location = os.path.join(polishing_outfolder, "consensus.fasta/q")
+            logging.debug(f"Saving medaka reference to file: {medaka_ref_location}")
+            for f in outfiles:
+                if os.path.isfile(f):
+                    with open(f, 'r') as cf:
+                        centers[i][2] = cf.readlines()[1].strip()  # the second line is the nucleotide sequence
+                        break
+            assert centers[i][2], "Medaka consensus sequence not found"
+        elif args.racon:
+            logging.debug("running racon on spoa reference {0} using {1} reads for polishing.".format(c_id, nr_reads_used))
+            polishing_outfolder = os.path.join(args.outfolder, "racon_cl_id_{0}".format(c_id))
+            help_functions.mkdir_p(polishing_outfolder)
+            run_racon(all_reads_file, spoa_center_file, polishing_outfolder, "1", args.racon_iter)
+            racon_ref_location = os.path.join(args.outfolder, "racon_cl_id_{0}/consensus.fasta".format(c_id))
+            logging.debug(f"Saving racon reference to file: {racon_ref_location}")
+            with open(os.path.join(polishing_outfolder, "consensus.fasta"), 'r') as cf:
+                l = cf.readlines()
+            center_polished = l[1].strip()
+            centers[i][2] = center_polished
+
+    return centers
+
+
+def form_draft_consensus(clusters, representatives, sorted_reads_fastq_file, work_dir, abundance_cutoff, args):
+    centers = []
+    singletons = 0
+    discarded_clusters = []
+    reads = { acc : (seq, qual) for acc, (seq, qual) in help_functions.readfq(open(sorted_reads_fastq_file, 'r'))}
+    for c_id, all_read_acc in sorted(clusters.items(), key = lambda x: (len(x[1]),representatives[x[0]][5]), reverse=True):
+        nr_reads_in_cluster = len(all_read_acc)
+        if nr_reads_in_cluster >= abundance_cutoff:
+            reads_path_name = os.path.join(work_dir, "reads_c_id_{0}.fq".format(c_id))
+            with open(reads_path_name, "w") as reads_file:
+                for i, acc in enumerate(all_read_acc):
+                    if (args.max_seqs_for_consensus) >=0 and (i >= args.max_seqs_for_consensus):
+                        break
+                    seq, qual = reads[acc]
+                    reads_file.write("@{0}\n{1}\n{2}\n{3}\n".format(acc, seq, "+", qual))
+            tmp_param = args.max_seqs_for_consensus if args.max_seqs_for_consensus > 0 else 2**32
+            logging.debug("creating center of {0} sequences.".format(min(nr_reads_in_cluster, tmp_param)))
+            center = run_spoa(reads_path_name, os.path.join(work_dir,"spoa_tmp.fa"), "spoa")
+            centers.append( [nr_reads_in_cluster, c_id, center, reads_path_name])
+        elif nr_reads_in_cluster == 1:
+            singletons += 1
+        elif nr_reads_in_cluster > 1:
+            discarded_clusters.append(nr_reads_in_cluster)
+    logging.debug(f"{singletons} singletons were discarded")
+    logging.debug(
+        f"{len(discarded_clusters)} clusters were discarded due to not passing the abundance_cutoff: "
+        f"a total of {sum(discarded_clusters)} reads were discarded. "
+        f"Highest abundance among them: {max(discarded_clusters or [0])} reads."
+    )
+    return centers

modules/get_sorted_fastq_for_cluster.py

@@ -0,0 +1,218 @@
+from __future__ import print_function
+import os,sys
+import argparse
+
+import signal
+from multiprocessing import Pool
+import multiprocessing as mp
+
+import operator
+import functools
+from time import time
+from collections import deque
+import sys
+import itertools
+import math
+import logging
+
+from modules import help_functions
+
+D = {chr(i) : min( 10**( - (ord(chr(i)) - 33)/10.0 ), 0.79433)  for i in range(128)}
+D_no_min = {chr(i) : 10**( - (ord(chr(i)) - 33)/10.0 )  for i in range(128)}
+
+def expected_number_of_erroneous_kmers(quality_string, k):
+    prob_error = [D[char_] for char_ in quality_string]
+    window = deque([ (1.0 - p_e) for p_e in prob_error[:k]])
+    qurrent_prob_no_error = functools.reduce(operator.mul, window, 1)
+    sum_of_expectations = qurrent_prob_no_error # initialization
+    for p_e in prob_error[k:]:
+        p_to_leave = window.popleft()
+        qurrent_prob_no_error *= ((1.0 -p_e)/(p_to_leave))
+        sum_of_expectations += qurrent_prob_no_error
+        window.append(1.0 -p_e)
+    return len(quality_string) - k + 1 - sum_of_expectations 
+
+
+def reverse_complement(string):
+    rev_nuc = {'A':'T', 'C':'G', 'G':'C', 'T':'A', 'a':'t', 'c':'g', 'g':'c', 't':'a', 'N':'N', 'X':'X', 'n':'n', 'Y':'R', 'R':'Y', 'K':'M', 'M':'K', 'S':'S', 'W':'W', 'B':'V', 'V':'B', 'H':'D', 'D':'H', 'y':'r', 'r':'y', 'k':'m', 'm':'k', 's':'s', 'w':'w', 'b':'v', 'v':'b', 'h':'d', 'd':'h'}
+    rev_comp = ''.join([rev_nuc[nucl] for nucl in reversed(string)])
+    return(rev_comp)
+
+def batch(iterable, n=1):
+    l = len(iterable)
+    for ndx in range(0, l, n):
+        yield iterable[ndx:min(ndx + n, l)]
+
+
+def calc_score_new(d):
+    for key,value in d.items():
+        l, k, q_threshold = value
+
+    read_array = []
+    error_rates = []
+    for i, (acc, seq, qual) in enumerate(l):
+        if i % 10000 == 0:
+            logging.debug(f"{i} reads processed.")
+
+        # skip very short reads or degenerate reads
+        seq_hpol_comp = ''.join(ch for ch, _ in itertools.groupby(seq))
+        if len(seq) < 2*k or len(seq_hpol_comp) < k:
+            continue
+
+        poisson_mean = sum([ qual.count(char_) * D_no_min[char_] for char_ in set(qual)])
+        error_rate = poisson_mean/float(len(qual))
+        if 10*-math.log(error_rate, 10) <= q_threshold:
+            continue
+
+        error_rates.append(error_rate)
+        exp_errors_in_kmers = expected_number_of_erroneous_kmers(qual, k)
+        p_no_error_in_kmers = 1.0 - exp_errors_in_kmers/ float((len(seq) - k +1))
+        score =  p_no_error_in_kmers  * (len(seq) - k +1)
+        read_array.append((acc, seq, qual, score) )
+    return {key : (read_array, error_rates)}
+
+
+def fastq_parallel(args):
+    k = args.k
+    q_threshold = args.quality_threshold
+    error_rates = []
+    reads = [ (acc,seq, qual) for acc, (seq, qual) in help_functions.readfq(open(args.fastq, 'r'))]
+    start = time()
+    read_chunk_size = int( len(reads)/args.nr_cores ) + 1
+    read_batches = [b for b in batch(reads, read_chunk_size)]
+    del reads
+    ####### parallelize alignment #########
+    original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
+    signal.signal(signal.SIGINT, original_sigint_handler)
+    mp.set_start_method('spawn')
+    logging.debug(f"{mp.get_context()}")
+    logging.debug(f"Environment set: {mp.get_context()}")
+    logging.debug(f"Using {args.nr_cores} cores.")
+    start_multi = time()
+    pool = Pool(processes=int(args.nr_cores))
+    try:
+        batch_lengths = [len(b) for b in read_batches]
+        logging.debug(f"{batch_lengths}")
+        data = [ {i : (b,k, q_threshold)} for i, b in enumerate(read_batches)] #[ {i+1 :((cluster_batches[i], cluster_seq_origin_batches[i], read_batches[i], p_emp_probs, lowest_batch_index_db[i], i+1, args), {})} for i in range(len(read_batches))]
+        res = pool.map_async(calc_score_new, data)
+        score_results =res.get(999999999) # Without the timeout this blocking call ignores all signals.
+    except KeyboardInterrupt:
+        logging.warning("Caught KeyboardInterrupt, terminating workers")
+        pool.terminate()
+        sys.exit()
+    else:
+        pool.close()
+    pool.join()
+
+    logging.debug(f"Time elapesd multiprocessing: {time() - start_multi}")
+    read_array, error_rates = [], []
+
+    for output_dict in score_results:
+        for k, v in output_dict.items():
+            r_a, err_rates = v
+            logging.debug(f"Batch index {k}")
+            for item in r_a:
+                read_array.append(item)
+            for item2 in err_rates:
+                error_rates.append(item2)
+
+    read_array.sort(key=lambda x: x[3], reverse=True)
+    error_rates.sort()
+    return read_array, error_rates
+
+
+def fastq_single_core(args):
+    k = args.k
+    q_threshold = args.quality_threshold
+    error_rates = []
+    read_array = []
+    for i, (acc, (seq, qual)) in enumerate(help_functions.readfq(open(args.fastq, 'r'))):
+        if i % 10000 == 0:
+            logging.debug(f"{i} reads processed.")
+
+        # skip very short reads or degenerate reads
+        seq_hpol_comp = ''.join(ch for ch, _ in itertools.groupby(seq))
+        if len(seq) < 2*k or len(seq_hpol_comp) < args.k:
+            continue
+        ########################
+    
+        exp_errors_in_kmers = expected_number_of_erroneous_kmers(qual, k)
+        p_no_error_in_kmers = 1.0 - exp_errors_in_kmers/ float((len(seq) - k +1))
+        score =  p_no_error_in_kmers  * (len(seq) - k +1)
+        
+        ## For (inferred) average error rate only, based on quality values
+        ### These values are used in evaluations in the paper only, and are not used in clustering
+        poisson_mean = sum([ qual.count(char_) * D_no_min[char_] for char_ in set(qual)])
+        error_rate = poisson_mean/float(len(qual))
+        if 10*-math.log(error_rate, 10) <= q_threshold:
+            continue
+        error_rates.append(error_rate)
+        ##############################################
+        
+        read_array.append((acc, seq, qual, score) )
+
+    read_array.sort(key=lambda x: x[3], reverse=True)
+    return read_array, error_rates
+
+
+
+def main(args):
+    start = time()
+    logfile = open(os.path.join(args.outfolder, "logfile.txt"), 'w')
+    if os.path.isfile(args.outfile) and args.use_old_sorted_file:
+        logging.warning("Using already existing sorted file in specified directory, in not intended, specify different outfolder or delete the current file.")
+        return args.outfile
+
+    elif args.fastq:
+        if args.nr_cores > 1: 
+            read_array, error_rates = fastq_parallel(args)
+        else:
+            read_array, error_rates = fastq_single_core(args)
+
+
+
+    reads_sorted_outfile = open(args.outfile, "w")
+    for i, (acc, seq, qual, score) in enumerate(read_array):
+        reads_sorted_outfile.write("@{0}\n{1}\n+\n{2}\n".format(acc + "_{0}".format(score), seq, qual))
+    reads_sorted_outfile.close()
+    logging.debug(f"{len(read_array)} reads passed quality critera (avg phred Q val over {args.quality_threshold} and length > 2*k) and will be clustered.")
+    error_rates.sort()
+    min_e = error_rates[0]
+    max_e = error_rates[-1]
+    median_e = error_rates[int(len(error_rates)/2)]
+    mean_e = sum(error_rates)/len(error_rates)
+    logfile.write("Lowest read error rate:{0}\n".format(min_e))
+    logfile.write("Highest read error rate:{0}\n".format(max_e))
+    logfile.write("Median read error rate:{0}\n".format(median_e))
+    logfile.write("Mean read error rate:{0}\n".format(mean_e))
+    logfile.write("\n")
+    logfile.close()
+    logging.debug("Sorted all reads in {0} seconds.".format(time() - start) )
+    return reads_sorted_outfile.name
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Evaluate pacbio IsoSeq transcripts.")
+    reads_file = parser.add_mutually_exclusive_group(required=True)
+    reads_file.add_argument('--fastq', type=str, help='Path to consensus fastq file(s)')
+    reads_file.add_argument('--use_old_sorted_file', action='store_true', help='Using already existing sorted file if present in specified output directory.')
+    parser.add_argument('--outfile', type=str,  default=None, help='A fasta file with transcripts that are shared between samples and have perfect illumina support.')
+    parser.add_argument('--k', type=int, default=15, help='kmer size')
+    parser.add_argument('--debug', action='store_true', help='Enable debug logging')
+
+    args = parser.parse_args()
+
+    loglevel = logging.debug if args.debug else logging.INFO
+
+    logging.basicConfig(
+        level=loglevel,
+        format='%(message)s'
+    )
+
+    if len(sys.argv)==1:
+        parser.print_help()
+        sys.exit()
+    path_, file_prefix = os.path.split(args.outfile)
+    help_functions.mkdir_p(path_)
+
+    main(args)

modules/help_functions.py

@@ -0,0 +1,104 @@
+import os
+import errno
+import re
+import logging
+import sys
+
+
+
+'''
+    Below code taken from https://github.com/lh3/readfq/blob/master/readfq.py
+'''
+
+def readfq(fp): # this is a generator function
+    last = None # this is a buffer keeping the last unprocessed line
+    while True: # mimic closure; is it a bad idea?
+        if not last: # the first record or a record following a fastq
+            for l in fp: # search for the start of the next record
+                if l[0] in '>@': # fasta/q header line
+                    last = l[:-1] # save this line
+                    break
+        if not last: break
+        name, seqs, last = last[1:], [], None
+        for l in fp: # read the sequence
+            if l[0] in '@+>':
+                last = l[:-1]
+                break
+            seqs.append(l[:-1])
+        if not last or last[0] != '+': # this is a fasta record
+            yield name, (''.join(seqs), None) # yield a fasta record
+            if not last: break
+        else: # this is a fastq record
+            seq, leng, seqs = ''.join(seqs), 0, []
+            for l in fp: # read the quality
+                seqs.append(l[:-1])
+                leng += len(l) - 1
+                if leng >= len(seq): # have read enough quality
+                    last = None
+                    yield name, (seq, ''.join(seqs)); # yield a fastq record
+                    break
+            if last: # reach EOF before reading enough quality
+                yield name, (seq, None) # yield a fasta record instead
+                break
+
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+        logging.debug(f"creating {path}")
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+
+def cigar_to_seq(cigar, query, ref):
+    cigar_tuples = []
+    result = re.split(r'[=DXSMI]+', cigar)
+    i = 0
+    for length in result[:-1]:
+        i += len(length)
+        type_ = cigar[i]
+        i += 1
+        cigar_tuples.append((int(length), type_ ))
+
+    r_index = 0
+    q_index = 0
+    q_aln = []
+    r_aln = []
+    for length_ , type_ in cigar_tuples:
+        if type_ == "=" or type_ == "X":
+            q_aln.append(query[q_index : q_index + length_])
+            r_aln.append(ref[r_index : r_index + length_])
+
+            r_index += length_
+            q_index += length_
+        
+        elif  type_ == "I":
+            # insertion w.r.t. reference
+            r_aln.append('-' * length_)
+            q_aln.append(query[q_index: q_index + length_])
+            #  only query index change
+            q_index += length_
+
+        elif type_ == 'D':
+            # deletion w.r.t. reference
+            r_aln.append(ref[r_index: r_index + length_])
+            q_aln.append('-' * length_)
+            #  only ref index change
+            r_index += length_
+        
+        else:
+            logging.error("Error processing cigar")
+            logging.error(cigar)
+            sys.exit()
+
+    return  "".join([s for s in q_aln]), "".join([s for s in r_aln])
+
+
+
+
+
+
+