tb-consensus-aligner 1.0.0__py3-none-any.whl

tb_consensus_aligner/consensus_galaxy.py

@@ -0,0 +1,491 @@
+import argparse
+import re
+import os
+from collections import defaultdict
+from pathlib import Path
+from Bio import SeqIO
+import gzip
+'''
+This script creates consensus fastas from a collection of VCF files and the MTBC ancestor reference genome.
+SNPs with a frequency >90% are encoded with the alternative base from the VCF.
+SNPs with a frequency between 10% and 90% are encoded with the ambiguity base.
+SNPs with a frequency <10% are encoded with the ancestral base from the reference genome.
+
+Large deletions (coverage 0) are encoded by dashes (-).
+Small deletions with frequency >90% are encoded by dashes (-).
+Small deletions with frequency between 10% and 90% are encoded with a 'N'.
+Small deletions with frequency <10% are encoded with the ancestral base from the reference genome.
+
+Small insertions (alternative bases longer than reference base) are encoded with the ancestral state from the reference genome.
+
+Sites to exclude specified in the bed files are encoded with a 'N'.
+Variants that do not have the 'PASS' quality filter are encoded with a 'N'.
+Sites that are not in the VCF and are covered by less than 5 reads (via depth file) are encoded with a 'N'.
+
+'''
+
+# Extract the basename to use in output file naming and consensus file filling
+def get_basename(vcf_path):
+    # Get the filename from the path (e.g., 'input_vcfs/Sample_01.recal.vcf.gz')
+    filename = os.path.basename(vcf_path)
+    
+    # Use regex to "look ahead" for .vcf and take everything before it.
+    match = re.split(r'\.vcf', filename, flags=re.IGNORECASE)
+    
+    return match[0]
+
+def flatten(t):
+    return [item for sublist in t for item in sublist]
+
+
+# Split a string of characters into a list of the individual characters (list comprehension) useful to examine each base of f.e. ACTG individually
+def split(word):
+    return [char for char in word]
+
+
+# Function to open either gzipped or unzipped vcf files
+def open_vcf(path):
+
+    path = Path(path)
+
+    if path.suffix == ".gz":
+        return gzip.open(path, "rt")
+    else:
+        return open(path, "r")
+    
+
+# Function to get the length of the reference sequence in case of testing with a smaller reference genome
+def get_reference_length(args):
+    record = SeqIO.read(str(args.reference), "fasta")    # Create object with key info of the sequence
+    return len(record.seq)
+
+# Parse the BED files to get a dictionary of positions to exclude
+def get_pos_to_exclude(bed_files):
+
+    pos_to_exclude = {}
+    if bed_files:
+
+        for bed_path in bed_files:
+            bed_path = Path(bed_path)
+
+            with bed_path.open() as file:
+                table = [position.strip().split('\t') for position in file] # Create a list of lists, each representing a line with the entries as items
+
+                for i in range(1,len(table)):           # Iterate over every element of [table] -> line in BED
+                    StartPosition = int(table[i][1])    
+                    EndPosition = int(table[i][2])
+
+                    ranges_of_coordinates = [i for i in range(StartPosition, EndPosition)]  # Create a list of every position of the interval of a BED line
+
+                    for pos in ranges_of_coordinates:   # If the position of the range is not yet in the dict, add it
+                        if pos not in pos_to_exclude:
+                            pos_to_exclude[pos] = ''
+                        else:
+                            continue
+            
+    return dict(sorted(pos_to_exclude.items()))
+   
+        
+# Create a dictionary where the keys are the genomic positions and values are the bases of the reference genome
+def fasta2dict(args):
+
+    fasta_dict = {}
+
+    record = SeqIO.read(str(args.reference), "fasta")    # Create object with key info of the sequence
+    number_of_refbases = len(record.seq)
+
+    for i in range(number_of_refbases):     # Add the each refbase as value to the key (genomic position)
+        fasta_dict[i+1]=record.seq[i]
+
+    return(fasta_dict)
+
+# Create a dictionary from the VCFs where POS is the key and REF, ALT, QUAL and AF(as list) are values
+def vcf2dict(vcf_files):
+
+    vcf_dict = {}   
+    
+    for vcf_path in vcf_files:
+        vcf_path = Path(vcf_path)
+
+        with open_vcf(vcf_path) as vcf_file:
+
+            for row in vcf_file:
+
+                if row.startswith("#"):
+                    continue
+
+                row = row.strip().split('\t')
+
+                position = row[1]
+                ref = row[3]
+                alt = row[4]
+                qual = float(row[5])
+
+                # Compute the AF from AO and RO for one or multiple ALT alleles with AF = AO / AO + RO
+                value_fields = row[-1].split(":") # Split column with the values at the :
+                RO = int(value_fields[2])    # As int to do calculations with it
+                AO_list = [int(ao) for ao in value_fields[4].split(",")]  # A list in case calculation of AF has do be done for multiple alleles
+
+                AF = [ao / (sum(AO_list) + RO) for ao in AO_list]   # sum(AO_list) to accomodate possibility of many alleles
+
+                vcf_dict[int(position)] = [ref,alt,qual,AF]
+        
+    return(vcf_dict)
+
+# Parse the depth file to get a nested dictionary with with the depths of each genomic position for each BAM file 
+# {{ BAM0: {pos1: depth1, pos2: depth2, ...}, BAM1: {pos1: depth1, pos2: depth2, ...}}
+def get_sample_depth(depth_file_path, column_index):
+    """
+    column_index: the integer index of the sample (0, 1, 2...) 
+    This corresponds to the order Galaxy passed the VCFs.
+    """
+    sample_depths = {}
+    with open(depth_file_path, 'rt') as f:
+        # Skip the header
+        header = f.readline() 
+        
+        # We add 2 to column_index because:
+        # Col 0 = CHROM, Col 1 = POS, Col 2 = First Sample (Index 0)
+        actual_col = column_index + 2
+        
+        for line in f:
+            if line.startswith("#"):
+                continue
+            parts = line.strip().split("\t")
+            # Pull the POS and the specific DEPTH column
+            sample_depths[int(parts[1])] = int(parts[actual_col])
+            
+    return sample_depths
+
+# Define the ambiguity bases to return ambiguity base if SNP between AF 10% and 90%
+def ambiguity_code(ref, alt):
+
+    ambiguity_base = ''
+
+    if ref == 'C' and alt == 'T':
+        ambiguity_base = 'Y'
+
+    elif ref == 'T' and alt == 'C':
+        ambiguity_base = 'Y'
+
+    elif ref == 'A' and alt == 'G':
+        ambiguity_base = 'R'
+
+    elif ref == 'G' and alt == 'A':
+        ambiguity_base = 'R'
+
+    elif ref == 'A' and alt == 'T':
+        ambiguity_base = 'W'
+
+    elif ref == 'T' and alt == 'A':
+        ambiguity_base = 'W'
+
+    elif ref == 'G' and alt == 'C':
+        ambiguity_base = 'S'
+
+    elif ref == 'C' and alt == 'G':
+        ambiguity_base = 'S'
+
+    elif ref == 'T' and alt == 'G':
+        ambiguity_base = 'K'
+
+    elif ref == 'G' and alt == 'T':
+        ambiguity_base = 'K'
+
+    elif ref == 'C' and alt == 'A':
+        ambiguity_base = 'M'
+
+    elif ref == 'A' and alt == 'C':
+        ambiguity_base = 'M'
+
+    elif ref == alt:
+        ambiguity_base = ref
+    
+    return(ambiguity_base)
+
+# Main function to loop over VCFs in input directory and create the respective consensus genomes
+
+def main(args):
+    
+    REF = fasta2dict(args)
+    EXCLUDED_POS = get_pos_to_exclude(args.bed_files)
+    print(EXCLUDED_POS)
+
+    # Define the vcf_dir from the argument -v/vcf, sort the vcf files alphanumerically, extract all keys (columns of depth file) alphanumerically sorted
+    
+    vcf_files = args.vcf_files
+    
+
+    # Loop over each vcf file with the matching column from the depth file
+    for sample_idx, vcf_file in enumerate(vcf_files):
+
+        VCF = vcf2dict([vcf_file])
+        VCF_DEPTH = get_sample_depth(args.depth, sample_idx)
+
+
+        fasta_sequence = []
+
+        i = 1
+
+        # Go over each position in the reference sequence and modify the base if it is in the VCF
+        while(i <= len(REF)):
+
+            if i in EXCLUDED_POS:   # Check if the position falls in the BED file
+
+                fasta_sequence += ['N']
+                i += 1
+                
+
+            else:
+
+                if i not in VCF:    # Position not in VCF -> must be ancestral or deletion
+
+                    if i in VCF_DEPTH:
+
+                        if 1 <= VCF_DEPTH[i] <=5:       # Covered by less than 5 reads -> N
+                            fasta_sequence += ['N']
+                            i += 1
+                        
+                        elif VCF_DEPTH[i] == 0:         # Covered by 0 reads -> deletion
+                            fasta_sequence += ['-']
+                            i += 1
+
+                        elif VCF_DEPTH[i] > 5:          # Not in VCF but covered >5 reads -> ancestral base
+                            fasta_sequence += [REF[i]]
+                            i += 1
+                        
+                elif i in VCF:  # Position is in VCF -> Variant
+
+                    # for i define:
+                    reference_base = VCF[i][0]
+                    alternative_base = VCF[i][1]
+                    quality = VCF[i][2]
+                    allele_freq = VCF[i][3]
+
+                    if quality >= 20:   # Variants with a 99% confidence (phred-score >20)
+
+                        if len(allele_freq) == 1:  # Only one ALT allele
+
+                            allele_freq = allele_freq[0]
+
+                            if len(alternative_base) == 1 and len(reference_base) == 1: # REF and ALT = 1 -> SNP
+
+                                if allele_freq >= 0.90: # Take alt base from VCF
+                                    fasta_sequence +=[alternative_base]
+                                    i += 1
+                                    
+                                elif 0.10 <= allele_freq < 0.90:    # Take ambiguity base
+                                    fasta_sequence += [ambiguity_code(reference_base,alternative_base)]
+                                    i += 1
+
+                                elif allele_freq < 0.10:    # Take the ancestral base
+                                    fasta_sequence += [REF[i]]
+                                    i += 1
+                            
+
+                            elif len(alternative_base) < len(reference_base):   # Small deletions
+
+                                if len(alternative_base) == 1 and len(reference_base) > 1:
+                                    small_deletion_length = len(reference_base)    # Get length of REF to skip correct number of bases
+
+                                    if allele_freq >=0.90:  # Encode with "-"
+                                        fasta_sequence += [alternative_base[0]]
+                                        fasta_sequence += split('-'*(small_deletion_length-1))
+                                        i += small_deletion_length
+
+                                    elif 0.10 <= allele_freq < 0.90:    # Take ambiguity base
+                                        fasta_sequence += split('N'*small_deletion_length)
+                                        i += small_deletion_length
+
+                                    elif allele_freq < 0.10:    # take reference base
+                                        fasta_sequence += [REF[i]]
+                                        i += 1
+                                
+                                if len(alternative_base) > 1 and len(reference_base) > 1:   # If both REF & ALT are more than one base but still REF>ALT
+                                    fasta_sequence += [REF[i]]
+                                    i += 1
+
+                            elif len(alternative_base) > len(reference_base):   # Small insertion
+                                fasta_sequence += [REF[i]]
+                                i += 1
+                            
+                            elif (len(alternative_base) > 1 ) and (len(reference_base) > 1) and (len(alternative_base) == len(reference_base)): # MNP
+
+                                length_mnp = len(alternative_base)
+
+                                if allele_freq >= 0.90: # Take alt bases from VCF
+                                    fasta_sequence += split(alternative_base)
+                                    i += length_mnp
+                                    
+                                elif 0.10 <= allele_freq < 0.90:    # Take ambiguity bases
+                                    
+                                    for f, b in zip(reference_base, alternative_base):
+                                        fasta_sequence += [ambiguity_code(f, b)]
+                                    i += length_mnp
+
+                                elif allele_freq < 0.10:    # Take the ancestral bases
+                                    fasta_sequence += split(reference_base)
+                                    i += len(reference_base)
+
+                        elif len(allele_freq) > 1:  # More than one ALT allele, we consider the ALT allele with the highest allele frequency
+
+                            highest_AF = max(allele_freq)   # Find highest AF
+
+                            alternative_base = alternative_base.split(',')  # Get different alleles split by ','
+
+                            index_of_nucleotide_with_highest_AF = allele_freq.index(highest_AF) # Index of the allele with highest AF
+
+                            ALT_allele_with_highest_frequency = alternative_base[index_of_nucleotide_with_highest_AF]   # ALT allele for highest AF
+
+                            print(i,VCF[i],highest_AF,split(ALT_allele_with_highest_frequency),len(ALT_allele_with_highest_frequency),reference_base,REF[i])
+
+                            # Now again check for SNPs, deletions, insertions and MNPs
+                            if len(ALT_allele_with_highest_frequency) == 1 and len(reference_base) == 1: # SNP
+
+                                if highest_AF >= 0.90: # Take ALT base
+                                    fasta_sequence += [ALT_allele_with_highest_frequency]
+                                    i += 1
+
+                                elif 0.10 <= highest_AF < 0.90: # take ambiguity_base
+                                    fasta_sequence += [ambiguity_code(reference_base,ALT_allele_with_highest_frequency)]
+                                    i += 1
+
+                                elif highest_AF < 0.10: # Take reference base
+                                    fasta_sequence += [REF[i]]
+                                    i +=1
+
+                            elif len(ALT_allele_with_highest_frequency) < len(reference_base):  # Small deletion 
+
+                                if len(ALT_allele_with_highest_frequency) == 1 and len(reference_base) > 1: # Small deletion with ALT allele = 1bp
+                                    small_deletion_length = len(reference_base) # To move correct amount of bases
+
+                                    if highest_AF >= 0.90:  # Encode with a '-'
+                                        fasta_sequence += [ALT_allele_with_highest_frequency[0]]
+                                        fasta_sequence += split('-'*(small_deletion_length-1))
+                                        i += small_deletion_length
+
+                                    elif 0.10 <= highest_AF < 0.90: # Encode with N
+                                        fasta_sequence += split('N'*small_deletion_length)
+                                        i += small_deletion_length
+
+                                    elif highest_AF < 0.10: # Take reference base
+                                        fasta_sequence += [REF[i]]
+                                        i += 1
+
+                                if len(ALT_allele_with_highest_frequency) > 1 and len(reference_base) > 1:  # Although ALT < REF, ALT > 1
+                                    fasta_sequence += [REF[i]]
+                                    i += 1
+
+                            elif len(ALT_allele_with_highest_frequency) > len(reference_base): # Small insertion, no matter AF -> take anc base
+                                fasta_sequence += [REF[i]]
+                                i += 1
+
+                            elif (len(ALT_allele_with_highest_frequency) > 1) and (len(reference_base) > 1) and (len(ALT_allele_with_highest_frequency) == len(reference_base)): # MNP
+
+                                length_mnp = len(ALT_allele_with_highest_frequency)
+
+                                if highest_AF >= 0.90:  # Take alt base
+                                    fasta_sequence += split(ALT_allele_with_highest_frequency)
+                                    i += length_mnp
+
+                                elif 0.10 <= highest_AF < 0.90: # Take ambiguity base
+                                    for f, b in zip(reference_base, ALT_allele_with_highest_frequency):
+                                        fasta_sequence += [ambiguity_code(f, b)]
+                                    i += length_mnp
+
+                                elif highest_AF < 0.10: # Take reference base
+                                    fasta_sequence += split(reference_base)
+                                    i += len(reference_base)
+                                    
+
+                    else:   # QUAL < 20
+                        fasta_sequence += ['N']
+                        i += 1
+                    
+                else:   # Pos i not in VCF
+                    print("i not in VCF",i,VCF[i])
+                    break
+
+
+        print("Lenght of sequence:",len(flatten(fasta_sequence)))
+
+        # Find and print empty strings in the list fasta_sequence
+        for b in range(len(fasta_sequence)):
+            if len(fasta_sequence[b]) == 0:
+                print(b, fasta_sequence[b])
+
+        # Check if the script is running in test mode (shorter genomes)
+        if args.test_mode:
+
+            print("Running in test mode - smaller genomes accepted")
+            reference_length = get_reference_length(args)
+
+            # Created consensus sequence too short
+            if len(fasta_sequence) < reference_length:
+                print("Fasta file not created")
+                print("Fasta has less than {reference_length} bp:", len(fasta_sequence))
+
+            # Consensus sequence has correct length
+            elif len(fasta_sequence) == reference_length:
+                
+                sample_name = get_basename(vcf_file)
+                output_filename = f"{sample_name}.consensus.fasta"
+                output_path = os.path.join(args.output_dir, output_filename)
+                with open(output_path, 'w') as output_file:
+                    output_file.write(f">{sample_name}\n")
+                    output_file.write("".join(fasta_sequence))
+        
+            # Consensus sequence too long
+            else:
+                print("Fasta file not created")
+                print("Fasta more than {reference_length} bp:", len(fasta_sequence))
+
+        else: 
+            # Default reference length is set to ancestral MTBC ref length
+            # Get calculated ref len from object args if not available set to default 4411532
+            target_len = getattr(args, 'calculated_ref_len', 4411532)
+
+            # Created consensus sequence too short
+            if len(fasta_sequence) < target_len:
+                print("Fasta file not created")
+                print(f"Fasta has less than {target_len} bp: {len(fasta_sequence)}")
+            
+            # Consensus sequence has correct length
+            elif len(fasta_sequence) == target_len:
+                sample_name = get_basename(vcf_file)
+                output_filename = f"{sample_name}.consensus.fasta"
+                output_path = os.path.join(args.output_dir, output_filename)
+                with open(output_path, 'w') as output_file:
+                    output_file.write(f">{sample_name}\n")
+                    output_file.write("".join(fasta_sequence))
+        
+            # Consensus sequence too long
+            else:
+                print("Fasta file not created")
+                print(f"Fasta more than {target_len} bp: {len(fasta_sequence)}")
+        
+
+        
+
+
+        
+        
+                    
+
+                            
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

tb_consensus_aligner/main_galaxy.py

@@ -0,0 +1,161 @@
+import os
+import glob
+import sys
+import argparse
+from collections import defaultdict
+from pathlib import Path
+from Bio import SeqIO
+import gzip
+from . import consensus_galaxy
+from . import snp_aligner_galaxy
+
+'''
+
+This script is the main script for the "insert name here". 
+It is built on top of consensus.py and snp_aligner.py.
+
+The user can choose the following with the -s option.
+-s all:         Input = VCFs, output = SNP alignment -> runs consensus.py and snp_aligner.py subsequently.
+-s consensus:   Input = VCFs, output = consensus fasta files -> runs just consensus.py
+When -s all is chosen, -m option gives choice whether to just output the SNP alignment
+with -m alignment_only or both the SNP alignment and the consensus fasta files with  -m everything
+
+'''
+
+# Define arguments used in the script
+def get_args():
+
+    parser = argparse.ArgumentParser(description='Main script for consensus.py and snp_aligner.py')
+
+    # Use dest to set the name of the argument for further handling and clarity 
+    parser.add_argument('-s', choices=['consensus', 'all'],dest='step', help='Run either consensus, or both consensus and snp_aligner', required=True)
+    parser.add_argument('-m', choices=['alignment_only', 'everything'],dest='mode', help='If -s all then output either just SNP alignment or both the alignment and the consensus fasta files', required=False)
+    parser.add_argument('-v', action='append', dest='vcf_files',help='path to the input directory with all the vcf files', required=True)
+    parser.add_argument('-r', dest='reference',help='path to reference genome file', required=True)
+    parser.add_argument('-d',dest='depth',help='Depth file per position, output of samtools depth', required= True)
+    parser.add_argument('-c', dest='outgroup_vcf', help='Outgroup VCF required for variable alignment', required=False)
+    parser.add_argument('-n', dest='outgroup_name', type=str, help='Clean name for outgroup header in Galaxy')
+    parser.add_argument('-b', dest='bed_files', action='append', default=[], help='Optional BED files to mask certain genomic regions', required=False)
+    parser.add_argument('-o', dest='output_dir', help='Output directory for consensus files and the variable alignment' ,required=False)
+    parser.add_argument('-g', dest='undefined_states', help='Percentage of undefined states allowed per polymorphic position in the alignment', type=float, default=0.9, required=False)
+    parser.add_argument('-t', dest='test_mode',help='allows to use smaller files with less genomic positions', action='store_true')
+
+
+    # Run the parser and place data in the parser object for later use
+    args = parser.parse_args()
+
+    return args
+
+
+# Check if all arguments for the consensus script are there
+def check_consensus_args(args):
+    
+    required_consensus_args = ['vcf_files', 'reference', 'depth']
+
+    # List comprehension to iterate over field in required_consensus_args and if the field is missing, adds it to the list
+    missing_consensus_args = [field for field in required_consensus_args if getattr(args, field, None) is None]
+
+    # If Missing_consensus_args has elements, it raises and error and outputs what arguments are missing, separated by a ','
+    if missing_consensus_args:
+        raise ValueError(f"Missing required arguments: {', '.join(missing_consensus_args)}")
+
+# Check if all arguments for the snp_aligner script are there
+def check_snp_aligner_args(args):
+
+    required_snp_aligner_args = ['outgroup_vcf', 'undefined_states']
+
+    # List comprehension to iterate over field in required_snp_aligner_args and if the field is missing, adds it to the list
+    missing_snp_aligner_args = [field for field in required_snp_aligner_args if getattr(args, field, None) is None]
+
+    # If missing_snp_aligner_args has elements, it raises and error and outputs what arguments are missing, separated by a ','
+    if missing_snp_aligner_args:
+        raise ValueError(f"Missing required arguments: {', '.join(missing_snp_aligner_args)}")
+
+# Function to prevent crashing by checking for 1) reference genome length and 2) amounts of variants in VCF
+def safety_check(args, max_total_variants=200000, max_ref_length=16100000):
+
+    # If a VCF has more than 200'000 entries, it is most likely not bacterial or filled with unfiltered sequencing errors
+    # The largest bacterial genome is Minicystis rosea with 16.04 Mbp
+
+    print("Executing safety check")
+
+    # Check length of reference genome and throw error if file not parsed or too long
+    try:
+        record = SeqIO.read(str(args.reference), "fasta")
+        genome_len = len(record.seq)
+    except Exception as e:
+        sys.exit(f"ERROR: Failed to parse the reference genome: {e}")
+
+    if genome_len > max_ref_length:
+        sys.exit(f"ERROR: Reference genome length ({genome_len} bp) exceeds the maximum allowed bacterial ceiling of {max_ref_length} bp.")
+
+    # Check number of variants in each VCF file
+    for vcf_path in args.vcf_files:
+        try:
+            with consensus_galaxy.open_vcf(vcf_path) as file:
+                variant_count = sum(1 for line in file if not line.startswith('#'))
+
+            # If one VCF file is to big, throw error
+            if variant_count > max_total_variants:
+                sys.exit(
+                    f"ERROR: Variant overload. File '{os.path.basename(vcf_path)}' "
+                    f"contains {variant_count} variant lines.\n"
+                    f"This exceeds the safety limit of {max_total_variants} variants per sample. "
+                    f"Filter or remove this VCF running the pipeline."
+                )
+        
+        except Exception as e:
+            sys.exit(f"ERROR: Failed reading VCF file {vcf_path}: {e}")
+    
+    print(f"Safety check passed, Reference size: {genome_len} bp. Variants of all VCFs within safety margin.")
+    return genome_len
+
+
+# Main logic of the program
+def main():
+
+    args = get_args()
+
+    if args.output_dir is None:
+        args.output_dir = "output"
+
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Find input files and attribute them to arguments
+    #args.reference, args.depth, args.outgroup_vcf, args.vcf_files, args.bed_files = find_input_files(args.input_dir)
+    #print("REFERENCE:", args.reference)
+    #print("DEPTH:", args.depth)
+    #print("OUTGROUP:", args.outgroup_vcf)
+    #print("VCFS:", args.vcf_files)
+    #print("BEDS:", args.bed_files)
+
+    # Execute safety check and get the length of the reference genome
+    ref_genome_length = safety_check(args)
+
+    # Relay reference genome length to consensus_galaxy
+    args.calculated_ref_len = ref_genome_length
+
+    # -s consensus: Input = VCFs, output = consensus fasta files -> runs just consensus.py
+    if args.step == 'consensus':
+
+        check_consensus_args(args)
+
+        consensus_galaxy.main(args)
+
+    # -s all: Input = VCFs, output = SNP alignment -> runs consensus.py and snp_aligner.py subsequently
+    elif args.step == 'all':
+        
+        check_consensus_args(args)
+        check_snp_aligner_args(args)
+
+        consensus_galaxy.main(args)
+        snp_aligner_galaxy.main(args)
+
+        # After consensus is run and -m alignment_only is chosen, delete the fasta files again
+        if args.mode == 'alignment_only':
+
+            for file in glob.glob(os.path.join(args.output_dir, "*.fasta")):
+                os.remove(file)
+
+if __name__ == '__main__':
+    main()

tb_consensus_aligner/snp_aligner_galaxy.py

@@ -0,0 +1,264 @@
+import argparse
+import os
+from collections import defaultdict
+from pathlib import Path
+from Bio import SeqIO
+from pprint import pprint
+import gzip
+'''
+
+This script creates a SNP alignment in multi fasta format from a collection of consensus genomes
+one of which should be the outgroup if intended to be used later for phylogenetic analyses.
+
+The user can choose, how many undefined states ('-' or 'N') for a polymorphic position are accepted with
+the -g argument. By default, if a position has more than 90% undefined states, it will be excluded (-g 0.9).
+For a polymorphic site with Ns or - or both to be included in the final SNP alignment, the site has to be poly-
+morphic in terms of at least 2 different bases, plus Ns or -s or both. 
+A site with just one base and Ns and or -s is not considered polymorphic.
+
+'''
+  
+
+def flatten(t):
+    return [item for sublist in t for item in sublist]
+
+
+# Function to open both zipped and unzipped vcf files
+def open_vcf(path):
+    path = Path(path)
+    # Check the first two bytes for the gzip magic number
+    with open(path, 'rb') as f:
+        is_gzip = f.read(2) == b'\x1f\x8b'
+    
+    if is_gzip:
+        return gzip.open(path, "rt")
+    else:
+        return open(path, "r")
+
+# Create a dictionary of fasta sequences where the sample name is the key and the value the string of bases
+def fastas2dict(args):
+    sequences_dict = {}
+    consensus_dir = Path(args.output_dir)
+    consensus_files = sorted(consensus_dir.glob("*.fasta"))
+
+    for fasta in consensus_files:
+        with fasta.open() as fh:
+            header = None
+            seq_parts = []
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                if line.startswith(">"):
+                    header = line[1:]
+                else:
+                    seq_parts.append(line)
+            
+            if header:
+                # Join only once at the very end for this sample
+                sequences_dict[header] = "".join(seq_parts)
+                print(f"Loaded {header}: {len(sequences_dict[header])} bases")
+    
+    return sequences_dict
+
+
+
+
+# Check if all sequences (values, string of characters) in a dictionary are the same length, return True if so
+def check_length(sequences_dict):
+
+    sequence_lengths = [len(seq) for seq in sequences_dict.values()]
+
+    return all(x == sequence_lengths[0] for x in sequence_lengths)
+
+
+# Create a 2D array from dictionary of sequences with sequence names at pos 0 and first genomic position at python pos 1
+def dict2array(sequences_dict):
+
+
+    print("DEBUG dict2array:")
+    print("  Number of sequences:", len(sequences_dict))
+    print("  Unique sequence lengths:", set(len(seq) for seq in sequences_dict.values()))
+
+    if check_length(sequences_dict) == True:
+        
+        array = [[name] + list(seq) for name, seq in sequences_dict.items()]
+        
+        
+
+        return array
+
+    else: 
+        print("Not all sequences are of equal length")
+        return None
+
+# Get positions from outgroup VCF corresponding to genomic positions of polymorphic sites found in alignment
+def get_outgroup_vcf_var_pos(args, var_positions):
+    outgroup_vcf_dict = {}
+    
+    # 1. Clean the outgroup name for the FASTA header
+    outgroup_name = args.outgroup_name if args.outgroup_name else Path(args.outgroup_vcf).stem
+    outgroup_name = outgroup_name.replace(".vcf", "").replace(".gz", "")
+    
+    # 2. Populate the dictionary FIRST so debug prints work
+    with open_vcf(args.outgroup_vcf) as outgroup_vcf:
+        for var in outgroup_vcf:
+            if var.startswith('#') or not var.strip():
+                continue
+
+            # Strict tab splitting and stripping to avoid hidden characters in Galaxy
+            fields = var.strip().split('\t')
+
+            if len(fields) < 5:
+                continue
+
+            # Force POS to be a clean string
+            pos = fields[1].strip()
+            ref = fields[3].strip()
+            alt = fields[4].strip()
+
+            outgroup_vcf_dict[pos] = [ref, alt]
+
+    # 3. Handle the polymorphic coordinates
+    # Ensure var_positions is a list of clean strings to match the dict keys
+    unique_var_positions = [str(p) for p in dict.fromkeys(var_positions)]
+
+    # DEBUG SECTION
+    print(f"DEBUG: Input positions count: {len(unique_var_positions)}")
+    print(f"DEBUG: Outgroup VCF dict entries: {len(outgroup_vcf_dict)}")
+    if unique_var_positions:
+        print(f"DEBUG: First search key: '{unique_var_positions[0]}'")
+    
+    outgroup_sequence = list()
+    for pos in unique_var_positions:
+        # Check dictionary using the clean string key
+        key = str(pos)
+
+        if key in outgroup_vcf_dict:
+            ref_val = outgroup_vcf_dict[pos][0]
+            alt_val = outgroup_vcf_dict[pos][1]
+
+            if alt_val == '.':   # No ALT allele -> use REF
+                outgroup_sequence.append(ref_val)
+            
+            elif alt_val != '.': # ALT allele present
+                if len(alt_val) > 1 or len(ref_val) > 1: # Indel
+                    outgroup_sequence.append('N')
+                else:   # SNP
+                    outgroup_sequence.append(alt_val)
+            print(f"MATCH FOUND: Position {key} is in Outgroup VCF. Base: {outgroup_vcf_dict[key]}")
+        else:
+            # Position not in VCF means it's the Reference base (or missing)
+            print(f"MATCH FAILED: Position {key} not found in Outgroup VCF keys.")
+            outgroup_sequence.append("-")
+
+    # Add the header at the start
+    result = [outgroup_name] + outgroup_sequence
+    
+    print(f"DEBUG: Finalll outgroup row length: {len(result)}")
+    return result
+
+
+# Main script to extract polymorphic positions from the alignment
+def main(args):
+    UNDEF_STAT = args.undefined_states  # Use the arg from your parser
+    SEQ_DICT = fastas2dict(args)
+    
+    seq_names = list(SEQ_DICT.keys())
+    num_seqs = len(seq_names)
+    genome_length = len(SEQ_DICT[seq_names[0]])
+
+    # Initialize pol_pos with sequence names as the first element of each row
+    pol_pos = [[name] for name in seq_names]
+    polymorphic_indices = []
+
+    print(f"Analyzing {num_seqs} sequences across {genome_length} bp...")
+
+    # Iterate through each genomic position one by one (Memory Efficient)
+    for col_index in range(genome_length):
+        # Build the column on the fly
+        column = [SEQ_DICT[name][col_index].replace('X', 'N') for name in seq_names]
+        
+        # Check for polymorphism (more than 1 unique state)
+        if len(set(column)) > 1:
+            
+            # Logic for adding column to alignment
+            should_add = False
+            
+            if 'N' not in column and '-' not in column:
+                should_add = True
+            else:
+                count_undef = column.count('N') + column.count('-')
+                if (count_undef / num_seqs) <= UNDEF_STAT:
+                    unique_states = set(column)
+                    # Your specific conditions for missing data sites
+                    if ('N' in column and '-' not in column) and len(unique_states) > 2:
+                        should_add = True
+                    elif ('-' in column and 'N' not in column) and len(unique_states) > 2:
+                        should_add = True
+                    elif ('N' in column and '-' in column) and len(unique_states) > 3:
+                        should_add = True
+
+            if should_add:
+                for i, base in enumerate(column):
+                    pol_pos[i].append(base)
+                # Store the 1-based genomic index
+                polymorphic_indices.append(col_index + 1)
+
+    print("\npol_pos preview (first 5 sequences, first 10 entries):")
+    for row in pol_pos[:5]:
+        print(row[:10])
+
+    # Append the outgroup sequence if provided via VCF
+    if args.outgroup_vcf:
+        # Use a clean, unique, and sorted list of indices
+        unique_indices = sorted(list(set(polymorphic_indices)))
+    
+        # Get the outgroup sequence row
+        outgroup_line = get_outgroup_vcf_var_pos(args, var_positions=unique_indices)
+
+        # Safety check: Row length must match (Name + Bases)
+        if len(outgroup_line) != len(pol_pos[0]):
+            print(f"CRITICAL ERROR: Outgroup row length ({len(outgroup_line)}) "
+                  f"mismatch with Sample row length ({len(pol_pos[0])})")
+        
+        pol_pos.append(outgroup_line)
+
+    print("Number of polymorphic sites:", len(pol_pos[0]) - 1)   
+    print("Number of polymorphic_indices:", len(polymorphic_indices))
+   
+    # Write the output file
+    output_alignment_path = os.path.join(args.output_dir, 'snp_alignment.fasta')
+    with open(output_alignment_path, 'w') as output_file:
+        for i, row in enumerate(pol_pos):
+            sequence_name = row[0]
+            # Convert everything to string just in case a None or list slipped in
+            sequence = "".join(str(base) for base in row[1:])
+
+            # Standard FASTA format: >Header\nSequence\n
+            output_file.write(f">{sequence_name}\n")
+            output_file.write(f"{sequence}\n")
+    
+    print(f"Successfully wrote {len(pol_pos)} sequences to {output_alignment_path}")
+
+
+
+
+
+
+
+
+
+
+        
+
+
+
+
+
+
+
+
+    
+
+

tb_consensus_aligner-1.0.0.dist-info/METADATA

@@ -0,0 +1,151 @@
+Metadata-Version: 2.4
+Name: tb_consensus_aligner
+Version: 1.0.0
+Summary: Building consensus fasta files and variable multi-sequence alignments for mycobacterial genomes.
+Author-email: scg40 <gian.schuepbach@swisstph.ch>
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: biopython>=1.80
+
+# TBConsensusAligner
+TBConsensusAligner is a tool designed to create a multi-sequence alignment via consensus 
+genomes created from VCF files. 
+It can currently be used in different modes as follows
+- VCF to SNP alignment `-s all`
+    - input = VCFs, output = SNP alignment
+    - runs `consensus_galaxy.py` and `snp_aligner_galaxy.py`
+
+- VCF to consensus FASTA files `-s consensus`
+    - input = VCFs, output = consensus FASTA files
+    - runs only `consensus_galaxy.py`
+
+If the script produces a SNP alignment from VCFs, the user can choose whether to ouptut just the SNP alignment
+or the used consensus FASTA files as well with the `-m` option. 
+
+When running the `consensus_galaxy.py` the user has to provide the VCF files, the reference genome used in their creation and the multisample
+depth file from bamtools depth. Optionally, the user can provide one or several BED files to mask certain regions of the genome.
+
+## Usage
+The script is run via the command line.
+
+***usage***:
+
+```
+Usage:   TBConsensusAligner [options] 
+
+Options: -s STR         ['consensus' or 'all'] Create either consensus files or consensus files and a variable alignment
+
+         -m STR         ['everything' or 'alignment_only'] If -s all is chosen, output either consensus files and the alignment or just the alignment
+
+         -v list[STR]   Path to the input VCF files
+
+         -r STR         Path to the reference genome
+
+         -d STR         Path to the depth file created by samtools depth
+
+         -c STR         Path to the outgroup VCF for the alignment
+
+         -b list[STR]   Path to the BED files to mask certain genomic regions
+
+         -o STR         Path to the output directory
+
+         -g FLOAT       Percentage of undefined states allowed per polymorphic position in the alignment default at 90 percent value=0.9
+
+         -t BOOLEAN     Test Mode to run the script with smaller genomes
+
+```
+
+## Example usages command line
+
+### From 3 VCFs to alignment, output consensus FASTAs and SNP alignment, masked by two bedfiles 
+
+```
+python yourfolder/galaxy_main.py \
+-s all \
+-m everything \
+-v path/to/vcf1/vcf1.vcf -v path/to/vcf2/vcf2.vcf -v path/to/vcf2/vcf2.vcf \
+-r path/to/reference/reference_genome.reference.fasta \
+-d path/to/depthfile/depthfile.tabular \
+-c path/to/outgroupvcf/outgroup.vcf \
+-b path/to/bedfile1/bed1.bed -b path/to/bedfile2/bed2.bed \
+-o output
+-g 0.9
+
+```
+
+## Logic consensus_galaxy.py 
+
+This script produces consensus FASTA files from VCFs. The user provides the VCFs, the reference genome,
+the multisample depth file obtained from `bamtools depth` and optionally BED files to mask certain genomic regions.
+
+***algorithm***
+
+We loop through each position in the reference genome and build the consensus sequence base per base with the following rules:
+
+```
+- SNPs and MNPs with a frequency >90% are encoded with the alternative base from the VCF.
+- SNPs and MNPs with a frequency between 10% and 90% are encoded with the ambiguity base.
+- SNPs and MNPs with a frequency <10% are encoded with the ancestral base from the reference genome.
+
+- Large deletions (coverage 0) are encoded by dashes (-).
+- Small deletions with frequency >90% are encoded by dashes (-).
+- Small deletions with frequency between 10% and 90% are encoded with a 'N'.
+- Small deletions with frequency <10% are encoded with the ancestral base from the reference genome.
+
+- Small insertions (alternative bases longer than reference base) are encoded with the ancestral state from the reference genome.
+
+- Sites to exclude specified in the bed files are encoded with a 'N'.
+- Variants that have a phred score < 20 are encoded with a 'N'.
+- Sites that are not in the VCF and are covered by less than 5 reads (via depth file) are encoded with a 'N'.
+
+- If more than one ALT allele is present, we consider the one with the highest allele frequency.
+```
+Since the VCF for which the script is tailored to does not have the allele frequency `AF` calculated, we calculate it using 
+`RO = REF allele occurance` and `AO = ALT allele occurance` with 
+`AF = AO / ( sum(all AO's) + RO )`.
+
+## Logic snp_aligner_galaxy.py
+
+This script produces a SNP alignment i.e. multi-sequence alignment of polymorphic positions in FASTA format from consensus FASTA files.
+The consensus FASTAs are generated in the previous step by `consensus_galaxy.py` wrapped in `TBConsensusAligner`.
+
+***algorithm***
+
+The script creates a multi-sequence alignment from the consensus FASTAs in form of an 2D array where each row represents a sequence
+and each column a genomic position. Each column is checked for the abundance of a polymorphism. If such a polymorphism is detected,
+the column is appended to the alignment of polymorphic positions. For each polymorphic position, the corresponding nucleotide from 
+outgroup VCF is retrieved to get the outgroup's sequence.
+
+The algorithm to populate the multi-sequence alignment of polymorphic positions is asd follows.
+The user can control the proportion of gaps or undefined states (`-`or`N`) for a polymorphic position
+to be kept in the alignment (argument `-g`). By default, this is set to `0.9`, meaning that if a polymorphic position has 
+more than 90% gaps or undefined states, it will not be in the final alignment.
+
+## Directory structure used for development
+
+* TBConsensusAligner
+    * test_data
+        - snp_alignment.fasta
+        - test_G77777.consensus.fasta
+        - test_G77777.vcf.gz
+        - test_G88888_k1.consensus.fasta
+        - test_G88888_k1.vcf.gz
+        - test_G99999.k2.consensus.fasta
+        - test_G99999.k2.vcf.gz
+        - test_Galaxy_multiple_depths_header.tabular
+        - test_reference_200bp.reference.fasta
+        - test_regions_blindspots_modlin_farhat_and_PE_PPE_PGRS.bed
+        - test.outgroup.all.pos.vcf.gz
+    - consensus_galaxy.py
+    - main_galaxy.py
+    - README.md
+    - snp_aligner_galaxy.py
+    - TBConsensusAligner.xml
+
+
+
+

tb_consensus_aligner-1.0.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+tb_consensus_aligner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tb_consensus_aligner/consensus_galaxy.py,sha256=NnwO3_rT43NquvkOFQR0si_ivbAP0JRfOn4o1ezkdJw,20165
+tb_consensus_aligner/main_galaxy.py,sha256=8m_73RdxwDM3W_sjtNtKyvxhEmMYWEXFp2BJoCd1kR4,7376
+tb_consensus_aligner/snp_aligner_galaxy.py,sha256=tv8J2LmBsHzr9K-T5fLhd82B7q7CYDJEuE3UDK38UkY,9218
+tb_consensus_aligner-1.0.0.dist-info/METADATA,sha256=2WWLuwLTnqcbzQXzKW9EhSREDX8CAVGMliPVb2z_ALk,6415
+tb_consensus_aligner-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+tb_consensus_aligner-1.0.0.dist-info/entry_points.txt,sha256=cBK81ccXI6DIOvhvxfZNr4v2j21Rdq_7HEJT_1RGKUw,79
+tb_consensus_aligner-1.0.0.dist-info/top_level.txt,sha256=Q8H2spBV19grmt3Mw6TTiZ_D-1CvaK0LeqkqTT84gbA,21
+tb_consensus_aligner-1.0.0.dist-info/RECORD,,

tb_consensus_aligner-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tb_consensus_aligner-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tb-consensus-aligner = tb_consensus_aligner.main_galaxy:main

tb_consensus_aligner-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+tb_consensus_aligner