PyamilySeq 0.5.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

PyamilySeq/Seq_Combiner.py

@@ -11,15 +11,16 @@ except (ModuleNotFoundError, ImportError, NameError, TypeError) as error:
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Seq-Combiner ' + PyamilySeq_Version + ': Seq-Combiner Run Parameters.')
+    parser = argparse.ArgumentParser(description='Seq-Combiner ' + PyamilySeq_Version + ': A tool to extract sequences from GFF/FASTA files.')
     ### Required Arguments
     required = parser.add_argument_group('Required Arguments')
     required.add_argument('-input_dir', action='store', dest='input_dir',
                           help='Directory location where the files are located.',
                           required=True)
-    required.add_argument("-input_type", action="store", dest="input_type", choices=['separate', 'combined'],
-                          help="Type of input files: 'separate' for separate FASTA and GFF files,"
-                             " 'combined' for GFF files with embedded FASTA sequences.",
+    required.add_argument('-input_type', action='store', dest='input_type', choices=['separate', 'combined', 'fasta'],
+                          help='Type of input files: "separate" for separate FASTA and GFF files,'
+                             ' "combined" for GFF files with embedded FASTA sequences and "fasta" for combining multiple '
+                               'FASTA files together.',
                           required=True)
     required.add_argument("-name_split", action="store", dest="name_split",
                           help="substring used to split the filename and extract the genome name ('_combined.gff3' or '.gff').",
@@ -30,15 +31,33 @@ def main():
     required.add_argument("-output_name", action="store", dest="output_file",
                           help="Output file name.",
                           required=True)
+    optional = parser.add_argument_group('Optional Arguments')
+    optional.add_argument('-gene_ident', action='store', dest='gene_ident', default='CDS',
+                          help='Default - "CDS": Identifier used for extraction of sequences such as "misc_RNA,gene,mRNA,CDS,rRNA,tRNA,tmRNA,CRISPR,ncRNA,regulatory_region,oriC,pseudo"'
+                               ' - Not compatible with "fasta" input mode.',
+                          required=False)
+    optional.add_argument('-translate', action='store_true', dest='translate', default=None,
+                          help='Default - False: Translate extracted sequences to their AA counterpart?',
+                          required=False)
+    misc = parser.add_argument_group('Misc Arguments')
+    misc.add_argument('-v', action='store_true', dest='version',
+                         help='Print out version number and exit',
+                         required=False)
+
     options = parser.parse_args()
 
+    if options.version:
+        sys.exit(PyamilySeq_Version)
+
     output_path = os.path.abspath(options.output_dir)
     combined_out_file = os.path.join(output_path, options.output_file)
 
     if options.input_type == 'separate':
-        read_separate_files(options.input_dir, options.name_split, )
-    else:
-        read_combined_files(options.input_dir, options.name_split, combined_out_file)
+        read_separate_files(options.input_dir, options.name_split, options.gene_ident, combined_out_file, options.translate)
+    elif options.input_type == 'combined':
+        read_combined_files(options.input_dir, options.name_split, options.gene_ident, combined_out_file, options.translate)
+    elif options.input_type == 'fasta':
+        read_fasta_files(options.input_dir, options.name_split, combined_out_file, options.translate)
 
 if __name__ == "__main__":
     main()

PyamilySeq/clusterings.py

@@ -0,0 +1,362 @@
+
+import sys
+import copy
+from collections import OrderedDict
+from collections import defaultdict
+from collections import Counter
+
+def cluster_CDHIT(options, splitter):
+    First_in = open(options.clusters, 'r')
+    clusters = OrderedDict()
+    pangenome_clusters_First = OrderedDict()
+    pangenome_clusters_First_genomes = OrderedDict()
+    pangenome_clusters_First_sequences = OrderedDict()
+    first = True
+    taxa_dict = defaultdict(int)
+    reps = OrderedDict()
+    tmp_genomes = None
+    ## Load in all data for easier reuse later
+    for line in First_in:
+        if line.startswith('>'):
+            if tmp_genomes != None:
+                pangenome_clusters_First_genomes[rep] = tmp_genomes
+            tmp_genomes = []
+            if first == False:
+                cluster_size = len(clusters[cluster_id])
+                reps.update({rep: [cluster_size, len(pangenome_clusters_First[cluster_id])]})
+            cluster_id = line.strip('>')
+            cluster_id = cluster_id.strip('\n')
+            cluster_id = cluster_id.split(' ')[1]
+            clusters.update({cluster_id: []})
+            pangenome_clusters_First.update({cluster_id: []})
+            pangenome_clusters_First_sequences.update({cluster_id: []})
+            first = False
+        else:
+            clustered = line.split('\t')[1]
+            clustered = clustered.split('>')[1]
+            clustered = clustered.split('...')[0]
+            taxa = clustered.split(splitter)[0]
+            taxa_dict[taxa] += 1
+            if '*' in line:
+                rep = clustered
+                reps.update({rep: [0, 0]})
+            if first == False:
+                clusters[cluster_id].append(clustered)
+                clustered_taxa = clustered.split(splitter)[0]
+                if clustered_taxa not in pangenome_clusters_First[cluster_id]:
+                    pangenome_clusters_First[cluster_id].append(clustered_taxa)
+                    tmp_genomes.append(clustered_taxa)
+                pangenome_clusters_First_sequences[cluster_id].append(clustered)
+
+    pangenome_clusters_First_genomes[rep] = tmp_genomes
+
+    return taxa_dict, pangenome_clusters_First, pangenome_clusters_First_genomes, pangenome_clusters_First_sequences, reps
+
+
+
+#@profile
+def combined_clustering_counting(options, pangenome_clusters_First, reps, combined_pangenome_clusters_First_Second_clustered, pangenome_clusters_First_genomes, splitter):
+    num_clustered_First = defaultdict(list)
+    pangenome_clusters_Type = copy.deepcopy(pangenome_clusters_First)
+    list_of_reps = list(reps.keys())
+    for cluster, First_genomes in pangenome_clusters_First.items():
+        rep = list_of_reps[int(cluster)]  # get the rep of the current pep cluster
+        Com_PEP_Genomes = 0
+        Seconds = 0
+        seen_Seconds = []
+        added_Second_genomes = 0
+        temp_pep_genomes = copy.deepcopy(First_genomes)
+        try:  # get the cluster from the Second clusters which contains this rep
+            clustered_combined = combined_pangenome_clusters_First_Second_clustered[rep]
+            #We have to do this to correctly account for Seconds grouping multiple original First clusters
+            for clust in clustered_combined:
+                ### Get the original clustered genomes first:
+                if options.sequence_tag not in clust:
+                    original_clustered_genomes = pangenome_clusters_First_genomes[clust]
+                    for genome in original_clustered_genomes:
+                        if genome not in temp_pep_genomes:
+                            temp_pep_genomes.append(genome)
+
+            seen_clust_Genomes = []
+            num_clustered_First[cluster].append(rep + '_' + str(len(First_genomes)))
+            for clust in clustered_combined:
+                if options.sequence_tag not in clust:  # Not good enough at the moment
+                    clust_Genome = clust.split(splitter)[0]
+                    if clust_Genome not in seen_clust_Genomes:
+                        seen_clust_Genomes.append(clust_Genome)
+                        if clust_Genome not in First_genomes:
+                            Com_PEP_Genomes += 1
+                    num_clustered_First[cluster].append(clust + '_' + str(reps[clust][1]))
+                elif options.sequence_tag in clust:
+                    Seconds += 1
+                    clust_Genome = clust.split(splitter)[0]
+                    if clust_Genome not in seen_Seconds:
+                        seen_Seconds.append(clust_Genome)
+                    if clust_Genome not in seen_clust_Genomes:
+                        seen_clust_Genomes.append(clust_Genome)
+                        if clust_Genome not in temp_pep_genomes:
+                            added_Second_genomes += 1
+                            temp_pep_genomes.append(clust_Genome)
+                else:
+                    sys.exit("Error: looking for sequence_tag")
+
+            size_of_pep_clusters = []
+            genomes = num_clustered_First[cluster]
+
+
+            if len(genomes) > 1: #!!# So that we don't double count - This still needs to account for whether the same genome/genus is present however. Probably need to unique ti
+               collecting_genomes = []
+               for genome in genomes:
+                   genome = genome.rsplit('_', 1)
+                   collecting_genomes.append(pangenome_clusters_First[str(list_of_reps.index(genome[0]))])
+                   size_of_pep_clusters.append([str(list_of_reps.index(genome[0])) + ':' + genome[1]])
+               flattened_list = [item for sublist in collecting_genomes for item in sublist]
+               element_counts = Counter(flattened_list)
+               unique_elements = [element for element, count in element_counts.items() if count == 1]
+               sum_size_of_pep_clusters = len(unique_elements)
+            else:
+                genome = genomes[0].rsplit('_', 1)
+                size_of_pep_clusters.append([str(list_of_reps.index(genome[0]))+':'+genome[1]])
+                sum_size_of_pep_clusters = int(genome[1])
+
+            pangenome_clusters_Type[cluster] = [len(num_clustered_First[cluster]), sum_size_of_pep_clusters,
+                                                size_of_pep_clusters, added_Second_genomes, Seconds, len(seen_Seconds)]
+
+        except KeyError:
+            ###Singleton
+            num_First_genomes = [[str(cluster)+':'+str(len(First_genomes))]]
+            pangenome_clusters_Type[cluster] = [1, len(First_genomes), num_First_genomes, added_Second_genomes, Seconds,
+                                                len(seen_Seconds)]
+    # pangenome_clusters_Type = [Number of First clustered genomes or genera, Size of the cluster, Ditto, Added Seconds,Number of Seconds,Unique Seconds ]
+    return pangenome_clusters_Type
+
+
+#@profile
+def single_clustering_counting(pangenome_clusters_First, reps):
+    num_clustered_First = defaultdict(list)
+    recorded_First = []
+    pangenome_clusters_Type = copy.deepcopy(pangenome_clusters_First)
+    list_of_reps = list(reps.keys())
+    for cluster, First_taxa in pangenome_clusters_First.items():
+        rep = list_of_reps[int(cluster)]  # get the rep of the current pep cluster
+
+        try:  # get the cluster from the storf clusters which contains this rep
+            num_clustered_First[str(cluster)].append(rep + '_' + str(len(First_taxa)))
+            size_of_First_clusters = []
+            Firsts = num_clustered_First[str(cluster)]
+            for First in Firsts:
+                First = First.rsplit('_', 1)
+                size_of_First_clusters.append(int(First[1]))
+                recorded_First.append(First[0])
+            num_First_genomes = [[str(cluster) + ':' + str(len(First_taxa))]]
+            pangenome_clusters_Type[cluster] = [len(num_clustered_First[cluster]), sum(size_of_First_clusters),
+                                                num_First_genomes, 0, 0, 0]
+
+        except KeyError:
+            ###Singleton
+            num_First_genomes = [[str(cluster)+':'+str(len(First_taxa))]]
+            pangenome_clusters_Type[cluster] = [1, len(First_taxa), num_First_genomes, 0, 0, 0]
+
+    # pangenome_clusters_Type = [Number of First clustered genomes or genera, Size of the cluster, Ditto, 0,0,0 ]
+    return pangenome_clusters_Type
+
+
+
+#@profile
+def combined_clustering_CDHIT(options, taxa_dict, splitter):
+    Second_in = open(options.reclustered, 'r')
+    combined_pangenome_clusters_First = OrderedDict()
+    combined_pangenome_clusters_First_sequences = OrderedDict()
+    combined_pangenome_clusters_Second = OrderedDict()
+    combined_pangenome_clusters_Second_sequences = OrderedDict()
+    combined_pangenome_clusters_First_Second_clustered = OrderedDict()
+
+    not_Second_only_cluster_ids = []
+    already_seen_PEP = []
+    Combined_clusters = OrderedDict()
+    Combined_reps = OrderedDict()
+    first = True
+    for line in Second_in:
+        if line.startswith('>'):
+            if first == False:
+                cluster_size = len(Combined_clusters[cluster_id])
+                Combined_reps.update({rep: cluster_size})
+                for pep in combined_pangenome_clusters_First_sequences[cluster_id]:
+                    if pep != []:
+                        if pep in already_seen_PEP:
+                            continue
+                        else:
+                            already_seen_PEP.append(pep)
+                if len(combined_pangenome_clusters_Second_sequences[cluster_id]) > 0 and len(combined_pangenome_clusters_First_sequences[cluster_id]) > 0:
+                    if len(combined_pangenome_clusters_First_sequences[cluster_id]) > 1:  # If we have clustered >1 First family, we need to record 1 as key and all others are val
+                        all_but_first = combined_pangenome_clusters_First_sequences[cluster_id][1:]
+                        storfs_clustered = combined_pangenome_clusters_Second_sequences[cluster_id]
+                        VALUE = all_but_first + storfs_clustered
+                    else:
+                        VALUE = combined_pangenome_clusters_Second_sequences[cluster_id]
+                    KEY = combined_pangenome_clusters_First_sequences[cluster_id][0]
+                    combined_pangenome_clusters_First_Second_clustered.update({KEY: VALUE})
+            cluster_id = line.strip('>')
+            cluster_id = cluster_id.strip('\n')
+            cluster_id = cluster_id.split(' ')[1]
+            Combined_clusters.update({cluster_id: []})
+            combined_pangenome_clusters_First.update({cluster_id: []})
+            combined_pangenome_clusters_First_sequences.update({cluster_id: []})
+            combined_pangenome_clusters_Second.update({cluster_id: []})
+            combined_pangenome_clusters_Second_sequences.update({cluster_id: []})
+
+            first = False
+        else:
+            clustered = line.split('\t')[1]
+            clustered = clustered.split('>')[1]
+            clustered = clustered.split('...')[0]
+            genome = clustered.split(splitter)[0]
+            taxa_dict[genome] += 1
+            if '*' in line:
+                rep = clustered
+                Combined_reps.update({rep: 0})
+            if first == False:
+                Combined_clusters[cluster_id].append(clustered)
+                clustered_taxa = clustered.split(splitter)[0]
+                if options.sequence_tag in line:
+                    if clustered_taxa not in combined_pangenome_clusters_Second[cluster_id]:
+                        combined_pangenome_clusters_Second[cluster_id].append(clustered_taxa)
+                    combined_pangenome_clusters_Second_sequences[cluster_id].append(clustered)
+                else:
+                    if cluster_id not in not_Second_only_cluster_ids:
+                        not_Second_only_cluster_ids.append(cluster_id)
+                    if clustered_taxa not in combined_pangenome_clusters_First[cluster_id]:
+                        combined_pangenome_clusters_First[cluster_id].append(clustered_taxa)
+                    combined_pangenome_clusters_First_sequences[cluster_id].append(clustered)
+
+
+    return combined_pangenome_clusters_First_Second_clustered,not_Second_only_cluster_ids, combined_pangenome_clusters_Second, combined_pangenome_clusters_Second_sequences
+
+
+def cluster_EdgeList(options,splitter):
+    if options.cluster_format == 'TSV':
+        separator = '\t'
+    elif options.cluster_format == 'CSV':
+        separator = ','
+    cluster_id = 0
+    last_rep = ''
+    first = True
+    First_in = open(options.clusters, 'r')
+    pangenome_clusters_First = OrderedDict()
+    pangenome_clusters_First_genomes = OrderedDict()
+    pangenome_clusters_First_sequences = OrderedDict()
+    taxa_dict = defaultdict(int)
+    reps = OrderedDict()
+    tmp_genomes = None
+    for line in First_in:
+        rep, child = line.strip().split(separator)
+        child_taxa = child.split(splitter)[0]  # Extracting the genome identifier from the child sequence
+        # Counting occurrences of genomes
+        taxa_dict[child_taxa] += 1
+        if first == True:
+            pangenome_clusters_First['0'] = []
+            pangenome_clusters_First_sequences['0'] = []
+            first = False
+            tmp_genomes = []
+
+        if rep != last_rep and last_rep != '':
+            pangenome_clusters_First_genomes[rep] = tmp_genomes
+            tmp_genomes = []
+            cluster_id +=1
+            pangenome_clusters_First[str(cluster_id)] = []
+            pangenome_clusters_First_sequences[str(cluster_id)] = []
+            cluster_size = len(pangenome_clusters_First_sequences[str(cluster_id-1)])
+            reps.update({last_rep: [cluster_size, len(pangenome_clusters_First[str(cluster_id-1)])]})
+            pangenome_clusters_First[str(cluster_id)] = []
+            pangenome_clusters_First_sequences[str(cluster_id)] = []
+        if child_taxa not in pangenome_clusters_First[str(cluster_id)]:
+            pangenome_clusters_First[str(cluster_id)].append(child_taxa)
+            tmp_genomes.append(child_taxa)
+
+        pangenome_clusters_First_sequences[str(cluster_id)].append(child)
+        last_rep = rep
+        cluster_size = len(pangenome_clusters_First_sequences[str(cluster_id)])
+        reps.update({rep: [cluster_size, len(pangenome_clusters_First[str(cluster_id)])]})
+
+    #!!# May not be needed below
+    pangenome_clusters_First_genomes[rep] = tmp_genomes
+
+    return taxa_dict, pangenome_clusters_First, pangenome_clusters_First_genomes, pangenome_clusters_First_sequences, reps
+
+
+def combined_clustering_Edge_List(options, splitter):
+    if options.cluster_format == 'TSV':
+        separator = '\t'
+    elif options.cluster_format == 'CSV':
+        separator = ','
+
+    cluster_id = 0
+    last_rep = ''
+    Second_in = open(options.reclustered, 'r')
+    combined_pangenome_clusters_First = OrderedDict()
+    combined_pangenome_clusters_First_sequences = OrderedDict()
+    combined_pangenome_clusters_Second = OrderedDict()
+    combined_pangenome_clusters_Second_sequences = OrderedDict()
+    combined_pangenome_clusters_First_Second_clustered = OrderedDict()
+
+    not_Second_only_cluster_ids = []
+    already_seen_PEP = []
+    Combined_clusters = OrderedDict()
+    Combined_reps = OrderedDict()
+    first = True
+    for line in Second_in:
+        rep, child = line.strip().split(separator)
+        child_taxa = child.split(splitter)[0]  # Extracting the genome identifier from the child sequence
+
+        if first == True:
+            Combined_clusters.update({str(cluster_id): []})
+            combined_pangenome_clusters_First.update({str(cluster_id): []})
+            combined_pangenome_clusters_First_sequences.update({str(cluster_id): []})
+            combined_pangenome_clusters_Second.update({str(cluster_id): []})
+            combined_pangenome_clusters_Second_sequences.update({str(cluster_id): []})
+            Combined_reps.update({rep: 0})
+            first = False
+
+        if first == False:
+            if rep != last_rep and last_rep != '':
+                cluster_size = len(Combined_clusters[str(cluster_id)])
+                Combined_reps.update({rep: cluster_size})
+                for pep in combined_pangenome_clusters_First_sequences[str(cluster_id)]:
+                    if pep != []:
+                        if pep in already_seen_PEP:
+                            continue
+                        else:
+                            already_seen_PEP.append(pep)
+                if len(combined_pangenome_clusters_Second_sequences[str(cluster_id)]) > 0 and len(combined_pangenome_clusters_First_sequences[str(cluster_id)]) > 0:
+                    if len(combined_pangenome_clusters_First_sequences[str(cluster_id)]) > 1:  # If we have clustered >1 PEP family, we need to record 1 as key and all others are val
+                        all_but_first = combined_pangenome_clusters_First_sequences[str(cluster_id)][1:]
+                        storfs_clustered = combined_pangenome_clusters_Second_sequences[str(cluster_id)]
+                        VALUE = all_but_first + storfs_clustered
+                    else:
+                        VALUE = combined_pangenome_clusters_Second_sequences[str(cluster_id)]
+                    KEY = combined_pangenome_clusters_First_sequences[str(cluster_id)][0]
+                    combined_pangenome_clusters_First_Second_clustered.update({KEY: VALUE})
+
+                cluster_id += 1
+                Combined_clusters.update({str(cluster_id): []})
+                combined_pangenome_clusters_First.update({str(cluster_id): []})
+                combined_pangenome_clusters_First_sequences.update({str(cluster_id): []})
+                combined_pangenome_clusters_Second.update({str(cluster_id): []})
+                combined_pangenome_clusters_Second_sequences.update({str(cluster_id): []})
+                Combined_reps.update({rep: 0})
+
+        Combined_clusters[str(cluster_id)].append(child)
+        if options.sequence_tag in line:
+            if child_taxa not in combined_pangenome_clusters_Second[str(cluster_id)]:
+                combined_pangenome_clusters_Second[str(cluster_id)].append(child_taxa)
+            combined_pangenome_clusters_Second_sequences[str(cluster_id)].append(child)
+        else:
+            if str(cluster_id) not in not_Second_only_cluster_ids:
+                not_Second_only_cluster_ids.append(str(cluster_id))  # Tell us which StORF_Reporter clustered are unmatched to a PEP
+            if child_taxa not in combined_pangenome_clusters_First[str(cluster_id)]:
+                combined_pangenome_clusters_First[str(cluster_id)].append(child_taxa)
+            combined_pangenome_clusters_First_sequences[str(cluster_id)].append(child)
+
+        last_rep = rep
+
+    return combined_pangenome_clusters_First_Second_clustered,not_Second_only_cluster_ids, combined_pangenome_clusters_Second, combined_pangenome_clusters_Second_sequences