PyamilySeq 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

PyamilySeq/CD-Hit_StORF-Reporter_Cross-Genera_Builder.py

@@ -0,0 +1,600 @@
+import collections
+import copy
+import math
+import sys
+import numpy as np
+from itertools import chain
+
+def get_Genus(clustered):
+    clustered_genus = clustered.split('|')[0]
+    if '_' in clustered_genus[0]:  # Remove name error
+        clustered_genus = clustered_genus.split('_')[1]
+    else:
+        clustered_genus = clustered_genus.split('_')[0]
+    return str(clustered_genus).capitalize()
+
+def get_Species(clustered):
+    clustered_species = clustered.split('|')[0]
+    if '_' in clustered_species[0]:  # Remove name error
+        clustered_species = clustered_species.split('_')[1]
+    else:
+        clustered_species = clustered_species.split('_')[:2]
+    return str('_'.join(clustered_species)).capitalize()
+
+
+PEP_In = open('/home/nick/Documents/Single_Genome/All_Ensembl_PEP_CD_Clustered_90_60.clstr','r')
+StORF_In = open('/home/nick/Documents/Single_Genome/All_Ensem_PEP_CD_Clustered_90_60_Unclustered_UR_StORFs_AA_CD.clstr','r') # Clusters for single Genera
+
+clusters = collections.OrderedDict()
+
+
+pangenome_clusters_PEP_Genera = collections.OrderedDict()
+pangenome_clusters_PEP_Species = collections.OrderedDict()
+pangenome_clusters_PEP_Strains = collections.OrderedDict()
+pangenome_clusters_PEP_SEQS = collections.OrderedDict()
+
+
+max_storf_only_genera = 0
+
+
+
+
+count = 0
+first = True
+genome_dict = collections.defaultdict(int)
+reps = collections.OrderedDict()
+county = 0
+#singleton_cluster = "Null"
+clusters_With_Con_StORFs = []
+## Load in all data for easier reuse later
+for line in PEP_In:
+    if line.startswith('>'):
+        if first == False:
+            Ensem_Con = set(Ensem_genomes).intersection(Con_genomes)
+            cluster_size = len(clusters[cluster_id])
+            reps.update({rep: [cluster_size,len(pangenome_clusters_PEP_Genera[cluster_id])]}) # Add strains, species here if wanted
+            #if len(clusters[cluster_id]) == 1 and "Null" not in singleton_cluster: # Stop at clusters smaller than 10
+            #    singleton_cluster = cluster_id
+            #if len(clusters[cluster_id]) < 10: # Stop at clusters smaller than 10
+                # pangenome_clusters_PEP_Species.popitem()
+                # pangenome_clusters_PEP_Genera.popitem()
+                # pangenome_clusters_PEP_SEQS.popitem()
+                # reps.popitem()
+            # if len(clusters[cluster_id]) == 1:
+            #    break    # REMEMBER
+        Ensem_genomes, Con_genomes = [], []
+        cluster_id = line.strip('>')
+        cluster_id = cluster_id.strip('\n')
+        cluster_id = cluster_id.split(' ')[1]
+        clusters.update({cluster_id: []})
+        pangenome_clusters_PEP_Genera.update({cluster_id: []})
+        pangenome_clusters_PEP_Species.update({cluster_id:[]})
+        pangenome_clusters_PEP_Strains.update({cluster_id: []})
+        # pangenome_clusters_PEP_SEQS.update({cluster_id:[]})
+
+        first = False
+    else:
+        clustered = line.split('\t')[1]
+        clustered = clustered.split('>')[1]
+        clustered = clustered.split('...')[0]
+        genome = clustered.split('|')[0]
+        genome_dict[genome] +=1
+        if '*' in line:
+            rep = clustered
+            reps.update({rep:[0,0]})
+        if first == False:
+            clusters[cluster_id].append(clustered)
+            clustered_genus = get_Genus(clustered)
+            clustered_species = get_Species(clustered)
+            clustered_strain = clustered.split('|')[0]
+
+            if clustered_genus not in pangenome_clusters_PEP_Genera[cluster_id]:
+                pangenome_clusters_PEP_Genera[cluster_id].append(clustered_genus)
+            #if clustered_species not in pangenome_clusters_PEP_Species[cluster_id]:
+            #    pangenome_clusters_PEP_Species[cluster_id].append(clustered_species)
+            if genome not in pangenome_clusters_PEP_Strains[cluster_id]:
+                pangenome_clusters_PEP_Strains[cluster_id].append(genome)
+            # pangenome_clusters_PEP_SEQS[cluster_id].append(clustered)
+print("PEP DONE")
+######################################
+Combined_pangenome_clusters_PEP_Genera = collections.OrderedDict()
+Combined_pangenome_clusters_PEP_Species = collections.OrderedDict()
+Combined_pangenome_clusters_PEP_Strains = collections.OrderedDict()
+Combined_pangenome_clusters_PEP_SEQS = collections.OrderedDict()
+
+Combined_pangenome_clusters_StORF_Genera = collections.OrderedDict()
+Combined_pangenome_clusters_StORF_Species = collections.OrderedDict()
+Combined_pangenome_clusters_StORF_Strains = collections.OrderedDict()
+Combined_pangenome_clusters_StORF_SEQS = collections.OrderedDict()
+
+Combined_pangenome_clusters_PEP_StORF_Clustered_Genera = collections.OrderedDict()
+Combined_pangenome_clusters_PEP_StORF_Clustered = collections.OrderedDict()
+
+not_StORF_Only_Cluster_IDs = []
+
+Combined_clusters = collections.OrderedDict()
+Combined_reps = collections.OrderedDict()
+first = True
+###############
+## We load in the combined PEP and StORF_Reporter data separately
+for line in StORF_In:
+    if line.startswith('>'):
+        if first == False:
+            cluster_size = len(Combined_clusters[cluster_id])
+            Combined_reps.update({rep: cluster_size})
+            # if len(Combined_pangenome_clusters_PEP_SEQS[cluster_id]) > 1:
+            #     print("Here")
+            if len(Combined_pangenome_clusters_StORF_SEQS[cluster_id]) > 0 and len(Combined_pangenome_clusters_PEP_SEQS[cluster_id]) > 0:
+                if len(Combined_pangenome_clusters_PEP_SEQS[cluster_id]) > 1: # If we have clustered >1 PEP family, we need to record 1 as key and all others are val
+                    all_but_first = Combined_pangenome_clusters_PEP_SEQS[cluster_id][1:]
+                    storfs_clustered = Combined_pangenome_clusters_StORF_SEQS[cluster_id]
+                    VALUE = all_but_first+storfs_clustered
+                else:
+                    VALUE = Combined_pangenome_clusters_StORF_SEQS[cluster_id]
+                KEY = Combined_pangenome_clusters_PEP_SEQS[cluster_id][0]
+                Combined_pangenome_clusters_PEP_StORF_Clustered.update({KEY:VALUE})
+
+
+            ## Below needs to be rewritten. With >1 genus - be able to record multiple  PEPs for each combined...
+            # if len(Combined_pangenome_clusters_StORF_Genera[cluster_id]) > 0 and len(Combined_pangenome_clusters_PEP_Genera[cluster_id]) > 0:
+            #     KEY = Combined_pangenome_clusters_PEP_SEQS[cluster_id][0]
+            #     VALUE = Combined_pangenome_clusters_StORF_SEQS[cluster_id]
+            #     Combined_pangenome_clusters_PEP_StORF_Clustered_Genera.update({KEY:VALUE})
+            if len(Combined_clusters[cluster_id]) == 1: # Stop at clusters smaller than 10
+                print("First Singleton Cluster is: " +str(cluster_id))
+                break
+        cluster_id = line.strip('>')
+        cluster_id = cluster_id.strip('\n')
+        cluster_id = cluster_id.split(' ')[1]
+        Combined_clusters.update({cluster_id: []})
+        # Combined_pangenome_clusters_PEP_Genera.update({cluster_id:[]})
+        # Combined_pangenome_clusters_PEP_Species.update({cluster_id: []})
+        Combined_pangenome_clusters_PEP_Strains.update({cluster_id: []})
+        Combined_pangenome_clusters_PEP_SEQS.update({cluster_id: []})
+        #
+        Combined_pangenome_clusters_StORF_Genera.update({cluster_id: []})
+        # Combined_pangenome_clusters_StORF_Species.update({cluster_id: []})
+        Combined_pangenome_clusters_StORF_Strains.update({cluster_id: []})
+        Combined_pangenome_clusters_StORF_SEQS.update({cluster_id: []})
+        first = False
+    else:
+        clustered = line.split('\t')[1]
+        clustered = clustered.split('>')[1]
+        clustered = clustered.split('...')[0]
+        if '*' in line:
+            rep = clustered
+            Combined_reps.update({rep:0})
+        if first == False:
+            Combined_clusters[cluster_id].append(clustered)
+            clustered_genus = get_Genus(clustered)
+            clustered_species = get_Species(clustered)
+            clustered_strain = clustered.split('|')[0]
+            if '_' in clustered_strain[0]:  # Remove name error
+                clustered_strain = clustered_strain.split('_')[1]
+
+            if "StORF_Type" in line:
+            #     if cluster_id not in clusters_With_Con_StORFs: # For counting?
+            #         clusters_With_Con_StORFs.append(cluster_id)
+                if clustered_genus not in Combined_pangenome_clusters_StORF_Genera[cluster_id]:
+                    Combined_pangenome_clusters_StORF_Genera[cluster_id].append(clustered_genus)
+            #     if clustered_species not in Combined_pangenome_clusters_StORF_Species[cluster_id]:
+            #         Combined_pangenome_clusters_StORF_Species[cluster_id].append(clustered_species)
+                if clustered_strain not in Combined_pangenome_clusters_StORF_Strains[cluster_id]:
+                    Combined_pangenome_clusters_StORF_Strains[cluster_id].append(clustered_strain)
+                Combined_pangenome_clusters_StORF_SEQS[cluster_id].append(clustered)
+            #
+            else:
+            #     if clustered_genus not in Combined_pangenome_clusters_PEP_Genera[cluster_id]:
+            #         Combined_pangenome_clusters_PEP_Genera[cluster_id].append(clustered_genus)
+            #     if clustered_species not in Combined_pangenome_clusters_PEP_Species[cluster_id]:
+            #         Combined_pangenome_clusters_PEP_Species[cluster_id].append(clustered_species)
+                if clustered_strain not in Combined_pangenome_clusters_PEP_Strains[cluster_id]:
+                    Combined_pangenome_clusters_PEP_Strains[cluster_id].append(clustered_strain)
+                if cluster_id not in not_StORF_Only_Cluster_IDs:
+                    not_StORF_Only_Cluster_IDs.append(cluster_id)  # Tell us which StORF_Reporter clustered are unmatched to a PEP
+                Combined_pangenome_clusters_PEP_SEQS[cluster_id].append(clustered)
+
+
+
+###HERE for tomorrow - copy the updated work from single to here and repeat for genus,species and strain
+list_of_reps = list(reps.keys())
+num_clustered_PEP_Genera = collections.defaultdict(list)
+recorded_PEP = []
+################################# Genera
+pangenome_clusters_Type_Genera = copy.deepcopy(pangenome_clusters_PEP_Genera)
+pangenome_clusters_Type_Strains = collections.defaultdict(list)
+
+for cluster, pep_genomes in pangenome_clusters_PEP_Genera.items():
+    recorded_PEP.append(cluster)
+    rep = list_of_reps[int(cluster)]
+    Com_PEPs = 0
+    Com_PEP_Genomes = 0
+    StORFs = 0
+    Added_StORF_Genera = 0
+    seen_clust_Strains = []
+
+    PEP_Strains = pangenome_clusters_PEP_Strains[cluster]
+    for clustered_strain in PEP_Strains:
+        if '_' in clustered_strain[0]:  # Remove name error
+            clustered_strain = clustered_strain[1:]
+        if clustered_strain not in seen_clust_Strains:
+            seen_clust_Strains.append(clustered_strain)
+
+
+    try:
+        clustered_combined = Combined_pangenome_clusters_PEP_StORF_Clustered[rep]
+        seen_clust_Genera = []
+        num_clustered_PEP_Genera[cluster].append(rep + '_' + str(len(pep_genomes)))
+        for clust in clustered_combined:
+            if 'StORF_Type' not in clust:
+                ### Need to get the number of pep genomes for each pep clustered into this
+                Com_PEPs += 1
+                clustered_genus = get_Genus(clust)
+                #clust_Genome = clust.split('|')[0]
+                if clustered_genus not in seen_clust_Genera:
+                    seen_clust_Genera.append(clustered_genus)
+                    if clustered_genus not in pep_genomes:
+                        Com_PEP_Genomes += 1
+                try:
+                    num_clustered_PEP_Genera[cluster].append(clust + '_' + str(reps[clust][1]))
+                except TypeError:
+                    sys.exit("Broken")
+
+            elif 'StORF_Type' in clust:
+                StORFs += 1
+                clustered_genus = get_Genus(clust)
+                #clust_Genome = clust.split('|')[0]
+                if clustered_genus not in seen_clust_Genera:
+                    seen_clust_Genera.append(clustered_genus)
+                    if clustered_genus not in pep_genomes:
+                        Added_StORF_Genera += 1
+            else:
+                print("WHAT")
+
+        size_of_pep_clusters = []
+        peps = num_clustered_PEP_Genera[cluster]
+        for pep in peps:
+            pep = pep.rsplit('_', 1)
+            size_of_pep_clusters.append(int(pep[1]))
+        pangenome_clusters_Type_Genera[cluster] = [len(num_clustered_PEP_Genera[cluster]), sum(size_of_pep_clusters),
+                                            size_of_pep_clusters, Added_StORF_Genera, StORFs]
+        pangenome_clusters_Type_Strains[cluster] = seen_clust_Strains
+    except KeyError:
+        ###Singleton
+        num_pep_genomes = [len(pep_genomes)]
+        pangenome_clusters_Type_Genera[cluster] = [1, len(pep_genomes), num_pep_genomes, Added_StORF_Genera, StORFs]
+        pangenome_clusters_Type_Strains[cluster] = seen_clust_Strains
+
+print("S")
+
+
+#######################################
+
+Without_StORF = open('./Ensem_Clusters_Without_StORFs_To_Be_Nogged_min2','w')
+With_StORF = open('./Ensem_Clusters_With_StORFs_To_Be_Nogged','w')
+#With_Extending_StORF = open('./Ensem_Clusters_With_Extending_Con-StORFs_To_Be_Nogged','w')
+
+for key, value in pangenome_clusters_Type_Genera.items():
+    pep_strains = pangenome_clusters_Type_Strains[key]
+    if value[4] == 0 and len(pep_strains) >=2:
+        Without_StORF.write(str(key)+',')
+ #   elif value[3] != 0:
+ #       With_Extending_StORF.write(str(key)+',')
+ #       With_StORF.write(str(key) + ',')
+    elif value[4] >=1:
+        With_StORF.write(str(key) + ',')
+
+With_StORF.close()
+Without_StORF.close()
+#With_Extending_StORF.close()
+
+############## Typing for the StORF_Reporter-Data
+
+
+multi_PEP_Combined_By_StORFs = collections.OrderedDict()
+
+StORF_Seqs_Extended = []
+StORF_Genomes_Extended = []
+
+####################################
+#cores = collections.OrderedDict({'pep_genera_single':[],'pep_genera_multi':[],'extended_genera':[],'comb_extended_genera_single':[],'comb_extended_genera_multi':[],'extended_genera_single':[],'extended_genera_multi':0,'storf_genera_single':0,'storf_genera_multi':0,
+#                                 'only_storf_genera_single':0,'only_storf_genera_multi':0})
+
+cores = collections.OrderedDict({'pep_genera':[],'extended_genera_single_pep':[],'many_extended_genera_pep':[],'extended_genera':[],'comb_extended_genera':[],'storf_genera':[],'only_storf_genera':[],'only_storf_genera_recording':[]})
+
+extended = collections.OrderedDict()
+############################
+
+clsuters_to_be_validated = collections.defaultdict(list)
+
+
+############################
+def calc_pep_only(pep_num):
+    cores['pep_genera'].append(pep_num)
+    # if pep_num == 1:# and StORF_num == 0:
+    #     cores['pep_genera_single'] += 1
+    # elif pep_num > 1:# and StORF_num == 0:
+    #     cores['pep_genera_multi'] += 1
+##########################
+def calc_pep_extended_StORF(cluster,pep_num,storf_num):
+    if pep_num != 0 and storf_num >= 1:
+        cores['extended_genera'].append(pep_num+storf_num)
+        clsuters_to_be_validated['extended_genera'].append(cluster)
+    if pep_num != 0 and storf_num >= 10:
+        cores['many_extended_genera_pep'].append([cluster,pep_num+storf_num])
+
+    if pep_num == 1 and storf_num >= 1:
+        cores['extended_genera_single_pep'].append([cluster,pep_num + storf_num])
+    #     cores['extended_genera_single'] +=1
+    # if pep_num != 0 and storf_num > 1:
+    #     cores['extended_genera_multi'] +=1
+##########################
+def calc_multi_pep_extended_StORF(cluster,number_of_pep_clustered,pep_num,storf_num):
+    if pep_num !=0 and storf_num >= 1:
+        cores['comb_extended_genera'].append(pep_num+storf_num)
+        clsuters_to_be_validated['comb_extended_genera'].append(cluster)
+
+#########################
+def calc_StORF_only_when_with_pep(cluster,storf_num):
+    cores['storf_genera'].append(storf_num)
+    clsuters_to_be_validated['storf_genera'].append(cluster)
+    # if storf_num == 1:# and StORF_num == 0:
+    #     cores['storf_genera_single'] += 1
+    # elif storf_num > 1:# and StORF_num == 0:
+    #     cores['storf_genera_multi'] += 1
+########################  What is the difference with these?
+def calc_only_StORF(cluster,storf_num,max_storf_only_genera): # only count the true storf onlies
+    cores['only_storf_genera'].append(storf_num)
+    clsuters_to_be_validated['only_storf_genera'].append(cluster)
+    if storf_num>=6:
+        cores['only_storf_genera_recording'].append([cluster, storf_num])
+    if storf_num > max_storf_only_genera:
+        max_storf_only_genera = storf_num
+    # if storf_num == 1:# and StORF_num == 0:
+    #     cores['only_storf_genera_single'] += 1
+    # elif storf_num > 1:# and StORF_num == 0:
+    #     cores['only_storf_genera_multi'] += 1
+    return max_storf_only_genera
+#########################
+
+
+
+###########################
+print("Running")
+check_all_calced = 0
+for cluster, numbers in pangenome_clusters_Type_Genera.items():
+    pep_strains = pangenome_clusters_Type_Strains[cluster]
+    if numbers[3] >=1:
+        StORF_Genomes_Extended.append(numbers[3])
+    if numbers[4] >=1:
+        StORF_Seqs_Extended.append(numbers[4])
+############################### Calc PEP only
+    if numbers[0] == 1 and len(pep_strains) >= 2: # If StORFs did not combine PEP reps
+        calc_pep_only(numbers[1])#,numbers[3])
+        check_all_calced +=1
+    elif numbers[0] >1: # IF StORFs combined multiple PEP
+        calc_pep_only(numbers[2][0])
+        check_all_calced += 1
+        # for num in numbers[2]:
+        #     calc_pep_only(num)  # ,numbers[3])
+
+############################# Calc PEP and StORF_Reporter
+    if numbers[0] == 1 and numbers[3] >1: # If StORFs did not combine PEP reps
+        calc_pep_extended_StORF(cluster,numbers[1],numbers[3])
+        extended.update({cluster:numbers})
+        check_all_calced += 1
+    elif numbers[0] >1 and numbers[3] >1: # IF StORFs combined multiple PEP - Genera added
+        #grouped_pep = sum(numbers[2])
+        #for num in numbers[2]:
+        calc_multi_pep_extended_StORF(cluster,numbers[2],numbers[1],numbers[3]) # same here
+        print("combined: " + str(cluster))
+
+        extended.update({cluster: numbers})
+        check_all_calced += 1
+    elif numbers[0] >1 and numbers[4] >1: # IF StORFs combined multiple PEP
+        multi_PEP_Combined_By_StORFs.update({cluster: numbers})
+
+
+import os
+###########################
+############################### Calc StORF_Reporter only
+Combined_pangenome_clusters_ONLY_StORF_Type = collections.defaultdict(list)
+Combined_pangenome_clusters_StORF_Type = collections.defaultdict(list)
+
+biggest_genera = ""
+big_genera = 0
+biggest_strains = ""
+big_strains = 0
+
+
+#Without_StORF = open('./Ensem_Clusters_Without_Con-StORFs_To_Be_Nogged_min2','w')
+#With_StORF = open('./Ensem_Clusters_With_Con-StORFs_To_Be_Nogged','w')
+#With_Extending_StORF = open('./Ensem_Clusters_With_Extending_Con-StORFs_To_Be_Nogged','w')
+StORF_Only = open("./StORF_Only_Clusters_To_Be_Nogged_min2",'w')
+
+for cluster, genera in Combined_pangenome_clusters_StORF_Genera.items():
+    storf_strains = Combined_pangenome_clusters_StORF_Strains[cluster]
+    pep_strains = Combined_pangenome_clusters_PEP_Strains[cluster]
+    if cluster in not_StORF_Only_Cluster_IDs:
+        Combined_pangenome_clusters_StORF_Type[cluster] = [cluster,len(genera)]
+        #if len(genera) >= 1:
+        calc_StORF_only_when_with_pep(cluster,len(genera))  # ,numbers[3])
+    else:
+        if len(storf_strains) >= 2:
+            StORF_Only.write(str(cluster) + ',')
+            Combined_pangenome_clusters_ONLY_StORF_Type[cluster] = [cluster,len(genera)]
+            max_storf_only_genera = calc_only_StORF(cluster,len(genera),max_storf_only_genera)
+            if len(genera) > big_genera:
+                big_genera = len(genera)
+                biggest_genera = cluster
+            if len(storf_strains) >= big_strains:
+                big_strains = len(storf_strains)
+                biggest_strains = cluster
+
+
+
+
+
+
+
+
+print("Biggest: " +biggest_genera)
+
+
+
+############################### Calc StORF_Reporter only
+# for cluster, data in Combined_pangenome_clusters_StORF_Type.items():
+#     if data[1] >=1:
+#         calc_StORF_only_when_with_pep(data[1])  # ,numbers[3])
+#
+#
+
+
+#################
+print(cores)
+#print(extended)
+
+from collections import Counter
+
+#print(Counter(cores['pep_genera']))
+#print(Counter(cores['extended_genera']))
+#print(Counter(cores['comb_extended_genera']))
+#print(Counter(cores['storf_genera']))
+print(Counter(cores['only_storf_genera']))
+print(cores['only_storf_genera_recording'])
+
+print("END")
+
+### Emoty file ready for interesting storfs
+# interesting_out = "./StORF_Only_Clusters_To_Be_Swissed.fa"
+# with open(interesting_out, 'r+') as f:
+#     f.truncate(4)
+# for cluster, data in Combined_pangenome_clusters_ONLY_StORF_Type.items():
+#     #if number >1:
+#     if data[1] >=1:
+#         calc_only_StORF(data[1])  # ,numbers[3])
+#         # if data[1] >= 2:
+#         #     print("Interesting:" + str(cluster))
+#         #     os.system(
+#         #         "python3 Extract_FASTA_From_Cluster.py -f ./All_Ensem_Filtered_PEP_Clustered_With_Unclustered_UR-StORFS_s.fa_CD_c90_s60.fa  "
+#         #         "-c  ./All_Ensem_Filtered_PEP_Clustered_With_Unclustered_UR-StORFS_s.fa_CD_c90_s60.clstr -id " + str(
+#         #             data[0]) + " -o "+ interesting_out)
+# #
+#
+
+
+
+
+#
+# ################################## Species
+# pangenome_clusters_Type_Species = copy.deepcopy(pangenome_clusters_PEP_Species)
+# for cluster, genomes in pangenome_clusters_PEP_Species.items():
+#     print(str(len(genomes)) + '\t' + str(len(pangenome_clusters_StORF_Species[cluster])))
+#     Con_StORFs = pangenome_clusters_StORF_Species[cluster]
+#     unique_con = 0
+#     all_con = 0
+#     for con in Con_StORFs:
+#         all_con +=1
+#         if con not in genomes:
+#             unique_con +=1
+#     pangenome_clusters_Type_Species[cluster] = [len(genomes),all_con,unique_con]
+# ################################# Strains
+# pangenome_clusters_Type_Strains = copy.deepcopy(pangenome_clusters_PEP_Strains)
+# for cluster, genomes in pangenome_clusters_PEP_Strains.items():
+#     print(str(len(genomes))+'\t'+str(len(pangenome_clusters_StORF_Strains[cluster])))
+#     Con_StORFs = pangenome_clusters_StORF_Strains[cluster]
+#     unique_con = 0
+#     all_con = 0
+#     for con in Con_StORFs:
+#         all_con +=1
+#         if con not in genomes:
+#             unique_con +=1
+#     pangenome_clusters_Type_Strains[cluster] = [len(genomes),all_con,unique_con]
+# ###################################
+# Chris_Out = open('./Chris_Clusters.txt','w')
+#
+# clusters_For_Chris = collections.OrderedDict()
+# clusters_For_Chris_PEP_0 = collections.OrderedDict()
+#
+# Chris_Out.write("Cluster\tSize\tEnsem_Genera_Num\tCon-StORF_Genera_Num\tCon-StORF_Only_Genera_Num\tEnsem_Species_Num\tCon-StORF_Species_Num\tCon-StORF_Only_Species_Num\tEnsem_Strain_Num\tCon-StORF_Strain_Num\tCon-StORF_Only_Strain_Num\n")
+# #This for-loop will go through ALL Clusters allowing for the extraction of ALL different groupings
+# for cluster, data in clusters.items():
+#     genera_numbers = pangenome_clusters_Type_Genera[cluster]
+#     species_numbers = pangenome_clusters_Type_Species[cluster]
+#     strain_numbers = pangenome_clusters_Type_Strains[cluster]
+#
+#     Chris_Out.write(str(cluster)+'\t'+str(len(data))+'\t'+str(genera_numbers[0])+'\t'+str(genera_numbers[1])+'\t'+str(genera_numbers[2])+'\t'+str(species_numbers[0])+'\t'
+#                                           +str(species_numbers[1])+'\t'+str(species_numbers[2])+'\t'+str(strain_numbers[0])+'\t'+str(strain_numbers[1])+'\t'+str(species_numbers[2])+'\n')
+
+    # if cluster in clusters_With_Con_StORFs:
+    #     print("Current")
+    #     size_Of_Cluster = len(clusters[cluster])
+    #     ensem_Num = 0
+    #     con_StORF_Num = 0
+    #     for i in clusters[cluster]:
+    #         print(i)
+    #         if 'Con-Stop' in i:
+    #             con_StORF_Num +=1
+    #         else:
+    #             ensem_Num +=1
+    #     clusters_For_Chris.update({cluster:[size_Of_Cluster,pep_Num,con_StORF_Num,numbers[0],numbers[1]]})
+    #     ############# Add - Num of
+    #     Chris_Out.write(str(cluster)+'\t'+str(size_Of_Cluster)+'\t'+str(pep_Num)+'\t'+str(ensem_Genera)+          str(con_StORF_Num)+'\t'+str(numbers[0])+'\t'+str(numbers[1])+'\n')
+    #     if pep_Num == 0:
+    #         clusters_For_Chris_PEP_0.update({cluster:[size_Of_Cluster,pep_Num,con_StORF_Num,numbers[0],numbers[1]]})
+
+print("Da Da!!!!")
+
+
+#
+#
+#
+#
+# ###################################
+#
+# core_99 = 9.9/10 * len(genome_dict)
+# core_95 = 9.5/10 * len(genome_dict)
+# core_90 = 9/10 * len(genome_dict)
+# core_15 = 1.5/10 * len(genome_dict)
+#
+# pep_core_99 = 0
+# pep_core_95 = 0
+# pep_core_90 = 0
+# pep_core_15 = 0
+#
+#
+# extended_99 = 0
+# extended_95 = 0
+# extended_90 = 0
+# extended_15 = 0
+#             ############### Needs to be redone with new 'numbers'
+# for cluster, numbers in pangenome_clusters_Type_Genera.items():
+#     if numbers[0] >= math.floor(core_99) and numbers[1] == 0:
+#         pep_core_99 +=1
+#     elif numbers[0] >= math.floor(core_95) and numbers[0] < math.floor(core_99) and numbers[1] == 0:
+#         pep_core_95 +=1
+#     elif numbers[0] >= math.floor(core_90) and numbers[0] < math.floor(core_95) and numbers[1] == 0:
+#         pep_core_90 +=1
+#     if numbers[0] >= math.floor(core_15) and numbers[0] < math.floor(core_95) and numbers[1] == 0: # this catch captures some from pep_core_90
+#         pep_core_15 +=1
+#     ############ With Con-StORFs
+#     if numbers[0] < math.floor(core_99) and numbers[0] != 0 and numbers[0]+numbers[1] >= math.floor(core_99):
+#         extended_99 +=1
+#     elif numbers[0] < math.floor(core_95) and numbers[0] != 0 and numbers[0]+numbers[1] >= math.floor(core_95) and numbers[0]+numbers[1] < math.floor(core_99):
+#         extended_95 +=1
+#     elif numbers[0] < math.floor(core_90) and numbers[0] != 0 and numbers[0]+numbers[1] >= math.floor(core_90) and numbers[0]+numbers[1] < math.floor(core_95):
+#         extended_90 +=1
+#     if numbers[0] < math.floor(core_15) and numbers[0] != 0 and numbers[0]+numbers[1] >= math.floor(core_15) and numbers[0]+numbers[1] < math.floor(core_95):
+#         extended_15 +=1
+#
+# print("Out")
+# print(pep_core_99)
+# print(pep_core_95)
+# print(pep_core_90)
+# print(pep_core_15)
+#
+# print(extended_99)
+# print(extended_95)
+# print(extended_90)
+# print(extended_15)
+#

PyamilySeq/Constants.py

@@ -0,0 +1 @@
+PyamilySeq_Version = 'v0.0.1'

PyamilySeq/PyamilySeq_Species.py

@@ -0,0 +1,586 @@
+#from line_profiler_pycharm import profile
+
+from collections import OrderedDict,defaultdict
+import copy
+import math
+import sys
+import argparse
+import os
+
+try:
+    from .Constants import *
+except (ModuleNotFoundError, ImportError, NameError, TypeError) as error:
+    from Constants import *
+
+
+def custom_sort_key(k, dict1, dict2):
+    return (len(dict1[k]), len(dict2[k]))
+
+def sort_keys_by_values(dict1, dict2):
+    sorted_keys = sorted(dict1.keys(), key=lambda k: custom_sort_key(k, dict1, dict2), reverse=True)
+    return sorted_keys
+
+def gene_presence_absence_output(options, genome_dict, pangenome_clusters_First_sorted, pangenome_clusters_First_sequences_sorted):
+    print("Outputting gene_presence_absence file")
+    in_name = options.clusters.split('.')[0]
+    gpa_outfile = open(in_name+'_gene_presence_absence.csv','w')
+    gpa_outfile.write('"Gene","Non-unique Gene name","Annotation","No. isolates","No. sequences","Avg sequences per isolate","Genome Fragment","Order within Fragment","'
+                     '"Accessory Fragment","Accessory Order with Fragment","QC","Min group size nuc","Max group size nuc","Avg group size nuc","')
+    gpa_outfile.write('","'.join(genome_dict.keys()))
+    gpa_outfile.write('"\n')
+    for cluster, sequences in pangenome_clusters_First_sequences_sorted.items():
+        average_sequences_per_genome = len(sequences) / len(pangenome_clusters_First_sorted[cluster])
+        gpa_outfile.write('"group_'+str(cluster)+'","","'+str(len(pangenome_clusters_First_sorted[cluster]))+'","'+str(len(sequences))+'","'+str(average_sequences_per_genome)+
+                         '","","","","","","","","",""')
+
+        full_out = ''
+        for genome in genome_dict.keys():
+            tmp_list = []
+            for value in sequences:
+                if value.split('|')[0] == genome:
+                    tmp_list.append(value)
+            if tmp_list:
+                full_out += ',"'+''.join(tmp_list)+'"'
+            gpa_outfile.write(full_out)
+        gpa_outfile.write('\n')
+
+### Below is some unfinished code
+    # edge_list_outfile = open(in_name+'_edge_list.csv','w')
+    # for cluster, sequences in pangenome_clusters_First_sequences_sorted.items():
+    #     output = []
+    #     for entry in sequences:
+    #         # Split each entry at '|'
+    #         genome, gene = entry.split('|')
+    #         # Format the result as "gene  genome"
+    #         output.append(f"{gene}\t{genome}")
+    #     for line in output:
+    #         edge_list_outfile.write(line + '\n')
+
+
+
+
+def reorder_dict_by_keys(original_dict, sorted_keys):
+    return {k: original_dict[k] for k in sorted_keys}
+
+def get_cores(options,genome_dict):
+    ##Calculate core groups
+    groups = OrderedDict()
+    cores = OrderedDict()
+    prev_top = len(genome_dict)
+    first = True
+    for group in options.core_groups.split(','):
+        calculated_floor = math.floor(int(group) / 100 * len(genome_dict))
+        if first == False:
+            groups[group] = (calculated_floor,prev_top -1)
+        else:
+            groups[group] = (calculated_floor, prev_top)
+            first = False
+        prev_top = calculated_floor
+        first_core_group = 'first_core_' + group
+        cores[first_core_group] = 0
+        if options.reclustered != None:
+            extended_core_group = 'extended_core_' + group
+            cores[extended_core_group] = 0
+            combined_core_group = 'combined_core_' + group
+            cores[combined_core_group] = 0
+            second_core_group = 'second_core_' + group
+            cores[second_core_group] = 0
+            only_second_core_group = 'only_second_core_' + group
+            cores[only_second_core_group] = 0
+    return cores, groups
+
+#@profile
+def calc_First_only_core(pep_num, groups, cores):
+    groups_as_list = list(groups.values())
+    for idx in (idx for idx, (sec, fir) in enumerate(groups_as_list) if sec <= pep_num <= fir):
+        res = idx
+    family_group = list(groups)[res]
+    cores['first_core_'+family_group] +=1
+
+#@profile
+def calc_single_First_extended_Second_only_core(pep_num, groups, cores, second_num): # Count gene families extended with StORFs
+    groups_as_list = list(groups.values())
+    for idx in (idx for idx, (sec, fir) in enumerate(groups_as_list) if sec <= pep_num+second_num <= fir):
+        res = idx
+    family_group = list(groups)[res]
+    cores['extended_core_' + family_group] += 1
+
+
+#@profile
+def calc_multi_First_extended_Second_only_core(pep_num, groups, cores, second_num): # Count seperately those gene families extended with StORF_Reporter but combined >1 PEP
+    groups_as_list = list(groups.values())
+    for idx in (idx for idx, (sec, fir) in enumerate(groups_as_list) if sec <= pep_num+second_num <= fir):
+        res = idx
+    family_group = list(groups)[res]
+    cores['combined_core_' + family_group] += 1
+
+
+#@profile
+def calc_Second_only_core(groups, cores, second_num):
+    groups_as_list = list(groups.values())
+    for idx in (idx for idx, (sec, fir) in enumerate(groups_as_list) if sec <= second_num <= fir):
+        res = idx
+    family_group = list(groups)[res]
+    cores['second_core_' + family_group] += 1
+
+#@profile
+def calc_only_Second_only_core(groups, cores, second_num): # only count the true storf onlies
+    groups_as_list = list(groups.values())
+    for idx in (idx for idx, (sec, fir) in enumerate(groups_as_list) if sec <= second_num <= fir):
+        res = idx
+    family_group = list(groups)[res]
+    cores['only_second_core_' + family_group] += 1
+
+
+
+
+
+#@profile
+def combined_clustering_counting(options, pangenome_clusters_First, reps, combined_pangenome_clusters_First_Second_clustered):
+    num_clustered_First = defaultdict(list)
+    pangenome_clusters_Type = copy.deepcopy(pangenome_clusters_First)
+    list_of_reps = list(reps.keys())
+    for cluster, pep_genomes in pangenome_clusters_First.items():
+        rep = list_of_reps[int(cluster)]  # get the rep of the current pep cluster
+        Com_PEP_Genomes = 0
+        Seconds = 0
+        seen_Seconds = []
+        added_Second_genomes = 0
+        try:  # get the cluster from the storf clusters which contains this rep
+            clustered_combined = combined_pangenome_clusters_First_Second_clustered[rep]  # Not true clusters - I put a PEP as key myself
+            seen_clust_Genomes = []
+            num_clustered_First[cluster].append(rep + '_' + str(len(pep_genomes)))
+            for clust in clustered_combined:
+                if options.sequence_tag not in clust:  # Not good enough at the moment
+                    clust_Genome = clust.split('|')[0]
+                    if clust_Genome not in seen_clust_Genomes:
+                        seen_clust_Genomes.append(clust_Genome)
+                        if clust_Genome not in pep_genomes:
+                            Com_PEP_Genomes += 1
+                    num_clustered_First[cluster].append(clust + '_' + str(reps[clust][1]))
+                elif options.sequence_tag in clust:
+                    Seconds += 1
+                    clust_Genome = clust.split('|')[0]
+                    if clust_Genome not in seen_Seconds:
+                        seen_Seconds.append(clust_Genome)
+                    if clust_Genome not in seen_clust_Genomes:
+                        seen_clust_Genomes.append(clust_Genome)
+                        if clust_Genome not in pep_genomes:
+                            added_Second_genomes += 1
+                else:
+                    sys.exit("Error: looking for sequence_tag")
+
+            size_of_pep_clusters = []
+            peps = num_clustered_First[cluster]
+            for pep in peps:
+                pep = pep.rsplit('_', 1)
+                size_of_pep_clusters.append(int(pep[1]))
+            pangenome_clusters_Type[cluster] = [len(num_clustered_First[cluster]), sum(size_of_pep_clusters),
+                                                size_of_pep_clusters, added_Second_genomes, Seconds, len(seen_Seconds)]
+
+        except KeyError:
+            ###Singleton
+            num_pep_genomes = [len(pep_genomes)]
+            pangenome_clusters_Type[cluster] = [1, len(pep_genomes), num_pep_genomes, added_Second_genomes, Seconds,
+                                                len(seen_Seconds)]
+
+    return pangenome_clusters_Type
+
+#@profile
+def single_clustering_counting(options, pangenome_clusters_First, reps):
+    num_clustered_PEP = defaultdict(list)
+    recorded_PEP = []
+    pangenome_clusters_Type = copy.deepcopy(pangenome_clusters_First)
+    list_of_reps = list(reps.keys())
+    for cluster, pep_genomes in pangenome_clusters_First.items():
+        rep = list_of_reps[int(cluster)]  # get the rep of the current pep cluster
+
+        try:  # get the cluster from the storf clusters which contains this rep
+            num_clustered_PEP[cluster].append(rep + '_' + str(len(pep_genomes)))
+            size_of_pep_clusters = []
+            peps = num_clustered_PEP[cluster]
+            for pep in peps:
+                pep = pep.rsplit('_', 1)
+                size_of_pep_clusters.append(int(pep[1]))
+                recorded_PEP.append(pep[0])
+            pangenome_clusters_Type[cluster] = [len(num_clustered_PEP[cluster]), sum(size_of_pep_clusters),
+                                                size_of_pep_clusters, 0, 0, 0]
+
+        except KeyError:
+            ###Singleton
+            num_pep_genomes = [len(pep_genomes)]
+            pangenome_clusters_Type[cluster] = [1, len(pep_genomes), num_pep_genomes, 0, 0, 0]
+
+    return pangenome_clusters_Type
+
+
+
+#@profile
+def combined_clustering_CDHIT(options, genome_dict):
+    unique_genomes = []
+    Second_in = open(options.reclustered, 'r')
+    combined_pangenome_clusters_First = OrderedDict()
+    combined_pangenome_clusters_First_sequences = OrderedDict()
+    combined_pangenome_clusters_Second = OrderedDict()
+    combined_pangenome_clusters_Second_sequences = OrderedDict()
+    combined_pangenome_clusters_First_Second_clustered = OrderedDict()
+
+    not_Second_only_cluster_ids = []
+    already_seen_PEP = []
+    Combined_clusters = OrderedDict()
+    Combined_reps = OrderedDict()
+    first = True
+    for line in Second_in:
+        if line.startswith('>'):
+            if first == False:
+                cluster_size = len(Combined_clusters[cluster_id])
+                Combined_reps.update({rep: cluster_size})
+                for pep in combined_pangenome_clusters_First_sequences[cluster_id]:
+                    if pep != []:
+                        if pep in already_seen_PEP:
+                            continue
+                        else:
+                            already_seen_PEP.append(pep)
+                if len(combined_pangenome_clusters_Second_sequences[cluster_id]) > 0 and len(combined_pangenome_clusters_First_sequences[cluster_id]) > 0:
+                    if len(combined_pangenome_clusters_First_sequences[cluster_id]) > 1:  # If we have clustered >1 PEP family, we need to record 1 as key and all others are val
+                        all_but_first = combined_pangenome_clusters_First_sequences[cluster_id][1:]
+                        storfs_clustered = combined_pangenome_clusters_Second_sequences[cluster_id]
+                        VALUE = all_but_first + storfs_clustered
+                    else:
+                        VALUE = combined_pangenome_clusters_Second_sequences[cluster_id]
+                    KEY = combined_pangenome_clusters_First_sequences[cluster_id][0]
+                    combined_pangenome_clusters_First_Second_clustered.update({KEY: VALUE})
+            cluster_id = line.strip('>')
+            cluster_id = cluster_id.strip('\n')
+            cluster_id = cluster_id.split(' ')[1]
+            Combined_clusters.update({cluster_id: []})
+            combined_pangenome_clusters_First.update({cluster_id: []})
+            combined_pangenome_clusters_First_sequences.update({cluster_id: []})
+            combined_pangenome_clusters_Second.update({cluster_id: []})
+            combined_pangenome_clusters_Second_sequences.update({cluster_id: []})
+
+            first = False
+        else:
+            clustered = line.split('\t')[1]
+            clustered = clustered.split('>')[1]
+            clustered = clustered.split('...')[0]
+            genome = clustered.split('|')[0]
+            genome_dict[genome] += 1
+            if '*' in line:
+                rep = clustered
+                Combined_reps.update({rep: 0})
+            if first == False:
+                Combined_clusters[cluster_id].append(clustered)
+                clustered_genome = clustered.split('|')[0]
+                if options.sequence_tag in line:
+                    if clustered_genome not in combined_pangenome_clusters_Second[cluster_id]:
+                        combined_pangenome_clusters_Second[cluster_id].append(clustered_genome)
+                    combined_pangenome_clusters_Second_sequences[cluster_id].append(clustered)
+                else:
+                    if cluster_id not in not_Second_only_cluster_ids:
+                        not_Second_only_cluster_ids.append(cluster_id)  # Tell us which StORF_Reporter clustered are unmatched to a PEP
+                    if clustered_genome not in combined_pangenome_clusters_First[cluster_id]:
+                        combined_pangenome_clusters_First[cluster_id].append(clustered_genome)
+                    combined_pangenome_clusters_First_sequences[cluster_id].append(clustered)
+
+
+    return combined_pangenome_clusters_First_Second_clustered,not_Second_only_cluster_ids, combined_pangenome_clusters_Second, unique_genomes
+
+def combined_clustering_Edge_List(options, genome_dict):
+    if options.format == 'TSV':
+        separator = '\t'
+    elif options.format == 'CSV':
+        separator = ','
+    unique_genomes = []
+    cluster_id = 0
+    last_rep = ''
+    Second_in = open(options.reclustered, 'r')
+    combined_pangenome_clusters_First = OrderedDict()
+    combined_pangenome_clusters_First_sequences = OrderedDict()
+    combined_pangenome_clusters_Second = OrderedDict()
+    combined_pangenome_clusters_Second_sequences = OrderedDict()
+    combined_pangenome_clusters_First_Second_clustered = OrderedDict()
+
+    not_Second_only_cluster_ids = []
+    already_seen_PEP = []
+    Combined_clusters = OrderedDict()
+    Combined_reps = OrderedDict()
+    first = True
+    for line in Second_in:
+        rep, child = line.strip().split(separator)
+        child_genome = child.split('|')[0]  # Extracting the genome identifier from the child sequence
+
+        if first == True:
+            Combined_clusters.update({cluster_id: []})
+            combined_pangenome_clusters_First.update({cluster_id: []})
+            combined_pangenome_clusters_First_sequences.update({cluster_id: []})
+            combined_pangenome_clusters_Second.update({cluster_id: []})
+            combined_pangenome_clusters_Second_sequences.update({cluster_id: []})
+            Combined_reps.update({rep: 0})
+            first = False
+
+        if first == False:
+            if rep != last_rep and last_rep != '':
+                cluster_size = len(Combined_clusters[cluster_id])
+                Combined_reps.update({rep: cluster_size})
+                for pep in combined_pangenome_clusters_First_sequences[cluster_id]:
+                    if pep != []:
+                        if pep in already_seen_PEP:
+                            continue
+                        else:
+                            already_seen_PEP.append(pep)
+                if len(combined_pangenome_clusters_Second_sequences[cluster_id]) > 0 and len(combined_pangenome_clusters_First_sequences[cluster_id]) > 0:
+                    if len(combined_pangenome_clusters_First_sequences[cluster_id]) > 1:  # If we have clustered >1 PEP family, we need to record 1 as key and all others are val
+                        all_but_first = combined_pangenome_clusters_First_sequences[cluster_id][1:]
+                        storfs_clustered = combined_pangenome_clusters_Second_sequences[cluster_id]
+                        VALUE = all_but_first + storfs_clustered
+                    else:
+                        VALUE = combined_pangenome_clusters_Second_sequences[cluster_id]
+                    KEY = combined_pangenome_clusters_First_sequences[cluster_id][0]
+                    combined_pangenome_clusters_First_Second_clustered.update({KEY: VALUE})
+
+                cluster_id += 1
+                Combined_clusters.update({cluster_id: []})
+                combined_pangenome_clusters_First.update({cluster_id: []})
+                combined_pangenome_clusters_First_sequences.update({cluster_id: []})
+                combined_pangenome_clusters_Second.update({cluster_id: []})
+                combined_pangenome_clusters_Second_sequences.update({cluster_id: []})
+                Combined_reps.update({rep: 0})
+
+
+        Combined_clusters[cluster_id].append(child)
+        if options.sequence_tag in line:
+            if child_genome not in combined_pangenome_clusters_Second[cluster_id]:
+                combined_pangenome_clusters_Second[cluster_id].append(child_genome)
+            combined_pangenome_clusters_Second_sequences[cluster_id].append(child)
+        else:
+            if cluster_id not in not_Second_only_cluster_ids:
+                not_Second_only_cluster_ids.append(cluster_id)  # Tell us which StORF_Reporter clustered are unmatched to a PEP
+            if child_genome not in combined_pangenome_clusters_First[cluster_id]:
+                combined_pangenome_clusters_First[cluster_id].append(child_genome)
+            combined_pangenome_clusters_First_sequences[cluster_id].append(child)
+
+        last_rep = rep
+
+    return combined_pangenome_clusters_First_Second_clustered,not_Second_only_cluster_ids, combined_pangenome_clusters_Second, unique_genomes
+
+
+def cluster_EdgeList(options):
+    if options.format == 'TSV':
+        separator = '\t'
+    elif options.format == 'CSV':
+        separator = ','
+    cluster_id = 0
+    last_rep = ''
+    first = True
+    First_in = open(options.clusters, 'r')
+    pangenome_clusters_First = OrderedDict()
+    pangenome_clusters_First_sequences = OrderedDict()
+    genome_dict = defaultdict(int)
+    reps = OrderedDict()
+    for line in First_in:
+        rep, child = line.strip().split(separator)
+        child_genome = child.split('|')[0]  # Extracting the genome identifier from the child sequence
+        # Counting occurrences of genomes
+        genome_dict[child_genome] += 1
+        if first == True:
+            pangenome_clusters_First[0] = []
+            pangenome_clusters_First_sequences[0] = []
+            first = False
+
+        if rep != last_rep and last_rep != '':
+            cluster_id +=1
+            pangenome_clusters_First[cluster_id] = []
+            pangenome_clusters_First_sequences[cluster_id] = []
+            cluster_size = len(pangenome_clusters_First_sequences[cluster_id-1])
+            reps.update({last_rep: [cluster_size, len(pangenome_clusters_First[cluster_id-1])]})
+            pangenome_clusters_First[cluster_id] = []
+            pangenome_clusters_First_sequences[cluster_id] = []
+        if child_genome not in pangenome_clusters_First[cluster_id]:
+            pangenome_clusters_First[cluster_id].append(child_genome)
+
+        pangenome_clusters_First_sequences[cluster_id].append(child)
+        last_rep = rep
+        cluster_size = len(pangenome_clusters_First_sequences[cluster_id])
+        reps.update({rep: [cluster_size, len(pangenome_clusters_First[cluster_id])]})
+
+
+    return genome_dict, pangenome_clusters_First, pangenome_clusters_First_sequences, reps
+
+
+
+def cluster_CDHIT(options):
+    First_in = open(options.clusters, 'r')
+    clusters = OrderedDict()
+    pangenome_clusters_First = OrderedDict()
+    pangenome_clusters_First_sequences = OrderedDict()
+    first = True
+    genome_dict = defaultdict(int)
+    reps = OrderedDict()
+    ## Load in all data for easier reuse later
+    for line in First_in:
+        if line.startswith('>'):
+            if first == False:
+                cluster_size = len(clusters[cluster_id])
+                reps.update({rep: [cluster_size, len(pangenome_clusters_First[cluster_id])]})
+            cluster_id = line.strip('>')
+            cluster_id = cluster_id.strip('\n')
+            cluster_id = cluster_id.split(' ')[1]
+            clusters.update({cluster_id: []})
+            pangenome_clusters_First.update({cluster_id: []})
+            pangenome_clusters_First_sequences.update({cluster_id: []})
+
+            first = False
+        else:
+            clustered = line.split('\t')[1]
+            clustered = clustered.split('>')[1]
+            clustered = clustered.split('...')[0]
+            genome = clustered.split('|')[0]
+            genome_dict[genome] += 1
+            if '*' in line:
+                rep = clustered
+                reps.update({rep: [0, 0]})
+            if first == False:
+                clusters[cluster_id].append(clustered)
+                clustered_genome = clustered.split('|')[0]
+                if clustered_genome not in pangenome_clusters_First[cluster_id]:
+                    pangenome_clusters_First[cluster_id].append(clustered_genome)
+                pangenome_clusters_First_sequences[cluster_id].append(clustered)
+    return genome_dict, pangenome_clusters_First, pangenome_clusters_First_sequences, reps
+
+#@profile
+def cluster(options):
+
+    if options.format == 'CD-HIT':
+        genome_dict, pangenome_clusters_First, pangenome_clusters_First_sequences, reps = cluster_CDHIT(options)
+    elif options.format in ['TSV','CSV']:
+        genome_dict, pangenome_clusters_First, pangenome_clusters_First_sequences, reps = cluster_EdgeList(options)
+
+    ######################################
+    cores, groups = get_cores(options, genome_dict)
+    ###
+
+    if options.reclustered != None:
+        if options.format == 'CD-HIT':
+            combined_pangenome_clusters_First_Second_clustered,not_Second_only_cluster_ids,combined_pangenome_clusters_Second,\
+                unique_genomes = combined_clustering_CDHIT(options, genome_dict)
+        if options.format == 'TSV':
+            combined_pangenome_clusters_First_Second_clustered,not_Second_only_cluster_ids,combined_pangenome_clusters_Second,\
+                unique_genomes = combined_clustering_Edge_List(options, genome_dict)
+        pangenome_clusters_Type = combined_clustering_counting(options, pangenome_clusters_First, reps, combined_pangenome_clusters_First_Second_clustered)
+    else:
+        pangenome_clusters_Type = single_clustering_counting(options, pangenome_clusters_First, reps)
+
+
+    counter = 0
+    Number_Of_StORF_Extending_But_Same_Genomes = 0
+
+    sorted_first_keys = sort_keys_by_values(pangenome_clusters_First, pangenome_clusters_First_sequences)
+    pangenome_clusters_First_sorted = reorder_dict_by_keys(pangenome_clusters_First, sorted_first_keys)
+    pangenome_clusters_First_sequences_sorted = reorder_dict_by_keys(pangenome_clusters_First_sequences, sorted_first_keys)
+    pangenome_clusters_Type_sorted = reorder_dict_by_keys(pangenome_clusters_Type, sorted_first_keys)
+
+    print("Calculating Groups")
+    for cluster, numbers in pangenome_clusters_Type_sorted.items():
+    ############################### Calculate First only
+        if numbers[0] == 1 and numbers[1] >=2:
+            calc_First_only_core(numbers[1],groups,cores)
+            counter +=1
+        elif numbers[0] >1 and numbers[1] >=2:
+            calc_First_only_core(numbers[2][0],groups,cores)
+            counter += 1
+
+    if options.reclustered != None:
+        ############################# Calculate First and Reclustered-Second
+        if numbers[0] == 1 and numbers[3] >= 1:  # If Seconds did not combine First reps
+            calc_single_First_extended_Second_only_core(numbers[1], groups, cores, numbers[3])
+        elif numbers[0] > 1 and numbers[3] >= 1:  # If unique Secondss combined multiple Firsts
+            calc_multi_First_extended_Second_only_core(numbers[1], groups, cores, numbers[3])
+        elif numbers[4] >= 1:
+            Number_Of_StORF_Extending_But_Same_Genomes += 1
+        combined_pangenome_clusters_ONLY_Second_Type = defaultdict(list)
+        combined_pangenome_clusters_Second_Type = defaultdict(list)
+        for cluster, genomes in combined_pangenome_clusters_Second.items():
+            if cluster in not_Second_only_cluster_ids:
+                combined_pangenome_clusters_Second_Type[cluster] = [cluster, len(genomes)]
+            else:
+                combined_pangenome_clusters_ONLY_Second_Type[cluster] = [cluster, len(genomes)]
+        for cluster, data in combined_pangenome_clusters_Second_Type.items():
+            calc_Second_only_core(groups, cores, data[1])
+        for cluster, data in combined_pangenome_clusters_ONLY_Second_Type.items():
+            if data[1] >= 2:
+                calc_only_Second_only_core(groups, cores, data[1])
+    ###########################
+    print("End")
+    key_order = ['first_core_', 'extended_core_', 'combined_core_', 'second_core_','only_second_core_']
+    print("Gene Family Groups:")
+    for key_prefix in key_order:
+        for key, value in cores.items():
+            if key.startswith(key_prefix):
+                print(f"{key}: {value}")
+
+    if options.gene_presence_absence_out != None:
+        gene_presence_absence_output(options,genome_dict, pangenome_clusters_First_sorted, pangenome_clusters_First_sequences_sorted)
+
+
+def main():
+
+    parser = argparse.ArgumentParser(description='PyamilySeq ' + PyamilySeq_Version + ': PyamilySeq Run Parameters.')
+    parser._action_groups.pop()
+
+    required = parser.add_argument_group('Required Arguments')
+    required.add_argument('-c', action='store', dest='clusters', help='Clustering output file from CD-HIT, TSV or CSV Edge List',
+                        required=True)
+    required.add_argument('-f', action='store', dest='format', choices=['CD-HIT', 'CSV', 'TSV'],
+                        help='Which format to use (CD-HIT or  Comma/Tab Separated Edge-List (such as MMseqs2 tsv output))', required=True)
+
+
+    optional = parser.add_argument_group('Optional Arguments')
+    optional.add_argument('-rc', action='store', dest='reclustered', help='Clustering output file from secondary round of clustering',
+                        required=False)
+    optional.add_argument('-st', action='store', dest='sequence_tag', help='Default - "StORF": Unique identifier to be used to distinguish the second of two rounds of clustered sequences',
+                        required=False)
+    optional.add_argument('-groups', action="store", dest='core_groups', default="99,80,15",
+                        help='Default - (\'99,95,90,80,15\'): Gene family groups to use')
+    optional.add_argument('-gpa', action='store', dest='gene_presence_absence_out', help='Default - False: If selected, a Roary formatted gene_presence_absence.csv will be created - Required for Coinfinder and other downstream tools',
+                        required=False)
+
+    misc = parser.add_argument_group('Misc')
+    misc.add_argument('-verbose', action='store', dest='verbose', default=False, type=eval, choices=[True, False],
+                        help='Default - False: Print out runtime messages')
+    misc.add_argument('-v', action='store_true', dest='version',
+                        help='Default - False: Print out version number and exit')
+
+
+    options = parser.parse_args()
+    if options.clusters == None or options.format == None:
+        if options.version:
+            sys.exit(PyamilySeq_Version)
+        else:
+            exit('PyamilySeq: error: the following arguments are required: -c, -f')
+
+    if options.sequence_tag == None:
+        options.sequence_tag = 'StORF'
+
+    options.clusters = os.path.normpath(options.clusters)
+    options.clusters = os.path.realpath(options.clusters)
+    if options.reclustered:
+        options.reclustered = os.path.normpath(options.reclustered)
+        options.reclustered = os.path.realpath(options.reclustered)
+
+
+    options.core_groups = options.core_groups + ',0'
+
+    cluster(options)
+
+    print("Thank you for using PyamilySeq -- A detailed user manual can be found at https://github.com/NickJD/PyamilySeq\n"
+          "Please report any issues to: https://github.com/NickJD/PyamilySeq/issues\n#####")
+
+
+
+
+
+if __name__ == "__main__":
+    main()
+    print("Complete")
+

PyamilySeq/combine_FASTA_with_genome_IDs.py

@@ -0,0 +1,49 @@
+
+import argparse
+import gzip
+import glob
+
+def combine_files(files, split, glob_location, combined_out):
+    count = 0
+
+    for file in glob.glob(glob_location + '/' + files):
+        count += 1
+        try:
+            with gzip.open(file, 'rb') as genome:
+
+                for line in genome:
+                    if line.startswith(b'#'):
+                        continue
+                    elif line.startswith(b'>'):
+                        genome_name = bytes(file.split(split)[0].split('/')[-1], 'utf-8')
+                        line = line.split(b' ')[0]
+                        line = line.replace(b'>', b'>' + genome_name + b'|')
+                        combined_out.write(line.decode('utf-8')+'\n')
+                    else:
+                        combined_out.write(line.decode('utf-8'))
+        except gzip.BadGzipFile:
+            with open(file, 'r') as genome:
+
+                for line in genome:
+                    if line.startswith('#'):
+                        continue
+                    elif line.startswith('>'):
+                        genome_name = file.split(split)[0].split('/')[-1]
+                        line = line.replace('>', '>' + genome_name + '|')
+                        combined_out.write(line)
+                    else:
+                        combined_out.write(line)
+
+def main():
+    parser = argparse.ArgumentParser(description="Combine gzipped fasta files.")
+    parser.add_argument("files", help="File pattern to match within the specified directory.")
+    parser.add_argument("split", help="String used to split the file path and extract the genome name.")
+    parser.add_argument("glob_location", help="Directory location where the files are located.")
+    parser.add_argument("combined_out", help="Output file where the combined data will be written.")
+    args = parser.parse_args()
+
+    with open(args.combined_out, 'w') as combined_out:
+        combine_files(args.files, args.split, args.glob_location, combined_out)
+
+if __name__ == "__main__":
+    main()

{PyamilySeq-0.0.1.dist-info → PyamilySeq-0.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyamilySeq
-Version: 0.0.1
+Version: 0.0.2
 Summary: PyamilySeq - A a tool to look for sequence-based gene families identified by clustering methods such as CD-HIT, DIAMOND, BLAST or MMseqs2.
 Home-page: https://github.com/NickJD/PyamilySeq
 Author: Nicholas Dimonaco
@@ -12,7 +12,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: numpy
 
 # PyamilySeq
 PyamilySeq (Family Seek) is a Python tool for clustering gene sequences into families based on sequence similarity identified by tools such as CD-HIT, DIAMOND or MMseqs2.

PyamilySeq-0.0.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+PyamilySeq/CD-Hit_StORF-Reporter_Cross-Genera_Builder.py,sha256=UzQ5iOKCNfurxmj1pnkowF11YfWBO5vnBCKxQK6goB8,26538
+PyamilySeq/Constants.py,sha256=hrbTdmPUFEzLfGZOPoQPV0NsAG-VnfIX51291vqb1C8,30
+PyamilySeq/PyamilySeq_Species.py,sha256=34NHcViENyAdvGRltNUbfWjEcNCYnsmbuhDdl8__mH0,28209
+PyamilySeq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+PyamilySeq/combine_FASTA_with_genome_IDs.py,sha256=aMUVSk6jKnKX0g04RMM360QueZS83lRLqLLysBtQbLo,2009
+PyamilySeq-0.0.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+PyamilySeq-0.0.2.dist-info/METADATA,sha256=v6hOL3kekqt8H5YhjpS6uQOF1QSFcBh4Zy-jNW3xDTk,2550
+PyamilySeq-0.0.2.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+PyamilySeq-0.0.2.dist-info/entry_points.txt,sha256=zGtA2Ycf0LG3PR7zuuT0wjaAKLFxtyGgBc0O_W7E250,66
+PyamilySeq-0.0.2.dist-info/top_level.txt,sha256=J6JhugUQTq4rq96yibAlQu3o4KCM9WuYfqr3w1r119M,11
+PyamilySeq-0.0.2.dist-info/RECORD,,

PyamilySeq-0.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+PyamilySeq

PyamilySeq-0.0.1.dist-info/RECORD

@@ -1,6 +0,0 @@
-PyamilySeq-0.0.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-PyamilySeq-0.0.1.dist-info/METADATA,sha256=gYAN6guZiV3POfjJJTn20Usj3PJZ-UTsdV5gruMo86g,2571
-PyamilySeq-0.0.1.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-PyamilySeq-0.0.1.dist-info/entry_points.txt,sha256=zGtA2Ycf0LG3PR7zuuT0wjaAKLFxtyGgBc0O_W7E250,66
-PyamilySeq-0.0.1.dist-info/top_level.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-PyamilySeq-0.0.1.dist-info/RECORD,,

PyamilySeq-0.0.1.dist-info/top_level.txt

@@ -1 +0,0 @@
-