gsrap 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

gsrap/mkmodel/.ipynb_checkpoints/pruner-checkpoint.py

@@ -1,6 +1,7 @@
 import os
 import warnings
 import logging
+import pickle
 
 
 import pandas as pnd
@@ -43,22 +44,57 @@ def load_input_eggnog(logger, eggnog):
     
     
     # load eggnog annotations
-    eggnog = pnd.read_csv(eggnog, sep='\t', comment='#', header=None)
-    eggnog.columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
-    eggnog = eggnog.set_index('query', drop=True, verify_integrity=True)
+    df_eggnog = pnd.read_csv(eggnog, sep='\t', comment='#', header=None)
+    df_eggnog.columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
+    df_eggnog = df_eggnog.set_index('query', drop=True, verify_integrity=True)
     
     
-    return eggnog
+    return df_eggnog
 
 
 
-def parse_eggnog(eggnog):
+def load_keggorg_like_eggnog(logger, keggorg, outdir):
+    
+    
+    # load raw data, downloaded form kegg: 
+    df_keggorg = pickle.load(open(os.path.join(outdir, f'{keggorg}.keggorg'), 'rb'))
+    df_keggorg = df_keggorg.set_index('gid', drop=True, verify_integrity=True)
+
+
+    # create an eggnog-like dataframe:
+    df_eggnog_like = []   # list of dict future df
+    for gid in df_keggorg.index:
+        row_dict = {}
+
+        row_dict['query'] = gid
+        row_dict['PFAMs'] = ','.join(df_keggorg.loc[gid, 'Pfam']) if type(df_keggorg.loc[gid, 'Pfam'])==list else '-'
+        row_dict['KEGG_ko'] = df_keggorg.loc[gid, 'ko'] if type(df_keggorg.loc[gid, 'ko'])==str else '-'
+
+        df_eggnog_like.append(row_dict)
+    df_eggnog_like = pnd.DataFrame.from_records(df_eggnog_like)
+
+
+    # appen missing coluns and sort
+    eggnog_columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
+    for c in eggnog_columns:  
+        if c not in df_eggnog_like.columns:
+            df_eggnog_like[c] = '-'
+    df_eggnog_like = df_eggnog_like[eggnog_columns]
+
+
+    # set the index like in eggnog
+    df_eggnog_like = df_eggnog_like.set_index('query', drop=True, verify_integrity=True)
+    return df_eggnog_like
+
+
+
+def parse_eggnog(df_eggnog):
     
     
     # PART 1. get KO codes available
     gid_to_kos = {}
     ko_to_gids = {}
-    for gid, kos in eggnog['KEGG_ko'].items():
+    for gid, kos in df_eggnog['KEGG_ko'].items():
         if kos == '-': 
             continue
             
@@ -229,8 +265,37 @@ def restore_gene_annotations(logger, model, universe, eggonog_gid_to_kos):
             # collect names
             names.append(uni_g.name)
         g.name = '; '.join(names)
+        
+        
+        
+def append_keggorg_gene_annots(logger, model, keggorg, outdir):
+    
 
-
+    # load raw data, downloaded form kegg: 
+    logger.info("Adding gene annotations retrieved from KEGG...")
+    df_keggorg = pickle.load(open(os.path.join(outdir, f'{keggorg}.keggorg'), 'rb'))
+    df_keggorg = df_keggorg.set_index('gid', drop=True, verify_integrity=True)
+    
+    
+    # KEGG can provide some useful (ie, used in Memote) gene annotations:
+    for g in model.genes:
+        if g.id in df_keggorg.index:
+            
+            g.annotation['kegg.genes'] = [keggorg + ':' + g.id]
+            
+            if 'NCBI-GeneID' in df_keggorg.columns:
+                g.annotation['ncbigene'] = df_keggorg.loc[g.id, 'NCBI-GeneID'] if type(df_keggorg.loc[g.id, 'NCBI-GeneID'])==list else []
+            if 'NCBI-ProteinID' in df_keggorg.columns:
+                g.annotation['ncbiprotein'] = df_keggorg.loc[g.id, 'NCBI-ProteinID'] if type(df_keggorg.loc[g.id, 'NCBI-ProteinID'])==list else []
+            if 'ASAP' in df_keggorg.columns:
+                g.annotation['asap'] = df_keggorg.loc[g.id, 'ASAP'] if type(df_keggorg.loc[g.id, 'ASAP'])==list else []
+            if 'UniProt' in df_keggorg.columns:
+                g.annotation['uniprot'] = df_keggorg.loc[g.id, 'UniProt'] if type(df_keggorg.loc[g.id, 'UniProt'])==list else []
+                
+                
+                
+            
+            
 
 
     

gsrap/mkmodel/mkmodel.py

@@ -12,10 +12,12 @@ import gempipe
 
 from .pruner import load_input_universe
 from .pruner import load_input_eggnog
+from .pruner import load_keggorg_like_eggnog
 from .pruner import parse_eggnog
 from .pruner import subtract_kos
 from .pruner import translate_remaining_kos
 from .pruner import restore_gene_annotations
+from .pruner import append_keggorg_gene_annots
 
 from .gapfillutils import include_forced
 
@@ -38,26 +40,37 @@ from ..commons import log_metrics
 from ..commons import log_unbalances
 from ..commons import format_expansion
 from ..commons import comparative_table
+from ..commons import download_keggorg
 
 from ..runsims.biosynth import biosynthesis_on_media
 
 
 
 def create_model_incore(params):
-    universe, eggpath, dbexp, args, multistrain = params
+    annotation_source, universe, eggpath, dbexp, args, multistrain = params
+    
+    # get the logger:
     logger = get_logger('gsrap_queued', args.verbose)  # loggers can't be pickled!
+    
+    
+    # only errors will be recorded if multistrain mode
     if multistrain:
-        # only errors will be recorded
         logger.setLevel(logging.ERROR)  
 
 
     # load the annotation
-    eggnog = load_input_eggnog(logger, eggpath)
+    if annotation_source == 'keggorg':
+        eggnog_style_table = load_keggorg_like_eggnog(logger, args.keggorg, args.outdir)
+    elif annotation_source == 'eggnog':
+        eggnog_style_table = load_input_eggnog(logger, eggpath)
 
 
-    # create a copy of the universe
+    # create a copy of the universe and define the model ID
     model = universe.copy()
-    model.id = Path(eggpath).stem 
+    if annotation_source == 'keggorg':
+        model.id = args.keggorg
+    elif annotation_source == 'eggnog':
+        model.id = Path(eggpath).stem 
 
 
     ###### POLISHING 1
@@ -67,9 +80,10 @@ def create_model_incore(params):
     
 
     ###### PRUNING
-    logger.info("Reading provided eggnog-mapper annotation...")
+    if   annotation_source == 'keggorg': logger.info(f"Reading annotation for organism code '{args.keggorg}'...")
+    elif annotation_source == 'eggnog':  logger.info("Reading provided eggnog-mapper annotation...")
     # get important dictionaries: 'eggnog_ko_to_gids' and 'eggonog_gid_to_kos'
-    eggnog_ko_to_gids, eggonog_gid_to_kos = parse_eggnog(eggnog)    
+    eggnog_ko_to_gids, eggonog_gid_to_kos = parse_eggnog(eggnog_style_table)    
 
     # prune reactions
     subtract_kos(logger, model, eggnog_ko_to_gids)
@@ -77,6 +91,10 @@ def create_model_incore(params):
     # translate KOs to the actual genes
     translate_remaining_kos(logger, model, eggnog_ko_to_gids)
     restore_gene_annotations(logger, model, universe, eggonog_gid_to_kos)
+    
+    # insert gene annotation if starting from kegg organisms:
+    if annotation_source == 'keggorg': 
+        append_keggorg_gene_annots(logger, model, args.keggorg, args.outdir)
 
     
 
@@ -141,7 +159,7 @@ def create_model_incore(params):
     cobra.io.write_sbml_model(model, f'{args.outdir}/{model.id}.xml')        # SBML   # groups are saved only to SBML
     logger.info(f"'{args.outdir}/{model.id}.xml' created!")
     force_id_on_sbml(f'{args.outdir}/{model.id}.xml', model.id)   # force introduction of the 'id=""' field
-    sheets_dict = write_excel_model(model, f'{args.outdir}/{model.id}.mkmodel.xlsx', None, df_B, df_P, df_S)  
+    sheets_dict = write_excel_model(model, f'{args.outdir}/{model.id}.mkmodel.xlsx', args.nofigs, None, df_B, df_P, df_S)  
     logger.info(f"'{args.outdir}/{model.id}.mkmodel.xlsx' created!")
         
         
@@ -171,13 +189,28 @@ def main(args, logger):
     
     
     # format the --eggnog param
-    args.eggnog = format_expansion(logger, args.eggnog)
-    if args.eggnog == '-':
-        logger.error("No valid eggnog-mapper annotations provided.")
+    args.eggnog = format_expansion(logger, args.eggnog)  # now 'args.eggnog' could still be '-'
+    
+    # get the kegg organism if requested
+    if args.keggorg != '-':
+        response = download_keggorg(logger, args.keggorg, args.outdir)
+        if response == 1: return 1
+        
+    
+    
+    # determine the source of functional annotation:
+    annotation_source = None
+    if args.keggorg != '-':  # keggorg has precedence
+        annotation_source = 'keggorg'
+    elif args.eggnog != '-':
+        annotation_source = 'eggnog'
+        if args.cores > len(args.eggnog):
+            logger.debug(f"Parameter --cores {args.cores} is greater than the number of strains ({len(args.eggnog)}): reset to {len(args.eggnog)}.")
+            args.cores = len(args.eggnog)
+    else:
+        logger.error("No valid functional annotations provided: please use '--keggorg' or '--eggnog'.")
         return 1
-    if args.cores > len(args.eggnog):
-        logger.debug(f"Parameter --cores {args.cores} is greater than the number of strains ({len(args.eggnog)}): reset to {len(args.eggnog)}.")
-        args.cores = len(args.eggnog)
+    
         
     
     # check compatibility of input parameters:
@@ -201,17 +234,26 @@ def main(args, logger):
     
 
     # disable logging (swith to txt) if strains are more than 1:
-    multistrain = len(args.eggnog) > 1
-    if multistrain:
-        logger.info(f"Number of provided strains is >1: logging will be disabled.")
-        logger.info(f"Performing {len(args.eggnog)} reconstructions relying on {args.cores} cores... ")
-        # actualy this is done inside child processess!
+    if annotation_source == 'keggorg':
+        multistrain = False
+    elif annotation_source == 'eggnog':
+        multistrain = len(args.eggnog) > 1
+        if multistrain:
+            logger.info(f"Number of provided strains is >1: logging will be disabled.")
+            logger.info(f"Performing {len(args.eggnog)} reconstructions relying on {args.cores} cores... ")
+            # actualy this is done inside child processess!
+            
             
     # create strain-specific GSMMs using multi-core
     error_raised = False
     sheets_dicts = []
     executor =  confu.ProcessPoolExecutor(max_workers=args.cores)
-    futures = [executor.submit(create_model_incore, (universe, eggpath, dbexp, args, multistrain)) for eggpath in args.eggnog]
+    
+    if annotation_source == 'keggorg':
+        futures = [executor.submit(create_model_incore, (annotation_source, universe, None, dbexp, args, multistrain))]
+    elif annotation_source == 'eggnog':
+        futures = [executor.submit(create_model_incore, (annotation_source, universe, eggpath, dbexp, args, multistrain)) for eggpath in args.eggnog]
+        
     for f in confu.as_completed(futures):
         sheets_dict = f.result() 
         
@@ -226,12 +268,14 @@ def main(args, logger):
                 sheets_dicts.append(sheets_dict)
                 print(f"{len(sheets_dicts)}/{len(args.eggnog)} ({int(len(sheets_dicts)/len(args.eggnog)*100)}%) completed!", end='\r', file=sys.stderr)
     
+    
     # hide last progress trace ('sheets_dicts' unused if not in multi-strain mode):
     if multistrain and sheets_dicts != []:
         last_trace = f"{len(sheets_dicts)}/{len(args.eggnog)} ({int(len(sheets_dicts)/len(args.eggnog)*100)}%) completed!"
         whitewash = ''.join([' ' for i in range(len(last_trace))])
         print(whitewash, end='\r', file=sys.stderr)   
     
+    
     # multiproces part terminated: safely shut down the executor
     executor.shutdown(wait=True)
     

gsrap/mkmodel/pruner.py

@@ -1,6 +1,7 @@
 import os
 import warnings
 import logging
+import pickle
 
 
 import pandas as pnd
@@ -43,22 +44,57 @@ def load_input_eggnog(logger, eggnog):
     
     
     # load eggnog annotations
-    eggnog = pnd.read_csv(eggnog, sep='\t', comment='#', header=None)
-    eggnog.columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
-    eggnog = eggnog.set_index('query', drop=True, verify_integrity=True)
+    df_eggnog = pnd.read_csv(eggnog, sep='\t', comment='#', header=None)
+    df_eggnog.columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
+    df_eggnog = df_eggnog.set_index('query', drop=True, verify_integrity=True)
     
     
-    return eggnog
+    return df_eggnog
 
 
 
-def parse_eggnog(eggnog):
+def load_keggorg_like_eggnog(logger, keggorg, outdir):
+    
+    
+    # load raw data, downloaded form kegg: 
+    df_keggorg = pickle.load(open(os.path.join(outdir, f'{keggorg}.keggorg'), 'rb'))
+    df_keggorg = df_keggorg.set_index('gid', drop=True, verify_integrity=True)
+
+
+    # create an eggnog-like dataframe:
+    df_eggnog_like = []   # list of dict future df
+    for gid in df_keggorg.index:
+        row_dict = {}
+
+        row_dict['query'] = gid
+        row_dict['PFAMs'] = ','.join(df_keggorg.loc[gid, 'Pfam']) if type(df_keggorg.loc[gid, 'Pfam'])==list else '-'
+        row_dict['KEGG_ko'] = df_keggorg.loc[gid, 'ko'] if type(df_keggorg.loc[gid, 'ko'])==str else '-'
+
+        df_eggnog_like.append(row_dict)
+    df_eggnog_like = pnd.DataFrame.from_records(df_eggnog_like)
+
+
+    # appen missing coluns and sort
+    eggnog_columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
+    for c in eggnog_columns:  
+        if c not in df_eggnog_like.columns:
+            df_eggnog_like[c] = '-'
+    df_eggnog_like = df_eggnog_like[eggnog_columns]
+
+
+    # set the index like in eggnog
+    df_eggnog_like = df_eggnog_like.set_index('query', drop=True, verify_integrity=True)
+    return df_eggnog_like
+
+
+
+def parse_eggnog(df_eggnog):
     
     
     # PART 1. get KO codes available
     gid_to_kos = {}
     ko_to_gids = {}
-    for gid, kos in eggnog['KEGG_ko'].items():
+    for gid, kos in df_eggnog['KEGG_ko'].items():
         if kos == '-': 
             continue
             
@@ -229,8 +265,37 @@ def restore_gene_annotations(logger, model, universe, eggonog_gid_to_kos):
             # collect names
             names.append(uni_g.name)
         g.name = '; '.join(names)
+        
+        
+        
+def append_keggorg_gene_annots(logger, model, keggorg, outdir):
+    
 
-
+    # load raw data, downloaded form kegg: 
+    logger.info("Adding gene annotations retrieved from KEGG...")
+    df_keggorg = pickle.load(open(os.path.join(outdir, f'{keggorg}.keggorg'), 'rb'))
+    df_keggorg = df_keggorg.set_index('gid', drop=True, verify_integrity=True)
+    
+    
+    # KEGG can provide some useful (ie, used in Memote) gene annotations:
+    for g in model.genes:
+        if g.id in df_keggorg.index:
+            
+            g.annotation['kegg.genes'] = [keggorg + ':' + g.id]
+            
+            if 'NCBI-GeneID' in df_keggorg.columns:
+                g.annotation['ncbigene'] = df_keggorg.loc[g.id, 'NCBI-GeneID'] if type(df_keggorg.loc[g.id, 'NCBI-GeneID'])==list else []
+            if 'NCBI-ProteinID' in df_keggorg.columns:
+                g.annotation['ncbiprotein'] = df_keggorg.loc[g.id, 'NCBI-ProteinID'] if type(df_keggorg.loc[g.id, 'NCBI-ProteinID'])==list else []
+            if 'ASAP' in df_keggorg.columns:
+                g.annotation['asap'] = df_keggorg.loc[g.id, 'ASAP'] if type(df_keggorg.loc[g.id, 'ASAP'])==list else []
+            if 'UniProt' in df_keggorg.columns:
+                g.annotation['uniprot'] = df_keggorg.loc[g.id, 'UniProt'] if type(df_keggorg.loc[g.id, 'UniProt'])==list else []
+                
+                
+                
+            
+            
 
 
     

gsrap/parsedb/.ipynb_checkpoints/completeness-checkpoint.py

@@ -1,3 +1,8 @@
+from pathlib import Path
+import pickle
+import os
+
+
 import pandas as pnd
 
 
@@ -32,14 +37,39 @@ def parse_eggnog(model, eggnog, idcollection_dict):
     return krs_org
     
     
+
+def parse_keggorg(keggorg, outdir, idcollection_dict):
+
+    df_keggorg = pickle.load(open(os.path.join(outdir, f'{keggorg}.keggorg'), 'rb'))
+    df_keggorg = df_keggorg.set_index('gid', drop=True, verify_integrity=True)
+
+    
+    # PART 1. get KO codes available
+    kos_org = set([i for i in df_keggorg['ko'] if pnd.isna(i)==False])
+    
+    
+    # PART 2. get reactions in the organism (even the GPR is not complete)
+    kr_to_kos = idcollection_dict['kr_to_kos']
+    krs_org = set()
+    for kr, kos in kr_to_kos.items(): 
+        if any([ko in kos_org for ko in kos]):
+            krs_org.add(kr)
     
-def check_completeness(logger, model, progress, module, focus, eggnog, zeroes, idcollection_dict, summary_dict): 
+    
+    return krs_org
+
+    
+    
+def check_completeness(logger, model, progress, module, focus, eggnog, keggorg, idcollection_dict, summary_dict, outdir): 
     # check KEGG annotations in the universe model to get '%' of completeness per pathway/module.
     
             
     # get the reference set of kr codes (all kegg or organism specific): 
     kr_uni = set()
-    if eggnog != '-':
+    if keggorg != '-':  # keggorg has precedence
+        kr_uni = parse_keggorg(keggorg, outdir, idcollection_dict)
+        kr_uni_label = f"organism code '{keggorg}'"
+    elif eggnog != '-':
         for eggfile in eggnog:
             eggset = parse_eggnog(model, eggfile, idcollection_dict)
             kr_uni = kr_uni.union(eggset)
@@ -55,10 +85,22 @@ def check_completeness(logger, model, progress, module, focus, eggnog, zeroes, i
         if 'kegg.reaction' in r.annotation.keys():
             for kr_id in r.annotation['kegg.reaction']:
                 kr_ids_modeled.add(kr_id)
-    kr_uni_missing = len(kr_uni - kr_ids_modeled.intersection(kr_uni))
+    kr_uni_missing = kr_uni - kr_ids_modeled
     kr_uni_coverage = len(kr_ids_modeled.intersection(kr_uni)) / len(kr_uni) * 100
-    logger.info(f"Coverage for '{kr_uni_label}': {round(kr_uni_coverage, 0)}% ({kr_uni_missing} missing).")
+    logger.info(f"Coverage for {kr_uni_label}: {round(kr_uni_coverage, 0)}% ({len(kr_uni_missing)} missing).")
+    
     
+    # define the map?????, containing krs not included in maps
+    krs_in_maps = set()
+    for i in summary_dict: krs_in_maps = krs_in_maps.union(i['kr_ids']) 
+    krs_not_in_maps = idcollection_dict['kr'] - krs_in_maps
+    summary_dict.append({
+        'map_id': 'map?????',
+        'map_name': 'Not included in maps',
+        'kr_ids': krs_not_in_maps,
+        'cnt_r': len(krs_not_in_maps),
+        'mds': []
+    })    
     
     
     # get all the map / md codes:
@@ -112,52 +154,77 @@ def check_completeness(logger, model, progress, module, focus, eggnog, zeroes, i
             missing_logger = (map_id, missing)
 
         
+        # put the map in the right bucket:
         if missing == set() and map_krs != set():
             maps_finished.add(map_id)
-            
         elif map_krs == set():
             maps_noreac.add(map_id)
-            
         elif missing == map_krs:
             maps_missing.add(map_id)
-            
-            if zeroes:
-                list_coverage.append({
-                    'map_id': map_id,
-                    'map_name_short': map_name_short, 
-                    'perc_completeness': 0,
-                    'perc_completeness_str': ' 0',
-                    'present': present,
-                    'missing': missing,
-                    'md_ids': [j['md_id'] for j in i['mds']],
-                })
-            
         elif len(missing) < len(map_krs):
             maps_partial.add(map_id)
             
-            # get '%' of completeness:
-            perc_completeness = len(present)/len(map_krs)*100
-            perc_completeness_str = str(round(perc_completeness))   # version to be printed
-            if len(perc_completeness_str)==1: 
-                perc_completeness_str = ' ' + perc_completeness_str
-                
-            list_coverage.append({
-                'map_id': map_id,
-                'map_name_short': map_name_short, 
-                'perc_completeness': perc_completeness,
-                'perc_completeness_str': perc_completeness_str,
-                'present': present,
-                'missing': missing,
-                'md_ids': [j['md_id'] for j in i['mds']],
-            })
             
+        # get '%' of completeness:
+        if len(map_krs) != 0: perc_completeness = len(present)/len(map_krs)*100
+        else: perc_completeness = 100   # for maps_noreac
+        perc_completeness_str = str(round(perc_completeness))   # version to be printed
+        if len(perc_completeness_str)==1: 
+            perc_completeness_str = ' ' + perc_completeness_str
+
             
-    # order list by '%' of completness and print:
+        # append map to list:
+        list_coverage.append({
+            'map_id': map_id,
+            'map_name_short': map_name_short, 
+            'perc_completeness': perc_completeness,
+            'perc_completeness_str': perc_completeness_str,
+            'present': present,
+            'missing': missing,
+            'md_ids': [j['md_id'] for j in i['mds']],
+        })
+        
+        
+    
+    # create coverage dataframe
+    if eggnog != '-' and len(eggnog) >= 2:  
+        df_coverage = {}
+        for i in list_coverage:
+            for kr in i['present'].union(i['missing']):
+                if kr not in df_coverage.keys(): 
+                    df_coverage[kr] = {'map_ids': set()}
+                df_coverage[kr]['map_ids'].add(i['map_id'])
+        df_coverage = pnd.DataFrame.from_records(df_coverage).T
+        df_coverage['modeled'] = False
+        for kr, row in df_coverage.iterrows():
+            if kr in kr_ids_modeled:
+                df_coverage.loc[kr, 'modeled'] = True
+        # build strain columns all at once
+        df_strains = []  # list of small DataFrames
+        for eggfile in eggnog:
+            strain = Path(eggfile).stem
+            eggset = parse_eggnog(model, eggfile, idcollection_dict)
+            col = df_coverage.index.to_series().isin(eggset).astype(int)  # integer: 0 or 1
+            df_strains.append(col.rename(strain))
+        df_strains = pnd.concat(df_strains, axis=1)
+        # sort rows: upper rows are present in more strains
+        #df_strains = df_strains.loc[df_strains.sum(axis=1).sort_values(ascending=False).index]   # commented: now in charge of figures.py
+        df_coverage = df_coverage.loc[df_strains.index]
+        df_coverage = pnd.concat([df_coverage, df_strains], axis=1)
+        # split in 2: modeled above, non-modeled below:
+        #df_coverage = pnd.concat([df_coverage[df_coverage['modeled']==True], df_coverage[df_coverage['modeled']==False]])   # commented: now in charge of figures.py
+    else:  # not interesting in a super-long table without strains in column
+        df_coverage = None
+    
+           
+            
+    # order list by '%' of completness and print if needed:
     list_coverage = sorted(list_coverage, key=lambda x: x['perc_completeness'], reverse=True)
     for i in list_coverage:
         if progress:
             if focus=='-' or focus in i['md_ids'] or focus==i['map_id']:
-                logger.info(f"{i['map_id']}: {i['map_name_short']} {i['perc_completeness_str']}% completed, {len(i['present'])} added, {len(i['missing'])} missing.")
+                if i['map_id'] in maps_missing or i['map_id'] in maps_partial:
+                    logger.info(f"{i['map_id']}: {i['map_name_short']} {i['perc_completeness_str']}% completed, {len(i['present'])} added, {len(i['missing'])} missing.")
         
         
         # get the correspondent pathway element of the 'summary_dict'
@@ -199,50 +266,43 @@ def check_completeness(logger, model, progress, module, focus, eggnog, zeroes, i
                 missing_logger = (md_id, missing)
             
             
+            # put the map in the right bucket:
             if missing == set() and md_krs != set():
                 mds_completed.add(md_id)
-
             elif md_krs == set():
                 mds_noreac.add(md_id)
-
             elif missing == md_krs:
                 mds_missing.add(md_id)
-                
-                if zeroes:
-                    list_coverage_md.append({
-                        'md_id': md_id,
-                        'md_name_short': md_name_short, 
-                        'perc_completeness': 0,
-                        'perc_completeness_str': ' 0',
-                        'present': present,
-                        'missing': missing,
-                    })
-                
             elif len(missing) < len(md_krs):
                 mds_partial.add(md_id)
 
-                # get '%' of completeness:
-                perc_completeness = len(present)/len(md_krs)*100
-                perc_completeness_str = str(round(perc_completeness))   # version to be printed
-                if len(perc_completeness_str)==1: 
-                    perc_completeness_str = ' ' + perc_completeness_str
+                
+            # get '%' of completeness:
+            if len(md_krs) != 0: perc_completeness = len(present)/len(md_krs)*100
+            else: perc_completeness = 100   # for mds_noreac
+            perc_completeness_str = str(round(perc_completeness))   # version to be printed
+            if len(perc_completeness_str)==1: 
+                perc_completeness_str = ' ' + perc_completeness_str
 
-                list_coverage_md.append({
-                    'md_id': md_id,
-                    'md_name_short': md_name_short, 
-                    'perc_completeness': perc_completeness,
-                    'perc_completeness_str': perc_completeness_str,
-                    'present': present,
-                    'missing': missing,
-                })
+                
+            # append md to list:
+            list_coverage_md.append({
+                'md_id': md_id,
+                'md_name_short': md_name_short, 
+                'perc_completeness': perc_completeness,
+                'perc_completeness_str': perc_completeness_str,
+                'present': present,
+                'missing': missing,
+            })
                
             
-        # order list by '%' of completness and print:
+        # order list by '%' of completness and print if needed:
         list_coverage_md = sorted(list_coverage_md, key=lambda x: x['perc_completeness'], reverse=True)
         for z in list_coverage_md:
             if module:
                 if focus=='-' or focus==z['md_id']:
-                    logger.info(f"{spacer}{z['md_id']}: {z['md_name_short']} {z['perc_completeness_str']}% completed, {len(z['present'])} added, {len(z['missing'])} missing.")
+                    if z['md_id'] in mds_missing or z['md_id'] in mds_partial:
+                        logger.info(f"{spacer}{z['md_id']}: {z['md_name_short']} {z['perc_completeness_str']}% completed, {len(z['present'])} added, {len(z['missing'])} missing.")
         
         
         # print summary:
@@ -254,6 +314,6 @@ def check_completeness(logger, model, progress, module, focus, eggnog, zeroes, i
         logger.info(f"Maps: finished {len(maps_finished)} - partial {len(maps_partial)} - missing {len(maps_missing)} - noreac {len(maps_noreac)}")
             
         
-    return 0     
+    return df_coverage     
 
 

gsrap/parsedb/.ipynb_checkpoints/introduce-checkpoint.py

@@ -143,6 +143,14 @@ def introduce_metabolites(logger, db, model, idcollection_dict, kegg_compound_to
             m.annotation[ankey] = list(m.annotation[ankey])
             
             
+        # replace inchikey with manually-curated
+        if m.annotation['inchikey'] != [] and m.annotation['inchikey'] != [row['inchikey']]:
+            logger.debug(f"Metabolite '{pure_mid}': manual-curated inchikey ({[row['inchikey']]}) is diferent from the one derived from MNX ({m.annotation['inchikey']}).")
+        m.annotation['inchikey'] = [row['inchikey']]  # force the manual-curated version
+        if m.annotation['inchikey'] == ['XXXXXXXXXXXXXX-XXXXXXXXXX-X']:
+            m.annotation['inchikey'] = []
+            
+            
         # add SBO annotation
         m.annotation['sbo'] = ['SBO:0000247']  # generic metabolite