gsrap 0.7.2__py3-none-any.whl → 0.8.0__py3-none-any.whl

gsrap/commons/keggutils.py

@@ -0,0 +1,145 @@
+import time
+import os
+import sys
+import pickle
+
+
+import pandas as pnd
+from Bio.KEGG import REST
+
+
+
+def download_keggorg(logger, keggorg='lpl', outdir='./', ):
+
+
+    # check if already downloaded
+    outfile = os.path.join(outdir, f'{keggorg}.keggorg')
+    if os.path.exists(outfile):
+        logger.info(f"Organism code '{keggorg}' already downloaded ('{os.path.join(outdir, f'{keggorg}.keggorg')}').")
+        return 0
+    
+    
+    # donwload entire txt:
+    logger.info(f"Verifying existence of organism code '{keggorg}' on KEGG...")
+    time.sleep(0.5)   # be respectful
+    try: response = REST.kegg_list(keggorg).read()
+    except: 
+        logger.error(f"Organism code '{keggorg}' not found in KEGG database.")
+        return 1
+    # response is now a string similar to:
+    """
+    lpl:lp_0026	CDS	31317..32084	hydrolase, HAD superfamily, Cof family
+    lpl:lp_0027	CDS	complement(32236..32907)	pgmB1; beta-phosphoglucomutase
+    """
+
+    
+    # extract the gene IDs list:
+    gene_ids = [line.split('\t')[0] for line in response.strip().split('\n')]
+    # example of gene_id: "lpl:lp_0005"
+    logger.info(f"Respectfully downloading {len(gene_ids)} genes from KEGG...")
+    
+    
+    
+    # respectfully download in batch
+    # 10 is the max number of elements that can be downloaded
+    batch_size = 10
+    n_batches = len(gene_ids) // batch_size + (1 if (len(gene_ids) % batch_size) > 0 else 0) 
+
+
+    n_attempts = 5
+    attempts_left = n_attempts
+    default_sleep = 0.5
+    sleep_time = default_sleep
+
+
+    completed_batches = 0
+    completed_genes = 0
+    res_string_list = []
+    while completed_batches < n_batches:
+
+        # be respectful
+        time.sleep(sleep_time)
+
+        # extract batch 
+        start_index = completed_batches *batch_size
+        end_index = (completed_batches+1) *batch_size
+        if end_index > len(gene_ids): end_index = len(gene_ids)
+        curr_batch = gene_ids[start_index: end_index]
+
+
+        # download batch
+        try:
+            res_string = REST.kegg_get(curr_batch).read()
+            for item in res_string.split("///\n\n"):
+                res_string_list.append(item.replace('///\n', ''))
+            completed_batches += 1
+            completed_genes += len(curr_batch)
+
+            print(f"{completed_genes}/{len(gene_ids)} ({int(completed_genes/len(gene_ids)*100)}%) completed!", end='\r', file=sys.stderr)
+
+            attempts_left = n_attempts
+            sleep_time = default_sleep
+        except: 
+            attempts_left -= 1
+            sleep_time = default_sleep *4  # increase sleep time to be more respectful
+            logger.warning(f"An error occurred during kegg_get() of batch {curr_batch}. Remaining attempts: {attempts_left}.")
+
+            
+            if attempts_left == 0:
+                logger.error("No attemps left! Shutting down...")
+                return 1
+
+
+    # hide last progress trace ('sheets_dicts' unused if not in multi-strain mode):
+    last_trace = f"{completed_genes}/{len(gene_ids)} ({int(completed_genes/len(gene_ids)*100)}%) completed!"
+    whitewash = ''.join([' ' for i in range(len(last_trace))])
+    print(whitewash, end='\r', file=sys.stderr)   
+    
+    
+    
+    # extract info into a formatted df:
+    df = []  # list of dicts, future df
+    for entry in res_string_list:
+
+        entry_dict = {}
+        curr_header = None
+
+        for line in entry.split('\n'):
+            if line == '': continue
+
+            header = line[:12]
+            content = line[12:]
+            if header != ' '*12:
+                curr_header = header
+
+            if curr_header == 'ENTRY       ':
+                gid = content.split(' ', 1)[0]
+                entry_dict['gid'] = gid
+
+            if curr_header == 'POSITION    ':
+                entry_dict['pos'] = content.strip()
+
+            if curr_header == 'ORTHOLOGY   ':
+                ko = content.split(' ', 1)[0]
+                entry_dict['ko'] = ko
+
+            if curr_header == 'MOTIF       ':
+                db, value = content.strip().split(': ', 1)
+                entry_dict[db] = value.split(' ')
+
+            if curr_header == 'DBLINKS     ':
+                db, value = content.strip().split(': ', 1)
+                entry_dict[db] = value.split(' ')
+
+        df.append(entry_dict)
+    df = pnd.DataFrame.from_records(df)
+    
+    
+    # save dataframe in the output dir:
+    with open(outfile, 'wb') as wb_handler:
+        pickle.dump(df, wb_handler)
+    logger.info(f"'{outfile}' created!")
+        
+        
+        
+    return 0

gsrap/mkmodel/.ipynb_checkpoints/mkmodel-checkpoint.py

@@ -12,10 +12,12 @@ import gempipe
 
 from .pruner import load_input_universe
 from .pruner import load_input_eggnog
+from .pruner import load_keggorg_like_eggnog
 from .pruner import parse_eggnog
 from .pruner import subtract_kos
 from .pruner import translate_remaining_kos
 from .pruner import restore_gene_annotations
+from .pruner import append_keggorg_gene_annots
 
 from .gapfillutils import include_forced
 
@@ -38,26 +40,37 @@ from ..commons import log_metrics
 from ..commons import log_unbalances
 from ..commons import format_expansion
 from ..commons import comparative_table
+from ..commons import download_keggorg
 
 from ..runsims.biosynth import biosynthesis_on_media
 
 
 
 def create_model_incore(params):
-    universe, eggpath, dbexp, args, multistrain = params
+    annotation_source, universe, eggpath, dbexp, args, multistrain = params
+    
+    # get the logger:
     logger = get_logger('gsrap_queued', args.verbose)  # loggers can't be pickled!
+    
+    
+    # only errors will be recorded if multistrain mode
     if multistrain:
-        # only errors will be recorded
         logger.setLevel(logging.ERROR)  
 
 
     # load the annotation
-    eggnog = load_input_eggnog(logger, eggpath)
+    if annotation_source == 'keggorg':
+        eggnog_style_table = load_keggorg_like_eggnog(logger, args.keggorg, args.outdir)
+    elif annotation_source == 'eggnog':
+        eggnog_style_table = load_input_eggnog(logger, eggpath)
 
 
-    # create a copy of the universe
+    # create a copy of the universe and define the model ID
     model = universe.copy()
-    model.id = Path(eggpath).stem 
+    if annotation_source == 'keggorg':
+        model.id = args.keggorg
+    elif annotation_source == 'eggnog':
+        model.id = Path(eggpath).stem 
 
 
     ###### POLISHING 1
@@ -67,9 +80,10 @@ def create_model_incore(params):
     
 
     ###### PRUNING
-    logger.info("Reading provided eggnog-mapper annotation...")
+    if   annotation_source == 'keggorg': logger.info(f"Reading annotation for organism code '{args.keggorg}'...")
+    elif annotation_source == 'eggnog':  logger.info("Reading provided eggnog-mapper annotation...")
     # get important dictionaries: 'eggnog_ko_to_gids' and 'eggonog_gid_to_kos'
-    eggnog_ko_to_gids, eggonog_gid_to_kos = parse_eggnog(eggnog)    
+    eggnog_ko_to_gids, eggonog_gid_to_kos = parse_eggnog(eggnog_style_table)    
 
     # prune reactions
     subtract_kos(logger, model, eggnog_ko_to_gids)
@@ -77,6 +91,10 @@ def create_model_incore(params):
     # translate KOs to the actual genes
     translate_remaining_kos(logger, model, eggnog_ko_to_gids)
     restore_gene_annotations(logger, model, universe, eggonog_gid_to_kos)
+    
+    # insert gene annotation if starting from kegg organisms:
+    if annotation_source == 'keggorg': 
+        append_keggorg_gene_annots(logger, model, args.keggorg, args.outdir)
 
     
 
@@ -171,13 +189,28 @@ def main(args, logger):
     
     
     # format the --eggnog param
-    args.eggnog = format_expansion(logger, args.eggnog)
-    if args.eggnog == '-':
-        logger.error("No valid eggnog-mapper annotations provided.")
+    args.eggnog = format_expansion(logger, args.eggnog)  # now 'args.eggnog' could still be '-'
+    
+    # get the kegg organism if requested
+    if args.keggorg != '-':
+        response = download_keggorg(logger, args.keggorg, args.outdir)
+        if response == 1: return 1
+        
+    
+    
+    # determine the source of functional annotation:
+    annotation_source = None
+    if args.keggorg != '-':  # keggorg has precedence
+        annotation_source = 'keggorg'
+    elif args.eggnog != '-':
+        annotation_source = 'eggnog'
+        if args.cores > len(args.eggnog):
+            logger.debug(f"Parameter --cores {args.cores} is greater than the number of strains ({len(args.eggnog)}): reset to {len(args.eggnog)}.")
+            args.cores = len(args.eggnog)
+    else:
+        logger.error("No valid functional annotations provided: please use '--keggorg' or '--eggnog'.")
         return 1
-    if args.cores > len(args.eggnog):
-        logger.debug(f"Parameter --cores {args.cores} is greater than the number of strains ({len(args.eggnog)}): reset to {len(args.eggnog)}.")
-        args.cores = len(args.eggnog)
+    
         
     
     # check compatibility of input parameters:
@@ -201,17 +234,26 @@ def main(args, logger):
     
 
     # disable logging (swith to txt) if strains are more than 1:
-    multistrain = len(args.eggnog) > 1
-    if multistrain:
-        logger.info(f"Number of provided strains is >1: logging will be disabled.")
-        logger.info(f"Performing {len(args.eggnog)} reconstructions relying on {args.cores} cores... ")
-        # actualy this is done inside child processess!
+    if annotation_source == 'keggorg':
+        multistrain = False
+    elif annotation_source == 'eggnog':
+        multistrain = len(args.eggnog) > 1
+        if multistrain:
+            logger.info(f"Number of provided strains is >1: logging will be disabled.")
+            logger.info(f"Performing {len(args.eggnog)} reconstructions relying on {args.cores} cores... ")
+            # actualy this is done inside child processess!
+            
             
     # create strain-specific GSMMs using multi-core
     error_raised = False
     sheets_dicts = []
     executor =  confu.ProcessPoolExecutor(max_workers=args.cores)
-    futures = [executor.submit(create_model_incore, (universe, eggpath, dbexp, args, multistrain)) for eggpath in args.eggnog]
+    
+    if annotation_source == 'keggorg':
+        futures = [executor.submit(create_model_incore, (annotation_source, universe, None, dbexp, args, multistrain))]
+    elif annotation_source == 'eggnog':
+        futures = [executor.submit(create_model_incore, (annotation_source, universe, eggpath, dbexp, args, multistrain)) for eggpath in args.eggnog]
+        
     for f in confu.as_completed(futures):
         sheets_dict = f.result() 
         
@@ -226,12 +268,14 @@ def main(args, logger):
                 sheets_dicts.append(sheets_dict)
                 print(f"{len(sheets_dicts)}/{len(args.eggnog)} ({int(len(sheets_dicts)/len(args.eggnog)*100)}%) completed!", end='\r', file=sys.stderr)
     
+    
     # hide last progress trace ('sheets_dicts' unused if not in multi-strain mode):
     if multistrain and sheets_dicts != []:
         last_trace = f"{len(sheets_dicts)}/{len(args.eggnog)} ({int(len(sheets_dicts)/len(args.eggnog)*100)}%) completed!"
         whitewash = ''.join([' ' for i in range(len(last_trace))])
         print(whitewash, end='\r', file=sys.stderr)   
     
+    
     # multiproces part terminated: safely shut down the executor
     executor.shutdown(wait=True)
     

gsrap/mkmodel/.ipynb_checkpoints/pruner-checkpoint.py

@@ -1,6 +1,7 @@
 import os
 import warnings
 import logging
+import pickle
 
 
 import pandas as pnd
@@ -43,22 +44,57 @@ def load_input_eggnog(logger, eggnog):
     
     
     # load eggnog annotations
-    eggnog = pnd.read_csv(eggnog, sep='\t', comment='#', header=None)
-    eggnog.columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
-    eggnog = eggnog.set_index('query', drop=True, verify_integrity=True)
+    df_eggnog = pnd.read_csv(eggnog, sep='\t', comment='#', header=None)
+    df_eggnog.columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
+    df_eggnog = df_eggnog.set_index('query', drop=True, verify_integrity=True)
     
     
-    return eggnog
+    return df_eggnog
 
 
 
-def parse_eggnog(eggnog):
+def load_keggorg_like_eggnog(logger, keggorg, outdir):
+    
+    
+    # load raw data, downloaded form kegg: 
+    df_keggorg = pickle.load(open(os.path.join(outdir, f'{keggorg}.keggorg'), 'rb'))
+    df_keggorg = df_keggorg.set_index('gid', drop=True, verify_integrity=True)
+
+
+    # create an eggnog-like dataframe:
+    df_eggnog_like = []   # list of dict future df
+    for gid in df_keggorg.index:
+        row_dict = {}
+
+        row_dict['query'] = gid
+        row_dict['PFAMs'] = ','.join(df_keggorg.loc[gid, 'Pfam']) if type(df_keggorg.loc[gid, 'Pfam'])==list else '-'
+        row_dict['KEGG_ko'] = df_keggorg.loc[gid, 'ko'] if type(df_keggorg.loc[gid, 'ko'])==str else '-'
+
+        df_eggnog_like.append(row_dict)
+    df_eggnog_like = pnd.DataFrame.from_records(df_eggnog_like)
+
+
+    # appen missing coluns and sort
+    eggnog_columns = 'query	seed_ortholog	evalue	score	eggNOG_OGs	max_annot_lvl	COG_category	Description	Preferred_name	GOs	EC	KEGG_ko	KEGG_Pathway	KEGG_Module	KEGG_Reaction	KEGG_rclass	BRITE	KEGG_TC	CAZy	BiGG_Reaction	PFAMs'.split('\t')
+    for c in eggnog_columns:  
+        if c not in df_eggnog_like.columns:
+            df_eggnog_like[c] = '-'
+    df_eggnog_like = df_eggnog_like[eggnog_columns]
+
+
+    # set the index like in eggnog
+    df_eggnog_like = df_eggnog_like.set_index('query', drop=True, verify_integrity=True)
+    return df_eggnog_like
+
+
+
+def parse_eggnog(df_eggnog):
     
     
     # PART 1. get KO codes available
     gid_to_kos = {}
     ko_to_gids = {}
-    for gid, kos in eggnog['KEGG_ko'].items():
+    for gid, kos in df_eggnog['KEGG_ko'].items():
         if kos == '-': 
             continue
             
@@ -229,8 +265,37 @@ def restore_gene_annotations(logger, model, universe, eggonog_gid_to_kos):
             # collect names
             names.append(uni_g.name)
         g.name = '; '.join(names)
+        
+        
+        
+def append_keggorg_gene_annots(logger, model, keggorg, outdir):
+    
 
-
+    # load raw data, downloaded form kegg: 
+    logger.info("Adding gene annotations retrieved from KEGG...")
+    df_keggorg = pickle.load(open(os.path.join(outdir, f'{keggorg}.keggorg'), 'rb'))
+    df_keggorg = df_keggorg.set_index('gid', drop=True, verify_integrity=True)
+    
+    
+    # KEGG can provide some useful (ie, used in Memote) gene annotations:
+    for g in model.genes:
+        if g.id in df_keggorg.index:
+            
+            g.annotation['kegg.genes'] = [keggorg + ':' + g.id]
+            
+            if 'NCBI-GeneID' in df_keggorg.columns:
+                g.annotation['ncbigene'] = df_keggorg.loc[g.id, 'NCBI-GeneID'] if type(df_keggorg.loc[g.id, 'NCBI-GeneID'])==list else []
+            if 'NCBI-ProteinID' in df_keggorg.columns:
+                g.annotation['ncbiprotein'] = df_keggorg.loc[g.id, 'NCBI-ProteinID'] if type(df_keggorg.loc[g.id, 'NCBI-ProteinID'])==list else []
+            if 'ASAP' in df_keggorg.columns:
+                g.annotation['asap'] = df_keggorg.loc[g.id, 'ASAP'] if type(df_keggorg.loc[g.id, 'ASAP'])==list else []
+            if 'UniProt' in df_keggorg.columns:
+                g.annotation['uniprot'] = df_keggorg.loc[g.id, 'UniProt'] if type(df_keggorg.loc[g.id, 'UniProt'])==list else []
+                
+                
+                
+            
+            
 
 
     

gsrap/mkmodel/mkmodel.py

@@ -12,10 +12,12 @@ import gempipe
 
 from .pruner import load_input_universe
 from .pruner import load_input_eggnog
+from .pruner import load_keggorg_like_eggnog
 from .pruner import parse_eggnog
 from .pruner import subtract_kos
 from .pruner import translate_remaining_kos
 from .pruner import restore_gene_annotations
+from .pruner import append_keggorg_gene_annots
 
 from .gapfillutils import include_forced
 
@@ -38,26 +40,37 @@ from ..commons import log_metrics
 from ..commons import log_unbalances
 from ..commons import format_expansion
 from ..commons import comparative_table
+from ..commons import download_keggorg
 
 from ..runsims.biosynth import biosynthesis_on_media
 
 
 
 def create_model_incore(params):
-    universe, eggpath, dbexp, args, multistrain = params
+    annotation_source, universe, eggpath, dbexp, args, multistrain = params
+    
+    # get the logger:
     logger = get_logger('gsrap_queued', args.verbose)  # loggers can't be pickled!
+    
+    
+    # only errors will be recorded if multistrain mode
     if multistrain:
-        # only errors will be recorded
         logger.setLevel(logging.ERROR)  
 
 
     # load the annotation
-    eggnog = load_input_eggnog(logger, eggpath)
+    if annotation_source == 'keggorg':
+        eggnog_style_table = load_keggorg_like_eggnog(logger, args.keggorg, args.outdir)
+    elif annotation_source == 'eggnog':
+        eggnog_style_table = load_input_eggnog(logger, eggpath)
 
 
-    # create a copy of the universe
+    # create a copy of the universe and define the model ID
     model = universe.copy()
-    model.id = Path(eggpath).stem 
+    if annotation_source == 'keggorg':
+        model.id = args.keggorg
+    elif annotation_source == 'eggnog':
+        model.id = Path(eggpath).stem 
 
 
     ###### POLISHING 1
@@ -67,9 +80,10 @@ def create_model_incore(params):
     
 
     ###### PRUNING
-    logger.info("Reading provided eggnog-mapper annotation...")
+    if   annotation_source == 'keggorg': logger.info(f"Reading annotation for organism code '{args.keggorg}'...")
+    elif annotation_source == 'eggnog':  logger.info("Reading provided eggnog-mapper annotation...")
     # get important dictionaries: 'eggnog_ko_to_gids' and 'eggonog_gid_to_kos'
-    eggnog_ko_to_gids, eggonog_gid_to_kos = parse_eggnog(eggnog)    
+    eggnog_ko_to_gids, eggonog_gid_to_kos = parse_eggnog(eggnog_style_table)    
 
     # prune reactions
     subtract_kos(logger, model, eggnog_ko_to_gids)
@@ -77,6 +91,10 @@ def create_model_incore(params):
     # translate KOs to the actual genes
     translate_remaining_kos(logger, model, eggnog_ko_to_gids)
     restore_gene_annotations(logger, model, universe, eggonog_gid_to_kos)
+    
+    # insert gene annotation if starting from kegg organisms:
+    if annotation_source == 'keggorg': 
+        append_keggorg_gene_annots(logger, model, args.keggorg, args.outdir)
 
     
 
@@ -171,13 +189,28 @@ def main(args, logger):
     
     
     # format the --eggnog param
-    args.eggnog = format_expansion(logger, args.eggnog)
-    if args.eggnog == '-':
-        logger.error("No valid eggnog-mapper annotations provided.")
+    args.eggnog = format_expansion(logger, args.eggnog)  # now 'args.eggnog' could still be '-'
+    
+    # get the kegg organism if requested
+    if args.keggorg != '-':
+        response = download_keggorg(logger, args.keggorg, args.outdir)
+        if response == 1: return 1
+        
+    
+    
+    # determine the source of functional annotation:
+    annotation_source = None
+    if args.keggorg != '-':  # keggorg has precedence
+        annotation_source = 'keggorg'
+    elif args.eggnog != '-':
+        annotation_source = 'eggnog'
+        if args.cores > len(args.eggnog):
+            logger.debug(f"Parameter --cores {args.cores} is greater than the number of strains ({len(args.eggnog)}): reset to {len(args.eggnog)}.")
+            args.cores = len(args.eggnog)
+    else:
+        logger.error("No valid functional annotations provided: please use '--keggorg' or '--eggnog'.")
         return 1
-    if args.cores > len(args.eggnog):
-        logger.debug(f"Parameter --cores {args.cores} is greater than the number of strains ({len(args.eggnog)}): reset to {len(args.eggnog)}.")
-        args.cores = len(args.eggnog)
+    
         
     
     # check compatibility of input parameters:
@@ -201,17 +234,26 @@ def main(args, logger):
     
 
     # disable logging (swith to txt) if strains are more than 1:
-    multistrain = len(args.eggnog) > 1
-    if multistrain:
-        logger.info(f"Number of provided strains is >1: logging will be disabled.")
-        logger.info(f"Performing {len(args.eggnog)} reconstructions relying on {args.cores} cores... ")
-        # actualy this is done inside child processess!
+    if annotation_source == 'keggorg':
+        multistrain = False
+    elif annotation_source == 'eggnog':
+        multistrain = len(args.eggnog) > 1
+        if multistrain:
+            logger.info(f"Number of provided strains is >1: logging will be disabled.")
+            logger.info(f"Performing {len(args.eggnog)} reconstructions relying on {args.cores} cores... ")
+            # actualy this is done inside child processess!
+            
             
     # create strain-specific GSMMs using multi-core
     error_raised = False
     sheets_dicts = []
     executor =  confu.ProcessPoolExecutor(max_workers=args.cores)
-    futures = [executor.submit(create_model_incore, (universe, eggpath, dbexp, args, multistrain)) for eggpath in args.eggnog]
+    
+    if annotation_source == 'keggorg':
+        futures = [executor.submit(create_model_incore, (annotation_source, universe, None, dbexp, args, multistrain))]
+    elif annotation_source == 'eggnog':
+        futures = [executor.submit(create_model_incore, (annotation_source, universe, eggpath, dbexp, args, multistrain)) for eggpath in args.eggnog]
+        
     for f in confu.as_completed(futures):
         sheets_dict = f.result() 
         
@@ -226,12 +268,14 @@ def main(args, logger):
                 sheets_dicts.append(sheets_dict)
                 print(f"{len(sheets_dicts)}/{len(args.eggnog)} ({int(len(sheets_dicts)/len(args.eggnog)*100)}%) completed!", end='\r', file=sys.stderr)
     
+    
     # hide last progress trace ('sheets_dicts' unused if not in multi-strain mode):
     if multistrain and sheets_dicts != []:
         last_trace = f"{len(sheets_dicts)}/{len(args.eggnog)} ({int(len(sheets_dicts)/len(args.eggnog)*100)}%) completed!"
         whitewash = ''.join([' ' for i in range(len(last_trace))])
         print(whitewash, end='\r', file=sys.stderr)   
     
+    
     # multiproces part terminated: safely shut down the executor
     executor.shutdown(wait=True)