py-semtools 1.3.3__tar.gz → 1.3.4__tar.gz

{py_semtools-1.3.3 → py_semtools-1.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py_semtools
-Version: 1.3.3
+Version: 1.3.4
 Summary: Library to handle ontologies that allows queries and calculations such as information coefficients, semantic similarity, ontology representations, etc in a easy way. It can load any ontology that complies the obo format supported by OBO Foundry.
 Home-page: https://github.com/seoanezonjic/py_semtools
 Author: seoanezonjic
@@ -26,10 +26,12 @@ Requires-Dist: py_report_html
 Requires-Dist: loguru
 Requires-Dist: loky
 Requires-Dist: bm25s
+Requires-Dist: langchain-text-splitters
 Provides-Extra: models
 Requires-Dist: sentence_transformers; extra == "models"
 Requires-Dist: langchain; extra == "models"
-Requires-Dist: langchain-text-splitters; extra == "models"
+Requires-Dist: ontogpt; extra == "models"
+Requires-Dist: faiss-cpu; extra == "models"
 Provides-Extra: testing
 Requires-Dist: setuptools; extra == "testing"
 Requires-Dist: pytest; extra == "testing"

{py_semtools-1.3.3 → py_semtools-1.3.4}/setup.cfg

@@ -36,6 +36,7 @@ install_requires =
 	loguru
 	loky
 	bm25s
+	langchain-text-splitters
 
 [options.packages.find]
 where = src
@@ -49,7 +50,7 @@ py_semtools.templates =
 	*.txt
 
 [options.extras_require]
-MODELS = sentence_transformers; langchain; langchain-text-splitters
+MODELS = sentence_transformers; langchain; ontogpt; faiss-cpu
 testing = 
 	setuptools
 	pytest
@@ -62,6 +63,7 @@ console_scripts =
 	get_sorted_suggestions = py_semtools.cli_manager:get_sorted_suggestions
 	get_sorted_profs = py_semtools.cli_manager:get_sorted_profs
 	remote_retriever = py_semtools.cli_manager:remote_retriever
+	llmEngine = py_semtools.cli_manager:llmEngine
 	fmEngine = py_semtools.cli_manager:fmEngine
 	stEngine = py_semtools.cli_manager:stEngine
 	stEngine_report = py_semtools.cli_manager:stEngine_report

{py_semtools-1.3.3 → py_semtools-1.3.4}/src/py_semtools/cli_manager.py

@@ -1,4 +1,4 @@
-import argparse, re, sys, inspect
+import argparse, re, sys, inspect, codecs
 from py_semtools.main_modules import *
 from collections import defaultdict
 
@@ -42,6 +42,9 @@ def split_keyword_and_fields(string):
 def split_column_numbers(string):
     return [int(number) for number in string.split(",")]
 
+def unescaped_str(arg_str):
+    return codecs.decode(str(arg_str), 'unicode_escape')
+
 #########################################################################
 #CLI PARSERS 
 #########################################################################
@@ -187,7 +190,10 @@ def semtools(args = None):
               help="With flags 'get_LCA_from_profile' and 'get_MICA_from_profile' constraint the ancestors to the given list of terms (one per line).") 
     parser.add_argument("--get_representative_profile", dest="get_representative_profile", default=None, type=float,
               help="For a list of profiles, give one single profile with terms presented with al least FLOAT frecuency in the profile list. Parents are infered and computed too. The most spefic terms are retained.") 
-
+    parser.add_argument("--set_a", dest="set_a", default=None, type=text_list, help="Set of terms A to be used in set operations. It must be a file with one term per line.")
+    parser.add_argument("--set_b", dest="set_b", default=None, type=text_list, help="Set of terms B to be used in set operations. It must be a file with one term per line.")
+    parser.add_argument("--set_operation", dest="set_operation", default=None, type=str, help="Set operation to perform between set A and B. Options: 'is_parental', 'is_descendant'")
+    parser.add_argument("--set_operation_output", dest="set_operation_output", default=None, type=str, help="Path to save the output of the set operation between set A and B.")
     opts =  parser.parse_args(args)
     
     main_semtools(opts)
@@ -229,11 +235,47 @@ def get_sorted_suggestions(args = None):
             help="Path to the output report file. If not used, no report will be generated")
     parser.add_argument("--heatmap_color_preset", dest="heatmap_color_preset", default= "1",
             help="Use to set the color preset for the heatmap. Options: '1' (default) or '2'. Each preset has a different color scheme.")
+    parser.add_argument("--color_heatmap_by", dest="color_heatmap_by", default="IC",
+            help="Use to set the values to color the heatmap. Options: 'IC' (information content of the target term) or "
+            "'rel_value' (the value of the relation between query and target term)")
 
     opts =  parser.parse_args(args)
     main_get_sorted_suggestions(opts)
 
 
+def llmEngine(args = None):
+    parser = argparse.ArgumentParser(description='Perform Ontology driven analysis with LLMs')
+    parser.add_argument('-P', "--model_provider", dest="model_provider", default= None,
+            help="Provider of the LLM model to be used")
+    parser.add_argument('-m', "--model_name", dest="model_name", default= None,
+            help="Model to be used")
+    parser.add_argument('-A', "--listening_addresses", dest="listening_addresses", default= None,
+            help="Addresses to listen for incoming requests in the format 'host:port'. For example: 'http://localhost:11434'")
+    parser.add_argument("-t", "--template", dest="template", default= None,
+            help="For ontoGPT, it is the template to be used for the prompt.")
+    parser.add_argument("-T", "--regex_tag", dest="regex_tag", default= None,
+            help="Regex tag for filtering grounded terms (like HP:), etc")
+    parser.add_argument("--logging_level", dest="logging_level", default=None,
+            help="Logging level for the process (if the model support it). Options: 'DEBUG', 'INFO', 'WARNING' or 'ERROR'. By default, logging is not activated.")
+    parser.add_argument('-q', "--query", dest="query", default= None,
+            help="Path to the query file. Wildcards are accepted to process multiple files")    
+    parser.add_argument('-c', "--corpus", dest="corpus", default= None,
+            help="Path to the corpus file. Wildcards are accepted to process multiple files")    
+    parser.add_argument('-o', "--output_file", dest="output_file", default= None,
+            help="Path to save the output file with semantic scores")
+    parser.add_argument('-s', "--split_level", dest="split_level", default= "sentence",
+            help="Use it if your corpus comes splitted in smaller parts and want to unify it at document level. Options: 'doc', 'paragraph', or 'sentence'")        
+    parser.add_argument('-v', "--verbose", dest="verbose", default= False, action='store_true',
+            help="Toogle on to get verbose output")
+    parser.add_argument("--print_relevant_pairs", dest="print_relevant_pairs", default= False, action='store_true',
+            help="Use it to print the relevant pairs of query-corpus with their scores")
+    parser.add_argument("--get_total_time_file", dest="get_total_time_file", default= None,
+            help="Use it to save the calculation time of the process to a file") 
+    parser.add_argument("--stream_write_results", dest="stream_write_results", default= False, action='store_true',
+            help="Use it to write results in streaming mode, which means that results will be written as they are calculated instead of waiting for the whole process to end.")   
+    opts = parser.parse_args(args)
+    main_llmEngine(opts)
+
 def fmEngine(args = None):
     if args is None:
         args = sys.argv[1:]
@@ -241,7 +283,7 @@ def fmEngine(args = None):
     parser = argparse.ArgumentParser(description='Perform Ontology driven analysis with Sentence Transformer')
 
     parser.add_argument('-m', "--model_name", dest="model_name", default= None,
-            help="Model to be used. Current options include 'bm25' and 'tfidf'")
+            help="Model to be used. Current options include 'bm25', 'tfidf', 'tfidf_faiss'")
     parser.add_argument('-q', "--query", dest="query", default= None,
             help="Path to the query file. Wildcards are accepted to process multiple files")
     parser.add_argument('-c', "--corpus", dest="corpus", default= None,
@@ -256,10 +298,15 @@ def fmEngine(args = None):
             help="Toogle on to get verbose output")
     parser.add_argument("--order", dest="order", default= "corpus-query",
             help="Order of the semantic search. Options: 'corpus-query' or 'query-corpus'")    
-    parser.add_argument('-s', "--split", dest="split", default= False, action='store_true',
-            help="Use it if your corpus comes splitted in smaller parts as list of lists (embedded in a json)")
+    parser.add_argument('-s', "--split_level", dest="split_level", default= "sentence",
+            help="Use it if your corpus comes splitted in smaller parts and want to unify it at document level. Options: 'doc', 'paragraph', or 'sentence'")
     parser.add_argument("--print_relevant_pairs", dest="print_relevant_pairs", default= False, action='store_true',
-            help="Use it to print the relevant pairs of query-corpus with their scores")        
+            help="Use it to print the relevant pairs of query-corpus with their scores")
+    parser.add_argument("--get_total_time_file", dest="get_total_time_file", default= None,
+            help="Use it to save the calculation time of the process to a file")    
+    parser.add_argument("--write_sims_mode", dest="write_sims_mode", default= "w",
+            help="Use it to set the mode for writing similarities to the output file. Options: 'w' for write (overwrite) and 'a' for append.") 
+
     opts =  parser.parse_args(args)
     main_fmEngine(opts)
 
@@ -292,8 +339,8 @@ def stEngine(args = None):
             help="Similarity threshold to filter results to write")    
     parser.add_argument('-v', "--verbose", dest="verbose", default= False, action='store_true',
             help="Toogle on to get verbose output")    
-    parser.add_argument('-s', "--split", dest="split", default= False, action='store_true',
-            help="Use it if your corpus comes splitted in smaller parts as list of lists (embedded in a json)")
+    parser.add_argument('-s', "--split_level", dest="split_level", default= "sentence",
+            help="Use it if your corpus comes splitted in smaller parts and want to unify it at document level. Options: 'doc', 'paragraph', or 'sentence'")
     parser.add_argument("--order", dest="order", default= "corpus-query",
             help="Order of the semantic search. Options: 'corpus-query' or 'query-corpus'")    
     parser.add_argument("--print_relevant_pairs", dest="print_relevant_pairs", default= False, action='store_true',
@@ -309,7 +356,12 @@ def stEngine(args = None):
     parser.add_argument("--single_worker_chunk_size", dest="single_worker_chunk_size", default=10000, type=int,
             help="(Only for multi GPU processing) Chunk size for each worker when multi-GPU processing is used.")
     parser.add_argument("-b", "--batch_size", dest="batch_size", default= 32, type=int,
-            help="Use to specify batch size for multi-GPU embedding step")             
+            help="Use to specify batch size for multi-GPU embedding step")
+    parser.add_argument("--get_total_time_file", dest="get_total_time_file", default= None,
+            help="Use it to save the calculation time of the process to a file")
+    parser.add_argument("--write_sims_mode", dest="write_sims_mode", default= "w",
+            help="Use it to set the mode for writing similarities to the output file. Options: 'w' for write (overwrite) and 'a' for append.") 
+                    
     opts =  parser.parse_args(args)
     main_stEngine(opts)
 
@@ -386,7 +438,7 @@ def get_corpus_index(args = None):
             help="When using 'filter_by_blacklist' use this option to choose between 'exact' or 'partial' match. Default is 'partial'")
     parser.add_argument("--clean_type", dest="clean_type", default= 'hard',
             help="Use to set the type of cleaning to be performed on the text. Options: basic (do not clean at all), soft or hard (default).")
-    parser.add_argument("--split_type", dest="split_type", default= 'classic',
+    parser.add_argument("--split_type", dest="split_type", default= 'classic', type=unescaped_str,
             help="Use to set the type of splitting to be performed on the text. Options: classic (default, splitting by '\n\n', \n', '.', ';', ',') or space_overlap ('\n\n', '\n', ' ', '' with overlap to avoid losing information).")
     parser.add_argument("--extra_fields", dest="extra_fields", default= defaultdict(lambda: False), type=text_to_default_dict,
             help="Comma-separated list of extra fields to include in the processed corpus. Available: title and/or keywords. Default: none")
@@ -431,6 +483,10 @@ def get_sorted_profs(args=None):
                         help="When reference profile is not given, a general ine is computed with all profiles. If a freq is defined (0-1), all terms with freq minor than limit are removed")
     parser.add_argument("-L", "--matrix_limits", dest="matrix_limits", default= [20, 40], type= lambda data: [int(i) for i in data.split(",")],
                         help="Number of rows and columns to show in heatmap defined as 'Nrows,Ncols'. Default 20,40")
+    parser.add_argument("--sim_bidirectional", dest="sim_bidirectional", default= False, action="store_true",
+                        help="Use to activate similarity average calculation onto term profiles when both directios are computed (A to B, and B to A profiles)")                                        
+    parser.add_argument("--sim_direction", dest="sim_direction", default= None,
+                        help="If not defined or set to 'ext-int' are external profiles against reference profile. If set 'int-ext' is the ref profile against the external profiles")
     parser.add_argument("--header_id", dest="header_id", default="HP", help="Header ID to use in plotted heatmaps")
     opts =  parser.parse_args(args)
     main_get_sorted_profs(opts)

{py_semtools-1.3.3 → py_semtools-1.3.4}/src/py_semtools/indexers/text_indexer.py

@@ -105,7 +105,7 @@ class TextIndexer:
         for parsed_text in parsed_texts:
             doc_id, file, text = parsed_text
             if doc_id == None or text == "": continue
-            pmid_content_and_stats = cls.prepare_indexes(text, doc_id, file, "None", "None", "None", "None", options)
+            pmid_content_and_stats = cls.prepare_indexes(text, doc_id, file, "None", "None", "None", "None", "None", options)
             texts.append(pmid_content_and_stats)
         if logger != None: logger.warning(f"stats:file={file_path},total={stats['total']}")
         if logger != None: logger.warning(f"logs_errors:file={file_path},errors_number={stats['errors']}")
@@ -209,6 +209,8 @@ class TextIndexer:
             sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 100, chunk_overlap  = 20, length_function = len, separators=["\n", " ", ""], keep_separator=False, is_separator_regex=False)
             #sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 20, chunk_overlap  = 2, length_function = cls._len_by_words_func, separators=["\n", " ", ""], keep_separator=False, is_separator_regex=False)
         #sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 120, chunk_overlap  = 20, length_function = len, separators=["\n", " ", ""], keep_separator=False, is_separator_regex=False)
+        else:
+            sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 10, chunk_overlap  = 0, length_function = len, separators=split_type.split("%-%"), keep_separator=False, is_separator_regex=False)
         paragraphs = [paragraph.strip() for paragraph in paragraph_splitter.split_text(text)]
         paragraph_sentences = [ sentences_splitter.split_text(paragraph) for paragraph in paragraphs ]
         

{py_semtools-1.3.3 → py_semtools-1.3.4}/src/py_semtools/lexical_engines/engine_baseclass.py

@@ -1,12 +1,14 @@
 #JPG: THIS IS AN ABSTRACT BASE CLASS FOR LEXICAL ENGINES AND SHOULD NOT BE INSTANTIATED DIRECTLY
 
-import os, warnings, json, gzip
+import os, warnings, json, gzip, time, sys
 from abc import ABC, abstractmethod
 
 class LexicalEngineBaseClass(ABC):
 
     def __init__(self):
+        self.model_name = None
         self.embedder = None
+        self.reranker = None
         self.queries_content = {}
 
     @abstractmethod
@@ -68,7 +70,7 @@ class LexicalEngineBaseClass(ABC):
         return queries, query_ids
 
     
-    def get_splitted_document(self, id, text):
+    def get_splitted_document(self, id, text, split_level = "sentence"):
         """
         Process a document that has been split by get_corpus_index. It is a json formmated list of lists, where each sublist is a paragraph and each element inside the sublist is a sentence.
         Each sentence is stored in a dictionary with the key being the ID of the document in the format "id_paragraphNumber_sentenceNumber" and the value being the sentence text.
@@ -78,29 +80,42 @@ class LexicalEngineBaseClass(ABC):
         Returns:
             dict: A dictionary where keys are IDs in the format "id_paragraph_sentence" and values are the corresponding sentences.
         """
+        from py_exp_calc.exp_calc import flatten
         pubmed_index = {}
         abstract_parts = json.loads(text)
         paragraph_number = 0
-        for paragraph in abstract_parts:
-            if paragraph[0] == "TITLE": paragraph_number = -2
-            if paragraph[0] == "KEYWORDS": paragraph_number = -1
-            sentence_number = 0
-            for sentence in paragraph:
-                if sentence in ["TITLE", "KEYWORDS", "None"]: 
-                    sentence_number += 1
-                    continue
-                id_tag = f"{id}_{paragraph_number}_{sentence_number}"
-                pubmed_index[id_tag] = sentence
-                sentence_number += 1
-            paragraph_number += 1
+        sentence_number = 0
+
+        if split_level == "doc":
+            joined_text = self._join_text_back(abstract_parts, join_back_to = "doc")
+            pubmed_index[f"{id}_{paragraph_number}_{sentence_number}"] = joined_text
+        else:
+            for paragraph in abstract_parts:
+                if paragraph[0] == "TITLE": paragraph_number = -2
+                if paragraph[0] == "KEYWORDS": paragraph_number = -1
+                sentence_number = 0
+
+                if split_level == "paragraph":
+                    joined_paragraph = self._join_text_back(paragraph, join_back_to = "paragraph")
+                    pubmed_index[f"{id}_{paragraph_number}_{sentence_number}"] = joined_paragraph
+                    paragraph_number += 1
+                else: #case for split_level == "sentence"
+                    for sentence in paragraph:
+                        if sentence in ["TITLE", "KEYWORDS", "None"]: 
+                            sentence_number += 1
+                            continue
+                        id_tag = f"{id}_{paragraph_number}_{sentence_number}"
+                        pubmed_index[id_tag] = sentence
+                        sentence_number += 1
+                paragraph_number += 1
         return pubmed_index
 
-    def load_pubmed_index(self, file, is_splitted):
+    def load_pubmed_index(self, file, split_level = "sentence"):
         """
         Load a PubMed processed index file get by 'get_corpus_index' binary. The file is expected to be in a specific format, where each line contains an ID and the corresponding text (and other columns with possible metadata not loaded).
         Args:
             file (str): Path to the file containing the PubMed index.
-            is_splitted (bool): If True, the text is expected to be in a JSON format with sentences split into paragraphs.
+            split_level (str): The level of splitting to use. Can be "doc", "paragraph", or "sentence".
         Returns:
             tuple: A tuple containing a dictionary where keys are IDs in the format "id_paragraphNumber_sentenceNumber" and values are the corresponding sentences, and the total number of papers processed.
         """
@@ -110,7 +125,7 @@ class LexicalEngineBaseClass(ABC):
             for line in f:
                 try:
                     id, text, *_rest = line.rstrip().split("\t")
-                    pubmed_index_iter = self.get_splitted_document(id, text)
+                    pubmed_index_iter = self.get_splitted_document(id, text, split_level = split_level)
                     pubmed_index.update(pubmed_index_iter)
                     n_papers += 1
                 except Exception as e:
@@ -120,17 +135,22 @@ class LexicalEngineBaseClass(ABC):
     def calculate_similarities(self, options, corpus_info):
         if options.get("output_file"):
           for query_basename, query_info in self.queries_content.items():
+            self.current_query_basename = query_basename
+            self.start_time = time.time()
             best_matches = self.calculate_similarity(query_info, corpus_info, options)
+            end = time.time()
+            if options['get_total_time_file']:
+                with open(options['get_total_time_file'], 'a') as f: f.write(f"TIMELOG:{self.model_name}:{self.reranker}:Total time calculating similarities for {query_basename} query ({len(query_info.get('query_ids', []))} names and synonims) and {len(corpus_info['all_corpus'])} sentences: {end - self.start_time}\n")
             if options['print_relevant_pairs']: self.print_similarities(query_info, corpus_info, best_matches, options)
             output_filename = os.path.join(options["output_file"],query_basename)
             self.save_similarities(output_filename, best_matches, options)
 
     def save_similarities(self, filepath, best_matches, options):
         #with gzip.open(filepath, "a") as f: #TODO: add it later
-        with open(filepath, 'a') as f:
+        with open(filepath, options["write_sims_mode"]) as f:
           for kwdID, matches in best_matches.items():
             for textID, score in matches.items():
-              if score >= options["threshold"]: 
+              if score == "-" or score >= options["threshold"]: #"-" means full match for ontogpt
                 if options["order"] == "corpus-query":
                   f.write(f"{textID}\t{kwdID}\t{score}\n")
                 else:
@@ -142,7 +162,7 @@ class LexicalEngineBaseClass(ABC):
             textIDX = corpus_info["textIDs"].index(textID)
             text = corpus_info["all_corpus"][textIDX]
             for kwdID, score in matches.items():
-                if score >= options["threshold"]: 
+                if score == "-" or score >= options["threshold"]: 
                     kwIDXs = (idx for idx, char in enumerate(query_info['query_ids']) if char == kwdID) 
                     term = " -- ".join([query_info["queries"][kwIDX] for kwIDX in kwIDXs])
                     if term not in term_related_sentences: term_related_sentences[term] = {"term_id": kwdID, "sentences": []}
@@ -153,4 +173,19 @@ class LexicalEngineBaseClass(ABC):
             for textID, text, score in data["sentences"]:
                 print(f"  - Text ID: {textID}, Score: {score}")
                 print(f"    Text: {text}")
-            print("-"*30)                      
+            print("-"*30)   
+
+    def _join_text_back(self, text, join_back_to = "doc"):
+        final_text = ""
+
+        if join_back_to == "doc":
+            for paragraph in text:
+                for sentence in paragraph:              
+                    final_text += sentence + ", "
+                final_text += ".\n\n"
+
+        elif join_back_to == "paragraph":
+            for sentence in text:
+                final_text += sentence + ", "
+
+        return final_text

{py_semtools-1.3.3 → py_semtools-1.3.4}/src/py_semtools/lexical_engines/fmEngine.py

@@ -3,10 +3,12 @@ from py_semtools.lexical_engines.engine_baseclass import LexicalEngineBaseClass
 class FMengine(LexicalEngineBaseClass):
     def __init__(self, gpu_devices = []):
         self.model_name = None
+        self.embedder = None
+        self.reranker = None
         self.queries_content = {}
 
     def init_model(self, model_name: str, verbose: bool = False) -> None:
-        if model_name not in ["bm25", "tfidf"]:
+        if model_name not in ["bm25", "tfidf", "tfidf_faiss"]:
             raise Exception(f"Model {model_name} not supported in FMengine")
         if verbose: print(f"\n-Loading {model_name} model")
         self.model_name = model_name
@@ -21,7 +23,7 @@ class FMengine(LexicalEngineBaseClass):
         all_textIDs = []; all_corpus = []; total_papers = 0 
         for corpus_filename in corpus_filenames:
             if verbose: print(f"---Loading corpus of {corpus_filename}")
-            pubmed_index, n_papers = self.load_pubmed_index(corpus_filename, options["split"]) # abstracts
+            pubmed_index, n_papers = self.load_pubmed_index(corpus_filename, split_level=options["split_level"]) # abstracts
             if verbose: print(f"------Loaded {n_papers} documents with {len(pubmed_index)} sentences from {corpus_filename}")
             total_papers += n_papers
             all_textIDs.extend(pubmed_index.keys())
@@ -45,6 +47,9 @@ class FMengine(LexicalEngineBaseClass):
             results_indexes, scores = self.calculate_bm25_similarity(query_text, corpus_text, options)
         elif self.model_name == "tfidf":
             results_indexes, scores = self.calculate_tfidf_similarity(query_text, corpus_text, options)
+        elif self.model_name == "tfidf_faiss":
+            results_indexes, scores = self.calculate_tfidf_similarity_faiss(query_text, corpus_text, options)
+
         matches = self.find_best_matches(results_indexes, scores, query_ids, corpus_ids)
         return matches
     
@@ -88,6 +93,31 @@ class FMengine(LexicalEngineBaseClass):
             
         return batch_indices, batch_scores
 
+    def calculate_tfidf_similarity_faiss(self, query_text, corpus_text, options):
+        from sklearn.feature_extraction.text import TfidfVectorizer
+        import faiss
+        # 1. Vectorization (remains the same)
+        vectorizer = TfidfVectorizer().fit(corpus_text)
+        tfidf_matrix = vectorizer.transform(corpus_text).toarray().astype('float32')
+        query_vecs = vectorizer.transform(query_text).toarray().astype('float32')
+
+        # 2. Normalize vectors for Cosine Similarity
+        faiss.normalize_L2(tfidf_matrix)
+        faiss.normalize_L2(query_vecs)
+
+        # 3. Initialize FAISS Index (Inner Product)
+        d = tfidf_matrix.shape[1]  # dimensionality
+        index = faiss.IndexFlatIP(d)
+        
+        # 4. Add corpus to index (Fast process)
+        index.add(tfidf_matrix)
+
+        # 5. Search (This handles batching and parallelization internally)
+        k = min(options['top_k'], tfidf_matrix.shape[0])
+        batch_scores, batch_indices = index.search(query_vecs, k)
+
+        return batch_indices, batch_scores
+
     def find_best_matches(self, results_indexes, scores, query_ids, corpus_ids):
         matches = {}
         for row_idx, res_idxs  in enumerate(results_indexes):

py_semtools-1.3.4/src/py_semtools/lexical_engines/llmEngine.py

@@ -0,0 +1,102 @@
+import os, re, logging, requests, warnings
+from py_semtools.lexical_engines.engine_baseclass import LexicalEngineBaseClass
+from ontogpt.io.template_loader import get_template_details, get_template_path
+from ontogpt.engines.spires_engine import SPIRESEngine
+
+class LLMengine(LexicalEngineBaseClass):
+    def __init__(self, gpu_devices = []):
+        self.model_name = None
+        self.embedder = None
+        self.reranker = None
+        self.queries_content = {}
+
+    def init_model(self, model_name: str, model_provider: str = None, verbose: bool = False) -> None:
+        if verbose: print(f"\n-Using {model_name} model with {model_provider} provider")
+        self.model_name = model_name
+        self.model_provider = model_provider
+
+    def load_several_queries(self, options, query_filenames, verbose=False):
+        if verbose: print("\n-Loading and embedding queries:")
+        for query_filename in query_filenames:
+            query_basename = os.path.splitext(os.path.basename(query_filename))[0]
+            self.queries_content[query_basename] = {'template': options["template"]}
+            if options["verbose"]: print(f"---Using query template {options['template']} for {query_basename}")
+
+    def load_several_corpora(self, options, corpus_filenames, verbose=False):
+        all_textIDs = []; all_corpus = []; total_papers = 0 
+        for corpus_filename in corpus_filenames:
+            if verbose: print(f"---Loading corpus of {corpus_filename}")
+            pubmed_index, n_papers = self.load_pubmed_index(corpus_filename, split_level=options["split_level"]) # abstracts
+            if verbose: print(f"------Loaded {n_papers} documents with {len(pubmed_index)} sentences from {corpus_filename}")
+            total_papers += n_papers
+            all_textIDs.extend(pubmed_index.keys())
+            all_corpus.extend(pubmed_index.values())
+        corpus_info = {'textIDs': all_textIDs, "all_corpus": all_corpus}
+        if verbose: print(f"---Loaded a total of {total_papers} documents with {len(all_corpus)} sentences from {len(corpus_filenames)} corpora")
+        return corpus_info
+
+    def calculate_similarity(self, query_info, corpus_info, options):
+        kwargs = {'recurse': True, 'auto_prefix': 'AUTO'}
+        corpus_texts = corpus_info["all_corpus"]
+        corpus_ids = corpus_info["textIDs"]
+        template = options["template"]
+        model = options["model_name"]
+        address = options["listening_addresses"]
+        
+        if options["logging_level"]:
+            loggin_levels = {"DEBUG": logging.DEBUG, "INFO":logging.INFO, "WARNING":logging.WARNING, "ERROR":logging.ERROR}
+            logger = logging.getLogger()
+            logger.setLevel(level=loggin_levels[options["logging_level"]])
+
+        if options['verbose']: print(f"\n-Calculating similarities between queries and corpus using {model} model with {template} template:")
+
+        if template:
+            template_details = get_template_details(template=template)
+        else:
+            raise ValueError("No template specified. Use -t/--template option.")
+
+        ke = SPIRESEngine(template_details=template_details, model=model, temperature=1,
+                          api_base=address, api_version=None, model_provider=None, system_message=None, max_text_length=None,
+                          **kwargs)
+
+        best_matches = {}
+        for idx, text in enumerate(corpus_texts):
+            if options['verbose']: print(f"Extracting for text ID {corpus_ids[idx]}")  # Print the beginning of the text for context
+
+            #results = ke.extract_from_text( text=text, cls=None, show_prompt=None)
+            results = self._execute_with_retry( func=ke.extract_from_text, 
+                exception_type=requests.exceptions.ReadTimeout, max_retries=3, text_id=corpus_ids[idx], 
+                text=text, cls=None, show_prompt=None)
+            
+            if results != None:
+                hpos = self._get_genuine_terms(results.named_entities, options["regex_tag"])
+                best_matches[corpus_ids[idx]] = hpos
+
+                if options['stream_write_results']:
+                    backup_file = os.path.join(options["output_file"], self.current_query_basename + "_backup")
+                    warnings.warn(f"Streaming mode enabled: writing {len(hpos)}terms for text ID {corpus_ids[idx]} immediately after processing (inside file {backup_file} ).")
+                    self.save_similarities(backup_file, {corpus_ids[idx]: hpos}, options) # Write results in streaming mode if enabled
+        
+        return best_matches
+    
+    def find_best_matches(self, results_indexes, scores, query_ids, corpus_ids):
+        pass
+
+
+    def _get_genuine_terms(self, named_entities, regex_tag):
+        if regex_tag:
+            return {entity.id : "-" for entity in named_entities if re.match(regex_tag, entity.id) and len(entity.original_spans) > 0}
+        else:
+            return {entity.id : "-" for entity in named_entities if len(entity.original_spans) > 0}
+        
+    def _execute_with_retry(self, func, exception_type, max_retries, text_id=None, *args, **kwargs):
+        tries = 0
+        while True:
+            try:
+                return func(*args, **kwargs)                
+            except exception_type as e:
+                tries += 1
+                warnings.warn(f"{exception_type.__name__} occurred for text ID {text_id}. Retrying... Error: {e}")
+                if max_retries > 0 and tries >= max_retries:
+                    warnings.warn(f"Failed to process text ID {text_id} after {max_retries} attempts. Skipping this text.")
+                    return None  # Salimos devolviendo None tras agotar intentos

{py_semtools-1.3.3 → py_semtools-1.3.4}/src/py_semtools/lexical_engines/stEngine.py

@@ -62,7 +62,8 @@ class STengine(LexicalEngineBaseClass):
         full_model_path = os.path.join(cache_folder, model_name)
         if verbose: print(f"\n-Downloading or loading bi encoder model {model_name} inside path {full_model_path}")
         self.model_name = model_name
-        has_cached_model = os.path.exists(os.path.join(full_model_path, f'models--sentence-transformers--{model_name}')) #SentenceTransfomers now uses local_files_only to control internet access, even if the model is cached, and trying to download the model with this variable false gives error, so we have to control it
+        has_cached_model = os.path.exists(os.path.join(full_model_path, f'models--{model_name.replace("/", "--")}')) #SentenceTransfomers now uses local_files_only to control internet access, even if the model is cached, and trying to download the model with this variable false gives error, so we have to control it
+        if verbose: print(f"Seeking cached model (found:{has_cached_model}) in", os.path.join(full_model_path, f'models--{model_name.replace("/", "--")}'))
         self.embedder = SentenceTransformer(model_name, cache_folder = full_model_path, local_files_only=has_cached_model)
 
     def init_rerank_model(self, model_name, cache_folder = None, verbose = False):
@@ -70,8 +71,9 @@ class STengine(LexicalEngineBaseClass):
         full_model_path = os.path.join(cache_folder, model_name)
         if verbose: print(f"\n-Downloading or loading cross-encoder model {model_name} inside path {full_model_path}")
         self.reranker_model_name = model_name
-        has_cached_model = os.path.exists(os.path.join(full_model_path, f'models--cross-encoder--{model_name}')) #SentenceTransfomers now uses local_files_only to control internet access, even if the model is cached, and trying to download the model with this variable false gives error, so we have to control it
-        self.reranker = CrossEncoder(model_name, cache_folder = full_model_path, local_files_only=has_cached_model)
+        has_cached_model = os.path.exists(os.path.join(full_model_path, f'models--{model_name.replace("/", "--")}')) #SentenceTransfomers now uses local_files_only to control internet access, even if the model is cached, and trying to download the model with this variable false gives error, so we have to control it
+        if verbose: print(f"Seeking cached model (found:{has_cached_model}) in ", os.path.join(full_model_path, f'models--{model_name.replace("/", "--")}'))
+        self.reranker = CrossEncoder(model_name_or_path = model_name, cache_folder = full_model_path, local_files_only=has_cached_model)
      
     def embed_save_corpus(self, options, corpus_basename, all_textIDs, all_corpus, total_papers):
         if options["verbose"]: print(f"---Embedding corpus of {corpus_basename} comprised by {total_papers} initial papers with {len(all_textIDs)} sentences, with {'GPU' if options.get('gpu_device') else 'CPU'}")
@@ -198,7 +200,6 @@ class STengine(LexicalEngineBaseClass):
             results[query_idx].append({'score': score, 'corpus_id': corpus_idx})
         return results
 
-
     def make_rerank(self, sentence_pairs, options):
         import torch
         if options['verbose']: print(f"---Reranking {len(sentence_pairs)} sentence pairs with cross-encoder model")
@@ -208,7 +209,10 @@ class STengine(LexicalEngineBaseClass):
                 scores = self.reranker.predict(sentence_pairs, pool=pool, batch_size=options["batch_size"], chunk_size=options['single_worker_chunk_size'], activation_fn=torch.nn.Sigmoid())
                 self.reranker.stop_multi_process_pool(pool)
         elif len(options["gpu_device"]) == 1:
-                scores = self.reranker.encode(sentence_pairs, device= options["gpu_device"][0], batch_size=options["batch_size"], activation_fn=torch.nn.Sigmoid()) #convert_to_tensor=True 
+                scores = self.reranker.predict(sentence_pairs, device= options["gpu_device"][0], batch_size=options["batch_size"], activation_fn=torch.nn.Sigmoid()) #convert_to_tensor=True 
+        else:
+                scores = self.reranker.predict(sentence_pairs, batch_size=options["batch_size"], activation_fn=torch.nn.Sigmoid()) #convert_to_tensor=True 
+                #raise Exception("You need to provide GPU to use reranker. It should be a list of GPU devices like: ['cuda:0'] or ['cuda:0', 'cuda:1']")                
         if options["verbose"]: print(f"---Reranking time with {0 if options.get('gpu_device') == None else len(options['gpu_device'])} GPUs: {time.time() - start} seconds")
         return scores
 
@@ -271,7 +275,7 @@ class STengine(LexicalEngineBaseClass):
           
           if options.get("corpus") != None: #LOAD RAW CORPUS AND EMBEDD (AND MAYBE SAVE)
             if verbose: print(f"---Loading corpus of {corpus_filename}")
-            pubmed_index, n_papers = self.load_pubmed_index(corpus_filename, options["split"]) # abstracts
+            pubmed_index, n_papers = self.load_pubmed_index(corpus_filename, split_level=options["split_level"]) # abstracts
             total_papers += n_papers
             all_textIDs.extend(pubmed_index.keys())
             all_corpus.extend(pubmed_index.values())