py-semtools 1.3__tar.gz → 1.3.3__tar.gz

{py_semtools-1.3 → py_semtools-1.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py_semtools
-Version: 1.3
+Version: 1.3.3
 Summary: Library to handle ontologies that allows queries and calculations such as information coefficients, semantic similarity, ontology representations, etc in a easy way. It can load any ontology that complies the obo format supported by OBO Foundry.
 Home-page: https://github.com/seoanezonjic/py_semtools
 Author: seoanezonjic
@@ -18,8 +18,6 @@ Requires-Dist: numpy
 Requires-Dist: networkx
 Requires-Dist: entrezpy
 Requires-Dist: scipy
-Requires-Dist: sentence_transformers
-Requires-Dist: langchain
 Requires-Dist: pynvml
 Requires-Dist: requests
 Requires-Dist: py_exp_calc
@@ -27,6 +25,11 @@ Requires-Dist: py_cmdtabs
 Requires-Dist: py_report_html
 Requires-Dist: loguru
 Requires-Dist: loky
+Requires-Dist: bm25s
+Provides-Extra: models
+Requires-Dist: sentence_transformers; extra == "models"
+Requires-Dist: langchain; extra == "models"
+Requires-Dist: langchain-text-splitters; extra == "models"
 Provides-Extra: testing
 Requires-Dist: setuptools; extra == "testing"
 Requires-Dist: pytest; extra == "testing"

py_semtools-1.3.3/README.md

@@ -0,0 +1,6 @@
+If you want to use lexical models capabilities (stEngine and fmEngine binaries), you should install extra required dependencies that are not installed
+with the base installation. For that end, you can install semtools like the following:
+
+```bash
+pip install py_semtools[MODELS]
+``` 

{py_semtools-1.3 → py_semtools-1.3.3}/setup.cfg

@@ -28,8 +28,6 @@ install_requires =
 	networkx
 	entrezpy
 	scipy
-	sentence_transformers
-	langchain
 	pynvml
 	requests
 	py_exp_calc
@@ -37,6 +35,7 @@ install_requires =
 	py_report_html
 	loguru
 	loky
+	bm25s
 
 [options.packages.find]
 where = src
@@ -50,6 +49,7 @@ py_semtools.templates =
 	*.txt
 
 [options.extras_require]
+MODELS = sentence_transformers; langchain; langchain-text-splitters
 testing = 
 	setuptools
 	pytest
@@ -62,11 +62,13 @@ console_scripts =
 	get_sorted_suggestions = py_semtools.cli_manager:get_sorted_suggestions
 	get_sorted_profs = py_semtools.cli_manager:get_sorted_profs
 	remote_retriever = py_semtools.cli_manager:remote_retriever
+	fmEngine = py_semtools.cli_manager:fmEngine
 	stEngine = py_semtools.cli_manager:stEngine
 	stEngine_report = py_semtools.cli_manager:stEngine_report
 	get_corpus_index = py_semtools.cli_manager:get_corpus_index
 
 [tool:pytest]
+python_files = tests.py test_*.py *_tests.py
 addopts = 
 	--cov py_semtools --cov-report term-missing
 	--verbose
@@ -74,6 +76,13 @@ norecursedirs =
 	dist
 	build
 	.tox
+	.git
+	docs
+	src/py_semtools/templates
+	tests/cli_examples
+	tests/data
+	tests/demo_examples
+	tests/__pycache__
 testpaths = tests
 
 [devpi:upload]

py_semtools-1.3.3/src/py_semtools/__init__.py

@@ -0,0 +1,16 @@
+import sys
+
+if sys.version_info[:2] >= (3, 8):
+    # TODO: Import directly (no need for conditional) when `python_requires = >= 3.8`
+    from importlib.metadata import PackageNotFoundError, version  # pragma: no cover
+else:
+    from importlib_metadata import PackageNotFoundError, version  # pragma: no cover
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = __name__
+    __version__ = version(dist_name)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "unknown"
+finally:
+    del version, PackageNotFoundError

{py_semtools-1.3 → py_semtools-1.3.3}/src/py_semtools/cli_manager.py

@@ -1,5 +1,7 @@
 import argparse, re, sys, inspect
 from py_semtools.main_modules import *
+from collections import defaultdict
+
 
 ###########################################################################
 ## TYPES
@@ -9,6 +11,12 @@ def one_column_file(file): return [line.strip() for line in open(file).readlines
 
 def text_list(string): return string.split(',')
 
+def text_to_default_dict(string):
+    d = defaultdict(lambda: False)
+    for item in text_list(string):
+        d[item] = True
+    return d
+
 def filter_regex(string):
     filters = []
     pattern = re.compile(r"([pn])\(([A-Za-z:0-9,]*)\)")
@@ -73,8 +81,8 @@ def remote_retriever(args = None):
     main_remote_retriever(opts)
  
 def semtools(args = None):
-    if args is None:
-        args = sys.argv[1:]
+    
+    if args is None: args = sys.argv[1:]
 
     parser = argparse.ArgumentParser(description='Perform Ontology driven analysis ')
     parser.add_argument("--return_all_terms_with_user_defined_attributes", dest="return_all_terms_with_user_defined_attributes", default=None,
@@ -98,10 +106,10 @@ def semtools(args = None):
     parser.add_argument("-t", "--translate", dest="translate", default= None,
               help="Translate to 'names' or to 'codes'")
     parser.add_argument("-s", "--similarity_method", dest="similarity", default= None, 
-              help="Calculate similarity between profile IDs computed by 'resnik', 'lin' or 'jiang_conrath' methods.")
+              help="Calculate similarity between profile IDs computed by 'resnik', 'lin' or 'jiang_conrath' methods. Recently added 'eric', 'neric', 'nweric' and 'erlin' methods.")
     parser.add_argument("--reference_profiles", dest="reference_profiles", default= None, 
               help="Path to file tabulated file with first column as id profile and second column with ontology terms separated by separator.")
-    parser.add_argument('-c', "--clean_profiles", dest="clean_profiles", default= False, action='store_true', 
+    parser.add_argument("--disable_cleaning_profiles", dest="clean_profiles", default= True, action='store_false', 
               help="Removes ancestors, descendants and obsolete terms from profiles.")
     parser.add_argument('-r', "--removed_path", dest="removed_path", default= 'rejected_profs', 
               help="Desired path to write removed profiles file.")
@@ -157,6 +165,8 @@ def semtools(args = None):
               help="For the output report, it sets the root term to show in center of ontoplot")
     parser.add_argument("--ref_term", dest="ref_term", default=None,
               help="For the output report, it sets the term whose childs will be in the color legend to indicate the branches of descendants")
+    parser.add_argument("--similarity_cluster", dest="similarity_cluster", default=None,
+              help="Use to activate profiles clustering with a similarity method and generate basefiles (resnik', 'lin' or 'jiang_conrath). Not active by default")
     parser.add_argument("--similarity_cluster_plot", dest="similarity_cluster_plot", default=None,
               help="For output report, use to activate profiles clustering and clustermap plot with a similarity method (resnik', 'lin' or 'jiang_conrath). Not active by default")
     parser.add_argument("--cl_size_factor", dest="cl_size_factor", default=1.0, type=float,
@@ -167,9 +177,22 @@ def semtools(args = None):
               help="For the ontoplot, use it to deactivate propagating frequency to parentals terms")
     parser.add_argument("--onto_min_freq", dest="onto_min_freq", default=0.005, type=float,
               help="For the ontoplot, it sets the minimum frequency of terms to be shown in the plot. Default is 0.5 percent. Set to 0 to show all terms.") 
+    parser.add_argument("--similarity_index", dest="similarity_index", default=None,
+              help="Term to term similarity index to perform profile similarity calculations") 
+    parser.add_argument("--get_LCA_from_profile", dest="get_LCA_from_profile", default=None, type=float,
+              help="For a list if profiles, give for each one the list of LCA that occurs for the selected frequency in the profile terms.") 
+    parser.add_argument("--get_MICA_from_profile", dest="get_MICA_from_profile", default=None, type=float,
+              help="For a list of profiles, give for each one the list of LCA that occurs for the selected frequency in the profile terms.") 
+    parser.add_argument("--LCA_list", dest="LCA_list", default=None, type=str,
+              help="With flags 'get_LCA_from_profile' and 'get_MICA_from_profile' constraint the ancestors to the given list of terms (one per line).") 
+    parser.add_argument("--get_representative_profile", dest="get_representative_profile", default=None, type=float,
+              help="For a list of profiles, give one single profile with terms presented with al least FLOAT frecuency in the profile list. Parents are infered and computed too. The most spefic terms are retained.") 
+
     opts =  parser.parse_args(args)
+    
     main_semtools(opts)
 
+
 def get_sorted_suggestions(args = None):
     if args is None:
         args = sys.argv[1:]
@@ -210,6 +233,37 @@ def get_sorted_suggestions(args = None):
     opts =  parser.parse_args(args)
     main_get_sorted_suggestions(opts)
 
+
+def fmEngine(args = None):
+    if args is None:
+        args = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(description='Perform Ontology driven analysis with Sentence Transformer')
+
+    parser.add_argument('-m', "--model_name", dest="model_name", default= None,
+            help="Model to be used. Current options include 'bm25' and 'tfidf'")
+    parser.add_argument('-q', "--query", dest="query", default= None,
+            help="Path to the query file. Wildcards are accepted to process multiple files")
+    parser.add_argument('-c', "--corpus", dest="corpus", default= None,
+            help="Path to the corpus file. Wildcards are accepted to process multiple files")    
+    parser.add_argument('-o', "--output_file", dest="output_file", default= None,
+            help="Path to save the output file with semantic scores")
+    parser.add_argument('-k', "--top_k", dest="top_k", default= 20, type = int,
+            help="Get top scores per keyword")
+    parser.add_argument('-t', "--threshold", dest="threshold", default= 0, type = float,
+            help="Similarity threshold to filter results to write")    
+    parser.add_argument('-v', "--verbose", dest="verbose", default= False, action='store_true',
+            help="Toogle on to get verbose output")
+    parser.add_argument("--order", dest="order", default= "corpus-query",
+            help="Order of the semantic search. Options: 'corpus-query' or 'query-corpus'")    
+    parser.add_argument('-s', "--split", dest="split", default= False, action='store_true',
+            help="Use it if your corpus comes splitted in smaller parts as list of lists (embedded in a json)")
+    parser.add_argument("--print_relevant_pairs", dest="print_relevant_pairs", default= False, action='store_true',
+            help="Use it to print the relevant pairs of query-corpus with their scores")        
+    opts =  parser.parse_args(args)
+    main_fmEngine(opts)
+
+
 def stEngine(args = None):
     if args is None:
         args = sys.argv[1:]
@@ -218,6 +272,8 @@ def stEngine(args = None):
 
     parser.add_argument('-m', "--model_name", dest="model_name", default= None,
             help="Name of the model to be used")
+    parser.add_argument('-r', "--reranker_model_name", dest="reranker_model_name", default= None,
+            help="(Optional) Name of the reranker model to be used")    
     parser.add_argument('-p', "--model_path", dest="model_path", default= None,
             help="Path where the model is cached or where it will be stored")
     parser.add_argument('-q', "--query", dest="query", default= None,
@@ -235,21 +291,25 @@ def stEngine(args = None):
     parser.add_argument('-t', "--threshold", dest="threshold", default= 0, type = float,
             help="Similarity threshold to filter results to write")    
     parser.add_argument('-v', "--verbose", dest="verbose", default= False, action='store_true',
-            help="Toogle on to get verbose output")
-    parser.add_argument('-g', "--gpu_device", dest="gpu_device", default= [], type=text_list,
-            help="Use to specify the GPU device to be used for speed-up (if available). The format is like: 'cuda:0' or 'cuda:0,cuda:1' to use multiple GPUs or cpu,cpu to use multiple CPUs")    
-    parser.add_argument("-b", "--batch_size", dest="batch_size", default= 32, type=int,
-            help="Use to specify batch size for multi-GPU embedding step") 
-    parser.add_argument("--use_gpu_for_sim_calculation", dest="use_gpu_for_sim_calculation", default= False, action='store_true',
-            help="Toogle on if you want to use GPU not only for the embedding process, but also for calculating query-corpus similarities (then dot score if used instead of cosine similarity)")
+            help="Toogle on to get verbose output")    
     parser.add_argument('-s', "--split", dest="split", default= False, action='store_true',
             help="Use it if your corpus comes splitted in smaller parts as list of lists (embedded in a json)")
     parser.add_argument("--order", dest="order", default= "corpus-query",
-            help="Order of the semantic search. Options: 'corpus-query' or 'query-corpus'")
-    parser.add_argument("--chunk_size", dest="chunk_size", default=10000, type=int,
-            help="Size to be accumulating corpora until a threshold is reached before proceeding to embedd")
+            help="Order of the semantic search. Options: 'corpus-query' or 'query-corpus'")    
     parser.add_argument("--print_relevant_pairs", dest="print_relevant_pairs", default= False, action='store_true',
-            help="Use it to print the relevant pairs of query-corpus with their scores")    
+            help="Use it to print the relevant pairs of query-corpus with their scores")
+    parser.add_argument('-g', "--gpu_device", dest="gpu_device", default= [], type=text_list,
+            help="Use to specify the GPU device to be used for speed-up (if available). The format is like: 'cuda:0' or 'cuda:0,cuda:1' to use multiple GPUs or cpu,cpu to use multiple CPUs")
+    parser.add_argument("--use_gpu_for_sim_calculation", dest="use_gpu_for_sim_calculation", default= False, action='store_true',
+            help="Toogle on if you want to use GPU not only for the embedding process, but also for calculating query-corpus similarities (then dot score if used instead of cosine similarity)")        
+    parser.add_argument("--chunk_size", dest="chunk_size", default=10000, type=int,
+            help="Size to be accumulating corpora (documents) until a threshold is reached before proceeding to embedd")
+    parser.add_argument("--chunk_size_sentences", dest="chunk_size_sentences", default=0, type=int,
+            help="(If toogled on, takes precedence over --chunk_size) Size to be accumulating sentences until a threshold is reached before proceeding to embedd")    
+    parser.add_argument("--single_worker_chunk_size", dest="single_worker_chunk_size", default=10000, type=int,
+            help="(Only for multi GPU processing) Chunk size for each worker when multi-GPU processing is used.")
+    parser.add_argument("-b", "--batch_size", dest="batch_size", default= 32, type=int,
+            help="Use to specify batch size for multi-GPU embedding step")             
     opts =  parser.parse_args(args)
     main_stEngine(opts)
 
@@ -328,6 +388,10 @@ def get_corpus_index(args = None):
             help="Use to set the type of cleaning to be performed on the text. Options: basic (do not clean at all), soft or hard (default).")
     parser.add_argument("--split_type", dest="split_type", default= 'classic',
             help="Use to set the type of splitting to be performed on the text. Options: classic (default, splitting by '\n\n', \n', '.', ';', ',') or space_overlap ('\n\n', '\n', ' ', '' with overlap to avoid losing information).")
+    parser.add_argument("--extra_fields", dest="extra_fields", default= defaultdict(lambda: False), type=text_to_default_dict,
+            help="Comma-separated list of extra fields to include in the processed corpus. Available: title and/or keywords. Default: none")
+    parser.add_argument("--remain_empty_documents", dest="remain_empty_documents", default= False, action='store_true',
+            help="Use this option to keep entries with (main text) empty documents in the processed corpus (if they have at least title or keywords)")
     opts =  parser.parse_args(args)
     main_get_corpus_index(opts)
 
@@ -358,7 +422,9 @@ def get_sorted_profs(args=None):
     parser.add_argument("-X", "--excluded_terms", dest="excluded_terms", default= [], type = one_column_file,
                         help="File with excluded terms. One term code per line.")
     parser.add_argument("--hard_check", dest="hard_check", default= True, action="store_false",
-                        help="Set to disable hard check cleaning. Default true")    
+                        help="Set to disable hard check cleaning. Default true")
+    parser.add_argument("-s", "--similarity_method", dest="similarity", default= 'lin', 
+                        help="Calculate similarity between profile IDs computed by 'resnik', 'lin' or 'jiang_conrath' methods. Recently added 'eric', 'neric', 'nweric' and 'erlin' methods.")    
     parser.add_argument("-o", "--output_file", dest="output_file", default= 'report.html',
                         help="Output report file")
     parser.add_argument("-f", "--general_prof_freq", dest="term_freq", default= 0, type= float,

{py_semtools-1.3 → py_semtools-1.3.3}/src/py_semtools/indexers/text_indexer.py

@@ -1,8 +1,9 @@
+from collections import defaultdict
 import sys, os, glob, re, warnings, json, gzip
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from py_exp_calc.exp_calc import invert_nested_hash, flatten
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from py_exp_calc.exp_calc import flatten
 
-from py_cmdtabs import CmdTabs
+from py_cmdtabs.cmdtabs import CmdTabs
 from py_semtools.parallelizer import Parallelizer
 from py_semtools.parsers.text.text_basic_parser import TextBasicParser
 from py_semtools.parsers.text.text_pubmed_abstract_parser import TextPubmedAbstractParser
@@ -31,7 +32,11 @@ class TextIndexer:
             else:
                 chunks = manager.get_chunks(filenames, workload_balance='disperse_max', workload_function= lambda filename: os.stat(filename).st_size)
                 items = [[[options, chunk, idx], {}] for idx,chunk in enumerate(chunks)]
-            manager.execute(items, cls.process_files)
+            res_status= manager.execute(items, cls.process_files)
+            if set(list(res_status)) == {True}: 
+                print("All processes finished succesfully")
+            else:
+                print("Some processes finished with errors")
 
     @classmethod
     def process_files(cls, options, filenames, sup_counter, logger = None):
@@ -54,6 +59,7 @@ class TextIndexer:
                 counter += 1    
         # For records saved in accumulated_texts that the loop has not writed
         cls.write_indexes(accumulated_texts, options["output"], options["tag"], a_suffix=sup_counter, b_suffix=counter, balancer = balancer, split_output_files = options["split_output_files"], items_per_file = options["text_balancing_size"])
+        return True
 
     @classmethod
     def write_indexes(cls, indexes, folder, name, a_suffix=None, b_suffix=None, balancer = None, split_output_files = False, items_per_file = 0):
@@ -71,13 +77,13 @@ class TextIndexer:
             f = gzip.open(out_filename, 'wt')
             item_count = 0
             while indexes:
-                pmid, text, original_filename, year, text_length, number_of_sentences, length_of_sentences, title, article_type, article_category  =  indexes.pop()
+                pmid, text, original_filename, year, text_length, number_of_sentences, length_of_sentences, title, article_type, article_category, keywords  =  indexes.pop()
                 if split_output_files and item_count >= items_per_file:
                     f.close()
                     file_count += 1
                     item_count = 0
                     f = gzip.open(os.path.join(folder, name+f"{a_suffix}{b_suffix}_{file_count}.gz" ), 'wt')
-                f.write(f"{pmid}\t{text}\t{original_filename}\t{year}\t{text_length}\t{number_of_sentences}\t{length_of_sentences}\t{title}\t{article_type}\t{article_category}\n")
+                f.write(f"{pmid}\t{text}\t{original_filename}\t{year}\t{text_length}\t{number_of_sentences}\t{length_of_sentences}\t{title}\t{article_type}\t{article_category}\t{keywords}\n")
                 item_count += 1
             f.close()
 
@@ -112,19 +118,17 @@ class TextIndexer:
         if logger != None: logger.info(f"The file {file} {file_exist}")
         parsed_texts, stats = TextPubmedAbstractParser.parse(file, logger= logger, options=options)
         for parsed_text in parsed_texts:
-            pmid, text, year, title, article_type, article_category = parsed_text
-            if pmid == None or text == "" or (len(text.split(" ")) < 10): continue
-
-            #If a blacklist word file is given, check to filter out documents whose title or article_category contains any of the words
-            if options['filter_by_blacklist'] != None:
-                blacklisted_words = [word.strip() for word in open(options['filter_by_blacklist']).readlines()]
-                filtered_out, w, c, t = TextIndexer._check_to_filter_out(blacklisted_words, title, article_category, options['blacklisted_mode'])
-                if filtered_out:
-                    if logger != None: logger.warning(f"Blacklisted PMID {pmid} for having word {w} in {c} with content: {t}")
-                    continue
-
-            pmid_content_and_stats = cls.prepare_indexes(text, pmid, file, year, title, article_type, article_category, options)
-            texts.append(pmid_content_and_stats)
+            pmid, text, year, title, article_type, article_category, keywords = parsed_text
+            if pmid == "None": continue
+            if text == "None" and title == "None" and len(keywords) == 0: continue
+
+            if options['remain_empty_documents'] or text != "None":
+                if options['filter_by_blacklist'] != None: #If a blacklist word file is given, check to filter out documents whose title or article_category contains any of the words
+                    is_blacklisted = TextIndexer._check_if_blacklisted_pmid(pmid, title, article_category, options, logger)
+                    if is_blacklisted: continue                       
+
+                pmid_content_and_stats = cls.prepare_indexes(text, pmid, file, year, title, article_type, article_category, keywords, options)
+                texts.append(pmid_content_and_stats)
 
         if logger != None: logger.warning(f"stats:file={file},total={stats['total']},no_abstract={stats['no_abstract']},no_pmid={stats['no_pmid']}")
         return texts
@@ -139,54 +143,58 @@ class TextIndexer:
         if options["equivalences_file"] != None: PMC_PMID_dict = dict(CmdTabs.load_input_data(options["equivalences_file"]))
         texts = [] # aggregate all papers in XML file (Technically it is just one for each xml file, but it is emulating the original abstract part logic)
         for parsed_text in parsed_texts:
-            pmid, pmc, filename, year, whole_content, title, article_type, article_category = parsed_text
-            if pmid == None and PMC_PMID_dict != None: pmid = PMC_PMID_dict.get(pmc)
+            pmid, pmc, filename, year, whole_content, title, article_type, article_category, keywords = parsed_text
+            pmid = cls._set_alternative_pmid_if_needed_and_available(pmid, pmc, PMC_PMID_dict, file_path, logger)           
 
-            #If there is no pmid in the paper or in the PMC_PMID_dict, we will use the pmc as the document identifier, specifying it is PMC in the index. 
-            #If not PMC nor PMID, we will warn the user
-            if pmid == None and pmc == None: logger.warning(f"ERROR: The file {file_path} has a paper without PMID and PMC")
-            elif pmid == None and pmc != None: pmid = pmc
+            if pmid == "None": continue
+            if whole_content == "None" and title == "None" and len(keywords) == 0: continue
             
-            #If a blacklist word file is given, check to filter out documents whose title or article_category contains any of the words
-            if options['filter_by_blacklist'] != None:
-                if os.path.exists(options['filter_by_blacklist']):
-                    blacklisted_words = [word.strip() for word in open(options['filter_by_blacklist']).readlines()]
-                    filtered_out, w, c, t = TextIndexer._check_to_filter_out(blacklisted_words, title, article_category, options['blacklisted_mode'])
-                    if filtered_out:
-                        if logger != None: logger.warning(f"Blacklisted PMID {pmid} for having word {w} in {c} with content: {t}")
-                        continue
-                else: 
-                    raise Exception("Blacklisted words filepath given does not exist")
-
-            if pmid != None and whole_content != "" and len(whole_content) > 1000:
+            if options['remain_empty_documents'] or (whole_content != "None"):
+                if options['filter_by_blacklist'] != None: #If a blacklist word file is given, check to filter out documents whose title or article_category contains any of the words
+                    is_blacklisted = TextIndexer._check_if_blacklisted_pmid(pmid, title, article_category, options, logger)
+                    if is_blacklisted: continue
+                    
                 #pmid_content_and_stats = cls.prepare_indexes(whole_content, pmc+"-"+pmid, filename, year, options)
-                pmid_content_and_stats = cls.prepare_indexes(whole_content, pmid, filename, year, title, article_type, article_category, options)
+                pmid_content_and_stats = cls.prepare_indexes(whole_content, pmid, filename, year, title, article_type, article_category, keywords, options)
                 texts.append(pmid_content_and_stats)
 
         if logger != None: logger.warning(f"stats:file={file_path},total={stats['total']},no_abstract={stats['no_abstract']},no_pmid={stats['no_pmid']}")
         if logger != None: logger.warning(f"logs_errors:file={file_path},errors_number={stats['errors']}")
         return texts
-
+      
 
     @classmethod
-    def prepare_indexes(cls, text, pmid, file, year, title, article_type, article_category, options):
+    def prepare_indexes(cls, text, pmid, file, year, title, article_type, article_category, keywords, options):
+        extra_fields = options.get("extra_fields", defaultdict(lambda: False))
         pmid = pmid.replace("\n", "")
         file = file.replace("\n", "")
         year = str(year).replace("\n", "")
-
         document_length = str(len(text))
-        if options["split"]:
-          document_parts = cls.split_document(text, pmid, split_type = options["split_type"])
+
+        cleaned_text = TextPubmedParser.perform_soft_cleaning(text, type=options["clean_type"])
+        if text == "None":
+          doc_to_jsonify = []
+          document_length = "0"
+          number_of_sentences = "0"
+          length_of_sentences = "0"
+        elif options["split"]:
+          document_parts = cls.split_document(cleaned_text, pmid, split_type = options["split_type"])
+          doc_to_jsonify = document_parts
           flattened_document = flatten(document_parts)
           number_of_sentences = str(len(flattened_document))
           length_of_sentences = ",".join([str(len(sentence)) for sentence in flattened_document])
-          document_parts_json = json.dumps(document_parts)
-          prepared_index = [pmid, document_parts_json, file, year, document_length, number_of_sentences, length_of_sentences, 
-                            title, article_type, article_category] 
         else:
-          cleaned_document = TextPubmedParser.perform_soft_cleaning(text)
-          prepared_index = [pmid, cleaned_document, file, year, document_length, "1", document_length, 
-                            title, article_type, article_category]
+          number_of_sentences = "1"
+          length_of_sentences = document_length
+          doc_to_jsonify = [[cleaned_text]]
+
+        keywords_to_add = keywords if len(keywords) > 0 else ["None"]
+        if extra_fields["keywords"]: doc_to_jsonify.insert(0, ["KEYWORDS"] + keywords_to_add)
+        if extra_fields["title"]: doc_to_jsonify.insert(0, ["TITLE", title])
+        
+        document_json = json.dumps(doc_to_jsonify)
+        prepared_index = [pmid, document_json, file, year, document_length, number_of_sentences, length_of_sentences, 
+                            title, article_type, article_category, ",".join(keywords_to_add)]
         return prepared_index
 
 
@@ -198,25 +206,54 @@ class TextIndexer:
         if split_type == "classic":
             sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 10, chunk_overlap  = 0, length_function = len, separators=["\n",".", ";", ","], keep_separator=False, is_separator_regex=False)
         elif split_type == "space_overlap":
-            sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 60, chunk_overlap  = 20, length_function = len, separators=["\n", " ", ""], keep_separator=False, is_separator_regex=False)
+            sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 100, chunk_overlap  = 20, length_function = len, separators=["\n", " ", ""], keep_separator=False, is_separator_regex=False)
+            #sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 20, chunk_overlap  = 2, length_function = cls._len_by_words_func, separators=["\n", " ", ""], keep_separator=False, is_separator_regex=False)
         #sentences_splitter = RecursiveCharacterTextSplitter(chunk_size = 120, chunk_overlap  = 20, length_function = len, separators=["\n", " ", ""], keep_separator=False, is_separator_regex=False)
         paragraphs = [paragraph.strip() for paragraph in paragraph_splitter.split_text(text)]
-        sentences = [ sentences_splitter.split_text(paragraph.replace("\n", "")) for paragraph in paragraphs ]
-        sentences = [ list(map(lambda sentence: sentence.strip(), paragraph)) for paragraph in sentences ]
+        paragraph_sentences = [ sentences_splitter.split_text(paragraph) for paragraph in paragraphs ]
         
         formated = []       
-        for paragraph in sentences:
+        for paragraph in paragraph_sentences:
             formated_paragraph = [] 
             for sentence in paragraph:
                 if len(sentence) > 2:
                     if len(sentence) > 3000: warnings.warn(f"ERROR: The pmid {pmid} has an unusual sentence lenght even after splitting. Total length of characters: {len(sentence)}. Content: {sentence}")
-                    else: formated_paragraph.append(sentence)
+                    else: formated_paragraph.append(sentence.strip())
             if len(formated_paragraph) > 0: formated.append(formated_paragraph)
         
         return formated
     
 
     ######### UTILS METHODS
+    @classmethod
+    def _len_by_words_func(cls, text):
+        return text.count(" ")
+
+    @classmethod
+    def _set_alternative_pmid_if_needed_and_available(cls, pmid, pmc, PMC_PMID_dict, file_path, logger):
+        #If there is no pmid in the paper or in the PMC_PMID_dict, we will use the pmc as the document identifier, specifying it is PMC in the index. 
+        #If not PMC nor PMID, we will warn the user
+        if pmid == "None" and PMC_PMID_dict != None: 
+            pmid = PMC_PMID_dict.get(pmc, "None")
+        if pmid == "None" and pmc == "None": 
+            logger.warning(f"ERROR: The file {file_path} has a paper without PMID and PMC")
+        elif pmid == "None" and pmc != "None": 
+            pmid = pmc
+        return pmid
+
+    @classmethod
+    def _check_if_blacklisted_pmid(cls, pmid, title, article_category, options, logger):
+        if os.path.exists(options['filter_by_blacklist']):
+            is_blacklisted = False
+            blacklisted_words = [word.strip() for word in open(options['filter_by_blacklist']).readlines()]
+            filtered_out, w, c, t = TextIndexer._check_to_filter_out(blacklisted_words, title, article_category, options['blacklisted_mode'])
+            if filtered_out:
+                if logger != None: logger.warning(f"Blacklisted PMID {pmid} for having word {w} in {c} with content: {t}")
+                is_blacklisted = True
+            return is_blacklisted
+        else: 
+            raise Exception("Blacklisted words filepath given does not exist")  
+
 
     @classmethod
     def _check_to_filter_out(cls, blacklisted_words, title, article_category, mode):