pembot 0.0.6__tar.gz → 0.0.8__tar.gz

{pembot-0.0.6 → pembot-0.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pembot
-Version: 0.0.6
+Version: 0.0.8
 Summary: A Python Package to convert PEM blog content to usseful information by leveraging LLMs
 Author-email: cyto <aryan_sidhwani@protonmail.com>
 License-Expression: MIT

pembot-0.0.8/pembot/.git/COMMIT_EDITMSG

@@ -0,0 +1 @@
+added model name to convertor

{pembot-0.0.6 → pembot-0.0.8}/pembot/.git/logs/HEAD

@@ -6,3 +6,5 @@ e91172752e9a421ae463112d2b0506b37498c98d 0c8d9b2690545bf1906b05cd9f18b783b3eb74f
 0c8d9b2690545bf1906b05cd9f18b783b3eb74f1 eb75e1c49f1e5b79dca17ccdbec8067756523238 cyto <silverstone965@gmail.com> 1750856653 +0530	commit: made arrangements for the cases when custom file bytes are to be processed to text output; handled a ollama running / crashing error
 eb75e1c49f1e5b79dca17ccdbec8067756523238 0bdb4169fc0f312b8698f1df17a258fff163aeaa cyto <silverstone965@gmail.com> 1750937276 +0530	commit: fixed the output_dir bug; fixed the excel to json function; ran some tests on convertor; incremented the version on the package; removed dependency on schema / structure, and shifted required fields to a pickle file path in the cli args;
 0bdb4169fc0f312b8698f1df17a258fff163aeaa 9528bbccd167e3f4ad583a1ae9fac98a52620e27 cyto <silverstone965@gmail.com> 1750947488 +0530	commit: handled local llm nonexistent error properly for choice of just passing None as llm_client;
+9528bbccd167e3f4ad583a1ae9fac98a52620e27 ef0503a60244391590b16042019032e91d7cc30d cyto <silverstone965@gmail.com> 1751872559 +0530	commit: added a model_name_parameter to change models quicky
+ef0503a60244391590b16042019032e91d7cc30d af80ddb5890f062e364ea8ade2d602df4e12de8c cyto <silverstone965@gmail.com> 1751896700 +0530	commit: added model name to convertor

{pembot-0.0.6 → pembot-0.0.8}/pembot/.git/logs/refs/heads/main

@@ -6,3 +6,5 @@ e91172752e9a421ae463112d2b0506b37498c98d 0c8d9b2690545bf1906b05cd9f18b783b3eb74f
 0c8d9b2690545bf1906b05cd9f18b783b3eb74f1 eb75e1c49f1e5b79dca17ccdbec8067756523238 cyto <silverstone965@gmail.com> 1750856653 +0530	commit: made arrangements for the cases when custom file bytes are to be processed to text output; handled a ollama running / crashing error
 eb75e1c49f1e5b79dca17ccdbec8067756523238 0bdb4169fc0f312b8698f1df17a258fff163aeaa cyto <silverstone965@gmail.com> 1750937276 +0530	commit: fixed the output_dir bug; fixed the excel to json function; ran some tests on convertor; incremented the version on the package; removed dependency on schema / structure, and shifted required fields to a pickle file path in the cli args;
 0bdb4169fc0f312b8698f1df17a258fff163aeaa 9528bbccd167e3f4ad583a1ae9fac98a52620e27 cyto <silverstone965@gmail.com> 1750947488 +0530	commit: handled local llm nonexistent error properly for choice of just passing None as llm_client;
+9528bbccd167e3f4ad583a1ae9fac98a52620e27 ef0503a60244391590b16042019032e91d7cc30d cyto <silverstone965@gmail.com> 1751872559 +0530	commit: added a model_name_parameter to change models quicky
+ef0503a60244391590b16042019032e91d7cc30d af80ddb5890f062e364ea8ade2d602df4e12de8c cyto <silverstone965@gmail.com> 1751896700 +0530	commit: added model name to convertor

{pembot-0.0.6 → pembot-0.0.8}/pembot/.git/logs/refs/remotes/origin/main

@@ -5,3 +5,5 @@ e91172752e9a421ae463112d2b0506b37498c98d 0c8d9b2690545bf1906b05cd9f18b783b3eb74f
 0c8d9b2690545bf1906b05cd9f18b783b3eb74f1 eb75e1c49f1e5b79dca17ccdbec8067756523238 cyto <silverstone965@gmail.com> 1750856672 +0530	update by push
 eb75e1c49f1e5b79dca17ccdbec8067756523238 0bdb4169fc0f312b8698f1df17a258fff163aeaa cyto <silverstone965@gmail.com> 1750937389 +0530	update by push
 0bdb4169fc0f312b8698f1df17a258fff163aeaa 9528bbccd167e3f4ad583a1ae9fac98a52620e27 cyto <silverstone965@gmail.com> 1750947502 +0530	update by push
+9528bbccd167e3f4ad583a1ae9fac98a52620e27 ef0503a60244391590b16042019032e91d7cc30d cyto <silverstone965@gmail.com> 1751872581 +0530	update by push
+ef0503a60244391590b16042019032e91d7cc30d af80ddb5890f062e364ea8ade2d602df4e12de8c cyto <silverstone965@gmail.com> 1751896713 +0530	update by push

pembot-0.0.8/pembot/.git/objects/ef/0503a60244391590b16042019032e91d7cc30d

@@ -0,0 +1,3 @@
+x
��Kj1D��)z� ��ĴZ-{�h���
�>2�AVU��Qom��9��j�5�d��,��#��tQF&T�J|��ۀ�t̙�(�T��E�
+9U��i�%� �>�}z�Dz��~��q����pYߩ�OP���s	ޤ3R�v
+�*�.��z���a��Ԟ�t��¯����B���k�U�

pembot-0.0.8/pembot/.git/refs/heads/main

@@ -0,0 +1 @@
+af80ddb5890f062e364ea8ade2d602df4e12de8c

pembot-0.0.8/pembot/.git/refs/remotes/origin/main

@@ -0,0 +1 @@
+af80ddb5890f062e364ea8ade2d602df4e12de8c

{pembot-0.0.6 → pembot-0.0.8}/pembot/AnyToText/convertor.py

@@ -31,12 +31,14 @@ EXCEL_FILE_TYPES= [
 class Convertor():
 
 
-    def __init__(self, myfile: Path | None= None, output_dir: Path | None= None, file_bytes: bytes | None= None, suffix: str | None= None, file_type: str | None= None):
+    def __init__(self, myfile: Path | None= None, output_dir: Path | None= None, file_bytes: bytes | None= None, suffix: str | None= None, file_type: str | None= None, model_name: str | None = None):
 
         self.output= ""
 
-        # model_name=  "gemini-2.5-flash"
-        model_name= None
+        if model_name is None:
+            # model_name=  "gemini-2.5-flash"
+            model_name=  "Nanonets-OCR-s"
+
         # file_type can be pdf, excel, etc.
         if output_dir is None and myfile is None and file_bytes is not None and suffix is not None:
             with tempfile.TemporaryDirectory() as dp:

{pembot-0.0.6 → pembot-0.0.8}/pembot/TextEmbedder/mongodb_embedder.py

@@ -29,6 +29,7 @@ def search_within_document(
     limit: int = 5,
     index_name: str = "test_search",
     embeddings_collection_name: str= "doc_chunks",
+    document_belongs_to_a_type = "",
 ):
     """
     Performs a vector similarity search within the chunks of a specific document
@@ -42,6 +43,7 @@ def search_within_document(
         index_name: The name of your MongoDB Atlas Vector Search index.
                     You MUST have a vector search index created on the 'embedding' field
                     of the 'embeddings_collection' collection for this to work efficiently.
+        document_belongs_to_a_type: When search spaces intersect for different docIds, such that docId is an array field,
 
     Returns:
         A list of dictionaries, where each dictionary represents a matching chunk
@@ -50,10 +52,23 @@ def search_within_document(
     embeddings_collection = db_client[embeddings_collection_name]
 
     print(f"Searching within document (docId: {document_name_id})...")
+    # print(f" filter (slug: {document_belongs_to_a_type})...")
 
     # MongoDB Atlas Vector Search aggregation pipeline
     # The 'path' should point to the field containing the embeddings.
     # The 'filter' stage is crucial for searching within a specific document.
+    #
+    project_dict= {
+        '_id': 0,
+        'docId': 1,
+        'chunk_number': 1,
+        'chunk_text': 1,
+        'score': { '$meta': 'vectorSearchScore' } # Get the similarity score
+    }
+
+    if document_belongs_to_a_type:
+        project_dict['type']= 1
+
     pipeline = [
         {
             '$vectorSearch': {
@@ -66,25 +81,21 @@ def search_within_document(
                 'index': index_name,
 
                 #filter to search only within the specified document
-                'filter': {
-                    'docId': document_name_id
-                }
+                'filter':
+                    { "type": {"$in": [document_belongs_to_a_type ]} } if document_belongs_to_a_type else
+                    { 'docId': document_name_id }
             }
         },
 
         # to exclude the MongoDB internal _id
         {
-            '$project': {
-                '_id': 0,
-                'docId': 1,
-                'chunk_number': 1,
-                'chunk_text': 1,
-                'score': { '$meta': 'vectorSearchScore' } # Get the similarity score
-            }
+            '$project': project_dict
         }
     ]
 
+    # print("sesraching now:")
     results = list(embeddings_collection.aggregate(pipeline))
+    # print("search results: ", results)
 
     if not results:
         print(f"No relevant chunks found for document '{document_name_id}' with the given query.")
@@ -100,15 +111,18 @@ def search_within_document(
 
 
 
-def process_document_and_embed(db_client,
+def process_document_and_embed(
+    db_client,
     llm_client,
     inference_client,
     file_path: Path,
     chunk_size: int,
-    embedding_model: str = 'nomic-embed-text:v1.5',
+    embedding_model: str = 'BAAI/bge-en-icl',
     embeddings_collection_name= "doc_chunks",
     use_custom_id: str | None = None,
-    use_custom_input: str | None = None
+    use_custom_input: str | None = None,
+    document_belongs_to_a_type= "",
+    type_info= []
 ) -> list[dict]:
     """
     Processes an input document by chunking its text, generating embeddings using
@@ -228,13 +242,30 @@ def process_document_and_embed(db_client,
                 'chunk_text': chunk,
                 'embedding': embedding,
                 'chunk_id_global': chunk_id_global,
-                'chunk_id_doc_specific': chunk_id_doc_specific
+                'chunk_id_doc_specific': chunk_id_doc_specific,
             }
-            embeddings_collection.update_one(
-                {'docId': document_name_id, 'chunk_number': i + 1},
-                {'$set': doc_set},
-                upsert=True
-            )
+
+
+            # TBD: this is NOT pushing array, this is creating a "$push" field with type: "" object
+
+            if len(type_info) > 0:
+                embeddings_collection.update_one(
+                    {'docId': document_name_id, 'chunk_number': i + 1},
+                    {
+                        '$set': doc_set,
+                        '$push': {
+                            "type": type_info
+                        }
+                    },
+                    upsert=True
+                )
+            else:
+
+                embeddings_collection.update_one(
+                    {'docId': document_name_id, 'chunk_number': i + 1},
+                    {'$set': doc_set},
+                    upsert=True
+                )
             print(f"Successfully stored chunk {i+1} for '{file_path.name}' in MongoDB.")
             res.append({**doc_set, "docId": document_name_id, "chunk_number": i + 1})
 

{pembot-0.0.6 → pembot-0.0.8}/pembot/TextEmbedder/mongodb_index_creator.py

@@ -4,7 +4,7 @@ from pymongo.operations import SearchIndexModel
 import time
 import os
 
-def create_vector_index(collection: Collection, index_name: str, num_dimensions: int = 768):
+def create_vector_index(collection: Collection, index_name: str, num_dimensions: int = 768, document_belongs_to_a_type= ""):
     """
     Creates a MongoDB Atlas Vector Search index if it does not already exist.
 
@@ -13,14 +13,14 @@ def create_vector_index(collection: Collection, index_name: str, num_dimensions:
         index_name: The desired name for the vector search index.
         num_dimensions: The number of dimensions for the embedding vectors.
     """
-    
+
     # 1. Check if the index already exists
     existing_indexes = list(collection.list_search_indexes())
-    
+
     for index in existing_indexes:
         if index.get('name') == index_name:
             print(f"Search index '{index_name}' already exists. Skipping creation.")
-            
+
             # Optional: You can also check if the existing index is "READY"
             if index.get('status') == 'READY':
                 print(f"Index '{index_name}' is already ready for querying.")
@@ -33,20 +33,27 @@ def create_vector_index(collection: Collection, index_name: str, num_dimensions:
     # 2. If the index does not exist, proceed to create it
     print(f"Search index '{index_name}' does not exist. Creating it now...")
 
+    fields_arr= [
+        {
+            "type": "vector",
+            "path": "embedding",
+            "similarity": "dotProduct", # Or "cosine", "euclidean"
+            "numDimensions": num_dimensions,
+            "quantization": "scalar" # Or "none"
+        },
+        {
+            "type": "filter",
+            "path": "docId"
+        }
+    ]
+
+    if document_belongs_to_a_type:
+        fields_arr.append({
+            "type": "filter",
+            "path": "type"
+        })
     search_index_model = SearchIndexModel(definition={
-            "fields": [
-                {
-                    "type": "vector",
-                    "path": "embedding",
-                    "similarity": "dotProduct", # Or "cosine", "euclidean"
-                    "numDimensions": num_dimensions,
-                    "quantization": "scalar" # Or "none"
-                },
-                {
-                    "type": "filter",
-                    "path": "docId"
-                }
-            ]
+            "fields": fields_arr
         },
         name=index_name,
         type="vectorSearch"
@@ -70,7 +77,7 @@ def _wait_for_index_ready(collection: Collection, index_name: str):
     Helper function to poll the index status until it's ready.
     """
     print("Polling to check if the index is ready. This may take some time (up to a few minutes for large indexes).")
-    
+
     start_time = time.time()
     timeout = 300 # 5 minutes timeout, adjust as needed
 
@@ -89,7 +96,7 @@ def _wait_for_index_ready(collection: Collection, index_name: str):
                 print(f"Index '{index_name}' status: {current_status}. Waiting...")
         except Exception as e:
             print(f"Error while polling index status: {e}. Retrying...")
-        
+
         if time.time() - start_time > timeout:
             status= indices[0].get('status') if indices else 'N/A'
             print(f"Timeout: Index '{index_name}' did not become ready within {timeout} seconds. Current status: {status}")
@@ -99,9 +106,9 @@ def _wait_for_index_ready(collection: Collection, index_name: str):
 
 # --- Example Usage ---
 if __name__ == "__main__":
-    
+
     # Replace with your database and collection names
-    DATABASE_NAME = "pembot" 
+    DATABASE_NAME = "pembot"
     COLLECTION_NAME = "doc_chunks"
     VECTOR_INDEX_NAME = "test_search"
 
@@ -119,7 +126,7 @@ if __name__ == "__main__":
 
         # Call the function to create the index, with existence check
         create_vector_index(collection, VECTOR_INDEX_NAME, num_dimensions=EMBEDDING_DIMENSIONS)
-        
+
         # Test calling it again to see the "already exists" message
         create_vector_index(collection, VECTOR_INDEX_NAME, num_dimensions=EMBEDDING_DIMENSIONS)
 
@@ -129,5 +136,3 @@ if __name__ == "__main__":
         if 'mongo_client' in locals() and mongo_client:
             mongo_client.close()
             print("MongoDB connection closed.")
-
-

{pembot-0.0.6 → pembot-0.0.8}/pembot/__init__.py

@@ -1,6 +1,6 @@
 """
 A Python Package to convert PEM blog content to usseful information by leveraging LLMs
 """
-__version__ = '0.0.6'
+__version__ = '0.0.8'
 from .main import save_to_json_file, make_query
 __all__ = ["save_to_json_file", "make_query"]

{pembot-0.0.6 → pembot-0.0.8}/pembot/config/config.yaml

@@ -2,4 +2,4 @@ OUTPUT_DIR: /home/cyto/dev/pembotdir
 PAGE_DELIMITER: ___________________________ NEXT PAGE ___________________________
 app:
   name: pembot
-  version: 0.0.6
+  version: 0.0.8

pembot-0.0.8/pembot/pdf2markdown/.git/COMMIT_EDITMSG

@@ -0,0 +1 @@
+handled the gpu errors non-gracefully so that it stops

{pembot-0.0.6 → pembot-0.0.8}/pembot/pdf2markdown/.git/logs/HEAD

@@ -2,3 +2,4 @@
 ffb759ee4605b232366a9ee58134532913c3f9e0 b8702320e56074e9680181d8b7897d6a0a552e2d cyto <silverstone965@gmail.com> 1750947962 +0530	commit: handled config loading errors gracefully; added gemini support, as an option; added huggingface nanonets transformers support (as an option); redesigned the extract markdown for captioning and image ocr (block image and full-page image);
 b8702320e56074e9680181d8b7897d6a0a552e2d 14251b198e0bac39a3dc3b42f9e57b20c01465fb cyto <silverstone965@gmail.com> 1751604763 +0530	commit: removed deps on torch and transformers; used gradio client for ocr through public spaces;
 14251b198e0bac39a3dc3b42f9e57b20c01465fb b48d697aa9fd97151eb2a84a1af5d408b7630232 cyto <silverstone965@gmail.com> 1751871887 +0530	commit: cyto/argument-list-bug-fix;authentication-used-in-gradio-client
+b48d697aa9fd97151eb2a84a1af5d408b7630232 f3b2d76c75bbd50e04fc4c2ad17fc94ca6daed32 cyto <silverstone965@gmail.com> 1751896628 +0530	commit: handled the gpu errors non-gracefully so that it stops

{pembot-0.0.6 → pembot-0.0.8}/pembot/pdf2markdown/.git/logs/refs/heads/main

@@ -2,3 +2,4 @@
 ffb759ee4605b232366a9ee58134532913c3f9e0 b8702320e56074e9680181d8b7897d6a0a552e2d cyto <silverstone965@gmail.com> 1750947962 +0530	commit: handled config loading errors gracefully; added gemini support, as an option; added huggingface nanonets transformers support (as an option); redesigned the extract markdown for captioning and image ocr (block image and full-page image);
 b8702320e56074e9680181d8b7897d6a0a552e2d 14251b198e0bac39a3dc3b42f9e57b20c01465fb cyto <silverstone965@gmail.com> 1751604763 +0530	commit: removed deps on torch and transformers; used gradio client for ocr through public spaces;
 14251b198e0bac39a3dc3b42f9e57b20c01465fb b48d697aa9fd97151eb2a84a1af5d408b7630232 cyto <silverstone965@gmail.com> 1751871887 +0530	commit: cyto/argument-list-bug-fix;authentication-used-in-gradio-client
+b48d697aa9fd97151eb2a84a1af5d408b7630232 f3b2d76c75bbd50e04fc4c2ad17fc94ca6daed32 cyto <silverstone965@gmail.com> 1751896628 +0530	commit: handled the gpu errors non-gracefully so that it stops

{pembot-0.0.6 → pembot-0.0.8}/pembot/pdf2markdown/.git/logs/refs/remotes/myorigin/main

@@ -1,3 +1,4 @@
 0000000000000000000000000000000000000000 b8702320e56074e9680181d8b7897d6a0a552e2d cyto <silverstone965@gmail.com> 1750948073 +0530	update by push
 b8702320e56074e9680181d8b7897d6a0a552e2d 14251b198e0bac39a3dc3b42f9e57b20c01465fb cyto <silverstone965@gmail.com> 1751604904 +0530	update by push
 14251b198e0bac39a3dc3b42f9e57b20c01465fb b48d697aa9fd97151eb2a84a1af5d408b7630232 cyto <silverstone965@gmail.com> 1751872077 +0530	update by push
+b48d697aa9fd97151eb2a84a1af5d408b7630232 f3b2d76c75bbd50e04fc4c2ad17fc94ca6daed32 cyto <silverstone965@gmail.com> 1751896663 +0530	update by push

pembot-0.0.8/pembot/pdf2markdown/.git/objects/f3/b2d76c75bbd50e04fc4c2ad17fc94ca6daed32

@@ -0,0 +1 @@
+x
��Kj!3vw� �l媷?`k�v��>�!�C'��:'Hk�f't:�lȺ�6g�u2j�߈G�TV��ةN��gb�rp���F��ɚ���RI��<Z���

pembot-0.0.8/pembot/pdf2markdown/.git/refs/heads/main

@@ -0,0 +1 @@
+f3b2d76c75bbd50e04fc4c2ad17fc94ca6daed32

pembot-0.0.8/pembot/pdf2markdown/.git/refs/remotes/myorigin/main

@@ -0,0 +1 @@
+f3b2d76c75bbd50e04fc4c2ad17fc94ca6daed32

{pembot-0.0.6 → pembot-0.0.8}/pembot/pdf2markdown/extract.py

@@ -115,6 +115,10 @@ class MarkdownPDFExtractor(PDFExtractor):
         except Exception as e:
             self.logger.error(f"Error processing PDF: {e}")
             self.logger.exception(traceback.format_exc())
+
+            error_message= str(e).lower()
+            if "GPU" in error_message and "quota" in error_message:
+                return "GPU quota error", []
             return "", []
 
 
@@ -176,6 +180,12 @@ class MarkdownPDFExtractor(PDFExtractor):
                     print("ocr'd: ", result[:100] + "...")
             except Exception as e:
                 print("Error during nanonet inference", e)
+                error_message = str(e)
+                if "You have exceeded your Pro GPU quota" in error_message:
+                    # print("\n\n\nFALLING BACK TO TESS\n\n\n")
+                    # return pytesseract.image_to_string(pil_image)
+                    raise e
+
             
             return result
         else:
@@ -262,6 +272,9 @@ class MarkdownPDFExtractor(PDFExtractor):
                             except Exception as e:
                                 self.logger.error(f"    Error processing embedded image block for OCR: {e}")
                                 current_page_markdown_blocks.append("\n\n![Image Processing Error](error_on_page_{page_num+1}_block_{block_num+1}.png)\n\n")
+                                error_message= str(e).lower()
+                                if "GPU" in error_message and "quota" in error_message:
+                                    raise e
 
 
                     # Insert tables at their approximate positions (after blocks are processed for the page)
@@ -306,6 +319,9 @@ class MarkdownPDFExtractor(PDFExtractor):
                                 self.logger.info(f"  Full-page OCR yielded no text for page {page_num+1}.")
                         except Exception as e:
                             self.logger.error(f"  Error during full-page OCR on page {page_num+1}: {e}")
+                            error_message= str(e).lower()
+                            if "GPU" in error_message and "quota" in error_message:
+                                raise e
                     else:
                         self.logger.info(f"  Page {page_num + 1} has sufficient searchable text or embedded image OCR; skipping full-page OCR.")
 
@@ -329,7 +345,12 @@ class MarkdownPDFExtractor(PDFExtractor):
             except Exception as e:
                 self.logger.critical(f"An unexpected error occurred during markdown extraction: {e}")
                 self.logger.exception(traceback.format_exc())
-                return "", []
+
+                error_message= str(e).lower()
+                if "GPU" in error_message and "quota" in error_message:
+                    return "GPU quota error", []
+                else:
+                    return "", []
 
     def extract_tables(self):
         """Extract tables from PDF using pdfplumber."""
@@ -412,6 +433,9 @@ class MarkdownPDFExtractor(PDFExtractor):
         except Exception as e:
             self.logger.error(f"Error captioning image: {e}")
             self.logger.exception(traceback.format_exc())
+            error_message= str(e)
+            if "GPU" in error_message and "quota" in error_message:
+                raise e
             return ""
 
     def clean_text(self, text):
@@ -726,6 +750,7 @@ class MarkdownPDFExtractor(PDFExtractor):
             self.logger.exception(traceback.format_exc())
             return ""
 
+
     def get_header_level(self, font_size):
         """Determine header level based on font size."""
         if font_size > 24:

{pembot-0.0.6 → pembot-0.0.8}/pembot/query.py

@@ -68,7 +68,8 @@ def multi_embedding_average(llm_client, inference_client, descriptions, model= "
         except Exception as e:
             print(f"Error generating embedding for description '{desc}': {e}")
             # Decide how to handle errors: skip, raise, or use a placeholder
-            continue
+            # continue
+            raise e
         time.sleep(1)
 
     if not description_embeddings:
@@ -81,7 +82,7 @@ def multi_embedding_average(llm_client, inference_client, descriptions, model= "
 
 
 
-def rag_query_llm(db_client, llm_client, inference_client, user_query: str, document_id: str, required_fields_descriptions: list[str], model_name: str = "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B", ollama_base_url: str = "http://localhost:11434", no_of_fields= 4, embedding_model= "BAAI/bge-en-icl", llm_provider_name: PROVIDER_T= "novita", index_name: str= "test_search", embeddings_collection= "doc_chunks"):
+def rag_query_llm(db_client, llm_client, inference_client, user_query: str, document_id: str, required_fields_descriptions: list[str], model_name: str = "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B", ollama_base_url: str = "http://localhost:11434", no_of_fields= 4, embedding_model= "BAAI/bge-en-icl", llm_provider_name: PROVIDER_T= "novita", index_name: str= "test_search", embeddings_collection= "doc_chunks", document_belongs_to_a_type= ""):
     """
     Performs a RAG (Retrieval Augmented Generation) query using a Hugging Face
     embedding model, ChromaDB for retrieval, and a local Ollama model for generation.
@@ -119,10 +120,10 @@ def rag_query_llm(db_client, llm_client, inference_client, user_query: str, docu
     aggregate_query_embedding= multi_embedding_average(llm_client, inference_client, required_fields_descriptions, model= embedding_model, embed_locally= embed_locally)
     print("Aggregate query embedding generated. length: ", len(aggregate_query_embedding))
 
-    create_vector_index(db_client[embeddings_collection], index_name, num_dimensions= len(aggregate_query_embedding))
+    create_vector_index(db_client[embeddings_collection], index_name, num_dimensions= len(aggregate_query_embedding), document_belongs_to_a_type= document_belongs_to_a_type)
 
     # check the order of args
-    relevant_chunks= search_within_document(db_client, aggregate_query_embedding, document_id, limit= no_of_fields, index_name= index_name, embeddings_collection_name= embeddings_collection)
+    relevant_chunks= search_within_document(db_client, aggregate_query_embedding, document_id, limit= no_of_fields, index_name= index_name, embeddings_collection_name= embeddings_collection, document_belongs_to_a_type= document_belongs_to_a_type)
     relevant_chunks= list(map(lambda x: x['chunk_text'], relevant_chunks))
 
     if not relevant_chunks:

pembot-0.0.6/pembot/.git/COMMIT_EDITMSG

@@ -1 +0,0 @@
-handled config loading errors gracefully; added gemini support, as an option; added huggingface nanonets transformers support (as an option); redesigned the extract markdown for captioning and image ocr (block image and full-page image);

pembot-0.0.6/pembot/.git/refs/heads/main

@@ -1 +0,0 @@
-9528bbccd167e3f4ad583a1ae9fac98a52620e27

pembot-0.0.6/pembot/.git/refs/remotes/origin/main

@@ -1 +0,0 @@
-9528bbccd167e3f4ad583a1ae9fac98a52620e27

pembot-0.0.6/pembot/pdf2markdown/.git/COMMIT_EDITMSG

@@ -1 +0,0 @@
-cyto/argument-list-bug-fix;authentication-used-in-gradio-client

pembot-0.0.6/pembot/pdf2markdown/.git/refs/heads/main

@@ -1 +0,0 @@
-b48d697aa9fd97151eb2a84a1af5d408b7630232

pembot-0.0.6/pembot/pdf2markdown/.git/refs/remotes/myorigin/main

@@ -1 +0,0 @@
-b48d697aa9fd97151eb2a84a1af5d408b7630232