pembot 0.0.7__tar.gz → 0.0.9__tar.gz

{pembot-0.0.7 → pembot-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pembot
-Version: 0.0.7
+Version: 0.0.9
 Summary: A Python Package to convert PEM blog content to usseful information by leveraging LLMs
 Author-email: cyto <aryan_sidhwani@protonmail.com>
 License-Expression: MIT

pembot-0.0.9/pembot/.git/COMMIT_EDITMSG

@@ -0,0 +1 @@
+fixed the type_info array pushing bug

{pembot-0.0.7 → pembot-0.0.9}/pembot/.git/logs/HEAD

@@ -8,3 +8,5 @@ eb75e1c49f1e5b79dca17ccdbec8067756523238 0bdb4169fc0f312b8698f1df17a258fff163aea
 0bdb4169fc0f312b8698f1df17a258fff163aeaa 9528bbccd167e3f4ad583a1ae9fac98a52620e27 cyto <silverstone965@gmail.com> 1750947488 +0530	commit: handled local llm nonexistent error properly for choice of just passing None as llm_client;
 9528bbccd167e3f4ad583a1ae9fac98a52620e27 ef0503a60244391590b16042019032e91d7cc30d cyto <silverstone965@gmail.com> 1751872559 +0530	commit: added a model_name_parameter to change models quicky
 ef0503a60244391590b16042019032e91d7cc30d af80ddb5890f062e364ea8ade2d602df4e12de8c cyto <silverstone965@gmail.com> 1751896700 +0530	commit: added model name to convertor
+af80ddb5890f062e364ea8ade2d602df4e12de8c 0d28f73897db0c9a9351ee9e64d2a0fe27db2705 cyto <silverstone965@gmail.com> 1752081786 +0530	commit: upgraded to 0.0.8; added ability to classify the document chunks into one more filterable parameter 'type'; other query related improvements..
+0d28f73897db0c9a9351ee9e64d2a0fe27db2705 a898d2c3947d30d8be64bd2bbcef68f956d5456b cyto <silverstone965@gmail.com> 1752136060 +0530	commit: fixed the type_info array pushing bug

{pembot-0.0.7 → pembot-0.0.9}/pembot/.git/logs/refs/heads/main

@@ -8,3 +8,5 @@ eb75e1c49f1e5b79dca17ccdbec8067756523238 0bdb4169fc0f312b8698f1df17a258fff163aea
 0bdb4169fc0f312b8698f1df17a258fff163aeaa 9528bbccd167e3f4ad583a1ae9fac98a52620e27 cyto <silverstone965@gmail.com> 1750947488 +0530	commit: handled local llm nonexistent error properly for choice of just passing None as llm_client;
 9528bbccd167e3f4ad583a1ae9fac98a52620e27 ef0503a60244391590b16042019032e91d7cc30d cyto <silverstone965@gmail.com> 1751872559 +0530	commit: added a model_name_parameter to change models quicky
 ef0503a60244391590b16042019032e91d7cc30d af80ddb5890f062e364ea8ade2d602df4e12de8c cyto <silverstone965@gmail.com> 1751896700 +0530	commit: added model name to convertor
+af80ddb5890f062e364ea8ade2d602df4e12de8c 0d28f73897db0c9a9351ee9e64d2a0fe27db2705 cyto <silverstone965@gmail.com> 1752081786 +0530	commit: upgraded to 0.0.8; added ability to classify the document chunks into one more filterable parameter 'type'; other query related improvements..
+0d28f73897db0c9a9351ee9e64d2a0fe27db2705 a898d2c3947d30d8be64bd2bbcef68f956d5456b cyto <silverstone965@gmail.com> 1752136060 +0530	commit: fixed the type_info array pushing bug

{pembot-0.0.7 → pembot-0.0.9}/pembot/.git/logs/refs/remotes/origin/main

@@ -7,3 +7,5 @@ eb75e1c49f1e5b79dca17ccdbec8067756523238 0bdb4169fc0f312b8698f1df17a258fff163aea
 0bdb4169fc0f312b8698f1df17a258fff163aeaa 9528bbccd167e3f4ad583a1ae9fac98a52620e27 cyto <silverstone965@gmail.com> 1750947502 +0530	update by push
 9528bbccd167e3f4ad583a1ae9fac98a52620e27 ef0503a60244391590b16042019032e91d7cc30d cyto <silverstone965@gmail.com> 1751872581 +0530	update by push
 ef0503a60244391590b16042019032e91d7cc30d af80ddb5890f062e364ea8ade2d602df4e12de8c cyto <silverstone965@gmail.com> 1751896713 +0530	update by push
+af80ddb5890f062e364ea8ade2d602df4e12de8c 0d28f73897db0c9a9351ee9e64d2a0fe27db2705 cyto <silverstone965@gmail.com> 1752081801 +0530	update by push
+0d28f73897db0c9a9351ee9e64d2a0fe27db2705 a898d2c3947d30d8be64bd2bbcef68f956d5456b cyto <silverstone965@gmail.com> 1752136119 +0530	update by push

pembot-0.0.9/pembot/.git/objects/0d/28f73897db0c9a9351ee9e64d2a0fe27db2705

@@ -0,0 +1,3 @@
+x
��An� E��)��E�B`���z�	*�!#��%W�����߷�)�
+J��Z��K%�����HZMl���4"�� cz�mXx���
+�Y�IxaVfd�H<����@l]�G]r
w�_{�o.{�+OF�<�ػ��A�� ��[�B+�5�
V��jwl��P���Eo�t�8��y�.��ꅁ�;��-���CX�D���K�M������c��Vύo�m����rBስل���������jv�

pembot-0.0.9/pembot/.git/objects/a8/98d2c3947d30d8be64bd2bbcef68f956d5456b

@@ -0,0 +1 @@
+x
��݊� @�^�s�P������2�MfR���<��~pश,�A�;�HC*:�h]��8T���	)��,ņ�V��X���Q���XE�

pembot-0.0.9/pembot/.git/refs/heads/main

@@ -0,0 +1 @@
+a898d2c3947d30d8be64bd2bbcef68f956d5456b

pembot-0.0.9/pembot/.git/refs/remotes/origin/main

@@ -0,0 +1 @@
+a898d2c3947d30d8be64bd2bbcef68f956d5456b

{pembot-0.0.7 → pembot-0.0.9}/pembot/TextEmbedder/mongodb_embedder.py

@@ -29,6 +29,7 @@ def search_within_document(
     limit: int = 5,
     index_name: str = "test_search",
     embeddings_collection_name: str= "doc_chunks",
+    document_belongs_to_a_type = "",
 ):
     """
     Performs a vector similarity search within the chunks of a specific document
@@ -42,6 +43,7 @@ def search_within_document(
         index_name: The name of your MongoDB Atlas Vector Search index.
                     You MUST have a vector search index created on the 'embedding' field
                     of the 'embeddings_collection' collection for this to work efficiently.
+        document_belongs_to_a_type: When search spaces intersect for different docIds, such that docId is an array field,
 
     Returns:
         A list of dictionaries, where each dictionary represents a matching chunk
@@ -50,10 +52,23 @@ def search_within_document(
     embeddings_collection = db_client[embeddings_collection_name]
 
     print(f"Searching within document (docId: {document_name_id})...")
+    # print(f" filter (slug: {document_belongs_to_a_type})...")
 
     # MongoDB Atlas Vector Search aggregation pipeline
     # The 'path' should point to the field containing the embeddings.
     # The 'filter' stage is crucial for searching within a specific document.
+    #
+    project_dict= {
+        '_id': 0,
+        'docId': 1,
+        'chunk_number': 1,
+        'chunk_text': 1,
+        'score': { '$meta': 'vectorSearchScore' } # Get the similarity score
+    }
+
+    if document_belongs_to_a_type:
+        project_dict['type']= 1
+
     pipeline = [
         {
             '$vectorSearch': {
@@ -66,25 +81,21 @@ def search_within_document(
                 'index': index_name,
 
                 #filter to search only within the specified document
-                'filter': {
-                    'docId': document_name_id
-                }
+                'filter':
+                    { "type": {"$in": [document_belongs_to_a_type ]} } if document_belongs_to_a_type else
+                    { 'docId': document_name_id }
             }
         },
 
         # to exclude the MongoDB internal _id
         {
-            '$project': {
-                '_id': 0,
-                'docId': 1,
-                'chunk_number': 1,
-                'chunk_text': 1,
-                'score': { '$meta': 'vectorSearchScore' } # Get the similarity score
-            }
+            '$project': project_dict
         }
     ]
 
+    # print("sesraching now:")
     results = list(embeddings_collection.aggregate(pipeline))
+    # print("search results: ", results)
 
     if not results:
         print(f"No relevant chunks found for document '{document_name_id}' with the given query.")
@@ -100,15 +111,18 @@ def search_within_document(
 
 
 
-def process_document_and_embed(db_client,
+def process_document_and_embed(
+    db_client,
     llm_client,
     inference_client,
     file_path: Path,
     chunk_size: int,
-    embedding_model: str = 'nomic-embed-text:v1.5',
+    embedding_model: str = 'BAAI/bge-en-icl',
     embeddings_collection_name= "doc_chunks",
     use_custom_id: str | None = None,
-    use_custom_input: str | None = None
+    use_custom_input: str | None = None,
+    document_belongs_to_a_type= "",
+    type_info= []
 ) -> list[dict]:
     """
     Processes an input document by chunking its text, generating embeddings using
@@ -228,13 +242,30 @@ def process_document_and_embed(db_client,
                 'chunk_text': chunk,
                 'embedding': embedding,
                 'chunk_id_global': chunk_id_global,
-                'chunk_id_doc_specific': chunk_id_doc_specific
+                'chunk_id_doc_specific': chunk_id_doc_specific,
             }
-            embeddings_collection.update_one(
-                {'docId': document_name_id, 'chunk_number': i + 1},
-                {'$set': doc_set},
-                upsert=True
-            )
+
+
+            # TBD: this is NOT pushing array, this is creating a "$push" field with type: "" object
+
+            if len(type_info) > 0:
+                embeddings_collection.update_one(
+                    {'docId': document_name_id, 'chunk_number': i + 1},
+                    {
+                        '$set': doc_set,
+                        '$addToSet': {
+                            "type": { '$each': type_info }
+                        }
+                    },
+                    upsert=True
+                )
+            else:
+
+                embeddings_collection.update_one(
+                    {'docId': document_name_id, 'chunk_number': i + 1},
+                    {'$set': doc_set},
+                    upsert=True
+                )
             print(f"Successfully stored chunk {i+1} for '{file_path.name}' in MongoDB.")
             res.append({**doc_set, "docId": document_name_id, "chunk_number": i + 1})
 

{pembot-0.0.7 → pembot-0.0.9}/pembot/TextEmbedder/mongodb_index_creator.py

@@ -4,7 +4,7 @@ from pymongo.operations import SearchIndexModel
 import time
 import os
 
-def create_vector_index(collection: Collection, index_name: str, num_dimensions: int = 768):
+def create_vector_index(collection: Collection, index_name: str, num_dimensions: int = 768, document_belongs_to_a_type= ""):
     """
     Creates a MongoDB Atlas Vector Search index if it does not already exist.
 
@@ -13,14 +13,14 @@ def create_vector_index(collection: Collection, index_name: str, num_dimensions:
         index_name: The desired name for the vector search index.
         num_dimensions: The number of dimensions for the embedding vectors.
     """
-    
+
     # 1. Check if the index already exists
     existing_indexes = list(collection.list_search_indexes())
-    
+
     for index in existing_indexes:
         if index.get('name') == index_name:
             print(f"Search index '{index_name}' already exists. Skipping creation.")
-            
+
             # Optional: You can also check if the existing index is "READY"
             if index.get('status') == 'READY':
                 print(f"Index '{index_name}' is already ready for querying.")
@@ -33,20 +33,27 @@ def create_vector_index(collection: Collection, index_name: str, num_dimensions:
     # 2. If the index does not exist, proceed to create it
     print(f"Search index '{index_name}' does not exist. Creating it now...")
 
+    fields_arr= [
+        {
+            "type": "vector",
+            "path": "embedding",
+            "similarity": "dotProduct", # Or "cosine", "euclidean"
+            "numDimensions": num_dimensions,
+            "quantization": "scalar" # Or "none"
+        },
+        {
+            "type": "filter",
+            "path": "docId"
+        }
+    ]
+
+    if document_belongs_to_a_type:
+        fields_arr.append({
+            "type": "filter",
+            "path": "type"
+        })
     search_index_model = SearchIndexModel(definition={
-            "fields": [
-                {
-                    "type": "vector",
-                    "path": "embedding",
-                    "similarity": "dotProduct", # Or "cosine", "euclidean"
-                    "numDimensions": num_dimensions,
-                    "quantization": "scalar" # Or "none"
-                },
-                {
-                    "type": "filter",
-                    "path": "docId"
-                }
-            ]
+            "fields": fields_arr
         },
         name=index_name,
         type="vectorSearch"
@@ -70,7 +77,7 @@ def _wait_for_index_ready(collection: Collection, index_name: str):
     Helper function to poll the index status until it's ready.
     """
     print("Polling to check if the index is ready. This may take some time (up to a few minutes for large indexes).")
-    
+
     start_time = time.time()
     timeout = 300 # 5 minutes timeout, adjust as needed
 
@@ -89,7 +96,7 @@ def _wait_for_index_ready(collection: Collection, index_name: str):
                 print(f"Index '{index_name}' status: {current_status}. Waiting...")
         except Exception as e:
             print(f"Error while polling index status: {e}. Retrying...")
-        
+
         if time.time() - start_time > timeout:
             status= indices[0].get('status') if indices else 'N/A'
             print(f"Timeout: Index '{index_name}' did not become ready within {timeout} seconds. Current status: {status}")
@@ -99,9 +106,9 @@ def _wait_for_index_ready(collection: Collection, index_name: str):
 
 # --- Example Usage ---
 if __name__ == "__main__":
-    
+
     # Replace with your database and collection names
-    DATABASE_NAME = "pembot" 
+    DATABASE_NAME = "pembot"
     COLLECTION_NAME = "doc_chunks"
     VECTOR_INDEX_NAME = "test_search"
 
@@ -119,7 +126,7 @@ if __name__ == "__main__":
 
         # Call the function to create the index, with existence check
         create_vector_index(collection, VECTOR_INDEX_NAME, num_dimensions=EMBEDDING_DIMENSIONS)
-        
+
         # Test calling it again to see the "already exists" message
         create_vector_index(collection, VECTOR_INDEX_NAME, num_dimensions=EMBEDDING_DIMENSIONS)
 
@@ -129,5 +136,3 @@ if __name__ == "__main__":
         if 'mongo_client' in locals() and mongo_client:
             mongo_client.close()
             print("MongoDB connection closed.")
-
-

{pembot-0.0.7 → pembot-0.0.9}/pembot/__init__.py

@@ -1,6 +1,6 @@
 """
 A Python Package to convert PEM blog content to usseful information by leveraging LLMs
 """
-__version__ = '0.0.7'
+__version__ = '0.0.9'
 from .main import save_to_json_file, make_query
 __all__ = ["save_to_json_file", "make_query"]

{pembot-0.0.7 → pembot-0.0.9}/pembot/config/config.yaml

@@ -2,4 +2,4 @@ OUTPUT_DIR: /home/cyto/dev/pembotdir
 PAGE_DELIMITER: ___________________________ NEXT PAGE ___________________________
 app:
   name: pembot
-  version: 0.0.7
+  version: 0.0.9

{pembot-0.0.7 → pembot-0.0.9}/pembot/query.py

@@ -68,7 +68,8 @@ def multi_embedding_average(llm_client, inference_client, descriptions, model= "
         except Exception as e:
             print(f"Error generating embedding for description '{desc}': {e}")
             # Decide how to handle errors: skip, raise, or use a placeholder
-            continue
+            # continue
+            raise e
         time.sleep(1)
 
     if not description_embeddings:
@@ -81,7 +82,7 @@ def multi_embedding_average(llm_client, inference_client, descriptions, model= "
 
 
 
-def rag_query_llm(db_client, llm_client, inference_client, user_query: str, document_id: str, required_fields_descriptions: list[str], model_name: str = "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B", ollama_base_url: str = "http://localhost:11434", no_of_fields= 4, embedding_model= "BAAI/bge-en-icl", llm_provider_name: PROVIDER_T= "novita", index_name: str= "test_search", embeddings_collection= "doc_chunks"):
+def rag_query_llm(db_client, llm_client, inference_client, user_query: str, document_id: str, required_fields_descriptions: list[str], model_name: str = "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B", ollama_base_url: str = "http://localhost:11434", no_of_fields= 4, embedding_model= "BAAI/bge-en-icl", llm_provider_name: PROVIDER_T= "novita", index_name: str= "test_search", embeddings_collection= "doc_chunks", document_belongs_to_a_type= ""):
     """
     Performs a RAG (Retrieval Augmented Generation) query using a Hugging Face
     embedding model, ChromaDB for retrieval, and a local Ollama model for generation.
@@ -119,10 +120,10 @@ def rag_query_llm(db_client, llm_client, inference_client, user_query: str, docu
     aggregate_query_embedding= multi_embedding_average(llm_client, inference_client, required_fields_descriptions, model= embedding_model, embed_locally= embed_locally)
     print("Aggregate query embedding generated. length: ", len(aggregate_query_embedding))
 
-    create_vector_index(db_client[embeddings_collection], index_name, num_dimensions= len(aggregate_query_embedding))
+    create_vector_index(db_client[embeddings_collection], index_name, num_dimensions= len(aggregate_query_embedding), document_belongs_to_a_type= document_belongs_to_a_type)
 
     # check the order of args
-    relevant_chunks= search_within_document(db_client, aggregate_query_embedding, document_id, limit= no_of_fields, index_name= index_name, embeddings_collection_name= embeddings_collection)
+    relevant_chunks= search_within_document(db_client, aggregate_query_embedding, document_id, limit= no_of_fields, index_name= index_name, embeddings_collection_name= embeddings_collection, document_belongs_to_a_type= document_belongs_to_a_type)
     relevant_chunks= list(map(lambda x: x['chunk_text'], relevant_chunks))
 
     if not relevant_chunks:

{pembot-0.0.7 → pembot-0.0.9}/pembot/requirements.txt

@@ -40,7 +40,7 @@ pandas==2.3.0
 pathlib==1.0.1
 pdfminer.six==20250506
 pdfplumber==0.11.7
-pembot==0.0.6
+pembot==0.0.8
 pillow==11.3.0
 pyasn1==0.6.1
 pyasn1_modules==0.4.2

pembot-0.0.7/pembot/.git/COMMIT_EDITMSG

@@ -1 +0,0 @@
-added model name to convertor

pembot-0.0.7/pembot/.git/refs/heads/main

@@ -1 +0,0 @@
-af80ddb5890f062e364ea8ade2d602df4e12de8c

pembot-0.0.7/pembot/.git/refs/remotes/origin/main

@@ -1 +0,0 @@
-af80ddb5890f062e364ea8ade2d602df4e12de8c