sunholo 0.80.5__py3-none-any.whl → 0.80.6__py3-none-any.whl

sunholo/chunker/process_chunker_data.py

@@ -15,6 +15,7 @@ from ..discovery_engine.chunker_handler import discovery_engine_chunker_check
 from .publish import process_docs_chunks_vector_name
 from .splitter import chunk_doc_to_docs
 from ..azure.blobs import is_azure_blob
+from ..utils import ConfigManager
 
 from ..custom_logging import log
 
@@ -29,13 +30,15 @@ def process_chunker_data(message_data, metadata, vector_name):
 
     log.debug(f"Found metadata in pubsub: {metadata=}")
 
+    config=ConfigManager(vector_name)
+
     # checks if only a llamaindex chunking/embedder, return early as no other processing needed
     llamacheck = llamaindex_chunker_check(message_data, metadata, vector_name)
     if llamacheck:
         return llamacheck
     
     # if only a discovery engine memory, return early as no other processing needed
-    discovery_check = discovery_engine_chunker_check(message_data, metadata, vector_name)
+    discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config)
     if discovery_check:
         return discovery_check
 

sunholo/discovery_engine/__init__.py

@@ -1 +1,2 @@
 from .discovery_engine_client import DiscoveryEngineClient
+from .get_ai_search_chunks import get_all_chunks

sunholo/discovery_engine/chunker_handler.py

@@ -1,5 +1,5 @@
 from ..custom_logging import log
-from ..utils import load_config_key, ConfigManager
+from ..utils import ConfigManager
 from ..utils.gcp_project import get_gcp_project
 from ..components import load_memories
 
@@ -7,7 +7,7 @@ from .discovery_engine_client import DiscoveryEngineClient
 from .create_new import create_new_discovery_engine
     
 
-def do_discovery_engine(message_data, metadata, vector_name):
+def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=None):
     """
 
     Example:
@@ -15,13 +15,13 @@ def do_discovery_engine(message_data, metadata, vector_name):
     message_data = "gs://bucket_name/path_to_file.txt"
     metadata = {"user": "admin"}
     vector_name = "example_vector"
-    response = do_discovery_engine(message_data, metadata, vector_name)
+    response = do_discovery_engine(message_data, metadata, config=config)
     print(response)
     # Imported file to corpus: {'status': 'success'}
     ```
     """
 
-    memories = load_memories(vector_name)
+    memories = load_memories(config=config)
     tools = []
 
     if not memories:
@@ -38,7 +38,7 @@ def do_discovery_engine(message_data, metadata, vector_name):
                     continue
                 #location = gcp_config.get('location') 
                 corpus = DiscoveryEngineClient(
-                    data_store_id=vector_name, 
+                    data_store_id=config.vector_name, 
                     project_id=get_gcp_project(),
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
                     #location=location or global_location
@@ -65,14 +65,14 @@ def do_discovery_engine(message_data, metadata, vector_name):
                 log.error(f"Error importing {message_data} - {corp=} - {str(err)}")
 
                 if str(err).startswith("404"):
-                    log.info(f"Attempting to create a new DiscoveryEngine corpus: {vector_name}")
+                    log.info(f"Attempting to create a new DiscoveryEngine corpus: {config.vector_name}")
                     try:
-                        new_corp = create_new_discovery_engine(vector_name)
+                        new_corp = create_new_discovery_engine(config)
                     except Exception as err:
-                        log.error(f"Failed to create new DiscoveryEngine {vector_name} - {str(err)}")
+                        log.error(f"Failed to create new DiscoveryEngine {config.vector_name} - {str(err)}")
                         continue
                     if new_corp:
-                        log.info(f"Found new DiscoveryEngine {vector_name=} - {new_corp=}")
+                        log.info(f"Found new DiscoveryEngine {config.vector_name=} - {new_corp=}")
                         response = corp.import_documents(
                             gcs_uri=message_data
                         )
@@ -126,7 +126,7 @@ def discovery_engine_chunker_check(message_data, metadata, vector_name:str=None,
     total_memories = len(check_write_memories(config))
     llama = None
     if check_discovery_engine_in_memory(config):
-        llama = do_discovery_engine(message_data, metadata, vector_name)
+        llama = do_discovery_engine(message_data, metadata, config=config)
         log.info(f"Processed discovery engine: {llama}")
 
     # If discovery engine is the only entry, return

sunholo/discovery_engine/create_new.py

@@ -1,10 +1,10 @@
 from .discovery_engine_client import DiscoveryEngineClient
-from ..utils.config import load_config_key
+from ..utils import ConfigManager
 from ..utils.gcp_project import get_gcp_project
 
-def create_new_discovery_engine(vector_name):
+def create_new_discovery_engine(config:ConfigManager):
 
-    chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
+    chunker_config = config.vacConfig("chunker")
 
     chunk_size = 500
     if chunker_config:
@@ -18,7 +18,7 @@ def create_new_discovery_engine(vector_name):
     #location = gcp_config.get('location')
 
     de = DiscoveryEngineClient(
-                    data_store_id=vector_name, 
+                    data_store_id=config.vector_name, 
                     project_id=project_id,
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
                     #location=location

sunholo/discovery_engine/get_ai_search_chunks.py

@@ -0,0 +1,53 @@
+from ..utils import ConfigManager
+from ..utils.gcp_project  import get_gcp_project
+from ..custom_logging import log
+from .discovery_engine_client import DiscoveryEngineClient
+from ..components import load_memories
+
+def get_all_chunks(question:str, config:ConfigManager):
+    """
+    Look through a config memory key and find all Vertex AI Search retrievers, call them and return a joined string of chunks
+
+        args: question - question to search similarity for
+        config: A ConfigManager object
+
+        returns: a big string of chunks
+    """
+    memories = load_memories(config=config)
+    chunks = []
+
+    if not memories:
+        return None
+
+    vector_name = config.vector_name
+    for memory in memories:
+        for key, value in memory.items():  # Now iterate over the dictionary
+            log.info(f"Found memory {key}")
+            vectorstore = value.get('vectorstore')
+            if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
+                if value.get('read_only'):
+                    new_vector_name = value.get('vector_name')
+                    if not new_vector_name:
+                        log.warning("read_only specified but no new vector_name to read from")
+                    vector_name = new_vector_name
+                
+                num_chunks = value.get('num_chunks') or 3
+
+                chunk = get_chunks(question, vector_name, num_chunks)
+                if chunk:
+                    chunks.append(chunk)
+    if chunks:
+        return "\n".join(chunks)
+
+    log.warning(f"No chunks found for {vector_name}")
+    return None
+
+def get_chunks(question, vector_name, num_chunks):
+    de = DiscoveryEngineClient(vector_name, project_id=get_gcp_project())
+    try:
+        return de.get_chunks(question, num_previous_chunks=num_chunks, num_next_chunks=num_chunks)
+    except Exception as err:
+        log.error(f"No discovery engine chunks found: {str(err)}")
+    
+
+

{sunholo-0.80.5.dist-info → sunholo-0.80.6.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.80.5
+Version: 0.80.6
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.80.5.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.80.6.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.80.5.dist-info → sunholo-0.80.6.dist-info}/RECORD

@@ -37,7 +37,7 @@ sunholo/chunker/images.py,sha256=id2PBu6XyGEOtgafq2v0c9_O6kxaC_pYFMnbsIitkSg,186
 sunholo/chunker/loaders.py,sha256=CCB0IGigNAWT__2ImVin_j83W3eGS2Qe5I6U18YQzoM,10275
 sunholo/chunker/message_data.py,sha256=EaiY7_HClpcfPUAYaAm6Zk5ReeZ9s9F_jBVd0kDgI-4,10836
 sunholo/chunker/pdfs.py,sha256=njDPop751GMHi3cOwIKd2Yct-_lWR2gqcB7WykfHphs,2480
-sunholo/chunker/process_chunker_data.py,sha256=OnMvXHRv3rGpFsU50FyUNkNIwC1D8TkhaWWbn72yQss,3523
+sunholo/chunker/process_chunker_data.py,sha256=uO-YOEHIjAOy0ZMJ0vea9OMNsQBISHfhbtgoyuHiP6s,3598
 sunholo/chunker/publish.py,sha256=AX5u-fcyDytED67IfizMzvOMcYPXEo6XBJvyk_7maK8,2939
 sunholo/chunker/pubsub.py,sha256=48bhuAcszN7LGe3-ksPSLHHhq0uKxiXOrizck5qpcP0,1012
 sunholo/chunker/splitter.py,sha256=QLAEsJOpEYFZr9-UGZUuAlNVyjfCWb8jvzCHg0rVShE,6751
@@ -70,10 +70,11 @@ sunholo/database/sql/sb/create_table.sql,sha256=SbcOrf5tUiVKGUohu1lau7IsbDRbTFbr
 sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjsmfHZNNOo1ptwLLSo,75
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
-sunholo/discovery_engine/__init__.py,sha256=qUKWzuHApDRJIUoynukVdGRBEq8eC9T7l9a3bWckgI0,59
-sunholo/discovery_engine/chunker_handler.py,sha256=NgLCnnJ3mcrvy4BPNxmBxJX4BOYXNdLKf2YxC5TwX0o,5088
-sunholo/discovery_engine/create_new.py,sha256=7oZG78T6lW0EspRzlo7-qRyXFSuFxDn2dfSAVEaqlqY,978
+sunholo/discovery_engine/__init__.py,sha256=P00bB8aVVWefOZbCQvzHsVMuP_sd-_d_4o5xCuCpN3g,108
+sunholo/discovery_engine/chunker_handler.py,sha256=Fv4BLOBi_7ap3AiAy4TlTN48CLZSMurJ3TkvC75Euro,5123
+sunholo/discovery_engine/create_new.py,sha256=NzhSh6nG6nQ5J9gZh8IDph4JiEVT_DC5GGvP0GuwTWs,943
 sunholo/discovery_engine/discovery_engine_client.py,sha256=oORB2SVVqrYrz7E3srPrknyuR6Dl3SJJwaVrbVXJER4,17726
+sunholo/discovery_engine/get_ai_search_chunks.py,sha256=7yMpGaLU1nL3ttFLE-cIhNao-Vq9SY1edFC3T-7wH2I,1944
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=MCbTePWjUbIRVDFFhHJ94BvOZvIom62-mTr0PmfQyt0,6951
 sunholo/gcs/__init__.py,sha256=SZvbsMFDko40sIRHTHppA37IijvJTae54vrhooEF5-4,90
@@ -133,9 +134,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=pgSahVDh7GPEulu3nl-w0jb5lTClb4TCnVxPnMokNZY,7533
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.80.5.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.80.5.dist-info/METADATA,sha256=VkZcaejfXjHhZxi3Lkhwb0_MuScBe27ZfS6sgvEVm-Q,7348
-sunholo-0.80.5.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-sunholo-0.80.5.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.80.5.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.80.5.dist-info/RECORD,,
+sunholo-0.80.6.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.80.6.dist-info/METADATA,sha256=CQEUn95SrlED6d_I0m_V-ZhLHTOXD0nVugPZbWl3N4k,7348
+sunholo-0.80.6.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+sunholo-0.80.6.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.80.6.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.80.6.dist-info/RECORD,,