sunholo 0.80.3__py3-none-any.whl → 0.80.5__py3-none-any.whl

sunholo/components/retriever.py

@@ -60,11 +60,14 @@ def pick_retriever(vector_name:str=None, config:ConfigManager=None, embeddings=N
 
                 if vectorstore == "vertex_ai_search" or vectorstore == "discovery_engine":
                     # use direct retriever
+                    if value.get('chunks'):
+                        log.warning(f"{config.vector_name} will not be using GoogleVertexAISearchRetriever with chunks vertex AI search as not supported yet")
+                        continue
                     from langchain.retrievers import GoogleVertexAISearchRetriever
                     gcp_config = config.vacConfig('gcp_config')
                     try:
                         gcp_retriever = GoogleVertexAISearchRetriever(
-                            data_store_id=None if value.get("search_engine_id") else vector_name,
+                            data_store_id=None if value.get("search_engine_id") else config.vector_name,
                             max_documents=value.get('max_documents', 5),
                             project_id=gcp_config.get('project_id') or get_gcp_project(),
                             search_engine_id=value.get("search_engine_id"),
@@ -76,6 +79,7 @@ def pick_retriever(vector_name:str=None, config:ConfigManager=None, embeddings=N
                         continue
                     
                     retriever_list.append(gcp_retriever)
+                    continue
 
                 from_metadata_id = value.get('from_metadata_id')
                 if from_metadata_id:

sunholo/discovery_engine/chunker_handler.py

@@ -1,5 +1,5 @@
 from ..custom_logging import log
-from ..utils.config import load_config_key
+from ..utils import load_config_key, ConfigManager
 from ..utils.gcp_project import get_gcp_project
 from ..components import load_memories
 
@@ -86,8 +86,9 @@ def do_discovery_engine(message_data, metadata, vector_name):
         log.warning("Only gs:// data is supported for Discovery Engine")
 
 
-def check_discovery_engine_in_memory(vector_name):
-    memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")
+def check_discovery_engine_in_memory(config:ConfigManager):
+    memories = config.vacConfig("memory")
+
     for memory in memories:  # Iterate over the list
         for key, value in memory.items():  # Now iterate over the dictionary
             log.info(f"Found memory {key}")
@@ -99,15 +100,32 @@ def check_discovery_engine_in_memory(vector_name):
     
     return False
 
-def discovery_engine_chunker_check(message_data, metadata, vector_name):
+def check_write_memories(config:ConfigManager):
+    write_mem = []
+    memories = config.vacConfig("memory")
+    for memory in memories:
+        for key, value in memory.items():
+            if value.get('read_only'):
+                continue
+            write_mem.append(memory)
+    
+    return write_mem
+
+def discovery_engine_chunker_check(message_data, metadata, vector_name:str=None, config:ConfigManager=None):
+
+    if config is None:
+        if vector_name is None:
+            raise ValueError("Must provide config or vector_name")
+        config = ConfigManager(vector_name=vector_name)
+
     # discovery engine handles its own chunking/embedding
-    memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")
+    memories = config.vacConfig("memory")
     if not memories:
         return None
     
-    total_memories = len(memories)
+    total_memories = len(check_write_memories(config))
     llama = None
-    if check_discovery_engine_in_memory(vector_name):
+    if check_discovery_engine_in_memory(config):
         llama = do_discovery_engine(message_data, metadata, vector_name)
         log.info(f"Processed discovery engine: {llama}")
 

sunholo/gcs/add_file.py

@@ -118,12 +118,13 @@ def add_file_to_gcs(filename: str,
     if os.getenv('EXTENSIONS_BUCKET'):
         bucket_filepath = os.path.basename(filename)
 
-    if vector_name is None:
-            vector_name = "global"
+    if not vector_name:
+        vector_name = "global"
     
     if not bucket_filepath:
         
         bucket_filepath = f"{vector_name}/{year}/{month}/{day}/{hour}/{os.path.basename(filename)}"
+    
     bucket_filepath_prev = f"{vector_name}/{year}/{month}/{day}/{hour_prev}/{os.path.basename(filename)}"
 
     blob = bucket.blob(bucket_filepath)

{sunholo-0.80.3.dist-info → sunholo-0.80.5.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.80.3
+Version: 0.80.5
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.80.3.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.80.5.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.80.3.dist-info → sunholo-0.80.5.dist-info}/RECORD

@@ -55,7 +55,7 @@ sunholo/cli/swagger.py,sha256=absYKAU-7Yd2eiVNUY-g_WLl2zJfeRUNdWQ0oH8M_HM,1564
 sunholo/cli/vertex.py,sha256=8130YCarxHL1UC3aqblNmUwGZTXbkdL4Y_FOnZJsWiI,2056
 sunholo/components/__init__.py,sha256=IDoylb74zFKo6NIS3RQqUl0PDFBGVxM1dfUmO7OJ44U,176
 sunholo/components/llm.py,sha256=5wRVf7lIb7q1vRADNcdQp26L9l4vGHFIvjtUDurZN_s,11488
-sunholo/components/retriever.py,sha256=qhlnhzFeMI41vm_aT8z7rSIksnsBirZepbIpaxoFpp8,7381
+sunholo/components/retriever.py,sha256=hKF3Az6DfDwUaKLkSTuzGg_5THS_lv7C51tfnzhb960,7653
 sunholo/components/vectorstore.py,sha256=xKk7micTRwZckaI7U6PxvFz_ZSjCH48xPTDYiDcv2tc,5913
 sunholo/database/__init__.py,sha256=Zz0Shcq-CtStf9rJGIYB_Ybzb8rY_Q9mfSj-nviM490,241
 sunholo/database/alloydb.py,sha256=c1PEmK9fJCxYaVmKv4emvOoXrajV7KqaVK5mqpeksvM,11527
@@ -71,13 +71,13 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=qUKWzuHApDRJIUoynukVdGRBEq8eC9T7l9a3bWckgI0,59
-sunholo/discovery_engine/chunker_handler.py,sha256=kCprXDwb-6BQkuVkP2SVEY2_UZbo7xF7NAnPAC_3xrs,4616
+sunholo/discovery_engine/chunker_handler.py,sha256=NgLCnnJ3mcrvy4BPNxmBxJX4BOYXNdLKf2YxC5TwX0o,5088
 sunholo/discovery_engine/create_new.py,sha256=7oZG78T6lW0EspRzlo7-qRyXFSuFxDn2dfSAVEaqlqY,978
 sunholo/discovery_engine/discovery_engine_client.py,sha256=oORB2SVVqrYrz7E3srPrknyuR6Dl3SJJwaVrbVXJER4,17726
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=MCbTePWjUbIRVDFFhHJ94BvOZvIom62-mTr0PmfQyt0,6951
 sunholo/gcs/__init__.py,sha256=SZvbsMFDko40sIRHTHppA37IijvJTae54vrhooEF5-4,90
-sunholo/gcs/add_file.py,sha256=l04xuFoVzw1AbjME_cABvMVZ7bnrT2JHNG04OuF25zU,7127
+sunholo/gcs/add_file.py,sha256=0GruAKsvVO9qVddwJ1ugr4ldpk_QKXmhKKVio2QwuPE,7124
 sunholo/gcs/download_folder.py,sha256=ijJTnS595JqZhBH8iHFErQilMbkuKgL-bnTCMLGuvlA,1614
 sunholo/gcs/download_url.py,sha256=i_LKd3fJQNDqpUzDgSSehWVSzOPA-HPM7o0Tf8nLrM4,5235
 sunholo/gcs/metadata.py,sha256=oQLcXi4brsZ74aegWyC1JZmhlaEV270HS5_UWtAYYWE,898
@@ -133,9 +133,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=pgSahVDh7GPEulu3nl-w0jb5lTClb4TCnVxPnMokNZY,7533
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.80.3.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.80.3.dist-info/METADATA,sha256=wF5hQP2bHEi51bX7ECH7mwCaURtuqCOa95jxgVGiKD4,7348
-sunholo-0.80.3.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-sunholo-0.80.3.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.80.3.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.80.3.dist-info/RECORD,,
+sunholo-0.80.5.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.80.5.dist-info/METADATA,sha256=VkZcaejfXjHhZxi3Lkhwb0_MuScBe27ZfS6sgvEVm-Q,7348
+sunholo-0.80.5.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+sunholo-0.80.5.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.80.5.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.80.5.dist-info/RECORD,,