sunholo 0.69.4__py3-none-any.whl → 0.69.5__py3-none-any.whl

sunholo/discovery_engine/chunker_handler.py

@@ -3,6 +3,7 @@ from ..utils.config import load_config_key
 from ..components import load_memories
 
 from .discovery_engine_client import DiscoveryEngineClient
+from .create_new import create_new_discovery_engine
     
 
 def do_discovery_engine(message_data, metadata, vector_name):
@@ -19,14 +20,11 @@ def do_discovery_engine(message_data, metadata, vector_name):
     ```
     """
 
+    global_gcp_config = load_config_key("gcp_config", vector_name="global", kind="vacConfig")
     gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
-    if not gcp_config:
+    if not gcp_config and not global_gcp_config:
         raise ValueError(f"Need config.{vector_name}.gcp_config to configure discovery engine")
 
-    global_project_id = gcp_config.get('project_id')
-    #global_location = gcp_config.get('location')
-    global_data_store_id = gcp_config.get('data_store_id')
-
     memories = load_memories(vector_name)
     tools = []
 
@@ -40,12 +38,11 @@ def do_discovery_engine(message_data, metadata, vector_name):
             vectorstore = value.get('vectorstore')
             if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                 log.info(f"Found vectorstore {vectorstore}")
-                data_store_id = value.get('data_store_id')
-                project_id = gcp_config.get('project_id')
+                project_id = gcp_config.get('project_id') or global_gcp_config['project_id']
                 #location = gcp_config.get('location') 
                 corpus = DiscoveryEngineClient(
-                    data_store_id=data_store_id or global_data_store_id, 
-                    project_id=project_id or global_project_id,
+                    data_store_id=vector_name, 
+                    project_id=project_id,
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
                     #location=location or global_location
                     )
@@ -67,6 +64,20 @@ def do_discovery_engine(message_data, metadata, vector_name):
                 log.info(f"Imported file to corpus: {response} with metadata: {metadata}")
             except Exception as err:
                 log.error(f"Error importing {message_data} - {corp=} - {str(err)}")
+
+                if str(err).startswith("404"):
+                    log.info(f"Attempting to create a new DiscoveryEngine corpus: {vector_name}")
+                    try:
+                        new_corp = create_new_discovery_engine(vector_name)
+                    except Exception as err:
+                        log.error(f"Failed to create new DiscoveryEngine {vector_name} - {str(err)}")
+                        continue
+                    if new_corp:
+                        log.info(f"Found new DiscoveryEngine {vector_name=} - {new_corp=}")
+                        response = corp.import_documents(
+                            gcs_uri=message_data
+                        )
+                    
                 continue
 
         metadata["source"] = message_data

sunholo/discovery_engine/discovery_engine_client.py

@@ -124,8 +124,9 @@ class DiscoveryEngineClient:
         # Make the request
         operation = self.store_client.create_data_store(request=request)
 
-        log.info(f"Waiting for operation to complete: {operation.operation.name}")
+        log.info(f"Waiting for datastore operation to complete: {operation.operation.name}")
         response = operation.result()
+        log.info(f"Datastore operation creation complete: {response=}")
 
         # Once the operation is complete,
         # get information from operation metadata

{sunholo-0.69.4.dist-info → sunholo-0.69.5.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.4
+Version: 0.69.5
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.4.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.5.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.4.dist-info → sunholo-0.69.5.dist-info}/RECORD

@@ -61,9 +61,9 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sunholo/discovery_engine/chunker_handler.py,sha256=4NT4fRAWY-kKM4X3imZ2RISYLMvpIVsPDpsAgUOhdlA,4149
+sunholo/discovery_engine/chunker_handler.py,sha256=eqRI8R-AVIhtqpzw62glCucvX_r1MXzdrF_DAKcPhtM,4793
 sunholo/discovery_engine/create_new.py,sha256=6sowAvblLDBZq6skYwdc6n99YlyHvNLaIs6kYMnkdO8,921
-sunholo/discovery_engine/discovery_engine_client.py,sha256=yjMu_MXYzXzQMSxp_PATDcM7PXlB8VTgJT56fRbD0xs,11693
+sunholo/discovery_engine/discovery_engine_client.py,sha256=PDoRO-6uc9u3lYUy6oUc-jbgxijfJMNaitScoz5nUCk,11775
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=P744zUQJgqrjILunzaqtTerB9AwoXFU6tXBtz4rjWgQ,6673
 sunholo/gcs/__init__.py,sha256=DtVw_AZwQn-IguR5BJuIi2XJeF_FQXizhJikzRNrXiE,50
@@ -111,9 +111,9 @@ sunholo/vertex/__init__.py,sha256=JvHcGFuv6R_nAhY2AdoqqhMpJ5ugeWPZ_svGhWrObBk,13
 sunholo/vertex/init.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
 sunholo/vertex/memory_tools.py,sha256=8F1iTWnqEK9mX4W5RzCVKIjydIcNp6OFxjn_dtQ3GXo,5379
 sunholo/vertex/safety.py,sha256=3meAX0HyGZYrH7rXPUAHxtI_3w_zoy_RX7Shtkoa660,1275
-sunholo-0.69.4.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.69.4.dist-info/METADATA,sha256=gaLbxVufOz5-8Juy1fMBHvw_BXCv5MiUPoSMI1hJotQ,6155
-sunholo-0.69.4.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-sunholo-0.69.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.69.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.69.4.dist-info/RECORD,,
+sunholo-0.69.5.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.69.5.dist-info/METADATA,sha256=z_-h8LDKsg8u64WDv7MY41Id1MYczH7HJabwdfOeW5Y,6155
+sunholo-0.69.5.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+sunholo-0.69.5.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.69.5.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.69.5.dist-info/RECORD,,