sunholo 0.69.3__tar.gz → 0.69.5__tar.gz

{sunholo-0.69.3 → sunholo-0.69.5}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.3
+Version: 0.69.5
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.3.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.5.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.3 → sunholo-0.69.5}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.69.3'
+version = '0.69.5'
 
 setup(
     name='sunholo',

{sunholo-0.69.3 → sunholo-0.69.5}/sunholo/discovery_engine/chunker_handler.py

@@ -3,6 +3,7 @@ from ..utils.config import load_config_key
 from ..components import load_memories
 
 from .discovery_engine_client import DiscoveryEngineClient
+from .create_new import create_new_discovery_engine
     
 
 def do_discovery_engine(message_data, metadata, vector_name):
@@ -19,14 +20,11 @@ def do_discovery_engine(message_data, metadata, vector_name):
     ```
     """
 
+    global_gcp_config = load_config_key("gcp_config", vector_name="global", kind="vacConfig")
     gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
-    if not gcp_config:
+    if not gcp_config and not global_gcp_config:
         raise ValueError(f"Need config.{vector_name}.gcp_config to configure discovery engine")
 
-    global_project_id = gcp_config.get('project_id')
-    #global_location = gcp_config.get('location')
-    global_data_store_id = gcp_config.get('data_store_id')
-
     memories = load_memories(vector_name)
     tools = []
 
@@ -40,12 +38,11 @@ def do_discovery_engine(message_data, metadata, vector_name):
             vectorstore = value.get('vectorstore')
             if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                 log.info(f"Found vectorstore {vectorstore}")
-                data_store_id = value.get('data_store_id')
-                project_id = gcp_config.get('project_id')
+                project_id = gcp_config.get('project_id') or global_gcp_config['project_id']
                 #location = gcp_config.get('location') 
                 corpus = DiscoveryEngineClient(
-                    data_store_id=data_store_id or global_data_store_id, 
-                    project_id=project_id or global_project_id,
+                    data_store_id=vector_name, 
+                    project_id=project_id,
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
                     #location=location or global_location
                     )
@@ -67,6 +64,20 @@ def do_discovery_engine(message_data, metadata, vector_name):
                 log.info(f"Imported file to corpus: {response} with metadata: {metadata}")
             except Exception as err:
                 log.error(f"Error importing {message_data} - {corp=} - {str(err)}")
+
+                if str(err).startswith("404"):
+                    log.info(f"Attempting to create a new DiscoveryEngine corpus: {vector_name}")
+                    try:
+                        new_corp = create_new_discovery_engine(vector_name)
+                    except Exception as err:
+                        log.error(f"Failed to create new DiscoveryEngine {vector_name} - {str(err)}")
+                        continue
+                    if new_corp:
+                        log.info(f"Found new DiscoveryEngine {vector_name=} - {new_corp=}")
+                        response = corp.import_documents(
+                            gcs_uri=message_data
+                        )
+                    
                 continue
 
         metadata["source"] = message_data

{sunholo-0.69.3 → sunholo-0.69.5}/sunholo/discovery_engine/discovery_engine_client.py

@@ -61,16 +61,13 @@ class DiscoveryEngineClient:
             if location != "global"
             else None
         )
-        self.client = discoveryengine.DataStoreServiceClient(client_options=client_options)
-        self.parent = self.client.branch_path(
-            project=project_id,
-            location=location,
-            data_store=data_store_id,
-            branch="default_branch",
-        )
+        self.store_client  = discoveryengine.DataStoreServiceClient(client_options=client_options)
+        self.doc_client    = discoveryengine.DocumentServiceClient(client_options=client_options)
+        self.search_client = discoveryengine.SearchServiceClient(client_options=client_options)
 
     def create_data_store(
-        self, chunk_size: int = 500
+        self, chunk_size: int = 500,
+        collection: str = "default_collection"
     ) -> str:
         """
         Creates a new data store with default configuration.
@@ -109,9 +106,15 @@ class DiscoveryEngineClient:
             document_processing_config=doc_config
         )
 
+        parent = self.store_client.collection_path(
+            project=self.project_id,
+            location=self.location,
+            collection=collection,
+        )
+
         # https://cloud.google.com/python/docs/reference/discoveryengine/0.11.4/google.cloud.discoveryengine_v1alpha.types.CreateDataStoreRequest
         request = discoveryengine.CreateDataStoreRequest(
-            parent=self.parent,
+            parent=parent,
             data_store_id=self.data_store_id,
             data_store=data_store,
             # Optional: For Advanced Site Search Only
@@ -119,10 +122,11 @@ class DiscoveryEngineClient:
         )
 
         # Make the request
-        operation = self.client.create_data_store(request=request)
+        operation = self.store_client.create_data_store(request=request)
 
-        log.info(f"Waiting for operation to complete: {operation.operation.name}")
+        log.info(f"Waiting for datastore operation to complete: {operation.operation.name}")
         response = operation.result()
+        log.info(f"Datastore operation creation complete: {response=}")
 
         # Once the operation is complete,
         # get information from operation metadata
@@ -140,6 +144,7 @@ class DiscoveryEngineClient:
         num_next_chunks: int = 3,
         page_size: int = 10,
         doc_or_chunks: str = "CHUNKS",  # or DOCUMENTS
+        serving_config: str = "default_serving_config",
     ):
         """Retrieves chunks or documents based on a query.
 
@@ -161,22 +166,19 @@ class DiscoveryEngineClient:
                     print(f"Chunk: {chunk.snippet}, document name: {chunk.document_name}")
             ```
         """
-        serving_config = self.client.get_default_serving_config(
-            name=self.client.serving_config_path(
-                project=self.project_id, 
-                location=self.location, 
-                data_store=self.data_store_id, 
-                serving_config="default_serving_config")
-                ).name
-        
+
+        serving_config_path = self.search_client.serving_config_path(
+            self.project_id,
+            self.location,
+            self.data_store_id,
+            serving_config
+        )
+
         search_request = discoveryengine.SearchRequest(
-            serving_config=serving_config,
+            serving_config=serving_config_path,
             query=query,
             page_size=page_size, 
             content_search_spec=discoveryengine.SearchRequest.ContentSearchSpec(
-                #snippet_spec=discoveryengine.SearchRequest.ContentSearchSpec.SnippetSpec(
-                #    return_snippet=True
-                #),
                 search_result_mode=doc_or_chunks,  # CHUNKS or DOCUMENTS
                 chunk_spec=discoveryengine.SearchRequest.ContentSearchSpec.ChunkSpec(
                     num_previous_chunks=num_previous_chunks,
@@ -185,13 +187,14 @@ class DiscoveryEngineClient:
             ),
         )
 
-        search_response = self.client.search(search_request)
+        search_response = self.search_client.search(search_request)
 
         return search_response
 
     def import_documents(self,
         gcs_uri: Optional[str] = None,
         data_schema="content",
+        branch="default_branch",
         bigquery_dataset: Optional[str] = None,
         bigquery_table: Optional[str] = None,
         bigquery_project_id: Optional[str] = None,
@@ -203,9 +206,16 @@ class DiscoveryEngineClient:
         
         """
 
+        parent = self.doc_client.branch_path(
+            self.project_id, 
+            self.location, 
+            self.data_store_id, 
+            branch
+        )
+
         if gcs_uri:
             request = discoveryengine.ImportDocumentsRequest(
-                parent=self.parent,
+                parent=parent,
                 # https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1alpha.types.GcsSource
                 gcs_source=discoveryengine.GcsSource(
                     input_uris=[gcs_uri], data_schema=data_schema,
@@ -215,7 +225,7 @@ class DiscoveryEngineClient:
             )
         else:
             request = discoveryengine.ImportDocumentsRequest(
-                parent=self.parent,
+                parent=parent,
                 bigquery_source=discoveryengine.BigQuerySource(
                     project_id=bigquery_project_id or self.project_id,
                     dataset_id=bigquery_dataset,
@@ -227,7 +237,7 @@ class DiscoveryEngineClient:
             )
 
         # Make the request
-        operation = self.client.import_documents(request=request)
+        operation = self.doc_client.import_documents(request=request)
 
         log.info(f"Waiting for operation to complete: {operation.operation.name}")
         response = operation.result()

{sunholo-0.69.3 → sunholo-0.69.5}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.3
+Version: 0.69.5
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.3.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.5.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0