sunholo 0.69.6__tar.gz → 0.69.9__tar.gz

{sunholo-0.69.6 → sunholo-0.69.9}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.6
+Version: 0.69.9
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.6.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.9.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.6 → sunholo-0.69.9}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.69.6'
+version = '0.69.9'
 
 setup(
     name='sunholo',

{sunholo-0.69.6 → sunholo-0.69.9}/sunholo/discovery_engine/chunker_handler.py

@@ -1,5 +1,6 @@
 from ..logging import log
 from ..utils.config import load_config_key
+from ..utils.gcp_project import get_gcp_project
 from ..components import load_memories
 
 from .discovery_engine_client import DiscoveryEngineClient
@@ -20,11 +21,6 @@ def do_discovery_engine(message_data, metadata, vector_name):
     ```
     """
 
-    global_gcp_config = load_config_key("gcp_config", vector_name="global", kind="vacConfig")
-    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
-    if not gcp_config and not global_gcp_config:
-        raise ValueError(f"Need config.{vector_name}.gcp_config to configure discovery engine")
-
     memories = load_memories(vector_name)
     tools = []
 
@@ -38,11 +34,10 @@ def do_discovery_engine(message_data, metadata, vector_name):
             vectorstore = value.get('vectorstore')
             if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                 log.info(f"Found vectorstore {vectorstore}")
-                project_id = gcp_config.get('project_id') or global_gcp_config['project_id']
                 #location = gcp_config.get('location') 
                 corpus = DiscoveryEngineClient(
                     data_store_id=vector_name, 
-                    project_id=project_id,
+                    project_id=get_gcp_project(),
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
                     #location=location or global_location
                     )

{sunholo-0.69.6 → sunholo-0.69.9}/sunholo/discovery_engine/create_new.py

@@ -1,9 +1,8 @@
 from .discovery_engine_client import DiscoveryEngineClient
 from ..utils.config import load_config_key
+from ..utils.gcp_project import get_gcp_project
 
 def create_new_discovery_engine(vector_name):
-    global_config = load_config_key("gcp_config", "global", kind="vacConfig")
-    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
 
     chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
 
@@ -12,7 +11,7 @@ def create_new_discovery_engine(vector_name):
         if "chunk_size" in chunker_config:
             chunk_size = chunker_config["chunk_size"]    
 
-    project_id = gcp_config.get('project_id') or global_config.get('project_id')
+    project_id = get_gcp_project()
     if not project_id:
         raise ValueError("Could not find project_id in gcp_config")
     

{sunholo-0.69.6 → sunholo-0.69.9}/sunholo/discovery_engine/discovery_engine_client.py

@@ -1,6 +1,8 @@
 try:
     from google.api_core.client_options import ClientOptions
     from google.cloud import discoveryengine_v1alpha as discoveryengine
+    from google.api_core.retry import Retry, if_exception_type
+    from google.api_core.exceptions import ResourceExhausted
 except ImportError:
     ClientOptions = None
     discoveryengine = None
@@ -65,6 +67,16 @@ class DiscoveryEngineClient:
         self.doc_client    = discoveryengine.DocumentServiceClient(client_options=client_options)
         self.search_client = discoveryengine.SearchServiceClient(client_options=client_options)
 
+    @classmethod
+    def my_retry(cls):
+        return Retry(
+            initial=1.0,       # Initial delay before retrying (in seconds)
+            maximum=60.0,      # Maximum delay between retries (in seconds)
+            multiplier=2.0,    # Multiplier for the delay between retries
+            timeout=300.0,     # Maximum total time to wait before giving up (in seconds)
+            predicate=if_exception_type(ResourceExhausted)  # Retry if a ResourceExhausted error occurs
+        )
+    
     def create_data_store(
         self, chunk_size: int = 500,
         collection: str = "default_collection"
@@ -237,17 +249,16 @@ class DiscoveryEngineClient:
             )
 
         # Make the request
-        operation = self.doc_client.import_documents(request=request)
-
-        log.info(f"Waiting for operation to complete: {operation.operation.name}")
-        response = operation.result()
-
-        # Once the operation is complete,
-        # get information from operation metadata
-        metadata = discoveryengine.ImportDocumentsMetadata(operation.metadata)
-
-        # Handle the response
-        log.info(f"{response=} {metadata=}")
+        @self.my_retry()
+        def import_documents_with_retry(doc_client, request):
+            return doc_client.import_documents(request=request)
+        
+        try:
+            operation = import_documents_with_retry(self.doc_client, request)
+        except ResourceExhausted as e:
+            print(f"Operation failed after retries due to quota exceeded: {e}")
+        except Exception as e:
+            print(f"An unexpected error occurred: {e}")
 
         return operation.operation.name
 

{sunholo-0.69.6 → sunholo-0.69.9}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.6
+Version: 0.69.9
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.6.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.9.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0