sunholo 0.69.5__py3-none-any.whl → 0.69.8__py3-none-any.whl

sunholo/discovery_engine/create_new.py

@@ -2,18 +2,20 @@ from .discovery_engine_client import DiscoveryEngineClient
 from ..utils.config import load_config_key
 
 def create_new_discovery_engine(vector_name):
+    global_config = load_config_key("gcp_config", "global", kind="vacConfig")
     gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
 
     chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
 
+    chunk_size = 500
     if chunker_config:
-        chunk_size = chunker_config.get("chunk_size")
-    
-    if not chunk_size:
-        chunk_size = 500
-
+        if "chunk_size" in chunker_config:
+            chunk_size = chunker_config["chunk_size"]    
 
-    project_id = gcp_config.get('project_id')
+    project_id = gcp_config.get('project_id') or global_config.get('project_id')
+    if not project_id:
+        raise ValueError("Could not find project_id in gcp_config")
+    
     #location = gcp_config.get('location')
 
     de = DiscoveryEngineClient(

sunholo/discovery_engine/discovery_engine_client.py

@@ -1,6 +1,8 @@
 try:
     from google.api_core.client_options import ClientOptions
     from google.cloud import discoveryengine_v1alpha as discoveryengine
+    from google.api_core.retry import Retry, if_exception_type
+    from google.api_core.exceptions import ResourceExhausted
 except ImportError:
     ClientOptions = None
     discoveryengine = None
@@ -65,6 +67,16 @@ class DiscoveryEngineClient:
         self.doc_client    = discoveryengine.DocumentServiceClient(client_options=client_options)
         self.search_client = discoveryengine.SearchServiceClient(client_options=client_options)
 
+    @classmethod
+    def my_retry(cls):
+        return Retry(
+            initial=1.0,       # Initial delay before retrying (in seconds)
+            maximum=60.0,      # Maximum delay between retries (in seconds)
+            multiplier=2.0,    # Multiplier for the delay between retries
+            timeout=300.0,     # Maximum total time to wait before giving up (in seconds)
+            predicate=if_exception_type(ResourceExhausted)  # Retry if a ResourceExhausted error occurs
+        )
+    
     def create_data_store(
         self, chunk_size: int = 500,
         collection: str = "default_collection"
@@ -237,17 +249,16 @@ class DiscoveryEngineClient:
             )
 
         # Make the request
-        operation = self.doc_client.import_documents(request=request)
-
-        log.info(f"Waiting for operation to complete: {operation.operation.name}")
-        response = operation.result()
-
-        # Once the operation is complete,
-        # get information from operation metadata
-        metadata = discoveryengine.ImportDocumentsMetadata(operation.metadata)
-
-        # Handle the response
-        log.info(f"{response=} {metadata=}")
+        @self.my_retry()
+        def import_documents_with_retry(doc_client, request):
+            return doc_client.import_documents(request=request)
+        
+        try:
+            operation = import_documents_with_retry(self.doc_client, request)
+        except ResourceExhausted as e:
+            print(f"Operation failed after retries due to quota exceeded: {e}")
+        except Exception as e:
+            print(f"An unexpected error occurred: {e}")
 
         return operation.operation.name
 

{sunholo-0.69.5.dist-info → sunholo-0.69.8.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.5
+Version: 0.69.8
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.5.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.8.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.5.dist-info → sunholo-0.69.8.dist-info}/RECORD

@@ -62,8 +62,8 @@ sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUt
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/discovery_engine/chunker_handler.py,sha256=eqRI8R-AVIhtqpzw62glCucvX_r1MXzdrF_DAKcPhtM,4793
-sunholo/discovery_engine/create_new.py,sha256=6sowAvblLDBZq6skYwdc6n99YlyHvNLaIs6kYMnkdO8,921
-sunholo/discovery_engine/discovery_engine_client.py,sha256=PDoRO-6uc9u3lYUy6oUc-jbgxijfJMNaitScoz5nUCk,11775
+sunholo/discovery_engine/create_new.py,sha256=31NK8ZzgKURFT2e1kBJCiyfFxfP6I_WCSbcjFpbp_fM,1144
+sunholo/discovery_engine/discovery_engine_client.py,sha256=UYMhXtGWXuyyg4fA1piVjqfrWXkSP_MFimh6NzaUlZ8,12413
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=P744zUQJgqrjILunzaqtTerB9AwoXFU6tXBtz4rjWgQ,6673
 sunholo/gcs/__init__.py,sha256=DtVw_AZwQn-IguR5BJuIi2XJeF_FQXizhJikzRNrXiE,50
@@ -111,9 +111,9 @@ sunholo/vertex/__init__.py,sha256=JvHcGFuv6R_nAhY2AdoqqhMpJ5ugeWPZ_svGhWrObBk,13
 sunholo/vertex/init.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
 sunholo/vertex/memory_tools.py,sha256=8F1iTWnqEK9mX4W5RzCVKIjydIcNp6OFxjn_dtQ3GXo,5379
 sunholo/vertex/safety.py,sha256=3meAX0HyGZYrH7rXPUAHxtI_3w_zoy_RX7Shtkoa660,1275
-sunholo-0.69.5.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.69.5.dist-info/METADATA,sha256=z_-h8LDKsg8u64WDv7MY41Id1MYczH7HJabwdfOeW5Y,6155
-sunholo-0.69.5.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-sunholo-0.69.5.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.69.5.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.69.5.dist-info/RECORD,,
+sunholo-0.69.8.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.69.8.dist-info/METADATA,sha256=SEe57-886plXFH-T-Kkn2acgw4WlJKYqh3Ab6hJzKMQ,6155
+sunholo-0.69.8.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+sunholo-0.69.8.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.69.8.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.69.8.dist-info/RECORD,,