sunholo 0.69.16__py3-none-any.whl → 0.70.0__py3-none-any.whl

sunholo/chunker/data_to_embed_pubsub.py

@@ -60,10 +60,10 @@ def process_chunker_data(message_data, metadata, vector_name):
         metadata["vector_name"] = vector_name
 
     if message_data is None:
-        log.error(f"No message_data was found in data: {message_data}")
+        log.warning(f"No message_data was found in data: {metadata=}")
         return
 
-    log.debug(f"Found metadata in pubsub: {metadata}")
+    log.debug(f"Found metadata in pubsub: {metadata=}")
 
     # checks if only a llamaindex chunking/embedder, return early as no other processing needed
     llamacheck = llamaindex_chunker_check(message_data, metadata, vector_name)

sunholo/cli/chat_vac.py

@@ -211,7 +211,11 @@ def headless_mode(service_url, service_name, user_input, chat_history=None, stre
             if isinstance(token, bytes):
                 token = token.decode('utf-8')
             print(token, end='', flush=True)
-            answer += token
+            if isinstance(token, dict):
+                # ?
+                pass
+            elif isinstance(token, str):
+                answer += token
 
     if answer:
         chat_history.append({"name": "Human", "content": user_input})

sunholo/discovery_engine/discovery_engine_client.py

@@ -80,6 +80,13 @@ class DiscoveryEngineClient:
             predicate=if_exception_type(ResourceExhausted)  # Retry if a ResourceExhausted error occurs
         )
     
+    def data_store_path(self, collection: str = "default_collection"):
+        return self.store_client.collection_path(
+            project=self.project_id,
+            location=self.location,
+            collection=collection,
+        )
+
     def create_data_store(
         self, chunk_size: int = 500,
         collection: str = "default_collection"
@@ -121,15 +128,9 @@ class DiscoveryEngineClient:
             document_processing_config=doc_config
         )
 
-        parent = self.store_client.collection_path(
-            project=self.project_id,
-            location=self.location,
-            collection=collection,
-        )
-
         # https://cloud.google.com/python/docs/reference/discoveryengine/0.11.4/google.cloud.discoveryengine_v1alpha.types.CreateDataStoreRequest
         request = discoveryengine.CreateDataStoreRequest(
-            parent=parent,
+            parent=self.data_store_path(collection),
             data_store_id=self.data_store_id,
             data_store=data_store,
             # Optional: For Advanced Site Search Only

sunholo/vertex/init.py

@@ -1,6 +1,7 @@
 from ..logging import log
+from ..utils.gcp_project import get_gcp_project
 
-def init_vertex(gcp_config):
+def init_vertex(gcp_config=None, location="eu"):
     """
     Initializes the Vertex AI environment using the provided Google Cloud Platform configuration.
 
@@ -12,6 +13,7 @@ def init_vertex(gcp_config):
         gcp_config (dict): A dictionary containing the Google Cloud Platform configuration with keys:
             - 'project_id': The Google Cloud project ID to configure for Vertex AI.
             - 'location': The Google Cloud region to configure for Vertex AI.
+            If default None it will derive it from the environment
 
     Raises:
         KeyError: If the necessary keys ('project_id' or 'location') are missing in the gcp_config dictionary.
@@ -37,7 +39,11 @@ def init_vertex(gcp_config):
 
         return None
     
-    # Initialize Vertex AI API once per session
-    project_id = gcp_config.get('project_id')
-    location = gcp_config.get('location')
+    if gcp_config:
+        # Initialize Vertex AI API once per session
+        project_id = gcp_config.get('project_id')
+        location = gcp_config.get('location') or location
+    else:
+        project_id = get_gcp_project()
+
     vertexai.init(project=project_id, location=location)

sunholo/vertex/memory_tools.py

@@ -8,6 +8,8 @@ from ..logging import log
 from ..utils.config import load_config_key
 from ..components import load_memories
 from ..llamaindex.get_files import fetch_corpus
+from ..discovery_engine.discovery_engine_client import DiscoveryEngineClient
+from ..utils.gcp_project import get_gcp_project
 
 def get_vertex_memories(vector_name):
     """
@@ -42,10 +44,7 @@ def get_vertex_memories(vector_name):
     if not rag:
         raise ValueError("Need to install vertexai module via `pip install sunholo[gcp]`")
     
-    global_project_id = gcp_config.get('project_id')
     global_location = gcp_config.get('location')
-    global_rag_id = gcp_config.get('rag_id')
-    global_data_store_id = gcp_config.get('data_store_id')
 
     memories = load_memories(vector_name)
     tools = []
@@ -60,12 +59,16 @@ def get_vertex_memories(vector_name):
             if vectorstore == "llamaindex":
                 log.info(f"Found vectorstore {vectorstore}")
                 rag_id = value.get('rag_id')
+                if rag_id is None:
+                    raise ValueError("Must specify rag_id if using vectorstore: llamaindex")
+                
                 project_id = gcp_config.get('project_id')
                 location = gcp_config.get('location')
+
                 corpus = fetch_corpus(
-                    project_id=project_id or global_project_id,
+                    project_id=project_id or get_gcp_project(),
                     location=location or global_location,
-                    rag_id=rag_id or global_rag_id
+                    rag_id=rag_id
                 )
                 corpus_tool = Tool.from_retrieval(
                     retrieval=rag.Retrieval(
@@ -76,13 +79,12 @@ def get_vertex_memories(vector_name):
                     )
                 )
                 tools.append(corpus_tool)
-            elif vectorstore == "vertexai_agent_builder":
+            elif vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
+
+                de = DiscoveryEngineClient(vector_name, project_id=get_gcp_project())
                 log.info(f"Found vectorstore {vectorstore}")
-                data_store_id = value.get('data_store_id') or global_data_store_id
-                project_id = gcp_config.get('project_id') or global_project_id
-                location = gcp_config.get('location') or global_location
-                data_store_path=f"projects/{project_id}/locations/{location}/collections/default_collection/dataStores/{data_store_id}"
 
+                data_store_path = de.data_store_path()
                 corpus_tool = Tool.from_retrieval(
                     grounding.Retrieval(grounding.VertexAISearch(datastore=data_store_path))
                 )

{sunholo-0.69.16.dist-info → sunholo-0.70.0.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.16
+Version: 0.70.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.16.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.70.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.16.dist-info → sunholo-0.70.0.dist-info}/RECORD

@@ -24,7 +24,7 @@ sunholo/bots/discord.py,sha256=cCFae5K1BCa6JVkWGLh_iZ9qFO1JpXb6K4eJrlDfEro,2442
 sunholo/bots/github_webhook.py,sha256=5pQPRLM_wxxcILVaIzUDV8Kt7Arcm2dL1r1kMMHA524,9629
 sunholo/bots/webapp.py,sha256=EIMxdAJ_xtufwJmvnn7N_Fb_1hZ9DjhJ0Kf_hp02vEU,1926
 sunholo/chunker/__init__.py,sha256=yWYwpejyYxDpZv1joTrFMsh2SWAkd0z7a1VKtmOfMhA,77
-sunholo/chunker/data_to_embed_pubsub.py,sha256=tL9J7s-F4szmpbZYU4dpvLr5R-LgiuZLpRvhQnrTD1E,4203
+sunholo/chunker/data_to_embed_pubsub.py,sha256=-bXtm3Tn6YczUWL9kSAc6OWdD6dDY2rKhgpD-90H6ms,4203
 sunholo/chunker/doc_handling.py,sha256=rIyknpzDyj5A0u_DqSQVD_CXLRNZPOU6TCL4bhCdjOI,8563
 sunholo/chunker/images.py,sha256=Xmh1vwHrVhoXm5iH2dhCc52O8YgdzE8KrDSdL-pGnp8,1861
 sunholo/chunker/loaders.py,sha256=xiToUVgPz2ZzcqpUAq7aNP3PTenb_rBUAFzu0JPycIg,10268
@@ -33,7 +33,7 @@ sunholo/chunker/pdfs.py,sha256=daCZ1xjn1YvxlifIyxskWNpLJLe-Q9D_Jq12MWx3tZo,2473
 sunholo/chunker/publish.py,sha256=tiO615A2uo_ZjzdFDzNH1PL_1kJeLMUQwLJ4w67rNIc,2932
 sunholo/chunker/splitter.py,sha256=FLkDhkePkg_zGQpFBK13Cznw575D-Rf9pcaCpc1HUxY,6726
 sunholo/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sunholo/cli/chat_vac.py,sha256=CceXT-nNVhXIGtVUg07FT8Azd0LBZN18kvCGIkJq7zM,18659
+sunholo/cli/chat_vac.py,sha256=NcXQcz0HHE0epfUPmDat5Fk8XaDrTFGs3MINQlBu1vo,18785
 sunholo/cli/cli.py,sha256=8e00HBN6eYIUJ8cnvKteBJNn7aZPRMk4b82jwcGg9D4,3741
 sunholo/cli/cli_init.py,sha256=JMZ9AX2cPDZ-_mv3adiv2ToFVNyRPtjk9Biszl1kiR0,2358
 sunholo/cli/configs.py,sha256=QUM9DvKOdZmEQRM5uI3Nh887T0YDiSMr7O240zTLqws,4546
@@ -63,7 +63,7 @@ sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3
 sunholo/discovery_engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/discovery_engine/chunker_handler.py,sha256=puNnV6vKnjNqk28FLnsIPMNsC-1Y6s20PK7ioi8qwzc,4491
 sunholo/discovery_engine/create_new.py,sha256=7oZG78T6lW0EspRzlo7-qRyXFSuFxDn2dfSAVEaqlqY,978
-sunholo/discovery_engine/discovery_engine_client.py,sha256=aiEfn0511pIaS4lj6GwVA6LOiBn_PPphmMZGfjLfj5o,14486
+sunholo/discovery_engine/discovery_engine_client.py,sha256=n4euIZjc3pWUYfoFhpvReIzHXnIYO8J9nKC8euKSTgY,14581
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=P744zUQJgqrjILunzaqtTerB9AwoXFU6tXBtz4rjWgQ,6673
 sunholo/gcs/__init__.py,sha256=DtVw_AZwQn-IguR5BJuIi2XJeF_FQXizhJikzRNrXiE,50
@@ -108,12 +108,12 @@ sunholo/utils/timedelta.py,sha256=BbLabEx7_rbErj_YbNM0MBcaFN76DC4PTe4zD2ucezg,49
 sunholo/utils/user_ids.py,sha256=SQd5_H7FE7vcTZp9AQuQDWBXd4FEEd7TeVMQe1H4Ny8,292
 sunholo/utils/version.py,sha256=jjU_4anXBikJxPg0Wur0X-B7-ec1tC7jToykAnAG9Dg,108
 sunholo/vertex/__init__.py,sha256=JvHcGFuv6R_nAhY2AdoqqhMpJ5ugeWPZ_svGhWrObBk,136
-sunholo/vertex/init.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
-sunholo/vertex/memory_tools.py,sha256=8F1iTWnqEK9mX4W5RzCVKIjydIcNp6OFxjn_dtQ3GXo,5379
+sunholo/vertex/init.py,sha256=RLjQppTUwubWgwf2PoAke-EtcwlVkFPaPMYvUsMw1KQ,2029
+sunholo/vertex/memory_tools.py,sha256=Qo6-uV0_SlwBsCEYsPnrr1l_2UH_zLT_xPxiC9XxGQg,5278
 sunholo/vertex/safety.py,sha256=3meAX0HyGZYrH7rXPUAHxtI_3w_zoy_RX7Shtkoa660,1275
-sunholo-0.69.16.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.69.16.dist-info/METADATA,sha256=VHUSz1m6mWug70lj_pLHCxZFeMK72-xyE3xZqbtkxSQ,6242
-sunholo-0.69.16.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-sunholo-0.69.16.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.69.16.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.69.16.dist-info/RECORD,,
+sunholo-0.70.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.70.0.dist-info/METADATA,sha256=Jc55TzM3gNp1oMX2mF6sW-55-BjYesOKO_mVa9U15y4,6240
+sunholo-0.70.0.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+sunholo-0.70.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.70.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.70.0.dist-info/RECORD,,