sunholo 0.69.8__py3-none-any.whl → 0.69.10__py3-none-any.whl

sunholo/discovery_engine/chunker_handler.py

@@ -1,5 +1,6 @@
 from ..logging import log
 from ..utils.config import load_config_key
+from ..utils.gcp_project import get_gcp_project
 from ..components import load_memories
 
 from .discovery_engine_client import DiscoveryEngineClient
@@ -20,11 +21,6 @@ def do_discovery_engine(message_data, metadata, vector_name):
     ```
     """
 
-    global_gcp_config = load_config_key("gcp_config", vector_name="global", kind="vacConfig")
-    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
-    if not gcp_config and not global_gcp_config:
-        raise ValueError(f"Need config.{vector_name}.gcp_config to configure discovery engine")
-
     memories = load_memories(vector_name)
     tools = []
 
@@ -38,11 +34,10 @@ def do_discovery_engine(message_data, metadata, vector_name):
             vectorstore = value.get('vectorstore')
             if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                 log.info(f"Found vectorstore {vectorstore}")
-                project_id = gcp_config.get('project_id') or global_gcp_config['project_id']
                 #location = gcp_config.get('location') 
                 corpus = DiscoveryEngineClient(
                     data_store_id=vector_name, 
-                    project_id=project_id,
+                    project_id=get_gcp_project(),
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
                     #location=location or global_location
                     )

sunholo/discovery_engine/create_new.py

@@ -1,9 +1,8 @@
 from .discovery_engine_client import DiscoveryEngineClient
 from ..utils.config import load_config_key
+from ..utils.gcp_project import get_gcp_project
 
 def create_new_discovery_engine(vector_name):
-    global_config = load_config_key("gcp_config", "global", kind="vacConfig")
-    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
 
     chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
 
@@ -12,7 +11,7 @@ def create_new_discovery_engine(vector_name):
         if "chunk_size" in chunker_config:
             chunk_size = chunker_config["chunk_size"]    
 
-    project_id = gcp_config.get('project_id') or global_config.get('project_id')
+    project_id = get_gcp_project()
     if not project_id:
         raise ValueError("Could not find project_id in gcp_config")
     

sunholo/discovery_engine/discovery_engine_client.py

@@ -155,6 +155,7 @@ class DiscoveryEngineClient:
         num_previous_chunks: int = 3,
         num_next_chunks: int = 3,
         page_size: int = 10,
+        parse_chunks_to_string: bool = True,
         doc_or_chunks: str = "CHUNKS",  # or DOCUMENTS
         serving_config: str = "default_serving_config",
     ):
@@ -166,6 +167,7 @@ class DiscoveryEngineClient:
             num_previous_chunks (int, optional): Number of previous chunks to return for context (default is 3).
             num_next_chunks (int, optional): Number of next chunks to return for context (default is 3).
             page_size (int, optional): The maximum number of results to return per page (default is 10).
+            parse_chunks_to_string: If True will put chunks in one big string, False will return object
 
         Returns:
             discoveryengine.SearchResponse: The search response object containing the search results.
@@ -201,8 +203,61 @@ class DiscoveryEngineClient:
 
         search_response = self.search_client.search(search_request)
 
+        if parse_chunks_to_string:
+            
+            return self.process_chunks(search_response)
+        
         return search_response
 
+    def process_chunks(self, response):
+        all_chunks = []
+
+        if 'results' not in response:
+            raise ValueError('No results found in response')
+        
+        for result in response['results']:
+            chunk = result['chunk']
+            chunk_metadata = chunk['chunkMetadata']
+
+            if 'previousChunks' in chunk_metadata:
+                # Process previous chunks
+                for prev_chunk in chunk['chunkMetadata']['previousChunks']:
+                    prev_chunk_string = (
+                        f"# {prev_chunk['id']}\n"
+                        f"{prev_chunk['content']}\n"
+                        f"## metadata\n"
+                        f"Document URI: {prev_chunk['documentMetadata']['uri']}\n"
+                        f"Document Title: {prev_chunk['documentMetadata']['title']}\n"
+                    )
+                    all_chunks.append(prev_chunk_string)
+
+            # Process fetched chunk
+            fetched_chunk_string = (
+                f"# {chunk['id']}\n"
+                f"{chunk['content']}\n"
+                f"## metadata\n"
+                f"Document URI: {chunk['documentMetadata']['uri']}\n"
+                f"Document Title: {chunk['documentMetadata']['title']}\n"
+            )
+            all_chunks.append(fetched_chunk_string)
+
+            # Process next chunks
+            if 'nextChunks' in chunk_metadata:
+                for next_chunk in chunk_metadata['nextChunks']:
+                    next_chunk_string = (
+                        f"# {next_chunk['id']}\n"
+                        f"{next_chunk['content']}\n"
+                        f"## metadata\n"
+                        f"Document URI: {next_chunk['documentMetadata']['uri']}\n"
+                        f"Document Title: {next_chunk['documentMetadata']['title']}\n"
+                    )
+                    all_chunks.append(next_chunk_string)
+
+        # Combine all chunks into one long string
+        result_string = "\n".join(all_chunks)
+
+        return result_string
+
     def import_documents(self,
         gcs_uri: Optional[str] = None,
         data_schema="content",

sunholo/llamaindex/import_files.py

@@ -61,7 +61,6 @@ def do_llamaindex(message_data, metadata, vector_name):
     corpuses = []
     for memory in memories:
         for key, value in memory.items():  # Now iterate over the dictionary
-            log.info(f"Found memory {key}")
             vectorstore = value.get('vectorstore')
             if vectorstore == "llamaindex":
                 log.info(f"Found vectorstore {vectorstore}")

{sunholo-0.69.8.dist-info → sunholo-0.69.10.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.69.8
+Version: 0.69.10
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.8.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.69.10.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.69.8.dist-info → sunholo-0.69.10.dist-info}/RECORD

@@ -61,9 +61,9 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sunholo/discovery_engine/chunker_handler.py,sha256=eqRI8R-AVIhtqpzw62glCucvX_r1MXzdrF_DAKcPhtM,4793
-sunholo/discovery_engine/create_new.py,sha256=31NK8ZzgKURFT2e1kBJCiyfFxfP6I_WCSbcjFpbp_fM,1144
-sunholo/discovery_engine/discovery_engine_client.py,sha256=UYMhXtGWXuyyg4fA1piVjqfrWXkSP_MFimh6NzaUlZ8,12413
+sunholo/discovery_engine/chunker_handler.py,sha256=9zv72CxgKlKpOcSAt-XFtq0Ra-Z6tBTwMuXQhRPE5mY,4425
+sunholo/discovery_engine/create_new.py,sha256=7oZG78T6lW0EspRzlo7-qRyXFSuFxDn2dfSAVEaqlqY,978
+sunholo/discovery_engine/discovery_engine_client.py,sha256=xgRv1WMYgjjKE5e2SwV3ZeaodzQ1twSoaRwfQkiTOSo,14687
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=P744zUQJgqrjILunzaqtTerB9AwoXFU6tXBtz4rjWgQ,6673
 sunholo/gcs/__init__.py,sha256=DtVw_AZwQn-IguR5BJuIi2XJeF_FQXizhJikzRNrXiE,50
@@ -76,7 +76,7 @@ sunholo/langfuse/prompts.py,sha256=HO4Zy9usn5tKooBPCKksuw4Lff3c03Ny5wqn4ce_xZM,1
 sunholo/llamaindex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/llamaindex/generate.py,sha256=l1Picr-hVwkmAUD7XmTCa63qY9ERliFHQXwyX3BqB2Q,686
 sunholo/llamaindex/get_files.py,sha256=6rhXCDqQ_lrIapISQ_OYQDjiSATXvS_9m3qq53-oIl0,781
-sunholo/llamaindex/import_files.py,sha256=WPaVfYKS08GN0HXodPH_5WCs6d_fqj-N4MQIzvlz5Xw,5610
+sunholo/llamaindex/import_files.py,sha256=LIFBXxgXgRYMdVpq3FkycIoMMhLSXkKdKzK7qDuqOmQ,5566
 sunholo/lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/lookup/model_lookup.yaml,sha256=O7o-jP53MLA06C8pI-ILwERShO-xf6z_258wtpZBv6A,739
 sunholo/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -111,9 +111,9 @@ sunholo/vertex/__init__.py,sha256=JvHcGFuv6R_nAhY2AdoqqhMpJ5ugeWPZ_svGhWrObBk,13
 sunholo/vertex/init.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
 sunholo/vertex/memory_tools.py,sha256=8F1iTWnqEK9mX4W5RzCVKIjydIcNp6OFxjn_dtQ3GXo,5379
 sunholo/vertex/safety.py,sha256=3meAX0HyGZYrH7rXPUAHxtI_3w_zoy_RX7Shtkoa660,1275
-sunholo-0.69.8.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.69.8.dist-info/METADATA,sha256=SEe57-886plXFH-T-Kkn2acgw4WlJKYqh3Ab6hJzKMQ,6155
-sunholo-0.69.8.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-sunholo-0.69.8.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.69.8.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.69.8.dist-info/RECORD,,
+sunholo-0.69.10.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.69.10.dist-info/METADATA,sha256=t0FhARCqbXEulRWUtzijoEFrssaFlYtG5dBG8xzEpg4,6157
+sunholo-0.69.10.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+sunholo-0.69.10.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.69.10.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.69.10.dist-info/RECORD,,