sunholo 0.66.1__py3-none-any.whl → 0.66.2__py3-none-any.whl

sunholo/llamaindex/import_files.py

@@ -7,6 +7,7 @@ from ..logging import log
 from ..utils.config import load_config_key
 from ..vertex import init_vertex
 from .get_files import fetch_corpus
+from ..components import load_memories
     
 
 def do_llamaindex(message_data, metadata, vector_name):
@@ -45,30 +46,60 @@ def do_llamaindex(message_data, metadata, vector_name):
         raise ValueError(f"Need config.{vector_name}.gcp_config to configure llamaindex on VertexAI")
 
     init_vertex(gcp_config)
-    corpus = fetch_corpus(gcp_config)
-    #display_name = load_config_key("display_name", vector_name=vector_name, filename="config/llm_config.yaml")
-    #description = load_config_key("description", vector_name=vector_name, filename="config/llm_config.yaml")
 
+    global_project_id = gcp_config.get('project_id')
+    global_location = gcp_config.get('location')
+    global_rag_id = gcp_config.get('rag_id')
+    #global_data_store_id = gcp_config.get('data_store_id')
+
+    memories = load_memories(vector_name)
+    tools = []
+
+    if not memories:
+        return tools
+    
+    corpuses = None
+    for memory in memories:
+        for key, value in memory.items():  # Now iterate over the dictionary
+            log.info(f"Found memory {key}")
+            vectorstore = value.get('vectorstore')
+            if vectorstore == "llamaindex":
+                log.info(f"Found vectorstore {vectorstore}")
+                rag_id = value.get('rag_id')
+                project_id = gcp_config.get('project_id')
+                location = gcp_config.get('location')
+                corpus = fetch_corpus(
+                    project_id=project_id or global_project_id,
+                    location=location or global_location,
+                    rag_id=rag_id or global_rag_id
+                )
+                corpuses.append(corpus)
+    if not corpuses:
+        log.error("Could not find a RAG corpus to import data to")
+        return None
+    
     try:
         corpura = rag.list_corpora()
-        log.info(f"Corpora: {corpura} - {type(corpura)}")
+        log.info(f"All Corpora: {corpura} - {type(corpura)}")
     except Exception as err:
         log.warning(f"Could not list any corpora - {str(err)}")
 
-    log.info(f"Found llamaindex corpus: {corpus}")
+    log.info(f"Found llamaindex corpus: {corpuses}")
 
     # native support for cloud storage and drive links
     chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
 
+    
     if message_data.startswith("gs://") or message_data.startswith("https://drive.google.com"):
         log.info(f"rag.import_files for {message_data}")
-        response = rag.import_files(
-            corpus_name=corpus.name,
-            paths=[message_data],
-            chunk_size=chunker_config.get("chunk_size"),  # Optional
-            chunk_overlap=chunker_config.get("overlap"),  # Optional
-        )
-        log.info(f"Imported file to corpus: {response} with metadata: {metadata}")
+        for corp in corpuses:
+            response = rag.import_files(
+                corpus_name=corp.name,
+                paths=[message_data],
+                chunk_size=chunker_config.get("chunk_size"),  # Optional
+                chunk_overlap=chunker_config.get("overlap"),  # Optional
+            )
+            log.info(f"Imported file to corpus: {response} with metadata: {metadata}")
 
         metadata["source"] = message_data
         return metadata

{sunholo-0.66.1.dist-info → sunholo-0.66.2.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.66.1
+Version: 0.66.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.66.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.66.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.66.1.dist-info → sunholo-0.66.2.dist-info}/RECORD

@@ -70,7 +70,7 @@ sunholo/langfuse/prompts.py,sha256=HO4Zy9usn5tKooBPCKksuw4Lff3c03Ny5wqn4ce_xZM,1
 sunholo/llamaindex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/llamaindex/generate.py,sha256=l1Picr-hVwkmAUD7XmTCa63qY9ERliFHQXwyX3BqB2Q,686
 sunholo/llamaindex/get_files.py,sha256=6rhXCDqQ_lrIapISQ_OYQDjiSATXvS_9m3qq53-oIl0,781
-sunholo/llamaindex/import_files.py,sha256=eGRVYoMzeEjxtg4sQw5NLPIBWjlYDu7r63DPRarLGpQ,5349
+sunholo/llamaindex/import_files.py,sha256=i4YvrDOg-eq1Ep0SgmgpoUjC2y2D7kNLb31U-UipPcE,6373
 sunholo/lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/lookup/model_lookup.yaml,sha256=O7o-jP53MLA06C8pI-ILwERShO-xf6z_258wtpZBv6A,739
 sunholo/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -104,9 +104,9 @@ sunholo/vertex/__init__.py,sha256=JvHcGFuv6R_nAhY2AdoqqhMpJ5ugeWPZ_svGhWrObBk,13
 sunholo/vertex/init.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
 sunholo/vertex/memory_tools.py,sha256=8F1iTWnqEK9mX4W5RzCVKIjydIcNp6OFxjn_dtQ3GXo,5379
 sunholo/vertex/safety.py,sha256=3meAX0HyGZYrH7rXPUAHxtI_3w_zoy_RX7Shtkoa660,1275
-sunholo-0.66.1.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.66.1.dist-info/METADATA,sha256=Mv0TUMCU4fHkOPMw2fCWKfiL6sJtkWG8MPDlA7JstiI,5998
-sunholo-0.66.1.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-sunholo-0.66.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.66.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.66.1.dist-info/RECORD,,
+sunholo-0.66.2.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.66.2.dist-info/METADATA,sha256=oIbyKC0s3PKDXd2fQlmaEWuNiXQkRJlJ54VBpWn-G9k,5998
+sunholo-0.66.2.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+sunholo-0.66.2.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.66.2.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.66.2.dist-info/RECORD,,