sunholo 0.79.1__py3-none-any.whl → 0.79.3__py3-none-any.whl

sunholo/components/retriever.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 from ..custom_logging import log
 from .vectorstore import pick_vectorstore
-from ..utils import load_config_key, ConfigManager
+from ..utils import ConfigManager
 from .llm import get_embeddings
 from ..utils.gcp_project import get_gcp_project
 

sunholo/llamaindex/llamaindex_class.py

@@ -1,3 +1,6 @@
+import os
+import tempfile
+
 try:
     from vertexai.preview import rag
 except ImportError:
@@ -46,6 +49,60 @@ class LlamaIndexVertexCorpusManager:
             self.location = self.config.vacConfig('location') or location
 
         init_vertex(location=self.location, project_id=self.project_id)
+
+    def upload_text(self, text: str, corpus_display_name: str, description: str = None):
+        """
+        Uploads a text string to a specified corpus by saving it to a temporary file first.
+
+        Args:
+            text (str): The text content to upload.
+            corpus_display_name (str): The display name of the corpus.
+            description (str, optional): Description of the text upload.
+
+        Returns:
+            The uploaded file object.
+        """
+        # Create a temporary file and write the text to it
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.txt') as temp_file:
+            temp_file.write(text.encode('utf-8'))
+            temp_filename = temp_file.name
+        
+        try:
+            # Upload the temporary file
+            uploaded_file = self.upload_file(temp_filename, corpus_display_name, description or text[:50])
+        finally:
+            # Clean up the temporary file
+            os.remove(temp_filename)
+
+        return uploaded_file
+    
+    def upload_file(self, filename:str, corpus_display_name:str, description:str=None):
+        corpus = self.find_corpus_from_list(corpus_display_name)
+
+        rag_file = rag.upload_file(
+            corpus_name=corpus.name,
+            path=filename,
+            display_name=filename,
+            description=description or f"Upload for {filename}",
+        )
+        log.info("Uploaded file: {rag_file}")
+
+        return rag_file
+    
+    def import_files(self, file_imports:list[str], corpus_display_name:str):
+        corpus = self.find_corpus_from_list(corpus_display_name)
+
+        log.info(f"Importing files: {file_imports} into {corpus.name}")
+        response = rag.import_files(
+            corpus_name=corpus.name,
+            paths=file_imports,
+            chunk_size=512,  # Optional
+            chunk_overlap=100,  # Optional
+            max_embedding_requests_per_min=900,  # Optional
+        )
+        log.info(f"Imported files: {response}")
+
+        return response
     
     def list_corpora(self):
         """
@@ -83,7 +140,6 @@ class LlamaIndexVertexCorpusManager:
         """
         corp = self.find_corpus_from_list(display_name)
         if corp:
-
             return corp
         
         corpus = rag.create_corpus(display_name=display_name, description=description or f"Corpus for {display_name}")
@@ -130,7 +186,6 @@ class LlamaIndexVertexCorpusManager:
         return rag.get_corpus(name=corp.name)
 
 def llamaindex_command(args):
-
     if console is None:
         raise ImportError("Need cli tools to use `sunholo llamaindex` - install via `pip install sunholo[cli]`")
     
@@ -148,8 +203,14 @@ def llamaindex_command(args):
         corpus = manager.find_corpus_from_list(display_name=args.display_name)
         console.print(corpus)
     elif args.action == "list":
-        corpura = manager.list_corpora()
-        console.print(corpura)
+        corpora = manager.list_corpora()
+        console.print(corpora)
+    elif args.action == "import_files":
+        manager.import_files(file_imports=args.file_imports, corpus_display_name=args.display_name)
+    elif args.action == "upload_file":
+        manager.upload_file(filename=args.filename, corpus_display_name=args.display_name, description=args.description)
+    elif args.action == "upload_text":
+        manager.upload_text(text=args.text, corpus_display_name=args.display_name, description=args.description)
     else:
         console.print(f"Unknown action: {args.action}")
 
@@ -186,7 +247,28 @@ def setup_llamaindex_subparser(subparsers):
     find_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
 
     # LlamaIndex list command
-    find_parser = llamaindex_subparsers.add_parser('list', help='List all corpus')
-    find_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    list_parser = llamaindex_subparsers.add_parser('list', help='List all corpus')
+    list_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex import_files command
+    import_files_parser = llamaindex_subparsers.add_parser('import_files', help='Import files from URLs to a corpus')
+    import_files_parser.add_argument('display_name', help='The name of the corpus')
+    import_files_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    import_files_parser.add_argument('file_imports', nargs='+', help='The list of file URLs to import')
+
+    # LlamaIndex upload_file command
+    upload_file_parser = llamaindex_subparsers.add_parser('upload_file', help='Upload a local file to a corpus')
+    upload_file_parser.add_argument('display_name', help='The name of the corpus')
+    upload_file_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    upload_file_parser.add_argument('filename', help='The local file path to upload')
+    upload_file_parser.add_argument('--description', help='Description of the file upload', default=None)
+
+    # LlamaIndex upload_text command
+    upload_text_parser = llamaindex_subparsers.add_parser('upload_text', help='Upload text to a corpus')
+    upload_text_parser.add_argument('display_name', help='The name of the corpus')
+    upload_text_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    upload_text_parser.add_argument('text', help='The text content to upload')
+    upload_text_parser.add_argument('--description', help='Description of the text upload', default=None)
 
     llamaindex_parser.set_defaults(func=llamaindex_command)
+    

sunholo/utils/config_class.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 from .timedelta import format_timedelta
 
 class ConfigManager:
-    def __init__(self, vector_name: str):
+    def __init__(self, vector_name: str, validate:bool=True):
         """
         Initialize the ConfigManager with a vector name.
         Requires a local config/ folder holding your configuration files or the env var VAC_CONFIG_FOLDER to be set.
@@ -15,6 +15,7 @@ class ConfigManager:
 
         Args:
             vector_name (str): The name of the vector in the configuration files.
+            validate (bool): Whether to validate the configurations
         
         Example:
         ```python
@@ -38,7 +39,7 @@ class ConfigManager:
         self.configs_by_kind = self.load_all_configs()
 
         test_agent = self.vacConfig("agent")
-        if not test_agent and self.vector_name != "global":
+        if not test_agent and self.vector_name != "global" and validate:
             print(f"WARNING: No vacConfig.agent found for {self.vector_name} - are you in right folder? {local_config_folder=} {self.config_folder=}")
 
     def load_all_configs(self):

sunholo/vertex/memory_tools.py

@@ -12,7 +12,7 @@ from ..discovery_engine.discovery_engine_client import DiscoveryEngineClient
 from ..utils.gcp_project import get_gcp_project
 from ..utils import ConfigManager
 
-def get_vertex_memories(vector_name:str=None, config:ConfigManager=None):
+def get_vertex_memories(config:ConfigManager):
     """
     Retrieves a LlamaIndex corpus from Vertex AI based on the provided Google Cloud configuration.
     
@@ -40,10 +40,6 @@ def get_vertex_memories(vector_name:str=None, config:ConfigManager=None):
         print("Error fetching corpus:", str(e))
     ```
     """
-    if config is None:
-        if vector_name is None:
-            raise ValueError("config or vector_name are required")
-        config = ConfigManager(vector_name)
 
     gcp_config = config.vacConfig("gcp_config")
 
@@ -52,7 +48,7 @@ def get_vertex_memories(vector_name:str=None, config:ConfigManager=None):
     
     global_location = gcp_config.get('location')
 
-    memories = load_memories(vector_name)
+    memories = load_memories(config=config)
     tools = []
 
     if not memories:
@@ -76,9 +72,9 @@ def get_vertex_memories(vector_name:str=None, config:ConfigManager=None):
                             rag_id=rag_id
                         )
                 else:
-                    log.info(f"Using display_name {vector_name} to derive rag_id")
+                    log.info(f"Using display_name {config.vector_name} to derive rag_id")
                     manager = LlamaIndexVertexCorpusManager(project_id=project_id, location=location)
-                    corpus = manager.find_corpus_from_list(vector_name)
+                    corpus = manager.find_corpus_from_list(config.vector_name)
 
                 try:
                     corpus_tool = Tool.from_retrieval(

{sunholo-0.79.1.dist-info → sunholo-0.79.3.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.79.1
+Version: 0.79.3
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.3.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.79.1.dist-info → sunholo-0.79.3.dist-info}/RECORD

@@ -55,7 +55,7 @@ sunholo/cli/swagger.py,sha256=absYKAU-7Yd2eiVNUY-g_WLl2zJfeRUNdWQ0oH8M_HM,1564
 sunholo/cli/vertex.py,sha256=8130YCarxHL1UC3aqblNmUwGZTXbkdL4Y_FOnZJsWiI,2056
 sunholo/components/__init__.py,sha256=IDoylb74zFKo6NIS3RQqUl0PDFBGVxM1dfUmO7OJ44U,176
 sunholo/components/llm.py,sha256=5wRVf7lIb7q1vRADNcdQp26L9l4vGHFIvjtUDurZN_s,11488
-sunholo/components/retriever.py,sha256=vT5ouRUDcpdL4PMZVK57SYm31wZREbMsXkgCq8PnCUo,6922
+sunholo/components/retriever.py,sha256=KK8Df3NH8jNpEymPBfR2913rrQZN-PRMUGedtYNIV6k,6905
 sunholo/components/vectorstore.py,sha256=xKk7micTRwZckaI7U6PxvFz_ZSjCH48xPTDYiDcv2tc,5913
 sunholo/database/__init__.py,sha256=Zz0Shcq-CtStf9rJGIYB_Ybzb8rY_Q9mfSj-nviM490,241
 sunholo/database/alloydb.py,sha256=c1PEmK9fJCxYaVmKv4emvOoXrajV7KqaVK5mqpeksvM,11527
@@ -90,7 +90,7 @@ sunholo/langfuse/prompts.py,sha256=27BsVfihM6-h1jscbkGSO4HsATl-d4ZN6tcNCVztWoY,1
 sunholo/llamaindex/__init__.py,sha256=DlY_cHWCsVEV1C5WBgDdHRgOMlJc8pDoCRukUJ8PT9w,88
 sunholo/llamaindex/get_files.py,sha256=6rhXCDqQ_lrIapISQ_OYQDjiSATXvS_9m3qq53-oIl0,781
 sunholo/llamaindex/import_files.py,sha256=Bnic5wz8c61af9Kwq8KSrNBbc4imYnzMtBCb2jzSImI,6224
-sunholo/llamaindex/llamaindex_class.py,sha256=KAD9Unby2INIWVfxarthA4F5P6SiJC2fnAj3Cu1pEhw,7062
+sunholo/llamaindex/llamaindex_class.py,sha256=45N_xU0gDdgUPNv5m36DopRmG9JHFErwmgssKZ3wOqA,11058
 sunholo/lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/lookup/model_lookup.yaml,sha256=O7o-jP53MLA06C8pI-ILwERShO-xf6z_258wtpZBv6A,739
 sunholo/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -116,7 +116,7 @@ sunholo/utils/__init__.py,sha256=Hv02T5L2zYWvCso5hzzwm8FQogwBq0OgtUbN_7Quzqc,89
 sunholo/utils/api_key.py,sha256=Ct4bIAQZxzPEw14hP586LpVxBAVi_W9Serpy0BK-7KI,244
 sunholo/utils/big_context.py,sha256=gJIP7_ZL-YSLhOMq8jmFTMqH1wq8eB1NK7oKPeZAq2s,5578
 sunholo/utils/config.py,sha256=aG29MXcL5qzQMtCMqcdy-2ysDCYf9Zn_ZLk5NNOQNSE,8982
-sunholo/utils/config_class.py,sha256=BewBxf59vSKJs4jcYvuLi6PW9M6so8w3qz9HfNFIu0M,8862
+sunholo/utils/config_class.py,sha256=DEY2w7ks5fXmtJ4SwqELhUF4NJFVOuRwxeVFZJlYPf0,8963
 sunholo/utils/config_schema.py,sha256=Wv-ncitzljOhgbDaq9qnFqH5LCuxNv59dTGDWgd1qdk,4189
 sunholo/utils/gcp.py,sha256=uueODEpA-P6O15-t0hmcGC9dONLO_hLfzSsSoQnkUss,4854
 sunholo/utils/gcp_project.py,sha256=0ozs6tzI4qEvEeXb8MxLnCdEVoWKxlM6OH05htj7_tc,1325
@@ -129,12 +129,12 @@ sunholo/vertex/extensions_call.py,sha256=VIVBYpJPMaDzGWxuBU5RF9GfAbrvrskGWH4eXpJ
 sunholo/vertex/extensions_class.py,sha256=2QGW28lNjoMEnaoVb3QcqEDwphclIsZthnpLUi5_Ivo,21033
 sunholo/vertex/genai_functions.py,sha256=2z6grM9H0Z79Yzx88l8mE1wXck3bRa0TWvnqZZ9ifDc,2051
 sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
-sunholo/vertex/memory_tools.py,sha256=7nDQXjrF9IpCPKbX3Focn4A2PdKb7JhfZDLv4le5kWc,7498
+sunholo/vertex/memory_tools.py,sha256=6SM9ANuG2OcgFarJNZxAXobWpWbdz0DNZ6ooMsAGfzA,7321
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.79.1.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.79.1.dist-info/METADATA,sha256=lNLvJFenzRDXsw-prEyzwa6NY2jSvWpDyDZ-t-pRCYo,7348
-sunholo-0.79.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-sunholo-0.79.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.79.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.79.1.dist-info/RECORD,,
+sunholo-0.79.3.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.79.3.dist-info/METADATA,sha256=qN4H8g0Xe_6N39qmwrYvI25G2aYZA3x-N218wLL-yjE,7348
+sunholo-0.79.3.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+sunholo-0.79.3.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.79.3.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.79.3.dist-info/RECORD,,