sunholo 0.79.1__py3-none-any.whl → 0.79.2__py3-none-any.whl

sunholo/components/retriever.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 from ..custom_logging import log
 from .vectorstore import pick_vectorstore
-from ..utils import load_config_key, ConfigManager
+from ..utils import ConfigManager
 from .llm import get_embeddings
 from ..utils.gcp_project import get_gcp_project
 

sunholo/llamaindex/llamaindex_class.py

@@ -1,3 +1,6 @@
+import os
+import tempfile
+
 try:
     from vertexai.preview import rag
 except ImportError:
@@ -46,6 +49,60 @@ class LlamaIndexVertexCorpusManager:
             self.location = self.config.vacConfig('location') or location
 
         init_vertex(location=self.location, project_id=self.project_id)
+
+    def upload_text(self, text: str, corpus_display_name: str, description: str = None):
+        """
+        Uploads a text string to a specified corpus by saving it to a temporary file first.
+
+        Args:
+            text (str): The text content to upload.
+            corpus_display_name (str): The display name of the corpus.
+            description (str, optional): Description of the text upload.
+
+        Returns:
+            The uploaded file object.
+        """
+        # Create a temporary file and write the text to it
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.txt') as temp_file:
+            temp_file.write(text.encode('utf-8'))
+            temp_filename = temp_file.name
+        
+        try:
+            # Upload the temporary file
+            uploaded_file = self.upload_file(temp_filename, corpus_display_name, description or text[:50])
+        finally:
+            # Clean up the temporary file
+            os.remove(temp_filename)
+
+        return uploaded_file
+    
+    def upload_file(self, filename:str, corpus_display_name:str, description:str=None):
+        corpus = self.find_corpus_from_list(corpus_display_name)
+
+        rag_file = rag.upload_file(
+            corpus_name=corpus.name,
+            path=filename,
+            display_name=filename,
+            description=description or f"Upload for {filename}",
+        )
+        log.info("Uploaded file: {rag_file}")
+
+        return rag_file
+    
+    def import_files(self, file_imports:list[str], corpus_display_name:str):
+        corpus = self.find_corpus_from_list(corpus_display_name)
+
+        log.info(f"Importing files: {file_imports} into {corpus.name}")
+        response = rag.import_files(
+            corpus_name=corpus.name,
+            paths=file_imports,
+            chunk_size=512,  # Optional
+            chunk_overlap=100,  # Optional
+            max_embedding_requests_per_min=900,  # Optional
+        )
+        log.info(f"Imported files: {response}")
+
+        return response
     
     def list_corpora(self):
         """
@@ -83,7 +140,6 @@ class LlamaIndexVertexCorpusManager:
         """
         corp = self.find_corpus_from_list(display_name)
         if corp:
-
             return corp
         
         corpus = rag.create_corpus(display_name=display_name, description=description or f"Corpus for {display_name}")
@@ -130,7 +186,6 @@ class LlamaIndexVertexCorpusManager:
         return rag.get_corpus(name=corp.name)
 
 def llamaindex_command(args):
-
     if console is None:
         raise ImportError("Need cli tools to use `sunholo llamaindex` - install via `pip install sunholo[cli]`")
     
@@ -148,8 +203,14 @@ def llamaindex_command(args):
         corpus = manager.find_corpus_from_list(display_name=args.display_name)
         console.print(corpus)
     elif args.action == "list":
-        corpura = manager.list_corpora()
-        console.print(corpura)
+        corpora = manager.list_corpora()
+        console.print(corpora)
+    elif args.action == "import_files":
+        manager.import_files(file_imports=args.file_imports, corpus_display_name=args.display_name)
+    elif args.action == "upload_file":
+        manager.upload_file(filename=args.filename, corpus_display_name=args.display_name, description=args.description)
+    elif args.action == "upload_text":
+        manager.upload_text(text=args.text, corpus_display_name=args.display_name, description=args.description)
     else:
         console.print(f"Unknown action: {args.action}")
 
@@ -186,7 +247,28 @@ def setup_llamaindex_subparser(subparsers):
     find_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
 
     # LlamaIndex list command
-    find_parser = llamaindex_subparsers.add_parser('list', help='List all corpus')
-    find_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    list_parser = llamaindex_subparsers.add_parser('list', help='List all corpus')
+    list_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex import_files command
+    import_files_parser = llamaindex_subparsers.add_parser('import_files', help='Import files from URLs to a corpus')
+    import_files_parser.add_argument('display_name', help='The name of the corpus')
+    import_files_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    import_files_parser.add_argument('file_imports', nargs='+', help='The list of file URLs to import')
+
+    # LlamaIndex upload_file command
+    upload_file_parser = llamaindex_subparsers.add_parser('upload_file', help='Upload a local file to a corpus')
+    upload_file_parser.add_argument('display_name', help='The name of the corpus')
+    upload_file_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    upload_file_parser.add_argument('filename', help='The local file path to upload')
+    upload_file_parser.add_argument('--description', help='Description of the file upload', default=None)
+
+    # LlamaIndex upload_text command
+    upload_text_parser = llamaindex_subparsers.add_parser('upload_text', help='Upload text to a corpus')
+    upload_text_parser.add_argument('display_name', help='The name of the corpus')
+    upload_text_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    upload_text_parser.add_argument('text', help='The text content to upload')
+    upload_text_parser.add_argument('--description', help='Description of the text upload', default=None)
 
     llamaindex_parser.set_defaults(func=llamaindex_command)
+    

{sunholo-0.79.1.dist-info → sunholo-0.79.2.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.79.1
+Version: 0.79.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.79.1.dist-info → sunholo-0.79.2.dist-info}/RECORD

@@ -55,7 +55,7 @@ sunholo/cli/swagger.py,sha256=absYKAU-7Yd2eiVNUY-g_WLl2zJfeRUNdWQ0oH8M_HM,1564
 sunholo/cli/vertex.py,sha256=8130YCarxHL1UC3aqblNmUwGZTXbkdL4Y_FOnZJsWiI,2056
 sunholo/components/__init__.py,sha256=IDoylb74zFKo6NIS3RQqUl0PDFBGVxM1dfUmO7OJ44U,176
 sunholo/components/llm.py,sha256=5wRVf7lIb7q1vRADNcdQp26L9l4vGHFIvjtUDurZN_s,11488
-sunholo/components/retriever.py,sha256=vT5ouRUDcpdL4PMZVK57SYm31wZREbMsXkgCq8PnCUo,6922
+sunholo/components/retriever.py,sha256=KK8Df3NH8jNpEymPBfR2913rrQZN-PRMUGedtYNIV6k,6905
 sunholo/components/vectorstore.py,sha256=xKk7micTRwZckaI7U6PxvFz_ZSjCH48xPTDYiDcv2tc,5913
 sunholo/database/__init__.py,sha256=Zz0Shcq-CtStf9rJGIYB_Ybzb8rY_Q9mfSj-nviM490,241
 sunholo/database/alloydb.py,sha256=c1PEmK9fJCxYaVmKv4emvOoXrajV7KqaVK5mqpeksvM,11527
@@ -90,7 +90,7 @@ sunholo/langfuse/prompts.py,sha256=27BsVfihM6-h1jscbkGSO4HsATl-d4ZN6tcNCVztWoY,1
 sunholo/llamaindex/__init__.py,sha256=DlY_cHWCsVEV1C5WBgDdHRgOMlJc8pDoCRukUJ8PT9w,88
 sunholo/llamaindex/get_files.py,sha256=6rhXCDqQ_lrIapISQ_OYQDjiSATXvS_9m3qq53-oIl0,781
 sunholo/llamaindex/import_files.py,sha256=Bnic5wz8c61af9Kwq8KSrNBbc4imYnzMtBCb2jzSImI,6224
-sunholo/llamaindex/llamaindex_class.py,sha256=KAD9Unby2INIWVfxarthA4F5P6SiJC2fnAj3Cu1pEhw,7062
+sunholo/llamaindex/llamaindex_class.py,sha256=45N_xU0gDdgUPNv5m36DopRmG9JHFErwmgssKZ3wOqA,11058
 sunholo/lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/lookup/model_lookup.yaml,sha256=O7o-jP53MLA06C8pI-ILwERShO-xf6z_258wtpZBv6A,739
 sunholo/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -132,9 +132,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=7nDQXjrF9IpCPKbX3Focn4A2PdKb7JhfZDLv4le5kWc,7498
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.79.1.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.79.1.dist-info/METADATA,sha256=lNLvJFenzRDXsw-prEyzwa6NY2jSvWpDyDZ-t-pRCYo,7348
-sunholo-0.79.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-sunholo-0.79.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.79.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.79.1.dist-info/RECORD,,
+sunholo-0.79.2.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.79.2.dist-info/METADATA,sha256=qtezkTfwv1CJNsCg-T2vPY1zsZMixC7t-teHOjFwPI0,7348
+sunholo-0.79.2.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+sunholo-0.79.2.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.79.2.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.79.2.dist-info/RECORD,,