sunholo 0.79.4__py3-none-any.whl → 0.79.6__py3-none-any.whl

sunholo/chunker/splitter.py

@@ -113,8 +113,9 @@ def choose_splitter(extension: str, chunk_size: int=1024, chunk_overlap:int=200,
 
     if vector_name:
         # check if there is a chunking configuration
-        from ..utils import load_config_key
-        chunk_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
+        from ..utils import ConfigManager
+        config = ConfigManager(vector_name)
+        chunk_config = config.vacConfig("chunker")
         if chunk_config:
             if chunk_config.get("type") == "semantic":
                 embedding_str = chunk_config.get("llm")
@@ -124,7 +125,7 @@ def choose_splitter(extension: str, chunk_size: int=1024, chunk_overlap:int=200,
                     log.info(f"Semantic chunking for {vector_name}")
                     from langchain_experimental.text_splitter import SemanticChunker
                     from ..components import pick_embedding
-                    embeddings = pick_embedding(embedding_str)
+                    embeddings = pick_embedding(embedding_str, config=config)
                     semantic_splitter = SemanticChunker(
                         embeddings, breakpoint_threshold_type="percentile"
                     )

sunholo/llamaindex/llamaindex_class.py

@@ -43,6 +43,8 @@ class LlamaIndexVertexCorpusManager:
         self.config = config
         self.project_id = project_id
         self.location = location
+        self.corpus_display_name = ""
+        self.corpus = ""
 
         if config:
             self.project_id = self.config.vacConfig('project_id') or project_id
@@ -104,6 +106,88 @@ class LlamaIndexVertexCorpusManager:
 
         return response
     
+    def list_files(self, corpus_display_name:str):
+        corpus = self.find_corpus_from_list(corpus_display_name)
+        files = rag.list_files(corpus_name=corpus.name)
+
+        log.info(f"--Files in {corpus.name}:\n{files}")
+        
+        return files
+
+    def find_file_from_list(self, display_name: str, corpus_display_name:str):
+        """
+        Finds a file from the list of files by its display name.
+
+        Args:
+            display_name (str): The display name of the file.
+            corpus_display_name (str): The display name of the corpus to look within
+
+        Returns:
+            The found file object if it exists, otherwise None.
+        """
+        files = self.list_files(corpus_display_name)
+        for file in files:
+            if display_name == file.display_name:
+                log.info(f"Found existing file with display name: {display_name}")
+
+                return file
+        
+        return None
+    
+    def get_file(self, file_display_name:str=None, file_name:str=None, corpus_display_name:str=None):
+        
+        if file_display_name:
+            rag_file = self.find_file_from_list(file_display_name, corpus_display_name)
+            log.info(f"Found {rag_file} via display name: {file_display_name}")
+            
+            return rag_file
+        
+        if not file_name:
+            raise ValueError("Need to supply one of file_display_name or file_name")
+
+        corpus = self.find_corpus_from_list(corpus_display_name)
+        if file_name.startswith("projects/"):
+            rag_file = rag.get_file(name=file_name)
+        else:
+            if not corpus_display_name:
+                raise ValueError("Must supply corpus_display_name if not a full file_name")
+            rag_file = rag.get_file(name=file_name, corpus_name=corpus.name)
+        
+        log.info(f"Found {rag_file}")
+
+        return rag_file
+    
+    def delete_file(self, file_name, corpus_display_name:str):
+
+        corpus = self.find_corpus_from_list(corpus_display_name)
+        if file_name.startswith("projects/"):
+            rag.delete_file(name=file_name)
+        else:
+            if not corpus_display_name:
+                raise ValueError("Must supply corpus_display_name if not a full file_name")
+            rag.delete_file(name=file_name, corpus_name=corpus.name)
+        
+        log.info(f"File {file_name} deleted.")
+
+        return True
+    
+    def query_corpus(self, query:str, corpus_disply_name:str):
+        corpus = self.find_corpus_from_list(corpus_disply_name)
+        response = rag.retrieval_query(
+            rag_resources=[
+                rag.RagResource(
+                    rag_corpus=corpus.name,
+                    # Supply IDs from `rag.list_files()`.
+                    # rag_file_ids=["rag-file-1", "rag-file-2", ...],
+                )
+            ],
+            text=query,
+            similarity_top_k=10,  # Optional
+            vector_distance_threshold=0.5,  # Optional
+        )
+
+        return response
+
     def list_corpora(self):
         """
         List all VertexAI Corpus for the project/location
@@ -120,10 +204,13 @@ class LlamaIndexVertexCorpusManager:
         Returns:
             The found corpus object if it exists, otherwise None.
         """
+        if display_name == self.corpus_display_name:
+            return self.corpus
         corpora = self.list_corpora()
         for corp in corpora:
             if display_name == corp.display_name:
                 log.info(f"Found existing corpus with display name: {display_name}")
+                self.corpus = corp
                 return corp
         return None
 
@@ -211,6 +298,29 @@ def llamaindex_command(args):
         manager.upload_file(filename=args.filename, corpus_display_name=args.display_name, description=args.description)
     elif args.action == "upload_text":
         manager.upload_text(text=args.text, corpus_display_name=args.display_name, description=args.description)
+    elif args.action == "list_files":
+        files = manager.list_files(corpus_display_name=args.display_name)
+        if files:
+            console.print(files)
+        else:
+            console.print("No files found for {args.display_name}")
+    elif args.action == "get_file":
+        file = manager.get_file(file_display_name=args.file_name, corpus_display_name=args.display_name)
+        console.print(file)
+        return file
+    elif args.action == "delete_file":
+        deleted = manager.delete_file(args.file_name, corpus_display_name=args.display_name)
+        if deleted:
+            console.print(f"Deleted {args.file_name}")
+        else:
+            console.print(f"ERROR: Could not delete {args.file_name}")
+
+    elif args.action == "query":
+        answer = manager.query_corpus(args.query, corpus_disply_name=args.display_name)
+        if answer:
+            console.print(answer)
+        else:
+            console.print(f"No answer found for {args.query} in {args.display_name}")
     else:
         console.print(f"Unknown action: {args.action}")
 
@@ -270,5 +380,27 @@ def setup_llamaindex_subparser(subparsers):
     upload_text_parser.add_argument('text', help='The text content to upload')
     upload_text_parser.add_argument('--description', help='Description of the text upload', default=None)
 
+    # LlamaIndex list_files command
+    list_files_parser = llamaindex_subparsers.add_parser('list_files', help='List all files in a corpus')
+    list_files_parser.add_argument('display_name', help='The name of the corpus')
+    list_files_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex get_file command
+    get_file_parser = llamaindex_subparsers.add_parser('get_file', help='Get a file from a corpus')
+    get_file_parser.add_argument('display_name', help='The name of the corpus')
+    get_file_parser.add_argument('file_name', help='The name of the file to get')
+    get_file_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex delete_file command
+    delete_file_parser = llamaindex_subparsers.add_parser('delete_file', help='Delete a file from a corpus')
+    delete_file_parser.add_argument('display_name', help='The name of the corpus')
+    delete_file_parser.add_argument('file_name', help='The name of the file to delete')
+    delete_file_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex query command
+    query_parser = llamaindex_subparsers.add_parser('query', help='Query a corpus')
+    query_parser.add_argument('display_name', help='The name of the corpus')
+    query_parser.add_argument('query', help='The query string')
+    query_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
     llamaindex_parser.set_defaults(func=llamaindex_command)
-    

{sunholo-0.79.4.dist-info → sunholo-0.79.6.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.79.4
+Version: 0.79.6
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.4.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.6.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.79.4.dist-info → sunholo-0.79.6.dist-info}/RECORD

@@ -40,7 +40,7 @@ sunholo/chunker/pdfs.py,sha256=njDPop751GMHi3cOwIKd2Yct-_lWR2gqcB7WykfHphs,2480
 sunholo/chunker/process_chunker_data.py,sha256=OnMvXHRv3rGpFsU50FyUNkNIwC1D8TkhaWWbn72yQss,3523
 sunholo/chunker/publish.py,sha256=AX5u-fcyDytED67IfizMzvOMcYPXEo6XBJvyk_7maK8,2939
 sunholo/chunker/pubsub.py,sha256=48bhuAcszN7LGe3-ksPSLHHhq0uKxiXOrizck5qpcP0,1012
-sunholo/chunker/splitter.py,sha256=liruCE6fqzPdQ8dwvoD5EjMgWe0PW5OmO-hKHmiG9Nk,6736
+sunholo/chunker/splitter.py,sha256=QLAEsJOpEYFZr9-UGZUuAlNVyjfCWb8jvzCHg0rVShE,6751
 sunholo/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/cli/chat_vac.py,sha256=UNLzZYAejjEvMR0EjpxIHXyvUpBDoqThohzzFL_m3Yg,23026
 sunholo/cli/cli.py,sha256=yuY7SLFiYDUKqJDOXy7jL1l6P0UVPMuAZK9bXXTG8ck,3939
@@ -90,7 +90,7 @@ sunholo/langfuse/prompts.py,sha256=27BsVfihM6-h1jscbkGSO4HsATl-d4ZN6tcNCVztWoY,1
 sunholo/llamaindex/__init__.py,sha256=DlY_cHWCsVEV1C5WBgDdHRgOMlJc8pDoCRukUJ8PT9w,88
 sunholo/llamaindex/get_files.py,sha256=6rhXCDqQ_lrIapISQ_OYQDjiSATXvS_9m3qq53-oIl0,781
 sunholo/llamaindex/import_files.py,sha256=Bnic5wz8c61af9Kwq8KSrNBbc4imYnzMtBCb2jzSImI,6224
-sunholo/llamaindex/llamaindex_class.py,sha256=45N_xU0gDdgUPNv5m36DopRmG9JHFErwmgssKZ3wOqA,11058
+sunholo/llamaindex/llamaindex_class.py,sha256=USb8zmF0DuEdaDjplJHvAItL4Ud2eE0iZ5ITwJ5Wflo,16763
 sunholo/lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/lookup/model_lookup.yaml,sha256=O7o-jP53MLA06C8pI-ILwERShO-xf6z_258wtpZBv6A,739
 sunholo/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -132,9 +132,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=pgSahVDh7GPEulu3nl-w0jb5lTClb4TCnVxPnMokNZY,7533
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.79.4.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.79.4.dist-info/METADATA,sha256=-mV_Y5Er002Y0OS4lEz913p-XMfqTuifDtkRMlExgA4,7348
-sunholo-0.79.4.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-sunholo-0.79.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.79.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.79.4.dist-info/RECORD,,
+sunholo-0.79.6.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.79.6.dist-info/METADATA,sha256=ykRVzCPMucoQbLQqvS9EydN33TnwAp5M4xfmK8rUPzc,7348
+sunholo-0.79.6.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+sunholo-0.79.6.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.79.6.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.79.6.dist-info/RECORD,,